You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2013/12/31 22:19:54 UTC
[01/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Updated Branches:
refs/heads/master e8a4db11c -> bc1635df8
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js
new file mode 100644
index 0000000..02694a4
--- /dev/null
+++ b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js
@@ -0,0 +1,155 @@
+/*
+ * File: jquery.dataTables.min.js
+ * Version: 1.9.4
+ * Author: Allan Jardine (www.sprymedia.co.uk)
+ * Info: www.datatables.net
+ *
+ * Copyright 2008-2012 Allan Jardine, all rights reserved.
+ *
+ * This source file is free software, under either the GPL v2 license or a
+ * BSD style license, available at:
+ * http://datatables.net/license_gpl2
+ * http://datatables.net/license_bsd
+ *
+ * This source file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
+ */
+(function(X,l,n){var L=function(h){var j=function(e){function o(a,b){var c=j.defaults.columns,d=a.aoColumns.length,c=h.extend({},j.models.oColumn,c,{sSortingClass:a.oClasses.sSortable,sSortingClassJUI:a.oClasses.sSortJUI,nTh:b?b:l.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.oDefaults:d});a.aoColumns.push(c);if(a.aoPreSearchCols[d]===n||null===a.aoPreSearchCols[d])a.aoPreSearchCols[d]=h.extend({},j.models.oSearch);else if(c=a.aoPreSearchCols[d],
+c.bRegex===n&&(c.bRegex=!0),c.bSmart===n&&(c.bSmart=!0),c.bCaseInsensitive===n)c.bCaseInsensitive=!0;m(a,d,null)}function m(a,b,c){var d=a.aoColumns[b];c!==n&&null!==c&&(c.mDataProp&&!c.mData&&(c.mData=c.mDataProp),c.sType!==n&&(d.sType=c.sType,d._bAutoType=!1),h.extend(d,c),p(d,c,"sWidth","sWidthOrig"),c.iDataSort!==n&&(d.aDataSort=[c.iDataSort]),p(d,c,"aDataSort"));var i=d.mRender?Q(d.mRender):null,f=Q(d.mData);d.fnGetData=function(a,b){var c=f(a,b);return d.mRender&&b&&""!==b?i(c,b,a):c};d.fnSetData=
+L(d.mData);a.oFeatures.bSort||(d.bSortable=!1);!d.bSortable||-1==h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortableNone,d.sSortingClassJUI=""):-1==h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortable,d.sSortingClassJUI=a.oClasses.sSortJUI):-1!=h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortableAsc,d.sSortingClassJUI=a.oClasses.sSortJUIAscAllowed):-1==
+h.inArray("asc",d.asSorting)&&-1!=h.inArray("desc",d.asSorting)&&(d.sSortingClass=a.oClasses.sSortableDesc,d.sSortingClassJUI=a.oClasses.sSortJUIDescAllowed)}function k(a){if(!1===a.oFeatures.bAutoWidth)return!1;da(a);for(var b=0,c=a.aoColumns.length;b<c;b++)a.aoColumns[b].nTh.style.width=a.aoColumns[b].sWidth}function G(a,b){var c=r(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function R(a,b){var c=r(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function t(a){return r(a,"bVisible").length}
+function r(a,b){var c=[];h.map(a.aoColumns,function(a,i){a[b]&&c.push(i)});return c}function B(a){for(var b=j.ext.aTypes,c=b.length,d=0;d<c;d++){var i=b[d](a);if(null!==i)return i}return"string"}function u(a,b){for(var c=b.split(","),d=[],i=0,f=a.aoColumns.length;i<f;i++)for(var g=0;g<f;g++)if(a.aoColumns[i].sName==c[g]){d.push(g);break}return d}function M(a){for(var b="",c=0,d=a.aoColumns.length;c<d;c++)b+=a.aoColumns[c].sName+",";return b.length==d?"":b.slice(0,-1)}function ta(a,b,c,d){var i,f,
+g,e,w;if(b)for(i=b.length-1;0<=i;i--){var j=b[i].aTargets;h.isArray(j)||D(a,1,"aTargets must be an array of targets, not a "+typeof j);f=0;for(g=j.length;f<g;f++)if("number"===typeof j[f]&&0<=j[f]){for(;a.aoColumns.length<=j[f];)o(a);d(j[f],b[i])}else if("number"===typeof j[f]&&0>j[f])d(a.aoColumns.length+j[f],b[i]);else if("string"===typeof j[f]){e=0;for(w=a.aoColumns.length;e<w;e++)("_all"==j[f]||h(a.aoColumns[e].nTh).hasClass(j[f]))&&d(e,b[i])}}if(c){i=0;for(a=c.length;i<a;i++)d(i,c[i])}}function H(a,
+b){var c;c=h.isArray(b)?b.slice():h.extend(!0,{},b);var d=a.aoData.length,i=h.extend(!0,{},j.models.oRow);i._aData=c;a.aoData.push(i);for(var f,i=0,g=a.aoColumns.length;i<g;i++)c=a.aoColumns[i],"function"===typeof c.fnRender&&c.bUseRendered&&null!==c.mData?F(a,d,i,S(a,d,i)):F(a,d,i,v(a,d,i)),c._bAutoType&&"string"!=c.sType&&(f=v(a,d,i,"type"),null!==f&&""!==f&&(f=B(f),null===c.sType?c.sType=f:c.sType!=f&&"html"!=c.sType&&(c.sType="string")));a.aiDisplayMaster.push(d);a.oFeatures.bDeferRender||ea(a,
+d);return d}function ua(a){var b,c,d,i,f,g,e;if(a.bDeferLoading||null===a.sAjaxSource)for(b=a.nTBody.firstChild;b;){if("TR"==b.nodeName.toUpperCase()){c=a.aoData.length;b._DT_RowIndex=c;a.aoData.push(h.extend(!0,{},j.models.oRow,{nTr:b}));a.aiDisplayMaster.push(c);f=b.firstChild;for(d=0;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)F(a,c,d,h.trim(f.innerHTML)),d++;f=f.nextSibling}}b=b.nextSibling}i=T(a);d=[];b=0;for(c=i.length;b<c;b++)for(f=i[b].firstChild;f;)g=f.nodeName.toUpperCase(),("TD"==
+g||"TH"==g)&&d.push(f),f=f.nextSibling;c=0;for(i=a.aoColumns.length;c<i;c++){e=a.aoColumns[c];null===e.sTitle&&(e.sTitle=e.nTh.innerHTML);var w=e._bAutoType,o="function"===typeof e.fnRender,k=null!==e.sClass,n=e.bVisible,m,p;if(w||o||k||!n){g=0;for(b=a.aoData.length;g<b;g++)f=a.aoData[g],m=d[g*i+c],w&&"string"!=e.sType&&(p=v(a,g,c,"type"),""!==p&&(p=B(p),null===e.sType?e.sType=p:e.sType!=p&&"html"!=e.sType&&(e.sType="string"))),e.mRender?m.innerHTML=v(a,g,c,"display"):e.mData!==c&&(m.innerHTML=v(a,
+g,c,"display")),o&&(p=S(a,g,c),m.innerHTML=p,e.bUseRendered&&F(a,g,c,p)),k&&(m.className+=" "+e.sClass),n?f._anHidden[c]=null:(f._anHidden[c]=m,m.parentNode.removeChild(m)),e.fnCreatedCell&&e.fnCreatedCell.call(a.oInstance,m,v(a,g,c,"display"),f._aData,g,c)}}if(0!==a.aoRowCreatedCallback.length){b=0;for(c=a.aoData.length;b<c;b++)f=a.aoData[b],A(a,"aoRowCreatedCallback",null,[f.nTr,f._aData,b])}}function I(a,b){return b._DT_RowIndex!==n?b._DT_RowIndex:null}function fa(a,b,c){for(var b=J(a,b),d=0,a=
+a.aoColumns.length;d<a;d++)if(b[d]===c)return d;return-1}function Y(a,b,c,d){for(var i=[],f=0,g=d.length;f<g;f++)i.push(v(a,b,d[f],c));return i}function v(a,b,c,d){var i=a.aoColumns[c];if((c=i.fnGetData(a.aoData[b]._aData,d))===n)return a.iDrawError!=a.iDraw&&null===i.sDefaultContent&&(D(a,0,"Requested unknown parameter "+("function"==typeof i.mData?"{mData function}":"'"+i.mData+"'")+" from the data source for row "+b),a.iDrawError=a.iDraw),i.sDefaultContent;if(null===c&&null!==i.sDefaultContent)c=
+i.sDefaultContent;else if("function"===typeof c)return c();return"display"==d&&null===c?"":c}function F(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d)}function Q(a){if(null===a)return function(){return null};if("function"===typeof a)return function(b,d,i){return a(b,d,i)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("["))){var b=function(a,d,i){var f=i.split("."),g;if(""!==i){var e=0;for(g=f.length;e<g;e++){if(i=f[e].match(U)){f[e]=f[e].replace(U,"");""!==f[e]&&(a=a[f[e]]);
+g=[];f.splice(0,e+1);for(var f=f.join("."),e=0,h=a.length;e<h;e++)g.push(b(a[e],d,f));a=i[0].substring(1,i[0].length-1);a=""===a?g:g.join(a);break}if(null===a||a[f[e]]===n)return n;a=a[f[e]]}}return a};return function(c,d){return b(c,d,a)}}return function(b){return b[a]}}function L(a){if(null===a)return function(){};if("function"===typeof a)return function(b,d){a(b,"set",d)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("["))){var b=function(a,d,i){var i=i.split("."),f,g,e=0;for(g=
+i.length-1;e<g;e++){if(f=i[e].match(U)){i[e]=i[e].replace(U,"");a[i[e]]=[];f=i.slice();f.splice(0,e+1);g=f.join(".");for(var h=0,j=d.length;h<j;h++)f={},b(f,d[h],g),a[i[e]].push(f);return}if(null===a[i[e]]||a[i[e]]===n)a[i[e]]={};a=a[i[e]]}a[i[i.length-1].replace(U,"")]=d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Z(a){for(var b=[],c=a.aoData.length,d=0;d<c;d++)b.push(a.aoData[d]._aData);return b}function ga(a){a.aoData.splice(0,a.aoData.length);a.aiDisplayMaster.splice(0,
+a.aiDisplayMaster.length);a.aiDisplay.splice(0,a.aiDisplay.length);y(a)}function ha(a,b){for(var c=-1,d=0,i=a.length;d<i;d++)a[d]==b?c=d:a[d]>b&&a[d]--; -1!=c&&a.splice(c,1)}function S(a,b,c){var d=a.aoColumns[c];return d.fnRender({iDataRow:b,iDataColumn:c,oSettings:a,aData:a.aoData[b]._aData,mDataProp:d.mData},v(a,b,c,"display"))}function ea(a,b){var c=a.aoData[b],d;if(null===c.nTr){c.nTr=l.createElement("tr");c.nTr._DT_RowIndex=b;c._aData.DT_RowId&&(c.nTr.id=c._aData.DT_RowId);c._aData.DT_RowClass&&
+(c.nTr.className=c._aData.DT_RowClass);for(var i=0,f=a.aoColumns.length;i<f;i++){var g=a.aoColumns[i];d=l.createElement(g.sCellType);d.innerHTML="function"===typeof g.fnRender&&(!g.bUseRendered||null===g.mData)?S(a,b,i):v(a,b,i,"display");null!==g.sClass&&(d.className=g.sClass);g.bVisible?(c.nTr.appendChild(d),c._anHidden[i]=null):c._anHidden[i]=d;g.fnCreatedCell&&g.fnCreatedCell.call(a.oInstance,d,v(a,b,i,"display"),c._aData,b,i)}A(a,"aoRowCreatedCallback",null,[c.nTr,c._aData,b])}}function va(a){var b,
+c,d;if(0!==h("th, td",a.nTHead).length){b=0;for(d=a.aoColumns.length;b<d;b++)if(c=a.aoColumns[b].nTh,c.setAttribute("role","columnheader"),a.aoColumns[b].bSortable&&(c.setAttribute("tabindex",a.iTabIndex),c.setAttribute("aria-controls",a.sTableId)),null!==a.aoColumns[b].sClass&&h(c).addClass(a.aoColumns[b].sClass),a.aoColumns[b].sTitle!=c.innerHTML)c.innerHTML=a.aoColumns[b].sTitle}else{var i=l.createElement("tr");b=0;for(d=a.aoColumns.length;b<d;b++)c=a.aoColumns[b].nTh,c.innerHTML=a.aoColumns[b].sTitle,
+c.setAttribute("tabindex","0"),null!==a.aoColumns[b].sClass&&h(c).addClass(a.aoColumns[b].sClass),i.appendChild(c);h(a.nTHead).html("")[0].appendChild(i);V(a.aoHeader,a.nTHead)}h(a.nTHead).children("tr").attr("role","row");if(a.bJUI){b=0;for(d=a.aoColumns.length;b<d;b++){c=a.aoColumns[b].nTh;i=l.createElement("div");i.className=a.oClasses.sSortJUIWrapper;h(c).contents().appendTo(i);var f=l.createElement("span");f.className=a.oClasses.sSortIcon;i.appendChild(f);c.appendChild(i)}}if(a.oFeatures.bSort)for(b=
+0;b<a.aoColumns.length;b++)!1!==a.aoColumns[b].bSortable?ia(a,a.aoColumns[b].nTh,b):h(a.aoColumns[b].nTh).addClass(a.oClasses.sSortableNone);""!==a.oClasses.sFooterTH&&h(a.nTFoot).children("tr").children("th").addClass(a.oClasses.sFooterTH);if(null!==a.nTFoot){c=N(a,null,a.aoFooter);b=0;for(d=a.aoColumns.length;b<d;b++)c[b]&&(a.aoColumns[b].nTf=c[b],a.aoColumns[b].sClass&&h(c[b]).addClass(a.aoColumns[b].sClass))}}function W(a,b,c){var d,i,f,g=[],e=[],h=a.aoColumns.length,j;c===n&&(c=!1);d=0;for(i=
+b.length;d<i;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=h-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);e.push([])}d=0;for(i=g.length;d<i;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(j=h=1,e[d][f]===n){a.appendChild(g[d][f].cell);for(e[d][f]=1;g[d+h]!==n&&g[d][f].cell==g[d+h][f].cell;)e[d+h][f]=1,h++;for(;g[d][f+j]!==n&&g[d][f].cell==g[d][f+j].cell;){for(c=0;c<h;c++)e[d+c][f+j]=1;j++}g[d][f].cell.rowSpan=h;g[d][f].cell.colSpan=j}}}function x(a){var b=
+A(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))E(a,!1);else{var c,d,b=[],i=0,f=a.asStripeClasses.length;c=a.aoOpenRows.length;a.bDrawing=!0;a.iInitDisplayStart!==n&&-1!=a.iInitDisplayStart&&(a._iDisplayStart=a.oFeatures.bServerSide?a.iInitDisplayStart:a.iInitDisplayStart>=a.fnRecordsDisplay()?0:a.iInitDisplayStart,a.iInitDisplayStart=-1,y(a));if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++;else if(a.oFeatures.bServerSide){if(!a.bDestroying&&!wa(a))return}else a.iDraw++;if(0!==a.aiDisplay.length){var g=
+a._iDisplayStart;d=a._iDisplayEnd;a.oFeatures.bServerSide&&(g=0,d=a.aoData.length);for(;g<d;g++){var e=a.aoData[a.aiDisplay[g]];null===e.nTr&&ea(a,a.aiDisplay[g]);var j=e.nTr;if(0!==f){var o=a.asStripeClasses[i%f];e._sRowStripe!=o&&(h(j).removeClass(e._sRowStripe).addClass(o),e._sRowStripe=o)}A(a,"aoRowCallback",null,[j,a.aoData[a.aiDisplay[g]]._aData,i,g]);b.push(j);i++;if(0!==c)for(e=0;e<c;e++)if(j==a.aoOpenRows[e].nParent){b.push(a.aoOpenRows[e].nTr);break}}}else b[0]=l.createElement("tr"),a.asStripeClasses[0]&&
+(b[0].className=a.asStripeClasses[0]),c=a.oLanguage,f=c.sZeroRecords,1==a.iDraw&&null!==a.sAjaxSource&&!a.oFeatures.bServerSide?f=c.sLoadingRecords:c.sEmptyTable&&0===a.fnRecordsTotal()&&(f=c.sEmptyTable),c=l.createElement("td"),c.setAttribute("valign","top"),c.colSpan=t(a),c.className=a.oClasses.sRowEmpty,c.innerHTML=ja(a,f),b[i].appendChild(c);A(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Z(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);A(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],
+Z(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);i=l.createDocumentFragment();c=l.createDocumentFragment();if(a.nTBody){f=a.nTBody.parentNode;c.appendChild(a.nTBody);if(!a.oScroll.bInfinite||!a._bInitComplete||a.bSorted||a.bFiltered)for(;c=a.nTBody.firstChild;)a.nTBody.removeChild(c);c=0;for(d=b.length;c<d;c++)i.appendChild(b[c]);a.nTBody.appendChild(i);null!==f&&f.appendChild(a.nTBody)}A(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1;a.oFeatures.bServerSide&&(E(a,!1),
+a._bInitComplete||$(a))}}function aa(a){a.oFeatures.bSort?O(a,a.oPreviousSearch):a.oFeatures.bFilter?K(a,a.oPreviousSearch):(y(a),x(a))}function xa(a){var b=h("<div></div>")[0];a.nTable.parentNode.insertBefore(b,a.nTable);a.nTableWrapper=h('<div id="'+a.sTableId+'_wrapper" class="'+a.oClasses.sWrapper+'" role="grid"></div>')[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var c=a.nTableWrapper,d=a.sDom.split(""),i,f,g,e,w,o,k,m=0;m<d.length;m++){f=0;g=d[m];if("<"==g){e=h("<div></div>")[0];w=d[m+
+1];if("'"==w||'"'==w){o="";for(k=2;d[m+k]!=w;)o+=d[m+k],k++;"H"==o?o=a.oClasses.sJUIHeader:"F"==o&&(o=a.oClasses.sJUIFooter);-1!=o.indexOf(".")?(w=o.split("."),e.id=w[0].substr(1,w[0].length-1),e.className=w[1]):"#"==o.charAt(0)?e.id=o.substr(1,o.length-1):e.className=o;m+=k}c.appendChild(e);c=e}else if(">"==g)c=c.parentNode;else if("l"==g&&a.oFeatures.bPaginate&&a.oFeatures.bLengthChange)i=ya(a),f=1;else if("f"==g&&a.oFeatures.bFilter)i=za(a),f=1;else if("r"==g&&a.oFeatures.bProcessing)i=Aa(a),f=
+1;else if("t"==g)i=Ba(a),f=1;else if("i"==g&&a.oFeatures.bInfo)i=Ca(a),f=1;else if("p"==g&&a.oFeatures.bPaginate)i=Da(a),f=1;else if(0!==j.ext.aoFeatures.length){e=j.ext.aoFeatures;k=0;for(w=e.length;k<w;k++)if(g==e[k].cFeature){(i=e[k].fnInit(a))&&(f=1);break}}1==f&&null!==i&&("object"!==typeof a.aanFeatures[g]&&(a.aanFeatures[g]=[]),a.aanFeatures[g].push(i),c.appendChild(i))}b.parentNode.replaceChild(a.nTableWrapper,b)}function V(a,b){var c=h(b).children("tr"),d,i,f,g,e,j,o,k,m,p;a.splice(0,a.length);
+f=0;for(j=c.length;f<j;f++)a.push([]);f=0;for(j=c.length;f<j;f++){d=c[f];for(i=d.firstChild;i;){if("TD"==i.nodeName.toUpperCase()||"TH"==i.nodeName.toUpperCase()){k=1*i.getAttribute("colspan");m=1*i.getAttribute("rowspan");k=!k||0===k||1===k?1:k;m=!m||0===m||1===m?1:m;g=0;for(e=a[f];e[g];)g++;o=g;p=1===k?!0:!1;for(e=0;e<k;e++)for(g=0;g<m;g++)a[f+g][o+e]={cell:i,unique:p},a[f+g].nTr=d}i=i.nextSibling}}}function N(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],V(c,b)));for(var b=0,i=c.length;b<i;b++)for(var f=
+0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function wa(a){if(a.bAjaxDataGet){a.iDraw++;E(a,!0);var b=Ea(a);ka(a,b);a.fnServerData.call(a.oInstance,a.sAjaxSource,b,function(b){Fa(a,b)},a);return!1}return!0}function Ea(a){var b=a.aoColumns.length,c=[],d,i,f,g;c.push({name:"sEcho",value:a.iDraw});c.push({name:"iColumns",value:b});c.push({name:"sColumns",value:M(a)});c.push({name:"iDisplayStart",value:a._iDisplayStart});c.push({name:"iDisplayLength",
+value:!1!==a.oFeatures.bPaginate?a._iDisplayLength:-1});for(f=0;f<b;f++)d=a.aoColumns[f].mData,c.push({name:"mDataProp_"+f,value:"function"===typeof d?"function":d});if(!1!==a.oFeatures.bFilter){c.push({name:"sSearch",value:a.oPreviousSearch.sSearch});c.push({name:"bRegex",value:a.oPreviousSearch.bRegex});for(f=0;f<b;f++)c.push({name:"sSearch_"+f,value:a.aoPreSearchCols[f].sSearch}),c.push({name:"bRegex_"+f,value:a.aoPreSearchCols[f].bRegex}),c.push({name:"bSearchable_"+f,value:a.aoColumns[f].bSearchable})}if(!1!==
+a.oFeatures.bSort){var e=0;d=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(f=0;f<d.length;f++){i=a.aoColumns[d[f][0]].aDataSort;for(g=0;g<i.length;g++)c.push({name:"iSortCol_"+e,value:i[g]}),c.push({name:"sSortDir_"+e,value:d[f][1]}),e++}c.push({name:"iSortingCols",value:e});for(f=0;f<b;f++)c.push({name:"bSortable_"+f,value:a.aoColumns[f].bSortable})}return c}function ka(a,b){A(a,"aoServerParams","serverParams",[b])}function Fa(a,b){if(b.sEcho!==n){if(1*b.sEcho<
+a.iDraw)return;a.iDraw=1*b.sEcho}(!a.oScroll.bInfinite||a.oScroll.bInfinite&&(a.bSorted||a.bFiltered))&&ga(a);a._iRecordsTotal=parseInt(b.iTotalRecords,10);a._iRecordsDisplay=parseInt(b.iTotalDisplayRecords,10);var c=M(a),c=b.sColumns!==n&&""!==c&&b.sColumns!=c,d;c&&(d=u(a,b.sColumns));for(var i=Q(a.sAjaxDataProp)(b),f=0,g=i.length;f<g;f++)if(c){for(var e=[],h=0,j=a.aoColumns.length;h<j;h++)e.push(i[f][d[h]]);H(a,e)}else H(a,i[f]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;x(a);a.bAjaxDataGet=
+!0;E(a,!1)}function za(a){var b=a.oPreviousSearch,c=a.oLanguage.sSearch,c=-1!==c.indexOf("_INPUT_")?c.replace("_INPUT_",'<input type="text" />'):""===c?'<input type="text" />':c+' <input type="text" />',d=l.createElement("div");d.className=a.oClasses.sFilter;d.innerHTML="<label>"+c+"</label>";a.aanFeatures.f||(d.id=a.sTableId+"_filter");c=h('input[type="text"]',d);d._DT_Input=c[0];c.val(b.sSearch.replace('"',"""));c.bind("keyup.DT",function(){for(var c=a.aanFeatures.f,d=this.value===""?"":this.value,
+g=0,e=c.length;g<e;g++)c[g]!=h(this).parents("div.dataTables_filter")[0]&&h(c[g]._DT_Input).val(d);d!=b.sSearch&&K(a,{sSearch:d,bRegex:b.bRegex,bSmart:b.bSmart,bCaseInsensitive:b.bCaseInsensitive})});c.attr("aria-controls",a.sTableId).bind("keypress.DT",function(a){if(a.keyCode==13)return false});return d}function K(a,b,c){var d=a.oPreviousSearch,i=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};if(a.oFeatures.bServerSide)f(b);
+else{Ga(a,b.sSearch,c,b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<a.aoPreSearchCols.length;b++)Ha(a,i[b].sSearch,b,i[b].bRegex,i[b].bSmart,i[b].bCaseInsensitive);Ia(a)}a.bFiltered=!0;h(a.oInstance).trigger("filter",a);a._iDisplayStart=0;y(a);x(a);la(a,0)}function Ia(a){for(var b=j.ext.afnFiltering,c=r(a,"bSearchable"),d=0,i=b.length;d<i;d++)for(var f=0,g=0,e=a.aiDisplay.length;g<e;g++){var h=a.aiDisplay[g-f];b[d](a,Y(a,h,"filter",c),h)||(a.aiDisplay.splice(g-f,1),f++)}}function Ha(a,b,c,
+d,i,f){if(""!==b)for(var g=0,b=ma(b,d,i,f),d=a.aiDisplay.length-1;0<=d;d--)i=Ja(v(a,a.aiDisplay[d],c,"filter"),a.aoColumns[c].sType),b.test(i)||(a.aiDisplay.splice(d,1),g++)}function Ga(a,b,c,d,i,f){d=ma(b,d,i,f);i=a.oPreviousSearch;c||(c=0);0!==j.ext.afnFiltering.length&&(c=1);if(0>=b.length)a.aiDisplay.splice(0,a.aiDisplay.length),a.aiDisplay=a.aiDisplayMaster.slice();else if(a.aiDisplay.length==a.aiDisplayMaster.length||i.sSearch.length>b.length||1==c||0!==b.indexOf(i.sSearch)){a.aiDisplay.splice(0,
+a.aiDisplay.length);la(a,1);for(b=0;b<a.aiDisplayMaster.length;b++)d.test(a.asDataSearch[b])&&a.aiDisplay.push(a.aiDisplayMaster[b])}else for(b=c=0;b<a.asDataSearch.length;b++)d.test(a.asDataSearch[b])||(a.aiDisplay.splice(b-c,1),c++)}function la(a,b){if(!a.oFeatures.bServerSide){a.asDataSearch=[];for(var c=r(a,"bSearchable"),d=1===b?a.aiDisplayMaster:a.aiDisplay,i=0,f=d.length;i<f;i++)a.asDataSearch[i]=na(a,Y(a,d[i],"filter",c))}}function na(a,b){var c=b.join(" ");-1!==c.indexOf("&")&&(c=h("<div>").html(c).text());
+return c.replace(/[\n\r]/g," ")}function ma(a,b,c,d){if(c)return a=b?a.split(" "):oa(a).split(" "),a="^(?=.*?"+a.join(")(?=.*?")+").*$",RegExp(a,d?"i":"");a=b?a:oa(a);return RegExp(a,d?"i":"")}function Ja(a,b){return"function"===typeof j.ext.ofnSearch[b]?j.ext.ofnSearch[b](a):null===a?"":"html"==b?a.replace(/[\r\n]/g," ").replace(/<.*?>/g,""):"string"===typeof a?a.replace(/[\r\n]/g," "):a}function oa(a){return a.replace(RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),
+"\\$1")}function Ca(a){var b=l.createElement("div");b.className=a.oClasses.sInfo;a.aanFeatures.i||(a.aoDrawCallback.push({fn:Ka,sName:"information"}),b.id=a.sTableId+"_info");a.nTable.setAttribute("aria-describedby",a.sTableId+"_info");return b}function Ka(a){if(a.oFeatures.bInfo&&0!==a.aanFeatures.i.length){var b=a.oLanguage,c=a._iDisplayStart+1,d=a.fnDisplayEnd(),i=a.fnRecordsTotal(),f=a.fnRecordsDisplay(),g;g=0===f?b.sInfoEmpty:b.sInfo;f!=i&&(g+=" "+b.sInfoFiltered);g+=b.sInfoPostFix;g=ja(a,g);
+null!==b.fnInfoCallback&&(g=b.fnInfoCallback.call(a.oInstance,a,c,d,i,f,g));a=a.aanFeatures.i;b=0;for(c=a.length;b<c;b++)h(a[b]).html(g)}}function ja(a,b){var c=a.fnFormatNumber(a._iDisplayStart+1),d=a.fnDisplayEnd(),d=a.fnFormatNumber(d),i=a.fnRecordsDisplay(),i=a.fnFormatNumber(i),f=a.fnRecordsTotal(),f=a.fnFormatNumber(f);a.oScroll.bInfinite&&(c=a.fnFormatNumber(1));return b.replace(/_START_/g,c).replace(/_END_/g,d).replace(/_TOTAL_/g,i).replace(/_MAX_/g,f)}function ba(a){var b,c,d=a.iInitDisplayStart;
+if(!1===a.bInitialised)setTimeout(function(){ba(a)},200);else{xa(a);va(a);W(a,a.aoHeader);a.nTFoot&&W(a,a.aoFooter);E(a,!0);a.oFeatures.bAutoWidth&&da(a);b=0;for(c=a.aoColumns.length;b<c;b++)null!==a.aoColumns[b].sWidth&&(a.aoColumns[b].nTh.style.width=q(a.aoColumns[b].sWidth));a.oFeatures.bSort?O(a):a.oFeatures.bFilter?K(a,a.oPreviousSearch):(a.aiDisplay=a.aiDisplayMaster.slice(),y(a),x(a));null!==a.sAjaxSource&&!a.oFeatures.bServerSide?(c=[],ka(a,c),a.fnServerData.call(a.oInstance,a.sAjaxSource,
+c,function(c){var f=a.sAjaxDataProp!==""?Q(a.sAjaxDataProp)(c):c;for(b=0;b<f.length;b++)H(a,f[b]);a.iInitDisplayStart=d;if(a.oFeatures.bSort)O(a);else{a.aiDisplay=a.aiDisplayMaster.slice();y(a);x(a)}E(a,false);$(a,c)},a)):a.oFeatures.bServerSide||(E(a,!1),$(a))}}function $(a,b){a._bInitComplete=!0;A(a,"aoInitComplete","init",[a,b])}function pa(a){var b=j.defaults.oLanguage;!a.sEmptyTable&&(a.sZeroRecords&&"No data available in table"===b.sEmptyTable)&&p(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&
+(a.sZeroRecords&&"Loading..."===b.sLoadingRecords)&&p(a,a,"sZeroRecords","sLoadingRecords")}function ya(a){if(a.oScroll.bInfinite)return null;var b='<select size="1" '+('name="'+a.sTableId+'_length"')+">",c,d,i=a.aLengthMenu;if(2==i.length&&"object"===typeof i[0]&&"object"===typeof i[1]){c=0;for(d=i[0].length;c<d;c++)b+='<option value="'+i[0][c]+'">'+i[1][c]+"</option>"}else{c=0;for(d=i.length;c<d;c++)b+='<option value="'+i[c]+'">'+i[c]+"</option>"}b+="</select>";i=l.createElement("div");a.aanFeatures.l||
+(i.id=a.sTableId+"_length");i.className=a.oClasses.sLength;i.innerHTML="<label>"+a.oLanguage.sLengthMenu.replace("_MENU_",b)+"</label>";h('select option[value="'+a._iDisplayLength+'"]',i).attr("selected",!0);h("select",i).bind("change.DT",function(){var b=h(this).val(),i=a.aanFeatures.l;c=0;for(d=i.length;c<d;c++)i[c]!=this.parentNode&&h("select",i[c]).val(b);a._iDisplayLength=parseInt(b,10);y(a);if(a.fnDisplayEnd()==a.fnRecordsDisplay()){a._iDisplayStart=a.fnDisplayEnd()-a._iDisplayLength;if(a._iDisplayStart<
+0)a._iDisplayStart=0}if(a._iDisplayLength==-1)a._iDisplayStart=0;x(a)});h("select",i).attr("aria-controls",a.sTableId);return i}function y(a){a._iDisplayEnd=!1===a.oFeatures.bPaginate?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength>a.aiDisplay.length||-1==a._iDisplayLength?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength}function Da(a){if(a.oScroll.bInfinite)return null;var b=l.createElement("div");b.className=a.oClasses.sPaging+a.sPaginationType;j.ext.oPagination[a.sPaginationType].fnInit(a,
+b,function(a){y(a);x(a)});a.aanFeatures.p||a.aoDrawCallback.push({fn:function(a){j.ext.oPagination[a.sPaginationType].fnUpdate(a,function(a){y(a);x(a)})},sName:"pagination"});return b}function qa(a,b){var c=a._iDisplayStart;if("number"===typeof b)a._iDisplayStart=b*a._iDisplayLength,a._iDisplayStart>a.fnRecordsDisplay()&&(a._iDisplayStart=0);else if("first"==b)a._iDisplayStart=0;else if("previous"==b)a._iDisplayStart=0<=a._iDisplayLength?a._iDisplayStart-a._iDisplayLength:0,0>a._iDisplayStart&&(a._iDisplayStart=
+0);else if("next"==b)0<=a._iDisplayLength?a._iDisplayStart+a._iDisplayLength<a.fnRecordsDisplay()&&(a._iDisplayStart+=a._iDisplayLength):a._iDisplayStart=0;else if("last"==b)if(0<=a._iDisplayLength){var d=parseInt((a.fnRecordsDisplay()-1)/a._iDisplayLength,10)+1;a._iDisplayStart=(d-1)*a._iDisplayLength}else a._iDisplayStart=0;else D(a,0,"Unknown paging action: "+b);h(a.oInstance).trigger("page",a);return c!=a._iDisplayStart}function Aa(a){var b=l.createElement("div");a.aanFeatures.r||(b.id=a.sTableId+
+"_processing");b.innerHTML=a.oLanguage.sProcessing;b.className=a.oClasses.sProcessing;a.nTable.parentNode.insertBefore(b,a.nTable);return b}function E(a,b){if(a.oFeatures.bProcessing)for(var c=a.aanFeatures.r,d=0,i=c.length;d<i;d++)c[d].style.visibility=b?"visible":"hidden";h(a.oInstance).trigger("processing",[a,b])}function Ba(a){if(""===a.oScroll.sX&&""===a.oScroll.sY)return a.nTable;var b=l.createElement("div"),c=l.createElement("div"),d=l.createElement("div"),i=l.createElement("div"),f=l.createElement("div"),
+g=l.createElement("div"),e=a.nTable.cloneNode(!1),j=a.nTable.cloneNode(!1),o=a.nTable.getElementsByTagName("thead")[0],k=0===a.nTable.getElementsByTagName("tfoot").length?null:a.nTable.getElementsByTagName("tfoot")[0],m=a.oClasses;c.appendChild(d);f.appendChild(g);i.appendChild(a.nTable);b.appendChild(c);b.appendChild(i);d.appendChild(e);e.appendChild(o);null!==k&&(b.appendChild(f),g.appendChild(j),j.appendChild(k));b.className=m.sScrollWrapper;c.className=m.sScrollHead;d.className=m.sScrollHeadInner;
+i.className=m.sScrollBody;f.className=m.sScrollFoot;g.className=m.sScrollFootInner;a.oScroll.bAutoCss&&(c.style.overflow="hidden",c.style.position="relative",f.style.overflow="hidden",i.style.overflow="auto");c.style.border="0";c.style.width="100%";f.style.border="0";d.style.width=""!==a.oScroll.sXInner?a.oScroll.sXInner:"100%";e.removeAttribute("id");e.style.marginLeft="0";a.nTable.style.marginLeft="0";null!==k&&(j.removeAttribute("id"),j.style.marginLeft="0");d=h(a.nTable).children("caption");0<
+d.length&&(d=d[0],"top"===d._captionSide?e.appendChild(d):"bottom"===d._captionSide&&k&&j.appendChild(d));""!==a.oScroll.sX&&(c.style.width=q(a.oScroll.sX),i.style.width=q(a.oScroll.sX),null!==k&&(f.style.width=q(a.oScroll.sX)),h(i).scroll(function(){c.scrollLeft=this.scrollLeft;if(k!==null)f.scrollLeft=this.scrollLeft}));""!==a.oScroll.sY&&(i.style.height=q(a.oScroll.sY));a.aoDrawCallback.push({fn:La,sName:"scrolling"});a.oScroll.bInfinite&&h(i).scroll(function(){if(!a.bDrawing&&h(this).scrollTop()!==
+0&&h(this).scrollTop()+h(this).height()>h(a.nTable).height()-a.oScroll.iLoadGap&&a.fnDisplayEnd()<a.fnRecordsDisplay()){qa(a,"next");y(a);x(a)}});a.nScrollHead=c;a.nScrollFoot=f;return b}function La(a){var b=a.nScrollHead.getElementsByTagName("div")[0],c=b.getElementsByTagName("table")[0],d=a.nTable.parentNode,i,f,g,e,j,o,k,m,p=[],n=[],l=null!==a.nTFoot?a.nScrollFoot.getElementsByTagName("div")[0]:null,R=null!==a.nTFoot?l.getElementsByTagName("table")[0]:null,r=a.oBrowser.bScrollOversize,s=function(a){k=
+a.style;k.paddingTop="0";k.paddingBottom="0";k.borderTopWidth="0";k.borderBottomWidth="0";k.height=0};h(a.nTable).children("thead, tfoot").remove();i=h(a.nTHead).clone()[0];a.nTable.insertBefore(i,a.nTable.childNodes[0]);g=a.nTHead.getElementsByTagName("tr");e=i.getElementsByTagName("tr");null!==a.nTFoot&&(j=h(a.nTFoot).clone()[0],a.nTable.insertBefore(j,a.nTable.childNodes[1]),o=a.nTFoot.getElementsByTagName("tr"),j=j.getElementsByTagName("tr"));""===a.oScroll.sX&&(d.style.width="100%",b.parentNode.style.width=
+"100%");var t=N(a,i);i=0;for(f=t.length;i<f;i++)m=G(a,i),t[i].style.width=a.aoColumns[m].sWidth;null!==a.nTFoot&&C(function(a){a.style.width=""},j);a.oScroll.bCollapse&&""!==a.oScroll.sY&&(d.style.height=d.offsetHeight+a.nTHead.offsetHeight+"px");i=h(a.nTable).outerWidth();if(""===a.oScroll.sX){if(a.nTable.style.width="100%",r&&(h("tbody",d).height()>d.offsetHeight||"scroll"==h(d).css("overflow-y")))a.nTable.style.width=q(h(a.nTable).outerWidth()-a.oScroll.iBarWidth)}else""!==a.oScroll.sXInner?a.nTable.style.width=
+q(a.oScroll.sXInner):i==h(d).width()&&h(d).height()<h(a.nTable).height()?(a.nTable.style.width=q(i-a.oScroll.iBarWidth),h(a.nTable).outerWidth()>i-a.oScroll.iBarWidth&&(a.nTable.style.width=q(i))):a.nTable.style.width=q(i);i=h(a.nTable).outerWidth();C(s,e);C(function(a){p.push(q(h(a).width()))},e);C(function(a,b){a.style.width=p[b]},g);h(e).height(0);null!==a.nTFoot&&(C(s,j),C(function(a){n.push(q(h(a).width()))},j),C(function(a,b){a.style.width=n[b]},o),h(j).height(0));C(function(a,b){a.innerHTML=
+"";a.style.width=p[b]},e);null!==a.nTFoot&&C(function(a,b){a.innerHTML="";a.style.width=n[b]},j);if(h(a.nTable).outerWidth()<i){g=d.scrollHeight>d.offsetHeight||"scroll"==h(d).css("overflow-y")?i+a.oScroll.iBarWidth:i;if(r&&(d.scrollHeight>d.offsetHeight||"scroll"==h(d).css("overflow-y")))a.nTable.style.width=q(g-a.oScroll.iBarWidth);d.style.width=q(g);a.nScrollHead.style.width=q(g);null!==a.nTFoot&&(a.nScrollFoot.style.width=q(g));""===a.oScroll.sX?D(a,1,"The table cannot fit into the current element which will cause column misalignment. The table has been drawn at its minimum possible width."):
+""!==a.oScroll.sXInner&&D(a,1,"The table cannot fit into the current element which will cause column misalignment. Increase the sScrollXInner value or remove it to allow automatic calculation")}else d.style.width=q("100%"),a.nScrollHead.style.width=q("100%"),null!==a.nTFoot&&(a.nScrollFoot.style.width=q("100%"));""===a.oScroll.sY&&r&&(d.style.height=q(a.nTable.offsetHeight+a.oScroll.iBarWidth));""!==a.oScroll.sY&&a.oScroll.bCollapse&&(d.style.height=q(a.oScroll.sY),r=""!==a.oScroll.sX&&a.nTable.offsetWidth>
+d.offsetWidth?a.oScroll.iBarWidth:0,a.nTable.offsetHeight<d.offsetHeight&&(d.style.height=q(a.nTable.offsetHeight+r)));r=h(a.nTable).outerWidth();c.style.width=q(r);b.style.width=q(r);c=h(a.nTable).height()>d.clientHeight||"scroll"==h(d).css("overflow-y");b.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px";null!==a.nTFoot&&(R.style.width=q(r),l.style.width=q(r),l.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px");h(d).scroll();if(a.bSorted||a.bFiltered)d.scrollTop=0}function C(a,b,c){for(var d=
+0,i=0,f=b.length,g,e;i<f;){g=b[i].firstChild;for(e=c?c[i].firstChild:null;g;)1===g.nodeType&&(c?a(g,e,d):a(g,d),d++),g=g.nextSibling,e=c?e.nextSibling:null;i++}}function Ma(a,b){if(!a||null===a||""===a)return 0;b||(b=l.body);var c,d=l.createElement("div");d.style.width=q(a);b.appendChild(d);c=d.offsetWidth;b.removeChild(d);return c}function da(a){var b=0,c,d=0,i=a.aoColumns.length,f,e,j=h("th",a.nTHead),o=a.nTable.getAttribute("width");e=a.nTable.parentNode;for(f=0;f<i;f++)a.aoColumns[f].bVisible&&
+(d++,null!==a.aoColumns[f].sWidth&&(c=Ma(a.aoColumns[f].sWidthOrig,e),null!==c&&(a.aoColumns[f].sWidth=q(c)),b++));if(i==j.length&&0===b&&d==i&&""===a.oScroll.sX&&""===a.oScroll.sY)for(f=0;f<a.aoColumns.length;f++)c=h(j[f]).width(),null!==c&&(a.aoColumns[f].sWidth=q(c));else{b=a.nTable.cloneNode(!1);f=a.nTHead.cloneNode(!0);d=l.createElement("tbody");c=l.createElement("tr");b.removeAttribute("id");b.appendChild(f);null!==a.nTFoot&&(b.appendChild(a.nTFoot.cloneNode(!0)),C(function(a){a.style.width=
+""},b.getElementsByTagName("tr")));b.appendChild(d);d.appendChild(c);d=h("thead th",b);0===d.length&&(d=h("tbody tr:eq(0)>td",b));j=N(a,f);for(f=d=0;f<i;f++){var k=a.aoColumns[f];k.bVisible&&null!==k.sWidthOrig&&""!==k.sWidthOrig?j[f-d].style.width=q(k.sWidthOrig):k.bVisible?j[f-d].style.width="":d++}for(f=0;f<i;f++)a.aoColumns[f].bVisible&&(d=Na(a,f),null!==d&&(d=d.cloneNode(!0),""!==a.aoColumns[f].sContentPadding&&(d.innerHTML+=a.aoColumns[f].sContentPadding),c.appendChild(d)));e.appendChild(b);
+""!==a.oScroll.sX&&""!==a.oScroll.sXInner?b.style.width=q(a.oScroll.sXInner):""!==a.oScroll.sX?(b.style.width="",h(b).width()<e.offsetWidth&&(b.style.width=q(e.offsetWidth))):""!==a.oScroll.sY?b.style.width=q(e.offsetWidth):o&&(b.style.width=q(o));b.style.visibility="hidden";Oa(a,b);i=h("tbody tr:eq(0)",b).children();0===i.length&&(i=N(a,h("thead",b)[0]));if(""!==a.oScroll.sX){for(f=d=e=0;f<a.aoColumns.length;f++)a.aoColumns[f].bVisible&&(e=null===a.aoColumns[f].sWidthOrig?e+h(i[d]).outerWidth():
+e+(parseInt(a.aoColumns[f].sWidth.replace("px",""),10)+(h(i[d]).outerWidth()-h(i[d]).width())),d++);b.style.width=q(e);a.nTable.style.width=q(e)}for(f=d=0;f<a.aoColumns.length;f++)a.aoColumns[f].bVisible&&(e=h(i[d]).width(),null!==e&&0<e&&(a.aoColumns[f].sWidth=q(e)),d++);i=h(b).css("width");a.nTable.style.width=-1!==i.indexOf("%")?i:q(h(b).outerWidth());b.parentNode.removeChild(b)}o&&(a.nTable.style.width=q(o))}function Oa(a,b){""===a.oScroll.sX&&""!==a.oScroll.sY?(h(b).width(),b.style.width=q(h(b).outerWidth()-
+a.oScroll.iBarWidth)):""!==a.oScroll.sX&&(b.style.width=q(h(b).outerWidth()))}function Na(a,b){var c=Pa(a,b);if(0>c)return null;if(null===a.aoData[c].nTr){var d=l.createElement("td");d.innerHTML=v(a,c,b,"");return d}return J(a,c)[b]}function Pa(a,b){for(var c=-1,d=-1,i=0;i<a.aoData.length;i++){var e=v(a,i,b,"display")+"",e=e.replace(/<.*?>/g,"");e.length>c&&(c=e.length,d=i)}return d}function q(a){if(null===a)return"0px";if("number"==typeof a)return 0>a?"0px":a+"px";var b=a.charCodeAt(a.length-1);
+return 48>b||57<b?a:a+"px"}function Qa(){var a=l.createElement("p"),b=a.style;b.width="100%";b.height="200px";b.padding="0px";var c=l.createElement("div"),b=c.style;b.position="absolute";b.top="0px";b.left="0px";b.visibility="hidden";b.width="200px";b.height="150px";b.padding="0px";b.overflow="hidden";c.appendChild(a);l.body.appendChild(c);b=a.offsetWidth;c.style.overflow="scroll";a=a.offsetWidth;b==a&&(a=c.clientWidth);l.body.removeChild(c);return b-a}function O(a,b){var c,d,i,e,g,k,o=[],m=[],p=
+j.ext.oSort,l=a.aoData,q=a.aoColumns,G=a.oLanguage.oAria;if(!a.oFeatures.bServerSide&&(0!==a.aaSorting.length||null!==a.aaSortingFixed)){o=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(c=0;c<o.length;c++)if(d=o[c][0],i=R(a,d),e=a.aoColumns[d].sSortDataType,j.ext.afnSortData[e])if(g=j.ext.afnSortData[e].call(a.oInstance,a,d,i),g.length===l.length){i=0;for(e=l.length;i<e;i++)F(a,i,d,g[i])}else D(a,0,"Returned data sort array (col "+d+") is the wrong length");c=
+0;for(d=a.aiDisplayMaster.length;c<d;c++)m[a.aiDisplayMaster[c]]=c;var r=o.length,s;c=0;for(d=l.length;c<d;c++)for(i=0;i<r;i++){s=q[o[i][0]].aDataSort;g=0;for(k=s.length;g<k;g++)e=q[s[g]].sType,e=p[(e?e:"string")+"-pre"],l[c]._aSortData[s[g]]=e?e(v(a,c,s[g],"sort")):v(a,c,s[g],"sort")}a.aiDisplayMaster.sort(function(a,b){var c,d,e,i,f;for(c=0;c<r;c++){f=q[o[c][0]].aDataSort;d=0;for(e=f.length;d<e;d++)if(i=q[f[d]].sType,i=p[(i?i:"string")+"-"+o[c][1]](l[a]._aSortData[f[d]],l[b]._aSortData[f[d]]),0!==
+i)return i}return p["numeric-asc"](m[a],m[b])})}(b===n||b)&&!a.oFeatures.bDeferRender&&P(a);c=0;for(d=a.aoColumns.length;c<d;c++)e=q[c].sTitle.replace(/<.*?>/g,""),i=q[c].nTh,i.removeAttribute("aria-sort"),i.removeAttribute("aria-label"),q[c].bSortable?0<o.length&&o[0][0]==c?(i.setAttribute("aria-sort","asc"==o[0][1]?"ascending":"descending"),i.setAttribute("aria-label",e+("asc"==(q[c].asSorting[o[0][2]+1]?q[c].asSorting[o[0][2]+1]:q[c].asSorting[0])?G.sSortAscending:G.sSortDescending))):i.setAttribute("aria-label",
+e+("asc"==q[c].asSorting[0]?G.sSortAscending:G.sSortDescending)):i.setAttribute("aria-label",e);a.bSorted=!0;h(a.oInstance).trigger("sort",a);a.oFeatures.bFilter?K(a,a.oPreviousSearch,1):(a.aiDisplay=a.aiDisplayMaster.slice(),a._iDisplayStart=0,y(a),x(a))}function ia(a,b,c,d){Ra(b,{},function(b){if(!1!==a.aoColumns[c].bSortable){var e=function(){var d,e;if(b.shiftKey){for(var f=!1,h=0;h<a.aaSorting.length;h++)if(a.aaSorting[h][0]==c){f=!0;d=a.aaSorting[h][0];e=a.aaSorting[h][2]+1;a.aoColumns[d].asSorting[e]?
+(a.aaSorting[h][1]=a.aoColumns[d].asSorting[e],a.aaSorting[h][2]=e):a.aaSorting.splice(h,1);break}!1===f&&a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0])}else 1==a.aaSorting.length&&a.aaSorting[0][0]==c?(d=a.aaSorting[0][0],e=a.aaSorting[0][2]+1,a.aoColumns[d].asSorting[e]||(e=0),a.aaSorting[0][1]=a.aoColumns[d].asSorting[e],a.aaSorting[0][2]=e):(a.aaSorting.splice(0,a.aaSorting.length),a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0]));O(a)};a.oFeatures.bProcessing?(E(a,!0),setTimeout(function(){e();
+a.oFeatures.bServerSide||E(a,!1)},0)):e();"function"==typeof d&&d(a)}})}function P(a){var b,c,d,e,f,g=a.aoColumns.length,j=a.oClasses;for(b=0;b<g;b++)a.aoColumns[b].bSortable&&h(a.aoColumns[b].nTh).removeClass(j.sSortAsc+" "+j.sSortDesc+" "+a.aoColumns[b].sSortingClass);c=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(b=0;b<a.aoColumns.length;b++)if(a.aoColumns[b].bSortable){f=a.aoColumns[b].sSortingClass;e=-1;for(d=0;d<c.length;d++)if(c[d][0]==b){f="asc"==c[d][1]?
+j.sSortAsc:j.sSortDesc;e=d;break}h(a.aoColumns[b].nTh).addClass(f);a.bJUI&&(f=h("span."+j.sSortIcon,a.aoColumns[b].nTh),f.removeClass(j.sSortJUIAsc+" "+j.sSortJUIDesc+" "+j.sSortJUI+" "+j.sSortJUIAscAllowed+" "+j.sSortJUIDescAllowed),f.addClass(-1==e?a.aoColumns[b].sSortingClassJUI:"asc"==c[e][1]?j.sSortJUIAsc:j.sSortJUIDesc))}else h(a.aoColumns[b].nTh).addClass(a.aoColumns[b].sSortingClass);f=j.sSortColumn;if(a.oFeatures.bSort&&a.oFeatures.bSortClasses){a=J(a);e=[];for(b=0;b<g;b++)e.push("");b=0;
+for(d=1;b<c.length;b++)j=parseInt(c[b][0],10),e[j]=f+d,3>d&&d++;f=RegExp(f+"[123]");var o;b=0;for(c=a.length;b<c;b++)j=b%g,d=a[b].className,o=e[j],j=d.replace(f,o),j!=d?a[b].className=h.trim(j):0<o.length&&-1==d.indexOf(o)&&(a[b].className=d+" "+o)}}function ra(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b,c;b=a.oScroll.bInfinite;var d={iCreate:(new Date).getTime(),iStart:b?0:a._iDisplayStart,iEnd:b?a._iDisplayLength:a._iDisplayEnd,iLength:a._iDisplayLength,aaSorting:h.extend(!0,[],a.aaSorting),
+oSearch:h.extend(!0,{},a.oPreviousSearch),aoSearchCols:h.extend(!0,[],a.aoPreSearchCols),abVisCols:[]};b=0;for(c=a.aoColumns.length;b<c;b++)d.abVisCols.push(a.aoColumns[b].bVisible);A(a,"aoStateSaveParams","stateSaveParams",[a,d]);a.fnStateSave.call(a.oInstance,a,d)}}function Sa(a,b){if(a.oFeatures.bStateSave){var c=a.fnStateLoad.call(a.oInstance,a);if(c){var d=A(a,"aoStateLoadParams","stateLoadParams",[a,c]);if(-1===h.inArray(!1,d)){a.oLoadedState=h.extend(!0,{},c);a._iDisplayStart=c.iStart;a.iInitDisplayStart=
+c.iStart;a._iDisplayEnd=c.iEnd;a._iDisplayLength=c.iLength;a.aaSorting=c.aaSorting.slice();a.saved_aaSorting=c.aaSorting.slice();h.extend(a.oPreviousSearch,c.oSearch);h.extend(!0,a.aoPreSearchCols,c.aoSearchCols);b.saved_aoColumns=[];for(d=0;d<c.abVisCols.length;d++)b.saved_aoColumns[d]={},b.saved_aoColumns[d].bVisible=c.abVisCols[d];A(a,"aoStateLoaded","stateLoaded",[a,c])}}}}function s(a){for(var b=0;b<j.settings.length;b++)if(j.settings[b].nTable===a)return j.settings[b];return null}function T(a){for(var b=
+[],a=a.aoData,c=0,d=a.length;c<d;c++)null!==a[c].nTr&&b.push(a[c].nTr);return b}function J(a,b){var c=[],d,e,f,g,h,j;e=0;var o=a.aoData.length;b!==n&&(e=b,o=b+1);for(f=e;f<o;f++)if(j=a.aoData[f],null!==j.nTr){e=[];for(d=j.nTr.firstChild;d;)g=d.nodeName.toLowerCase(),("td"==g||"th"==g)&&e.push(d),d=d.nextSibling;g=d=0;for(h=a.aoColumns.length;g<h;g++)a.aoColumns[g].bVisible?c.push(e[g-d]):(c.push(j._anHidden[g]),d++)}return c}function D(a,b,c){a=null===a?"DataTables warning: "+c:"DataTables warning (table id = '"+
+a.sTableId+"'): "+c;if(0===b)if("alert"==j.ext.sErrMode)alert(a);else throw Error(a);else X.console&&console.log&&console.log(a)}function p(a,b,c,d){d===n&&(d=c);b[c]!==n&&(a[d]=b[c])}function Ta(a,b){var c,d;for(d in b)b.hasOwnProperty(d)&&(c=b[d],"object"===typeof e[d]&&null!==c&&!1===h.isArray(c)?h.extend(!0,a[d],c):a[d]=c);return a}function Ra(a,b,c){h(a).bind("click.DT",b,function(b){a.blur();c(b)}).bind("keypress.DT",b,function(a){13===a.which&&c(a)}).bind("selectstart.DT",function(){return!1})}
+function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function A(a,b,c,d){for(var b=a[b],e=[],f=b.length-1;0<=f;f--)e.push(b[f].fn.apply(a.oInstance,d));null!==c&&h(a.oInstance).trigger(c,d);return e}function Ua(a){var b=h('<div style="position:absolute; top:0; left:0; height:1px; width:1px; overflow:hidden"><div style="position:absolute; top:1px; left:1px; width:100px; overflow:scroll;"><div id="DT_BrowserTest" style="width:100%; height:10px;"></div></div></div>')[0];l.body.appendChild(b);a.oBrowser.bScrollOversize=
+100===h("#DT_BrowserTest",b)[0].offsetWidth?!0:!1;l.body.removeChild(b)}function Va(a){return function(){var b=[s(this[j.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return j.ext.oApi[a].apply(this,b)}}var U=/\[.*?\]$/,Wa=X.JSON?JSON.stringify:function(a){var b=typeof a;if("object"!==b||null===a)return"string"===b&&(a='"'+a+'"'),a+"";var c,d,e=[],f=h.isArray(a);for(c in a)d=a[c],b=typeof d,"string"===b?d='"'+d+'"':"object"===b&&null!==d&&(d=Wa(d)),e.push((f?"":'"'+c+'":')+d);return(f?
+"[":"{")+e+(f?"]":"}")};this.$=function(a,b){var c,d,e=[],f;d=s(this[j.ext.iApiIndex]);var g=d.aoData,o=d.aiDisplay,k=d.aiDisplayMaster;b||(b={});b=h.extend({},{filter:"none",order:"current",page:"all"},b);if("current"==b.page){c=d._iDisplayStart;for(d=d.fnDisplayEnd();c<d;c++)(f=g[o[c]].nTr)&&e.push(f)}else if("current"==b.order&&"none"==b.filter){c=0;for(d=k.length;c<d;c++)(f=g[k[c]].nTr)&&e.push(f)}else if("current"==b.order&&"applied"==b.filter){c=0;for(d=o.length;c<d;c++)(f=g[o[c]].nTr)&&e.push(f)}else if("original"==
+b.order&&"none"==b.filter){c=0;for(d=g.length;c<d;c++)(f=g[c].nTr)&&e.push(f)}else if("original"==b.order&&"applied"==b.filter){c=0;for(d=g.length;c<d;c++)f=g[c].nTr,-1!==h.inArray(c,o)&&f&&e.push(f)}else D(d,1,"Unknown selection options");e=h(e);c=e.filter(a);e=e.find(a);return h([].concat(h.makeArray(c),h.makeArray(e)))};this._=function(a,b){var c=[],d,e,f=this.$(a,b);d=0;for(e=f.length;d<e;d++)c.push(this.fnGetData(f[d]));return c};this.fnAddData=function(a,b){if(0===a.length)return[];var c=[],
+d,e=s(this[j.ext.iApiIndex]);if("object"===typeof a[0]&&null!==a[0])for(var f=0;f<a.length;f++){d=H(e,a[f]);if(-1==d)return c;c.push(d)}else{d=H(e,a);if(-1==d)return c;c.push(d)}e.aiDisplay=e.aiDisplayMaster.slice();(b===n||b)&&aa(e);return c};this.fnAdjustColumnSizing=function(a){var b=s(this[j.ext.iApiIndex]);k(b);a===n||a?this.fnDraw(!1):(""!==b.oScroll.sX||""!==b.oScroll.sY)&&this.oApi._fnScrollDraw(b)};this.fnClearTable=function(a){var b=s(this[j.ext.iApiIndex]);ga(b);(a===n||a)&&x(b)};this.fnClose=
+function(a){for(var b=s(this[j.ext.iApiIndex]),c=0;c<b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a)return(a=b.aoOpenRows[c].nTr.parentNode)&&a.removeChild(b.aoOpenRows[c].nTr),b.aoOpenRows.splice(c,1),0;return 1};this.fnDeleteRow=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e,f,a="object"===typeof a?I(d,a):a,g=d.aoData.splice(a,1);e=0;for(f=d.aoData.length;e<f;e++)null!==d.aoData[e].nTr&&(d.aoData[e].nTr._DT_RowIndex=e);e=h.inArray(a,d.aiDisplay);d.asDataSearch.splice(e,1);ha(d.aiDisplayMaster,
+a);ha(d.aiDisplay,a);"function"===typeof b&&b.call(this,d,g);d._iDisplayStart>=d.fnRecordsDisplay()&&(d._iDisplayStart-=d._iDisplayLength,0>d._iDisplayStart&&(d._iDisplayStart=0));if(c===n||c)y(d),x(d);return g};this.fnDestroy=function(a){var b=s(this[j.ext.iApiIndex]),c=b.nTableWrapper.parentNode,d=b.nTBody,i,f,a=a===n?!1:a;b.bDestroying=!0;A(b,"aoDestroyCallback","destroy",[b]);if(!a){i=0;for(f=b.aoColumns.length;i<f;i++)!1===b.aoColumns[i].bVisible&&this.fnSetColumnVis(i,!0)}h(b.nTableWrapper).find("*").andSelf().unbind(".DT");
+h("tbody>tr>td."+b.oClasses.sRowEmpty,b.nTable).parent().remove();b.nTable!=b.nTHead.parentNode&&(h(b.nTable).children("thead").remove(),b.nTable.appendChild(b.nTHead));b.nTFoot&&b.nTable!=b.nTFoot.parentNode&&(h(b.nTable).children("tfoot").remove(),b.nTable.appendChild(b.nTFoot));b.nTable.parentNode.removeChild(b.nTable);h(b.nTableWrapper).remove();b.aaSorting=[];b.aaSortingFixed=[];P(b);h(T(b)).removeClass(b.asStripeClasses.join(" "));h("th, td",b.nTHead).removeClass([b.oClasses.sSortable,b.oClasses.sSortableAsc,
+b.oClasses.sSortableDesc,b.oClasses.sSortableNone].join(" "));b.bJUI&&(h("th span."+b.oClasses.sSortIcon+", td span."+b.oClasses.sSortIcon,b.nTHead).remove(),h("th, td",b.nTHead).each(function(){var a=h("div."+b.oClasses.sSortJUIWrapper,this),c=a.contents();h(this).append(c);a.remove()}));!a&&b.nTableReinsertBefore?c.insertBefore(b.nTable,b.nTableReinsertBefore):a||c.appendChild(b.nTable);i=0;for(f=b.aoData.length;i<f;i++)null!==b.aoData[i].nTr&&d.appendChild(b.aoData[i].nTr);!0===b.oFeatures.bAutoWidth&&
+(b.nTable.style.width=q(b.sDestroyWidth));if(f=b.asDestroyStripes.length){a=h(d).children("tr");for(i=0;i<f;i++)a.filter(":nth-child("+f+"n + "+i+")").addClass(b.asDestroyStripes[i])}i=0;for(f=j.settings.length;i<f;i++)j.settings[i]==b&&j.settings.splice(i,1);e=b=null};this.fnDraw=function(a){var b=s(this[j.ext.iApiIndex]);!1===a?(y(b),x(b)):aa(b)};this.fnFilter=function(a,b,c,d,e,f){var g=s(this[j.ext.iApiIndex]);if(g.oFeatures.bFilter){if(c===n||null===c)c=!1;if(d===n||null===d)d=!0;if(e===n||null===
+e)e=!0;if(f===n||null===f)f=!0;if(b===n||null===b){if(K(g,{sSearch:a+"",bRegex:c,bSmart:d,bCaseInsensitive:f},1),e&&g.aanFeatures.f){b=g.aanFeatures.f;c=0;for(d=b.length;c<d;c++)try{b[c]._DT_Input!=l.activeElement&&h(b[c]._DT_Input).val(a)}catch(o){h(b[c]._DT_Input).val(a)}}}else h.extend(g.aoPreSearchCols[b],{sSearch:a+"",bRegex:c,bSmart:d,bCaseInsensitive:f}),K(g,g.oPreviousSearch,1)}};this.fnGetData=function(a,b){var c=s(this[j.ext.iApiIndex]);if(a!==n){var d=a;if("object"===typeof a){var e=a.nodeName.toLowerCase();
+"tr"===e?d=I(c,a):"td"===e&&(d=I(c,a.parentNode),b=fa(c,d,a))}return b!==n?v(c,d,b,""):c.aoData[d]!==n?c.aoData[d]._aData:null}return Z(c)};this.fnGetNodes=function(a){var b=s(this[j.ext.iApiIndex]);return a!==n?b.aoData[a]!==n?b.aoData[a].nTr:null:T(b)};this.fnGetPosition=function(a){var b=s(this[j.ext.iApiIndex]),c=a.nodeName.toUpperCase();return"TR"==c?I(b,a):"TD"==c||"TH"==c?(c=I(b,a.parentNode),a=fa(b,c,a),[c,R(b,a),a]):null};this.fnIsOpen=function(a){for(var b=s(this[j.ext.iApiIndex]),c=0;c<
+b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a)return!0;return!1};this.fnOpen=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e=T(d);if(-1!==h.inArray(a,e)){this.fnClose(a);var e=l.createElement("tr"),f=l.createElement("td");e.appendChild(f);f.className=c;f.colSpan=t(d);"string"===typeof b?f.innerHTML=b:h(f).html(b);b=h("tr",d.nTBody);-1!=h.inArray(a,b)&&h(e).insertAfter(a);d.aoOpenRows.push({nTr:e,nParent:a});return e}};this.fnPageChange=function(a,b){var c=s(this[j.ext.iApiIndex]);qa(c,a);
+y(c);(b===n||b)&&x(c)};this.fnSetColumnVis=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e,f,g=d.aoColumns,h=d.aoData,o,m;if(g[a].bVisible!=b){if(b){for(e=f=0;e<a;e++)g[e].bVisible&&f++;m=f>=t(d);if(!m)for(e=a;e<g.length;e++)if(g[e].bVisible){o=e;break}e=0;for(f=h.length;e<f;e++)null!==h[e].nTr&&(m?h[e].nTr.appendChild(h[e]._anHidden[a]):h[e].nTr.insertBefore(h[e]._anHidden[a],J(d,e)[o]))}else{e=0;for(f=h.length;e<f;e++)null!==h[e].nTr&&(o=J(d,e)[a],h[e]._anHidden[a]=o,o.parentNode.removeChild(o))}g[a].bVisible=
+b;W(d,d.aoHeader);d.nTFoot&&W(d,d.aoFooter);e=0;for(f=d.aoOpenRows.length;e<f;e++)d.aoOpenRows[e].nTr.colSpan=t(d);if(c===n||c)k(d),x(d);ra(d)}};this.fnSettings=function(){return s(this[j.ext.iApiIndex])};this.fnSort=function(a){var b=s(this[j.ext.iApiIndex]);b.aaSorting=a;O(b)};this.fnSortListener=function(a,b,c){ia(s(this[j.ext.iApiIndex]),a,b,c)};this.fnUpdate=function(a,b,c,d,e){var f=s(this[j.ext.iApiIndex]),b="object"===typeof b?I(f,b):b;if(h.isArray(a)&&c===n){f.aoData[b]._aData=a.slice();
+for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(v(f,b,c),b,c,!1,!1)}else if(h.isPlainObject(a)&&c===n){f.aoData[b]._aData=h.extend(!0,{},a);for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(v(f,b,c),b,c,!1,!1)}else{F(f,b,c,a);var a=v(f,b,c,"display"),g=f.aoColumns[c];null!==g.fnRender&&(a=S(f,b,c),g.bUseRendered&&F(f,b,c,a));null!==f.aoData[b].nTr&&(J(f,b)[c].innerHTML=a)}c=h.inArray(b,f.aiDisplay);f.asDataSearch[c]=na(f,Y(f,b,"filter",r(f,"bSearchable")));(e===n||e)&&k(f);(d===n||d)&&aa(f);return 0};
+this.fnVersionCheck=j.ext.fnVersionCheck;this.oApi={_fnExternApiFunc:Va,_fnInitialise:ba,_fnInitComplete:$,_fnLanguageCompat:pa,_fnAddColumn:o,_fnColumnOptions:m,_fnAddData:H,_fnCreateTr:ea,_fnGatherData:ua,_fnBuildHead:va,_fnDrawHead:W,_fnDraw:x,_fnReDraw:aa,_fnAjaxUpdate:wa,_fnAjaxParameters:Ea,_fnAjaxUpdateDraw:Fa,_fnServerParams:ka,_fnAddOptionsHtml:xa,_fnFeatureHtmlTable:Ba,_fnScrollDraw:La,_fnAdjustColumnSizing:k,_fnFeatureHtmlFilter:za,_fnFilterComplete:K,_fnFilterCustom:Ia,_fnFilterColumn:Ha,
+_fnFilter:Ga,_fnBuildSearchArray:la,_fnBuildSearchRow:na,_fnFilterCreateSearch:ma,_fnDataToSearch:Ja,_fnSort:O,_fnSortAttachListener:ia,_fnSortingClasses:P,_fnFeatureHtmlPaginate:Da,_fnPageChange:qa,_fnFeatureHtmlInfo:Ca,_fnUpdateInfo:Ka,_fnFeatureHtmlLength:ya,_fnFeatureHtmlProcessing:Aa,_fnProcessingDisplay:E,_fnVisibleToColumnIndex:G,_fnColumnIndexToVisible:R,_fnNodeToDataIndex:I,_fnVisbleColumns:t,_fnCalculateEnd:y,_fnConvertToWidth:Ma,_fnCalculateColumnWidths:da,_fnScrollingWidthAdjust:Oa,_fnGetWidestNode:Na,
+_fnGetMaxLenString:Pa,_fnStringToCss:q,_fnDetectType:B,_fnSettingsFromNode:s,_fnGetDataMaster:Z,_fnGetTrNodes:T,_fnGetTdNodes:J,_fnEscapeRegex:oa,_fnDeleteIndex:ha,_fnReOrderIndex:u,_fnColumnOrdering:M,_fnLog:D,_fnClearTable:ga,_fnSaveState:ra,_fnLoadState:Sa,_fnCreateCookie:function(a,b,c,d,e){var f=new Date;f.setTime(f.getTime()+1E3*c);var c=X.location.pathname.split("/"),a=a+"_"+c.pop().replace(/[\/:]/g,"").toLowerCase(),g;null!==e?(g="function"===typeof h.parseJSON?h.parseJSON(b):eval("("+b+")"),
+b=e(a,g,f.toGMTString(),c.join("/")+"/")):b=a+"="+encodeURIComponent(b)+"; expires="+f.toGMTString()+"; path="+c.join("/")+"/";a=l.cookie.split(";");e=b.split(";")[0].length;f=[];if(4096<e+l.cookie.length+10){for(var j=0,o=a.length;j<o;j++)if(-1!=a[j].indexOf(d)){var k=a[j].split("=");try{(g=eval("("+decodeURIComponent(k[1])+")"))&&g.iCreate&&f.push({name:k[0],time:g.iCreate})}catch(m){}}for(f.sort(function(a,b){return b.time-a.time});4096<e+l.cookie.length+10;){if(0===f.length)return;d=f.pop();l.cookie=
+d.name+"=; expires=Thu, 01-Jan-1970 00:00:01 GMT; path="+c.join("/")+"/"}}l.cookie=b},_fnReadCookie:function(a){for(var b=X.location.pathname.split("/"),a=a+"_"+b[b.length-1].replace(/[\/:]/g,"").toLowerCase()+"=",b=l.cookie.split(";"),c=0;c<b.length;c++){for(var d=b[c];" "==d.charAt(0);)d=d.substring(1,d.length);if(0===d.indexOf(a))return decodeURIComponent(d.substring(a.length,d.length))}return null},_fnDetectHeader:V,_fnGetUniqueThs:N,_fnScrollBarWidth:Qa,_fnApplyToChildren:C,_fnMap:p,_fnGetRowData:Y,
+_fnGetCellData:v,_fnSetCellData:F,_fnGetObjectDataFn:Q,_fnSetObjectDataFn:L,_fnApplyColumnDefs:ta,_fnBindAction:Ra,_fnExtend:Ta,_fnCallbackReg:z,_fnCallbackFire:A,_fnJsonString:Wa,_fnRender:S,_fnNodeToColumnIndex:fa,_fnInfoMacros:ja,_fnBrowserDetect:Ua,_fnGetColumns:r};h.extend(j.ext.oApi,this.oApi);for(var sa in j.ext.oApi)sa&&(this[sa]=Va(sa));var ca=this;this.each(function(){var a=0,b,c,d;c=this.getAttribute("id");var i=!1,f=!1;if("table"!=this.nodeName.toLowerCase())D(null,0,"Attempted to initialise DataTables on a node which is not a table: "+
+this.nodeName);else{a=0;for(b=j.settings.length;a<b;a++){if(j.settings[a].nTable==this){if(e===n||e.bRetrieve)return j.settings[a].oInstance;if(e.bDestroy){j.settings[a].oInstance.fnDestroy();break}else{D(j.settings[a],0,"Cannot reinitialise DataTable.\n\nTo retrieve the DataTables object for this table, pass no arguments or see the docs for bRetrieve and bDestroy");return}}if(j.settings[a].sTableId==this.id){j.settings.splice(a,1);break}}if(null===c||""===c)this.id=c="DataTables_Table_"+j.ext._oExternConfig.iNextUnique++;
+var g=h.extend(!0,{},j.models.oSettings,{nTable:this,oApi:ca.oApi,oInit:e,sDestroyWidth:h(this).width(),sInstance:c,sTableId:c});j.settings.push(g);g.oInstance=1===ca.length?ca:h(this).dataTable();e||(e={});e.oLanguage&&pa(e.oLanguage);e=Ta(h.extend(!0,{},j.defaults),e);p(g.oFeatures,e,"bPaginate");p(g.oFeatures,e,"bLengthChange");p(g.oFeatures,e,"bFilter");p(g.oFeatures,e,"bSort");p(g.oFeatures,e,"bInfo");p(g.oFeatures,e,"bProcessing");p(g.oFeatures,e,"bAutoWidth");p(g.oFeatures,e,"bSortClasses");
+p(g.oFeatures,e,"bServerSide");p(g.oFeatures,e,"bDeferRender");p(g.oScroll,e,"sScrollX","sX");p(g.oScroll,e,"sScrollXInner","sXInner");p(g.oScroll,e,"sScrollY","sY");p(g.oScroll,e,"bScrollCollapse","bCollapse");p(g.oScroll,e,"bScrollInfinite","bInfinite");p(g.oScroll,e,"iScrollLoadGap","iLoadGap");p(g.oScroll,e,"bScrollAutoCss","bAutoCss");p(g,e,"asStripeClasses");p(g,e,"asStripClasses","asStripeClasses");p(g,e,"fnServerData");p(g,e,"fnFormatNumber");p(g,e,"sServerMethod");p(g,e,"aaSorting");p(g,
+e,"aaSortingFixed");p(g,e,"aLengthMenu");p(g,e,"sPaginationType");p(g,e,"sAjaxSource");p(g,e,"sAjaxDataProp");p(g,e,"iCookieDuration");p(g,e,"sCookiePrefix");p(g,e,"sDom");p(g,e,"bSortCellsTop");p(g,e,"iTabIndex");p(g,e,"oSearch","oPreviousSearch");p(g,e,"aoSearchCols","aoPreSearchCols");p(g,e,"iDisplayLength","_iDisplayLength");p(g,e,"bJQueryUI","bJUI");p(g,e,"fnCookieCallback");p(g,e,"fnStateLoad");p(g,e,"fnStateSave");p(g.oLanguage,e,"fnInfoCallback");z(g,"aoDrawCallback",e.fnDrawCallback,"user");
+z(g,"aoServerParams",e.fnServerParams,"user");z(g,"aoStateSaveParams",e.fnStateSaveParams,"user");z(g,"aoStateLoadParams",e.fnStateLoadParams,"user");z(g,"aoStateLoaded",e.fnStateLoaded,"user");z(g,"aoRowCallback",e.fnRowCallback,"user");z(g,"aoRowCreatedCallback",e.fnCreatedRow,"user");z(g,"aoHeaderCallback",e.fnHeaderCallback,"user");z(g,"aoFooterCallback",e.fnFooterCallback,"user");z(g,"aoInitComplete",e.fnInitComplete,"user");z(g,"aoPreDrawCallback",e.fnPreDrawCallback,"user");g.oFeatures.bServerSide&&
+g.oFeatures.bSort&&g.oFeatures.bSortClasses?z(g,"aoDrawCallback",P,"server_side_sort_classes"):g.oFeatures.bDeferRender&&z(g,"aoDrawCallback",P,"defer_sort_classes");e.bJQueryUI?(h.extend(g.oClasses,j.ext.oJUIClasses),e.sDom===j.defaults.sDom&&"lfrtip"===j.defaults.sDom&&(g.sDom='<"H"lfr>t<"F"ip>')):h.extend(g.oClasses,j.ext.oStdClasses);h(this).addClass(g.oClasses.sTable);if(""!==g.oScroll.sX||""!==g.oScroll.sY)g.oScroll.iBarWidth=Qa();g.iInitDisplayStart===n&&(g.iInitDisplayStart=e.iDisplayStart,
+g._iDisplayStart=e.iDisplayStart);e.bStateSave&&(g.oFeatures.bStateSave=!0,Sa(g,e),z(g,"aoDrawCallback",ra,"state_save"));null!==e.iDeferLoading&&(g.bDeferLoading=!0,a=h.isArray(e.iDeferLoading),g._iRecordsDisplay=a?e.iDeferLoading[0]:e.iDeferLoading,g._iRecordsTotal=a?e.iDeferLoading[1]:e.iDeferLoading);null!==e.aaData&&(f=!0);""!==e.oLanguage.sUrl?(g.oLanguage.sUrl=e.oLanguage.sUrl,h.getJSON(g.oLanguage.sUrl,null,function(a){pa(a);h.extend(true,g.oLanguage,e.oLanguage,a);ba(g)}),i=!0):h.extend(!0,
+g.oLanguage,e.oLanguage);null===e.asStripeClasses&&(g.asStripeClasses=[g.oClasses.sStripeOdd,g.oClasses.sStripeEven]);b=g.asStripeClasses.length;g.asDestroyStripes=[];if(b){c=!1;d=h(this).children("tbody").children("tr:lt("+b+")");for(a=0;a<b;a++)d.hasClass(g.asStripeClasses[a])&&(c=!0,g.asDestroyStripes.push(g.asStripeClasses[a]));c&&d.removeClass(g.asStripeClasses.join(" "))}c=[];a=this.getElementsByTagName("thead");0!==a.length&&(V(g.aoHeader,a[0]),c=N(g));if(null===e.aoColumns){d=[];a=0;for(b=
+c.length;a<b;a++)d.push(null)}else d=e.aoColumns;a=0;for(b=d.length;a<b;a++)e.saved_aoColumns!==n&&e.saved_aoColumns.length==b&&(null===d[a]&&(d[a]={}),d[a].bVisible=e.saved_aoColumns[a].bVisible),o(g,c?c[a]:null);ta(g,e.aoColumnDefs,d,function(a,b){m(g,a,b)});a=0;for(b=g.aaSorting.length;a<b;a++){g.aaSorting[a][0]>=g.aoColumns.length&&(g.aaSorting[a][0]=0);var k=g.aoColumns[g.aaSorting[a][0]];g.aaSorting[a][2]===n&&(g.aaSorting[a][2]=0);e.aaSorting===n&&g.saved_aaSorting===n&&(g.aaSorting[a][1]=
+k.asSorting[0]);c=0;for(d=k.asSorting.length;c<d;c++)if(g.aaSorting[a][1]==k.asSorting[c]){g.aaSorting[a][2]=c;break}}P(g);Ua(g);a=h(this).children("caption").each(function(){this._captionSide=h(this).css("caption-side")});b=h(this).children("thead");0===b.length&&(b=[l.createElement("thead")],this.appendChild(b[0]));g.nTHead=b[0];b=h(this).children("tbody");0===b.length&&(b=[l.createElement("tbody")],this.appendChild(b[0]));g.nTBody=b[0];g.nTBody.setAttribute("role","alert");g.nTBody.setAttribute("aria-live",
+"polite");g.nTBody.setAttribute("aria-relevant","all");b=h(this).children("tfoot");if(0===b.length&&0<a.length&&(""!==g.oScroll.sX||""!==g.oScroll.sY))b=[l.createElement("tfoot")],this.appendChild(b[0]);0<b.length&&(g.nTFoot=b[0],V(g.aoFooter,g.nTFoot));if(f)for(a=0;a<e.aaData.length;a++)H(g,e.aaData[a]);else ua(g);g.aiDisplay=g.aiDisplayMaster.slice();g.bInitialised=!0;!1===i&&ba(g)}});ca=null;return this};j.fnVersionCheck=function(e){for(var h=function(e,h){for(;e.length<h;)e+="0";return e},m=j.ext.sVersion.split("."),
+e=e.split("."),k="",n="",l=0,t=e.length;l<t;l++)k+=h(m[l],3),n+=h(e[l],3);return parseInt(k,10)>=parseInt(n,10)};j.fnIsDataTable=function(e){for(var h=j.settings,m=0;m<h.length;m++)if(h[m].nTable===e||h[m].nScrollHead===e||h[m].nScrollFoot===e)return!0;return!1};j.fnTables=function(e){var o=[];jQuery.each(j.settings,function(j,k){(!e||!0===e&&h(k.nTable).is(":visible"))&&o.push(k.nTable)});return o};j.version="1.9.4";j.settings=[];j.models={};j.models.ext={afnFiltering:[],afnSortData:[],aoFeatures:[],
+aTypes:[],fnVersionCheck:j.fnVersionCheck,iApiIndex:0,ofnSearch:{},oApi:{},oStdClasses:{},oJUIClasses:{},oPagination:{},oSort:{},sVersion:j.version,sErrMode:"alert",_oExternConfig:{iNextUnique:0}};j.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};j.models.oRow={nTr:null,_aData:[],_aSortData:[],_anHidden:[],_sRowStripe:""};j.models.oColumn={aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bUseRendered:null,bVisible:null,_bAutoType:!0,fnCreatedCell:null,fnGetData:null,
+fnRender:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};j.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,
+bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollAutoCss:!0,bScrollCollapse:!1,bScrollInfinite:!1,bServerSide:!1,bSort:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCookieCallback:null,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(e){if(1E3>e)return e;for(var h=e+"",e=h.split(""),j="",h=h.length,k=0;k<h;k++)0===k%3&&0!==k&&(j=this.oLanguage.sInfoThousands+j),j=e[h-k-1]+j;return j},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,
+fnRowCallback:null,fnServerData:function(e,j,m,k){k.jqXHR=h.ajax({url:e,data:j,success:function(e){e.sError&&k.oApi._fnLog(k,0,e.sError);h(k.oInstance).trigger("xhr",[k,e]);m(e)},dataType:"json",cache:!1,type:k.sServerMethod,error:function(e,h){"parsererror"==h&&k.oApi._fnLog(k,0,"DataTables warning: JSON data from server could not be parsed. This is caused by a JSON formatting error.")}})},fnServerParams:null,fnStateLoad:function(e){var e=this.oApi._fnReadCookie(e.sCookiePrefix+e.sInstance),j;try{j=
+"function"===typeof h.parseJSON?h.parseJSON(e):eval("("+e+")")}catch(m){j=null}return j},fnStateLoadParams:null,fnStateLoaded:null,fnStateSave:function(e,h){this.oApi._fnCreateCookie(e.sCookiePrefix+e.sInstance,this.oApi._fnJsonString(h),e.iCookieDuration,e.sCookiePrefix,e.fnCookieCallback)},fnStateSaveParams:null,iCookieDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iScrollLoadGap:100,iTabIndex:0,oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},
+oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sInfoThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},j.models.oSearch),sAjaxDataProp:"aaData",
+sAjaxSource:null,sCookiePrefix:"SpryMedia_DataTables_",sDom:"lfrtip",sPaginationType:"two_button",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET"};j.defaults.columns={aDataSort:null,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bUseRendered:!0,bVisible:!0,fnCreatedCell:null,fnRender:null,iDataSort:-1,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};j.models.oSettings={oFeatures:{bAutoWidth:null,
+bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortClasses:null,bStateSave:null},oScroll:{bAutoCss:null,bCollapse:null,bInfinite:null,iBarWidth:0,iLoadGap:null,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1},aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aoColumns:[],aoHeader:[],aoFooter:[],asDataSearch:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:null,
+asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,sPaginationType:"two_button",iCookieDuration:0,sCookiePrefix:"",fnCookieCallback:null,aoStateSave:[],aoStateLoad:[],
+oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iDisplayEnd:10,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsTotal,10):this.aiDisplayMaster.length},
+fnRecordsDisplay:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsDisplay,10):this.aiDisplay.length},fnDisplayEnd:function(){return this.oFeatures.bServerSide?!1===this.oFeatures.bPaginate||-1==this._iDisplayLength?this._iDisplayStart+this.aiDisplay.length:Math.min(this._iDisplayStart+this._iDisplayLength,this._iRecordsDisplay):this._iDisplayEnd},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null};j.ext=h.extend(!0,{},j.models.ext);h.extend(j.ext.oStdClasses,
+{sTable:"dataTable",sPagePrevEnabled:"paginate_enabled_previous",sPagePrevDisabled:"paginate_disabled_previous",sPageNextEnabled:"paginate_enabled_next",sPageNextDisabled:"paginate_disabled_next",sPageJUINext:"",sPageJUIPrev:"",sPageButton:"paginate_button",sPageButtonActive:"paginate_active",sPageButtonStaticDisabled:"paginate_button paginate_button_disabled",sPageFirst:"first",sPagePrevious:"previous",sPageNext:"next",sPageLast:"last",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",
+sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",
+sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sFooterTH:"",sJUIHeader:"",sJUIFooter:""});h.extend(j.ext.oJUIClasses,j.ext.oStdClasses,{sPagePrevEnabled:"fg-button ui-button ui-state-default ui-corner-left",sPagePrevDisabled:"fg-button ui-button ui-state-default ui-corner-left ui-state-disabled",sPageNextEnabled:"fg-button ui-button ui-state-default ui-corner-right",
+sPageNextDisabled:"fg-button ui-button ui-state-default ui-corner-right ui-state-disabled",sPageJUINext:"ui-icon ui-icon-circle-arrow-e",sPageJUIPrev:"ui-icon ui-icon-circle-arrow-w",sPageButton:"fg-button ui-button ui-state-default",sPageButtonActive:"fg-button ui-button ui-state-default ui-state-disabled",sPageButtonStaticDisabled:"fg-button ui-button ui-state-default ui-state-disabled",sPageFirst:"first ui-corner-tl ui-corner-bl",sPageLast:"last ui-corner-tr ui-corner-br",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",
+sSortAsc:"ui-state-default",sSortDesc:"ui-state-default",sSortable:"ui-state-default",sSortableAsc:"ui-state-default",sSortableDesc:"ui-state-default",sSortableNone:"ui-state-default",sSortJUIAsc:"css_right ui-icon ui-icon-triangle-1-n",sSortJUIDesc:"css_right ui-icon ui-icon-triangle-1-s",sSortJUI:"css_right ui-icon ui-icon-carat-2-n-s",sSortJUIAscAllowed:"css_right ui-icon ui-icon-carat-1-n",sSortJUIDescAllowed:"css_right ui-icon ui-icon-carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",
+sScrollHead:"dataTables_scrollHead ui-state-default",sScrollFoot:"dataTables_scrollFoot ui-state-default",sFooterTH:"ui-state-default",sJUIHeader:"fg-toolbar ui-toolbar ui-widget-header ui-corner-tl ui-corner-tr ui-helper-clearfix",sJUIFooter:"fg-toolbar ui-toolbar ui-widget-header ui-corner-bl ui-corner-br ui-helper-clearfix"});h.extend(j.ext.oPagination,{two_button:{fnInit:function(e,j,m){var k=e.oLanguage.oPaginate,n=function(h){e.oApi._fnPageChange(e,h.data.action)&&m(e)},k=!e.bJUI?'<a class="'+
+e.oClasses.sPagePrevDisabled+'" tabindex="'+e.iTabIndex+'" role="button">'+k.sPrevious+'</a><a class="'+e.oClasses.sPageNextDisabled+'" tabindex="'+e.iTabIndex+'" role="button">'+k.sNext+"</a>":'<a class="'+e.oClasses.sPagePrevDisabled+'" tabindex="'+e.iTabIndex+'" role="button"><span class="'+e.oClasses.sPageJUIPrev+'"></span></a><a class="'+e.oClasses.sPageNextDisabled+'" tabindex="'+e.iTabIndex+'" role="button"><span class="'+e.oClasses.sPageJUINext+'"></span></a>';h(j).append(k);var l=h("a",j),
+k=l[0],l=l[1];e.oApi._fnBindAction(k,{action:"previous"},n);e.oApi._fnBindAction(l,{action:"next"},n);e.aanFeatures.p||(j.id=e.sTableId+"_paginate",k.id=e.sTableId+"_previous",l.id=e.sTableId+"_next",k.setAttribute("aria-controls",e.sTableId),l.setAttribute("aria-controls",e.sTableId))},fnUpdate:function(e){if(e.aanFeatures.p)for(var h=e.oClasses,j=e.aanFeatures.p,k,l=0,n=j.length;l<n;l++)if(k=j[l].firstChild)k.className=0===e._iDisplayStart?h.sPagePrevDisabled:h.sPagePrevEnabled,k=k.nextSibling,
+k.className=e.fnDisplayEnd()==e.fnRecordsDisplay()?h.sPageNextDisabled:h.sPageNextEnabled}},iFullNumbersShowPages:5,full_numbers:{fnInit:function(e,j,m){var k=e.oLanguage.oPaginate,l=e.oClasses,n=function(h){e.oApi._fnPageChange(e,h.data.action)&&m(e)};h(j).append('<a tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPageFirst+'">'+k.sFirst+'</a><a tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPagePrevious+'">'+k.sPrevious+'</a><span></span><a tabindex="'+e.iTabIndex+'" class="'+
+l.sPageButton+" "+l.sPageNext+'">'+k.sNext+'</a><a tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPageLast+'">'+k.sLast+"</a>");var t=h("a",j),k=t[0],l=t[1],r=t[2],t=t[3];e.oApi._fnBindAction(k,{action:"first"},n);e.oApi._fnBindAction(l,{action:"previous"},n);e.oApi._fnBindAction(r,{action:"next"},n);e.oApi._fnBindAction(t,{action:"last"},n);e.aanFeatures.p||(j.id=e.sTableId+"_paginate",k.id=e.sTableId+"_first",l.id=e.sTableId+"_previous",r.id=e.sTableId+"_next",t.id=e.sTableId+"_last")},
+fnUpdate:function(e,o){if(e.aanFeatures.p){var m=j.ext.oPagination.iFullNumbersShowPages,k=Math.floor(m/2),l=Math.ceil(e.fnRecordsDisplay()/e._iDisplayLength),n=Math.ceil(e._iDisplayStart/e._iDisplayLength)+1,t="",r,B=e.oClasses,u,M=e.aanFeatures.p,L=function(h){e.oApi._fnBindAction(this,{page:h+r-1},function(h){e.oApi._fnPageChange(e,h.data.page);o(e);h.preventDefault()})};-1===e._iDisplayLength?n=k=r=1:l<m?(r=1,k=l):n<=k?(r=1,k=m):n>=l-k?(r=l-m+1,k=l):(r=n-Math.ceil(m/2)+1,k=r+m-1);for(m=r;m<=k;m++)t+=
+n!==m?'<a tabindex="'+e.iTabIndex+'" class="'+B.sPageButton+'">'+e.fnFormatNumber(m)+"</a>":'<a tabindex="'+e.iTabIndex+'" class="'+B.sPageButtonActive+'">'+e.fnFormatNumber(m)+"</a>";m=0;for(k=M.length;m<k;m++)u=M[m],u.hasChildNodes()&&(h("span:eq(0)",u).html(t).children("a").each(L),u=u.getElementsByTagName("a"),u=[u[0],u[1],u[u.length-2],u[u.length-1]],h(u).removeClass(B.sPageButton+" "+B.sPageButtonActive+" "+B.sPageButtonStaticDisabled),h([u[0],u[1]]).addClass(1==n?B.sPageButtonStaticDisabled:
+B.sPageButton),h([u[2],u[3]]).addClass(0===l||n===l||-1===e._iDisplayLength?B.sPageButtonStaticDisabled:B.sPageButton))}}}});h.extend(j.ext.oSort,{"string-pre":function(e){"string"!=typeof e&&(e=null!==e&&e.toString?e.toString():"");return e.toLowerCase()},"string-asc":function(e,h){return e<h?-1:e>h?1:0},"string-desc":function(e,h){return e<h?1:e>h?-1:0},"html-pre":function(e){return e.replace(/<.*?>/g,"").toLowerCase()},"html-asc":function(e,h){return e<h?-1:e>h?1:0},"html-desc":function(e,h){return e<
+h?1:e>h?-1:0},"date-pre":function(e){e=Date.parse(e);if(isNaN(e)||""===e)e=Date.parse("01/01/1970 00:00:00");return e},"date-asc":function(e,h){return e-h},"date-desc":function(e,h){return h-e},"numeric-pre":function(e){return"-"==e||""===e?0:1*e},"numeric-asc":function(e,h){return e-h},"numeric-desc":function(e,h){return h-e}});h.extend(j.ext.aTypes,[function(e){if("number"===typeof e)return"numeric";if("string"!==typeof e)return null;var h,j=!1;h=e.charAt(0);if(-1=="0123456789-".indexOf(h))return null;
+for(var k=1;k<e.length;k++){h=e.charAt(k);if(-1=="0123456789.".indexOf(h))return null;if("."==h){if(j)return null;j=!0}}return"numeric"},function(e){var h=Date.parse(e);return null!==h&&!isNaN(h)||"string"===typeof e&&0===e.length?"date":null},function(e){return"string"===typeof e&&-1!=e.indexOf("<")&&-1!=e.indexOf(">")?"html":null}]);h.fn.DataTable=j;h.fn.dataTable=j;h.fn.dataTableSettings=j.settings;h.fn.dataTableExt=j.ext};"function"===typeof define&&define.amd?define(["jquery"],L):jQuery&&!jQuery.fn.dataTable&&
+L(jQuery)})(window,document);
[43/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCoreImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCoreImpl.java b/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCoreImpl.java
deleted file mode 100644
index cf74e49..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCoreImpl.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-
-import java.util.Set;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Functions;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.TaskIdGenerator;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.ScheduleException;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
-import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.args.constraints.Positive;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
-import static com.twitter.aurora.gen.ScheduleStatus.RESTARTING;
-import static com.twitter.aurora.scheduler.base.Tasks.ACTIVE_STATES;
-
-/**
- * Implementation of the scheduler core.
- */
-class SchedulerCoreImpl implements SchedulerCore {
- @Positive
- @CmdLine(name = "max_tasks_per_job", help = "Maximum number of allowed tasks in a single job.")
- public static final Arg<Integer> MAX_TASKS_PER_JOB = Arg.create(4000);
-
- private static final Logger LOG = Logger.getLogger(SchedulerCoreImpl.class.getName());
-
- private final Storage storage;
-
- private final CronJobManager cronScheduler;
-
- // Schedulers that are responsible for triggering execution of jobs.
- private final ImmutableList<JobManager> jobManagers;
-
- // TODO(Bill Farner): Avoid using StateManagerImpl.
- // State manager handles persistence of task modifications and state transitions.
- private final StateManagerImpl stateManager;
-
- private final TaskIdGenerator taskIdGenerator;
- private final JobFilter jobFilter;
-
- /**
- * Creates a new core scheduler.
- *
- * @param storage Backing store implementation.
- * @param cronScheduler Cron scheduler.
- * @param immediateScheduler Immediate scheduler.
- * @param stateManager Persistent state manager.
- * @param taskIdGenerator Task ID generator.
- * @param jobFilter Job filter.
- */
- @Inject
- public SchedulerCoreImpl(
- Storage storage,
- CronJobManager cronScheduler,
- ImmediateJobManager immediateScheduler,
- StateManagerImpl stateManager,
- TaskIdGenerator taskIdGenerator,
- JobFilter jobFilter) {
-
- this.storage = checkNotNull(storage);
-
- // The immediate scheduler will accept any job, so it's important that other schedulers are
- // placed first.
- this.jobManagers = ImmutableList.of(cronScheduler, immediateScheduler);
- this.cronScheduler = cronScheduler;
- this.stateManager = checkNotNull(stateManager);
- this.taskIdGenerator = checkNotNull(taskIdGenerator);
- this.jobFilter = checkNotNull(jobFilter);
- }
-
- private boolean hasActiveJob(IJobConfiguration job) {
- return Iterables.any(jobManagers, managerHasJob(job));
- }
-
- @Override
- public synchronized void tasksDeleted(Set<String> taskIds) {
- setTaskStatus(Query.taskScoped(taskIds), ScheduleStatus.UNKNOWN, Optional.<String>absent());
- }
-
- @Override
- public synchronized void createJob(SanitizedConfiguration sanitizedConfiguration)
- throws ScheduleException {
-
- IJobConfiguration job = sanitizedConfiguration.getJobConfig();
- if (hasActiveJob(job)) {
- throw new ScheduleException("Job already exists: " + JobKeys.toPath(job));
- }
-
- runJobFilters(job.getKey(), job.getTaskConfig(), job.getInstanceCount(), false);
-
- boolean accepted = false;
- for (final JobManager manager : jobManagers) {
- if (manager.receiveJob(sanitizedConfiguration)) {
- LOG.info("Job accepted by manager: " + manager.getUniqueKey());
- accepted = true;
- break;
- }
- }
-
- if (!accepted) {
- LOG.severe("Job was not accepted by any of the configured schedulers, discarding.");
- LOG.severe("Discarded job: " + job);
- throw new ScheduleException("Job not accepted, discarding.");
- }
- }
-
- // This number is derived from the maximum file name length limit on most UNIX systems, less
- // the number of characters we've observed being added by mesos for the executor ID, prefix, and
- // delimiters.
- @VisibleForTesting
- static final int MAX_TASK_ID_LENGTH = 255 - 90;
-
- // TODO(maximk): Consider a better approach to quota checking. MESOS-4476.
- private void runJobFilters(IJobKey jobKey, ITaskConfig task, int count, boolean incremental)
- throws ScheduleException {
-
- int instanceCount = count;
- if (incremental) {
- instanceCount +=
- Storage.Util.weaklyConsistentFetchTasks(storage, Query.jobScoped(jobKey).active()).size();
- }
-
- // TODO(maximk): This is a short-term hack to stop the bleeding from
- // https://issues.apache.org/jira/browse/MESOS-691
- if (taskIdGenerator.generate(task, instanceCount).length() > MAX_TASK_ID_LENGTH) {
- throw new ScheduleException(
- "Task ID is too long, please shorten your role or job name.");
- }
-
- JobFilter.JobFilterResult filterResult = jobFilter.filter(task, instanceCount);
- // TODO(maximk): Consider deprecating JobFilterResult in favor of custom exception.
- if (!filterResult.isPass()) {
- throw new ScheduleException(filterResult.getReason());
- }
-
- if (instanceCount > MAX_TASKS_PER_JOB.get()) {
- throw new ScheduleException("Job exceeds task limit of " + MAX_TASKS_PER_JOB.get());
- }
- }
-
- @Override
- public void validateJobResources(SanitizedConfiguration sanitizedConfiguration)
- throws ScheduleException {
-
- IJobConfiguration job = sanitizedConfiguration.getJobConfig();
- runJobFilters(job.getKey(), job.getTaskConfig(), job.getInstanceCount(), false);
- }
-
- @Override
- public void addInstances(
- final IJobKey jobKey,
- final ImmutableSet<Integer> instanceIds,
- final ITaskConfig config) throws ScheduleException {
-
- runJobFilters(jobKey, config, instanceIds.size(), true);
- storage.write(new MutateWork.NoResult<ScheduleException>() {
- @Override
- protected void execute(MutableStoreProvider storeProvider)
- throws ScheduleException {
-
- ImmutableSet<IScheduledTask> tasks =
- storeProvider.getTaskStore().fetchTasks(Query.jobScoped(jobKey).active());
-
- Set<Integer> existingInstanceIds =
- FluentIterable.from(tasks).transform(Tasks.SCHEDULED_TO_INSTANCE_ID).toSet();
- if (!Sets.intersection(existingInstanceIds, instanceIds).isEmpty()) {
- throw new ScheduleException("Instance ID collision detected.");
- }
-
- stateManager.insertPendingTasks(Maps.asMap(instanceIds, Functions.constant(config)));
- }
- });
- }
-
- @Override
- public synchronized void startCronJob(IJobKey jobKey)
- throws ScheduleException, TaskDescriptionException {
-
- checkNotNull(jobKey);
-
- if (!cronScheduler.hasJob(jobKey)) {
- throw new ScheduleException("Cron job does not exist for " + JobKeys.toPath(jobKey));
- }
-
- cronScheduler.startJobNow(jobKey);
- }
-
- /**
- * Creates a predicate that will determine whether a job manager has a job matching a job key.
- *
- * @param job Job to match.
- * @return A new predicate matching the job owner and name given.
- */
- private static Predicate<JobManager> managerHasJob(final IJobConfiguration job) {
- return new Predicate<JobManager>() {
- @Override public boolean apply(JobManager manager) {
- return manager.hasJob(job.getKey());
- }
- };
- }
-
- @Override
- public synchronized void setTaskStatus(
- Query.Builder query,
- final ScheduleStatus status,
- Optional<String> message) {
-
- checkNotNull(query);
- checkNotNull(status);
-
- stateManager.changeState(query, status, message);
- }
-
- @Override
- public synchronized void killTasks(Query.Builder query, String user) throws ScheduleException {
- checkNotNull(query);
- LOG.info("Killing tasks matching " + query);
-
- boolean jobDeleted = false;
-
- if (Query.isOnlyJobScoped(query)) {
- // If this looks like a query for all tasks in a job, instruct the scheduler modules to
- // delete the job.
- IJobKey jobKey = JobKeys.from(query).get();
- for (JobManager manager : jobManagers) {
- if (manager.deleteJob(jobKey)) {
- jobDeleted = true;
- }
- }
- }
-
- // Unless statuses were specifically supplied, only attempt to kill active tasks.
- Query.Builder taskQuery = query.get().isSetStatuses() ? query.byStatus(ACTIVE_STATES) : query;
-
- int tasksAffected =
- stateManager.changeState(taskQuery, KILLING, Optional.of("Killed by " + user));
- if (!jobDeleted && (tasksAffected == 0)) {
- throw new ScheduleException("No jobs to kill");
- }
- }
-
- @Override
- public void restartShards(
- IJobKey jobKey,
- final Set<Integer> shards,
- final String requestingUser) throws ScheduleException {
-
- if (!JobKeys.isValid(jobKey)) {
- throw new ScheduleException("Invalid job key: " + jobKey);
- }
-
- if (shards.isEmpty()) {
- throw new ScheduleException("At least one shard must be specified.");
- }
-
- final Query.Builder query = Query.instanceScoped(jobKey, shards).active();
- storage.write(new MutateWork.NoResult<ScheduleException>() {
- @Override protected void execute(MutableStoreProvider storeProvider)
- throws ScheduleException {
-
- Set<IScheduledTask> matchingTasks = storeProvider.getTaskStore().fetchTasks(query);
- if (matchingTasks.size() != shards.size()) {
- throw new ScheduleException("Not all requested shards are active.");
- }
- LOG.info("Restarting shards matching " + query);
- stateManager.changeState(
- Query.taskScoped(Tasks.ids(matchingTasks)),
- RESTARTING,
- Optional.of("Restarted by " + requestingUser));
- }
- });
- }
-
-
- @Override
- public synchronized void preemptTask(IAssignedTask task, IAssignedTask preemptingTask) {
- checkNotNull(task);
- checkNotNull(preemptingTask);
- // TODO(William Farner): Throw SchedulingException if either task doesn't exist, etc.
-
- stateManager.changeState(Query.taskScoped(task.getTaskId()), ScheduleStatus.PREEMPTING,
- Optional.of("Preempting in favor of " + preemptingTask.getTaskId()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/SideEffectStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/SideEffectStorage.java b/src/main/java/com/twitter/aurora/scheduler/state/SideEffectStorage.java
deleted file mode 100644
index 23ffff9..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/SideEffectStorage.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Queue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-import com.twitter.aurora.scheduler.events.PubsubEvent;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.StorageException;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.common.base.Closure;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Wrapper around the persistent storage and mutable state.
- */
-class SideEffectStorage {
-
- private final Queue<PubsubEvent> events = Lists.newLinkedList();
- @VisibleForTesting
- Queue<PubsubEvent> getEvents() {
- return events;
- }
-
- private AtomicBoolean inOperation = new AtomicBoolean(false);
-
- private final Storage storage;
- private final OperationFinalizer operationFinalizer;
- private final Closure<PubsubEvent> taskEventSink;
-
- interface OperationFinalizer {
- /**
- * Performs any work necessary to complete the operation.
- * This is executed in the context of a write operation, immediately after the work
- * executes normally.
- * NOTE: At present, this is executed for every nesting level of operations, rather than
- * at the completion of the top-level operation.
- * See comment in {@link #SideEffectStorage#executeSideEffectsAfter(SideEffectWork)}
- * for more detail.
- *
- * @param work Work to finalize.
- * @param storeProvider Mutable store reference.
- */
- void finalize(SideEffectWork<?, ?> work, MutableStoreProvider storeProvider);
- }
-
- SideEffectStorage(
- Storage storage,
- OperationFinalizer operationFinalizer,
- Closure<PubsubEvent> taskEventSink) {
-
- this.storage = checkNotNull(storage);
- this.operationFinalizer = checkNotNull(operationFinalizer);
- this.taskEventSink = checkNotNull(taskEventSink);
- }
-
- /**
- * Perform a unit of work in a mutating operation. This supports nesting/reentrancy.
- *
- * @param work Work to perform.
- * @param <T> Work return type
- * @param <E> Work exception type.
- * @return The work return value.
- * @throws E The work exception.
- */
- <T, E extends Exception> T write(SideEffectWork<T, E> work) throws E {
- return storage.write(executeSideEffectsAfter(work));
- }
-
- <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
- return storage.consistentRead(work);
- }
-
- /**
- * Work that has side effects external to the storage system.
- * Work may add side effect and pubsub events, which will be executed/sent upon normal
- * completion of the operation.
- *
- * @param <T> Work return type.
- * @param <E> Work exception type.
- */
- abstract class SideEffectWork<T, E extends Exception> implements MutateWork<T, E> {
- protected final void addTaskEvent(PubsubEvent notice) {
- Preconditions.checkState(inOperation.get());
- events.add(Preconditions.checkNotNull(notice));
- }
- }
-
- /**
- * Work with side effects which does not throw checked exceptions.
- *
- * @param <T> Work return type.
- */
- abstract class QuietSideEffectWork<T> extends SideEffectWork<T, RuntimeException> {
- }
-
- /**
- * Work with side effects that does not have a return value.
- *
- * @param <E> Work exception type.
- */
- abstract class NoResultSideEffectWork<E extends Exception> extends SideEffectWork<Void, E> {
-
- @Override public final Void apply(MutableStoreProvider storeProvider) throws E {
- execute(storeProvider);
- return null;
- }
-
- abstract void execute(MutableStoreProvider storeProvider) throws E;
- }
-
- /**
- * Work with side effects which does not throw checked exceptions or have a return
- * value.
- */
- abstract class NoResultQuietSideEffectWork extends NoResultSideEffectWork<RuntimeException> {
- }
-
- private <T, E extends Exception> MutateWork<T, E> executeSideEffectsAfter(
- final SideEffectWork<T, E> work) {
-
- return new MutateWork<T, E>() {
- @Override public T apply(MutableStoreProvider storeProvider) throws E {
- boolean topLevelOperation = inOperation.compareAndSet(false, true);
-
- try {
- T result = work.apply(storeProvider);
-
- // TODO(William Farner): Maintaining this since it matches prior behavior, but this
- // seems wrong. Double-check whether this is necessary, or if only the top-level
- // operation should be executing the finalizer. Update doc on OperationFinalizer
- // once this is assessed.
- operationFinalizer.finalize(work, storeProvider);
- if (topLevelOperation) {
- while (!events.isEmpty()) {
- taskEventSink.execute(events.remove());
- }
- }
- return result;
- } finally {
- if (topLevelOperation) {
- inOperation.set(false);
- }
- }
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/StateManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/StateManager.java b/src/main/java/com/twitter/aurora/scheduler/state/StateManager.java
deleted file mode 100644
index 099ec70..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/StateManager.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.base.Optional;
-
-import org.apache.mesos.Protos.SlaveID;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * Thin interface for the state manager.
- */
-public interface StateManager {
-
- /**
- * Performs a simple state change, transitioning all tasks matching a query to the given
- * state and applying the given audit message.
- * TODO(William Farner): Consider removing the return value.
- *
- * @param query Builder of the query to perform, the results of which will be modified.
- * @param newState State to move the resulting tasks into.
- * @param auditMessage Audit message to apply along with the state change.
- * @return the number of successful state changes.
- */
- int changeState(Query.Builder query, ScheduleStatus newState, Optional<String> auditMessage);
-
- /**
- * Assigns a task to a specific slave.
- * This will modify the task record to reflect the host assignment and return the updated record.
- *
- * @param taskId ID of the task to mutate.
- * @param slaveHost Host name that the task is being assigned to.
- * @param slaveId ID of the slave that the task is being assigned to.
- * @param assignedPorts Ports on the host that are being assigned to the task.
- * @return The updated task record, or {@code null} if the task was not found.
- */
- IAssignedTask assignTask(
- String taskId,
- String slaveHost,
- SlaveID slaveId,
- Set<Integer> assignedPorts);
-
- /**
- * Inserts new tasks into the store. Tasks will immediately move into PENDING and will be eligible
- * for scheduling.
- *
- * @param tasks Tasks to insert, mapped by their instance IDs.
- */
- void insertPendingTasks(Map<Integer, ITaskConfig> tasks);
-
- /**
- * Deletes records of tasks from the task store.
- * This will not perform any state checking or state transitions, but will immediately remove
- * the tasks from the store. It will also silently ignore attempts to delete task IDs that do
- * not exist.
- *
- * @param taskIds IDs of tasks to delete.
- */
- void deleteTasks(final Set<String> taskIds);
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java b/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java
deleted file mode 100644
index 37d13f4..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/StateManagerImpl.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.PriorityQueue;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Atomics;
-
-import org.apache.mesos.Protos.SlaveID;
-
-import com.twitter.aurora.gen.AssignedTask;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.ScheduledTask;
-import com.twitter.aurora.scheduler.Driver;
-import com.twitter.aurora.scheduler.TaskIdGenerator;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.events.PubsubEvent;
-import com.twitter.aurora.scheduler.state.SideEffectStorage.SideEffectWork;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.TaskStore;
-import com.twitter.aurora.scheduler.storage.TaskStore.Mutable.TaskMutation;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.base.Closure;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.collect.Iterables.transform;
-
-import static com.twitter.aurora.gen.ScheduleStatus.INIT;
-import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
-import static com.twitter.aurora.gen.ScheduleStatus.UNKNOWN;
-import static com.twitter.aurora.scheduler.state.SideEffectStorage.OperationFinalizer;
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * Manager of all persistence-related operations for the scheduler. Acts as a controller for
- * persisted state machine transitions, and their side-effects.
- *
- * TODO(William Farner): Re-evaluate thread safety here, specifically risk of races that
- * modify managerState.
- */
-public class StateManagerImpl implements StateManager {
- private static final Logger LOG = Logger.getLogger(StateManagerImpl.class.getName());
-
- private final SideEffectStorage storage;
- @VisibleForTesting
- SideEffectStorage getStorage() {
- return storage;
- }
-
- private final TaskIdGenerator taskIdGenerator;
-
- // Work queue to receive state machine side effect work.
- // Items are sorted to place DELETE entries last. This is to ensure that within an operation,
- // a delete is always processed after a state transition.
- private final Queue<WorkEntry> workQueue = new PriorityQueue<>(10,
- new Comparator<WorkEntry>() {
- @Override public int compare(WorkEntry a, WorkEntry b) {
- if ((a.command == WorkCommand.DELETE) != (b.command == WorkCommand.DELETE)) {
- return (a.command == WorkCommand.DELETE) ? 1 : -1;
- } else {
- return 0;
- }
- }
- });
-
- // Adapt the work queue into a sink.
- private final TaskStateMachine.WorkSink workSink = new TaskStateMachine.WorkSink() {
- @Override public void addWork(
- WorkCommand work,
- TaskStateMachine stateMachine,
- Function<IScheduledTask, IScheduledTask> mutation) {
-
- workQueue.add(new WorkEntry(work, stateMachine, mutation));
- }
- };
-
- private final Function<Map.Entry<Integer, ITaskConfig>, IScheduledTask> taskCreator =
- new Function<Map.Entry<Integer, ITaskConfig>, IScheduledTask>() {
- @Override public IScheduledTask apply(Map.Entry<Integer, ITaskConfig> entry) {
- ITaskConfig task = entry.getValue();
- AssignedTask assigned = new AssignedTask()
- .setTaskId(taskIdGenerator.generate(task, entry.getKey()))
- .setInstanceId(entry.getKey())
- .setTask(task.newBuilder());
- return IScheduledTask.build(new ScheduledTask()
- .setStatus(INIT)
- .setAssignedTask(assigned));
- }
- };
-
- private final Driver driver;
- private final Clock clock;
-
- /**
- * An item of work on the work queue.
- */
- private static class WorkEntry {
- private final WorkCommand command;
- private final TaskStateMachine stateMachine;
- private final Function<IScheduledTask, IScheduledTask> mutation;
-
- WorkEntry(
- WorkCommand command,
- TaskStateMachine stateMachine,
- Function<IScheduledTask, IScheduledTask> mutation) {
-
- this.command = command;
- this.stateMachine = stateMachine;
- this.mutation = mutation;
- }
- }
-
- @Inject
- StateManagerImpl(
- final Storage storage,
- final Clock clock,
- Driver driver,
- TaskIdGenerator taskIdGenerator,
- Closure<PubsubEvent> taskEventSink) {
-
- checkNotNull(storage);
- this.clock = checkNotNull(clock);
-
- OperationFinalizer finalizer = new OperationFinalizer() {
- @Override public void finalize(SideEffectWork<?, ?> work, MutableStoreProvider store) {
- processWorkQueueInWriteOperation(work, store);
- }
- };
-
- this.storage = new SideEffectStorage(storage, finalizer, taskEventSink);
-
- this.driver = checkNotNull(driver);
- this.taskIdGenerator = checkNotNull(taskIdGenerator);
-
- Stats.exportSize("work_queue_depth", workQueue);
- }
-
- @Override
- public void insertPendingTasks(final Map<Integer, ITaskConfig> tasks) {
- checkNotNull(tasks);
-
- // Done outside the write transaction to minimize the work done inside a transaction.
- final Set<IScheduledTask> scheduledTasks =
- ImmutableSet.copyOf(transform(tasks.entrySet(), taskCreator));
-
- storage.write(storage.new NoResultQuietSideEffectWork() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- storeProvider.getUnsafeTaskStore().saveTasks(scheduledTasks);
-
- for (IScheduledTask task : scheduledTasks) {
- createStateMachine(task).updateState(PENDING);
- }
- }
- });
- }
-
- @Override
- public int changeState(
- Query.Builder query,
- final ScheduleStatus newState,
- final Optional<String> auditMessage) {
-
- return changeState(query, new Function<TaskStateMachine, Boolean>() {
- @Override public Boolean apply(TaskStateMachine stateMachine) {
- return stateMachine.updateState(newState, auditMessage);
- }
- });
- }
-
- @Override
- public IAssignedTask assignTask(
- String taskId,
- String slaveHost,
- SlaveID slaveId,
- Set<Integer> assignedPorts) {
-
- checkNotBlank(taskId);
- checkNotBlank(slaveHost);
- checkNotNull(assignedPorts);
-
- TaskAssignMutation mutation = assignHost(slaveHost, slaveId, assignedPorts);
- changeState(Query.taskScoped(taskId), mutation);
-
- return mutation.getAssignedTask();
- }
-
- private int changeStateInWriteOperation(
- Set<String> taskIds,
- Function<TaskStateMachine, Boolean> stateChange) {
-
- int count = 0;
- for (TaskStateMachine stateMachine : getStateMachines(taskIds).values()) {
- if (stateChange.apply(stateMachine)) {
- ++count;
- }
- }
- return count;
- }
-
- private int changeState(
- final Query.Builder query,
- final Function<TaskStateMachine, Boolean> stateChange) {
-
- return storage.write(storage.new QuietSideEffectWork<Integer>() {
- @Override public Integer apply(MutableStoreProvider storeProvider) {
- Set<String> ids = FluentIterable.from(storeProvider.getTaskStore().fetchTasks(query))
- .transform(Tasks.SCHEDULED_TO_ID)
- .toSet();
- return changeStateInWriteOperation(ids, stateChange);
- }
- });
- }
-
- private interface TaskAssignMutation extends Function<TaskStateMachine, Boolean> {
- IAssignedTask getAssignedTask();
- }
-
- private static Map<String, Integer> getNameMappedPorts(
- Set<String> portNames,
- Set<Integer> allocatedPorts) {
-
- Preconditions.checkNotNull(portNames);
-
- // Expand ports.
- Map<String, Integer> ports = Maps.newHashMap();
- Set<Integer> portsRemaining = Sets.newHashSet(allocatedPorts);
- Iterator<Integer> portConsumer = Iterables.consumingIterable(portsRemaining).iterator();
-
- for (String portName : portNames) {
- Preconditions.checkArgument(portConsumer.hasNext(),
- "Allocated ports %s were not sufficient to expand task.", allocatedPorts);
- int portNumber = portConsumer.next();
- ports.put(portName, portNumber);
- }
-
- if (!portsRemaining.isEmpty()) {
- LOG.warning("Not all allocated ports were used to map ports!");
- }
-
- return ports;
- }
-
- private TaskAssignMutation assignHost(
- final String slaveHost,
- final SlaveID slaveId,
- final Set<Integer> assignedPorts) {
-
- final TaskMutation mutation = new TaskMutation() {
- @Override public IScheduledTask apply(IScheduledTask task) {
- ScheduledTask builder = task.newBuilder();
- AssignedTask assigned = builder.getAssignedTask();
- assigned.setAssignedPorts(
- getNameMappedPorts(assigned.getTask().getRequestedPorts(), assignedPorts));
- assigned.setSlaveHost(slaveHost)
- .setSlaveId(slaveId.getValue());
- return IScheduledTask.build(builder);
- }
- };
-
- return new TaskAssignMutation() {
- private AtomicReference<IAssignedTask> assignedTask = Atomics.newReference();
- @Override public IAssignedTask getAssignedTask() {
- return assignedTask.get();
- }
-
- @Override public Boolean apply(final TaskStateMachine stateMachine) {
- TaskMutation wrapper = new TaskMutation() {
- @Override public IScheduledTask apply(IScheduledTask task) {
- IScheduledTask mutated = mutation.apply(task);
- Preconditions.checkState(
- assignedTask.compareAndSet(null, mutated.getAssignedTask()),
- "More than one result was found for an identity query.");
- return mutated;
- }
- };
- return stateMachine.updateState(ScheduleStatus.ASSIGNED, wrapper);
- }
- };
- }
-
- private void processWorkQueueInWriteOperation(
- SideEffectWork<?, ?> sideEffectWork,
- MutableStoreProvider storeProvider) {
-
- for (final WorkEntry work : Iterables.consumingIterable(workQueue)) {
- final TaskStateMachine stateMachine = work.stateMachine;
-
- if (work.command == WorkCommand.KILL) {
- driver.killTask(stateMachine.getTaskId());
- } else {
- TaskStore.Mutable taskStore = storeProvider.getUnsafeTaskStore();
- String taskId = stateMachine.getTaskId();
- Query.Builder idQuery = Query.taskScoped(taskId);
-
- switch (work.command) {
- case RESCHEDULE:
- ScheduledTask builder =
- Iterables.getOnlyElement(taskStore.fetchTasks(idQuery)).newBuilder();
- builder.getAssignedTask().unsetSlaveId();
- builder.getAssignedTask().unsetSlaveHost();
- builder.getAssignedTask().unsetAssignedPorts();
- builder.unsetTaskEvents();
- builder.setAncestorId(taskId);
- String newTaskId = taskIdGenerator.generate(
- ITaskConfig.build(builder.getAssignedTask().getTask()),
- builder.getAssignedTask().getInstanceId());
- builder.getAssignedTask().setTaskId(newTaskId);
-
- LOG.info("Task being rescheduled: " + taskId);
-
- IScheduledTask task = IScheduledTask.build(builder);
- taskStore.saveTasks(ImmutableSet.of(task));
-
- createStateMachine(task).updateState(PENDING, Optional.of("Rescheduled"));
- ITaskConfig taskInfo = task.getAssignedTask().getTask();
- sideEffectWork.addTaskEvent(
- new PubsubEvent.TaskRescheduled(
- taskInfo.getOwner().getRole(),
- taskInfo.getJobName(),
- task.getAssignedTask().getInstanceId()));
- break;
-
- case UPDATE_STATE:
- taskStore.mutateTasks(idQuery, new TaskMutation() {
- @Override public IScheduledTask apply(IScheduledTask task) {
- return work.mutation.apply(
- IScheduledTask.build(task.newBuilder().setStatus(stateMachine.getState())));
- }
- });
- sideEffectWork.addTaskEvent(
- new PubsubEvent.TaskStateChange(
- Iterables.getOnlyElement(taskStore.fetchTasks(idQuery)),
- stateMachine.getPreviousState()));
- break;
-
- case DELETE:
- deleteTasks(ImmutableSet.of(taskId));
- break;
-
- case INCREMENT_FAILURES:
- taskStore.mutateTasks(idQuery, new TaskMutation() {
- @Override public IScheduledTask apply(IScheduledTask task) {
- return IScheduledTask.build(
- task.newBuilder().setFailureCount(task.getFailureCount() + 1));
- }
- });
- break;
-
- default:
- LOG.severe("Unrecognized work command type " + work.command);
- }
- }
- }
- }
-
- @Override
- public void deleteTasks(final Set<String> taskIds) {
- storage.write(storage.new NoResultQuietSideEffectWork() {
- @Override protected void execute(final MutableStoreProvider storeProvider) {
- TaskStore.Mutable taskStore = storeProvider.getUnsafeTaskStore();
- Iterable<IScheduledTask> tasks = taskStore.fetchTasks(Query.taskScoped(taskIds));
- addTaskEvent(new PubsubEvent.TasksDeleted(ImmutableSet.copyOf(tasks)));
- taskStore.deleteTasks(taskIds);
- }
- });
- }
-
- private Map<String, TaskStateMachine> getStateMachines(final Set<String> taskIds) {
- return storage.consistentRead(new Work.Quiet<Map<String, TaskStateMachine>>() {
- @Override public Map<String, TaskStateMachine> apply(StoreProvider storeProvider) {
- Map<String, IScheduledTask> existingTasks = Maps.uniqueIndex(
- storeProvider.getTaskStore().fetchTasks(Query.taskScoped(taskIds)),
- new Function<IScheduledTask, String>() {
- @Override public String apply(IScheduledTask input) {
- return input.getAssignedTask().getTaskId();
- }
- });
-
- ImmutableMap.Builder<String, TaskStateMachine> builder = ImmutableMap.builder();
- for (String taskId : taskIds) {
- // Pass null get() values through.
- builder.put(taskId, getStateMachine(taskId, existingTasks.get(taskId)));
- }
- return builder.build();
- }
- });
- }
-
- private TaskStateMachine getStateMachine(String taskId, @Nullable IScheduledTask task) {
- if (task != null) {
- return createStateMachine(task, task.getStatus());
- }
-
- // The task is unknown, not present in storage.
- TaskStateMachine stateMachine = new TaskStateMachine(
- taskId,
- null,
- workSink,
- clock,
- INIT);
- stateMachine.updateState(UNKNOWN);
- return stateMachine;
- }
-
- private TaskStateMachine createStateMachine(IScheduledTask task) {
- return createStateMachine(task, INIT);
- }
-
- private TaskStateMachine createStateMachine(IScheduledTask task, ScheduleStatus initialState) {
- return new TaskStateMachine(
- Tasks.id(task),
- task,
- workSink,
- clock,
- initialState);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/StateModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/StateModule.java b/src/main/java/com/twitter/aurora/scheduler/state/StateModule.java
deleted file mode 100644
index 870d085..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/StateModule.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import javax.inject.Singleton;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.inject.AbstractModule;
-import com.google.inject.Binder;
-
-import com.twitter.aurora.scheduler.MesosTaskFactory;
-import com.twitter.aurora.scheduler.MesosTaskFactory.MesosTaskFactoryImpl;
-import com.twitter.aurora.scheduler.events.PubsubEventModule;
-import com.twitter.aurora.scheduler.state.MaintenanceController.MaintenanceControllerImpl;
-import com.twitter.aurora.scheduler.state.TaskAssigner.TaskAssignerImpl;
-import com.twitter.aurora.scheduler.state.UUIDGenerator.UUIDGeneratorImpl;
-
-/**
- * Binding module for scheduling logic and higher-level state management.
- */
-public class StateModule extends AbstractModule {
-
- @Override
- protected void configure() {
- bind(TaskAssigner.class).to(TaskAssignerImpl.class);
- bind(TaskAssignerImpl.class).in(Singleton.class);
- bind(MesosTaskFactory.class).to(MesosTaskFactoryImpl.class);
-
- bind(SchedulerCore.class).to(SchedulerCoreImpl.class).in(Singleton.class);
-
- bind(StateManager.class).to(StateManagerImpl.class);
- bind(StateManagerImpl.class).in(Singleton.class);
-
- bind(UUIDGenerator.class).to(UUIDGeneratorImpl.class);
- bind(UUIDGeneratorImpl.class).in(Singleton.class);
- bind(LockManager.class).to(LockManagerImpl.class);
- bind(LockManagerImpl.class).in(Singleton.class);
-
- bindCronJobManager(binder());
- bind(ImmediateJobManager.class).in(Singleton.class);
-
- bindMaintenanceController(binder());
- }
-
- @VisibleForTesting
- static void bindCronJobManager(Binder binder) {
- binder.bind(CronJobManager.class).in(Singleton.class);
- PubsubEventModule.bindSubscriber(binder, CronJobManager.class);
- }
-
- @VisibleForTesting
- static void bindMaintenanceController(Binder binder) {
- binder.bind(MaintenanceController.class).to(MaintenanceControllerImpl.class);
- binder.bind(MaintenanceControllerImpl.class).in(Singleton.class);
- PubsubEventModule.bindSubscriber(binder, MaintenanceControllerImpl.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/TaskAssigner.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/TaskAssigner.java b/src/main/java/com/twitter/aurora/scheduler/state/TaskAssigner.java
deleted file mode 100644
index c37feae..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/TaskAssigner.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Set;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Optional;
-
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.TaskInfo;
-
-import com.twitter.aurora.scheduler.MesosTaskFactory;
-import com.twitter.aurora.scheduler.ResourceSlot;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.Resources;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Responsible for matching a task against an offer.
- */
-public interface TaskAssigner {
-
- /**
- * Tries to match a task against an offer. If a match is found, the assigner should
- * make the appropriate changes to the task and provide a non-empty result.
- *
- * @param offer The resource offer.
- * @param task The task to match against and optionally assign.
- * @return Instructions for launching the task if matching and assignment were successful.
- */
- Optional<TaskInfo> maybeAssign(Offer offer, IScheduledTask task);
-
- class TaskAssignerImpl implements TaskAssigner {
- private static final Logger LOG = Logger.getLogger(TaskAssignerImpl.class.getName());
-
- private final StateManager stateManager;
- private final SchedulingFilter filter;
- private final MesosTaskFactory taskFactory;
-
- @Inject
- public TaskAssignerImpl(
- StateManager stateManager,
- SchedulingFilter filter,
- MesosTaskFactory taskFactory) {
-
- this.stateManager = checkNotNull(stateManager);
- this.filter = checkNotNull(filter);
- this.taskFactory = checkNotNull(taskFactory);
- }
-
- private TaskInfo assign(Offer offer, IScheduledTask task) {
- String host = offer.getHostname();
- Set<Integer> selectedPorts =
- Resources.getPorts(offer, task.getAssignedTask().getTask().getRequestedPorts().size());
- IAssignedTask assigned = stateManager.assignTask(
- Tasks.id(task),
- host,
- offer.getSlaveId(),
- selectedPorts);
- LOG.info(String.format("Offer on slave %s (id %s) is being assigned task for %s.",
- host, offer.getSlaveId(), Tasks.id(task)));
- return taskFactory.createFrom(assigned, offer.getSlaveId());
- }
-
- @Override
- public Optional<TaskInfo> maybeAssign(Offer offer, IScheduledTask task) {
- Set<Veto> vetoes = filter.filter(
- ResourceSlot.from(offer),
- offer.getHostname(),
- task.getAssignedTask().getTask(),
- Tasks.id(task));
- if (vetoes.isEmpty()) {
- return Optional.of(assign(offer, task));
- } else {
- LOG.fine("Slave " + offer.getHostname() + " vetoed task " + Tasks.id(task)
- + ": " + vetoes);
- return Optional.absent();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/TaskStateMachine.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/TaskStateMachine.java b/src/main/java/com/twitter/aurora/scheduler/state/TaskStateMachine.java
deleted file mode 100644
index f32fd14..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/TaskStateMachine.java
+++ /dev/null
@@ -1,622 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Functions;
-import com.google.common.base.Optional;
-import com.google.common.base.Supplier;
-import com.google.common.base.Suppliers;
-import com.google.common.base.Throwables;
-
-import org.apache.commons.lang.builder.HashCodeBuilder;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.ScheduledTask;
-import com.twitter.aurora.gen.TaskEvent;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.base.Closure;
-import com.twitter.common.base.Closures;
-import com.twitter.common.base.Command;
-import com.twitter.common.base.MorePreconditions;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-import com.twitter.common.util.StateMachine;
-import com.twitter.common.util.StateMachine.Rule;
-import com.twitter.common.util.StateMachine.Transition;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * State machine for a task.
- * <p>
- * This enforces the lifecycle of a task, and triggers the actions that should be taken in response
- * to different state transitions. These responses are externally communicated by populating a
- * provided work queue.
- * <p>
- * TODO(William Farner): Introduce an interface to allow state machines to be dealt with
- * abstractly from the consumption side.
- */
-class TaskStateMachine {
- private static final Logger LOG = Logger.getLogger(TaskStateMachine.class.getName());
-
- private static final AtomicLong ILLEGAL_TRANSITIONS =
- Stats.exportLong("scheduler_illegal_task_state_transitions");
-
- // Re-declarations of statuses as wrapped state objects.
- private static final State ASSIGNED = State.create(ScheduleStatus.ASSIGNED);
- private static final State FAILED = State.create(ScheduleStatus.FAILED);
- private static final State FINISHED = State.create(ScheduleStatus.FINISHED);
- private static final State INIT = State.create(ScheduleStatus.INIT);
- private static final State KILLED = State.create(ScheduleStatus.KILLED);
- private static final State KILLING = State.create(ScheduleStatus.KILLING);
- private static final State LOST = State.create(ScheduleStatus.LOST);
- private static final State PENDING = State.create(ScheduleStatus.PENDING);
- private static final State PREEMPTING = State.create(ScheduleStatus.PREEMPTING);
- private static final State RESTARTING = State.create(ScheduleStatus.RESTARTING);
- private static final State RUNNING = State.create(ScheduleStatus.RUNNING);
- private static final State STARTING = State.create(ScheduleStatus.STARTING);
- private static final State UNKNOWN = State.create(ScheduleStatus.UNKNOWN);
-
- @VisibleForTesting
- static final Supplier<String> LOCAL_HOST_SUPPLIER = Suppliers.memoize(
- new Supplier<String>() {
- @Override public String get() {
- try {
- return InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- LOG.log(Level.SEVERE, "Failed to get self hostname.");
- throw Throwables.propagate(e);
- }
- }
- });
-
- private final String taskId;
- private final WorkSink workSink;
- private final StateMachine<State> stateMachine;
- private ScheduleStatus previousState = null;
- private final Clock clock;
-
- /**
- * Composes a schedule status and a state change argument. Only the ScheduleStatuses in two
- * States must be equal for them to be considered equal.
- */
- private static class State {
- private final ScheduleStatus state;
- private final Function<IScheduledTask, IScheduledTask> mutation;
-
- State(ScheduleStatus state, Function<IScheduledTask, IScheduledTask> mutation) {
- this.state = state;
- this.mutation = mutation;
- }
-
- static State create(ScheduleStatus status) {
- return create(status, Functions.<IScheduledTask>identity());
- }
-
- static State create(
- ScheduleStatus status,
- Function<IScheduledTask, IScheduledTask> mutation) {
-
- return new State(status, mutation);
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof State)) {
- return false;
- }
-
- if (o == this) {
- return true;
- }
-
- State other = (State) o;
- return state == other.state;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder()
- .append(state)
- .toHashCode();
- }
-
- @Override
- public String toString() {
- return state.toString();
- }
-
- private ScheduleStatus getState() {
- return state;
- }
-
- private Function<IScheduledTask, IScheduledTask> getMutation() {
- return mutation;
- }
- }
-
- /**
- * A write-only work acceptor.
- */
- public interface WorkSink {
- /**
- * Appends external work that must be performed for a state machine transition to be fully
- * complete.
- *
- * @param work Description of the work to be performed.
- * @param stateMachine The state machine that the work is associated with.
- * @param mutation Mutate operation to perform along with the state transition.
- */
- void addWork(
- WorkCommand work,
- TaskStateMachine stateMachine,
- Function<IScheduledTask, IScheduledTask> mutation);
- }
-
- /**
- * Creates a new task state machine.
- *
- * @param taskId ID of the task managed by this state machine.
- * @param task Read-only task that this state machine manages.
- * @param workSink Work sink to receive transition response actions
- * @param clock Clock to use for reading the current time.
- * @param initialState The state to begin the state machine at. All legal transitions will be
- * added, but this allows the state machine to 'skip' states, for instance when a task is
- * loaded from a persistent store.
- */
- public TaskStateMachine(
- final String taskId,
- final IScheduledTask task,
- final WorkSink workSink,
- final Clock clock,
- final ScheduleStatus initialState) {
-
- this.taskId = MorePreconditions.checkNotBlank(taskId);
- this.workSink = checkNotNull(workSink);
- this.clock = checkNotNull(clock);
- checkNotNull(initialState);
-
- @SuppressWarnings("unchecked")
- Closure<Transition<State>> manageTerminatedTasks = Closures.combine(
- /* Kill a task that we believe to be terminated when an attempt is made to revive. */
- Closures.filter(Transition.to(ASSIGNED, STARTING, RUNNING),
- addWorkClosure(WorkCommand.KILL)),
- /* Remove a terminated task that is remotely removed. */
- Closures.filter(Transition.to(UNKNOWN), addWorkClosure(WorkCommand.DELETE)));
-
- final Closure<Transition<State>> manageRestartingTask = new Closure<Transition<State>>() {
- @SuppressWarnings("fallthrough")
- @Override public void execute(Transition<State> transition) {
- switch (transition.getTo().getState()) {
- case ASSIGNED:
- case STARTING:
- case RUNNING:
- addWork(WorkCommand.KILL);
- break;
-
- case LOST:
- addWork(WorkCommand.KILL);
- // fall through
-
- case FINISHED:
- case FAILED:
- case KILLED:
- addWork(WorkCommand.RESCHEDULE, transition.getTo().getMutation());
- break;
-
- case UNKNOWN:
- updateState(ScheduleStatus.LOST);
- break;
-
- default:
- // No-op.
- }
- }
- };
-
- // To be called on a task transitioning into the FINISHED state.
- final Command rescheduleIfService = new Command() {
- @Override public void execute() {
- if (task.getAssignedTask().getTask().isIsService()) {
- addWork(WorkCommand.RESCHEDULE);
- }
- }
- };
-
- // To be called on a task transitioning into the FAILED state.
- final Command incrementFailuresMaybeReschedule = new Command() {
- @Override public void execute() {
- addWork(WorkCommand.INCREMENT_FAILURES);
-
- // Max failures is ignored for service task.
- boolean isService = task.getAssignedTask().getTask().isIsService();
-
- // Max failures is ignored when set to -1.
- int maxFailures = task.getAssignedTask().getTask().getMaxTaskFailures();
- if (isService || (maxFailures == -1) || (task.getFailureCount() < (maxFailures - 1))) {
- addWork(WorkCommand.RESCHEDULE);
- } else {
- LOG.info("Task " + getTaskId() + " reached failure limit, not rescheduling");
- }
- }
- };
-
- stateMachine = StateMachine.<State>builder(taskId)
- .logTransitions()
- .initialState(State.create(initialState))
- .addState(
- Rule.from(INIT)
- .to(PENDING, UNKNOWN))
- .addState(
- Rule.from(PENDING)
- .to(ASSIGNED, KILLING)
- .withCallback(
- new Closure<Transition<State>>() {
- @Override public void execute(Transition<State> transition) {
- switch (transition.getTo().getState()) {
- case KILLING:
- addWork(WorkCommand.DELETE);
- break;
-
- default:
- // No-op.
- }
- }
- }
- ))
- .addState(
- Rule.from(ASSIGNED)
- .to(STARTING, RUNNING, FINISHED, FAILED, RESTARTING, KILLED,
- KILLING, LOST, PREEMPTING)
- .withCallback(
- new Closure<Transition<State>>() {
- @SuppressWarnings("fallthrough")
- @Override public void execute(Transition<State> transition) {
- switch (transition.getTo().getState()) {
- case FINISHED:
- rescheduleIfService.execute();
- break;
-
- case PREEMPTING:
- addWork(WorkCommand.KILL);
- break;
-
- case FAILED:
- incrementFailuresMaybeReschedule.execute();
- break;
-
- case RESTARTING:
- addWork(WorkCommand.KILL);
- break;
-
- case KILLED:
- addWork(WorkCommand.RESCHEDULE);
- break;
-
- case LOST:
- addWork(WorkCommand.RESCHEDULE);
- // fall through
- case KILLING:
- addWork(WorkCommand.KILL);
- break;
-
- case UNKNOWN:
- break;
-
- default:
- // No-op.
- }
- }
- }
- ))
- .addState(
- Rule.from(STARTING)
- .to(RUNNING, FINISHED, FAILED, RESTARTING, KILLING, KILLED, LOST, PREEMPTING)
- .withCallback(
- new Closure<Transition<State>>() {
- @SuppressWarnings("fallthrough")
- @Override public void execute(Transition<State> transition) {
- switch (transition.getTo().getState()) {
- case FINISHED:
- rescheduleIfService.execute();
- break;
-
- case RESTARTING:
- addWork(WorkCommand.KILL);
- break;
-
- case PREEMPTING:
- addWork(WorkCommand.KILL);
- break;
-
- case FAILED:
- incrementFailuresMaybeReschedule.execute();
- break;
-
- case KILLED:
- addWork(WorkCommand.RESCHEDULE);
- break;
-
- case KILLING:
- addWork(WorkCommand.KILL);
- break;
-
- case LOST:
- addWork(WorkCommand.RESCHEDULE);
- break;
-
- case UNKNOWN:
- // The slave previously acknowledged that it had the task, and now
- // stopped reporting it.
- updateState(ScheduleStatus.LOST);
- break;
-
- default:
- // No-op.
- }
- }
- }
- ))
- .addState(
- Rule.from(RUNNING)
- .to(FINISHED, RESTARTING, FAILED, KILLING, KILLED, LOST, PREEMPTING)
- .withCallback(
- new Closure<Transition<State>>() {
- @SuppressWarnings("fallthrough")
- @Override public void execute(Transition<State> transition) {
- switch (transition.getTo().getState()) {
- case FINISHED:
- rescheduleIfService.execute();
- break;
-
- case PREEMPTING:
- addWork(WorkCommand.KILL);
- break;
-
- case RESTARTING:
- addWork(WorkCommand.KILL);
- break;
-
- case FAILED:
- incrementFailuresMaybeReschedule.execute();
- break;
-
- case KILLED:
- addWork(WorkCommand.RESCHEDULE);
- break;
-
- case KILLING:
- addWork(WorkCommand.KILL);
- break;
-
- case LOST:
- addWork(WorkCommand.RESCHEDULE);
- break;
-
- case UNKNOWN:
- updateState(ScheduleStatus.LOST);
- break;
-
- default:
- // No-op.
- }
- }
- }
- ))
- .addState(
- Rule.from(FINISHED)
- .to(UNKNOWN)
- .withCallback(manageTerminatedTasks))
- .addState(
- Rule.from(PREEMPTING)
- .to(FINISHED, FAILED, KILLING, KILLED, LOST)
- .withCallback(manageRestartingTask))
- .addState(
- Rule.from(RESTARTING)
- .to(FINISHED, FAILED, KILLING, KILLED, LOST)
- .withCallback(manageRestartingTask))
- .addState(
- Rule.from(FAILED)
- .to(UNKNOWN)
- .withCallback(manageTerminatedTasks))
- .addState(
- Rule.from(KILLED)
- .to(UNKNOWN)
- .withCallback(manageTerminatedTasks))
- .addState(
- Rule.from(KILLING)
- .to(FINISHED, FAILED, KILLED, LOST, UNKNOWN)
- .withCallback(manageTerminatedTasks))
- .addState(
- Rule.from(LOST)
- .to(UNKNOWN)
- .withCallback(manageTerminatedTasks))
- .addState(
- Rule.from(UNKNOWN)
- .noTransitions()
- .withCallback(manageTerminatedTasks))
- // Since we want this action to be performed last in the transition sequence, the callback
- // must be the last chained transition callback.
- .onAnyTransition(
- new Closure<Transition<State>>() {
- @Override public void execute(final Transition<State> transition) {
- ScheduleStatus from = transition.getFrom().getState();
- ScheduleStatus to = transition.getTo().getState();
-
- if (transition.isValidStateChange() && (to != ScheduleStatus.UNKNOWN)
- // Prevent an update when killing a pending task, since the task is deleted
- // prior to the update.
- && !((from == ScheduleStatus.PENDING) && (to == ScheduleStatus.KILLING))) {
- addWork(WorkCommand.UPDATE_STATE, transition.getTo().getMutation());
- } else if (!transition.isAllowed()) {
- LOG.log(Level.SEVERE, "Illegal state transition attempted: " + transition);
- ILLEGAL_TRANSITIONS.incrementAndGet();
- }
-
- if (transition.isValidStateChange()) {
- previousState = from;
- }
- }
- }
- )
- // TODO(wfarner): Consider alternatives to allow exceptions to surface. This would allow
- // the state machine to surface illegal state transitions and propagate better information
- // to the caller. As it stands, the caller must implement logic that really belongs in
- // the state machine. For example, preventing RESTARTING->UPDATING transitions
- // (or for that matter, almost any user-initiated state transition) is awkward.
- .throwOnBadTransition(false)
- .build();
- }
-
- private Closure<Transition<State>> addWorkClosure(final WorkCommand work) {
- return new Closure<Transition<State>>() {
- @Override public void execute(Transition<State> item) {
- addWork(work);
- }
- };
- }
-
- private void addWork(WorkCommand work) {
- addWork(work, Functions.<IScheduledTask>identity());
- }
-
- private void addWork(WorkCommand work, Function<IScheduledTask, IScheduledTask> mutation) {
- LOG.info("Adding work command " + work + " for " + this);
- workSink.addWork(work, TaskStateMachine.this, mutation);
- }
-
- /**
- * Same as {@link #updateState(ScheduleStatus, Function)}, but uses a noop mutation.
- *
- * @param status Status to apply to the task.
- * @return {@code true} if the state change was allowed, {@code false} otherwise.
- */
- public synchronized boolean updateState(ScheduleStatus status) {
- return updateState(status, Functions.<IScheduledTask>identity());
- }
-
- /**
- * Same as {@link #updateState(ScheduleStatus, Function, Optional)}, but uses a noop mutation.
- *
- * @param status Status to apply to the task.
- * @param auditMessage The (optional) audit message to associate with the transition.
- * @return {@code true} if the state change was allowed, {@code false} otherwise.
- */
- public synchronized boolean updateState(ScheduleStatus status, Optional<String> auditMessage) {
- return updateState(status, Functions.<IScheduledTask>identity(), auditMessage);
- }
-
- /**
- * Same as {@link #updateState(ScheduleStatus, Function, Optional)}, but omits the audit message.
- *
- * @param status Status to apply to the task.
- * @param mutation Mutate operation to perform while updating the task.
- * @return {@code true} if the state change was allowed, {@code false} otherwise.
- */
- public synchronized boolean updateState(
- ScheduleStatus status,
- Function<IScheduledTask, IScheduledTask> mutation) {
-
- return updateState(status, mutation, Optional.<String>absent());
- }
-
- /**
- * Attempt to transition the state machine to the provided state.
- * At the time this method returns, any work commands required to satisfy the state transition
- * will be appended to the work queue.
- *
- * @param status Status to apply to the task.
- * @param auditMessage The audit message to associate with the transition.
- * @param mutation Mutate operation to perform while updating the task.
- * @return {@code true} if the state change was allowed, {@code false} otherwise.
- */
- public synchronized boolean updateState(
- final ScheduleStatus status,
- Function<IScheduledTask, IScheduledTask> mutation,
- final Optional<String> auditMessage) {
-
- checkNotNull(status);
- checkNotNull(mutation);
- checkNotNull(auditMessage);
-
- /**
- * Don't bother applying noop state changes. If we end up modifying task state without a
- * state transition (e.g. storing resource consumption of a running task), we need to find
- * a different way to suppress noop transitions.
- */
- if (stateMachine.getState().getState() != status) {
- Function<IScheduledTask, IScheduledTask> operation = Functions.compose(mutation,
- new Function<IScheduledTask, IScheduledTask>() {
- @Override public IScheduledTask apply(IScheduledTask task) {
- ScheduledTask builder = task.newBuilder();
- builder.addToTaskEvents(new TaskEvent()
- .setTimestamp(clock.nowMillis())
- .setStatus(status)
- .setMessage(auditMessage.orNull())
- .setScheduler(LOCAL_HOST_SUPPLIER.get()));
- return IScheduledTask.build(builder);
- }
- });
- return stateMachine.transition(State.create(status, operation));
- }
-
- return false;
- }
-
- /**
- * Fetch the current state from the state machine.
- *
- * @return The current state.
- */
- public synchronized ScheduleStatus getState() {
- return stateMachine.getState().getState();
- }
-
- /**
- * Gets the ID for the task that this state machine manages.
- *
- * @return The state machine's task ID.
- */
- public String getTaskId() {
- return taskId;
- }
-
- /**
- * Gets the previous state of this state machine.
- *
- * @return The state machine's previous state, or {@code null} if the state machine has not
- * transitioned since being created.
- */
- @Nullable
- ScheduleStatus getPreviousState() {
- return previousState;
- }
-
- @Override
- public String toString() {
- return getTaskId();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/UUIDGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/UUIDGenerator.java b/src/main/java/com/twitter/aurora/scheduler/state/UUIDGenerator.java
deleted file mode 100644
index d8de19c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/UUIDGenerator.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package com.twitter.aurora.scheduler.state;
-
-import java.util.UUID;
-
-/**
- * Wraps {@link java.util.UUID#randomUUID()} to facilitate unit testing.
- */
-interface UUIDGenerator {
- UUID createNew();
-
- class UUIDGeneratorImpl implements UUIDGenerator {
- @Override
- public UUID createNew() {
- return UUID.randomUUID();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/WorkCommand.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/WorkCommand.java b/src/main/java/com/twitter/aurora/scheduler/state/WorkCommand.java
deleted file mode 100644
index 6c5637d..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/WorkCommand.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-/**
- * Descriptions of the different types of external work commands that task state machines may
- * trigger.
- */
-enum WorkCommand {
- // Send an instruction for the runner of this task to kill the task.
- KILL,
- // Create a new state machine with a copy of this task.
- RESCHEDULE,
- // Update the task's state (schedule status) in the persistent store to match the state machine.
- UPDATE_STATE,
- // Delete this task from the persistent store.
- DELETE,
- // Increment the failure count for this task.
- INCREMENT_FAILURES
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/stats/AsyncStatsModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/stats/AsyncStatsModule.java b/src/main/java/com/twitter/aurora/scheduler/stats/AsyncStatsModule.java
deleted file mode 100644
index a52c5c5..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/stats/AsyncStatsModule.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.stats;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.common.base.Function;
-import com.google.common.collect.FluentIterable;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.inject.AbstractModule;
-import com.google.inject.BindingAnnotation;
-
-import org.apache.mesos.Protos.Offer;
-
-import com.twitter.aurora.gen.Quota;
-import com.twitter.aurora.scheduler.async.OfferQueue;
-import com.twitter.aurora.scheduler.configuration.Resources;
-import com.twitter.aurora.scheduler.stats.SlotSizeCounter.ResourceSlotProvider;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.common.application.modules.LifecycleModule;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.base.Command;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Data;
-import com.twitter.common.quantity.Time;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Module to configure export of cluster-wide resource allocation and consumption statistics.
- */
-public class AsyncStatsModule extends AbstractModule {
-
- @CmdLine(name = "async_task_stat_update_interval",
- help = "Interval on which to try to update resource consumption stats.")
- private static final Arg<Amount<Long, Time>> TASK_STAT_INTERVAL =
- Arg.create(Amount.of(1L, Time.HOURS));
-
- @CmdLine(name = "async_slot_stat_update_interval",
- help = "Interval on which to try to update open slot stats.")
- private static final Arg<Amount<Long, Time>> SLOT_STAT_INTERVAL =
- Arg.create(Amount.of(1L, Time.MINUTES));
-
- @BindingAnnotation
- @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
- private @interface StatExecutor { }
-
- @Override
- protected void configure() {
- final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
- new ThreadFactoryBuilder().setNameFormat("AsyncStat-%d").setDaemon(true).build());
-
- bind(TaskStatCalculator.class).in(Singleton.class);
- bind(CachedCounters.class).in(Singleton.class);
- bind(ResourceSlotProvider.class).to(OfferAdapter.class);
- bind(SlotSizeCounter.class).in(Singleton.class);
-
- bind(ScheduledExecutorService.class).annotatedWith(StatExecutor.class).toInstance(executor);
- LifecycleModule.bindStartupAction(binder(), StatUpdater.class);
- }
-
- static class StatUpdater implements Command {
- private final ScheduledExecutorService executor;
- private final TaskStatCalculator taskStats;
- private final SlotSizeCounter slotCounter;
-
- @Inject
- StatUpdater(
- @StatExecutor ScheduledExecutorService executor,
- TaskStatCalculator taskStats,
- SlotSizeCounter slotCounter) {
-
- this.executor = checkNotNull(executor);
- this.taskStats = checkNotNull(taskStats);
- this.slotCounter = checkNotNull(slotCounter);
- }
-
- @Override
- public void execute() {
- long taskInterval = TASK_STAT_INTERVAL.get().as(Time.SECONDS);
- executor.scheduleAtFixedRate(taskStats, taskInterval, taskInterval, TimeUnit.SECONDS);
- long slotInterval = SLOT_STAT_INTERVAL.get().as(Time.SECONDS);
- executor.scheduleAtFixedRate(slotCounter, slotInterval, slotInterval, TimeUnit.SECONDS);
- }
- }
-
- static class OfferAdapter implements ResourceSlotProvider {
- private static final Function<Offer, IQuota> TO_QUOTA = new Function<Offer, IQuota>() {
- @Override public IQuota apply(Offer offer) {
- Resources resources = Resources.from(offer);
- return IQuota.build(new Quota()
- .setNumCpus(resources.getNumCpus())
- .setRamMb(resources.getRam().as(Data.MB))
- .setDiskMb(resources.getDisk().as(Data.MB)));
- }
- };
-
- private final OfferQueue offerQueue;
-
- @Inject
- OfferAdapter(OfferQueue offerQueue) {
- this.offerQueue = checkNotNull(offerQueue);
- }
-
- @Override
- public Iterable<IQuota> get() {
- Iterable<Offer> offers = offerQueue.getOffers();
- return FluentIterable.from(offers).transform(TO_QUOTA);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/stats/CachedCounters.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/stats/CachedCounters.java b/src/main/java/com/twitter/aurora/scheduler/stats/CachedCounters.java
deleted file mode 100644
index 81d6811..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/stats/CachedCounters.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.stats;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.inject.Inject;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-import com.twitter.common.stats.StatsProvider;
-
-/**
- * A cache of stats, allowing counters to be fetched and reused based on their names.
- */
-class CachedCounters {
- private final LoadingCache<String, AtomicLong> cache;
-
- @Inject
- CachedCounters(final StatsProvider stats) {
- cache = CacheBuilder.newBuilder().build(
- new CacheLoader<String, AtomicLong>() {
- @Override public AtomicLong load(String key) {
- return stats.makeCounter(key);
- }
- }
- );
- }
-
- AtomicLong get(String name) {
- return cache.getUnchecked(name);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/stats/ResourceCounter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/stats/ResourceCounter.java b/src/main/java/com/twitter/aurora/scheduler/stats/ResourceCounter.java
deleted file mode 100644
index 7b96e86..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/stats/ResourceCounter.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.stats;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.StorageException;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * Computes aggregate metrics about resource allocation and consumption in the scheduler.
- */
-public class ResourceCounter {
- private final Storage storage;
-
- @Inject
- ResourceCounter(Storage storage) {
- this.storage = Preconditions.checkNotNull(storage);
- }
-
- private Iterable<ITaskConfig> getTasks(Query.Builder query) throws StorageException {
- return Iterables.transform(
- Storage.Util.consistentFetchTasks(storage, query),
- Tasks.SCHEDULED_TO_INFO);
- }
-
- /**
- * Computes totals for each of the {@link MetricType}s.
- *
- * @return aggregates for each global metric type.
- * @throws StorageException if there was a problem fetching tasks from storage.
- */
- public List<GlobalMetric> computeConsumptionTotals() throws StorageException {
- List<GlobalMetric> counts = Arrays.asList(
- new GlobalMetric(MetricType.TOTAL_CONSUMED),
- new GlobalMetric(MetricType.DEDICATED_CONSUMED),
- new GlobalMetric(MetricType.QUOTA_CONSUMED),
- new GlobalMetric(MetricType.FREE_POOL_CONSUMED));
-
- for (ITaskConfig task : getTasks(Query.unscoped().active())) {
- for (GlobalMetric count : counts) {
- count.accumulate(task);
- }
- }
- return counts;
- }
-
- /**
- * Computes total quota allocations.
- *
- * @return Total allocated quota.
- * @throws StorageException if there was a problem fetching quotas from storage.
- */
- public Metric computeQuotaAllocationTotals() throws StorageException {
- return storage.weaklyConsistentRead(new Work.Quiet<Metric>() {
- @Override public Metric apply(StoreProvider storeProvider) {
- Metric allocation = new Metric();
- for (IQuota quota : storeProvider.getQuotaStore().fetchQuotas().values()) {
- allocation.accumulate(quota);
- }
- return allocation;
- }
- });
- }
-
- /**
- * Computes arbitrary resource aggregates based on a query, a filter, and a grouping function.
- *
- * @param query Query to select tasks for aggregation.
- * @param filter Filter to apply on query result tasks.
- * @param keyFunction Function to define aggregation groupings.
- * @param <K> Key type.
- * @return A map from the keys to their aggregates based on the tasks fetched.
- * @throws StorageException if there was a problem fetching tasks from storage.
- */
- public <K> Map<K, Metric> computeAggregates(
- Query.Builder query,
- Predicate<ITaskConfig> filter,
- Function<ITaskConfig, K> keyFunction) throws StorageException {
-
- LoadingCache<K, Metric> metrics = CacheBuilder.newBuilder()
- .build(new CacheLoader<K, Metric>() {
- @Override public Metric load(K key) {
- return new Metric();
- }
- });
- for (ITaskConfig task : Iterables.filter(getTasks(query), filter)) {
- metrics.getUnchecked(keyFunction.apply(task)).accumulate(task);
- }
- return metrics.asMap();
- }
-
- public enum MetricType {
- TOTAL_CONSUMED(Predicates.<ITaskConfig>alwaysTrue()),
- DEDICATED_CONSUMED(new Predicate<ITaskConfig>() {
- @Override public boolean apply(ITaskConfig task) {
- return ConfigurationManager.isDedicated(task);
- }
- }),
- QUOTA_CONSUMED(new Predicate<ITaskConfig>() {
- @Override public boolean apply(ITaskConfig task) {
- return task.isProduction();
- }
- }),
- FREE_POOL_CONSUMED(new Predicate<ITaskConfig>() {
- @Override public boolean apply(ITaskConfig task) {
- return !ConfigurationManager.isDedicated(task) && !task.isProduction();
- }
- });
-
- public final Predicate<ITaskConfig> filter;
-
- MetricType(Predicate<ITaskConfig> filter) {
- this.filter = filter;
- }
- }
-
- public static class GlobalMetric extends Metric {
- public final MetricType type;
-
- public GlobalMetric(MetricType type) {
- this.type = type;
- }
-
- @Override
- protected void accumulate(ITaskConfig task) {
- if (type.filter.apply(task)) {
- super.accumulate(task);
- }
- }
- }
-
- public static class Metric {
- private long cpu = 0;
- private long ramMb = 0;
- private long diskMb = 0;
-
- public Metric() {
- this.cpu = 0;
- this.ramMb = 0;
- this.diskMb = 0;
- }
-
- public Metric(Metric copy) {
- this.cpu = copy.cpu;
- this.ramMb = copy.ramMb;
- this.diskMb = copy.diskMb;
- }
-
- protected void accumulate(ITaskConfig task) {
- cpu += task.getNumCpus();
- ramMb += task.getRamMb();
- diskMb += task.getDiskMb();
- }
-
- protected void accumulate(IQuota quota) {
- cpu += quota.getNumCpus();
- ramMb += quota.getRamMb();
- diskMb += quota.getDiskMb();
- }
-
- public long getCpu() {
- return cpu;
- }
-
- public long getRamGb() {
- return ramMb / 1024;
- }
-
- public long getDiskGb() {
- return diskMb / 1024;
- }
- }
-}
[29/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java
new file mode 100644
index 0000000..74f06aa
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorage.java
@@ -0,0 +1,715 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.log;
+
+import java.io.IOException;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Date;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.inject.BindingAnnotation;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.gen.storage.LogEntry;
+import com.twitter.aurora.gen.storage.Op;
+import com.twitter.aurora.gen.storage.RemoveJob;
+import com.twitter.aurora.gen.storage.RemoveLock;
+import com.twitter.aurora.gen.storage.RemoveQuota;
+import com.twitter.aurora.gen.storage.RemoveTasks;
+import com.twitter.aurora.gen.storage.RewriteTask;
+import com.twitter.aurora.gen.storage.SaveAcceptedJob;
+import com.twitter.aurora.gen.storage.SaveFrameworkId;
+import com.twitter.aurora.gen.storage.SaveHostAttributes;
+import com.twitter.aurora.gen.storage.SaveLock;
+import com.twitter.aurora.gen.storage.SaveQuota;
+import com.twitter.aurora.gen.storage.SaveTasks;
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.SchedulerException;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.log.Log.Stream.InvalidPositionException;
+import com.twitter.aurora.scheduler.log.Log.Stream.StreamAccessException;
+import com.twitter.aurora.scheduler.storage.AttributeStore;
+import com.twitter.aurora.scheduler.storage.DistributedSnapshotStore;
+import com.twitter.aurora.scheduler.storage.ForwardingStore;
+import com.twitter.aurora.scheduler.storage.JobStore;
+import com.twitter.aurora.scheduler.storage.LockStore;
+import com.twitter.aurora.scheduler.storage.QuotaStore;
+import com.twitter.aurora.scheduler.storage.SchedulerStore;
+import com.twitter.aurora.scheduler.storage.SnapshotStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
+import com.twitter.aurora.scheduler.storage.TaskStore;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.ILockKey;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.aurora.scheduler.storage.log.LogManager.StreamManager;
+import com.twitter.aurora.scheduler.storage.log.LogManager.StreamManager.StreamTransaction;
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.base.Closure;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.util.concurrent.ExecutorServiceShutdown;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A storage implementation that ensures committed transactions are written to a log.
+ *
+ * <p>In the classic write-ahead log usage we'd perform mutations as follows:
+ * <ol>
+ * <li>write op to log</li>
+ * <li>perform op locally</li>
+ * <li>*checkpoint</li>
+ * </ol>
+ *
+ * <p>Writing the operation to the log provides us with a fast persistence mechanism to ensure we
+ * have a record of our mutation in case we should need to recover state later after a crash or on
+ * a new host (assuming the log is distributed). We then apply the mutation to a local (in-memory)
+ * data structure for serving fast read requests and then optionally write down the position of the
+ * log entry we wrote in the first step to stable storage to allow for quicker recovery after a
+ * crash. Instead of reading the whole log, we can read all entries past the checkpoint. This
+ * design implies that all mutations must be idempotent and free from constraint and thus
+ * replayable over newer operations when recovering from an old checkpoint.
+ *
+ * <p>The important detail in our case is the possibility of writing an op to the log, and then
+ * failing to commit locally since we use a local database instead of an in-memory data structure.
+ * If we die after such a failure, then another instance can read and apply the logged op
+ * erroneously.
+ *
+ * <p>This implementation leverages a local transaction to handle this:
+ * <ol>
+ * <li>start local transaction</li>
+ * <li>perform op locally (uncommitted!)</li>
+ * <li>write op to log</li>
+ * <li>commit local transaction</li>
+ * <li>*checkpoint</li>
+ * </ol>
+ *
+ * <p>If the op fails to apply to local storage we will never write the op to the log and if the op
+ * fails to apply to the log, it'll throw and abort the local storage transaction as well.
+ */
+public class LogStorage extends ForwardingStore
+ implements NonVolatileStorage, DistributedSnapshotStore {
+
+ /**
+ * A service that can schedule an action to be executed periodically.
+ */
+ @VisibleForTesting
+ interface SchedulingService {
+
+ /**
+ * Schedules an action to execute periodically.
+ *
+ * @param interval The time period to wait until running the {@code action} again.
+ * @param action The action to execute periodically.
+ */
+ void doEvery(Amount<Long, Time> interval, Runnable action);
+ }
+
+ private static class ScheduledExecutorSchedulingService implements SchedulingService {
+ private final ScheduledExecutorService scheduledExecutor;
+
+ ScheduledExecutorSchedulingService(ShutdownRegistry shutdownRegistry,
+ Amount<Long, Time> shutdownGracePeriod) {
+ scheduledExecutor = Executors.newSingleThreadScheduledExecutor();
+ shutdownRegistry.addAction(
+ new ExecutorServiceShutdown(scheduledExecutor, shutdownGracePeriod));
+ }
+
+ @Override
+ public void doEvery(Amount<Long, Time> interval, Runnable action) {
+ checkNotNull(interval);
+ checkNotNull(action);
+
+ long delay = interval.getValue();
+ TimeUnit timeUnit = interval.getUnit().getTimeUnit();
+ scheduledExecutor.scheduleWithFixedDelay(action, delay, delay, timeUnit);
+ }
+ }
+
+ private static final Logger LOG = Logger.getLogger(LogStorage.class.getName());
+
+ private final LogManager logManager;
+ private final SchedulingService schedulingService;
+ private final SnapshotStore<Snapshot> snapshotStore;
+ private final Amount<Long, Time> snapshotInterval;
+
+ private StreamManager streamManager;
+
+ private boolean recovered = false;
+ private StreamTransaction transaction = null;
+
+ private final MutableStoreProvider logStoreProvider = new MutableStoreProvider() {
+ @Override public SchedulerStore.Mutable getSchedulerStore() {
+ return LogStorage.this;
+ }
+
+ @Override public JobStore.Mutable getJobStore() {
+ return LogStorage.this;
+ }
+
+ @Override public TaskStore getTaskStore() {
+ return LogStorage.this;
+ }
+
+ @Override public TaskStore.Mutable getUnsafeTaskStore() {
+ return LogStorage.this;
+ }
+
+ @Override public LockStore.Mutable getLockStore() {
+ return LogStorage.this;
+ }
+
+ @Override public QuotaStore.Mutable getQuotaStore() {
+ return LogStorage.this;
+ }
+
+ @Override public AttributeStore.Mutable getAttributeStore() {
+ return LogStorage.this;
+ }
+ };
+
+ /**
+ * Identifies the grace period to give in-process snapshots and checkpoints to complete during
+ * shutdown.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ ElementType.PARAMETER, ElementType.METHOD })
+ @BindingAnnotation
+ public @interface ShutdownGracePeriod { }
+
+ /**
+ * Identifies the interval between snapshots of local storage truncating the log.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ ElementType.PARAMETER, ElementType.METHOD })
+ @BindingAnnotation
+ public @interface SnapshotInterval { }
+
+ /**
+ * Identifies a local storage layer that is written to only after first ensuring the write
+ * operation is persisted in the log.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ ElementType.PARAMETER, ElementType.METHOD })
+ @BindingAnnotation
+ public @interface WriteBehind { }
+
+ @Inject
+ LogStorage(LogManager logManager,
+ ShutdownRegistry shutdownRegistry,
+ @ShutdownGracePeriod Amount<Long, Time> shutdownGracePeriod,
+ SnapshotStore<Snapshot> snapshotStore,
+ @SnapshotInterval Amount<Long, Time> snapshotInterval,
+ @WriteBehind Storage storage,
+ @WriteBehind SchedulerStore.Mutable schedulerStore,
+ @WriteBehind JobStore.Mutable jobStore,
+ @WriteBehind TaskStore.Mutable taskStore,
+ @WriteBehind LockStore.Mutable lockStore,
+ @WriteBehind QuotaStore.Mutable quotaStore,
+ @WriteBehind AttributeStore.Mutable attributeStore) {
+
+ this(logManager,
+ new ScheduledExecutorSchedulingService(shutdownRegistry, shutdownGracePeriod),
+ snapshotStore,
+ snapshotInterval,
+ storage,
+ schedulerStore,
+ jobStore,
+ taskStore,
+ lockStore,
+ quotaStore,
+ attributeStore);
+ }
+
+ @VisibleForTesting
+ LogStorage(LogManager logManager,
+ SchedulingService schedulingService,
+ SnapshotStore<Snapshot> snapshotStore,
+ Amount<Long, Time> snapshotInterval,
+ Storage storage,
+ SchedulerStore.Mutable schedulerStore,
+ JobStore.Mutable jobStore,
+ TaskStore.Mutable taskStore,
+ LockStore.Mutable lockStore,
+ QuotaStore.Mutable quotaStore,
+ AttributeStore.Mutable attributeStore) {
+
+ super(storage, schedulerStore, jobStore, taskStore, lockStore, quotaStore, attributeStore);
+ this.logManager = checkNotNull(logManager);
+ this.schedulingService = checkNotNull(schedulingService);
+ this.snapshotStore = checkNotNull(snapshotStore);
+ this.snapshotInterval = checkNotNull(snapshotInterval);
+ }
+
+ @Override
+ public synchronized void prepare() {
+ // Open the log to make a log replica available to the scheduler group.
+ try {
+ streamManager = logManager.open();
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to open the log, cannot continue", e);
+ }
+
+ // TODO(John Sirois): start incremental recovery here from the log and do a final recovery
+ // catchup in start after shutting down the incremental syncer.
+ }
+
+ @Override
+ public synchronized void start(final MutateWork.NoResult.Quiet initializationLogic) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ // Must have the underlying storage started so we can query it for the last checkpoint.
+ // We replay these entries in the forwarded storage system's transactions but not ours - we
+ // do not want to re-record these ops to the log.
+ recover();
+ recovered = true;
+
+ // Now that we're recovered we should let any mutations done in initializationLogic append
+ // to the log, so run it in one of our transactions.
+ write(initializationLogic);
+ }
+ });
+
+ scheduleSnapshots();
+ }
+
+ @Override
+ public void stop() {
+ // No-op.
+ }
+
+ @Timed("scheduler_log_recover")
+ void recover() throws RecoveryFailedException {
+ try {
+ streamManager.readFromBeginning(new Closure<LogEntry>() {
+ @Override public void execute(LogEntry logEntry) {
+ replay(logEntry);
+ }
+ });
+ } catch (CodingException | InvalidPositionException | StreamAccessException e) {
+ throw new RecoveryFailedException(e);
+ }
+ }
+
+ private static final class RecoveryFailedException extends SchedulerException {
+ private RecoveryFailedException(Throwable cause) {
+ super(cause);
+ }
+ }
+
+ void replay(final LogEntry logEntry) {
+ switch (logEntry.getSetField()) {
+ case SNAPSHOT:
+ Snapshot snapshot = logEntry.getSnapshot();
+ LOG.info("Applying snapshot taken on " + new Date(snapshot.getTimestamp()));
+ snapshotStore.applySnapshot(snapshot);
+ break;
+
+ case TRANSACTION:
+ for (Op op : logEntry.getTransaction().getOps()) {
+ replayOp(op);
+ }
+ break;
+
+ case NOOP:
+ // Nothing to do here
+ break;
+
+ case DEFLATED_ENTRY:
+ throw new IllegalArgumentException("Deflated entries are not handled at this layer.");
+
+ case FRAME:
+ throw new IllegalArgumentException("Framed entries are not handled at this layer.");
+
+ default:
+ throw new IllegalStateException("Unknown log entry type: " + logEntry);
+ }
+ }
+
+ private void replayOp(Op op) {
+ switch (op.getSetField()) {
+ case SAVE_FRAMEWORK_ID:
+ saveFrameworkId(op.getSaveFrameworkId().getId());
+ break;
+
+ case SAVE_ACCEPTED_JOB:
+ SaveAcceptedJob acceptedJob = op.getSaveAcceptedJob();
+ saveAcceptedJob(
+ acceptedJob.getManagerId(),
+ IJobConfiguration.build(acceptedJob.getJobConfig()));
+ break;
+
+ case REMOVE_JOB:
+ removeJob(IJobKey.build(op.getRemoveJob().getJobKey()));
+ break;
+
+ case SAVE_TASKS:
+ saveTasks(IScheduledTask.setFromBuilders(op.getSaveTasks().getTasks()));
+ break;
+
+ case REWRITE_TASK:
+ RewriteTask rewriteTask = op.getRewriteTask();
+ unsafeModifyInPlace(rewriteTask.getTaskId(), ITaskConfig.build(rewriteTask.getTask()));
+ break;
+
+ case REMOVE_TASKS:
+ deleteTasks(op.getRemoveTasks().getTaskIds());
+ break;
+
+ case SAVE_QUOTA:
+ SaveQuota saveQuota = op.getSaveQuota();
+ saveQuota(saveQuota.getRole(), IQuota.build(saveQuota.getQuota()));
+ break;
+
+ case REMOVE_QUOTA:
+ removeQuota(op.getRemoveQuota().getRole());
+ break;
+
+ case SAVE_HOST_ATTRIBUTES:
+ saveHostAttributes(op.getSaveHostAttributes().hostAttributes);
+ break;
+
+ case SAVE_LOCK:
+ saveLock(ILock.build(op.getSaveLock().getLock()));
+ break;
+
+ case REMOVE_LOCK:
+ removeLock(ILockKey.build(op.getRemoveLock().getLockKey()));
+ break;
+
+ default:
+ throw new IllegalStateException("Unknown transaction op: " + op);
+ }
+ }
+
+ private void scheduleSnapshots() {
+ if (snapshotInterval.getValue() > 0) {
+ schedulingService.doEvery(snapshotInterval, new Runnable() {
+ @Override public void run() {
+ try {
+ snapshot();
+ } catch (StorageException e) {
+ if (e.getCause() != null) {
+ LOG.log(Level.WARNING, e.getMessage(), e.getCause());
+ } else {
+ LOG.log(Level.WARNING, "StorageException when attempting to snapshot.", e);
+ }
+ }
+ }
+ });
+ }
+ }
+
+ /**
+ * Forces a snapshot of the storage state.
+ *
+ * @throws CodingException If there is a problem encoding the snapshot.
+ * @throws InvalidPositionException If the log stream cursor is invalid.
+ * @throws StreamAccessException If there is a problem writing the snapshot to the log stream.
+ */
+ @Timed("scheduler_log_snapshot")
+ void doSnapshot() throws CodingException, InvalidPositionException, StreamAccessException {
+ super.write(new MutateWork.NoResult<CodingException>() {
+ @Override protected void execute(MutableStoreProvider unused)
+ throws CodingException, InvalidPositionException, StreamAccessException {
+
+ persist(snapshotStore.createSnapshot());
+ }
+ });
+ }
+
+ @Timed("scheduler_log_snapshot_persist")
+ @Override
+ public void persist(Snapshot snapshot)
+ throws CodingException, InvalidPositionException, StreamAccessException {
+
+ streamManager.snapshot(snapshot);
+ }
+
+ @Override
+ public synchronized <T, E extends Exception> T write(final MutateWork<T, E> work)
+ throws StorageException, E {
+
+ // We don't want to use the log when recovering from it, we just want to update the underlying
+ // store - so pass mutations straight through to the underlying storage.
+ if (!recovered) {
+ return super.write(work);
+ }
+
+ // The log stream transaction has already been set up so we just need to delegate with our
+ // store provider so any mutations performed by work get logged.
+ if (transaction != null) {
+ return super.write(new MutateWork<T, E>() {
+ @Override public T apply(MutableStoreProvider unused) throws E {
+ return work.apply(logStoreProvider);
+ }
+ });
+ }
+
+ transaction = streamManager.startTransaction();
+ try {
+ return super.write(new MutateWork<T, E>() {
+ @Override public T apply(MutableStoreProvider unused) throws E {
+ T result = work.apply(logStoreProvider);
+ try {
+ transaction.commit();
+ } catch (CodingException e) {
+ throw new IllegalStateException(
+ "Problem encoding transaction operations to the log stream", e);
+ } catch (StreamAccessException e) {
+ throw new StorageException(
+ "There was a problem committing the transaction to the log.", e);
+ }
+ return result;
+ }
+ });
+ } finally {
+ transaction = null;
+ }
+ }
+
+ @Timed("scheduler_log_save_framework_id")
+ @Override
+ public void saveFrameworkId(final String frameworkId) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.saveFrameworkId(new SaveFrameworkId(frameworkId)));
+ LogStorage.super.saveFrameworkId(frameworkId);
+ }
+ });
+ }
+
+ @Timed("scheduler_log_job_save")
+ @Override
+ public void saveAcceptedJob(final String managerId, final IJobConfiguration jobConfig) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.saveAcceptedJob(new SaveAcceptedJob(managerId, jobConfig.newBuilder())));
+ LogStorage.super.saveAcceptedJob(managerId, jobConfig);
+ }
+ });
+ }
+
+ @Timed("scheduler_log_job_remove")
+ @Override
+ public void removeJob(final IJobKey jobKey) {
+ checkNotNull(jobKey);
+
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.removeJob(new RemoveJob().setJobKey(jobKey.newBuilder())));
+ LogStorage.super.removeJob(jobKey);
+ }
+ });
+ }
+
+ @Timed("scheduler_log_tasks_save")
+ @Override
+ public void saveTasks(final Set<IScheduledTask> newTasks) throws IllegalStateException {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.saveTasks(new SaveTasks(IScheduledTask.toBuildersSet(newTasks))));
+ LogStorage.super.saveTasks(newTasks);
+ }
+ });
+ }
+
+ @Override
+ public void deleteAllTasks() {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ Query.Builder query = Query.unscoped();
+ Set<String> ids = FluentIterable.from(storeProvider.getTaskStore().fetchTasks(query))
+ .transform(Tasks.SCHEDULED_TO_ID)
+ .toSet();
+ deleteTasks(ids);
+ }
+ });
+ }
+
+ @Timed("scheduler_log_tasks_remove")
+ @Override
+ public void deleteTasks(final Set<String> taskIds) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.removeTasks(new RemoveTasks(taskIds)));
+ LogStorage.super.deleteTasks(taskIds);
+ }
+ });
+ }
+
+ @Timed("scheduler_log_tasks_mutate")
+ @Override
+ public ImmutableSet<IScheduledTask> mutateTasks(
+ final Query.Builder query,
+ final Function<IScheduledTask, IScheduledTask> mutator) {
+
+ return write(new MutateWork.Quiet<ImmutableSet<IScheduledTask>>() {
+ @Override public ImmutableSet<IScheduledTask> apply(MutableStoreProvider unused) {
+ ImmutableSet<IScheduledTask> mutated = LogStorage.super.mutateTasks(query, mutator);
+
+ Map<String, IScheduledTask> tasksById = Tasks.mapById(mutated);
+ if (LOG.isLoggable(Level.FINE)) {
+ LOG.fine("Storing updated tasks to log: "
+ + Maps.transformValues(tasksById, Tasks.GET_STATUS));
+ }
+
+ // TODO(William Farner): Avoid writing an op when mutated is empty.
+ log(Op.saveTasks(new SaveTasks(IScheduledTask.toBuildersSet(mutated))));
+ return mutated;
+ }
+ });
+ }
+
+ @Timed("scheduler_log_unsafe_modify_in_place")
+ @Override
+ public boolean unsafeModifyInPlace(final String taskId, final ITaskConfig taskConfiguration) {
+ return write(new MutateWork.Quiet<Boolean>() {
+ @Override public Boolean apply(MutableStoreProvider storeProvider) {
+ boolean mutated = LogStorage.super.unsafeModifyInPlace(taskId, taskConfiguration);
+ if (mutated) {
+ log(Op.rewriteTask(new RewriteTask(taskId, taskConfiguration.newBuilder())));
+ }
+ return mutated;
+ }
+ });
+ }
+
+ @Timed("scheduler_log_quota_remove")
+ @Override
+ public void removeQuota(final String role) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.removeQuota(new RemoveQuota(role)));
+ LogStorage.super.removeQuota(role);
+ }
+ });
+ }
+
+ @Timed("scheduler_log_quota_save")
+ @Override
+ public void saveQuota(final String role, final IQuota quota) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.saveQuota(new SaveQuota(role, quota.newBuilder())));
+ LogStorage.super.saveQuota(role, quota);
+ }
+ });
+ }
+
+ @Timed("scheduler_save_host_attribute")
+ @Override
+ public void saveHostAttributes(final HostAttributes attrs) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ // Pass the updated attributes upstream, and then check if the stored value changes.
+ // We do this since different parts of the system write partial HostAttributes objects
+ // and they are merged together internally.
+ // TODO(William Farner): Split out a separate method
+ // saveAttributes(String host, Iterable<Attributes>) to simplify this.
+ Optional<HostAttributes> saved = LogStorage.super.getHostAttributes(attrs.getHost());
+ LogStorage.super.saveHostAttributes(attrs);
+ Optional<HostAttributes> updated = LogStorage.super.getHostAttributes(attrs.getHost());
+ if (!saved.equals(updated)) {
+ log(Op.saveHostAttributes(new SaveHostAttributes(updated.get())));
+ }
+ }
+ });
+ }
+
+ @Timed("scheduler_lock_save")
+ @Override
+ public void saveLock(final ILock lock) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.saveLock(new SaveLock(lock.newBuilder())));
+ LogStorage.super.saveLock(lock);
+ }
+ });
+ }
+
+ @Timed("scheduler_lock_remove")
+ @Override
+ public void removeLock(final ILockKey lockKey) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ log(Op.removeLock(new RemoveLock(lockKey.newBuilder())));
+ LogStorage.super.removeLock(lockKey);
+ }
+ });
+ }
+
+ @Override
+ public boolean setMaintenanceMode(final String host, final MaintenanceMode mode) {
+ write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider unused) {
+ Optional<HostAttributes> saved = LogStorage.super.getHostAttributes(host);
+ if (saved.isPresent()) {
+ HostAttributes attributes = saved.get().setMode(mode);
+ log(Op.saveHostAttributes(new SaveHostAttributes(attributes)));
+ LogStorage.super.saveHostAttributes(attributes);
+ }
+ }
+ });
+ return false;
+ }
+
+ @Override
+ public void snapshot() throws StorageException {
+ try {
+ doSnapshot();
+ } catch (CodingException e) {
+ throw new StorageException("Failed to encode a snapshot", e);
+ } catch (InvalidPositionException e) {
+ throw new StorageException("Saved snapshot but failed to truncate entries preceding it", e);
+ } catch (StreamAccessException e) {
+ throw new StorageException("Failed to create a snapshot", e);
+ }
+ super.snapshot();
+ }
+
+ private void log(Op op) {
+ if (recovered) {
+ transaction.add(op);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
new file mode 100644
index 0000000..92568c8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.log;
+
+import java.lang.annotation.Annotation;
+
+import javax.inject.Singleton;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.AbstractModule;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+
+import com.twitter.aurora.scheduler.log.Log;
+import com.twitter.aurora.scheduler.storage.CallOrderEnforcingStorage;
+import com.twitter.aurora.scheduler.storage.DistributedSnapshotStore;
+import com.twitter.aurora.scheduler.storage.log.LogManager.MaxEntrySize;
+import com.twitter.aurora.scheduler.storage.log.LogManager.SnapshotSetting;
+import com.twitter.aurora.scheduler.storage.log.LogStorage.ShutdownGracePeriod;
+import com.twitter.aurora.scheduler.storage.log.LogStorage.SnapshotInterval;
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.util.Clock;
+
+/**
+ * Bindings for scheduler distributed log based storage.
+ * <p/>
+ * Requires bindings for:
+ * <ul>
+ * <li>{@link Clock}</li>
+ * <li>{@link ShutdownRegistry}</li>
+ * <li>The concrete {@link Log} implementation.</li>
+ * </ul>
+ * <p/>
+ */
+public class LogStorageModule extends AbstractModule {
+
+ @CmdLine(name = "dlog_shutdown_grace_period",
+ help = "Specifies the maximum time to wait for scheduled checkpoint and snapshot "
+ + "actions to complete before forcibly shutting down.")
+ private static final Arg<Amount<Long, Time>> SHUTDOWN_GRACE_PERIOD =
+ Arg.create(Amount.of(2L, Time.SECONDS));
+
+ @CmdLine(name = "dlog_snapshot_interval",
+ help = "Specifies the frequency at which snapshots of local storage are taken and "
+ + "written to the log.")
+ private static final Arg<Amount<Long, Time>> SNAPSHOT_INTERVAL =
+ Arg.create(Amount.of(1L, Time.HOURS));
+
+ @CmdLine(name = "dlog_max_entry_size",
+ help = "Specifies the maximum entry size to append to the log. Larger entries will be "
+ + "split across entry Frames.")
+ @VisibleForTesting
+ public static final Arg<Amount<Integer, Data>> MAX_LOG_ENTRY_SIZE =
+ Arg.create(Amount.of(512, Data.KB));
+
+ @CmdLine(name = "deflate_snapshots", help = "Whether snapshots should be deflate-compressed.")
+ private static final Arg<Boolean> DEFLATE_SNAPSHOTS = Arg.create(true);
+
+ @Override
+ protected void configure() {
+ requireBinding(Log.class);
+ requireBinding(Clock.class);
+ requireBinding(ShutdownRegistry.class);
+
+ bindInterval(ShutdownGracePeriod.class, SHUTDOWN_GRACE_PERIOD);
+ bindInterval(SnapshotInterval.class, SNAPSHOT_INTERVAL);
+
+ bind(new TypeLiteral<Amount<Integer, Data>>() { }).annotatedWith(MaxEntrySize.class)
+ .toInstance(MAX_LOG_ENTRY_SIZE.get());
+ bind(LogManager.class).in(Singleton.class);
+ bind(Boolean.class).annotatedWith(SnapshotSetting.class).toInstance(DEFLATE_SNAPSHOTS.get());
+
+ bind(LogStorage.class).in(Singleton.class);
+ install(CallOrderEnforcingStorage.wrappingModule(LogStorage.class));
+ bind(DistributedSnapshotStore.class).to(LogStorage.class);
+ }
+
+ private void bindInterval(Class<? extends Annotation> key, Arg<Amount<Long, Time>> value) {
+ bind(Key.get(new TypeLiteral<Amount<Long, Time>>() { }, key)).toInstance(value.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java
new file mode 100644
index 0000000..df6b899
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.log;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Properties;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.collect.ImmutableSet;
+
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.Lock;
+import com.twitter.aurora.gen.storage.QuotaConfiguration;
+import com.twitter.aurora.gen.storage.SchedulerMetadata;
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.gen.storage.StoredJob;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.storage.SnapshotStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Volatile;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.util.BuildInfo;
+import com.twitter.common.util.Clock;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
+
+/**
+ * Snapshot store implementation that delegates to underlying snapshot stores by
+ * extracting/applying fields in a snapshot thrift struct.
+ */
+public class SnapshotStoreImpl implements SnapshotStore<Snapshot> {
+
+ private static final Logger LOG = Logger.getLogger(SnapshotStoreImpl.class.getName());
+
+ private static final SnapshotField ATTRIBUTE_FIELD = new SnapshotField() {
+ @Override public void saveToSnapshot(StoreProvider storeProvider, Snapshot snapshot) {
+ snapshot.setHostAttributes(storeProvider.getAttributeStore().getHostAttributes());
+ }
+
+ @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
+ store.getAttributeStore().deleteHostAttributes();
+
+ if (snapshot.isSetHostAttributes()) {
+ for (HostAttributes attributes : snapshot.getHostAttributes()) {
+ store.getAttributeStore().saveHostAttributes(attributes);
+ }
+ }
+ }
+ };
+
+ private static final Iterable<SnapshotField> SNAPSHOT_FIELDS = Arrays.asList(
+ ATTRIBUTE_FIELD,
+ new SnapshotField() {
+ @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
+ snapshot.setTasks(
+ IScheduledTask.toBuildersSet(store.getTaskStore().fetchTasks(Query.unscoped())));
+ }
+
+ @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
+ store.getUnsafeTaskStore().deleteAllTasks();
+
+ if (snapshot.isSetTasks()) {
+ store.getUnsafeTaskStore().saveTasks(
+ IScheduledTask.setFromBuilders(snapshot.getTasks()));
+ }
+ }
+ },
+ new SnapshotField() {
+ @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
+ ImmutableSet.Builder<StoredJob> jobs = ImmutableSet.builder();
+ for (String managerId : store.getJobStore().fetchManagerIds()) {
+ for (IJobConfiguration config : store.getJobStore().fetchJobs(managerId)) {
+ jobs.add(new StoredJob(managerId, config.newBuilder()));
+ }
+ }
+ snapshot.setJobs(jobs.build());
+ }
+
+ @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
+ store.getJobStore().deleteJobs();
+
+ if (snapshot.isSetJobs()) {
+ for (StoredJob job : snapshot.getJobs()) {
+ store.getJobStore().saveAcceptedJob(
+ job.getJobManagerId(),
+ IJobConfiguration.build(job.getJobConfiguration()));
+ }
+ }
+ }
+ },
+ new SnapshotField() {
+ @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
+ Properties props = new BuildInfo().getProperties();
+
+ snapshot.setSchedulerMetadata(
+ new SchedulerMetadata()
+ .setFrameworkId(store.getSchedulerStore().fetchFrameworkId())
+ .setRevision(props.getProperty(BuildInfo.Key.GIT_REVISION.value))
+ .setTag(props.getProperty(BuildInfo.Key.GIT_TAG.value))
+ .setTimestamp(props.getProperty(BuildInfo.Key.TIMESTAMP.value))
+ .setUser(props.getProperty(BuildInfo.Key.USER.value))
+ .setMachine(props.getProperty(BuildInfo.Key.MACHINE.value))
+ .setVersion(CURRENT_API_VERSION));
+ }
+
+ @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
+ if (snapshot.isSetSchedulerMetadata()) {
+ // No delete necessary here since this is a single value.
+
+ store.getSchedulerStore()
+ .saveFrameworkId(snapshot.getSchedulerMetadata().getFrameworkId());
+ }
+ }
+ },
+ new SnapshotField() {
+ @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
+ ImmutableSet.Builder<QuotaConfiguration> quotas = ImmutableSet.builder();
+ for (Map.Entry<String, IQuota> entry : store.getQuotaStore().fetchQuotas().entrySet()) {
+ quotas.add(new QuotaConfiguration(entry.getKey(), entry.getValue().newBuilder()));
+ }
+
+ snapshot.setQuotaConfigurations(quotas.build());
+ }
+
+ @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
+ store.getQuotaStore().deleteQuotas();
+
+ if (snapshot.isSetQuotaConfigurations()) {
+ for (QuotaConfiguration quota : snapshot.getQuotaConfigurations()) {
+ store.getQuotaStore().saveQuota(quota.getRole(), IQuota.build(quota.getQuota()));
+ }
+ }
+ }
+ },
+ new SnapshotField() {
+ @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
+ snapshot.setLocks(ILock.toBuildersSet(store.getLockStore().fetchLocks()));
+ }
+
+ @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
+ store.getLockStore().deleteLocks();
+
+ if (snapshot.isSetLocks()) {
+ for (Lock lock : snapshot.getLocks()) {
+ store.getLockStore().saveLock(ILock.build(lock));
+ }
+ }
+ }
+ }
+ );
+
+ private final Clock clock;
+ private final Storage storage;
+
+ @Inject
+ public SnapshotStoreImpl(Clock clock, @Volatile Storage storage) {
+ this.clock = checkNotNull(clock);
+ this.storage = checkNotNull(storage);
+ }
+
+ @Timed("snapshot_create")
+ @Override public Snapshot createSnapshot() {
+ return storage.consistentRead(new Work.Quiet<Snapshot>() {
+ @Override public Snapshot apply(StoreProvider storeProvider) {
+ Snapshot snapshot = new Snapshot();
+
+ // Capture timestamp to signify the beginning of a snapshot operation, apply after in case
+ // one of the field closures is mean and tries to apply a timestamp.
+ long timestamp = clock.nowMillis();
+ for (SnapshotField field : SNAPSHOT_FIELDS) {
+ field.saveToSnapshot(storeProvider, snapshot);
+ }
+ snapshot.setTimestamp(timestamp);
+ return snapshot;
+ }
+ });
+ }
+
+ @Timed("snapshot_apply")
+ @Override public void applySnapshot(final Snapshot snapshot) {
+ checkNotNull(snapshot);
+
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ LOG.info("Restoring snapshot.");
+
+ for (SnapshotField field : SNAPSHOT_FIELDS) {
+ field.restoreFromSnapshot(storeProvider, snapshot);
+ }
+ }
+ });
+ }
+
+ private interface SnapshotField {
+ void saveToSnapshot(StoreProvider storeProvider, Snapshot snapshot);
+
+ void restoreFromSnapshot(MutableStoreProvider storeProvider, Snapshot snapshot);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java b/src/main/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
new file mode 100644
index 0000000..a4c0126
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.log.testing;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import org.easymock.EasyMock;
+import org.easymock.IArgumentMatcher;
+import org.easymock.IExpectationSetters;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.storage.LogEntry;
+import com.twitter.aurora.gen.storage.Op;
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.gen.storage.Transaction;
+import com.twitter.aurora.gen.storage.storageConstants;
+import com.twitter.aurora.scheduler.log.Log.Position;
+import com.twitter.aurora.scheduler.log.Log.Stream;
+
+import static org.easymock.EasyMock.expect;
+
+/**
+ * A junit argument matcher that detects same-value {@link LogEntry} objects in a more human
+ * readable way than byte array comparison.
+ */
+public class LogOpMatcher implements IArgumentMatcher {
+ private final LogEntry expected;
+
+ public LogOpMatcher(LogEntry expected) {
+ this.expected = expected;
+ }
+
+ @Override
+ public boolean matches(Object argument) {
+ try {
+ return expected.equals(ThriftBinaryCodec.decodeNonNull(LogEntry.class, (byte[]) argument));
+ } catch (CodingException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void appendTo(StringBuffer buffer) {
+ buffer.append(expected);
+ }
+
+ /**
+ * Creates a stream matcher that will set expectations on the provided {@code stream}.
+ *
+ * @param stream Mocked stream.
+ * @return A stream matcher to set expectations against {@code stream}.
+ */
+ public static StreamMatcher matcherFor(Stream stream) {
+ return new StreamMatcher(stream);
+ }
+
+ public static final class StreamMatcher {
+ private final Stream stream;
+
+ private StreamMatcher(Stream stream) {
+ this.stream = Preconditions.checkNotNull(stream);
+ }
+
+ /**
+ * Sets an expectation for a stream transaction containing the provided {@code ops}.
+ *
+ * @param ops Operations to expect in the transaction.
+ * @return An expectation setter.
+ */
+ public IExpectationSetters<Position> expectTransaction(Op...ops) {
+ LogEntry entry = LogEntry.transaction(
+ new Transaction(ImmutableList.copyOf(ops), storageConstants.CURRENT_SCHEMA_VERSION));
+ return expect(stream.append(sameEntry(entry)));
+ }
+
+ /**
+ * Sets an expectation for a snapshot.
+ *
+ * @param snapshot Expected snapshot.
+ * @return An expectation setter.
+ */
+ public IExpectationSetters<Position> expectSnapshot(Snapshot snapshot) {
+ LogEntry entry = LogEntry.snapshot(snapshot);
+ return expect(stream.append(sameEntry(entry)));
+ }
+ }
+
+ /**
+ * Creates a matcher that supports value matching between a serialized {@link LogEntry} byte array
+ * and a log entry object.
+ *
+ * @param entry Entry to match against.
+ * @return {@code null}, return value included for easymock-style embedding.
+ */
+ private static byte[] sameEntry(LogEntry entry) {
+ EasyMock.reportMatcher(new LogOpMatcher(entry));
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/Interner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/Interner.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/Interner.java
new file mode 100644
index 0000000..bf05caa
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/Interner.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+/**
+ * An interning pool that can be used to retrieve the canonical instances of objects, while
+ * maintaining a reference count to the canonical instances.
+ *
+ * @param <T> The interned object type.
+ * @param <A> The type used for maintaining associations.
+ */
+class Interner<T, A> {
+
+ private final Map<T, InternEntry<A, T>> pool = Maps.newHashMap();
+
+ /**
+ * Retrieves the canonical instance of {@code t} and maintains {@code association} with the
+ * interned value. If {@code t} was not previously interned, the provided instance is stored.
+ *
+ * @param t The object to intern, or get the previously-interned value for.
+ * @param association A value to associate with {@code t}.
+ * @return The interned value, which may be reference-equivalent to {@code t}.
+ */
+ synchronized T addAssociation(T t, A association) {
+ InternEntry<A, T> entry = pool.get(t);
+ if (entry == null) {
+ entry = new InternEntry<>(t, association);
+ pool.put(t, entry);
+ } else {
+ entry.associations.add(association);
+ }
+ return entry.interned;
+ }
+
+ /**
+ * Removes an association with an interned value, effectively decrementing the reference count.
+ *
+ * @param t The interned value that {@code association} was associated with.
+ * @param association The association to remove.
+ */
+ synchronized void removeAssociation(T t, A association) {
+ InternEntry<A, T> entry = pool.get(t);
+ if (entry != null) {
+ entry.associations.remove(association);
+ if (entry.associations.isEmpty()) {
+ pool.remove(t);
+ }
+ }
+ }
+
+ /**
+ * Removes all interned values and associations.
+ */
+ synchronized void clear() {
+ pool.clear();
+ }
+
+ @VisibleForTesting
+ synchronized boolean isInterned(T t) {
+ return pool.containsKey(t);
+ }
+
+ @VisibleForTesting
+ synchronized Set<A> getAssociations(T t) {
+ return ImmutableSet.copyOf(pool.get(t).associations);
+ }
+
+ private static class InternEntry<A, T> {
+ private final T interned;
+ private final Set<A> associations = Sets.newHashSet();
+
+ InternEntry(T interned, A initialAssociation) {
+ this.interned = interned;
+ associations.add(initialAssociation);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java
new file mode 100644
index 0000000..6c383c7
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.scheduler.storage.AttributeStore.Mutable;
+
+/**
+ * An in-memory attribute store.
+ */
+class MemAttributeStore implements Mutable {
+ private final ConcurrentMap<String, HostAttributes> hostAttributes = Maps.newConcurrentMap();
+
+ @Override
+ public void deleteHostAttributes() {
+ hostAttributes.clear();
+ }
+
+ @Override
+ public void saveHostAttributes(HostAttributes attributes) {
+ hostAttributes.putIfAbsent(attributes.getHost(), attributes);
+
+ HostAttributes stored = hostAttributes.get(attributes.getHost());
+ if (!stored.isSetMode()) {
+ stored.setMode(attributes.isSetMode() ? attributes.getMode() : MaintenanceMode.NONE);
+ }
+ stored.setAttributes(attributes.isSetAttributes()
+ ? attributes.getAttributes() : ImmutableSet.<Attribute>of());
+ }
+
+ @Override
+ public boolean setMaintenanceMode(String host, MaintenanceMode mode) {
+ HostAttributes stored = hostAttributes.get(host);
+ if (stored != null) {
+ stored.setMode(mode);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public Optional<HostAttributes> getHostAttributes(String host) {
+ return Optional.fromNullable(hostAttributes.get(host));
+ }
+
+ @Override
+ public Set<HostAttributes> getHostAttributes() {
+ return ImmutableSet.copyOf(hostAttributes.values());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemJobStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemJobStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemJobStore.java
new file mode 100644
index 0000000..8268e8d
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemJobStore.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.storage.JobStore;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * An in-memory job store.
+ */
+class MemJobStore implements JobStore.Mutable {
+
+ private final LoadingCache<String, Manager> managers = CacheBuilder.newBuilder()
+ .build(new CacheLoader<String, Manager>() {
+ @Override public Manager load(String key) {
+ return new Manager();
+ }
+ });
+
+ @Override
+ public void saveAcceptedJob(String managerId, IJobConfiguration jobConfig) {
+ checkNotNull(managerId);
+ checkNotNull(jobConfig);
+
+ IJobKey key = JobKeys.assertValid(jobConfig.getKey());
+ managers.getUnchecked(managerId).jobs.put(key, jobConfig);
+ }
+
+ @Override
+ public void removeJob(IJobKey jobKey) {
+ checkNotNull(jobKey);
+
+ for (Manager manager : managers.asMap().values()) {
+ manager.jobs.remove(jobKey);
+ }
+ }
+
+ @Override
+ public void deleteJobs() {
+ managers.invalidateAll();
+ }
+
+ @Override
+ public Iterable<IJobConfiguration> fetchJobs(String managerId) {
+ checkNotNull(managerId);
+
+ @Nullable Manager manager = managers.getIfPresent(managerId);
+ if (manager == null) {
+ return ImmutableSet.of();
+ }
+
+ synchronized (manager.jobs) {
+ return ImmutableSet.copyOf(manager.jobs.values());
+ }
+ }
+
+ @Override
+ public Optional<IJobConfiguration> fetchJob(String managerId, IJobKey jobKey) {
+ checkNotNull(managerId);
+ checkNotNull(jobKey);
+
+ Optional<Manager> manager = Optional.fromNullable(managers.getIfPresent(managerId));
+ if (!manager.isPresent()) {
+ return Optional.absent();
+ } else {
+ return Optional.fromNullable(manager.get().jobs.get(jobKey));
+ }
+ }
+
+ @Override
+ public Set<String> fetchManagerIds() {
+ return ImmutableSet.copyOf(managers.asMap().keySet());
+ }
+
+ private static class Manager {
+ private final Map<IJobKey, IJobConfiguration> jobs = Maps.newConcurrentMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemLockStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemLockStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemLockStore.java
new file mode 100644
index 0000000..1ced973
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemLockStore.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+
+import com.twitter.aurora.scheduler.storage.LockStore;
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.ILockKey;
+
+/**
+ * An in-memory lock store.
+ */
+class MemLockStore implements LockStore.Mutable {
+
+ private final Map<ILockKey, ILock> locks = Maps.newConcurrentMap();
+
+ @Override
+ public void saveLock(ILock lock) {
+ locks.put(lock.getKey(), lock);
+ }
+
+ @Override
+ public void removeLock(ILockKey lockKey) {
+ locks.remove(lockKey);
+ }
+
+ @Override
+ public void deleteLocks() {
+ locks.clear();
+ }
+
+ @Override
+ public Set<ILock> fetchLocks() {
+ return ImmutableSet.copyOf(locks.values());
+ }
+
+ @Override
+ public Optional<ILock> fetchLock(ILockKey lockKey) {
+ return Optional.fromNullable(locks.get(lockKey));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemQuotaStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemQuotaStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemQuotaStore.java
new file mode 100644
index 0000000..855b39c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemQuotaStore.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.Map;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+import com.twitter.aurora.scheduler.storage.QuotaStore;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * An in-memory quota store.
+ */
+class MemQuotaStore implements QuotaStore.Mutable {
+
+ private final Map<String, IQuota> quotas = Maps.newConcurrentMap();
+
+ @Override
+ public void deleteQuotas() {
+ quotas.clear();
+ }
+
+ @Override
+ public void removeQuota(String role) {
+ checkNotNull(role);
+
+ quotas.remove(role);
+ }
+
+ @Override
+ public void saveQuota(String role, IQuota quota) {
+ checkNotNull(role);
+ checkNotNull(quota);
+
+ quotas.put(role, quota);
+ }
+
+ @Override
+ public Optional<IQuota> fetchQuota(String role) {
+ checkNotNull(role);
+ return Optional.fromNullable(quotas.get(role));
+ }
+
+ @Override
+ public Map<String, IQuota> fetchQuotas() {
+ return ImmutableMap.copyOf(quotas);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemSchedulerStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemSchedulerStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemSchedulerStore.java
new file mode 100644
index 0000000..19bf4ec
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemSchedulerStore.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+import com.google.common.util.concurrent.Atomics;
+
+import com.twitter.aurora.scheduler.storage.SchedulerStore;
+
+/**
+ * An in-memory scheduler store.
+ */
+class MemSchedulerStore implements SchedulerStore.Mutable {
+ private final AtomicReference<String> frameworkId = Atomics.newReference();
+
+ @Override
+ public void saveFrameworkId(String newFrameworkId) {
+ frameworkId.set(newFrameworkId);
+ }
+
+ @Nullable
+ @Override
+ public String fetchFrameworkId() {
+ return frameworkId.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java
new file mode 100644
index 0000000..438c023
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.twitter.aurora.scheduler.storage.AttributeStore;
+import com.twitter.aurora.scheduler.storage.JobStore;
+import com.twitter.aurora.scheduler.storage.LockStore;
+import com.twitter.aurora.scheduler.storage.QuotaStore;
+import com.twitter.aurora.scheduler.storage.ReadWriteLockManager;
+import com.twitter.aurora.scheduler.storage.SchedulerStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.TaskStore;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.stats.Stats;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A storage implementation comprised of individual in-memory store implementations.
+ * <p>
+ * This storage has a global read-write lock, which is used when invoking
+ * {@link #consistentRead(Work)} and {@link #write(MutateWork)}. However, no locks are used at this
+ * level for {@link #weaklyConsistentRead(Work)}. It is the responsibility of the
+ * individual stores to ensure that read operations are thread-safe (optimally supporting
+ * concurrency). Store implementations may assume that all methods invoked on {@code Mutable}
+ * store interfaces are protected by the global write lock, and thus invoked serially.
+ */
+public class MemStorage implements Storage {
+ private final AtomicLong readLockWaitNanos = Stats.exportLong("read_lock_wait_nanos");
+ private final AtomicLong writeLockWaitNanos = Stats.exportLong("write_lock_wait_nanos");
+
+ private final MutableStoreProvider storeProvider;
+ private final ReadWriteLockManager lockManager = new ReadWriteLockManager();
+
+ @Inject
+ MemStorage(
+ final SchedulerStore.Mutable schedulerStore,
+ final JobStore.Mutable jobStore,
+ final TaskStore.Mutable taskStore,
+ final LockStore.Mutable lockStore,
+ final QuotaStore.Mutable quotaStore,
+ final AttributeStore.Mutable attributeStore) {
+
+ storeProvider = new MutableStoreProvider() {
+ @Override public SchedulerStore.Mutable getSchedulerStore() {
+ return schedulerStore;
+ }
+
+ @Override public JobStore.Mutable getJobStore() {
+ return jobStore;
+ }
+
+ @Override public TaskStore getTaskStore() {
+ return taskStore;
+ }
+
+ @Override public TaskStore.Mutable getUnsafeTaskStore() {
+ return taskStore;
+ }
+
+ @Override public LockStore.Mutable getLockStore() {
+ return lockStore;
+ }
+
+ @Override public QuotaStore.Mutable getQuotaStore() {
+ return quotaStore;
+ }
+
+ @Override public AttributeStore.Mutable getAttributeStore() {
+ return attributeStore;
+ }
+ };
+ }
+
+ /**
+ * Creates a new empty in-memory storage for use in testing.
+ */
+ @VisibleForTesting
+ public static MemStorage newEmptyStorage() {
+ return new MemStorage(
+ new MemSchedulerStore(),
+ new MemJobStore(),
+ new MemTaskStore(),
+ new MemLockStore(),
+ new MemQuotaStore(),
+ new MemAttributeStore());
+ }
+
+ @Timed("mem_storage_consistent_read_operation")
+ @Override
+ public <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
+ checkNotNull(work);
+
+ long lockStartNanos = System.nanoTime();
+ boolean topLevelOperation = lockManager.readLock();
+ if (topLevelOperation) {
+ readLockWaitNanos.addAndGet(System.nanoTime() - lockStartNanos);
+ }
+ try {
+ return work.apply(storeProvider);
+ } finally {
+ lockManager.readUnlock();
+ }
+ }
+
+ @Timed("mem_storage_weakly_consistent_read_operation")
+ @Override
+ public <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work)
+ throws StorageException, E {
+
+ return work.apply(storeProvider);
+ }
+
+ @Timed("mem_storage_write_operation")
+ @Override
+ public <T, E extends Exception> T write(MutateWork<T, E> work)
+ throws StorageException, E {
+
+ checkNotNull(work);
+
+ long lockStartNanos = System.nanoTime();
+ boolean topLevelOperation = lockManager.writeLock();
+ if (topLevelOperation) {
+ writeLockWaitNanos.addAndGet(System.nanoTime() - lockStartNanos);
+ }
+ try {
+ return work.apply(storeProvider);
+ } finally {
+ lockManager.writeUnlock();
+ }
+ }
+
+ @Override
+ public void snapshot() {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java
new file mode 100644
index 0000000..a99aa12
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import javax.inject.Singleton;
+
+import com.google.inject.Key;
+import com.google.inject.PrivateModule;
+
+import com.twitter.aurora.scheduler.storage.AttributeStore;
+import com.twitter.aurora.scheduler.storage.JobStore;
+import com.twitter.aurora.scheduler.storage.LockStore;
+import com.twitter.aurora.scheduler.storage.QuotaStore;
+import com.twitter.aurora.scheduler.storage.SchedulerStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.Volatile;
+import com.twitter.aurora.scheduler.storage.TaskStore;
+import com.twitter.common.inject.Bindings.KeyFactory;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Binding module for an in-memory storage system.
+ * <p>
+ * Exposes bindings for storage components:
+ * <ul>
+ * <li>{@link com.twitter.aurora.scheduler.storage.Storage}</li>
+ * <li>Keyed with keys provided by the provided{@code keyFactory}:</li>
+ * <ul>
+ * <li>{@link com.twitter.aurora.scheduler.storage.SchedulerStore}</li>
+ * <li>{@link com.twitter.aurora.scheduler.storage.JobStore}</li>
+ * <li>{@link com.twitter.aurora.scheduler.storage.TaskStore}</li>
+ * <li>{@link com.twitter.aurora.scheduler.storage.LockStore}</li>
+ * <li>{@link com.twitter.aurora.scheduler.storage.QuotaStore}</li>
+ * <li>{@link com.twitter.aurora.scheduler.storage.AttributeStore}</li>
+ * </ul>
+ * </ul>
+ */
+public final class MemStorageModule extends PrivateModule {
+
+ private final KeyFactory keyFactory;
+
+ public MemStorageModule(KeyFactory keyFactory) {
+ this.keyFactory = checkNotNull(keyFactory);
+ }
+
+ private <T> void bindStore(Class<T> binding, Class<? extends T> impl) {
+ bind(binding).to(impl);
+ bind(impl).in(Singleton.class);
+ Key<T> key = keyFactory.create(binding);
+ bind(key).to(impl);
+ expose(key);
+ }
+
+ @Override
+ protected void configure() {
+ Key<Storage> storageKey = keyFactory.create(Storage.class);
+ bind(storageKey).to(MemStorage.class);
+ expose(storageKey);
+ Key<Storage> exposedMemStorageKey = Key.get(Storage.class, Volatile.class);
+ bind(exposedMemStorageKey).to(MemStorage.class);
+ expose(exposedMemStorageKey);
+ bind(MemStorage.class).in(Singleton.class);
+
+ bindStore(SchedulerStore.Mutable.class, MemSchedulerStore.class);
+ bindStore(JobStore.Mutable.class, MemJobStore.class);
+ bindStore(TaskStore.Mutable.class, MemTaskStore.class);
+ bindStore(LockStore.Mutable.class, MemLockStore.class);
+ bindStore(QuotaStore.Mutable.class, MemQuotaStore.class);
+ bindStore(AttributeStore.Mutable.class, MemAttributeStore.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java
new file mode 100644
index 0000000..d02511f
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemTaskStore.java
@@ -0,0 +1,304 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.twitter.aurora.gen.ScheduledTask;
+import com.twitter.aurora.gen.TaskConfig;
+import com.twitter.aurora.gen.TaskQuery;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.TaskStore;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.base.MorePreconditions;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * An in-memory task store.
+ */
+class MemTaskStore implements TaskStore.Mutable {
+
+ private static final Logger LOG = Logger.getLogger(MemTaskStore.class.getName());
+
+ @CmdLine(name = "slow_query_log_threshold",
+ help = "Log all queries that take at least this long to execute.")
+ private static final Arg<Amount<Long, Time>> SLOW_QUERY_LOG_THRESHOLD =
+ Arg.create(Amount.of(25L, Time.MILLISECONDS));
+
+ private final long slowQueryThresholdNanos = SLOW_QUERY_LOG_THRESHOLD.get().as(Time.NANOSECONDS);
+
+ private final Map<String, Task> tasks = Maps.newConcurrentMap();
+ private final Multimap<IJobKey, String> tasksByJobKey =
+ Multimaps.synchronizedSetMultimap(HashMultimap.<IJobKey, String>create());
+
+ // An interner is used here to collapse equivalent TaskConfig instances into canonical instances.
+ // Ideally this would fall out of the object hierarchy (TaskConfig being associated with the job
+ // rather than the task), but we intuit this detail here for performance reasons.
+ private final Interner<TaskConfig, String> configInterner = new Interner<TaskConfig, String>();
+
+ private final AtomicLong taskQueriesById = Stats.exportLong("task_queries_by_id");
+ private final AtomicLong taskQueriesByJob = Stats.exportLong("task_queries_by_job");
+ private final AtomicLong taskQueriesAll = Stats.exportLong("task_queries_all");
+
+ @Timed("mem_storage_fetch_tasks")
+ @Override
+ public ImmutableSet<IScheduledTask> fetchTasks(Query.Builder query) {
+ checkNotNull(query);
+
+ long start = System.nanoTime();
+ ImmutableSet<IScheduledTask> result = matches(query.get()).toSet();
+ long durationNanos = System.nanoTime() - start;
+ Level level = (durationNanos >= slowQueryThresholdNanos) ? Level.INFO : Level.FINE;
+ if (LOG.isLoggable(level)) {
+ Long time = Amount.of(durationNanos, Time.NANOSECONDS).as(Time.MILLISECONDS);
+ LOG.log(level, "Query took " + time + " ms: " + query.get());
+ }
+
+ return result;
+ }
+
+ private final Function<IScheduledTask, Task> toTask =
+ new Function<IScheduledTask, Task>() {
+ @Override public Task apply(IScheduledTask task) {
+ return new Task(task, configInterner);
+ }
+ };
+
+ @Timed("mem_storage_save_tasks")
+ @Override
+ public void saveTasks(Set<IScheduledTask> newTasks) {
+ checkNotNull(newTasks);
+ Preconditions.checkState(Tasks.ids(newTasks).size() == newTasks.size(),
+ "Proposed new tasks would create task ID collision.");
+
+ Iterable<Task> canonicalized = Iterables.transform(newTasks, toTask);
+ tasks.putAll(Maps.uniqueIndex(canonicalized, TO_ID));
+ tasksByJobKey.putAll(taskIdsByJobKey(canonicalized));
+ }
+
+ private Multimap<IJobKey, String> taskIdsByJobKey(Iterable<Task> toIndex) {
+ return Multimaps.transformValues(
+ Multimaps.index(toIndex, Functions.compose(Tasks.SCHEDULED_TO_JOB_KEY, TO_SCHEDULED)),
+ TO_ID);
+ }
+
+ @Timed("mem_storage_delete_all_tasks")
+ @Override
+ public void deleteAllTasks() {
+ tasks.clear();
+ tasksByJobKey.clear();
+ configInterner.clear();
+ }
+
+ @Timed("mem_storage_delete_tasks")
+ @Override
+ public void deleteTasks(Set<String> taskIds) {
+ checkNotNull(taskIds);
+
+ for (String id : taskIds) {
+ Task removed = tasks.remove(id);
+ if (removed != null) {
+ tasksByJobKey.remove(Tasks.SCHEDULED_TO_JOB_KEY.apply(removed.task), id);
+ configInterner.removeAssociation(removed.task.getAssignedTask().getTask().newBuilder(), id);
+ }
+ }
+ }
+
+ @Timed("mem_storage_mutate_tasks")
+ @Override
+ public ImmutableSet<IScheduledTask> mutateTasks(
+ Query.Builder query,
+ Function<IScheduledTask, IScheduledTask> mutator) {
+
+ checkNotNull(query);
+ checkNotNull(mutator);
+
+ ImmutableSet.Builder<IScheduledTask> mutated = ImmutableSet.builder();
+ for (IScheduledTask original : matches(query.get())) {
+ IScheduledTask maybeMutated = mutator.apply(original);
+ if (!original.equals(maybeMutated)) {
+ Preconditions.checkState(
+ Tasks.id(original).equals(Tasks.id(maybeMutated)),
+ "A task's ID may not be mutated.");
+ tasks.put(Tasks.id(maybeMutated), toTask.apply(maybeMutated));
+ mutated.add(maybeMutated);
+ }
+ }
+
+ return mutated.build();
+ }
+
+ @Timed("mem_storage_unsafe_modify_in_place")
+ @Override
+ public boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration) {
+ MorePreconditions.checkNotBlank(taskId);
+ checkNotNull(taskConfiguration);
+
+ Task stored = tasks.get(taskId);
+ if (stored == null) {
+ return false;
+ } else {
+ ScheduledTask updated = stored.task.newBuilder();
+ updated.getAssignedTask().setTask(taskConfiguration.newBuilder());
+ tasks.put(taskId, toTask.apply(IScheduledTask.build(updated)));
+ return true;
+ }
+ }
+
+ private static Predicate<IScheduledTask> queryFilter(final TaskQuery query) {
+ return new Predicate<IScheduledTask>() {
+ @Override public boolean apply(IScheduledTask task) {
+ ITaskConfig config = task.getAssignedTask().getTask();
+ if (query.getOwner() != null) {
+ if (!StringUtils.isBlank(query.getOwner().getRole())) {
+ if (!query.getOwner().getRole().equals(config.getOwner().getRole())) {
+ return false;
+ }
+ }
+ if (!StringUtils.isBlank(query.getOwner().getUser())) {
+ if (!query.getOwner().getUser().equals(config.getOwner().getUser())) {
+ return false;
+ }
+ }
+ }
+ if (query.getEnvironment() != null) {
+ if (!query.getEnvironment().equals(config.getEnvironment())) {
+ return false;
+ }
+ }
+ if (query.getJobName() != null) {
+ if (!query.getJobName().equals(config.getJobName())) {
+ return false;
+ }
+ }
+
+ if (query.getTaskIds() != null) {
+ if (!query.getTaskIds().contains(Tasks.id(task))) {
+ return false;
+ }
+ }
+
+ if (query.getStatusesSize() > 0) {
+ if (!query.getStatuses().contains(task.getStatus())) {
+ return false;
+ }
+ }
+ if (!StringUtils.isEmpty(query.getSlaveHost())) {
+ if (!query.getSlaveHost().equals(task.getAssignedTask().getSlaveHost())) {
+ return false;
+ }
+ }
+ if (query.getInstanceIdsSize() > 0) {
+ if (!query.getInstanceIds().contains(task.getAssignedTask().getInstanceId())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+ };
+ }
+
+ private Iterable<Task> fromIdIndex(Iterable<String> taskIds) {
+ ImmutableList.Builder<Task> matches = ImmutableList.builder();
+ for (String id : taskIds) {
+ Task match = tasks.get(id);
+ if (match != null) {
+ matches.add(match);
+ }
+ }
+ return matches.build();
+ }
+
+ private FluentIterable<IScheduledTask> matches(TaskQuery query) {
+ // Apply the query against the working set.
+ Iterable<Task> from;
+ Optional<IJobKey> jobKey = JobKeys.from(Query.arbitrary(query));
+ if (query.isSetTaskIds()) {
+ taskQueriesById.incrementAndGet();
+ from = fromIdIndex(query.getTaskIds());
+ } else if (jobKey.isPresent()) {
+ taskQueriesByJob.incrementAndGet();
+ Collection<String> taskIds = tasksByJobKey.get(jobKey.get());
+ if (taskIds == null) {
+ from = ImmutableList.of();
+ } else {
+ from = fromIdIndex(taskIds);
+ }
+ } else {
+ taskQueriesAll.incrementAndGet();
+ from = tasks.values();
+ }
+
+ return FluentIterable.from(from).transform(TO_SCHEDULED).filter(queryFilter(query));
+ }
+
+ private static final Function<Task, IScheduledTask> TO_SCHEDULED =
+ new Function<Task, IScheduledTask>() {
+ @Override public IScheduledTask apply(Task task) {
+ return task.task;
+ }
+ };
+
+ private static final Function<Task, String> TO_ID =
+ Functions.compose(Tasks.SCHEDULED_TO_ID, TO_SCHEDULED);
+
+ private static class Task {
+ private final IScheduledTask task;
+
+ Task(IScheduledTask task, Interner<TaskConfig, String> interner) {
+ interner.removeAssociation(task.getAssignedTask().getTask().newBuilder(), Tasks.id(task));
+ TaskConfig canonical = interner.addAssociation(
+ task.getAssignedTask().getTask().newBuilder(),
+ Tasks.id(task));
+ ScheduledTask builder = task.newBuilder();
+ builder.getAssignedTask().setTask(canonical);
+ this.task = IScheduledTask.build(builder);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/mem/Util.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/Util.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/Util.java
new file mode 100644
index 0000000..cf1b057
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/Util.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.mem;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+
+import org.apache.thrift.TBase;
+
+/**
+ * Utility class for common operations amongst in-memory store implementations.
+ */
+final class Util {
+
+ private Util() {
+ // Utility class.
+ }
+
+ /**
+ * Creates a function that performs a 'deep copy' on a thrift struct of a specific type. The
+ * resulting copied objects will be exactly identical to the original. Mutations to the original
+ * object will not be reflected in the copy, and vice versa.
+ *
+ * @return A copier for the provided type of thrift structs.
+ */
+ static <T extends TBase<T, ?>> Function<T, T> deepCopier() {
+ return new Function<T, T>() {
+ @Override public T apply(@Nullable T input) {
+ if (input == null) {
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ T t = (T) input.deepCopy();
+ return t;
+ }
+ };
+ }
+}
[09/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/core/runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/core/runner.py b/src/main/python/twitter/thermos/core/runner.py
deleted file mode 100644
index b10cf16..0000000
--- a/src/main/python/twitter/thermos/core/runner.py
+++ /dev/null
@@ -1,905 +0,0 @@
-""" Thermos runner.
-
-This module contains the TaskRunner, the core component of Thermos responsible for actually running
-tasks. It also contains several Handlers which define the behaviour on state transitions within the
-TaskRunner.
-
-There are three "active" states in a running Thermos task:
- ACTIVE
- CLEANING
- FINALIZING
-
-A task in ACTIVE state is running regular processes. The moment this task succeeds or goes over its
-failure limit, it then goes into CLEANING state, where it begins the staged termination of leftover
-processes (with SIGTERMs). Once all processes have terminated, the task goes into FINALIZING state,
-where the processes marked with the 'final' bit run. Once the task has gone into CLEANING state, it
-has a deadline for going into terminal state. If it doesn't make it in time (at any point, whether
-in CLEANING or FINALIZING state), it is forced into terminal state through SIGKILLs of all live
-processes (coordinators, shells and the full process trees rooted at the shells.)
-
-TaskRunner.kill is implemented by forcing the task into CLEANING state and setting its finalization
-deadline manually. So in practice, we implement Task preemption by calling kill with the
-finalization deadline = now + preemption wait, which gives the Task an opportunity to do graceful
-shutdown. If preemption_wait=0, it will result in immediate SIGKILLs and then transition to the
-terminal state.
-
-"""
-
-from contextlib import contextmanager
-import errno
-from functools import partial
-import os
-import socket
-import sys
-import time
-import traceback
-
-from twitter.common import log
-from twitter.common.dirutil import safe_mkdir
-from twitter.common.quantity import Amount, Time
-from twitter.common.recordio import ThriftRecordReader
-
-from twitter.thermos.common.ckpt import (
- CheckpointDispatcher,
- UniversalStateHandler,
- ProcessStateHandler,
- TaskStateHandler)
-from twitter.thermos.common.path import TaskPath
-from twitter.thermos.common.planner import TaskPlanner
-from twitter.thermos.config.loader import (
- ThermosConfigLoader,
- ThermosProcessWrapper,
- ThermosTaskWrapper,
- ThermosTaskValidator)
-from twitter.thermos.config.schema import ThermosContext
-
-from gen.twitter.thermos.ttypes import (
- ProcessState,
- ProcessStatus,
- RunnerCkpt,
- RunnerHeader,
- RunnerState,
- TaskState,
- TaskStatus,
-)
-
-from .helper import TaskRunnerHelper
-from .muxer import ProcessMuxer
-from .process import Process
-
-from pystachio import Environment
-
-
-# TODO(wickman) Currently this is messy because of all the private access into ._runner.
-# Clean this up by giving the TaskRunnerProcessHandler the components it should own, and
-# create a legitimate API contract into the Runner.
-class TaskRunnerProcessHandler(ProcessStateHandler):
- """
- Accesses these parts of the runner:
-
- | _task_processes [array set, pop]
- | _task_process_from_process_name [process name / sequence number => Process]
- | _watcher [ProcessMuxer.register, unregister]
- | _plan [add_success, add_failure, set_running]
- """
-
- def __init__(self, runner):
- self._runner = runner
-
- def on_waiting(self, process_update):
- log.debug('Process on_waiting %s' % process_update)
- self._runner._task_processes[process_update.process] = (
- self._runner._task_process_from_process_name(
- process_update.process, process_update.seq + 1))
- self._runner._watcher.register(process_update.process, process_update.seq - 1)
-
- def on_forked(self, process_update):
- log.debug('Process on_forked %s' % process_update)
- task_process = self._runner._task_processes[process_update.process]
- task_process.rebind(process_update.coordinator_pid, process_update.fork_time)
- self._runner._plan.set_running(process_update.process)
-
- def on_running(self, process_update):
- log.debug('Process on_running %s' % process_update)
- self._runner._plan.set_running(process_update.process)
-
- def _cleanup(self, process_update):
- if not self._runner._recovery:
- TaskRunnerHelper.kill_process(self._runner.state, process_update.process)
-
- def on_success(self, process_update):
- log.debug('Process on_success %s' % process_update)
- log.info('Process(%s) finished successfully [rc=%s]' % (
- process_update.process, process_update.return_code))
- self._cleanup(process_update)
- self._runner._task_processes.pop(process_update.process)
- self._runner._watcher.unregister(process_update.process)
- self._runner._plan.add_success(process_update.process)
-
- def _on_abnormal(self, process_update):
- log.info('Process %s had an abnormal termination' % process_update.process)
- self._runner._task_processes.pop(process_update.process)
- self._runner._watcher.unregister(process_update.process)
-
- def on_failed(self, process_update):
- log.debug('Process on_failed %s' % process_update)
- log.info('Process(%s) failed [rc=%s]' % (process_update.process, process_update.return_code))
- self._cleanup(process_update)
- self._on_abnormal(process_update)
- self._runner._plan.add_failure(process_update.process)
- if process_update.process in self._runner._plan.failed:
- log.info('Process %s reached maximum failures, marking process run failed.' %
- process_update.process)
- else:
- log.info('Process %s under maximum failure limit, restarting.' % process_update.process)
-
- def on_lost(self, process_update):
- log.debug('Process on_lost %s' % process_update)
- self._cleanup(process_update)
- self._on_abnormal(process_update)
- self._runner._plan.lost(process_update.process)
-
- def on_killed(self, process_update):
- log.debug('Process on_killed %s' % process_update)
- self._cleanup(process_update)
- self._runner._task_processes.pop(process_update.process)
- self._runner._watcher.unregister(process_update.process)
- log.debug('Process killed, marking it as a loss.')
- self._runner._plan.lost(process_update.process)
-
-
-class TaskRunnerTaskHandler(TaskStateHandler):
- """
- Accesses these parts of the runner:
- _plan [set to regular_plan or finalizing_plan]
- _recovery [boolean, whether or not to side-effect]
- _pathspec [path creation]
- _task [ThermosTask]
- _set_finalization_start
- _kill
- """
-
- def __init__(self, runner):
- self._runner = runner
- self._pathspec = self._runner._pathspec
-
- def on_active(self, task_update):
- log.debug('Task on_active(%s)' % task_update)
- self._runner._plan = self._runner._regular_plan
- if self._runner._recovery:
- return
- TaskRunnerHelper.initialize_task(self._pathspec,
- ThermosTaskWrapper(self._runner._task).to_json())
-
- def on_cleaning(self, task_update):
- log.debug('Task on_cleaning(%s)' % task_update)
- self._runner._finalization_start = task_update.timestamp_ms / 1000.0
- self._runner._terminate_plan(self._runner._regular_plan)
-
- def on_finalizing(self, task_update):
- log.debug('Task on_finalizing(%s)' % task_update)
- if not self._runner._recovery:
- self._runner._kill()
- self._runner._plan = self._runner._finalizing_plan
- if self._runner._finalization_start is None:
- self._runner._finalization_start = task_update.timestamp_ms / 1000.0
-
- def on_killed(self, task_update):
- log.debug('Task on_killed(%s)' % task_update)
- self._cleanup()
-
- def on_success(self, task_update):
- log.debug('Task on_success(%s)' % task_update)
- self._cleanup()
- log.info('Task succeeded.')
-
- def on_failed(self, task_update):
- log.debug('Task on_failed(%s)' % task_update)
- self._cleanup()
-
- def on_lost(self, task_update):
- log.debug('Task on_lost(%s)' % task_update)
- self._cleanup()
-
- def _cleanup(self):
- if not self._runner._recovery:
- self._runner._kill()
- TaskRunnerHelper.finalize_task(self._pathspec)
-
-
-class TaskRunnerUniversalHandler(UniversalStateHandler):
- """
- Universal handler to checkpoint every process and task transition of the runner.
-
- Accesses these parts of the runner:
- _ckpt_write
- """
-
- def __init__(self, runner):
- self._runner = runner
-
- def _checkpoint(self, record):
- self._runner._ckpt_write(record)
-
- def on_process_transition(self, state, process_update):
- log.debug('_on_process_transition: %s' % process_update)
- self._checkpoint(RunnerCkpt(process_status=process_update))
-
- def on_task_transition(self, state, task_update):
- log.debug('_on_task_transition: %s' % task_update)
- self._checkpoint(RunnerCkpt(task_status=task_update))
-
- def on_initialization(self, header):
- log.debug('_on_initialization: %s' % header)
- ThermosTaskValidator.assert_valid_task(self._runner.task)
- ThermosTaskValidator.assert_valid_ports(self._runner.task, header.ports)
- self._checkpoint(RunnerCkpt(runner_header=header))
-
-
-class TaskRunnerStage(object):
- """
- A stage of the task runner pipeline.
- """
- MAX_ITERATION_WAIT = Amount(1, Time.SECONDS)
-
- def __init__(self, runner):
- self.runner = runner
- self.clock = runner._clock
-
- def run(self):
- """
- Perform any work necessary at this stage of the task.
-
- If there is no more work to be done, return None. [This will invoke a state transition.]
-
- If there is still work to be done, return the number of seconds from now in which you'd like
- to be called to re-run the plan.
- """
- return None
-
- def transition_to(self):
- """
- The stage to which we should transition.
- """
- raise NotImplementedError
-
-
-class TaskRunnerStage_ACTIVE(TaskRunnerStage):
- """
- Run the regular plan (i.e. normal, non-finalizing processes.)
- """
- MAX_ITERATION_WAIT = Amount(15, Time.SECONDS)
- MIN_ITERATION_WAIT = Amount(1, Time.SECONDS)
-
- def __init__(self, runner):
- super(TaskRunnerStage_ACTIVE, self).__init__(runner)
-
- def run(self):
- launched = self.runner._run_plan(self.runner._regular_plan)
-
- # Have we terminated?
- terminal_state = None
- if self.runner._regular_plan.is_complete():
- log.info('Regular plan complete.')
- terminal_state = TaskState.SUCCESS if self.runner.is_healthy() else TaskState.FAILED
- elif not self.runner.is_healthy():
- log.error('Regular plan unhealthy!')
- terminal_state = TaskState.FAILED
-
- if terminal_state:
- # No more work to do
- return None
- elif launched > 0:
- # We want to run ASAP after updates have been collected
- return max(self.MIN_ITERATION_WAIT.as_(Time.SECONDS), self.runner._regular_plan.min_wait())
- else:
- # We want to run as soon as something is available to run or after a prescribed timeout.
- return min(self.MAX_ITERATION_WAIT.as_(Time.SECONDS), self.runner._regular_plan.min_wait())
-
- def transition_to(self):
- return TaskState.CLEANING
-
-
-class TaskRunnerStage_CLEANING(TaskRunnerStage):
- """
- Start the cleanup of the regular plan (e.g. if it failed.) On ACTIVE -> CLEANING,
- we send SIGTERMs to all still-running processes. We wait at most finalization_wait
- for all processes to complete before SIGKILLs are sent. If everything exits cleanly
- prior to that point in time, we transition to FINALIZING, which kicks into gear
- the finalization schedule (if any.)
- """
- def run(self):
- log.debug('TaskRunnerStage[CLEANING]: Finalization remaining: %s' %
- self.runner._finalization_remaining())
- if self.runner._finalization_remaining() > 0 and self.runner.has_running_processes():
- return min(self.runner._finalization_remaining(), self.MAX_ITERATION_WAIT.as_(Time.SECONDS))
-
- def transition_to(self):
- if self.runner._finalization_remaining() <= 0:
- log.info('Exceeded finalization wait, skipping finalization.')
- return self.runner.terminal_state()
- return TaskState.FINALIZING
-
-
-class TaskRunnerStage_FINALIZING(TaskRunnerStage):
- """
- Run the finalizing plan, specifically the plan of tasks with the 'final'
- bit marked (e.g. log savers, checkpointers and the like.) Anything in this
- plan will be SIGKILLed if we go over the finalization_wait.
- """
-
- def run(self):
- self.runner._run_plan(self.runner._finalizing_plan)
- log.debug('TaskRunnerStage[FINALIZING]: Finalization remaining: %s' %
- self.runner._finalization_remaining())
- if self.runner.deadlocked(self.runner._finalizing_plan):
- log.warning('Finalizing plan deadlocked.')
- return None
- if self.runner._finalization_remaining() > 0 and not self.runner._finalizing_plan.is_complete():
- return min(self.runner._finalization_remaining(), self.MAX_ITERATION_WAIT.as_(Time.SECONDS))
-
- def transition_to(self):
- if self.runner._finalization_remaining() <= 0:
- log.info('Exceeded finalization wait, terminating finalization.')
- return self.runner.terminal_state()
-
-
-class TaskRunner(object):
- """
- Run a ThermosTask.
-
- This class encapsulates the core logic to run and control the state of a Thermos task.
- Typically, it will be instantiated directly to control a new task, but a TaskRunner can also be
- synthesised from an existing task's checkpoint root
- """
- class Error(Exception): pass
- class InvalidTask(Error): pass
- class InternalError(Error): pass
- class PermissionError(Error): pass
- class StateError(Error): pass
-
- # Maximum amount of time we spend waiting for new updates from the checkpoint streams
- # before doing housecleaning (checking for LOST tasks, dead PIDs.)
- MAX_ITERATION_TIME = Amount(10, Time.SECONDS)
-
- # Minimum amount of time we wait between polls for updates on coordinator checkpoints.
- COORDINATOR_INTERVAL_SLEEP = Amount(1, Time.SECONDS)
-
- # Amount of time we're willing to wait after forking before we expect the runner to have
- # exec'ed the child process.
- LOST_TIMEOUT = Amount(60, Time.SECONDS)
-
- # Active task stages
- STAGES = {
- TaskState.ACTIVE: TaskRunnerStage_ACTIVE,
- TaskState.CLEANING: TaskRunnerStage_CLEANING,
- TaskState.FINALIZING: TaskRunnerStage_FINALIZING
- }
-
- @classmethod
- def get(cls, task_id, checkpoint_root):
- """
- Get a TaskRunner bound to the task_id in checkpoint_root.
- """
- path = TaskPath(root=checkpoint_root, task_id=task_id, state='active')
- task_json = path.getpath('task_path')
- task_checkpoint = path.getpath('runner_checkpoint')
- if not os.path.exists(task_json):
- return None
- task = ThermosConfigLoader.load_json(task_json)
- if task is None:
- return None
- if len(task.tasks()) == 0:
- return None
- try:
- checkpoint = CheckpointDispatcher.from_file(task_checkpoint)
- if checkpoint is None or checkpoint.header is None:
- return None
- return cls(task.tasks()[0].task(), checkpoint_root, checkpoint.header.sandbox,
- log_dir=checkpoint.header.log_dir, task_id=task_id,
- portmap=checkpoint.header.ports)
- except Exception as e:
- log.error('Failed to reconstitute checkpoint in TaskRunner.get: %s' % e, exc_info=True)
- return None
-
- def __init__(self, task, checkpoint_root, sandbox, log_dir=None,
- task_id=None, portmap=None, user=None, chroot=False, clock=time,
- universal_handler=None, planner_class=TaskPlanner):
- """
- required:
- task (config.Task) = the task to run
- checkpoint_root (path) = the checkpoint root
- sandbox (path) = the sandbox in which the path will be run
- [if None, cwd will be assumed, but garbage collection will be
- disabled for this task.]
-
- optional:
- log_dir (string) = directory to house stdout/stderr logs. If not specified, logs will be
- written into the sandbox directory under .logs/
- task_id (string) = bind to this task id. if not specified, will synthesize an id based
- upon task.name()
- portmap (dict) = a map (string => integer) from name to port, e.g. { 'http': 80 }
- user (string) = the user to run the task as. if not current user, requires setuid
- privileges.
- chroot (boolean) = whether or not to chroot into the sandbox prior to exec.
- clock (time interface) = the clock to use throughout
- universal_handler = checkpoint record handler (only used for testing)
- planner_class (TaskPlanner class) = TaskPlanner class to use for constructing the task
- planning policy.
- """
- if not issubclass(planner_class, TaskPlanner):
- raise TypeError('planner_class must be a TaskPlanner.')
- self._clock = clock
- launch_time = self._clock.time()
- launch_time_ms = '%06d' % int((launch_time - int(launch_time)) * 10**6)
- if not task_id:
- self._task_id = '%s-%s.%s' % (task.name(),
- time.strftime('%Y%m%d-%H%M%S', time.localtime(launch_time)),
- launch_time_ms)
- else:
- self._task_id = task_id
- current_user = TaskRunnerHelper.get_actual_user()
- self._user = user or current_user
- # TODO(wickman) This should be delegated to the ProcessPlatform / Helper
- if self._user != current_user:
- if os.geteuid() != 0:
- raise ValueError('task specifies user as %s, but %s does not have setuid permission!' % (
- self._user, current_user))
- self._portmap = portmap or {}
- self._launch_time = launch_time
- self._log_dir = log_dir or os.path.join(sandbox, '.logs')
- self._pathspec = TaskPath(root=checkpoint_root, task_id=self._task_id, log_dir=self._log_dir)
- try:
- ThermosTaskValidator.assert_valid_task(task)
- ThermosTaskValidator.assert_valid_ports(task, self._portmap)
- except ThermosTaskValidator.InvalidTaskError as e:
- raise self.InvalidTask('Invalid task: %s' % e)
- context = ThermosContext(
- task_id=self._task_id,
- ports=self._portmap,
- user=self._user)
- self._task, uninterp = (task % Environment(thermos=context)).interpolate()
- if len(uninterp) > 0:
- raise self.InvalidTask('Failed to interpolate task, missing: %s' %
- ', '.join(str(ref) for ref in uninterp))
- try:
- ThermosTaskValidator.assert_same_task(self._pathspec, self._task)
- except ThermosTaskValidator.InvalidTaskError as e:
- raise self.InvalidTask('Invalid task: %s' % e)
- self._plan = None # plan currently being executed (updated by Handlers)
- self._regular_plan = planner_class(self._task, clock=clock,
- process_filter=lambda proc: proc.final().get() == False)
- self._finalizing_plan = planner_class(self._task, clock=clock,
- process_filter=lambda proc: proc.final().get() == True)
- self._chroot = chroot
- self._sandbox = sandbox
- self._terminal_state = None
- self._ckpt = None
- self._process_map = dict((p.name().get(), p) for p in self._task.processes())
- self._task_processes = {}
- self._stages = dict((state, stage(self)) for state, stage in self.STAGES.items())
- self._finalization_start = None
- self._preemption_deadline = None
- self._watcher = ProcessMuxer(self._pathspec)
- self._state = RunnerState(processes = {})
-
- # create runner state
- universal_handler = universal_handler or TaskRunnerUniversalHandler
- self._dispatcher = CheckpointDispatcher()
- self._dispatcher.register_handler(universal_handler(self))
- self._dispatcher.register_handler(TaskRunnerProcessHandler(self))
- self._dispatcher.register_handler(TaskRunnerTaskHandler(self))
-
- # recover checkpointed runner state and update plan
- self._recovery = True
- self._replay_runner_ckpt()
-
- @property
- def task(self):
- return self._task
-
- @property
- def task_id(self):
- return self._task_id
-
- @property
- def state(self):
- return self._state
-
- @property
- def processes(self):
- return self._task_processes
-
- def task_state(self):
- return self._state.statuses[-1].state if self._state.statuses else TaskState.ACTIVE
-
- def close_ckpt(self):
- """Force close the checkpoint stream. This is necessary for runners terminated through
- exception propagation."""
- log.debug('Closing the checkpoint stream.')
- self._ckpt.close()
-
- @contextmanager
- def control(self, force=False):
- """
- Bind to the checkpoint associated with this task, position to the end of the log if
- it exists, or create it if it doesn't. Fails if we cannot get "leadership" i.e. a
- file lock on the checkpoint stream.
- """
- if self.is_terminal():
- raise TaskRunner.StateError('Cannot take control of a task in terminal state.')
- if self._sandbox:
- safe_mkdir(self._sandbox)
- ckpt_file = self._pathspec.getpath('runner_checkpoint')
- try:
- self._ckpt = TaskRunnerHelper.open_checkpoint(ckpt_file, force=force, state=self._state)
- except TaskRunnerHelper.PermissionError:
- raise TaskRunner.PermissionError('Unable to open checkpoint %s' % ckpt_file)
- log.debug('Flipping recovery mode off.')
- self._recovery = False
- self._set_task_status(self.task_state())
- self._resume_task()
- try:
- yield
- except Exception as e:
- log.error('Caught exception in self.control(): %s' % e)
- log.error(' %s' % traceback.format_exc())
- self._ckpt.close()
-
- def _resume_task(self):
- assert self._ckpt is not None
- unapplied_updates = self._replay_process_ckpts()
- if self.is_terminal():
- raise self.StateError('Cannot resume terminal task.')
- self._initialize_ckpt_header()
- self._replay(unapplied_updates)
-
- def _ckpt_write(self, record):
- """
- Write to the checkpoint stream if we're not in recovery mode.
- """
- if not self._recovery:
- self._ckpt.write(record)
-
- def _replay(self, checkpoints):
- """
- Replay a sequence of RunnerCkpts.
- """
- for checkpoint in checkpoints:
- self._dispatcher.dispatch(self._state, checkpoint)
-
- def _replay_runner_ckpt(self):
- """
- Replay the checkpoint stream associated with this task.
- """
- ckpt_file = self._pathspec.getpath('runner_checkpoint')
- if os.path.exists(ckpt_file):
- fp = open(ckpt_file, "r")
- ckpt_recover = ThriftRecordReader(fp, RunnerCkpt)
- for record in ckpt_recover:
- log.debug('Replaying runner checkpoint record: %s' % record)
- self._dispatcher.dispatch(self._state, record, recovery=True)
- ckpt_recover.close()
-
- def _replay_process_ckpts(self):
- """
- Replay the unmutating process checkpoints. Return the unapplied process updates that
- would mutate the runner checkpoint stream.
- """
- process_updates = self._watcher.select()
- unapplied_process_updates = []
- for process_update in process_updates:
- if self._dispatcher.would_update(self._state, process_update):
- unapplied_process_updates.append(process_update)
- else:
- self._dispatcher.dispatch(self._state, process_update, recovery=True)
- return unapplied_process_updates
-
- def _initialize_ckpt_header(self):
- """
- Initializes the RunnerHeader for this checkpoint stream if it has not already
- been constructed.
- """
- if self._state.header is None:
- header = RunnerHeader(
- task_id=self._task_id,
- launch_time_ms=int(self._launch_time*1000),
- sandbox=self._sandbox,
- log_dir=self._log_dir,
- hostname=socket.gethostname(),
- user=self._user,
- ports=self._portmap)
- runner_ckpt = RunnerCkpt(runner_header=header)
- self._dispatcher.dispatch(self._state, runner_ckpt)
-
- def _set_task_status(self, state):
- update = TaskStatus(state=state, timestamp_ms=int(self._clock.time() * 1000),
- runner_pid=os.getpid(), runner_uid=os.getuid())
- runner_ckpt = RunnerCkpt(task_status=update)
- self._dispatcher.dispatch(self._state, runner_ckpt, self._recovery)
-
- def _finalization_remaining(self):
- # If a preemption deadline has been set, use that.
- if self._preemption_deadline:
- return max(0, self._preemption_deadline - self._clock.time())
-
- # Otherwise, use the finalization wait provided in the configuration.
- finalization_allocation = self.task.finalization_wait().get()
- if self._finalization_start is None:
- return sys.float_info.max
- else:
- waited = max(0, self._clock.time() - self._finalization_start)
- return max(0, finalization_allocation - waited)
-
- def _set_process_status(self, process_name, process_state, **kw):
- if 'sequence_number' in kw:
- sequence_number = kw.pop('sequence_number')
- log.debug('_set_process_status(%s <= %s, seq=%s[force])' % (process_name,
- ProcessState._VALUES_TO_NAMES.get(process_state), sequence_number))
- else:
- current_run = self._current_process_run(process_name)
- if not current_run:
- assert process_state == ProcessState.WAITING
- sequence_number = 0
- else:
- sequence_number = current_run.seq + 1
- log.debug('_set_process_status(%s <= %s, seq=%s[auto])' % (process_name,
- ProcessState._VALUES_TO_NAMES.get(process_state), sequence_number))
- runner_ckpt = RunnerCkpt(process_status=ProcessStatus(
- process=process_name, state=process_state, seq=sequence_number, **kw))
- self._dispatcher.dispatch(self._state, runner_ckpt, self._recovery)
-
- def _task_process_from_process_name(self, process_name, sequence_number):
- """
- Construct a Process() object from a process_name, populated with its
- correct run number and fully interpolated commandline.
- """
- run_number = len(self.state.processes[process_name]) - 1
- pathspec = self._pathspec.given(process=process_name, run=run_number)
- process = self._process_map.get(process_name)
- if process is None:
- raise self.InternalError('FATAL: Could not find process: %s' % process_name)
- def close_ckpt_and_fork():
- pid = os.fork()
- if pid == 0 and self._ckpt is not None:
- self._ckpt.close()
- return pid
- return Process(
- process.name().get(),
- process.cmdline().get(),
- sequence_number,
- pathspec,
- self._sandbox,
- self._user,
- chroot=self._chroot,
- fork=close_ckpt_and_fork)
-
- def deadlocked(self, plan=None):
- """Check whether a plan is deadlocked, i.e. there are no running/runnable processes, and the
- plan is not complete."""
- plan = plan or self._regular_plan
- now = self._clock.time()
- running = list(plan.running)
- runnable = list(plan.runnable_at(now))
- waiting = list(plan.waiting_at(now))
- log.debug('running:%d runnable:%d waiting:%d complete:%s' % (
- len(running), len(runnable), len(waiting), plan.is_complete()))
- return len(running + runnable + waiting) == 0 and not plan.is_complete()
-
- def is_healthy(self):
- """Check whether the TaskRunner is healthy. A healthy TaskRunner is not deadlocked and has not
- reached its max_failures count."""
- max_failures = self._task.max_failures().get()
- deadlocked = self.deadlocked()
- under_failure_limit = max_failures == 0 or len(self._regular_plan.failed) < max_failures
- log.debug('max_failures:%d failed:%d under_failure_limit:%s deadlocked:%s ==> health:%s' % (
- max_failures, len(self._regular_plan.failed), under_failure_limit, deadlocked,
- not deadlocked and under_failure_limit))
- return not deadlocked and under_failure_limit
-
- def _current_process_run(self, process_name):
- if process_name not in self._state.processes or len(self._state.processes[process_name]) == 0:
- return None
- return self._state.processes[process_name][-1]
-
- def is_process_lost(self, process_name):
- """Determine whether or not we should mark a task as LOST and do so if necessary."""
- current_run = self._current_process_run(process_name)
- if not current_run:
- raise self.InternalError('No current_run for process %s!' % process_name)
-
- def forked_but_never_came_up():
- return current_run.state == ProcessState.FORKED and (
- self._clock.time() - current_run.fork_time > TaskRunner.LOST_TIMEOUT.as_(Time.SECONDS))
-
- def running_but_coordinator_died():
- if current_run.state != ProcessState.RUNNING:
- return False
- coordinator_pid, _, _ = TaskRunnerHelper.scan_process(self.state, process_name)
- if coordinator_pid is not None:
- return False
- elif self._watcher.has_data(process_name):
- return False
- return True
-
- if forked_but_never_came_up() or running_but_coordinator_died():
- log.info('Detected a LOST task: %s' % current_run)
- log.debug(' forked_but_never_came_up: %s' % forked_but_never_came_up())
- log.debug(' running_but_coordinator_died: %s' % running_but_coordinator_died())
- return True
-
- return False
-
- def _run_plan(self, plan):
- log.debug('Schedule pass:')
-
- running = list(plan.running)
- log.debug('running: %s' % ' '.join(plan.running))
- log.debug('finished: %s' % ' '.join(plan.finished))
-
- launched = []
- for process_name in plan.running:
- if self.is_process_lost(process_name):
- self._set_process_status(process_name, ProcessState.LOST)
-
- now = self._clock.time()
- runnable = list(plan.runnable_at(now))
- waiting = list(plan.waiting_at(now))
- log.debug('runnable: %s' % ' '.join(runnable))
- log.debug('waiting: %s' % ' '.join(
- '%s[T-%.1fs]' % (process, plan.get_wait(process)) for process in waiting))
-
- def pick_processes(process_list):
- if self._task.max_concurrency().get() == 0:
- return process_list
- num_to_pick = max(self._task.max_concurrency().get() - len(running), 0)
- return process_list[:num_to_pick]
-
- for process_name in pick_processes(runnable):
- tp = self._task_processes.get(process_name)
- if tp:
- current_run = self._current_process_run(process_name)
- assert current_run.state == ProcessState.WAITING
- else:
- self._set_process_status(process_name, ProcessState.WAITING)
- tp = self._task_processes[process_name]
- log.info('Forking Process(%s)' % process_name)
- tp.start()
- launched.append(tp)
-
- return len(launched) > 0
-
- def _terminate_plan(self, plan):
- for process in plan.running:
- last_run = self._current_process_run(process)
- if last_run and last_run.state in (ProcessState.FORKED, ProcessState.RUNNING):
- TaskRunnerHelper.terminate_process(self.state, process)
-
- def has_running_processes(self):
- """
- Returns True if any processes associated with this task have active pids.
- """
- process_tree = TaskRunnerHelper.scantree(self.state)
- return any(any(process_set) for process_set in process_tree.values())
-
- def has_active_processes(self):
- """
- Returns True if any processes are in non-terminal states.
- """
- return any(not TaskRunnerHelper.is_process_terminal(run.state) for run in
- filter(None, (self._current_process_run(process) for process in self.state.processes)))
-
- def collect_updates(self, timeout=None):
- """
- Collects and applies updates from process checkpoint streams. Returns the number
- of applied process checkpoints.
- """
- if self.has_active_processes():
- sleep_interval = self.COORDINATOR_INTERVAL_SLEEP.as_(Time.SECONDS)
- total_time = 0.0
- while True:
- process_updates = self._watcher.select()
- for process_update in process_updates:
- self._dispatcher.dispatch(self._state, process_update, self._recovery)
- if process_updates:
- return len(process_updates)
- if timeout and total_time >= timeout:
- break
- total_time += sleep_interval
- self._clock.sleep(sleep_interval)
- return 0
-
- def is_terminal(self):
- return TaskRunnerHelper.is_task_terminal(self.task_state())
-
- def terminal_state(self):
- if self._terminal_state:
- log.debug('Forced terminal state: %s' %
- TaskState._VALUES_TO_NAMES.get(self._terminal_state, 'UNKNOWN'))
- return self._terminal_state
- else:
- return TaskState.SUCCESS if self.is_healthy() else TaskState.FAILED
-
- def run(self, force=False):
- """
- Entrypoint to runner. Assume control of checkpoint stream, and execute TaskRunnerStages
- until runner is terminal.
- """
- if self.is_terminal():
- return
- with self.control(force):
- self._run()
-
- def _run(self):
- iteration_time = self.MAX_ITERATION_TIME.as_(Time.SECONDS)
- while not self.is_terminal():
- start = self._clock.time()
- # step 1: execute stage corresponding to the state we're currently in
- runner = self._stages[self.task_state()]
- iteration_wait = runner.run()
- if iteration_wait is None:
- log.debug('Run loop: No more work to be done in state %s' %
- TaskState._VALUES_TO_NAMES.get(self.task_state(), 'UNKNOWN'))
- self._set_task_status(runner.transition_to())
- continue
- log.debug('Run loop: Work to be done within %.1fs' % iteration_wait)
- # step 2: check child process checkpoint streams for updates
- if not self.collect_updates(iteration_wait):
- # If we don't collect any updates, at least 'touch' the checkpoint stream
- # so as to prevent garbage collection.
- elapsed = self._clock.time() - start
- if elapsed < iteration_wait:
- log.debug('Update collection only took %.1fs, idling %.1fs' % (
- elapsed, iteration_wait - elapsed))
- self._clock.sleep(iteration_wait - elapsed)
- log.debug('Run loop: No updates collected, touching checkpoint.')
- os.utime(self._pathspec.getpath('runner_checkpoint'), None)
- # step 3: reap any zombie child processes
- TaskRunnerHelper.reap_children()
-
- def kill(self, force=False, terminal_status=TaskState.KILLED,
- preemption_wait=Amount(1, Time.MINUTES)):
- """
- Kill all processes associated with this task and set task/process states as terminal_status
- (defaults to KILLED)
- """
- log.debug('Runner issued kill: force:%s, preemption_wait:%s' % (
- force, preemption_wait))
- assert terminal_status in (TaskState.KILLED, TaskState.LOST)
- self._preemption_deadline = self._clock.time() + preemption_wait.as_(Time.SECONDS)
- with self.control(force):
- if self.is_terminal():
- log.warning('Task is not in ACTIVE state, cannot issue kill.')
- return
- self._terminal_state = terminal_status
- if self.task_state() == TaskState.ACTIVE:
- self._set_task_status(TaskState.CLEANING)
- self._run()
-
- def lose(self, force=False):
- """
- Mark a task as LOST and kill any straggling processes.
- """
- self.kill(force, preemption_wait=Amount(0, Time.SECONDS), terminal_status=TaskState.LOST)
-
- def _kill(self):
- processes = TaskRunnerHelper.scantree(self._state)
- for process, pid_tuple in processes.items():
- current_run = self._current_process_run(process)
- coordinator_pid, pid, tree = pid_tuple
- if TaskRunnerHelper.is_process_terminal(current_run.state):
- if coordinator_pid or pid or tree:
- log.warning('Terminal process (%s) still has running pids:' % process)
- log.warning(' coordinator_pid: %s' % coordinator_pid)
- log.warning(' pid: %s' % pid)
- log.warning(' tree: %s' % tree)
- TaskRunnerHelper.kill_process(self.state, process)
- else:
- if coordinator_pid or pid or tree:
- log.info('Transitioning %s to KILLED' % process)
- self._set_process_status(process, ProcessState.KILLED,
- stop_time=self._clock.time(), return_code=-1)
- else:
- log.info('Transitioning %s to LOST' % process)
- if current_run.state != ProcessState.WAITING:
- self._set_process_status(process, ProcessState.LOST)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/BUILD b/src/main/python/twitter/thermos/monitoring/BUILD
deleted file mode 100644
index a977878..0000000
--- a/src/main/python/twitter/thermos/monitoring/BUILD
+++ /dev/null
@@ -1,97 +0,0 @@
-import os
-
-python_library(
- name = 'detector',
- sources = ['detector.py'],
- dependencies = [
- pants('src/main/python/twitter/thermos/common:path')
- ]
-)
-
-python_library(
- name = 'garbage',
- sources = ['garbage.py'],
- dependencies = [
- pants(':detector'),
- pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/python/twitter/thermos/common:path'),
- ]
-)
-
-python_library(
- name = 'monitor',
- sources = ['monitor.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'disk',
- sources = ['disk.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- python_requirement('watchdog'),
- ]
-)
-
-python_library(
- name = 'process',
- sources = ['process.py'],
-)
-
-python_library(
- name = 'process_collector_psutil',
- sources = ['process_collector_psutil.py'],
- dependencies = [
- pants(':process'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/thermos:psutil'),
- ]
-)
-
-python_library(
- name = 'resource',
- sources = ['resource.py'],
- dependencies = [
- pants(':disk'),
- pants(':monitor'),
- pants(':process'),
- pants(':process_collector_psutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/collections'),
- pants('aurora/twitterdeps/src/python/twitter/common/concurrent'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- ]
-)
-
-python_library(
- name = 'monitoring',
- dependencies = [
- pants(':detector'),
- pants(':disk'),
- pants(':garbage'),
- pants(':monitor'),
- pants(':process'),
- pants(':resource'),
-
- # covering dependency for common
- pants('src/main/python/twitter/thermos/common'),
- ],
- provides = setup_py(
- name = 'twitter.thermos.monitoring',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- description = 'Thermos monitoring library.',
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/__init__.py b/src/main/python/twitter/thermos/monitoring/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/detector.py b/src/main/python/twitter/thermos/monitoring/detector.py
deleted file mode 100644
index ed48c93..0000000
--- a/src/main/python/twitter/thermos/monitoring/detector.py
+++ /dev/null
@@ -1,91 +0,0 @@
-"""Detect Thermos tasks on disk
-
-This module contains the TaskDetector, used to detect Thermos tasks within a given checkpoint root.
-
-"""
-
-import glob
-import os
-import re
-
-from twitter.thermos.common.path import TaskPath
-
-
-class TaskDetector(object):
- """
- Helper class in front of TaskPath to detect active/finished/running tasks. Performs no
- introspection on the state of a task; merely detects based on file paths on disk.
- """
- class MatchingError(Exception): pass
-
- def __init__(self, root):
- self._root_dir = root
- self._pathspec = TaskPath()
-
- def get_task_ids(self, state=None):
- paths = glob.glob(self._pathspec.given(root=self._root_dir,
- task_id="*",
- state=state or '*')
- .getpath('task_path'))
- path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
- task_id="(\S+)",
- state='(\S+)')
- .getpath('task_path'))
- for path in paths:
- try:
- task_state, task_id = path_re.match(path).groups()
- except:
- continue
- if state is None or task_state == state:
- yield (task_state, task_id)
-
- def get_process_runs(self, task_id, log_dir):
- paths = glob.glob(self._pathspec.given(root=self._root_dir,
- task_id=task_id,
- log_dir=log_dir,
- process='*',
- run='*')
- .getpath('process_logdir'))
- path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
- task_id=re.escape(task_id),
- log_dir=log_dir,
- process='(\S+)',
- run='(\d+)')
- .getpath('process_logdir'))
- for path in paths:
- try:
- process, run = path_re.match(path).groups()
- except:
- continue
- yield process, int(run)
-
- def get_process_logs(self, task_id, log_dir):
- for process, run in self.get_process_runs(task_id, log_dir):
- for logtype in ('stdout', 'stderr'):
- path = (self._pathspec.with_filename(logtype).given(root=self._root_dir,
- task_id=task_id,
- log_dir=log_dir,
- process=process,
- run=run)
- .getpath('process_logdir'))
- if os.path.exists(path):
- yield path
-
- def get_checkpoint(self, task_id):
- return self._pathspec.given(root=self._root_dir, task_id=task_id).getpath('runner_checkpoint')
-
- def get_process_checkpoints(self, task_id):
- matching_paths = glob.glob(self._pathspec.given(root=self._root_dir,
- task_id=task_id,
- process='*')
- .getpath('process_checkpoint'))
- path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
- task_id=re.escape(task_id),
- process='(\S+)')
- .getpath('process_checkpoint'))
- for path in matching_paths:
- try:
- process, = path_re.match(path).groups()
- except:
- continue
- yield path
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/disk.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/disk.py b/src/main/python/twitter/thermos/monitoring/disk.py
deleted file mode 100644
index 9116746..0000000
--- a/src/main/python/twitter/thermos/monitoring/disk.py
+++ /dev/null
@@ -1,181 +0,0 @@
-"""Sample disk usage under a particular path
-
-This module provides threads which can be used to gather information on the disk utilisation
-under a particular path.
-
-Currently, there are two threads available:
- - DiskCollectorThread, which periodically uses a basic brute-force approach (os.stat()ing every
- file within the path)
- - InotifyDiskCollectorThread, which updates disk utilisation dynamically by using inotify to
- monitor disk changes within the path
-
-"""
-
-import os
-import threading
-import time
-from Queue import Empty, Queue
-
-from twitter.common import log
-from twitter.common.dirutil import du, safe_bsize
-from twitter.common.exceptions import ExceptionalThread
-from twitter.common.lang import Lockable
-from twitter.common.quantity import Amount, Time
-
-from watchdog.observers import Observer as WatchdogObserver
-from watchdog.events import (
- FileSystemEventHandler,
- FileCreatedEvent,
- FileDeletedEvent,
- FileModifiedEvent,
- FileMovedEvent,
-)
-
-
-class DiskCollectorThread(ExceptionalThread):
- """ Thread to calculate aggregate disk usage under a given path using a simple algorithm """
- def __init__(self, path):
- self.path = path
- self.value = None
- self.event = threading.Event()
- super(DiskCollectorThread, self).__init__()
- self.daemon = True
-
- def run(self):
- log.debug("DiskCollectorThread: starting collection of %s" % self.path)
- self.value = du(self.path)
- log.debug("DiskCollectorThread: finished collection of %s" % self.path)
- self.event.set()
-
- def finished(self):
- return self.event.is_set()
-
-
-class DiskCollector(Lockable):
- """ Spawn a background thread to sample disk usage """
- def __init__(self, root):
- self._root = root
- self._thread = None
- self._value = 0
- super(DiskCollector, self).__init__()
-
- @Lockable.sync
- def sample(self):
- """ Trigger collection of sample, if not already begun """
- if self._thread is None:
- self._thread = DiskCollectorThread(self._root)
- self._thread.start()
-
- @property
- @Lockable.sync
- def value(self):
- """ Retrieve value of disk usage """
- if self._thread is not None and self._thread.finished():
- self._value = self._thread.value
- self._thread = None
- return self._value
-
- @property
- @Lockable.sync
- def completed_event(self):
- """ Return a threading.Event that will block until an in-progress disk collection is complete,
- or block indefinitely otherwise. Use with caution! (i.e.: set a timeout) """
- if self._thread is not None:
- return self._thread.event
- else:
- return threading.Event()
-
-
-class InotifyDiskCollectorThread(ExceptionalThread, FileSystemEventHandler):
- """ Thread to calculate aggregate disk usage under a given path
-
- Note that while this thread uses inotify (through the watchdog module) to monitor disk events in
- "real time", the actual processing of events is only performed periodically (configured via
- COLLECTION_INTERVAL)
-
- """
- INTERESTING_EVENTS = (FileCreatedEvent, FileDeletedEvent, FileModifiedEvent, FileMovedEvent)
- COLLECTION_INTERVAL = Amount(5, Time.SECONDS)
-
- def __init__(self, path):
- self._path = path
- self._files = {} # file path => size (bytes)
- self._queue = Queue()
- self._observer = WatchdogObserver()
- super(InotifyDiskCollectorThread, self).__init__()
- self.daemon = True
-
- def dispatch(self, event):
- """ Dispatch all interesting events to the internal queue """
- if isinstance(event, self.INTERESTING_EVENTS):
- self._queue.put(event)
-
- def _initialize(self):
- """ Collect an initial snapshot of the disk usage in the path """
- log.debug("Starting watchdog observer to collect events...")
- self._observer.schedule(self, path=self._path, recursive=True)
- self._observer.start()
- log.debug("Collecting initial disk usage sample...")
- for root, _, files in os.walk(self._path):
- for filename in files:
- f = os.path.join(root, filename)
- self._files[f] = safe_bsize(f)
-
- def _process_events(self):
- """ Deduplicate and process watchdog events, updating the internal file store appropriately """
- file_ops = {}
-
- def remove_file(path):
- self._files.pop(path, None)
- def stat_file(path):
- self._files[path] = safe_bsize(path)
-
- while not self._to_process.empty():
- event = self._to_process.get()
- if isinstance(event, (FileCreatedEvent, FileModifiedEvent)):
- file_ops[event.src_path] = lambda: stat_file(event.src_path)
- elif isinstance(event, FileDeletedEvent):
- file_ops[event.src_path] = lambda: remove_file(event.src_path)
- elif isinstance(event, FileMovedEvent):
- file_ops[event.src_path] = lambda: remove_file(event.src_path)
- file_ops[event.dest_path] = lambda: stat_file(event.dest_path)
-
- for op in file_ops.values():
- op()
-
- def run(self):
- """ Loop indefinitely, periodically processing watchdog/inotify events. """
- self._initialize()
- log.debug("Initialization complete. Moving to handling events.")
- while True:
- next = time.time() + self.COLLECTION_INTERVAL.as_(Time.SECONDS)
- if not self._queue.empty():
- self._to_process, self._queue = self._queue, Queue()
- self._process_events()
- time.sleep(max(0, next - time.time()))
-
- @property
- def value(self):
- return sum(self._files.itervalues())
-
-
-class InotifyDiskCollector(object):
- """ Spawn a background thread to sample disk usage """
- def __init__(self, root):
- self._root = root
- self._thread = InotifyDiskCollectorThread(self._root)
-
- def sample(self):
- """ Trigger disk collection loop. """
- if not os.path.exists(self._root):
- log.error('Cannot start monitoring path until it exists')
- elif not self._thread.is_alive():
- self._thread.start()
-
- @property
- def value(self):
- return self._thread.value
-
- @property
- def completed_event(self):
- return threading.Event()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/garbage.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/garbage.py b/src/main/python/twitter/thermos/monitoring/garbage.py
deleted file mode 100644
index 4ac0d3a..0000000
--- a/src/main/python/twitter/thermos/monitoring/garbage.py
+++ /dev/null
@@ -1,183 +0,0 @@
-from abc import abstractmethod
-from collections import namedtuple
-import os
-import sys
-import time
-
-from twitter.common.dirutil import safe_delete, safe_rmtree, safe_bsize
-from twitter.common.lang import Interface
-from twitter.common.quantity import Amount, Data, Time
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-from twitter.thermos.common.path import TaskPath
-
-from .detector import TaskDetector
-
-
-class TaskGarbageCollector(object):
- def __init__(self, root):
- self._root = root
- self._detector = TaskDetector(root=self._root)
- self._states = {}
-
- def state(self, task_id):
- if task_id not in self._states:
- self._states[task_id] = CheckpointDispatcher.from_file(self._detector.get_checkpoint(task_id))
- return self._states[task_id]
-
- def get_age(self, task_id):
- return os.path.getmtime(self._detector.get_checkpoint(task_id))
-
- def get_finished_tasks(self):
- return [task_id for _, task_id in self._detector.get_task_ids(state='finished')]
-
- def get_metadata(self, task_id, with_size=True):
- runner_ckpt = self._detector.get_checkpoint(task_id)
- process_ckpts = [ckpt for ckpt in self._detector.get_process_checkpoints(task_id)]
- json_spec = TaskPath(root=self._root, task_id=task_id, state='finished').getpath('task_path')
- for path in [json_spec, runner_ckpt] + process_ckpts:
- if with_size:
- yield path, safe_bsize(path)
- else:
- yield path
-
- def get_logs(self, task_id, with_size=True):
- state = self.state(task_id)
- if state and state.header:
- for path in self._detector.get_process_logs(task_id, state.header.log_dir):
- if with_size:
- yield path, safe_bsize(path)
- else:
- yield path
-
- def get_data(self, task_id, with_size=True):
- state = self.state(task_id)
- if state and state.header and state.header.sandbox:
- for root, dirs, files in os.walk(state.header.sandbox):
- for file in files:
- filename = os.path.join(root, file)
- if with_size:
- yield filename, safe_bsize(filename)
- else:
- yield filename
-
- def erase_task(self, task_id):
- self.erase_data(task_id)
- self.erase_logs(task_id)
- self.erase_metadata(task_id)
-
- def erase_metadata(self, task_id):
- for fn in self.get_metadata(task_id, with_size=False):
- safe_delete(fn)
- safe_rmtree(TaskPath(root=self._root, task_id=task_id).getpath('checkpoint_path'))
-
- def erase_logs(self, task_id):
- for fn in self.get_logs(task_id, with_size=False):
- safe_delete(fn)
- state = self.state(task_id)
- if state and state.header:
- safe_rmtree(TaskPath(root=self._root, task_id=task_id, log_dir=state.header.log_dir)
- .getpath('process_logbase'))
-
- def erase_data(self, task_id):
- # TODO(wickman)
- # This could be potentially dangerous if somebody naively runs their sandboxes in e.g.
- # $HOME or / or similar. Perhaps put a guard somewhere?
- for fn in self.get_data(task_id, with_size=False):
- os.unlink(fn)
- state = self.state(task_id)
- if state and state.header and state.header.sandbox:
- safe_rmtree(state.header.sandbox)
-
-
-class TaskGarbageCollectionPolicy(Interface):
- def __init__(self, collector):
- if not isinstance(collector, TaskGarbageCollector):
- raise ValueError(
- "Expected collector to be a TaskGarbageCollector, got %s" % collector.__class__.__name__)
- self._collector = collector
-
- @property
- def collector(self):
- return self._collector
-
- @abstractmethod
- def run(self):
- """Returns a list of task_ids that should be garbage collected given the specified policy."""
-
-
-class DefaultCollector(TaskGarbageCollectionPolicy):
- def __init__(self, collector, **kw):
- """
- Default garbage collection policy.
-
- Arguments that may be specified:
- max_age: Amount(Time) (max age of a retained task) [default: infinity]
- max_space: Amount(Data) (max space to keep) [default: infinity]
- max_tasks: int (max number of tasks to keep) [default: infinity]
- include_metadata: boolean (Whether or not to include metadata in the
- space calculations.) [default: True]
- include_logs: boolean (Whether or not to include logs in the
- space calculations.) [default: True]
- verbose: boolean (whether or not to log) [default: False]
- logger: callable (function to call with log messages) [default: sys.stdout.write]
- """
- self._max_age = kw.get('max_age', Amount(10**10, Time.DAYS))
- self._max_space = kw.get('max_space', Amount(10**10, Data.TB))
- self._max_tasks = kw.get('max_tasks', 10**10)
- self._include_metadata = kw.get('include_metadata', True)
- self._include_logs = kw.get('include_logs', True)
- self._verbose = kw.get('verbose', False)
- self._logger = kw.get('logger', sys.stdout.write)
- TaskGarbageCollectionPolicy.__init__(self, collector)
-
- def log(self, msg):
- if self._verbose:
- self._logger(msg)
-
- def run(self):
- tasks = []
- now = time.time()
-
- TaskTuple = namedtuple('TaskTuple', 'task_id age metadata_size log_size data_size')
- for task_id in self.collector.get_finished_tasks():
- age = Amount(int(now - self.collector.get_age(task_id)), Time.SECONDS)
- self.log('Analyzing task %s (age: %s)... ' % (task_id, age))
- metadata_size = Amount(sum(sz for _, sz in self.collector.get_metadata(task_id)), Data.BYTES)
- self.log(' metadata %.1fKB ' % metadata_size.as_(Data.KB))
- log_size = Amount(sum(sz for _, sz in self.collector.get_logs(task_id)), Data.BYTES)
- self.log(' logs %.1fKB ' % log_size.as_(Data.KB))
- data_size = Amount(sum(sz for _, sz in self.collector.get_data(task_id)), Data.BYTES)
- self.log(' data %.1fMB ' % data_size.as_(Data.MB))
- tasks.append(TaskTuple(task_id, age, metadata_size, log_size, data_size))
-
- gc_tasks = set()
- gc_tasks.update(task for task in tasks if task.age > self._max_age)
- self.log('After age filter: %s tasks' % len(gc_tasks))
-
- def total_gc_size(task):
- return sum([task.data_size,
- task.metadata_size if self._include_metadata else Amount(0, Data.BYTES),
- task.log_size if self._include_logs else Amount(0, Data.BYTES)],
- Amount(0, Data.BYTES))
-
- total_used = Amount(0, Data.BYTES)
- for task in sorted(tasks, key=lambda tsk: tsk.age, reverse=True):
- if task not in gc_tasks:
- total_used += total_gc_size(task)
- if total_used > self._max_space:
- gc_tasks.add(task)
- self.log('After size filter: %s tasks' % len(gc_tasks))
-
- for task in sorted(tasks, key=lambda tsk: tsk.age, reverse=True):
- if task not in gc_tasks and len(tasks) - len(gc_tasks) > self._max_tasks:
- gc_tasks.add(task)
- self.log('After total task filter: %s tasks' % len(gc_tasks))
-
- self.log('Deciding to garbage collect the following tasks:')
- if gc_tasks:
- for task in gc_tasks:
- self.log(' %s' % repr(task))
- else:
- self.log(' None.')
-
- return gc_tasks
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/monitor.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/monitor.py b/src/main/python/twitter/thermos/monitoring/monitor.py
deleted file mode 100644
index 5af67fe..0000000
--- a/src/main/python/twitter/thermos/monitoring/monitor.py
+++ /dev/null
@@ -1,125 +0,0 @@
-"""Monitor the state of Thermos tasks on a system
-
-This module contains the TaskMonitor, used to reconstruct the state of active or finished Thermos
-tasks based on their checkpoint streams. It exposes two key pieces of information about a Task, both
-as their corresponding Thrift structs:
- - a RunnerState, representing the latest state of the Task
- - a list of ProcessStates, representing the processes currently running within the Task
-
-"""
-
-import os
-import copy
-import errno
-import threading
-
-from twitter.common import log
-from twitter.common.recordio import ThriftRecordReader
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-
-from gen.twitter.thermos.ttypes import (
- ProcessState,
- RunnerCkpt,
- RunnerState,
- TaskState,
-)
-
-
-class TaskMonitor(object):
- """
- Class responsible for reconstructing and monitoring the state of an individual Thermos task via
- its runner checkpoint. Also exports information on active processes in the task.
- """
-
- def __init__(self, pathspec, task_id):
- self._task_id = task_id
- self._dispatcher = CheckpointDispatcher()
- self._runnerstate = RunnerState(processes={})
- self._runner_ckpt = pathspec.given(task_id=task_id).getpath('runner_checkpoint')
- self._active_file, self._finished_file = (
- pathspec.given(task_id=task_id, state=state).getpath('task_path')
- for state in ('active', 'finished'))
- self._ckpt_head = 0
- self._apply_states()
- self._lock = threading.Lock()
-
- def _apply_states(self):
- """
- os.stat() the corresponding checkpoint stream of this task and determine if there are new ckpt
- records. Attempt to read those records and update the high watermark for that stream.
- Returns True if new states were applied, False otherwise.
- """
- ckpt_offset = None
- try:
- ckpt_offset = os.stat(self._runner_ckpt).st_size
-
- updated = False
- if self._ckpt_head < ckpt_offset:
- with open(self._runner_ckpt, 'r') as fp:
- fp.seek(self._ckpt_head)
- rr = ThriftRecordReader(fp, RunnerCkpt)
- while True:
- runner_update = rr.try_read()
- if not runner_update:
- break
- try:
- self._dispatcher.dispatch(self._runnerstate, runner_update)
- except CheckpointDispatcher.InvalidSequenceNumber as e:
- log.error('Checkpoint stream is corrupt: %s' % e)
- break
- new_ckpt_head = fp.tell()
- updated = self._ckpt_head != new_ckpt_head
- self._ckpt_head = new_ckpt_head
- return updated
- except OSError as e:
- if e.errno == errno.ENOENT:
- # The log doesn't yet exist, will retry later.
- log.warning('Could not read from discovered task %s.' % self._task_id)
- return False
- else:
- raise
-
- def refresh(self):
- """
- Check to see if there are new updates and apply them. Return true if
- updates were applied, false otherwise.
- """
- with self._lock:
- return self._apply_states()
-
- def get_state(self):
- """
- Get the latest state of this Task.
- """
- with self._lock:
- self._apply_states()
- return copy.deepcopy(self._runnerstate)
-
- def task_state(self):
- state = self.get_state()
- return state.statuses[-1].state if state.statuses else TaskState.ACTIVE
-
- @property
- def active(self):
- return os.path.exists(self._active_file)
-
- @property
- def finished(self):
- return os.path.exists(self._finished_file)
-
- def get_active_processes(self):
- """
- Get active processes. Returned is a list of tuples of the form:
- (ProcessStatus object of running object, its run number)
- """
- active_processes = []
- with self._lock:
- self._apply_states()
- state = self._runnerstate
- for process, runs in state.processes.items():
- if len(runs) == 0:
- continue
- last_run = runs[-1]
- if last_run.state == ProcessState.RUNNING:
- active_processes.append((last_run, len(runs) - 1))
- return active_processes
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/process.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/process.py b/src/main/python/twitter/thermos/monitoring/process.py
deleted file mode 100644
index 294682f..0000000
--- a/src/main/python/twitter/thermos/monitoring/process.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""Represent resource consumption statistics for processes
-
-This module exposes one class: the ProcessSample, used to represent resource consumption. A single
-ProcessSample might correspond to one individual process, or to an aggregate of multiple processes.
-
-"""
-
-from collections import namedtuple
-
-
-class ProcessSample(namedtuple('ProcessSample', 'rate user system rss vms nice status threads')):
- """ Sample of statistics about a process's resource consumption (either a single process or an
- aggregate of processes) """
-
- @staticmethod
- def empty():
- return ProcessSample(rate=0, user=0, system=0, rss=0, vms=0, nice=None, status=None, threads=0)
-
- def __add__(self, other):
- if self.nice is not None and other.nice is None:
- nice = self.nice
- else:
- nice = other.nice
- if self.status is not None and other.status is None:
- status = self.status
- else:
- status = other.status
- return ProcessSample(
- rate = self.rate + other.rate,
- user = self.user + other.user,
- system = self.system + other.system,
- rss = self.rss + other.rss,
- vms = self.vms + other.vms,
- nice = nice,
- status = status,
- threads = self.threads + other.threads)
-
- def to_dict(self):
- return dict(
- cpu = self.rate,
- ram = self.rss,
- user = self.user,
- system = self.system,
- rss = self.rss,
- vms = self.vms,
- nice = self.nice,
- status = str(self.status),
- threads = self.threads
- )
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/process_collector_psutil.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/process_collector_psutil.py b/src/main/python/twitter/thermos/monitoring/process_collector_psutil.py
deleted file mode 100644
index 7b68173..0000000
--- a/src/main/python/twitter/thermos/monitoring/process_collector_psutil.py
+++ /dev/null
@@ -1,92 +0,0 @@
-""" Sample resource consumption statistics for processes using psutil """
-
-from operator import attrgetter
-from time import time
-
-from twitter.common import log
-
-from .process import ProcessSample
-
-from psutil import Process
-from psutil import NoSuchProcess, AccessDenied, Error as PsutilError
-
-
-def process_to_sample(process):
- """ Given a psutil.Process, return a current ProcessSample """
- try:
- # the nonblocking get_cpu_percent call is stateful on a particular Process object, and hence
- # >2 consecutive calls are required before it will return a non-zero value
- rate = process.get_cpu_percent(0.0) / 100.0
- user, system = process.get_cpu_times()
- rss, vms = process.get_memory_info()
- nice = process.nice
- status = process.status
- threads = process.get_num_threads()
- return ProcessSample(rate, user, system, rss, vms, nice, status, threads)
- except (AccessDenied, NoSuchProcess) as e:
- log.warning('Error during process sampling [pid=%s]: %s' % (process.pid, e))
- return ProcessSample.empty()
-
-
-class ProcessTreeCollector(object):
- """ Collect resource consumption statistics for a process and its children """
- def __init__(self, pid):
- """ Given a pid """
- self._pid = pid
- self._process = None # psutil.Process
- self._sampled_tree = {} # pid => ProcessSample
- self._sample = ProcessSample.empty()
- self._stamp = None
- self._rate = 0.0
- self._procs = 1
-
- def sample(self):
- """ Collate and aggregate ProcessSamples for process and children
- Returns None: result is stored in self.value
- """
- try:
- last_sample, last_stamp = self._sample, self._stamp
- if self._process is None:
- self._process = Process(self._pid)
- parent = self._process
- parent_sample = process_to_sample(parent)
- new_samples = dict(
- (proc.pid, process_to_sample(proc))
- for proc in parent.get_children(recursive=True)
- )
- new_samples[self._pid] = parent_sample
-
- except PsutilError as e:
- log.warning('Error during process sampling: %s' % e)
- self._sample = ProcessSample.empty()
- self._rate = 0.0
-
- else:
- last_stamp = self._stamp
- self._stamp = time()
- # for most stats, calculate simple sum to aggregate
- self._sample = sum(new_samples.values(), ProcessSample.empty())
- # cpu consumption is more complicated
- # We require at least 2 generations of a process before we can calculate rate, so for all
- # current processes that were not running in the previous sample, compare to an empty sample
- if self._sampled_tree and last_stamp:
- new = new_samples.values()
- old = [self._sampled_tree.get(pid, ProcessSample.empty()) for pid in new_samples.keys()]
- new_user_sys = sum(map(attrgetter('user'), new)) + sum(map(attrgetter('system'), new))
- old_user_sys = sum(map(attrgetter('user'), old)) + sum(map(attrgetter('system'), old))
- self._rate = (new_user_sys - old_user_sys) / (self._stamp - last_stamp)
- log.debug("Calculated rate for pid=%s and children: %s" % (self._process.pid, self._rate))
- self._sampled_tree = new_samples
-
- @property
- def value(self):
- """ Aggregated ProcessSample representing resource consumption of the tree """
- # Since we don't trust the CPU consumption returned by psutil, replace it with our own in the
- # exported ProcessSample
- return self._sample._replace(rate=self._rate)
-
- @property
- def procs(self):
- """ Number of active processes in the tree """
- return len(self._sampled_tree)
-
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/monitoring/resource.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/monitoring/resource.py b/src/main/python/twitter/thermos/monitoring/resource.py
deleted file mode 100644
index 468f1b1..0000000
--- a/src/main/python/twitter/thermos/monitoring/resource.py
+++ /dev/null
@@ -1,222 +0,0 @@
-"""Monitor the resource consumption of Thermos tasks
-
-This module contains classes used to monitor the resource consumption (e.g. CPU, RAM, disk) of
-Thermos tasks. Resource monitoring of a Thermos task typically treats the task as an aggregate of
-all the processes within it. Importantly, this excludes the process(es) of Thermos itself (i.e. the
-TaskRunner and any other wrappers involved in launching a task).
-
-The ResourceMonitorBase defines the interface for other components (for example, the Thermos
-TaskObserver) to interact with and retrieve information about a Task's resource consumption. The
-canonical/reference implementation of a ResourceMonitor is the TaskResourceMonitor, a thread which
-actively monitors resources for a particular task by periodically polling process information and
-disk consumption and retaining a limited (FIFO) in-memory history of this data.
-
-"""
-
-from abc import abstractmethod
-from bisect import bisect_left
-from collections import namedtuple
-from operator import attrgetter
-import platform
-import threading
-import time
-
-from twitter.common import log
-from twitter.common.collections import RingBuffer
-from twitter.common.concurrent import EventMuxer
-from twitter.common.lang import Interface
-from twitter.common.quantity import Amount, Time
-
-from .disk import DiskCollector
-from .monitor import TaskMonitor
-from .process import ProcessSample
-from .process_collector_psutil import ProcessTreeCollector
-
-
-class ResourceMonitorBase(Interface):
- """ Defines the interface for interacting with a ResourceMonitor """
-
- class Error(Exception): pass
-
- class ResourceResult(namedtuple('ResourceResult', 'num_procs process_sample disk_usage')):
- pass
-
- @abstractmethod
- def sample(self):
- """ Return a sample of the resource consumption of the task right now
-
- Returns a tuple of (timestamp, ResourceResult)
- """
-
- @abstractmethod
- def sample_at(self, time):
- """ Return a sample of the resource consumption as close as possible to the specified time
-
- Returns a tuple of (timestamp, ResourceResult)
- """
-
- @abstractmethod
- def sample_by_process(self, process_name):
- """ Return a sample of the resource consumption of a specific process in the task right now
-
- Returns a ProcessSample
- """
-
-
-class ResourceHistory(object):
- """Simple class to contain a RingBuffer (fixed-length FIFO) history of resource samples, with the
- mapping: timestamp => (number_of_procs, ProcessSample, disk_usage_in_bytes)
- """
- def __init__(self, maxlen, initialize=True):
- if not maxlen >= 1:
- raise ValueError("maxlen must be greater than 0")
- self._maxlen = maxlen
- self._values = RingBuffer(maxlen, None)
- if initialize:
- self.add(time.time(), ResourceMonitorBase.ResourceResult(0, ProcessSample.empty(), 0))
-
- def add(self, timestamp, value):
- """Store a new resource sample corresponding to the given timestamp"""
- if self._values and not timestamp >= self._values[-1][0]:
- raise ValueError("Refusing to add timestamp in the past!")
- self._values.append((timestamp, value))
-
- def get(self, timestamp):
- """Get the resource sample nearest to the given timestamp"""
- closest = min(bisect_left(self._values, (timestamp, None)), len(self) - 1)
- return self._values[closest]
-
- def __iter__(self):
- return iter(self._values)
-
- def __len__(self):
- return len(self._values)
-
- def __repr__(self):
- return 'ResourceHistory(%s)' % ', '.join([str(r) for r in self._values])
-
-
-class TaskResourceMonitor(ResourceMonitorBase, threading.Thread):
- """ Lightweight thread to aggregate resource consumption for a task's constituent processes.
- Actual resource calculation is delegated to collectors; this class periodically polls the
- collectors and aggregates into a representation for the entire task. Also maintains a limited
- history of previous sample results.
- """
-
- MAX_HISTORY = 10000 # magic number
-
- def __init__(self, task_monitor, sandbox,
- process_collector=ProcessTreeCollector, disk_collector=DiskCollector,
- process_collection_interval=Amount(20, Time.SECONDS),
- disk_collection_interval=Amount(1, Time.MINUTES),
- history_time=Amount(1, Time.HOURS)):
- """
- task_monitor: TaskMonitor object specifying the task whose resources should be monitored
- sandbox: Directory for which to monitor disk utilisation
- """
- self._task_monitor = task_monitor # exposes PIDs, sandbox
- self._task_id = task_monitor._task_id
- log.debug('Initialising resource collection for task %s' % self._task_id)
- self._process_collectors = dict() # ProcessStatus => ProcessTreeCollector
- # TODO(jon): sandbox is also available through task_monitor, but typically the first checkpoint
- # isn't written (and hence the header is not available) by the time we initialise here
- self._sandbox = sandbox
- self._process_collector_factory = process_collector
- self._disk_collector = disk_collector(self._sandbox)
- self._process_collection_interval = process_collection_interval.as_(Time.SECONDS)
- self._disk_collection_interval = disk_collection_interval.as_(Time.SECONDS)
- min_collection_interval = min(self._process_collection_interval, self._disk_collection_interval)
- history_length = int(history_time.as_(Time.SECONDS) / min_collection_interval)
- if history_length > self.MAX_HISTORY:
- raise ValueError("Requested history length too large")
- log.debug("Initialising ResourceHistory of length %s" % history_length)
- self._history = ResourceHistory(history_length)
- self._kill_signal = threading.Event()
- threading.Thread.__init__(self)
- self.daemon = True
-
- def sample(self):
- if not self.is_alive():
- log.warning("TaskResourceMonitor not running - sample may be inaccurate")
- return self.sample_at(time.time())
-
- def sample_at(self, timestamp):
- return self._history.get(timestamp)
-
- def sample_by_process(self, process_name):
- try:
- process = [process for process in self._get_active_processes()
- if process.process == process_name].pop()
- except IndexError:
- raise ValueError('No active process found with name "%s" in this task' % process_name)
- else:
- # Since this might be called out of band (before the main loop is aware of the process)
- if process not in self._process_collectors:
- self._process_collectors[process] = self._process_collector_factory(process.pid)
-
- self._process_collectors[process].sample()
- return self._process_collectors[process].value
-
- def _get_active_processes(self):
- """Get a list of ProcessStatus objects representing currently-running processes in the task"""
- return [process for process, _ in self._task_monitor.get_active_processes()]
-
- def kill(self):
- """Signal that the thread should cease collecting resources and terminate"""
- self._kill_signal.set()
-
- def run(self):
- """Thread entrypoint. Loop indefinitely, polling collectors at self._collection_interval and
- collating samples."""
-
- log.debug('Commencing resource monitoring for task "%s"' % self._task_id)
- next_process_collection = 0
- next_disk_collection = 0
-
- while not self._kill_signal.is_set():
-
- now = time.time()
-
- if now > next_process_collection:
- next_process_collection = now + self._process_collection_interval
- actives = set(self._get_active_processes())
- current = set(self._process_collectors)
- for process in current - actives:
- log.debug('Process "%s" (pid %s) no longer active, removing from monitored processes' %
- (process.process, process.pid))
- self._process_collectors.pop(process)
- for process in actives - current:
- log.debug('Adding process "%s" (pid %s) to resource monitoring' %
- (process.process, process.pid))
- self._process_collectors[process] = self._process_collector_factory(process.pid)
- for process, collector in self._process_collectors.iteritems():
- log.debug('Collecting sample for process "%s" (pid %s) and children' %
- (process.process, process.pid))
- collector.sample()
-
- if now > next_disk_collection:
- next_disk_collection = now + self._disk_collection_interval
- log.debug('Collecting disk sample for %s' % self._sandbox)
- self._disk_collector.sample()
-
- try:
- aggregated_procs = sum(map(attrgetter('procs'), self._process_collectors.values()))
- aggregated_sample = sum(map(attrgetter('value'), self._process_collectors.values()),
- ProcessSample.empty())
- self._history.add(now, self.ResourceResult(aggregated_procs, aggregated_sample,
- self._disk_collector.value))
- log.debug("Recorded resource sample at %s" % now)
- except ValueError as err:
- log.warning("Error recording resource sample: %s" % err)
-
- # Sleep until any of the following conditions are met:
- # - it's time for the next disk collection
- # - it's time for the next process collection
- # - the result from the last disk collection is available via the DiskCollector
- # - the TaskResourceMonitor has been killed via self._kill_signal
- now = time.time()
- next_collection = min(next_process_collection - now, next_disk_collection - now)
- EventMuxer(self._kill_signal, self._disk_collector.completed_event
- ).wait(timeout=max(0, next_collection))
-
- log.debug('Stopping resource monitoring for task "%s"' % self._task_id)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/BUILD b/src/main/python/twitter/thermos/observer/BUILD
deleted file mode 100644
index 00ea9cb..0000000
--- a/src/main/python/twitter/thermos/observer/BUILD
+++ /dev/null
@@ -1,51 +0,0 @@
-import os
-
-python_library(
- name = 'observed_task',
- sources = ['observed_task.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/thermos:pystachio'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/python/twitter/thermos/config'),
- ]
-)
-
-python_library(
- name = 'task_observer',
- sources = ['task_observer.py'],
- dependencies = [
- pants(':observed_task'),
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/python/twitter/thermos/monitoring:detector'),
- pants('src/main/python/twitter/thermos/monitoring:monitor'),
- pants('src/main/python/twitter/thermos/monitoring:process'),
- pants('src/main/python/twitter/thermos/monitoring:resource'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'observer',
- dependencies = [
- pants(':task_observer'),
- pants('src/main/python/twitter/thermos/observer/http:http_observer'),
-
- # covering libraries
- pants('src/main/python/twitter/thermos/common'),
- pants('src/main/python/twitter/thermos/config'),
- pants('src/main/python/twitter/thermos/monitoring'),
- ],
- provides = setup_py(
- name = 'twitter.thermos.observer',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- description = 'The Thermos observer web interface.',
- ).with_binaries(
- thermos_observer = pants('src/main/python/twitter/thermos/observer/bin:thermos_observer'),
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/__init__.py b/src/main/python/twitter/thermos/observer/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/bin/BUILD b/src/main/python/twitter/thermos/observer/bin/BUILD
deleted file mode 100644
index 26eb148..0000000
--- a/src/main/python/twitter/thermos/observer/bin/BUILD
+++ /dev/null
@@ -1,14 +0,0 @@
-python_binary(
- name = 'thermos_observer',
- source = 'thermos_observer.py',
- entry_point = 'twitter.thermos.observer.bin.thermos_observer:proxy_main',
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('aurora/twitterdeps/src/python/twitter/common/http'),
- pants('src/main/python/twitter/thermos:cherrypy'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/python/twitter/thermos/observer/http:http_observer'),
- pants('src/main/python/twitter/thermos/observer:task_observer'),
- ],
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/bin/__init__.py b/src/main/python/twitter/thermos/observer/bin/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/bin/thermos_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/bin/thermos_observer.py b/src/main/python/twitter/thermos/observer/bin/thermos_observer.py
deleted file mode 100644
index 9e7f7a0..0000000
--- a/src/main/python/twitter/thermos/observer/bin/thermos_observer.py
+++ /dev/null
@@ -1,55 +0,0 @@
-from __future__ import print_function
-
-import socket
-import sys
-import time
-
-from twitter.common import app
-from twitter.common.exceptions import ExceptionalThread
-from twitter.common.http import HttpServer
-from twitter.common.http.diagnostics import DiagnosticsEndpoints
-from twitter.thermos.common.path import TaskPath
-from twitter.thermos.observer.task_observer import TaskObserver
-from twitter.thermos.observer.http.http_observer import BottleObserver
-
-
-app.add_option("--root",
- dest="root",
- metavar="DIR",
- default=TaskPath.DEFAULT_CHECKPOINT_ROOT,
- help="root checkpoint directory for thermos task runners")
-
-
-app.add_option("--port",
- dest="port",
- metavar="INT",
- default=1338,
- help="port number to listen on.")
-
-
-def proxy_main():
- def main(args, opts):
- if args:
- print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr)
- app.help()
- sys.exit(1)
-
- root_server = HttpServer()
- root_server.mount_routes(DiagnosticsEndpoints())
-
- task_observer = TaskObserver(opts.root)
- task_observer.start()
-
- bottle_wrapper = BottleObserver(task_observer)
-
- root_server.mount_routes(bottle_wrapper)
-
- def run():
- root_server.run('0.0.0.0', opts.port, 'cherrypy')
-
- et = ExceptionalThread(target=run)
- et.daemon = True
- et.start()
- et.join()
-
- app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/BUILD b/src/main/python/twitter/thermos/observer/http/BUILD
deleted file mode 100644
index 9f3d587..0000000
--- a/src/main/python/twitter/thermos/observer/http/BUILD
+++ /dev/null
@@ -1,47 +0,0 @@
-python_library(
- name = 'json',
- sources = ['json.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/http'),
- ]
-)
-
-python_library(
- name = 'static_assets',
- sources = ['static_assets.py'],
- resources = rglobs('assets/*'),
- dependencies = [
- pants('src/main/python/twitter/thermos:bottle'),
- ]
-)
-
-python_library(
- name = 'templating',
- sources = ['templating.py'],
- resources = globs('templates/*.tpl'),
-)
-
-python_library(
- name = 'file_browser',
- sources = ['file_browser.py'],
- dependencies = [
- pants(':templating'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/http'),
- pants('src/main/python/twitter/thermos:bottle'),
- pants('src/main/python/twitter/thermos:mako'),
- ]
-)
-
-python_library(
- name = 'http_observer',
- sources = ['__init__.py', 'http_observer.py'],
- dependencies = [
- pants(':file_browser'),
- pants(':json'),
- pants(':static_assets'),
- pants(':templating'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/http'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/__init__.py b/src/main/python/twitter/thermos/observer/http/__init__.py
deleted file mode 100644
index e69de29..0000000
[20/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/assets/bootstrap.css
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/assets/bootstrap.css b/src/main/python/apache/thermos/observer/http/assets/bootstrap.css
new file mode 100644
index 0000000..1c75d0c
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/assets/bootstrap.css
@@ -0,0 +1,9 @@
+/*!
+ * Bootstrap v2.0.3
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-app
earance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:28px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box}body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;line-height:18px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover{color:#005580;text-decoration:underline}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;content:""}.row:after{clear:both}[class*="span"]{float:left;margin-left:20px}.container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.span
12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offset1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:28px;margin-left:2.127659574%;*margin-left:2.0744680846382977%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:99.9999
9998999999%;*width:99.94680850063828%}.row-fluid .span11{width:91.489361693%;*width:91.4361702036383%}.row-fluid .span10{width:82.97872339599999%;*width:82.92553190663828%}.row-fluid .span9{width:74.468085099%;*width:74.4148936096383%}.row-fluid .span8{width:65.95744680199999%;*width:65.90425531263828%}.row-fluid .span7{width:57.446808505%;*width:57.3936170156383%}.row-fluid .span6{width:48.93617020799999%;*width:48.88297871863829%}.row-fluid .span5{width:40.425531911%;*width:40.3723404216383%}.row-fluid .span4{width:31.914893614%;*width:31.8617021246383%}.row-fluid .span3{width:23.404255317%;*width:23.3510638276383%}.row-fluid .span2{width:14.89361702%;*width:14.8404255306383%}.row-fluid .span1{width:6.382978723%;*width:6.329787233638298%}.container{margin-right:auto;margin-left:auto;*zoom:1}.container:before,.container:after{display:table;content:""}.container:after{clear:both}.container-fluid{padding-right:20px;padding-left:20px;*zoom:1}.container-fluid:before,.container-fluid:af
ter{display:table;content:""}.container-fluid:after{clear:both}p{margin:0 0 9px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;line-height:18px}p small{font-size:11px;color:#999}.lead{margin-bottom:18px;font-size:20px;font-weight:200;line-height:27px}h1,h2,h3,h4,h5,h6{margin:0;font-family:inherit;font-weight:bold;color:inherit;text-rendering:optimizelegibility}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small{font-weight:normal;color:#999}h1{font-size:30px;line-height:36px}h1 small{font-size:18px}h2{font-size:24px;line-height:36px}h2 small{font-size:18px}h3{font-size:18px;line-height:27px}h3 small{font-size:14px}h4,h5,h6{line-height:18px}h4{font-size:14px}h4 small{font-size:12px}h5{font-size:12px}h6{font-size:11px;color:#999;text-transform:uppercase}.page-header{padding-bottom:17px;margin:18px 0;border-bottom:1px solid #eee}.page-header h1{line-height:1}ul,ol{padding:0;margin:0 0 9px 25px}ul ul,ul ol,ol ol,ol ul{margin-bottom:0}ul{list-style:disc}ol{list-
style:decimal}li{line-height:18px}ul.unstyled,ol.unstyled{margin-left:0;list-style:none}dl{margin-bottom:18px}dt,dd{line-height:18px}dt{font-weight:bold;line-height:17px}dd{margin-left:9px}.dl-horizontal dt{float:left;width:120px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:130px}hr{margin:18px 0;border:0;border-top:1px solid #eee;border-bottom:1px solid #fff}strong{font-weight:bold}em{font-style:italic}.muted{color:#999}abbr[title]{cursor:help;border-bottom:1px dotted #ddd}abbr.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:0 0 0 15px;margin:0 0 18px;border-left:5px solid #eee}blockquote p{margin-bottom:0;font-size:16px;font-weight:300;line-height:22.5px}blockquote small{display:block;line-height:18px;color:#999}blockquote small:before{content:'\2014 \00A0'}blockquote.pull-right{float:right;padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0}blockquote.pull-right p,b
lockquote.pull-right small{text-align:right}q:before,q:after,blockquote:before,blockquote:after{content:""}address{display:block;margin-bottom:18px;font-style:normal;line-height:18px}small{font-size:100%}cite{font-style:normal}code,pre{padding:0 3px 2px;font-family:Menlo,Monaco,Consolas,"Courier New",monospace;font-size:12px;color:#333;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}code{padding:2px 4px;color:#d14;background-color:#f7f7f9;border:1px solid #e1e1e8}pre{display:block;padding:8.5px;margin:0 0 9px;font-size:12.025px;line-height:18px;word-break:break-all;word-wrap:break-word;white-space:pre;white-space:pre-wrap;background-color:#f5f5f5;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}pre.prettyprint{margin-bottom:18px}pre code{padding:0;color:inherit;background-color:transparent;border:0}.pre-scrollable{max-height:340px;overflow-y:scroll}form{margin:0 0 18px}fieldset{padding:0;mar
gin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:27px;font-size:19.5px;line-height:36px;color:#333;border:0;border-bottom:1px solid #eee}legend small{font-size:13.5px;color:#999}label,input,button,select,textarea{font-size:13px;font-weight:normal;line-height:18px}input,button,select,textarea{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}label{display:block;margin-bottom:5px;color:#333}input,textarea,select,.uneditable-input{display:inline-block;width:210px;height:18px;padding:4px;margin-bottom:9px;font-size:13px;line-height:18px;color:#555;background-color:#fff;border:1px solid #ccc;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.uneditable-textarea{width:auto;height:auto}label input,label textarea,label select{display:block}input[type="image"],input[type="checkbox"],input[type="radio"]{width:auto;height:auto;padding:0;margin:3px 0;*margin-top:0;line-height:normal;cursor:pointer;background-color:transparent;border:0 \9;-webkit-bord
er-radius:0;-moz-border-radius:0;border-radius:0}input[type="image"]{border:0}input[type="file"]{width:auto;padding:initial;line-height:initial;background-color:#fff;background-color:initial;border:initial;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}input[type="button"],input[type="reset"],input[type="submit"]{width:auto;height:auto}select,input[type="file"]{height:28px;*margin-top:4px;line-height:28px}input[type="file"]{line-height:18px \9}select{width:220px;background-color:#fff}select[multiple],select[size]{height:auto}input[type="image"]{-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}textarea{height:auto}input[type="hidden"]{display:none}.radio,.checkbox{min-height:18px;padding-left:18px}.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-18px}.controls>.radio:first-child,.controls>.checkbox:first-child{padding-top:5px}.radio.inline,.checkbox.inline{display:inline-block;padding-top:5px;margin-bottom:0;vertical-align:
middle}.radio.inline+.radio.inline,.checkbox.inline+.checkbox.inline{margin-left:10px}input,textarea{-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border linear .2s,box-shadow linear .2s;-moz-transition:border linear .2s,box-shadow linear .2s;-ms-transition:border linear .2s,box-shadow linear .2s;-o-transition:border linear .2s,box-shadow linear .2s;transition:border linear .2s,box-shadow linear .2s}input:focus,textarea:focus{border-color:rgba(82,168,236,0.8);outline:0;outline:thin dotted \9;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6)}input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus,select:focus{outline:thin dotted #333;outline:5px auto -webkit-f
ocus-ring-color;outline-offset:-2px;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.input-mini{width:60px}.input-small{width:90px}.input-medium{width:150px}.input-large{width:210px}.input-xlarge{width:270px}.input-xxlarge{width:530px}input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"]{float:none;margin-left:0}input,textarea,.uneditable-input{margin-left:0}input.span12,textarea.span12,.uneditable-input.span12{width:930px}input.span11,textarea.span11,.uneditable-input.span11{width:850px}input.span10,textarea.span10,.uneditable-input.span10{width:770px}input.span9,textarea.span9,.uneditable-input.span9{width:690px}input.span8,textarea.span8,.uneditable-input.span8{width:610px}input.span7,textarea.span7,.uneditable-input.span7{width:530px}input.span6,textarea.span6,.uneditable-input.s
pan6{width:450px}input.span5,textarea.span5,.uneditable-input.span5{width:370px}input.span4,textarea.span4,.uneditable-input.span4{width:290px}input.span3,textarea.span3,.uneditable-input.span3{width:210px}input.span2,textarea.span2,.uneditable-input.span2{width:130px}input.span1,textarea.span1,.uneditable-input.span1{width:50px}input[disabled],select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#eee;border-color:#ddd}input[type="radio"][disabled],input[type="checkbox"][disabled],input[type="radio"][readonly],input[type="checkbox"][readonly]{background-color:transparent}.control-group.warning>label,.control-group.warning .help-block,.control-group.warning .help-inline{color:#c09853}.control-group.warning input,.control-group.warning select,.control-group.warning textarea{color:#c09853;border-color:#c09853}.control-group.warning input:focus,.control-group.warning select:focus,.control-group.warning textarea:focus
{border-color:#a47e3c;-webkit-box-shadow:0 0 6px #dbc59e;-moz-box-shadow:0 0 6px #dbc59e;box-shadow:0 0 6px #dbc59e}.control-group.warning .input-prepend .add-on,.control-group.warning .input-append .add-on{color:#c09853;background-color:#fcf8e3;border-color:#c09853}.control-group.error>label,.control-group.error .help-block,.control-group.error .help-inline{color:#b94a48}.control-group.error input,.control-group.error select,.control-group.error textarea{color:#b94a48;border-color:#b94a48}.control-group.error input:focus,.control-group.error select:focus,.control-group.error textarea:focus{border-color:#953b39;-webkit-box-shadow:0 0 6px #d59392;-moz-box-shadow:0 0 6px #d59392;box-shadow:0 0 6px #d59392}.control-group.error .input-prepend .add-on,.control-group.error .input-append .add-on{color:#b94a48;background-color:#f2dede;border-color:#b94a48}.control-group.success>label,.control-group.success .help-block,.control-group.success .help-inline{color:#468847}.control-group.success
input,.control-group.success select,.control-group.success textarea{color:#468847;border-color:#468847}.control-group.success input:focus,.control-group.success select:focus,.control-group.success textarea:focus{border-color:#356635;-webkit-box-shadow:0 0 6px #7aba7b;-moz-box-shadow:0 0 6px #7aba7b;box-shadow:0 0 6px #7aba7b}.control-group.success .input-prepend .add-on,.control-group.success .input-append .add-on{color:#468847;background-color:#dff0d8;border-color:#468847}input:focus:required:invalid,textarea:focus:required:invalid,select:focus:required:invalid{color:#b94a48;border-color:#ee5f5b}input:focus:required:invalid:focus,textarea:focus:required:invalid:focus,select:focus:required:invalid:focus{border-color:#e9322d;-webkit-box-shadow:0 0 6px #f8b9b7;-moz-box-shadow:0 0 6px #f8b9b7;box-shadow:0 0 6px #f8b9b7}.form-actions{padding:17px 20px 18px;margin-top:18px;margin-bottom:18px;background-color:#f5f5f5;border-top:1px solid #ddd;*zoom:1}.form-actions:before,.form-actions:aft
er{display:table;content:""}.form-actions:after{clear:both}.uneditable-input{overflow:hidden;white-space:nowrap;cursor:not-allowed;background-color:#fff;border-color:#eee;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);box-shadow:inset 0 1px 2px rgba(0,0,0,0.025)}:-moz-placeholder{color:#999}::-webkit-input-placeholder{color:#999}.help-block,.help-inline{color:#555}.help-block{display:block;margin-bottom:9px}.help-inline{display:inline-block;*display:inline;padding-left:5px;vertical-align:middle;*zoom:1}.input-prepend,.input-append{margin-bottom:5px}.input-prepend input,.input-append input,.input-prepend select,.input-append select,.input-prepend .uneditable-input,.input-append .uneditable-input{position:relative;margin-bottom:0;*margin-left:0;vertical-align:middle;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend input:focus,.input-append input:focus,.input-prepend select:
focus,.input-append select:focus,.input-prepend .uneditable-input:focus,.input-append .uneditable-input:focus{z-index:2}.input-prepend .uneditable-input,.input-append .uneditable-input{border-left-color:#ccc}.input-prepend .add-on,.input-append .add-on{display:inline-block;width:auto;height:18px;min-width:16px;padding:4px 5px;font-weight:normal;line-height:18px;text-align:center;text-shadow:0 1px 0 #fff;vertical-align:middle;background-color:#eee;border:1px solid #ccc}.input-prepend .add-on,.input-append .add-on,.input-prepend .btn,.input-append .btn{margin-left:-1px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend .active,.input-append .active{background-color:#a9dba9;border-color:#46a546}.input-prepend .add-on,.input-prepend .btn{margin-right:-1px}.input-prepend .add-on:first-child,.input-prepend .btn:first-child{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append input,.input-append select,.input-appe
nd .uneditable-input{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append .uneditable-input{border-right-color:#ccc;border-left-color:#eee}.input-append .add-on:last-child,.input-append .btn:last-child{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend.input-append input,.input-prepend.input-append select,.input-prepend.input-append .uneditable-input{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend.input-append .add-on:first-child,.input-prepend.input-append .btn:first-child{margin-right:-1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-prepend.input-append .add-on:last-child,.input-prepend.input-append .btn:last-child{margin-left:-1px;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.search-query{padding-right:14px;padding-right:4px \9;padding-left:14px;pad
ding-left:4px \9;margin-bottom:0;-webkit-border-radius:14px;-moz-border-radius:14px;border-radius:14px}.form-search input,.form-inline input,.form-horizontal input,.form-search textarea,.form-inline textarea,.form-horizontal textarea,.form-search select,.form-inline select,.form-horizontal select,.form-search .help-inline,.form-inline .help-inline,.form-horizontal .help-inline,.form-search .uneditable-input,.form-inline .uneditable-input,.form-horizontal .uneditable-input,.form-search .input-prepend,.form-inline .input-prepend,.form-horizontal .input-prepend,.form-search .input-append,.form-inline .input-append,.form-horizontal .input-append{display:inline-block;*display:inline;margin-bottom:0;*zoom:1}.form-search .hide,.form-inline .hide,.form-horizontal .hide{display:none}.form-search label,.form-inline label{display:inline-block}.form-search .input-append,.form-inline .input-append,.form-search .input-prepend,.form-inline .input-prepend{margin-bottom:0}.form-search .radio,.form-s
earch .checkbox,.form-inline .radio,.form-inline .checkbox{padding-left:0;margin-bottom:0;vertical-align:middle}.form-search .radio input[type="radio"],.form-search .checkbox input[type="checkbox"],.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:left;margin-right:3px;margin-left:0}.control-group{margin-bottom:9px}legend+.control-group{margin-top:18px;-webkit-margin-top-collapse:separate}.form-horizontal .control-group{margin-bottom:18px;*zoom:1}.form-horizontal .control-group:before,.form-horizontal .control-group:after{display:table;content:""}.form-horizontal .control-group:after{clear:both}.form-horizontal .control-label{float:left;width:140px;padding-top:5px;text-align:right}.form-horizontal .controls{*display:inline-block;*padding-left:20px;margin-left:160px;*margin-left:0}.form-horizontal .controls:first-child{*padding-left:160px}.form-horizontal .help-block{margin-top:9px;margin-bottom:0}.form-horizontal .form-actions{padding-left:1
60px}table{max-width:100%;background-color:transparent;border-collapse:collapse;border-spacing:0}.table{width:100%;margin-bottom:18px}.table th,.table td{padding:8px;line-height:18px;text-align:left;vertical-align:top;border-top:1px solid #ddd}.table th{font-weight:bold}.table thead th{vertical-align:bottom}.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0}.table tbody+tbody{border-top:2px solid #ddd}.table-condensed th,.table-condensed td{padding:4px 5px}.table-bordered{border:1px solid #ddd;border-collapse:separate;*border-collapse:collapsed;border-left:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-bordered th,.table-bordered td{border-left:1px solid #ddd}.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child t
h,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child th:first-child,.table-bordered tbody:first-child tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered thead:first-child tr:first-child th:last-child,.table-bordered tbody:first-child tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topright:4px}.table-bordered thead:last-child tr:last-child th:first-child,.table-bordered tbody:last-child tr:last-child td:first-child{-webkit-border-radius:0 0 0 4px;-moz-border-radius:0 0 0 4px;bor
der-radius:0 0 0 4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px}.table-bordered thead:last-child tr:last-child th:last-child,.table-bordered tbody:last-child tr:last-child td:last-child{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#f9f9f9}.table tbody tr:hover td,.table tbody tr:hover th{background-color:#f5f5f5}table .span1{float:none;width:44px;margin-left:0}table .span2{float:none;width:124px;margin-left:0}table .span3{float:none;width:204px;margin-left:0}table .span4{float:none;width:284px;margin-left:0}table .span5{float:none;width:364px;margin-left:0}table .span6{float:none;width:444px;margin-left:0}table .span7{float:none;width:524px;margin-left:0}table .span8{float:none;width:604px;margin-left:0}table .span9{float:none;width:684px;margin-left:0}table .span10{
float:none;width:764px;margin-left:0}table .span11{float:none;width:844px;margin-left:0}table .span12{float:none;width:924px;margin-left:0}table .span13{float:none;width:1004px;margin-left:0}table .span14{float:none;width:1084px;margin-left:0}table .span15{float:none;width:1164px;margin-left:0}table .span16{float:none;width:1244px;margin-left:0}table .span17{float:none;width:1324px;margin-left:0}table .span18{float:none;width:1404px;margin-left:0}table .span19{float:none;width:1484px;margin-left:0}table .span20{float:none;width:1564px;margin-left:0}table .span21{float:none;width:1644px;margin-left:0}table .span22{float:none;width:1724px;margin-left:0}table .span23{float:none;width:1804px;margin-left:0}table .span24{float:none;width:1884px;margin-left:0}[class^="icon-"],[class*=" icon-"]{display:inline-block;width:14px;height:14px;*margin-right:.3em;line-height:14px;vertical-align:text-top;background-image:url("../img/glyphicons-halflings.png");background-position:14px 14px;backgroun
d-repeat:no-repeat}[class^="icon-"]:last-child,[class*=" icon-"]:last-child{*margin-left:0}.icon-white{background-image:url("../img/glyphicons-halflings-white.png")}.icon-glass{background-position:0 0}.icon-music{background-position:-24px 0}.icon-search{background-position:-48px 0}.icon-envelope{background-position:-72px 0}.icon-heart{background-position:-96px 0}.icon-star{background-position:-120px 0}.icon-star-empty{background-position:-144px 0}.icon-user{background-position:-168px 0}.icon-film{background-position:-192px 0}.icon-th-large{background-position:-216px 0}.icon-th{background-position:-240px 0}.icon-th-list{background-position:-264px 0}.icon-ok{background-position:-288px 0}.icon-remove{background-position:-312px 0}.icon-zoom-in{background-position:-336px 0}.icon-zoom-out{background-position:-360px 0}.icon-off{background-position:-384px 0}.icon-signal{background-position:-408px 0}.icon-cog{background-position:-432px 0}.icon-trash{background-position:-456px 0}.icon-home{ba
ckground-position:0 -24px}.icon-file{background-position:-24px -24px}.icon-time{background-position:-48px -24px}.icon-road{background-position:-72px -24px}.icon-download-alt{background-position:-96px -24px}.icon-download{background-position:-120px -24px}.icon-upload{background-position:-144px -24px}.icon-inbox{background-position:-168px -24px}.icon-play-circle{background-position:-192px -24px}.icon-repeat{background-position:-216px -24px}.icon-refresh{background-position:-240px -24px}.icon-list-alt{background-position:-264px -24px}.icon-lock{background-position:-287px -24px}.icon-flag{background-position:-312px -24px}.icon-headphones{background-position:-336px -24px}.icon-volume-off{background-position:-360px -24px}.icon-volume-down{background-position:-384px -24px}.icon-volume-up{background-position:-408px -24px}.icon-qrcode{background-position:-432px -24px}.icon-barcode{background-position:-456px -24px}.icon-tag{background-position:0 -48px}.icon-tags{background-position:-25px -48p
x}.icon-book{background-position:-48px -48px}.icon-bookmark{background-position:-72px -48px}.icon-print{background-position:-96px -48px}.icon-camera{background-position:-120px -48px}.icon-font{background-position:-144px -48px}.icon-bold{background-position:-167px -48px}.icon-italic{background-position:-192px -48px}.icon-text-height{background-position:-216px -48px}.icon-text-width{background-position:-240px -48px}.icon-align-left{background-position:-264px -48px}.icon-align-center{background-position:-288px -48px}.icon-align-right{background-position:-312px -48px}.icon-align-justify{background-position:-336px -48px}.icon-list{background-position:-360px -48px}.icon-indent-left{background-position:-384px -48px}.icon-indent-right{background-position:-408px -48px}.icon-facetime-video{background-position:-432px -48px}.icon-picture{background-position:-456px -48px}.icon-pencil{background-position:0 -72px}.icon-map-marker{background-position:-24px -72px}.icon-adjust{background-position:-48
px -72px}.icon-tint{background-position:-72px -72px}.icon-edit{background-position:-96px -72px}.icon-share{background-position:-120px -72px}.icon-check{background-position:-144px -72px}.icon-move{background-position:-168px -72px}.icon-step-backward{background-position:-192px -72px}.icon-fast-backward{background-position:-216px -72px}.icon-backward{background-position:-240px -72px}.icon-play{background-position:-264px -72px}.icon-pause{background-position:-288px -72px}.icon-stop{background-position:-312px -72px}.icon-forward{background-position:-336px -72px}.icon-fast-forward{background-position:-360px -72px}.icon-step-forward{background-position:-384px -72px}.icon-eject{background-position:-408px -72px}.icon-chevron-left{background-position:-432px -72px}.icon-chevron-right{background-position:-456px -72px}.icon-plus-sign{background-position:0 -96px}.icon-minus-sign{background-position:-24px -96px}.icon-remove-sign{background-position:-48px -96px}.icon-ok-sign{background-position:-72
px -96px}.icon-question-sign{background-position:-96px -96px}.icon-info-sign{background-position:-120px -96px}.icon-screenshot{background-position:-144px -96px}.icon-remove-circle{background-position:-168px -96px}.icon-ok-circle{background-position:-192px -96px}.icon-ban-circle{background-position:-216px -96px}.icon-arrow-left{background-position:-240px -96px}.icon-arrow-right{background-position:-264px -96px}.icon-arrow-up{background-position:-289px -96px}.icon-arrow-down{background-position:-312px -96px}.icon-share-alt{background-position:-336px -96px}.icon-resize-full{background-position:-360px -96px}.icon-resize-small{background-position:-384px -96px}.icon-plus{background-position:-408px -96px}.icon-minus{background-position:-433px -96px}.icon-asterisk{background-position:-456px -96px}.icon-exclamation-sign{background-position:0 -120px}.icon-gift{background-position:-24px -120px}.icon-leaf{background-position:-48px -120px}.icon-fire{background-position:-72px -120px}.icon-eye-ope
n{background-position:-96px -120px}.icon-eye-close{background-position:-120px -120px}.icon-warning-sign{background-position:-144px -120px}.icon-plane{background-position:-168px -120px}.icon-calendar{background-position:-192px -120px}.icon-random{background-position:-216px -120px}.icon-comment{background-position:-240px -120px}.icon-magnet{background-position:-264px -120px}.icon-chevron-up{background-position:-288px -120px}.icon-chevron-down{background-position:-313px -119px}.icon-retweet{background-position:-336px -120px}.icon-shopping-cart{background-position:-360px -120px}.icon-folder-close{background-position:-384px -120px}.icon-folder-open{background-position:-408px -120px}.icon-resize-vertical{background-position:-432px -119px}.icon-resize-horizontal{background-position:-456px -118px}.icon-hdd{background-position:0 -144px}.icon-bullhorn{background-position:-24px -144px}.icon-bell{background-position:-48px -144px}.icon-certificate{background-position:-72px -144px}.icon-thumbs-up
{background-position:-96px -144px}.icon-thumbs-down{background-position:-120px -144px}.icon-hand-right{background-position:-144px -144px}.icon-hand-left{background-position:-168px -144px}.icon-hand-up{background-position:-192px -144px}.icon-hand-down{background-position:-216px -144px}.icon-circle-arrow-right{background-position:-240px -144px}.icon-circle-arrow-left{background-position:-264px -144px}.icon-circle-arrow-up{background-position:-288px -144px}.icon-circle-arrow-down{background-position:-312px -144px}.icon-globe{background-position:-336px -144px}.icon-wrench{background-position:-360px -144px}.icon-tasks{background-position:-384px -144px}.icon-filter{background-position:-408px -144px}.icon-briefcase{background-position:-432px -144px}.icon-fullscreen{background-position:-456px -144px}.dropup,.dropdown{position:relative}.dropdown-toggle{*margin-bottom:-3px}.dropdown-toggle:active,.open .dropdown-toggle{outline:0}.caret{display:inline-block;width:0;height:0;vertical-align:top;
border-top:4px solid #000;border-right:4px solid transparent;border-left:4px solid transparent;content:"";opacity:.3;filter:alpha(opacity=30)}.dropdown .caret{margin-top:8px;margin-left:2px}.dropdown:hover .caret,.open .caret{opacity:1;filter:alpha(opacity=100)}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:4px 0;margin:1px 0 0;list-style:none;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{*width:100%;height:1px;margin:8px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bo
ttom:1px solid #fff}.dropdown-menu a{display:block;padding:3px 15px;clear:both;font-weight:normal;line-height:18px;color:#333;white-space:nowrap}.dropdown-menu li>a:hover,.dropdown-menu .active>a,.dropdown-menu .active>a:hover{color:#fff;text-decoration:none;background-color:#08c}.open{*z-index:1000}.open .dropdown-menu{display:block}.pull-right .dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid #000;content:"\2191"}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px}.typeahead{margin-top:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #eee;border:1px solid rgba(0,0,0,0.05);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);b
ox-shadow:inset 0 1px 1px rgba(0,0,0,0.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,0.15)}.well-large{padding:24px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.well-small{padding:9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.fade{opacity:0;filter:alpha(opacity=0);-webkit-transition:opacity .15s linear;-moz-transition:opacity .15s linear;-ms-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1;filter:alpha(opacity=100)}.collapse{position:relative;height:0;overflow:hidden;-webkit-transition:height .35s ease;-moz-transition:height .35s ease;-ms-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.collapse.in{height:auto}.close{float:right;font-size:20px;font-weight:bold;line-height:18px;color:#000;text-shadow:0 1px 0 #fff;opacity:.2;filter:alpha(opacity=20)}.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:
.4;filter:alpha(opacity=40)}button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none}.btn{display:inline-block;*display:inline;padding:4px 10px 4px;margin-bottom:0;*margin-left:.3em;font-size:13px;line-height:18px;*line-height:20px;color:#333;text-align:center;text-shadow:0 1px 1px rgba(255,255,255,0.75);vertical-align:middle;cursor:pointer;background-color:#f5f5f5;*background-color:#e6e6e6;background-image:-ms-linear-gradient(top,#fff,#e6e6e6);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));background-image:-webkit-linear-gradient(top,#fff,#e6e6e6);background-image:-o-linear-gradient(top,#fff,#e6e6e6);background-image:linear-gradient(top,#fff,#e6e6e6);background-image:-moz-linear-gradient(top,#fff,#e6e6e6);background-repeat:repeat-x;border:1px solid #ccc;*border:0;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);border-color:#e6e6e6 #e6e6e6 #bfbfbf;border-bottom-color:#b3b3b3;-webkit-border-radius:4px;-mo
z-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffff',endColorstr='#e6e6e6',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);*zoom:1;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn:hover,.btn:active,.btn.active,.btn.disabled,.btn[disabled]{background-color:#e6e6e6;*background-color:#d9d9d9}.btn:active,.btn.active{background-color:#ccc \9}.btn:first-child{*margin-left:0}.btn:hover{color:#333;text-decoration:none;background-color:#e6e6e6;*background-color:#d9d9d9;background-position:0 -15px;-webkit-transition:background-position .1s linear;-moz-transition:background-position .1s linear;-ms-transition:background-position .1s linear;-o-transition:background-position .1s linear;transition:background-position .1
s linear}.btn:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.active,.btn:active{background-color:#e6e6e6;background-color:#d9d9d9 \9;background-image:none;outline:0;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn.disabled,.btn[disabled]{cursor:default;background-color:#e6e6e6;background-image:none;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-large{padding:9px 14px;font-size:15px;line-height:normal;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.btn-large [class^="icon-"]{margin-top:1px}.btn-small{padding:5px 9px;font-size:11px;line-height:16px}.btn-small [class^="icon-"]{margin-top:-1px}.btn-mini{padding:2px 6px;font-size:11px;line-height:14px}.btn-primary,.btn-primary:hover,.
btn-warning,.btn-warning:hover,.btn-danger,.btn-danger:hover,.btn-success,.btn-success:hover,.btn-info,.btn-info:hover,.btn-inverse,.btn-inverse:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.btn-primary.active,.btn-warning.active,.btn-danger.active,.btn-success.active,.btn-info.active,.btn-inverse.active{color:rgba(255,255,255,0.75)}.btn{border-color:#ccc;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25)}.btn-primary{background-color:#0074cc;*background-color:#05c;background-image:-ms-linear-gradient(top,#08c,#05c);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#05c));background-image:-webkit-linear-gradient(top,#08c,#05c);background-image:-o-linear-gradient(top,#08c,#05c);background-image:-moz-linear-gradient(top,#08c,#05c);background-image:linear-gradient(top,#08c,#05c);background-repeat:repeat-x;border-color:#05c #05c #003580;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(sta
rtColorstr='#0088cc',endColorstr='#0055cc',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-primary:hover,.btn-primary:active,.btn-primary.active,.btn-primary.disabled,.btn-primary[disabled]{background-color:#05c;*background-color:#004ab3}.btn-primary:active,.btn-primary.active{background-color:#004099 \9}.btn-warning{background-color:#faa732;*background-color:#f89406;background-image:-ms-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;border-color:#f89406 #f89406 #ad6704;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450',endColorstr='#f89406
',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-warning:hover,.btn-warning:active,.btn-warning.active,.btn-warning.disabled,.btn-warning[disabled]{background-color:#f89406;*background-color:#df8505}.btn-warning:active,.btn-warning.active{background-color:#c67605 \9}.btn-danger{background-color:#da4f49;*background-color:#bd362f;background-image:-ms-linear-gradient(top,#ee5f5b,#bd362f);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#bd362f));background-image:-webkit-linear-gradient(top,#ee5f5b,#bd362f);background-image:-o-linear-gradient(top,#ee5f5b,#bd362f);background-image:-moz-linear-gradient(top,#ee5f5b,#bd362f);background-image:linear-gradient(top,#ee5f5b,#bd362f);background-repeat:repeat-x;border-color:#bd362f #bd362f #802420;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ee5f5b',endColorstr='#bd362f',GradientType=0);filter:progid:dximage
transform.microsoft.gradient(enabled=false)}.btn-danger:hover,.btn-danger:active,.btn-danger.active,.btn-danger.disabled,.btn-danger[disabled]{background-color:#bd362f;*background-color:#a9302a}.btn-danger:active,.btn-danger.active{background-color:#942a25 \9}.btn-success{background-color:#5bb75b;*background-color:#51a351;background-image:-ms-linear-gradient(top,#62c462,#51a351);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#51a351));background-image:-webkit-linear-gradient(top,#62c462,#51a351);background-image:-o-linear-gradient(top,#62c462,#51a351);background-image:-moz-linear-gradient(top,#62c462,#51a351);background-image:linear-gradient(top,#62c462,#51a351);background-repeat:repeat-x;border-color:#51a351 #51a351 #387038;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#62c462',endColorstr='#51a351',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.
btn-success:hover,.btn-success:active,.btn-success.active,.btn-success.disabled,.btn-success[disabled]{background-color:#51a351;*background-color:#499249}.btn-success:active,.btn-success.active{background-color:#408140 \9}.btn-info{background-color:#49afcd;*background-color:#2f96b4;background-image:-ms-linear-gradient(top,#5bc0de,#2f96b4);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#2f96b4));background-image:-webkit-linear-gradient(top,#5bc0de,#2f96b4);background-image:-o-linear-gradient(top,#5bc0de,#2f96b4);background-image:-moz-linear-gradient(top,#5bc0de,#2f96b4);background-image:linear-gradient(top,#5bc0de,#2f96b4);background-repeat:repeat-x;border-color:#2f96b4 #2f96b4 #1f6377;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de',endColorstr='#2f96b4',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-info:hover,.btn-info:active,.btn-info
.active,.btn-info.disabled,.btn-info[disabled]{background-color:#2f96b4;*background-color:#2a85a0}.btn-info:active,.btn-info.active{background-color:#24748c \9}.btn-inverse{background-color:#414141;*background-color:#222;background-image:-ms-linear-gradient(top,#555,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#555),to(#222));background-image:-webkit-linear-gradient(top,#555,#222);background-image:-o-linear-gradient(top,#555,#222);background-image:-moz-linear-gradient(top,#555,#222);background-image:linear-gradient(top,#555,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#555555',endColorstr='#222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-inverse:hover,.btn-inverse:active,.btn-inverse.active,.btn-inverse.disabled,.btn-inverse[disabled]{background-color:#222;*background-color:#1515
15}.btn-inverse:active,.btn-inverse.active{background-color:#080808 \9}button.btn,input[type="submit"].btn{*padding-top:2px;*padding-bottom:2px}button.btn::-moz-focus-inner,input[type="submit"].btn::-moz-focus-inner{padding:0;border:0}button.btn.btn-large,input[type="submit"].btn.btn-large{*padding-top:7px;*padding-bottom:7px}button.btn.btn-small,input[type="submit"].btn.btn-small{*padding-top:3px;*padding-bottom:3px}button.btn.btn-mini,input[type="submit"].btn.btn-mini{*padding-top:1px;*padding-bottom:1px}.btn-group{position:relative;*margin-left:.3em;*zoom:1}.btn-group:before,.btn-group:after{display:table;content:""}.btn-group:after{clear:both}.btn-group:first-child{*margin-left:0}.btn-group+.btn-group{margin-left:5px}.btn-toolbar{margin-top:9px;margin-bottom:9px}.btn-toolbar .btn-group{display:inline-block;*display:inline;*zoom:1}.btn-group>.btn{position:relative;float:left;margin-left:-1px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group>.btn:first-child{
margin-left:0;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-bottomleft:4px;-moz-border-radius-topleft:4px}.btn-group>.btn:last-child,.btn-group>.dropdown-toggle{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-bottomright:4px}.btn-group>.btn.large:first-child{margin-left:0;-webkit-border-bottom-left-radius:6px;border-bottom-left-radius:6px;-webkit-border-top-left-radius:6px;border-top-left-radius:6px;-moz-border-radius-bottomleft:6px;-moz-border-radius-topleft:6px}.btn-group>.btn.large:last-child,.btn-group>.large.dropdown-toggle{-webkit-border-top-right-radius:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;border-bottom-right-radius:6px;-moz-border-radius-topright:6px;-moz-border-radius-bottomright:6px}.btn-group>.btn:hover
,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:2}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.dropdown-toggle{*padding-top:4px;padding-right:8px;*padding-bottom:4px;padding-left:8px;-webkit-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn-group>.btn-mini.dropdown-toggle{padding-right:5px;padding-left:5px}.btn-group>.btn-small.dropdown-toggle{*padding-top:4px;*padding-bottom:4px}.btn-group>.btn-large.dropdown-toggle{padding-right:12px;padding-left:12px}.btn-group.open .dropdown-toggle{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgb
a(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn-group.open .btn.dropdown-toggle{background-color:#e6e6e6}.btn-group.open .btn-primary.dropdown-toggle{background-color:#05c}.btn-group.open .btn-warning.dropdown-toggle{background-color:#f89406}.btn-group.open .btn-danger.dropdown-toggle{background-color:#bd362f}.btn-group.open .btn-success.dropdown-toggle{background-color:#51a351}.btn-group.open .btn-info.dropdown-toggle{background-color:#2f96b4}.btn-group.open .btn-inverse.dropdown-toggle{background-color:#222}.btn .caret{margin-top:7px;margin-left:0}.btn:hover .caret,.open.btn-group .caret{opacity:1;filter:alpha(opacity=100)}.btn-mini .caret{margin-top:5px}.btn-small .caret{margin-top:6px}.btn-large .caret{margin-top:6px;border-top-width:5px;border-right-width:5px;border-left-width:5px}.dropup .btn-large .caret{border-top:0;border-bottom:5px solid #000}.btn-primary .caret,.btn-warning .caret,.btn-danger .caret,.btn
-info .caret,.btn-success .caret,.btn-inverse .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:.75;filter:alpha(opacity=75)}.alert{padding:8px 35px 8px 14px;margin-bottom:18px;color:#c09853;text-shadow:0 1px 0 rgba(255,255,255,0.5);background-color:#fcf8e3;border:1px solid #fbeed5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.alert-heading{color:inherit}.alert .close{position:relative;top:-2px;right:-21px;line-height:18px}.alert-success{color:#468847;background-color:#dff0d8;border-color:#d6e9c6}.alert-danger,.alert-error{color:#b94a48;background-color:#f2dede;border-color:#eed3d7}.alert-info{color:#3a87ad;background-color:#d9edf7;border-color:#bce8f1}.alert-block{padding-top:14px;padding-bottom:14px}.alert-block>p,.alert-block>ul{margin-bottom:0}.alert-block p+p{margin-top:5px}.nav{margin-bottom:18px;margin-left:0;list-style:none}.nav>li>a{display:block}.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>.pull-right{float:right}.nav .n
av-header{display:block;padding:3px 15px;font-size:11px;font-weight:bold;line-height:18px;color:#999;text-shadow:0 1px 0 rgba(255,255,255,0.5);text-transform:uppercase}.nav li+.nav-header{margin-top:9px}.nav-list{padding-right:15px;padding-left:15px;margin-bottom:0}.nav-list>li>a,.nav-list .nav-header{margin-right:-15px;margin-left:-15px;text-shadow:0 1px 0 rgba(255,255,255,0.5)}.nav-list>li>a{padding:3px 15px}.nav-list>.active>a,.nav-list>.active>a:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.2);background-color:#08c}.nav-list [class^="icon-"]{margin-right:2px}.nav-list .divider{*width:100%;height:1px;margin:8px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.nav-tabs,.nav-pills{*zoom:1}.nav-tabs:before,.nav-pills:before,.nav-tabs:after,.nav-pills:after{display:table;content:""}.nav-tabs:after,.nav-pills:after{clear:both}.nav-tabs>li,.nav-pills>li{float:left}.nav-tabs>li>a,.nav-pills>li>a{padding-right:12px;padding-left:12px;margi
n-right:2px;line-height:14px}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{margin-bottom:-1px}.nav-tabs>li>a{padding-top:8px;padding-bottom:8px;line-height:18px;border:1px solid transparent;-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>.active>a,.nav-tabs>.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-pills>li>a{padding-top:8px;padding-bottom:8px;margin-top:2px;margin-bottom:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.nav-pills>.active>a,.nav-pills>.active>a:hover{color:#fff;background-color:#08c}.nav-stacked>li{float:none}.nav-stacked>li>a{margin-right:0}.nav-tabs.nav-stacked{border-bottom:0}.nav-tabs.nav-stacked>li>a{border:1px solid #ddd;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.nav-tabs.nav-stacked>li:first-child>a{-webkit-border-radius:4px 4px 0 0;
-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs.nav-stacked>li:last-child>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.nav-tabs.nav-stacked>li>a:hover{z-index:2;border-color:#ddd}.nav-pills.nav-stacked>li>a{margin-bottom:3px}.nav-pills.nav-stacked>li:last-child>a{margin-bottom:1px}.nav-tabs .dropdown-menu{-webkit-border-radius:0 0 5px 5px;-moz-border-radius:0 0 5px 5px;border-radius:0 0 5px 5px}.nav-pills .dropdown-menu{-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-tabs .dropdown-toggle .caret,.nav-pills .dropdown-toggle .caret{margin-top:6px;border-top-color:#08c;border-bottom-color:#08c}.nav-tabs .dropdown-toggle:hover .caret,.nav-pills .dropdown-toggle:hover .caret{border-top-color:#005580;border-bottom-color:#005580}.nav-tabs .active .dropdown-toggle .caret,.nav-pills .active .dropdown-toggle .caret{border-top-color:#333;border-bottom-color:#333}.nav>.dropdown.active>a:hover{color:#000;
cursor:pointer}.nav-tabs .open .dropdown-toggle,.nav-pills .open .dropdown-toggle,.nav>li.dropdown.open.active>a:hover{color:#fff;background-color:#999;border-color:#999}.nav li.dropdown.open .caret,.nav li.dropdown.open.active .caret,.nav li.dropdown.open a:hover .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:1;filter:alpha(opacity=100)}.tabs-stacked .open>a:hover{border-color:#999}.tabbable{*zoom:1}.tabbable:before,.tabbable:after{display:table;content:""}.tabbable:after{clear:both}.tab-content{overflow:auto}.tabs-below>.nav-tabs,.tabs-right>.nav-tabs,.tabs-left>.nav-tabs{border-bottom:0}.tab-content>.tab-pane,.pill-content>.pill-pane{display:none}.tab-content>.active,.pill-content>.active{display:block}.tabs-below>.nav-tabs{border-top:1px solid #ddd}.tabs-below>.nav-tabs>li{margin-top:-1px;margin-bottom:0}.tabs-below>.nav-tabs>li>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.tabs-below>.nav-tabs>li>a:hover{border-top-
color:#ddd;border-bottom-color:transparent}.tabs-below>.nav-tabs>.active>a,.tabs-below>.nav-tabs>.active>a:hover{border-color:transparent #ddd #ddd #ddd}.tabs-left>.nav-tabs>li,.tabs-right>.nav-tabs>li{float:none}.tabs-left>.nav-tabs>li>a,.tabs-right>.nav-tabs>li>a{min-width:74px;margin-right:0;margin-bottom:3px}.tabs-left>.nav-tabs{float:left;margin-right:19px;border-right:1px solid #ddd}.tabs-left>.nav-tabs>li>a{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px}.tabs-left>.nav-tabs>li>a:hover{border-color:#eee #ddd #eee #eee}.tabs-left>.nav-tabs .active>a,.tabs-left>.nav-tabs .active>a:hover{border-color:#ddd transparent #ddd #ddd;*border-right-color:#fff}.tabs-right>.nav-tabs{float:right;margin-left:19px;border-left:1px solid #ddd}.tabs-right>.nav-tabs>li>a{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0}.tabs-right>.nav-tabs>li>a:hover{border-color:#eee #eee #ee
e #ddd}.tabs-right>.nav-tabs .active>a,.tabs-right>.nav-tabs .active>a:hover{border-color:#ddd #ddd #ddd transparent;*border-left-color:#fff}.navbar{*position:relative;*z-index:2;margin-bottom:18px;overflow:visible}.navbar-inner{min-height:40px;padding-right:20px;padding-left:20px;background-color:#2c2c2c;background-image:-moz-linear-gradient(top,#333,#222);background-image:-ms-linear-gradient(top,#333,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#333),to(#222));background-image:-webkit-linear-gradient(top,#333,#222);background-image:-o-linear-gradient(top,#333,#222);background-image:linear-gradient(top,#333,#222);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#333333',endColorstr='#222222',GradientType=0);-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1);
box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1)}.navbar .container{width:auto}.nav-collapse.collapse{height:auto}.navbar{color:#999}.navbar .brand:hover{text-decoration:none}.navbar .brand{display:block;float:left;padding:8px 20px 12px;margin-left:-20px;font-size:20px;font-weight:200;line-height:1;color:#999}.navbar .navbar-text{margin-bottom:0;line-height:40px}.navbar .navbar-link{color:#999}.navbar .navbar-link:hover{color:#fff}.navbar .btn,.navbar .btn-group{margin-top:5px}.navbar .btn-group .btn{margin:0}.navbar-form{margin-bottom:0;*zoom:1}.navbar-form:before,.navbar-form:after{display:table;content:""}.navbar-form:after{clear:both}.navbar-form input,.navbar-form select,.navbar-form .radio,.navbar-form .checkbox{margin-top:5px}.navbar-form input,.navbar-form select{display:inline-block;margin-bottom:0}.navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px}.navbar-form .input-append,.navbar-form .
input-prepend{margin-top:6px;white-space:nowrap}.navbar-form .input-append input,.navbar-form .input-prepend input{margin-top:0}.navbar-search{position:relative;float:left;margin-top:6px;margin-bottom:0}.navbar-search .search-query{padding:4px 9px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;font-weight:normal;line-height:1;color:#fff;background-color:#626262;border:1px solid #151515;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-webkit-transition:none;-moz-transition:none;-ms-transition:none;-o-transition:none;transition:none}.navbar-search .search-query:-moz-placeholder{color:#ccc}.navbar-search .search-query::-webkit-input-placeholder{color:#ccc}.navbar-search .search-query:focus,.navbar-search .search-query.focused{padding:5px 10px;color:#333;text-shadow:0 1px 0 #fff;
background-color:#fff;border:0;outline:0;-webkit-box-shadow:0 0 3px rgba(0,0,0,0.15);-moz-box-shadow:0 0 3px rgba(0,0,0,0.15);box-shadow:0 0 3px rgba(0,0,0,0.15)}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030;margin-bottom:0}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding-right:0;padding-left:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.navbar-fixed-top{top:0}.navbar-fixed-bottom{bottom:0}.navbar .nav{position:relative;left:0;display:block;float:left;margin:0 10px 0 0}.navbar .nav.pull-right{float:right}.navbar .nav>li{display:block;float:left}.navbar .nav>li>a{float:none;padding:9px 10px 11px;line-height:19px;color:#999;text-decoration:none;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.navbar .btn{display:inline-block;padding:4px 10px 4px;margin:5px 5px 6px;line-height:18px}.navbar .btn-group{padding:5px 5px 6px;margin:0}.navbar .nav>li>
a:hover{color:#fff;text-decoration:none;background-color:transparent}.navbar .nav .active>a,.navbar .nav .active>a:hover{color:#fff;text-decoration:none;background-color:#222}.navbar .divider-vertical{width:1px;height:40px;margin:0 9px;overflow:hidden;background-color:#222;border-right:1px solid #333}.navbar .nav.pull-right{margin-right:0;margin-left:10px}.navbar .btn-navbar{display:none;float:right;padding:7px 10px;margin-right:5px;margin-left:5px;background-color:#2c2c2c;*background-color:#222;background-image:-ms-linear-gradient(top,#333,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#333),to(#222));background-image:-webkit-linear-gradient(top,#333,#222);background-image:-o-linear-gradient(top,#333,#222);background-image:linear-gradient(top,#333,#222);background-image:-moz-linear-gradient(top,#333,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradi
ent(startColorstr='#333333',endColorstr='#222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075)}.navbar .btn-navbar:hover,.navbar .btn-navbar:active,.navbar .btn-navbar.active,.navbar .btn-navbar.disabled,.navbar .btn-navbar[disabled]{background-color:#222;*background-color:#151515}.navbar .btn-navbar:active,.navbar .btn-navbar.active{background-color:#080808 \9}.navbar .btn-navbar .icon-bar{display:block;width:18px;height:2px;background-color:#f5f5f5;-webkit-border-radius:1px;-moz-border-radius:1px;border-radius:1px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,0.25);-moz-box-shadow:0 1px 0 rgba(0,0,0,0.25);box-shadow:0 1px 0 rgba(0,0,0,0.25)}.btn-navbar .icon-bar+.icon-bar{margin-top:3px}.navbar .dropdown-men
u:before{position:absolute;top:-7px;left:9px;display:inline-block;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-left:7px solid transparent;border-bottom-color:rgba(0,0,0,0.2);content:''}.navbar .dropdown-menu:after{position:absolute;top:-6px;left:10px;display:inline-block;border-right:6px solid transparent;border-bottom:6px solid #fff;border-left:6px solid transparent;content:''}.navbar-fixed-bottom .dropdown-menu:before{top:auto;bottom:-7px;border-top:7px solid #ccc;border-bottom:0;border-top-color:rgba(0,0,0,0.2)}.navbar-fixed-bottom .dropdown-menu:after{top:auto;bottom:-6px;border-top:6px solid #fff;border-bottom:0}.navbar .nav li.dropdown .dropdown-toggle .caret,.navbar .nav li.dropdown.open .caret{border-top-color:#fff;border-bottom-color:#fff}.navbar .nav li.dropdown.active .caret{opacity:1;filter:alpha(opacity=100)}.navbar .nav li.dropdown.open>.dropdown-toggle,.navbar .nav li.dropdown.active>.dropdown-toggle,.navbar .nav li.dropdown.open.active>.drop
down-toggle{background-color:transparent}.navbar .nav li.dropdown.active>.dropdown-toggle:hover{color:#fff}.navbar .pull-right .dropdown-menu,.navbar .dropdown-menu.pull-right{right:0;left:auto}.navbar .pull-right .dropdown-menu:before,.navbar .dropdown-menu.pull-right:before{right:12px;left:auto}.navbar .pull-right .dropdown-menu:after,.navbar .dropdown-menu.pull-right:after{right:13px;left:auto}.breadcrumb{padding:7px 14px;margin:0 0 18px;list-style:none;background-color:#fbfbfb;background-image:-moz-linear-gradient(top,#fff,#f5f5f5);background-image:-ms-linear-gradient(top,#fff,#f5f5f5);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#f5f5f5));background-image:-webkit-linear-gradient(top,#fff,#f5f5f5);background-image:-o-linear-gradient(top,#fff,#f5f5f5);background-image:linear-gradient(top,#fff,#f5f5f5);background-repeat:repeat-x;border:1px solid #ddd;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;filter:progid:dximagetransform.microsoft.gra
dient(startColorstr='#ffffff',endColorstr='#f5f5f5',GradientType=0);-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.breadcrumb li{display:inline-block;*display:inline;text-shadow:0 1px 0 #fff;*zoom:1}.breadcrumb .divider{padding:0 5px;color:#999}.breadcrumb .active a{color:#333}.pagination{height:36px;margin:18px 0}.pagination ul{display:inline-block;*display:inline;margin-bottom:0;margin-left:0;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;*zoom:1;-webkit-box-shadow:0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:0 1px 2px rgba(0,0,0,0.05);box-shadow:0 1px 2px rgba(0,0,0,0.05)}.pagination li{display:inline}.pagination a{float:left;padding:0 14px;line-height:34px;text-decoration:none;border:1px solid #ddd;border-left-width:0}.pagination a:hover,.pagination .active a{background-color:#f5f5f5}.pagination .active a{color:#999;cursor:default}.pagination .disabled span,.pagination .disabled a,.pagination .disabled a:
hover{color:#999;cursor:default;background-color:transparent}.pagination li:first-child a{border-left-width:1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.pagination li:last-child a{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.pagination-centered{text-align:center}.pagination-right{text-align:right}.pager{margin-bottom:18px;margin-left:0;text-align:center;list-style:none;*zoom:1}.pager:before,.pager:after{display:table;content:""}.pager:after{clear:both}.pager li{display:inline}.pager a{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.pager a:hover{text-decoration:none;background-color:#f5f5f5}.pager .next a{float:right}.pager .previous a{float:left}.pager .disabled a,.pager .disabled a:hover{color:#999;cursor:default;background-color:#fff}.modal-open .dropdown-menu{z-index:2050}.modal-
open .dropdown.open{*z-index:2050}.modal-open .popover{z-index:2060}.modal-open .tooltip{z-index:2070}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop,.modal-backdrop.fade.in{opacity:.8;filter:alpha(opacity=80)}.modal{position:fixed;top:50%;left:50%;z-index:1050;width:560px;margin:-250px 0 0 -280px;overflow:auto;background-color:#fff;border:1px solid #999;border:1px solid rgba(0,0,0,0.3);*border:1px solid #999;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3);-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.modal.fade{top:-25%;-webkit-transition:opacity .3s linear,top .3s ease-out;-moz-transition:opacity .3s linear,top .3s ease-out;-ms-transition:opacity .3s linear,top .3s ease-out;-o-transition:opacity
.3s linear,top .3s ease-out;transition:opacity .3s linear,top .3s ease-out}.modal.fade.in{top:50%}.modal-header{padding:9px 15px;border-bottom:1px solid #eee}.modal-header .close{margin-top:2px}.modal-body{max-height:400px;padding:15px;overflow-y:auto}.modal-form{margin-bottom:0}.modal-footer{padding:14px 15px 15px;margin-bottom:0;text-align:right;background-color:#f5f5f5;border-top:1px solid #ddd;-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;*zoom:1;-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.modal-footer:before,.modal-footer:after{display:table;content:""}.modal-footer:after{clear:both}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.tooltip{position:absolute;z-index:1020;display:block;padding:5px;font-size:11px;opacity:0;filter:alpha(opacity=0);visibility:visible}.tooltip.in{opacity:.8;filter:alpha(opacity=80)}.toolti
p.top{margin-top:-2px}.tooltip.right{margin-left:2px}.tooltip.bottom{margin-top:2px}.tooltip.left{margin-left:-2px}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-top:5px solid #000;border-right:5px solid transparent;border-left:5px solid transparent}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-top:5px solid transparent;border-bottom:5px solid transparent;border-left:5px solid #000}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-right:5px solid transparent;border-bottom:5px solid #000;border-left:5px solid transparent}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-top:5px solid transparent;border-right:5px solid #000;border-bottom:5px solid transparent}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;text-decoration:none;background-color:#000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0}.popover{pos
ition:absolute;top:0;left:0;z-index:1010;display:none;padding:5px}.popover.top{margin-top:-5px}.popover.right{margin-left:5px}.popover.bottom{margin-top:5px}.popover.left{margin-left:-5px}.popover.top .arrow{bottom:0;left:50%;margin-left:-5px;border-top:5px solid #000;border-right:5px solid transparent;border-left:5px solid transparent}.popover.right .arrow{top:50%;left:0;margin-top:-5px;border-top:5px solid transparent;border-right:5px solid #000;border-bottom:5px solid transparent}.popover.bottom .arrow{top:0;left:50%;margin-left:-5px;border-right:5px solid transparent;border-bottom:5px solid #000;border-left:5px solid transparent}.popover.left .arrow{top:50%;right:0;margin-top:-5px;border-top:5px solid transparent;border-bottom:5px solid transparent;border-left:5px solid #000}.popover .arrow{position:absolute;width:0;height:0}.popover-inner{width:280px;padding:3px;overflow:hidden;background:#000;background:rgba(0,0,0,0.8);-webkit-border-radius:6px;-moz-border-radius:6px;border-ra
dius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3)}.popover-title{padding:9px 15px;line-height:1;background-color:#f5f5f5;border-bottom:1px solid #eee;-webkit-border-radius:3px 3px 0 0;-moz-border-radius:3px 3px 0 0;border-radius:3px 3px 0 0}.popover-content{padding:14px;background-color:#fff;-webkit-border-radius:0 0 3px 3px;-moz-border-radius:0 0 3px 3px;border-radius:0 0 3px 3px;-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.popover-content p,.popover-content ul,.popover-content ol{margin-bottom:0}.thumbnails{margin-left:-20px;list-style:none;*zoom:1}.thumbnails:before,.thumbnails:after{display:table;content:""}.thumbnails:after{clear:both}.row-fluid .thumbnails{margin-left:0}.thumbnails>li{float:left;margin-bottom:18px;margin-left:20px}.thumbnail{display:block;padding:4px;line-height:1;border:1px solid #ddd;-webkit-border-radius:4px;-moz-border-rad
ius:4px;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:0 1px 1px rgba(0,0,0,0.075);box-shadow:0 1px 1px rgba(0,0,0,0.075)}a.thumbnail:hover{border-color:#08c;-webkit-box-shadow:0 1px 4px rgba(0,105,214,0.25);-moz-box-shadow:0 1px 4px rgba(0,105,214,0.25);box-shadow:0 1px 4px rgba(0,105,214,0.25)}.thumbnail>img{display:block;max-width:100%;margin-right:auto;margin-left:auto}.thumbnail .caption{padding:9px}.label,.badge{font-size:10.998px;font-weight:bold;line-height:14px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);white-space:nowrap;vertical-align:baseline;background-color:#999}.label{padding:1px 4px 2px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.badge{padding:1px 9px 2px;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px}a.label:hover,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.label-important,.badge-important{background-color:#b94a48}.label-important[href],.badge-important[href]{backgr
ound-color:#953b39}.label-warning,.badge-warning{background-color:#f89406}.label-warning[href],.badge-warning[href]{background-color:#c67605}.label-success,.badge-success{background-color:#468847}.label-success[href],.badge-success[href]{background-color:#356635}.label-info,.badge-info{background-color:#3a87ad}.label-info[href],.badge-info[href]{background-color:#2d6987}.label-inverse,.badge-inverse{background-color:#333}.label-inverse[href],.badge-inverse[href]{background-color:#1a1a1a}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-moz-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-ms-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:0 0}to{background-position:40px 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:18px;mar
gin-bottom:18px;overflow:hidden;background-color:#f7f7f7;background-image:-moz-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-ms-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#f5f5f5),to(#f9f9f9));background-image:-webkit-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-o-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:linear-gradient(top,#f5f5f5,#f9f9f9);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#f5f5f5',endColorstr='#f9f9f9',GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress .bar{width:0;height:18px;font-size:12px;color:#fff;text-align:center;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e90d2;background-image:-moz-linear-gradient(top,#149bdf,#0480be);background-image:-webk
it-gradient(linear,0 0,0 100%,from(#149bdf),to(#0480be));background-image:-webkit-linear-gradient(top,#149bdf,#0480be);background-image:-o-linear-gradient(top,#149bdf,#0480be);background-image:linear-gradient(top,#149bdf,#0480be);background-image:-ms-linear-gradient(top,#149bdf,#0480be);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#149bdf',endColorstr='#0480be',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box;-webkit-transition:width .6s ease;-moz-transition:width .6s ease;-ms-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-striped .bar{background-color:#149bdf;background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15)
50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50
%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;-moz-background-size:40px 40px;-o-background-size:40px 40px;background-size:40px 40px}.progress.active .bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-moz-animation:progress-bar-stripes 2s linear infinite;-ms-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-danger .bar{background-color:#dd514c;background-image:-moz-linear-gradient(top,#ee5f5b,#c43c35);background-image:-ms-linear-gradient(top,#ee5f5b,#c43c35);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#c43c35));background-image:-webkit-linear-gradient(top,#ee5f5b,#c43c35);background-image:-o-linear-gradient(top,#ee5f5b,#c43c35);background-image:linear-gradient(top,#ee5f5b,#c43c35);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startCo
lorstr='#ee5f5b',endColorstr='#c43c35',GradientType=0)}.progress-danger.progress-striped .bar{background-color:#ee5f5b;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-4
5deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-success .bar{background-color:#5eb95e;background-image:-moz-linear-gradient(top,#62c462,#57a957);background-image:-ms-linear-gradient(top,#62c462,#57a957);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#57a957));background-image:-webkit-linear-gradient(top,#62c462,#57a957);background-image:-o-linear-gradient(top,#62c462,#57a957);background-image:linear-gradient(top,#62c462,#57a957);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#62c462',endColorstr='#57a957',GradientType=0)}.progress-success.progress-striped .bar{background-color:#62c462;background-image:-webkit-gradient(linear,0 100%
,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:li
near-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-info .bar{background-color:#4bb1cf;background-image:-moz-linear-gradient(top,#5bc0de,#339bb9);background-image:-ms-linear-gradient(top,#5bc0de,#339bb9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#339bb9));background-image:-webkit-linear-gradient(top,#5bc0de,#339bb9);background-image:-o-linear-gradient(top,#5bc0de,#339bb9);background-image:linear-gradient(top,#5bc0de,#339bb9);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de',endColorstr='#339bb9',GradientType=0)}.progress-info.progress-striped .bar{background-color:#5bc0de;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255
,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress
-warning .bar{background-color:#faa732;background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:-ms-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450',endColorstr='#f89406',GradientType=0)}.progress-warning.progress-striped .bar{background-color:#fbb450;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,r
gba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.accordion{margin-bottom:18px}.accordion-group{margin-bottom:2px;border:1px solid #e5e5e5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.accordion-heading
{border-bottom:0}.accordion-heading .accordion-toggle{display:block;padding:8px 15px}.accordion-toggle{cursor:pointer}.accordion-inner{padding:9px 15px;border-top:1px solid #e5e5e5}.carousel{position:relative;margin-bottom:18px;line-height:1}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel .item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-moz-transition:.6s ease-in-out left;-ms-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel .item>img{display:block;line-height:1}.carousel .active,.carousel .next,.carousel .prev{display:block}.carousel .active{left:0}.carousel .next,.carousel .prev{position:absolute;top:0;width:100%}.carousel .next{left:100%}.carousel .prev{left:-100%}.carousel .next.left,.carousel .prev.right{left:0}.carousel .active.left{left:-100%}.carousel .active.right{left:100%}.carousel-control{position:absolute;top:40%;left:15px;width:40px;height:40px;margin-top:-20p
x;font-size:60px;font-weight:100;line-height:30px;color:#fff;text-align:center;background:#222;border:3px solid #fff;-webkit-border-radius:23px;-moz-border-radius:23px;border-radius:23px;opacity:.5;filter:alpha(opacity=50)}.carousel-control.right{right:15px;left:auto}.carousel-control:hover{color:#fff;text-decoration:none;opacity:.9;filter:alpha(opacity=90)}.carousel-caption{position:absolute;right:0;bottom:0;left:0;padding:10px 15px 5px;background:#333;background:rgba(0,0,0,0.75)}.carousel-caption h4,.carousel-caption p{color:#fff}.hero-unit{padding:60px;margin-bottom:30px;background-color:#eee;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.hero-unit h1{margin-bottom:0;font-size:60px;line-height:1;letter-spacing:-1px;color:inherit}.hero-unit p{font-size:18px;font-weight:200;line-height:27px;color:inherit}.pull-right{float:right}.pull-left{float:left}.hide{display:none}.show{display:block}.invisible{visibility:hidden}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/assets/favicon.ico
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/assets/favicon.ico b/src/main/python/apache/thermos/observer/http/assets/favicon.ico
new file mode 100644
index 0000000..048d2e4
Binary files /dev/null and b/src/main/python/apache/thermos/observer/http/assets/favicon.ico differ
[40/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorage.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorage.java
deleted file mode 100644
index 438c023..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorage.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import com.twitter.aurora.scheduler.storage.AttributeStore;
-import com.twitter.aurora.scheduler.storage.JobStore;
-import com.twitter.aurora.scheduler.storage.LockStore;
-import com.twitter.aurora.scheduler.storage.QuotaStore;
-import com.twitter.aurora.scheduler.storage.ReadWriteLockManager;
-import com.twitter.aurora.scheduler.storage.SchedulerStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.TaskStore;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.stats.Stats;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A storage implementation comprised of individual in-memory store implementations.
- * <p>
- * This storage has a global read-write lock, which is used when invoking
- * {@link #consistentRead(Work)} and {@link #write(MutateWork)}. However, no locks are used at this
- * level for {@link #weaklyConsistentRead(Work)}. It is the responsibility of the
- * individual stores to ensure that read operations are thread-safe (optimally supporting
- * concurrency). Store implementations may assume that all methods invoked on {@code Mutable}
- * store interfaces are protected by the global write lock, and thus invoked serially.
- */
-public class MemStorage implements Storage {
- private final AtomicLong readLockWaitNanos = Stats.exportLong("read_lock_wait_nanos");
- private final AtomicLong writeLockWaitNanos = Stats.exportLong("write_lock_wait_nanos");
-
- private final MutableStoreProvider storeProvider;
- private final ReadWriteLockManager lockManager = new ReadWriteLockManager();
-
- @Inject
- MemStorage(
- final SchedulerStore.Mutable schedulerStore,
- final JobStore.Mutable jobStore,
- final TaskStore.Mutable taskStore,
- final LockStore.Mutable lockStore,
- final QuotaStore.Mutable quotaStore,
- final AttributeStore.Mutable attributeStore) {
-
- storeProvider = new MutableStoreProvider() {
- @Override public SchedulerStore.Mutable getSchedulerStore() {
- return schedulerStore;
- }
-
- @Override public JobStore.Mutable getJobStore() {
- return jobStore;
- }
-
- @Override public TaskStore getTaskStore() {
- return taskStore;
- }
-
- @Override public TaskStore.Mutable getUnsafeTaskStore() {
- return taskStore;
- }
-
- @Override public LockStore.Mutable getLockStore() {
- return lockStore;
- }
-
- @Override public QuotaStore.Mutable getQuotaStore() {
- return quotaStore;
- }
-
- @Override public AttributeStore.Mutable getAttributeStore() {
- return attributeStore;
- }
- };
- }
-
- /**
- * Creates a new empty in-memory storage for use in testing.
- */
- @VisibleForTesting
- public static MemStorage newEmptyStorage() {
- return new MemStorage(
- new MemSchedulerStore(),
- new MemJobStore(),
- new MemTaskStore(),
- new MemLockStore(),
- new MemQuotaStore(),
- new MemAttributeStore());
- }
-
- @Timed("mem_storage_consistent_read_operation")
- @Override
- public <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
- checkNotNull(work);
-
- long lockStartNanos = System.nanoTime();
- boolean topLevelOperation = lockManager.readLock();
- if (topLevelOperation) {
- readLockWaitNanos.addAndGet(System.nanoTime() - lockStartNanos);
- }
- try {
- return work.apply(storeProvider);
- } finally {
- lockManager.readUnlock();
- }
- }
-
- @Timed("mem_storage_weakly_consistent_read_operation")
- @Override
- public <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work)
- throws StorageException, E {
-
- return work.apply(storeProvider);
- }
-
- @Timed("mem_storage_write_operation")
- @Override
- public <T, E extends Exception> T write(MutateWork<T, E> work)
- throws StorageException, E {
-
- checkNotNull(work);
-
- long lockStartNanos = System.nanoTime();
- boolean topLevelOperation = lockManager.writeLock();
- if (topLevelOperation) {
- writeLockWaitNanos.addAndGet(System.nanoTime() - lockStartNanos);
- }
- try {
- return work.apply(storeProvider);
- } finally {
- lockManager.writeUnlock();
- }
- }
-
- @Override
- public void snapshot() {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorageModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorageModule.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorageModule.java
deleted file mode 100644
index a99aa12..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemStorageModule.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import javax.inject.Singleton;
-
-import com.google.inject.Key;
-import com.google.inject.PrivateModule;
-
-import com.twitter.aurora.scheduler.storage.AttributeStore;
-import com.twitter.aurora.scheduler.storage.JobStore;
-import com.twitter.aurora.scheduler.storage.LockStore;
-import com.twitter.aurora.scheduler.storage.QuotaStore;
-import com.twitter.aurora.scheduler.storage.SchedulerStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.Volatile;
-import com.twitter.aurora.scheduler.storage.TaskStore;
-import com.twitter.common.inject.Bindings.KeyFactory;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Binding module for an in-memory storage system.
- * <p>
- * Exposes bindings for storage components:
- * <ul>
- * <li>{@link com.twitter.aurora.scheduler.storage.Storage}</li>
- * <li>Keyed with keys provided by the provided{@code keyFactory}:</li>
- * <ul>
- * <li>{@link com.twitter.aurora.scheduler.storage.SchedulerStore}</li>
- * <li>{@link com.twitter.aurora.scheduler.storage.JobStore}</li>
- * <li>{@link com.twitter.aurora.scheduler.storage.TaskStore}</li>
- * <li>{@link com.twitter.aurora.scheduler.storage.LockStore}</li>
- * <li>{@link com.twitter.aurora.scheduler.storage.QuotaStore}</li>
- * <li>{@link com.twitter.aurora.scheduler.storage.AttributeStore}</li>
- * </ul>
- * </ul>
- */
-public final class MemStorageModule extends PrivateModule {
-
- private final KeyFactory keyFactory;
-
- public MemStorageModule(KeyFactory keyFactory) {
- this.keyFactory = checkNotNull(keyFactory);
- }
-
- private <T> void bindStore(Class<T> binding, Class<? extends T> impl) {
- bind(binding).to(impl);
- bind(impl).in(Singleton.class);
- Key<T> key = keyFactory.create(binding);
- bind(key).to(impl);
- expose(key);
- }
-
- @Override
- protected void configure() {
- Key<Storage> storageKey = keyFactory.create(Storage.class);
- bind(storageKey).to(MemStorage.class);
- expose(storageKey);
- Key<Storage> exposedMemStorageKey = Key.get(Storage.class, Volatile.class);
- bind(exposedMemStorageKey).to(MemStorage.class);
- expose(exposedMemStorageKey);
- bind(MemStorage.class).in(Singleton.class);
-
- bindStore(SchedulerStore.Mutable.class, MemSchedulerStore.class);
- bindStore(JobStore.Mutable.class, MemJobStore.class);
- bindStore(TaskStore.Mutable.class, MemTaskStore.class);
- bindStore(LockStore.Mutable.class, MemLockStore.class);
- bindStore(QuotaStore.Mutable.class, MemQuotaStore.class);
- bindStore(AttributeStore.Mutable.class, MemAttributeStore.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemTaskStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemTaskStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemTaskStore.java
deleted file mode 100644
index d02511f..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemTaskStore.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import com.google.common.base.Function;
-import com.google.common.base.Functions;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.twitter.aurora.gen.ScheduledTask;
-import com.twitter.aurora.gen.TaskConfig;
-import com.twitter.aurora.gen.TaskQuery;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.storage.TaskStore;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.base.MorePreconditions;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * An in-memory task store.
- */
-class MemTaskStore implements TaskStore.Mutable {
-
- private static final Logger LOG = Logger.getLogger(MemTaskStore.class.getName());
-
- @CmdLine(name = "slow_query_log_threshold",
- help = "Log all queries that take at least this long to execute.")
- private static final Arg<Amount<Long, Time>> SLOW_QUERY_LOG_THRESHOLD =
- Arg.create(Amount.of(25L, Time.MILLISECONDS));
-
- private final long slowQueryThresholdNanos = SLOW_QUERY_LOG_THRESHOLD.get().as(Time.NANOSECONDS);
-
- private final Map<String, Task> tasks = Maps.newConcurrentMap();
- private final Multimap<IJobKey, String> tasksByJobKey =
- Multimaps.synchronizedSetMultimap(HashMultimap.<IJobKey, String>create());
-
- // An interner is used here to collapse equivalent TaskConfig instances into canonical instances.
- // Ideally this would fall out of the object hierarchy (TaskConfig being associated with the job
- // rather than the task), but we intuit this detail here for performance reasons.
- private final Interner<TaskConfig, String> configInterner = new Interner<TaskConfig, String>();
-
- private final AtomicLong taskQueriesById = Stats.exportLong("task_queries_by_id");
- private final AtomicLong taskQueriesByJob = Stats.exportLong("task_queries_by_job");
- private final AtomicLong taskQueriesAll = Stats.exportLong("task_queries_all");
-
- @Timed("mem_storage_fetch_tasks")
- @Override
- public ImmutableSet<IScheduledTask> fetchTasks(Query.Builder query) {
- checkNotNull(query);
-
- long start = System.nanoTime();
- ImmutableSet<IScheduledTask> result = matches(query.get()).toSet();
- long durationNanos = System.nanoTime() - start;
- Level level = (durationNanos >= slowQueryThresholdNanos) ? Level.INFO : Level.FINE;
- if (LOG.isLoggable(level)) {
- Long time = Amount.of(durationNanos, Time.NANOSECONDS).as(Time.MILLISECONDS);
- LOG.log(level, "Query took " + time + " ms: " + query.get());
- }
-
- return result;
- }
-
- private final Function<IScheduledTask, Task> toTask =
- new Function<IScheduledTask, Task>() {
- @Override public Task apply(IScheduledTask task) {
- return new Task(task, configInterner);
- }
- };
-
- @Timed("mem_storage_save_tasks")
- @Override
- public void saveTasks(Set<IScheduledTask> newTasks) {
- checkNotNull(newTasks);
- Preconditions.checkState(Tasks.ids(newTasks).size() == newTasks.size(),
- "Proposed new tasks would create task ID collision.");
-
- Iterable<Task> canonicalized = Iterables.transform(newTasks, toTask);
- tasks.putAll(Maps.uniqueIndex(canonicalized, TO_ID));
- tasksByJobKey.putAll(taskIdsByJobKey(canonicalized));
- }
-
- private Multimap<IJobKey, String> taskIdsByJobKey(Iterable<Task> toIndex) {
- return Multimaps.transformValues(
- Multimaps.index(toIndex, Functions.compose(Tasks.SCHEDULED_TO_JOB_KEY, TO_SCHEDULED)),
- TO_ID);
- }
-
- @Timed("mem_storage_delete_all_tasks")
- @Override
- public void deleteAllTasks() {
- tasks.clear();
- tasksByJobKey.clear();
- configInterner.clear();
- }
-
- @Timed("mem_storage_delete_tasks")
- @Override
- public void deleteTasks(Set<String> taskIds) {
- checkNotNull(taskIds);
-
- for (String id : taskIds) {
- Task removed = tasks.remove(id);
- if (removed != null) {
- tasksByJobKey.remove(Tasks.SCHEDULED_TO_JOB_KEY.apply(removed.task), id);
- configInterner.removeAssociation(removed.task.getAssignedTask().getTask().newBuilder(), id);
- }
- }
- }
-
- @Timed("mem_storage_mutate_tasks")
- @Override
- public ImmutableSet<IScheduledTask> mutateTasks(
- Query.Builder query,
- Function<IScheduledTask, IScheduledTask> mutator) {
-
- checkNotNull(query);
- checkNotNull(mutator);
-
- ImmutableSet.Builder<IScheduledTask> mutated = ImmutableSet.builder();
- for (IScheduledTask original : matches(query.get())) {
- IScheduledTask maybeMutated = mutator.apply(original);
- if (!original.equals(maybeMutated)) {
- Preconditions.checkState(
- Tasks.id(original).equals(Tasks.id(maybeMutated)),
- "A task's ID may not be mutated.");
- tasks.put(Tasks.id(maybeMutated), toTask.apply(maybeMutated));
- mutated.add(maybeMutated);
- }
- }
-
- return mutated.build();
- }
-
- @Timed("mem_storage_unsafe_modify_in_place")
- @Override
- public boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration) {
- MorePreconditions.checkNotBlank(taskId);
- checkNotNull(taskConfiguration);
-
- Task stored = tasks.get(taskId);
- if (stored == null) {
- return false;
- } else {
- ScheduledTask updated = stored.task.newBuilder();
- updated.getAssignedTask().setTask(taskConfiguration.newBuilder());
- tasks.put(taskId, toTask.apply(IScheduledTask.build(updated)));
- return true;
- }
- }
-
- private static Predicate<IScheduledTask> queryFilter(final TaskQuery query) {
- return new Predicate<IScheduledTask>() {
- @Override public boolean apply(IScheduledTask task) {
- ITaskConfig config = task.getAssignedTask().getTask();
- if (query.getOwner() != null) {
- if (!StringUtils.isBlank(query.getOwner().getRole())) {
- if (!query.getOwner().getRole().equals(config.getOwner().getRole())) {
- return false;
- }
- }
- if (!StringUtils.isBlank(query.getOwner().getUser())) {
- if (!query.getOwner().getUser().equals(config.getOwner().getUser())) {
- return false;
- }
- }
- }
- if (query.getEnvironment() != null) {
- if (!query.getEnvironment().equals(config.getEnvironment())) {
- return false;
- }
- }
- if (query.getJobName() != null) {
- if (!query.getJobName().equals(config.getJobName())) {
- return false;
- }
- }
-
- if (query.getTaskIds() != null) {
- if (!query.getTaskIds().contains(Tasks.id(task))) {
- return false;
- }
- }
-
- if (query.getStatusesSize() > 0) {
- if (!query.getStatuses().contains(task.getStatus())) {
- return false;
- }
- }
- if (!StringUtils.isEmpty(query.getSlaveHost())) {
- if (!query.getSlaveHost().equals(task.getAssignedTask().getSlaveHost())) {
- return false;
- }
- }
- if (query.getInstanceIdsSize() > 0) {
- if (!query.getInstanceIds().contains(task.getAssignedTask().getInstanceId())) {
- return false;
- }
- }
-
- return true;
- }
- };
- }
-
- private Iterable<Task> fromIdIndex(Iterable<String> taskIds) {
- ImmutableList.Builder<Task> matches = ImmutableList.builder();
- for (String id : taskIds) {
- Task match = tasks.get(id);
- if (match != null) {
- matches.add(match);
- }
- }
- return matches.build();
- }
-
- private FluentIterable<IScheduledTask> matches(TaskQuery query) {
- // Apply the query against the working set.
- Iterable<Task> from;
- Optional<IJobKey> jobKey = JobKeys.from(Query.arbitrary(query));
- if (query.isSetTaskIds()) {
- taskQueriesById.incrementAndGet();
- from = fromIdIndex(query.getTaskIds());
- } else if (jobKey.isPresent()) {
- taskQueriesByJob.incrementAndGet();
- Collection<String> taskIds = tasksByJobKey.get(jobKey.get());
- if (taskIds == null) {
- from = ImmutableList.of();
- } else {
- from = fromIdIndex(taskIds);
- }
- } else {
- taskQueriesAll.incrementAndGet();
- from = tasks.values();
- }
-
- return FluentIterable.from(from).transform(TO_SCHEDULED).filter(queryFilter(query));
- }
-
- private static final Function<Task, IScheduledTask> TO_SCHEDULED =
- new Function<Task, IScheduledTask>() {
- @Override public IScheduledTask apply(Task task) {
- return task.task;
- }
- };
-
- private static final Function<Task, String> TO_ID =
- Functions.compose(Tasks.SCHEDULED_TO_ID, TO_SCHEDULED);
-
- private static class Task {
- private final IScheduledTask task;
-
- Task(IScheduledTask task, Interner<TaskConfig, String> interner) {
- interner.removeAssociation(task.getAssignedTask().getTask().newBuilder(), Tasks.id(task));
- TaskConfig canonical = interner.addAssociation(
- task.getAssignedTask().getTask().newBuilder(),
- Tasks.id(task));
- ScheduledTask builder = task.newBuilder();
- builder.getAssignedTask().setTask(canonical);
- this.task = IScheduledTask.build(builder);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/Util.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/Util.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/Util.java
deleted file mode 100644
index cf1b057..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/Util.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Function;
-
-import org.apache.thrift.TBase;
-
-/**
- * Utility class for common operations amongst in-memory store implementations.
- */
-final class Util {
-
- private Util() {
- // Utility class.
- }
-
- /**
- * Creates a function that performs a 'deep copy' on a thrift struct of a specific type. The
- * resulting copied objects will be exactly identical to the original. Mutations to the original
- * object will not be reflected in the copy, and vice versa.
- *
- * @return A copier for the provided type of thrift structs.
- */
- static <T extends TBase<T, ?>> Function<T, T> deepCopier() {
- return new Function<T, T>() {
- @Override public T apply(@Nullable T input) {
- if (input == null) {
- return null;
- }
-
- @SuppressWarnings("unchecked")
- T t = (T) input.deepCopy();
- return t;
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/testing/StorageTestUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/testing/StorageTestUtil.java b/src/main/java/com/twitter/aurora/scheduler/storage/testing/StorageTestUtil.java
deleted file mode 100644
index d735828..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/testing/StorageTestUtil.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.testing;
-
-import com.google.common.collect.ImmutableSet;
-
-import org.easymock.Capture;
-import org.easymock.IAnswer;
-import org.easymock.IExpectationSetters;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.storage.AttributeStore;
-import com.twitter.aurora.scheduler.storage.JobStore;
-import com.twitter.aurora.scheduler.storage.LockStore;
-import com.twitter.aurora.scheduler.storage.QuotaStore;
-import com.twitter.aurora.scheduler.storage.SchedulerStore;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.TaskStore;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.testing.easymock.EasyMockTest;
-
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.expect;
-
-/**
- * Auxiliary class to simplify testing against a mocked storage. This allows callers to directly
- * set up call expectations on individual stores rather than writing plumbing code to deal with
- * operations and {@link StoreProvider}.
- */
-public class StorageTestUtil {
-
- public final StoreProvider storeProvider;
- public final MutableStoreProvider mutableStoreProvider;
- public final TaskStore.Mutable taskStore;
- public final QuotaStore.Mutable quotaStore;
- public final AttributeStore.Mutable attributeStore;
- public final JobStore.Mutable jobStore;
- public final LockStore.Mutable lockStore;
- public final SchedulerStore.Mutable schedulerStore;
- public final NonVolatileStorage storage;
-
- /**
- * Creates a new storage test utility.
- *
- * @param easyMock Mocking framework to use for setting up mocks and expectations.
- */
- public StorageTestUtil(EasyMockTest easyMock) {
- this.storeProvider = easyMock.createMock(StoreProvider.class);
- this.mutableStoreProvider = easyMock.createMock(MutableStoreProvider.class);
- this.taskStore = easyMock.createMock(TaskStore.Mutable.class);
- this.quotaStore = easyMock.createMock(QuotaStore.Mutable.class);
- this.attributeStore = easyMock.createMock(AttributeStore.Mutable.class);
- this.jobStore = easyMock.createMock(JobStore.Mutable.class);
- this.lockStore = easyMock.createMock(LockStore.Mutable.class);
- this.schedulerStore = easyMock.createMock(SchedulerStore.Mutable.class);
- this.storage = easyMock.createMock(NonVolatileStorage.class);
- }
-
- private <T> IExpectationSetters<T> expectConsistentRead() {
- final Capture<Work<T, RuntimeException>> work = EasyMockTest.createCapture();
- return expect(storage.consistentRead(capture(work))).andAnswer(new IAnswer<T>() {
- @Override public T answer() {
- return work.getValue().apply(storeProvider);
- }
- });
- }
-
- private <T> IExpectationSetters<T> expectWeaklyConsistentRead() {
- final Capture<Work<T, RuntimeException>> work = EasyMockTest.createCapture();
- return expect(storage.weaklyConsistentRead(capture(work))).andAnswer(new IAnswer<T>() {
- @Override public T answer() {
- return work.getValue().apply(storeProvider);
- }
- });
- }
-
- private <T> IExpectationSetters<T> expectWriteOperation() {
- final Capture<MutateWork<T, RuntimeException>> work = EasyMockTest.createCapture();
- return expect(storage.write(capture(work))).andAnswer(new IAnswer<T>() {
- @Override public T answer() {
- return work.getValue().apply(mutableStoreProvider);
- }
- });
- }
-
- /**
- * Expects any number of read or write operations.
- */
- public void expectOperations() {
- expect(storeProvider.getTaskStore()).andReturn(taskStore).anyTimes();
- expect(storeProvider.getQuotaStore()).andReturn(quotaStore).anyTimes();
- expect(storeProvider.getAttributeStore()).andReturn(attributeStore).anyTimes();
- expect(storeProvider.getJobStore()).andReturn(jobStore).anyTimes();
- expect(storeProvider.getLockStore()).andReturn(lockStore).anyTimes();
- expect(storeProvider.getSchedulerStore()).andReturn(schedulerStore).anyTimes();
- expect(mutableStoreProvider.getTaskStore()).andReturn(taskStore).anyTimes();
- expect(mutableStoreProvider.getUnsafeTaskStore()).andReturn(taskStore).anyTimes();
- expect(mutableStoreProvider.getQuotaStore()).andReturn(quotaStore).anyTimes();
- expect(mutableStoreProvider.getAttributeStore()).andReturn(attributeStore).anyTimes();
- expect(mutableStoreProvider.getJobStore()).andReturn(jobStore).anyTimes();
- expect(mutableStoreProvider.getLockStore()).andReturn(lockStore).anyTimes();
- expect(mutableStoreProvider.getSchedulerStore()).andReturn(schedulerStore).anyTimes();
- expectConsistentRead().anyTimes();
- expectWeaklyConsistentRead().anyTimes();
- expectWriteOperation().anyTimes();
- }
-
- public IExpectationSetters<?> expectTaskFetch(
- Query.Builder query,
- ImmutableSet<IScheduledTask> result) {
-
- return expect(taskStore.fetchTasks(query)).andReturn(result);
- }
-
- public IExpectationSetters<?> expectTaskFetch(Query.Builder query, IScheduledTask... result) {
- return expectTaskFetch(query, ImmutableSet.<IScheduledTask>builder().add(result).build());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java
deleted file mode 100644
index 2acf5c8..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerAPIServlet.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package com.twitter.aurora.scheduler.thrift;
-
-import javax.inject.Inject;
-
-import org.apache.thrift.protocol.TJSONProtocol;
-import org.apache.thrift.server.TServlet;
-
-import com.twitter.aurora.gen.AuroraAdmin;
-
-/**
- * A servlet that exposes the scheduler Thrift API over HTTP/JSON.
- */
-class SchedulerAPIServlet extends TServlet {
-
- @Inject
- SchedulerAPIServlet(AuroraAdmin.Iface schedulerThriftInterface) {
- super(new AuroraAdmin.Processor<>(schedulerThriftInterface), new TJSONProtocol.Factory());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java b/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
deleted file mode 100644
index 3ff0f1c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/SchedulerThriftInterface.java
+++ /dev/null
@@ -1,1000 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Functions;
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.base.Throwables;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-import com.google.common.collect.Sets;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.twitter.aurora.auth.CapabilityValidator;
-import com.twitter.aurora.auth.CapabilityValidator.AuditCheck;
-import com.twitter.aurora.auth.CapabilityValidator.Capability;
-import com.twitter.aurora.auth.SessionValidator.AuthFailedException;
-import com.twitter.aurora.gen.AcquireLockResult;
-import com.twitter.aurora.gen.AddInstancesConfig;
-import com.twitter.aurora.gen.AuroraAdmin;
-import com.twitter.aurora.gen.ConfigRewrite;
-import com.twitter.aurora.gen.DrainHostsResult;
-import com.twitter.aurora.gen.EndMaintenanceResult;
-import com.twitter.aurora.gen.GetJobsResult;
-import com.twitter.aurora.gen.GetQuotaResult;
-import com.twitter.aurora.gen.Hosts;
-import com.twitter.aurora.gen.InstanceConfigRewrite;
-import com.twitter.aurora.gen.InstanceKey;
-import com.twitter.aurora.gen.JobConfigRewrite;
-import com.twitter.aurora.gen.JobConfigValidation;
-import com.twitter.aurora.gen.JobConfiguration;
-import com.twitter.aurora.gen.JobKey;
-import com.twitter.aurora.gen.JobSummary;
-import com.twitter.aurora.gen.JobSummaryResult;
-import com.twitter.aurora.gen.ListBackupsResult;
-import com.twitter.aurora.gen.Lock;
-import com.twitter.aurora.gen.LockKey;
-import com.twitter.aurora.gen.LockValidation;
-import com.twitter.aurora.gen.MaintenanceStatusResult;
-import com.twitter.aurora.gen.PopulateJobResult;
-import com.twitter.aurora.gen.QueryRecoveryResult;
-import com.twitter.aurora.gen.Quota;
-import com.twitter.aurora.gen.Response;
-import com.twitter.aurora.gen.ResponseCode;
-import com.twitter.aurora.gen.Result;
-import com.twitter.aurora.gen.RewriteConfigsRequest;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.ScheduleStatusResult;
-import com.twitter.aurora.gen.SessionKey;
-import com.twitter.aurora.gen.StartMaintenanceResult;
-import com.twitter.aurora.gen.TaskConfig;
-import com.twitter.aurora.gen.TaskQuery;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.ScheduleException;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
-import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
-import com.twitter.aurora.scheduler.quota.Quotas;
-import com.twitter.aurora.scheduler.state.CronJobManager;
-import com.twitter.aurora.scheduler.state.LockManager;
-import com.twitter.aurora.scheduler.state.LockManager.LockException;
-import com.twitter.aurora.scheduler.state.MaintenanceController;
-import com.twitter.aurora.scheduler.state.SchedulerCore;
-import com.twitter.aurora.scheduler.storage.JobStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.backup.Recovery;
-import com.twitter.aurora.scheduler.storage.backup.Recovery.RecoveryException;
-import com.twitter.aurora.scheduler.storage.backup.StorageBackup;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.ILockKey;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.aurora.scheduler.thrift.auth.DecoratedThrift;
-import com.twitter.aurora.scheduler.thrift.auth.Requires;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.base.MorePreconditions;
-import com.twitter.common.base.Supplier;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.util.BackoffHelper;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.auth.SessionValidator.SessionContext;
-import static com.twitter.aurora.gen.ResponseCode.AUTH_FAILED;
-import static com.twitter.aurora.gen.ResponseCode.ERROR;
-import static com.twitter.aurora.gen.ResponseCode.INVALID_REQUEST;
-import static com.twitter.aurora.gen.ResponseCode.LOCK_ERROR;
-import static com.twitter.aurora.gen.ResponseCode.OK;
-import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * Aurora scheduler thrift server implementation.
- * <p>
- * Interfaces between users and the scheduler to access/modify jobs and perform cluster
- * administration tasks.
- */
-@DecoratedThrift
-class SchedulerThriftInterface implements AuroraAdmin.Iface {
- private static final Logger LOG = Logger.getLogger(SchedulerThriftInterface.class.getName());
-
- @CmdLine(name = "kill_task_initial_backoff",
- help = "Initial backoff delay while waiting for the tasks to transition to KILLED.")
- private static final Arg<Amount<Long, Time>> KILL_TASK_INITIAL_BACKOFF =
- Arg.create(Amount.of(1L, Time.SECONDS));
-
- @CmdLine(name = "kill_task_max_backoff",
- help = "Max backoff delay while waiting for the tasks to transition to KILLED.")
- private static final Arg<Amount<Long, Time>> KILL_TASK_MAX_BACKOFF =
- Arg.create(Amount.of(30L, Time.SECONDS));
-
- private static final Function<IScheduledTask, String> GET_ROLE = Functions.compose(
- new Function<ITaskConfig, String>() {
- @Override public String apply(ITaskConfig task) {
- return task.getOwner().getRole();
- }
- },
- Tasks.SCHEDULED_TO_INFO);
-
- private final Storage storage;
- private final SchedulerCore schedulerCore;
- private final LockManager lockManager;
- private final CapabilityValidator sessionValidator;
- private final StorageBackup backup;
- private final Recovery recovery;
- private final MaintenanceController maintenance;
- private final CronJobManager cronJobManager;
- private final Amount<Long, Time> killTaskInitialBackoff;
- private final Amount<Long, Time> killTaskMaxBackoff;
-
- @Inject
- SchedulerThriftInterface(
- Storage storage,
- SchedulerCore schedulerCore,
- LockManager lockManager,
- CapabilityValidator sessionValidator,
- StorageBackup backup,
- Recovery recovery,
- CronJobManager cronJobManager,
- MaintenanceController maintenance) {
-
- this(storage,
- schedulerCore,
- lockManager,
- sessionValidator,
- backup,
- recovery,
- maintenance,
- cronJobManager,
- KILL_TASK_INITIAL_BACKOFF.get(),
- KILL_TASK_MAX_BACKOFF.get());
- }
-
- @VisibleForTesting
- SchedulerThriftInterface(
- Storage storage,
- SchedulerCore schedulerCore,
- LockManager lockManager,
- CapabilityValidator sessionValidator,
- StorageBackup backup,
- Recovery recovery,
- MaintenanceController maintenance,
- CronJobManager cronJobManager,
- Amount<Long, Time> initialBackoff,
- Amount<Long, Time> maxBackoff) {
-
- this.storage = checkNotNull(storage);
- this.schedulerCore = checkNotNull(schedulerCore);
- this.lockManager = checkNotNull(lockManager);
- this.sessionValidator = checkNotNull(sessionValidator);
- this.backup = checkNotNull(backup);
- this.recovery = checkNotNull(recovery);
- this.maintenance = checkNotNull(maintenance);
- this.cronJobManager = checkNotNull(cronJobManager);
- this.killTaskInitialBackoff = checkNotNull(initialBackoff);
- this.killTaskMaxBackoff = checkNotNull(maxBackoff);
- }
-
- @Override
- public Response createJob(
- JobConfiguration mutableJob,
- @Nullable Lock mutableLock,
- SessionKey session) {
-
- IJobConfiguration job = IJobConfiguration.build(mutableJob);
- IJobKey jobKey = JobKeys.assertValid(job.getKey());
- checkNotNull(session);
-
- Response response = new Response();
-
- try {
- sessionValidator.checkAuthenticated(session, ImmutableSet.of(job.getOwner().getRole()));
- } catch (AuthFailedException e) {
- return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- }
-
- try {
- SanitizedConfiguration sanitized = SanitizedConfiguration.fromUnsanitized(job);
-
- lockManager.validateIfLocked(
- ILockKey.build(LockKey.job(jobKey.newBuilder())),
- Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
-
- schedulerCore.createJob(sanitized);
- response.setResponseCode(OK)
- .setMessage(String.format("%d new tasks pending for job %s",
- sanitized.getJobConfig().getInstanceCount(), JobKeys.toPath(job)));
- } catch (LockException e) {
- response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
- } catch (TaskDescriptionException | ScheduleException e) {
- response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
- }
-
- return response;
- }
-
- @Override
- public Response replaceCronTemplate(
- JobConfiguration mutableConfig,
- @Nullable Lock mutableLock,
- SessionKey session) {
-
- checkNotNull(mutableConfig);
- IJobConfiguration job = IJobConfiguration.build(mutableConfig);
- IJobKey jobKey = JobKeys.assertValid(job.getKey());
- checkNotNull(session);
-
- Response response = new Response();
- try {
- sessionValidator.checkAuthenticated(session, ImmutableSet.of(job.getOwner().getRole()));
- } catch (AuthFailedException e) {
- return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- }
-
- try {
- lockManager.validateIfLocked(
- ILockKey.build(LockKey.job(jobKey.newBuilder())),
- Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
-
- SanitizedConfiguration sanitized = SanitizedConfiguration.fromUnsanitized(job);
-
- if (!cronJobManager.hasJob(jobKey)) {
- return response.setResponseCode(INVALID_REQUEST).setMessage(
- "No cron template found for the given key: " + jobKey);
- }
- cronJobManager.updateJob(sanitized);
- return response.setResponseCode(OK).setMessage("Replaced template for: " + jobKey);
-
- } catch (LockException e) {
- return response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
- } catch (TaskDescriptionException | ScheduleException e) {
- return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
- }
- }
-
- @Override
- public Response populateJobConfig(JobConfiguration description, JobConfigValidation validation) {
-
- checkNotNull(description);
-
- Response response = new Response();
- try {
- SanitizedConfiguration sanitized =
- SanitizedConfiguration.fromUnsanitized(IJobConfiguration.build(description));
-
- // TODO(maximk): Consider moving job validation logic into a dedicated RPC. MESOS-4476.
- if (validation != null && validation == JobConfigValidation.RUN_FILTERS) {
- schedulerCore.validateJobResources(sanitized);
- }
-
- PopulateJobResult result = new PopulateJobResult()
- .setPopulated(ITaskConfig.toBuildersSet(sanitized.getTaskConfigs().values()));
- response.setResult(Result.populateJobResult(result))
- .setResponseCode(OK)
- .setMessage("Tasks populated");
- } catch (TaskDescriptionException | ScheduleException e) {
- response.setResponseCode(INVALID_REQUEST)
- .setMessage("Invalid configuration: " + e.getMessage());
- }
- return response;
- }
-
- @Override
- public Response startCronJob(JobKey mutableJobKey, SessionKey session) {
- checkNotNull(session);
- IJobKey jobKey = JobKeys.assertValid(IJobKey.build(mutableJobKey));
-
- Response response = new Response();
- try {
- sessionValidator.checkAuthenticated(session, ImmutableSet.of(jobKey.getRole()));
- } catch (AuthFailedException e) {
- response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- return response;
- }
-
- try {
- schedulerCore.startCronJob(jobKey);
- response.setResponseCode(OK).setMessage("Cron run started.");
- } catch (ScheduleException e) {
- response.setResponseCode(INVALID_REQUEST)
- .setMessage("Failed to start cron job - " + e.getMessage());
- } catch (TaskDescriptionException e) {
- response.setResponseCode(ERROR).setMessage("Invalid task description: " + e.getMessage());
- }
-
- return response;
- }
-
- // TODO(William Farner): Provide status information about cron jobs here.
- @Override
- public Response getTasksStatus(TaskQuery query) {
- checkNotNull(query);
-
- Set<IScheduledTask> tasks =
- Storage.Util.weaklyConsistentFetchTasks(storage, Query.arbitrary(query));
-
- Response response = new Response();
-
- if (tasks.isEmpty()) {
- response.setResponseCode(INVALID_REQUEST)
- .setMessage("No tasks found for query: " + query);
- } else {
- response.setResponseCode(OK)
- .setResult(Result.scheduleStatusResult(
- new ScheduleStatusResult().setTasks(IScheduledTask.toBuildersList(tasks))));
- }
-
- return response;
- }
-
- @Override
- public Response getJobSummary() {
- Set<IScheduledTask> tasks = Storage.Util.weaklyConsistentFetchTasks(storage, Query.unscoped());
- Multimap<String, IJobKey> jobsByRole = Multimaps.index(
- FluentIterable.from(tasks).transform(Tasks.SCHEDULED_TO_JOB_KEY),
- JobKeys.TO_ROLE);
-
- Multimap<String, IJobKey> cronJobsByRole = Multimaps.index(
- FluentIterable.from(cronJobManager.getJobs()).transform(JobKeys.FROM_CONFIG),
- JobKeys.TO_ROLE);
-
- List<JobSummary> jobSummaries = Lists.newLinkedList();
- for (String role : Sets.union(jobsByRole.keySet(), cronJobsByRole.keySet())) {
- JobSummary summary = new JobSummary();
- summary.setRole(role);
- summary.setJobCount(jobsByRole.get(role).size());
- summary.setCronJobCount(cronJobsByRole.get(role).size());
- jobSummaries.add(summary);
- }
-
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.jobSummaryResult(new JobSummaryResult(jobSummaries)));
- }
-
- @Override
- public Response getJobs(@Nullable String maybeNullRole) {
- Optional<String> ownerRole = Optional.fromNullable(maybeNullRole);
-
-
- // Ensure we only return one JobConfiguration for each JobKey.
- Map<IJobKey, IJobConfiguration> jobs = Maps.newHashMap();
-
- // Query the task store, find immediate jobs, and synthesize a JobConfiguration for them.
- // This is necessary because the ImmediateJobManager doesn't store jobs directly and
- // ImmediateJobManager#getJobs always returns an empty Collection.
- Query.Builder scope = ownerRole.isPresent()
- ? Query.roleScoped(ownerRole.get())
- : Query.unscoped();
- Multimap<IJobKey, IScheduledTask> tasks =
- Tasks.byJobKey(Storage.Util.weaklyConsistentFetchTasks(storage, scope.active()));
-
- jobs.putAll(Maps.transformEntries(tasks.asMap(),
- new Maps.EntryTransformer<IJobKey, Collection<IScheduledTask>, IJobConfiguration>() {
- @Override
- public IJobConfiguration transformEntry(
- IJobKey jobKey,
- Collection<IScheduledTask> tasks) {
-
- // Pick an arbitrary task for each immediate job. The chosen task might not be the most
- // recent if the job is in the middle of an update or some shards have been selectively
- // created.
- TaskConfig firstTask = tasks.iterator().next().getAssignedTask().getTask().newBuilder();
- return IJobConfiguration.build(new JobConfiguration()
- .setKey(jobKey.newBuilder())
- .setOwner(firstTask.getOwner())
- .setTaskConfig(firstTask)
- .setInstanceCount(tasks.size()));
- }
- }));
-
- // Get cron jobs directly from the manager. Do this after querying the task store so the real
- // template JobConfiguration for a cron job will overwrite the synthesized one that could have
- // been created above.
- Predicate<IJobConfiguration> configFilter = ownerRole.isPresent()
- ? Predicates.compose(Predicates.equalTo(ownerRole.get()), JobKeys.CONFIG_TO_ROLE)
- : Predicates.<IJobConfiguration>alwaysTrue();
- jobs.putAll(Maps.uniqueIndex(
- FluentIterable.from(cronJobManager.getJobs()).filter(configFilter),
- JobKeys.FROM_CONFIG));
-
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.getJobsResult(new GetJobsResult()
- .setConfigs(IJobConfiguration.toBuildersSet(jobs.values()))));
- }
-
- private void validateLockForTasks(Optional<ILock> lock, Iterable<IScheduledTask> tasks)
- throws LockException {
-
- ImmutableSet<IJobKey> uniqueKeys = FluentIterable.from(tasks)
- .transform(Tasks.SCHEDULED_TO_JOB_KEY)
- .toSet();
-
- // Validate lock against every unique job key derived from the tasks.
- for (IJobKey key : uniqueKeys) {
- lockManager.validateIfLocked(ILockKey.build(LockKey.job(key.newBuilder())), lock);
- }
- }
-
- private SessionContext validateSessionKeyForTasks(
- SessionKey session,
- TaskQuery taskQuery,
- Iterable<IScheduledTask> tasks) throws AuthFailedException {
-
- // Authenticate the session against any affected roles, always including the role for a
- // role-scoped query. This papers over the implementation detail that dormant cron jobs are
- // authenticated this way.
- ImmutableSet.Builder<String> targetRoles = ImmutableSet.<String>builder()
- .addAll(FluentIterable.from(tasks).transform(GET_ROLE));
- if (taskQuery.isSetOwner()) {
- targetRoles.add(taskQuery.getOwner().getRole());
- }
- return sessionValidator.checkAuthenticated(session, targetRoles.build());
- }
-
- private Optional<SessionContext> isAdmin(SessionKey session) {
- try {
- return Optional.of(
- sessionValidator.checkAuthorized(session, Capability.ROOT, AuditCheck.REQUIRED));
- } catch (AuthFailedException e) {
- return Optional.absent();
- }
- }
-
- @Override
- public Response killTasks(final TaskQuery query, Lock mutablelock, SessionKey session) {
- checkNotNull(query);
- checkNotNull(session);
-
- Response response = new Response();
-
- if (query.getJobName() != null && StringUtils.isBlank(query.getJobName())) {
- response.setResponseCode(INVALID_REQUEST).setMessage(
- String.format("Invalid job name: '%s'", query.getJobName()));
- return response;
- }
-
- Set<IScheduledTask> tasks = Storage.Util.consistentFetchTasks(storage, Query.arbitrary(query));
-
- Optional<SessionContext> context = isAdmin(session);
- if (context.isPresent()) {
- LOG.info("Granting kill query to admin user: " + query);
- } else {
- try {
- context = Optional.of(validateSessionKeyForTasks(session, query, tasks));
- } catch (AuthFailedException e) {
- response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- return response;
- }
- }
-
- try {
- validateLockForTasks(Optional.fromNullable(mutablelock).transform(ILock.FROM_BUILDER), tasks);
- schedulerCore.killTasks(Query.arbitrary(query), context.get().getIdentity());
- } catch (LockException e) {
- return response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
- } catch (ScheduleException e) {
- return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
- }
-
- // TODO(William Farner): Move this into the client.
- BackoffHelper backoff = new BackoffHelper(killTaskInitialBackoff, killTaskMaxBackoff, true);
- final Query.Builder activeQuery = Query.arbitrary(query.setStatuses(Tasks.ACTIVE_STATES));
- try {
- backoff.doUntilSuccess(new Supplier<Boolean>() {
- @Override public Boolean get() {
- Set<IScheduledTask> tasks = Storage.Util.consistentFetchTasks(storage, activeQuery);
- if (tasks.isEmpty()) {
- LOG.info("Tasks all killed, done waiting.");
- return true;
- } else {
- LOG.info("Jobs not yet killed, waiting...");
- return false;
- }
- }
- });
- response.setResponseCode(OK).setMessage("Tasks killed.");
- } catch (InterruptedException e) {
- LOG.warning("Interrupted while trying to kill tasks: " + e);
- Thread.currentThread().interrupt();
- response.setResponseCode(ERROR).setMessage("killTasks thread was interrupted.");
- } catch (BackoffHelper.BackoffStoppedException e) {
- response.setResponseCode(ERROR).setMessage("Tasks were not killed in time.");
- }
- return response;
- }
-
- @Override
- public Response restartShards(
- JobKey mutableJobKey,
- Set<Integer> shardIds,
- Lock mutableLock,
- SessionKey session) {
-
- IJobKey jobKey = JobKeys.assertValid(IJobKey.build(mutableJobKey));
- MorePreconditions.checkNotBlank(shardIds);
- checkNotNull(session);
-
- Response response = new Response();
- SessionContext context;
- try {
- context = sessionValidator.checkAuthenticated(session, ImmutableSet.of(jobKey.getRole()));
- } catch (AuthFailedException e) {
- response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- return response;
- }
-
- try {
- lockManager.validateIfLocked(
- ILockKey.build(LockKey.job(jobKey.newBuilder())),
- Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
- schedulerCore.restartShards(jobKey, shardIds, context.getIdentity());
- response.setResponseCode(OK).setMessage("Shards are restarting.");
- } catch (LockException e) {
- response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
- } catch (ScheduleException e) {
- response.setResponseCode(ResponseCode.INVALID_REQUEST).setMessage(e.getMessage());
- }
-
- return response;
- }
-
- @Override
- public Response getQuota(final String ownerRole) {
- checkNotBlank(ownerRole);
-
- IQuota quota = storage.consistentRead(new Work.Quiet<IQuota>() {
- @Override public IQuota apply(StoreProvider storeProvider) {
- return storeProvider.getQuotaStore().fetchQuota(ownerRole).or(Quotas.noQuota());
- }
- });
-
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.getQuotaResult(new GetQuotaResult()
- .setQuota(quota.newBuilder())));
- }
-
- @Override
- public Response startMaintenance(Hosts hosts, SessionKey session) {
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.startMaintenanceResult(new StartMaintenanceResult()
- .setStatuses(maintenance.startMaintenance(hosts.getHostNames()))));
- }
-
- @Override
- public Response drainHosts(Hosts hosts, SessionKey session) {
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.drainHostsResult(new DrainHostsResult()
- .setStatuses(maintenance.drain(hosts.getHostNames()))));
- }
-
- @Override
- public Response maintenanceStatus(Hosts hosts, SessionKey session) {
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.maintenanceStatusResult(new MaintenanceStatusResult()
- .setStatuses(maintenance.getStatus(hosts.getHostNames()))));
- }
-
- @Override
- public Response endMaintenance(Hosts hosts, SessionKey session) {
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.endMaintenanceResult(new EndMaintenanceResult()
- .setStatuses(maintenance.endMaintenance(hosts.getHostNames()))));
- }
-
- @Requires(whitelist = Capability.PROVISIONER)
- @Override
- public Response setQuota(final String ownerRole, final Quota quota, SessionKey session) {
- checkNotBlank(ownerRole);
- checkNotNull(quota);
- checkNotNull(session);
-
- // TODO(Kevin Sweeney): Input validation for Quota.
-
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- storeProvider.getQuotaStore().saveQuota(ownerRole, IQuota.build(quota));
- }
- });
-
- return new Response().setResponseCode(OK).setMessage("Quota applied.");
- }
-
- @Override
- public Response forceTaskState(
- String taskId,
- ScheduleStatus status,
- SessionKey session) {
-
- checkNotBlank(taskId);
- checkNotNull(status);
- checkNotNull(session);
-
- Response response = new Response();
- SessionContext context;
- try {
- // TODO(Sathya): Remove this after AOP-style session validation passes in a SessionContext.
- context = sessionValidator.checkAuthorized(session, Capability.ROOT, AuditCheck.REQUIRED);
- } catch (AuthFailedException e) {
- response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- return response;
- }
-
- schedulerCore.setTaskStatus(
- Query.taskScoped(taskId), status, transitionMessage(context.getIdentity()));
- return new Response().setResponseCode(OK).setMessage("Transition attempted.");
- }
-
- @Override
- public Response performBackup(SessionKey session) {
- backup.backupNow();
- return new Response().setResponseCode(OK);
- }
-
- @Override
- public Response listBackups(SessionKey session) {
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.listBackupsResult(new ListBackupsResult()
- .setBackups(recovery.listBackups())));
- }
-
- @Override
- public Response stageRecovery(String backupId, SessionKey session) {
- Response response = new Response().setResponseCode(OK);
- try {
- recovery.stage(backupId);
- } catch (RecoveryException e) {
- response.setResponseCode(ERROR).setMessage(e.getMessage());
- LOG.log(Level.WARNING, "Failed to stage recovery: " + e, e);
- }
-
- return response;
- }
-
- @Override
- public Response queryRecovery(TaskQuery query, SessionKey session) {
- Response response = new Response();
- try {
- response.setResponseCode(OK)
- .setResult(Result.queryRecoveryResult(new QueryRecoveryResult()
- .setTasks(IScheduledTask.toBuildersSet(recovery.query(Query.arbitrary(query))))));
- } catch (RecoveryException e) {
- response.setResponseCode(ERROR).setMessage(e.getMessage());
- LOG.log(Level.WARNING, "Failed to query recovery: " + e, e);
- }
-
- return response;
- }
-
- @Override
- public Response deleteRecoveryTasks(TaskQuery query, SessionKey session) {
- Response response = new Response().setResponseCode(OK);
- try {
- recovery.deleteTasks(Query.arbitrary(query));
- } catch (RecoveryException e) {
- response.setResponseCode(ERROR).setMessage(e.getMessage());
- LOG.log(Level.WARNING, "Failed to delete recovery tasks: " + e, e);
- }
-
- return response;
- }
-
- @Override
- public Response commitRecovery(SessionKey session) {
- Response response = new Response().setResponseCode(OK);
- try {
- recovery.commit();
- } catch (RecoveryException e) {
- response.setResponseCode(ERROR).setMessage(e.getMessage());
- }
-
- return response;
- }
-
- @Override
- public Response unloadRecovery(SessionKey session) {
- recovery.unload();
- return new Response().setResponseCode(OK);
- }
-
- @Override
- public Response snapshot(SessionKey session) {
- Response response = new Response();
- try {
- storage.snapshot();
- return response.setResponseCode(OK).setMessage("Compaction successful.");
- } catch (Storage.StorageException e) {
- LOG.log(Level.WARNING, "Requested snapshot failed.", e);
- return response.setResponseCode(ERROR).setMessage(e.getMessage());
- }
- }
-
- private static Multimap<String, IJobConfiguration> jobsByKey(JobStore jobStore, IJobKey jobKey) {
- ImmutableMultimap.Builder<String, IJobConfiguration> matches = ImmutableMultimap.builder();
- for (String managerId : jobStore.fetchManagerIds()) {
- for (IJobConfiguration job : jobStore.fetchJobs(managerId)) {
- if (job.getKey().equals(jobKey)) {
- matches.put(managerId, job);
- }
- }
- }
- return matches.build();
- }
-
- @Override
- public Response rewriteConfigs(
- final RewriteConfigsRequest request,
- SessionKey session) {
-
- if (request.getRewriteCommandsSize() == 0) {
- return new Response()
- .setResponseCode(ResponseCode.ERROR)
- .setMessage("No rewrite commands provided.");
- }
-
- return storage.write(new MutateWork.Quiet<Response>() {
- @Override public Response apply(MutableStoreProvider storeProvider) {
- List<String> errors = Lists.newArrayList();
-
- for (ConfigRewrite command : request.getRewriteCommands()) {
- Optional<String> error = rewriteConfig(command, storeProvider);
- if (error.isPresent()) {
- errors.add(error.get());
- }
- }
-
- Response resp = new Response();
- if (!errors.isEmpty()) {
- resp.setResponseCode(ResponseCode.WARNING).setMessage(Joiner.on(", ").join(errors));
- } else {
- resp.setResponseCode(OK).setMessage("All rewrites completed successfully.");
- }
- return resp;
- }
- });
- }
-
- private Optional<String> rewriteConfig(
- ConfigRewrite command,
- MutableStoreProvider storeProvider) {
-
- Optional<String> error = Optional.absent();
- switch (command.getSetField()) {
- case JOB_REWRITE:
- JobConfigRewrite jobRewrite = command.getJobRewrite();
- IJobConfiguration existingJob = IJobConfiguration.build(jobRewrite.getOldJob());
- IJobConfiguration rewrittenJob;
- try {
- rewrittenJob = ConfigurationManager.validateAndPopulate(
- IJobConfiguration.build(jobRewrite.getRewrittenJob()));
- } catch (TaskDescriptionException e) {
- // We could add an error here, but this is probably a hint of something wrong in
- // the client that's causing a bad configuration to be applied.
- throw Throwables.propagate(e);
- }
- if (!existingJob.getKey().equals(rewrittenJob.getKey())) {
- error = Optional.of("Disallowing rewrite attempting to change job key.");
- } else if (!existingJob.getOwner().equals(rewrittenJob.getOwner())) {
- error = Optional.of("Disallowing rewrite attempting to change job owner.");
- } else {
- JobStore.Mutable jobStore = storeProvider.getJobStore();
- Multimap<String, IJobConfiguration> matches =
- jobsByKey(jobStore, existingJob.getKey());
- switch (matches.size()) {
- case 0:
- error = Optional.of("No jobs found for key " + JobKeys.toPath(existingJob));
- break;
-
- case 1:
- Map.Entry<String, IJobConfiguration> match =
- Iterables.getOnlyElement(matches.entries());
- IJobConfiguration storedJob = match.getValue();
- if (!storedJob.equals(existingJob)) {
- error = Optional.of("CAS compare failed for " + JobKeys.toPath(storedJob));
- } else {
- jobStore.saveAcceptedJob(match.getKey(), rewrittenJob);
- }
- break;
-
- default:
- error = Optional.of("Multiple jobs found for key " + JobKeys.toPath(existingJob));
- }
- }
- break;
-
- case INSTANCE_REWRITE:
- InstanceConfigRewrite instanceRewrite = command.getInstanceRewrite();
- InstanceKey instanceKey = instanceRewrite.getInstanceKey();
- Iterable<IScheduledTask> tasks = storeProvider.getTaskStore().fetchTasks(
- Query.instanceScoped(IJobKey.build(instanceKey.getJobKey()),
- instanceKey.getInstanceId())
- .active());
- Optional<IAssignedTask> task =
- Optional.fromNullable(Iterables.getOnlyElement(tasks, null))
- .transform(Tasks.SCHEDULED_TO_ASSIGNED);
- if (!task.isPresent()) {
- error = Optional.of("No active task found for " + instanceKey);
- } else if (!task.get().getTask().newBuilder().equals(instanceRewrite.getOldTask())) {
- error = Optional.of("CAS compare failed for " + instanceKey);
- } else {
- ITaskConfig newConfiguration = ITaskConfig.build(
- ConfigurationManager.applyDefaultsIfUnset(instanceRewrite.getRewrittenTask()));
- boolean changed = storeProvider.getUnsafeTaskStore().unsafeModifyInPlace(
- task.get().getTaskId(), newConfiguration);
- if (!changed) {
- error = Optional.of("Did not change " + task.get().getTaskId());
- }
- }
- break;
-
- default:
- throw new IllegalArgumentException("Unhandled command type " + command.getSetField());
- }
-
- return error;
- }
-
- @Override
- public Response getVersion() {
- return new Response()
- .setResponseCode(OK)
- .setResult(Result.getVersionResult(CURRENT_API_VERSION));
- }
-
- @Override
- public Response addInstances(
- AddInstancesConfig config,
- @Nullable Lock mutableLock,
- SessionKey session) {
-
- checkNotNull(config);
- checkNotNull(session);
- checkNotBlank(config.getInstanceIds());
- IJobKey jobKey = JobKeys.assertValid(IJobKey.build(config.getKey()));
-
- Response resp = new Response();
- try {
- sessionValidator.checkAuthenticated(session, ImmutableSet.of(jobKey.getRole()));
- ITaskConfig task = ConfigurationManager.validateAndPopulate(
- ITaskConfig.build(config.getTaskConfig()));
-
- if (cronJobManager.hasJob(jobKey)) {
- return resp.setResponseCode(INVALID_REQUEST)
- .setMessage("Cron jobs are not supported here.");
- }
-
- lockManager.validateIfLocked(
- ILockKey.build(LockKey.job(jobKey.newBuilder())),
- Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
-
- schedulerCore.addInstances(jobKey, ImmutableSet.copyOf(config.getInstanceIds()), task);
- return resp.setResponseCode(OK).setMessage("Successfully added instances.");
- } catch (AuthFailedException e) {
- return resp.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- } catch (LockException e) {
- return resp.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
- } catch (TaskDescriptionException | ScheduleException e) {
- return resp.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
- }
- }
-
- private String getRoleFromLockKey(ILockKey lockKey) {
- switch (lockKey.getSetField()) {
- case JOB:
- JobKeys.assertValid(lockKey.getJob());
- return lockKey.getJob().getRole();
- default:
- throw new IllegalArgumentException("Unhandled LockKey: " + lockKey.getSetField());
- }
- }
-
- @Override
- public Response acquireLock(LockKey mutableLockKey, SessionKey session) {
- checkNotNull(mutableLockKey);
- checkNotNull(session);
-
- ILockKey lockKey = ILockKey.build(mutableLockKey);
- Response response = new Response();
-
- try {
- SessionContext context = sessionValidator.checkAuthenticated(
- session,
- ImmutableSet.of(getRoleFromLockKey(lockKey)));
-
- ILock lock = lockManager.acquireLock(lockKey, context.getIdentity());
- response.setResult(Result.acquireLockResult(
- new AcquireLockResult().setLock(lock.newBuilder())));
-
- return response.setResponseCode(OK).setMessage("Lock has been acquired.");
- } catch (AuthFailedException e) {
- return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- } catch (LockException e) {
- return response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
- }
- }
-
- @Override
- public Response releaseLock(Lock mutableLock, LockValidation validation, SessionKey session) {
- checkNotNull(mutableLock);
- checkNotNull(validation);
- checkNotNull(session);
-
- Response response = new Response();
- ILock lock = ILock.build(mutableLock);
-
- try {
- sessionValidator.checkAuthenticated(
- session,
- ImmutableSet.of(getRoleFromLockKey(lock.getKey())));
-
- if (validation == LockValidation.CHECKED) {
- lockManager.validateIfLocked(lock.getKey(), Optional.of(lock));
- }
- lockManager.releaseLock(lock);
- return response.setResponseCode(OK).setMessage("Lock has been released.");
- } catch (AuthFailedException e) {
- return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
- } catch (LockException e) {
- return response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
- }
- }
-
- @VisibleForTesting
- static Optional<String> transitionMessage(String user) {
- return Optional.of("Transition forced by " + user);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftConfiguration.java b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftConfiguration.java
deleted file mode 100644
index c6e9b18..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftConfiguration.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import com.google.common.base.Optional;
-
-/**
- * Container for thrift server configuration options.
- */
-public interface ThriftConfiguration {
- /**
- * Gets a stream for the thrift socket SSL key if this server is configured to use SSL.
- *
- * @return A stream that contains the SSL key data if SSL is enabled, absent otherwise.
- * @throws IOException If the stream could not be opened.
- */
- Optional<? extends InputStream> getSslKeyStream() throws IOException;
-
- /**
- * Gets the port that the thrift server should listen on.
- *
- * @return Thrift server port.
- */
- int getServingPort();
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java
deleted file mode 100644
index cca9053..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftModule.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift;
-
-import javax.inject.Singleton;
-
-import com.google.inject.AbstractModule;
-
-import com.twitter.aurora.gen.AuroraAdmin;
-import com.twitter.aurora.scheduler.thrift.aop.AopModule;
-import com.twitter.common.application.http.Registration;
-import com.twitter.common.application.modules.LifecycleModule;
-
-/**
- * Binding module to configure a thrift server.
- */
-public class ThriftModule extends AbstractModule {
-
- @Override
- protected void configure() {
- bind(AuroraAdmin.Iface.class).to(SchedulerThriftInterface.class);
- bind(ThriftServer.class).in(Singleton.class);
- LifecycleModule.bindServiceRunner(binder(), ThriftServerLauncher.class);
-
- Registration.registerServlet(binder(), "/api", SchedulerAPIServlet.class, true);
-
- install(new AopModule());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java
deleted file mode 100644
index 7b9abd1..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServer.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift;
-
-import java.net.ServerSocket;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import org.apache.thrift.TProcessor;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.server.TServer;
-import org.apache.thrift.server.TThreadPoolServer;
-import org.apache.thrift.transport.TServerSocket;
-
-import com.twitter.thrift.Status;
-
-class ThriftServer {
- private static final Logger LOG = Logger.getLogger(ThriftServer.class.getName());
-
- private TServer server = null;
-
- // Current health status of the server.
- private Status status = Status.STARTING;
-
- /**
- * Starts the server.
- * This may be called at any point except when the server is already alive. That is, it's
- * allowable to start, stop, and re-start the server.
- *
- * @param socket The socket to use.
- * @param processor The processor to handle requests.
- */
- public synchronized void start(ServerSocket socket, TProcessor processor) {
- Preconditions.checkNotNull(socket);
- Preconditions.checkNotNull(processor);
- Preconditions.checkState(status != Status.ALIVE, "Server must only be started once.");
- setStatus(Status.ALIVE);
- TThreadPoolServer.Args args = new TThreadPoolServer.Args(new TServerSocket(socket))
- .processor(processor)
- .protocolFactory(new TBinaryProtocol.Factory(false, true));
-
- final TServer starting = new TThreadPoolServer(args);
- server = starting;
- LOG.info("Starting thrift server on port " + socket.getLocalPort());
-
- Thread listeningThread = new ThreadFactoryBuilder().setDaemon(false).build()
- .newThread(new Runnable() {
- @Override public void run() {
- try {
- starting.serve();
- } catch (Throwable t) {
- LOG.log(Level.WARNING,
- "Uncaught exception while attempting to handle service requests: " + t, t);
- setStatus(Status.DEAD);
- }
- }
- });
-
- listeningThread.start();
- }
-
- private synchronized void setStatus(Status status) {
- LOG.info("Moving from status " + this.status + " to " + status);
- this.status = status;
- }
-
- /**
- * Attempts to shut down the server.
- * The server may be shut down at any time, though the request will be ignored if the server is
- * already stopped.
- */
- public synchronized void shutdown() {
- if (status == Status.STOPPED) {
- LOG.info("Server already stopped, shutdown request ignored.");
- return;
- }
-
- LOG.info("Received shutdown request, stopping server.");
- setStatus(Status.STOPPING);
-
- // TODO(William Farner): Figure out what happens to queued / in-process requests when the server
- // is stopped. Might want to allow a sleep period for the active requests to be completed.
-
- if (server != null) {
- server.stop();
- }
-
- server = null;
- setStatus(Status.STOPPED);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java b/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java
deleted file mode 100644
index 6743060..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/ThriftServerLauncher.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.ServerSocket;
-import java.security.GeneralSecurityException;
-import java.security.KeyStore;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLServerSocket;
-import javax.net.ssl.SSLServerSocketFactory;
-
-import com.google.common.base.Optional;
-
-import com.twitter.aurora.gen.AuroraAdmin;
-import com.twitter.aurora.gen.AuroraAdmin.Iface;
-import com.twitter.common.application.modules.LifecycleModule.ServiceRunner;
-import com.twitter.common.application.modules.LocalServiceRegistry.LocalService;
-import com.twitter.common.base.Command;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Service launcher that starts up and registers the scheduler thrift server as a primary service
- * for the application.
- */
-class ThriftServerLauncher implements ServiceRunner {
-
- private static final Logger LOG = Logger.getLogger(ThriftServerLauncher.class.getName());
-
- private final ThriftConfiguration configuration;
-
- // Security is enforced via file permissions, not via this password, for what it's worth.
- private static final String SSL_KEYFILE_PASSWORD = "MesosKeyStorePassword";
-
- private final Iface schedulerThriftInterface;
- private final ThriftServer schedulerThriftServer;
-
- @Inject
- ThriftServerLauncher(
- Iface schedulerThriftInterface,
- ThriftServer schedulerThriftServer,
- ThriftConfiguration configuration) {
-
- this.schedulerThriftInterface = checkNotNull(schedulerThriftInterface);
- this.schedulerThriftServer = checkNotNull(schedulerThriftServer);
- this.configuration = checkNotNull(configuration);
- }
-
- @Override
- public LocalService launch() {
- ServerSocket socket = getServerSocket();
- schedulerThriftServer.start(
- socket,
- new AuroraAdmin.Processor<>(schedulerThriftInterface));
-
- Command shutdown = new Command() {
- @Override public void execute() {
- LOG.info("Stopping thrift server.");
- schedulerThriftServer.shutdown();
- }
- };
-
- return LocalService.primaryService(socket.getLocalPort(), shutdown);
- }
-
- private ServerSocket getServerSocket() {
- try {
- Optional<? extends InputStream> sslKeyStream = configuration.getSslKeyStream();
- if (!sslKeyStream.isPresent()) {
- LOG.warning("Running Thrift Server without SSL.");
- return new ServerSocket(configuration.getServingPort());
- } else {
- // TODO(Kevin Sweeney): Add helper to perform this keyfile import.
- KeyStore ks = KeyStore.getInstance("JKS");
- ks.load(sslKeyStream.get(), SSL_KEYFILE_PASSWORD.toCharArray());
-
- KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
- kmf.init(ks, SSL_KEYFILE_PASSWORD.toCharArray());
-
- SSLContext ctx = SSLContext.getInstance("TLS");
- ctx.init(kmf.getKeyManagers(), null, null);
-
- SSLServerSocketFactory ssf = ctx.getServerSocketFactory();
- SSLServerSocket serverSocket = (SSLServerSocket) ssf.createServerSocket(
- configuration.getServingPort());
- serverSocket.setEnabledCipherSuites(serverSocket.getSupportedCipherSuites());
- serverSocket.setNeedClientAuth(false);
- return serverSocket;
- }
- } catch (IOException e) {
- throw new RuntimeException("Failed to read key file.", e);
- } catch (GeneralSecurityException e) {
- throw new RuntimeException("SSL setup failed.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
deleted file mode 100644
index d66a2b2..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package com.twitter.aurora.scheduler.thrift.aop;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-
-import com.twitter.aurora.gen.Response;
-
-import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
-
-class APIVersionInterceptor implements MethodInterceptor {
-
- @Override
- public Object invoke(MethodInvocation invocation) throws Throwable {
- Response resp = (Response) invocation.proceed();
- if (resp.version == null) {
- resp.setVersion(CURRENT_API_VERSION);
- }
- return resp;
- }
-}
[44/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/periodic/GcExecutorLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/periodic/GcExecutorLauncher.java b/src/main/java/com/twitter/aurora/scheduler/periodic/GcExecutorLauncher.java
deleted file mode 100644
index df15854..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/periodic/GcExecutorLauncher.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.periodic;
-
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.collect.Maps;
-import com.google.protobuf.ByteString;
-
-import org.apache.mesos.Protos.ExecutorID;
-import org.apache.mesos.Protos.ExecutorInfo;
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.TaskID;
-import org.apache.mesos.Protos.TaskInfo;
-import org.apache.mesos.Protos.TaskStatus;
-
-import com.twitter.aurora.Protobufs;
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.comm.AdjustRetainedTasks;
-import com.twitter.aurora.scheduler.TaskLauncher;
-import com.twitter.aurora.scheduler.base.CommandUtil;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.Resources;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Data;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-import com.twitter.common.util.Random;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A task launcher that periodically initiates garbage collection on a host, re-using a single
- * garbage collection executor
- */
-public class GcExecutorLauncher implements TaskLauncher {
- private static final Logger LOG = Logger.getLogger(GcExecutorLauncher.class.getName());
-
- private final AtomicLong tasksCreated = Stats.exportLong("scheduler_gc_tasks_created");
-
- @VisibleForTesting
- static final Resources TOTAL_GC_EXECUTOR_RESOURCES =
- new Resources(0.2, Amount.of(128L, Data.MB), Amount.of(16L, Data.MB), 0);
-
- // An epsilon is used because we are required to supply executor and task resources.
- @VisibleForTesting
- static final Resources EPSILON =
- new Resources(0.01, Amount.of(1L, Data.MB), Amount.of(1L, Data.MB), 0);
-
- private static final Resources GC_EXECUTOR_RESOURCES =
- Resources.subtract(TOTAL_GC_EXECUTOR_RESOURCES, EPSILON);
-
- private static final String SYSTEM_TASK_PREFIX = "system-gc-";
- private static final String EXECUTOR_NAME = "aurora.gc";
-
- private final GcExecutorSettings settings;
- private final Storage storage;
- private final Clock clock;
- private final Cache<String, Long> pulses;
-
- @Inject
- GcExecutorLauncher(
- GcExecutorSettings settings,
- Storage storage,
- Clock clock) {
-
- this.settings = checkNotNull(settings);
- this.storage = checkNotNull(storage);
- this.clock = checkNotNull(clock);
-
- this.pulses = CacheBuilder.newBuilder()
- .expireAfterWrite(settings.getMaxGcInterval(), TimeUnit.MILLISECONDS)
- .build();
- }
-
- @Override
- public Optional<TaskInfo> createTask(Offer offer) {
- if (!settings.getGcExecutorPath().isPresent()
- || !Resources.from(offer).greaterThanOrEqual(TOTAL_GC_EXECUTOR_RESOURCES)
- || isAlive(offer.getHostname())) {
- return Optional.absent();
- }
-
- Set<IScheduledTask> tasksOnHost =
- Storage.Util.weaklyConsistentFetchTasks(storage, Query.slaveScoped(offer.getHostname()));
- AdjustRetainedTasks message = new AdjustRetainedTasks()
- .setRetainedTasks(Maps.transformValues(Tasks.mapById(tasksOnHost), Tasks.GET_STATUS));
- byte[] data;
- try {
- data = ThriftBinaryCodec.encode(message);
- } catch (CodingException e) {
- LOG.severe("Failed to encode retained tasks message: " + message);
- return Optional.absent();
- }
-
- tasksCreated.incrementAndGet();
- pulses.put(offer.getHostname(), clock.nowMillis() + settings.getDelayMs());
-
- ExecutorInfo.Builder executor = ExecutorInfo.newBuilder()
- .setExecutorId(ExecutorID.newBuilder().setValue(EXECUTOR_NAME))
- .setName(EXECUTOR_NAME)
- .setSource(offer.getHostname())
- .addAllResources(GC_EXECUTOR_RESOURCES.toResourceList())
- .setCommand(CommandUtil.create(settings.getGcExecutorPath().get()));
-
- return Optional.of(TaskInfo.newBuilder().setName("system-gc")
- .setTaskId(TaskID.newBuilder().setValue(SYSTEM_TASK_PREFIX + UUID.randomUUID().toString()))
- .setSlaveId(offer.getSlaveId())
- .setData(ByteString.copyFrom(data))
- .setExecutor(executor)
- .addAllResources(EPSILON.toResourceList())
- .build());
- }
-
- @Override
- public boolean statusUpdate(TaskStatus status) {
- if (status.getTaskId().getValue().startsWith(SYSTEM_TASK_PREFIX)) {
- LOG.info("Received status update for GC task: " + Protobufs.toString(status));
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public void cancelOffer(OfferID offer) {
- // No-op.
- }
-
- private boolean isAlive(String hostname) {
- Optional<Long> timestamp = Optional.fromNullable(pulses.getIfPresent(hostname));
- return timestamp.isPresent() && clock.nowMillis() < timestamp.get();
- }
-
- /**
- * Wraps configuration values for the {@code GcExecutorLauncher}.
- */
- public static class GcExecutorSettings {
- private final Amount<Long, Time> gcInterval;
- private final Optional<String> gcExecutorPath;
- private final Random rand = new Random.SystemRandom(new java.util.Random());
-
- public GcExecutorSettings(
- Amount<Long, Time> gcInterval,
- Optional<String> gcExecutorPath) {
-
- this.gcInterval = checkNotNull(gcInterval);
- this.gcExecutorPath = checkNotNull(gcExecutorPath);
- }
-
- @VisibleForTesting
- long getMaxGcInterval() {
- return gcInterval.as(Time.MILLISECONDS);
- }
-
- @VisibleForTesting
- int getDelayMs() {
- return rand.nextInt(gcInterval.as(Time.MILLISECONDS).intValue());
- }
-
- @VisibleForTesting
- Optional<String> getGcExecutorPath() {
- return gcExecutorPath;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/quota/QuotaComparisonResult.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaComparisonResult.java b/src/main/java/com/twitter/aurora/scheduler/quota/QuotaComparisonResult.java
deleted file mode 100644
index 2ba2db8..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaComparisonResult.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.quota;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-
-/**
- * Calculates and formats detailed quota comparison result.
- */
-class QuotaComparisonResult {
-
- private enum Resource {
- CPU("core(s)"),
- RAM("MB"),
- DISK("MB");
-
- private final String unit;
- private Resource(String unit) {
- this.unit = unit;
- }
- }
-
- enum Result {
- SUFFICIENT_QUOTA,
- INSUFFICIENT_QUOTA
- }
-
- private final String details;
- private final Result result;
-
- @VisibleForTesting
- QuotaComparisonResult(Result result, String details) {
- this.result = result;
- this.details = details;
- }
-
- Result result() {
- return result;
- }
-
- String details() {
- return details;
- }
-
- static QuotaComparisonResult greaterOrEqual(IQuota a, IQuota b) {
- StringBuilder details = new StringBuilder();
- boolean result = compare(a.getNumCpus(), b.getNumCpus(), Resource.CPU, details)
- & compare(a.getRamMb(), b.getRamMb(), Resource.RAM, details)
- & compare(a.getDiskMb(), b.getDiskMb(), Resource.DISK, details);
-
- return new QuotaComparisonResult(
- result ? Result.SUFFICIENT_QUOTA : Result.INSUFFICIENT_QUOTA,
- details.toString());
- }
-
- private static boolean compare(
- double a,
- double b,
- Resource resource,
- StringBuilder details) {
-
- boolean result = a >= b;
- if (!result) {
- details
- .append(details.length() > 0 ? "; " : "")
- .append(resource)
- .append(" quota exceeded by ")
- .append(String.format("%.2f", b - a))
- .append(" ")
- .append(resource.unit);
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/quota/QuotaFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaFilter.java b/src/main/java/com/twitter/aurora/scheduler/quota/QuotaFilter.java
deleted file mode 100644
index cc67ba5..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaFilter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.quota;
-
-import javax.inject.Inject;
-
-import com.google.common.collect.Iterables;
-
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.quota.QuotaManager.QuotaManagerImpl;
-import com.twitter.aurora.scheduler.state.JobFilter;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.scheduler.quota.QuotaComparisonResult.Result.INSUFFICIENT_QUOTA;
-
-/**
- * A filter that fails production jobs for roles that do not have sufficient quota to run them.
- */
-class QuotaFilter implements JobFilter {
- private final QuotaManagerImpl quotaManager;
- private final Storage storage;
-
- @Inject
- QuotaFilter(QuotaManagerImpl quotaManager, Storage storage) {
- this.quotaManager = checkNotNull(quotaManager);
- this.storage = checkNotNull(storage);
- }
-
- @Override
- public JobFilterResult filter(final IJobConfiguration job) {
- return filterByTask(job.getKey(), job.getTaskConfig(), job.getInstanceCount());
- }
-
- @Override
- public JobFilterResult filter(ITaskConfig template, int instanceCount) {
- return filterByTask(JobKeys.from(template), template, instanceCount);
- }
-
- private synchronized JobFilterResult filterByTask(
- IJobKey jobKey,
- ITaskConfig template,
- int instanceCount) {
-
- if (!template.isProduction()) {
- return JobFilterResult.pass();
- }
-
- IQuota currentUsage = Quotas.fromProductionTasks(
- Iterables.transform(
- Storage.Util.consistentFetchTasks(storage, Query.jobScoped(jobKey).active()),
- Tasks.SCHEDULED_TO_INFO));
-
- IQuota additionalRequested =
- Quotas.subtract(Quotas.fromTasks(template, instanceCount), currentUsage);
- QuotaComparisonResult comparisonResult =
- quotaManager.checkQuota(jobKey.getRole(), additionalRequested);
-
- if (comparisonResult.result() == INSUFFICIENT_QUOTA) {
- return JobFilterResult.fail("Insufficient resource quota: " + comparisonResult.details());
- }
-
- return JobFilterResult.pass();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/quota/QuotaManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaManager.java b/src/main/java/com/twitter/aurora/scheduler/quota/QuotaManager.java
deleted file mode 100644
index a2abedb..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaManager.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.quota;
-
-import com.google.common.collect.Iterables;
-import com.google.inject.Inject;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.Storage.Work.Quiet;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * Allows access to resource quotas, and tracks quota consumption.
- */
-public interface QuotaManager {
- /**
- * Fetches the current resource usage for the role.
- *
- * @param role to fetch quota usage for.
- * @return Resource quota used by {@code role}.
- */
- IQuota getConsumption(String role);
-
- /**
- * Quota provider that stores quotas in the canonical {@link Storage} system.
- */
- static class QuotaManagerImpl implements QuotaManager {
- private final Storage storage;
-
- @Inject
- public QuotaManagerImpl(Storage storage) {
- this.storage = checkNotNull(storage);
- }
-
- @Override
- public IQuota getConsumption(final String role) {
- checkNotBlank(role);
-
- final Query.Builder query = Query.roleScoped(role).active();
-
- return storage.consistentRead(
- new Work.Quiet<IQuota>() {
- @Override public IQuota apply(StoreProvider storeProvider) {
- return Quotas.fromProductionTasks(Iterables.transform(
- storeProvider.getTaskStore().fetchTasks(query), Tasks.SCHEDULED_TO_INFO));
- }
- });
- }
-
- /**
- * Tests whether the role has at least the specified amount of quota available.
- *
- * @param role Role to consume quota for.
- * @param quota Quota amount to check for availability.
- * @return QuotaComparisonResult with {@code result()} returning {@code true} if the role
- * currently has at least {@code quota} quota remaining, {@code false} otherwise.
- */
- QuotaComparisonResult checkQuota(final String role, final IQuota quota) {
- checkNotBlank(role);
- checkNotNull(quota);
-
- return storage.consistentRead(new Quiet<QuotaComparisonResult>() {
- @Override public QuotaComparisonResult apply(StoreProvider storeProvider) {
- IQuota reserved = storeProvider.getQuotaStore().fetchQuota(role).or(Quotas.noQuota());
- return Quotas.greaterOrEqual(reserved, Quotas.add(getConsumption(role), quota));
- }
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/quota/QuotaModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaModule.java b/src/main/java/com/twitter/aurora/scheduler/quota/QuotaModule.java
deleted file mode 100644
index 63e8888..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/quota/QuotaModule.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.quota;
-
-import javax.inject.Singleton;
-
-import com.google.inject.AbstractModule;
-
-import com.twitter.aurora.scheduler.quota.QuotaManager.QuotaManagerImpl;
-import com.twitter.aurora.scheduler.state.JobFilter;
-import com.twitter.aurora.scheduler.storage.Storage;
-
-/**
- * Guice module for the quota package.
- */
-public class QuotaModule extends AbstractModule {
-
- @Override
- protected void configure() {
- requireBinding(Storage.class);
-
- bind(QuotaManager.class).to(QuotaManagerImpl.class);
- bind(QuotaManagerImpl.class).in(Singleton.class);
-
- bind(JobFilter.class).to(QuotaFilter.class);
- bind(QuotaFilter.class).in(Singleton.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/quota/Quotas.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/quota/Quotas.java b/src/main/java/com/twitter/aurora/scheduler/quota/Quotas.java
deleted file mode 100644
index f7b6591..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/quota/Quotas.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.quota;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-
-import com.twitter.aurora.gen.Quota;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Convenience class for normalizing resource measures between tasks and offers.
- */
-public final class Quotas {
- private static final IQuota NO_QUOTA = IQuota.build(new Quota(0, 0, 0));
-
- private Quotas() {
- // Utility class.
- }
-
- /**
- * Returns a quota with all resource vectors zeroed.
- *
- * @return A quota with all resource vectors zeroed.
- */
- public static IQuota noQuota() {
- return NO_QUOTA;
- }
-
- /**
- * Determines the amount of quota required for a set of job tasks.
- *
- * @param taskConfig Task template to count quota from.
- * @return Quota requirement to run {@code job}.
- */
- public static IQuota fromTasks(ITaskConfig taskConfig, int instanceCount) {
- return scale(fromProductionTasks(ImmutableSet.of(taskConfig)), instanceCount);
- }
-
- // TODO(Suman Karumuri): Refactor this function in to a new class.
- // TODO(Suman Karumuri): Rename Quota to something more meaningful (ex: ResourceAggregate)
- /**
- * Determines the amount of quota required for production tasks among {@code tasks}.
- *
- * @param tasks Tasks to count quota from.
- * @return Quota requirement to run {@code tasks}.
- */
- public static IQuota fromProductionTasks(Iterable<ITaskConfig> tasks) {
- checkNotNull(tasks);
-
- return fromTasks(Iterables.filter(tasks, Tasks.IS_PRODUCTION));
- }
-
- /**
- * Determines the amount of quota required for the given tasks.
- *
- * @param tasks Tasks to count quota from.
- * @return Quota requirement to run {@code tasks}.
- */
- public static IQuota fromTasks(Iterable<ITaskConfig> tasks) {
- double cpu = 0;
- int ramMb = 0;
- int diskMb = 0;
- for (ITaskConfig task : tasks) {
- cpu += task.getNumCpus();
- ramMb += task.getRamMb();
- diskMb += task.getDiskMb();
- }
-
- return IQuota.build(new Quota()
- .setNumCpus(cpu)
- .setRamMb(ramMb)
- .setDiskMb(diskMb));
- }
-
- /**
- * a >= b
- */
- public static QuotaComparisonResult greaterOrEqual(IQuota a, IQuota b) {
- return QuotaComparisonResult.greaterOrEqual(a, b);
- }
-
- /**
- * a + b
- */
- public static IQuota add(IQuota a, IQuota b) {
- return IQuota.build(new Quota()
- .setNumCpus(a.getNumCpus() + b.getNumCpus())
- .setRamMb(a.getRamMb() + b.getRamMb())
- .setDiskMb(a.getDiskMb() + b.getDiskMb()));
- }
-
- /**
- * a - b
- */
- public static IQuota subtract(IQuota a, IQuota b) {
- return IQuota.build(new Quota()
- .setNumCpus(a.getNumCpus() - b.getNumCpus())
- .setRamMb(a.getRamMb() - b.getRamMb())
- .setDiskMb(a.getDiskMb() - b.getDiskMb()));
- }
-
- /**
- * a * m
- */
- public static IQuota scale(IQuota a, int m) {
- return IQuota.build(new Quota()
- .setNumCpus(a.getNumCpus() * m)
- .setRamMb(a.getRamMb() * m)
- .setDiskMb(a.getDiskMb() * m));
- }
-
- /**
- * a / b
- * <p>
- * This calculates how many times {@code b} "fits into" {@code a}. Behavior is undefined when
- * {@code b} contains resources with a value of zero.
- */
- public static int divide(IQuota a, IQuota b) {
- return Ordering.natural().min(
- a.getNumCpus() / b.getNumCpus(),
- (double) a.getRamMb() / b.getRamMb(),
- (double) a.getDiskMb() / b.getDiskMb()
- ).intValue();
- }
-
- /**
- * sum(qs)
- */
- public static IQuota sum(Iterable<IQuota> qs) {
- IQuota sum = noQuota();
- for (IQuota q : qs) {
- sum = Quotas.add(sum, q);
- }
- return sum;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/CronJobManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/CronJobManager.java b/src/main/java/com/twitter/aurora/scheduler/state/CronJobManager.java
deleted file mode 100644
index d47c4af..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/CronJobManager.java
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Collections;
-import java.util.Date;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Ordering;
-import com.google.common.eventbus.Subscribe;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.twitter.aurora.gen.CronCollisionPolicy;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.ScheduleException;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
-import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
-import com.twitter.aurora.scheduler.cron.CronException;
-import com.twitter.aurora.scheduler.cron.CronScheduler;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.application.ShutdownRegistry;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.base.ExceptionalCommand;
-import com.twitter.common.base.Supplier;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.BackoffHelper;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
-
-/**
- * A job scheduler that receives jobs that should be run periodically on a cron schedule.
- */
-public class CronJobManager extends JobManager implements EventSubscriber {
-
- public static final String MANAGER_KEY = "CRON";
-
- @VisibleForTesting
- static final String CRON_USER = "cron";
-
- private static final Logger LOG = Logger.getLogger(CronJobManager.class.getName());
-
- @CmdLine(name = "cron_start_initial_backoff", help =
- "Initial backoff delay while waiting for a previous cron run to start.")
- private static final Arg<Amount<Long, Time>> CRON_START_INITIAL_BACKOFF =
- Arg.create(Amount.of(1L, Time.SECONDS));
-
- @CmdLine(name = "cron_start_max_backoff", help =
- "Max backoff delay while waiting for a previous cron run to start.")
- private static final Arg<Amount<Long, Time>> CRON_START_MAX_BACKOFF =
- Arg.create(Amount.of(1L, Time.MINUTES));
-
- private final AtomicLong cronJobsTriggered = Stats.exportLong("cron_jobs_triggered");
- private final AtomicLong cronJobLaunchFailures = Stats.exportLong("cron_job_launch_failures");
-
- // Maps from the unique job identifier to the unique identifier used internally by the cron
- // scheduler.
- private final Map<IJobKey, String> scheduledJobs =
- Collections.synchronizedMap(Maps.<IJobKey, String>newHashMap());
-
- // Prevents runs from dogpiling while waiting for a run to transition out of the KILLING state.
- // This is necessary because killing a job (if dictated by cron collision policy) is an
- // asynchronous operation.
- private final Map<IJobKey, SanitizedConfiguration> pendingRuns =
- Collections.synchronizedMap(Maps.<IJobKey, SanitizedConfiguration>newHashMap());
-
- private final StateManager stateManager;
- private final Storage storage;
- private final CronScheduler cron;
- private final ShutdownRegistry shutdownRegistry;
- private final BackoffHelper delayedStartBackoff;
- private final Executor delayedRunExecutor;
-
- @Inject
- CronJobManager(
- StateManager stateManager,
- Storage storage,
- CronScheduler cron,
- ShutdownRegistry shutdownRegistry) {
-
- this(
- stateManager,
- storage,
- cron,
- shutdownRegistry,
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("CronDelay-%d").build()));
- }
-
- @VisibleForTesting
- CronJobManager(
- StateManager stateManager,
- Storage storage,
- CronScheduler cron,
- ShutdownRegistry shutdownRegistry,
- Executor delayedRunExecutor) {
-
- this.stateManager = checkNotNull(stateManager);
- this.storage = checkNotNull(storage);
- this.cron = checkNotNull(cron);
- this.shutdownRegistry = checkNotNull(shutdownRegistry);
- this.delayedStartBackoff =
- new BackoffHelper(CRON_START_INITIAL_BACKOFF.get(), CRON_START_MAX_BACKOFF.get());
- this.delayedRunExecutor = checkNotNull(delayedRunExecutor);
-
- Stats.exportSize("cron_num_pending_runs", pendingRuns);
- }
-
- private void mapScheduledJob(IJobConfiguration job, String scheduledJobKey) {
- IJobKey jobKey = job.getKey();
- synchronized (scheduledJobs) {
- Preconditions.checkState(
- !scheduledJobs.containsKey(jobKey),
- "Illegal state - cron schedule already exists for " + JobKeys.toPath(jobKey));
- scheduledJobs.put(jobKey, scheduledJobKey);
- }
- }
-
- /**
- * Notifies the cron job manager that storage is started, and job configurations are ready to
- * load.
- *
- * @param storageStarted Event.
- */
- @Subscribe
- public void storageStarted(StorageStarted storageStarted) {
- cron.start();
- shutdownRegistry.addAction(new ExceptionalCommand<CronException>() {
- @Override public void execute() throws CronException {
- cron.stop();
- }
- });
-
- Iterable<IJobConfiguration> crons =
- storage.consistentRead(new Work.Quiet<Iterable<IJobConfiguration>>() {
- @Override public Iterable<IJobConfiguration> apply(Storage.StoreProvider storeProvider) {
- return storeProvider.getJobStore().fetchJobs(MANAGER_KEY);
- }
- });
-
- for (IJobConfiguration job : crons) {
- try {
- mapScheduledJob(job, scheduleJob(SanitizedConfiguration.fromUnsanitized(job)));
- } catch (ScheduleException | TaskDescriptionException e) {
- logLaunchFailure(job, e);
- }
- }
- }
-
- private void logLaunchFailure(IJobConfiguration job, Exception e) {
- cronJobLaunchFailures.incrementAndGet();
- LOG.log(Level.SEVERE, "Scheduling failed for recovered job " + job, e);
- }
-
- /**
- * Triggers execution of a job.
- *
- * @param jobKey Key of the job to start.
- */
- public void startJobNow(IJobKey jobKey) throws TaskDescriptionException {
- checkNotNull(jobKey);
-
- Optional<IJobConfiguration> jobConfig = fetchJob(jobKey);
- checkArgument(jobConfig.isPresent(), "No such cron job " + JobKeys.toPath(jobKey));
-
- cronTriggered(SanitizedConfiguration.fromUnsanitized(jobConfig.get()));
- }
-
- private void delayedRun(final Query.Builder query, final SanitizedConfiguration config) {
- IJobConfiguration job = config.getJobConfig();
- final String jobPath = JobKeys.toPath(job);
- final IJobKey jobKey = job.getKey();
- LOG.info("Waiting for job to terminate before launching cron job " + jobPath);
- if (pendingRuns.put(jobKey, config) == null) {
- LOG.info("Launching a task to wait for job to finish: " + jobPath);
- // There was no run already pending for this job, launch a task to delay launch until the
- // existing run has terminated.
- delayedRunExecutor.execute(new Runnable() {
- @Override public void run() {
- runWhenTerminated(query, jobKey);
- }
- });
- }
- }
-
- private void runWhenTerminated(final Query.Builder query, final IJobKey jobKey) {
- try {
- delayedStartBackoff.doUntilSuccess(new Supplier<Boolean>() {
- @Override public Boolean get() {
- if (!hasTasks(query)) {
- LOG.info("Initiating delayed launch of cron " + jobKey);
- SanitizedConfiguration config = pendingRuns.remove(jobKey);
- checkNotNull(config, "Failed to fetch job for delayed run of " + jobKey);
- LOG.info("Launching " + config.getTaskConfigs().size() + " tasks.");
- stateManager.insertPendingTasks(config.getTaskConfigs());
- return true;
- } else {
- LOG.info("Not yet safe to run cron " + jobKey);
- return false;
- }
- }
- });
- } catch (InterruptedException e) {
- LOG.log(Level.WARNING, "Interrupted while trying to launch cron " + jobKey, e);
- Thread.currentThread().interrupt();
- }
- }
-
- private boolean hasTasks(Query.Builder query) {
- return !Storage.Util.consistentFetchTasks(storage, query).isEmpty();
- }
-
- public static CronCollisionPolicy orDefault(@Nullable CronCollisionPolicy policy) {
- return Optional.fromNullable(policy).or(CronCollisionPolicy.KILL_EXISTING);
- }
-
- /**
- * Triggers execution of a cron job, depending on the cron collision policy for the job.
- *
- * @param config The config of the job to be triggered.
- */
- @VisibleForTesting
- void cronTriggered(SanitizedConfiguration config) {
- IJobConfiguration job = config.getJobConfig();
- LOG.info(String.format("Cron triggered for %s at %s with policy %s",
- JobKeys.toPath(job), new Date(), job.getCronCollisionPolicy()));
- cronJobsTriggered.incrementAndGet();
-
- ImmutableMap.Builder<Integer, ITaskConfig> builder = ImmutableMap.builder();
- final Query.Builder activeQuery = Query.jobScoped(job.getKey()).active();
- Set<IScheduledTask> activeTasks = Storage.Util.consistentFetchTasks(storage, activeQuery);
-
- if (activeTasks.isEmpty()) {
- builder.putAll(config.getTaskConfigs());
- } else {
- // Assign a default collision policy.
- CronCollisionPolicy collisionPolicy = orDefault(job.getCronCollisionPolicy());
-
- switch (collisionPolicy) {
- case KILL_EXISTING:
- try {
- schedulerCore.killTasks(activeQuery, CRON_USER);
- // Check immediately if the tasks are gone. This could happen if the existing tasks
- // were pending.
- if (!hasTasks(activeQuery)) {
- builder.putAll(config.getTaskConfigs());
- } else {
- delayedRun(activeQuery, config);
- }
- } catch (ScheduleException e) {
- LOG.log(Level.SEVERE, "Failed to kill job.", e);
- }
- break;
-
- case CANCEL_NEW:
- break;
-
- case RUN_OVERLAP:
- Map<Integer, IScheduledTask> byInstance =
- Maps.uniqueIndex(activeTasks, Tasks.SCHEDULED_TO_INSTANCE_ID);
- Map<Integer, ScheduleStatus> existingTasks =
- Maps.transformValues(byInstance, Tasks.GET_STATUS);
- if (existingTasks.isEmpty()) {
- builder.putAll(config.getTaskConfigs());
- } else if (Iterables.any(existingTasks.values(), Predicates.equalTo(PENDING))) {
- LOG.info("Job " + JobKeys.toPath(job) + " has pending tasks, suppressing run.");
- } else {
- // To safely overlap this run, we need to adjust the instance IDs of the overlapping
- // run (maintaining the role/job/instance UUID invariant).
- int instanceOffset = Ordering.natural().max(existingTasks.keySet()) + 1;
- LOG.info("Adjusting instance IDs of " + JobKeys.toPath(job) + " by " + instanceOffset
- + " for overlapping cron run.");
- for (Map.Entry<Integer, ITaskConfig> entry : config.getTaskConfigs().entrySet()) {
- builder.put(entry.getKey() + instanceOffset, entry.getValue());
- }
- }
- break;
-
- default:
- LOG.severe("Unrecognized cron collision policy: " + job.getCronCollisionPolicy());
- }
- }
-
- Map<Integer, ITaskConfig> newTasks = builder.build();
- if (!newTasks.isEmpty()) {
- stateManager.insertPendingTasks(newTasks);
- }
- }
-
- /**
- * Updates (re-schedules) the existing cron job.
- *
- * @param config New job configuration to update to.
- * @throws ScheduleException If non-cron job confuration provided.
- */
- public void updateJob(SanitizedConfiguration config) throws ScheduleException {
- IJobConfiguration job = config.getJobConfig();
- if (!hasCronSchedule(job)) {
- throw new ScheduleException("A cron job may not be updated to a non-cron job.");
- }
- String key = scheduledJobs.remove(job.getKey());
- checkNotNull(key, "Attempted to update unknown job " + JobKeys.toPath(job));
- cron.deschedule(key);
- checkArgument(receiveJob(config));
- }
-
- @Override
- public String getUniqueKey() {
- return MANAGER_KEY;
- }
-
- private static boolean hasCronSchedule(IJobConfiguration job) {
- checkNotNull(job);
- return !StringUtils.isEmpty(job.getCronSchedule());
- }
-
- @Override
- public boolean receiveJob(SanitizedConfiguration config) throws ScheduleException {
- final IJobConfiguration job = config.getJobConfig();
- if (!hasCronSchedule(job)) {
- return false;
- }
-
- String scheduledJobKey = scheduleJob(config);
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(Storage.MutableStoreProvider storeProvider) {
- storeProvider.getJobStore().saveAcceptedJob(MANAGER_KEY, job);
- }
- });
- mapScheduledJob(job, scheduledJobKey);
-
- return true;
- }
-
- private String scheduleJob(final SanitizedConfiguration config) throws ScheduleException {
- final IJobConfiguration job = config.getJobConfig();
- final String jobPath = JobKeys.toPath(job);
- if (!hasCronSchedule(job)) {
- throw new ScheduleException(
- String.format("Not a valid cronjob, %s has no cron schedule", jobPath));
- }
-
- if (!cron.isValidSchedule(job.getCronSchedule())) {
- throw new ScheduleException("Invalid cron schedule: " + job.getCronSchedule());
- }
-
- LOG.info(String.format("Scheduling cron job %s: %s", jobPath, job.getCronSchedule()));
- try {
- return cron.schedule(job.getCronSchedule(), new Runnable() {
- @Override public void run() {
- // TODO(William Farner): May want to record information about job runs.
- LOG.info("Running cron job: " + jobPath);
- cronTriggered(config);
- }
- });
- } catch (CronException e) {
- throw new ScheduleException("Failed to schedule cron job: " + e.getMessage(), e);
- }
- }
-
- @Override
- public Iterable<IJobConfiguration> getJobs() {
- return storage.consistentRead(new Work.Quiet<Iterable<IJobConfiguration>>() {
- @Override
- public Iterable<IJobConfiguration> apply(Storage.StoreProvider storeProvider) {
- return storeProvider.getJobStore().fetchJobs(MANAGER_KEY);
- }
- });
- }
-
- @Override
- public boolean hasJob(IJobKey jobKey) {
- return fetchJob(jobKey).isPresent();
- }
-
- private Optional<IJobConfiguration> fetchJob(final IJobKey jobKey) {
- checkNotNull(jobKey);
- return storage.consistentRead(new Work.Quiet<Optional<IJobConfiguration>>() {
- @Override public Optional<IJobConfiguration> apply(Storage.StoreProvider storeProvider) {
- return storeProvider.getJobStore().fetchJob(MANAGER_KEY, jobKey);
- }
- });
- }
-
- @Override
- public boolean deleteJob(final IJobKey jobKey) {
- checkNotNull(jobKey);
-
- if (!hasJob(jobKey)) {
- return false;
- }
-
- String scheduledJobKey = scheduledJobs.remove(jobKey);
- if (scheduledJobKey != null) {
- cron.deschedule(scheduledJobKey);
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(Storage.MutableStoreProvider storeProvider) {
- storeProvider.getJobStore().removeJob(jobKey);
- }
- });
- LOG.info("Successfully deleted cron job " + jobKey);
- }
- return true;
- }
-
- private final Function<String, String> keyToSchedule = new Function<String, String>() {
- @Override public String apply(String key) {
- return cron.getSchedule(key).or("Not found.");
- }
- };
-
- public Map<IJobKey, String> getScheduledJobs() {
- synchronized (scheduledJobs) {
- return ImmutableMap.copyOf(Maps.transformValues(scheduledJobs, keyToSchedule));
- }
- }
-
- public Set<IJobKey> getPendingRuns() {
- synchronized (pendingRuns) {
- return ImmutableSet.copyOf(pendingRuns.keySet());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/ImmediateJobManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/ImmediateJobManager.java b/src/main/java/com/twitter/aurora/scheduler/state/ImmediateJobManager.java
deleted file mode 100644
index a085d7d..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/ImmediateJobManager.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Job scheduler that accepts any job and executes it immediately.
- */
-class ImmediateJobManager extends JobManager {
-
- private static final Logger LOG = Logger.getLogger(ImmediateJobManager.class.getName());
-
- private final StateManager stateManager;
- private final Storage storage;
-
- @Inject
- ImmediateJobManager(StateManager stateManager, Storage storage) {
- this.stateManager = checkNotNull(stateManager);
- this.storage = checkNotNull(storage);
- }
-
- @Override
- public String getUniqueKey() {
- return "IMMEDIATE";
- }
-
- @Override
- public boolean receiveJob(SanitizedConfiguration config) {
- LOG.info("Launching " + config.getTaskConfigs().size() + " tasks.");
- stateManager.insertPendingTasks(config.getTaskConfigs());
- return true;
- }
-
- @Override
- public boolean hasJob(final IJobKey jobKey) {
- return !Storage.Util.consistentFetchTasks(storage, Query.jobScoped(jobKey).active()).isEmpty();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/JobFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/JobFilter.java b/src/main/java/com/twitter/aurora/scheduler/state/JobFilter.java
deleted file mode 100644
index 3b8f5d8..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/JobFilter.java
+++ /dev/null
@@ -1,119 +0,0 @@
-package com.twitter.aurora.scheduler.state;
-
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * An action that either accepts a configuration or rejects it with a reason.
- */
-public interface JobFilter {
- /**
- * Accept the JobConfiguration or reject it with a reason.
- *
- * @param jobConfiguration The job configuration to filter.
- * @return A result and the reason the result was reached.
- */
- JobFilterResult filter(IJobConfiguration jobConfiguration);
-
- /**
- * Accept the TaskConfig with the specified number of instances
- * or reject it with a reason.
- *
- * @param template The task configuration to filter.
- * @param instanceCount Number of instances to apply taskConfig to.
- * @return A result and the reason the result was reached.
- */
- JobFilterResult filter(ITaskConfig template, int instanceCount);
-
- /**
- * An indication of whether a job passed a filter or not.
- */
- public static final class JobFilterResult {
- private final boolean pass;
- private final String reason;
-
- private JobFilterResult(boolean pass, String reason) {
- this.pass = pass;
- this.reason = Preconditions.checkNotNull(reason);
- }
-
- /**
- * Create a new result indicating the job has passed the filter.
- *
- * @return A result indicating the job has passed with a default reason.
- * @see #fail(String)
- */
- public static JobFilterResult pass() {
- return new JobFilterResult(true, "Accepted by filter.");
- }
-
- /**
- * Create a new result indicating the job has passed the filter.
- *
- * @param reason A reason that the job has passed.
- * @return A result indicating the job has passed because of the given reason.
- * @throws NullPointerException if reason is {@code null}.
- * @see #fail(String)
- */
- public static JobFilterResult pass(String reason) {
- return new JobFilterResult(true, reason);
- }
-
- /**
- * Create a new result indicating the job has failed the filter.
- *
- * @param reason A reason that the job has failed.
- * @return A result indicating the job has failed because of the given reason.
- * @throws NullPointerException if {@code reason} is {@code null}.
- * @see #pass()
- * @see #pass(String)
- */
- public static JobFilterResult fail(String reason) {
- return new JobFilterResult(false, reason);
- }
-
- /**
- * Indicates whether the result indicates a pass.
- *
- * @return {@code true} if the result indicates a pass.
- */
- public boolean isPass() {
- return pass;
- }
-
- /**
- * Indicates the reason for the result.
- *
- * @return The reason for the result.
- */
- public String getReason() {
- return reason;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof JobFilterResult)) {
- return false;
- }
- JobFilterResult that = (JobFilterResult) o;
- return Objects.equal(pass, that.pass)
- && Objects.equal(reason, that.reason);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(pass, reason);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("pass", pass)
- .add("reason", reason)
- .toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/JobManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/JobManager.java b/src/main/java/com/twitter/aurora/scheduler/state/JobManager.java
deleted file mode 100644
index 4cb4ff6..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/JobManager.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Collections;
-
-import javax.inject.Inject;
-
-import com.twitter.aurora.scheduler.base.ScheduleException;
-import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-
-/**
- * Interface for a job manager. A job manager is responsible for deciding whether and when to
- * trigger execution of a job.
- */
-public abstract class JobManager {
-
- // TODO(Bill Farner): Remove this. It is only used since the CronJobManager and SchedulerCoreImpl
- // have a circular dependency.
- @Inject
- protected SchedulerCore schedulerCore;
-
- /**
- * Gets a key that uniquely identifies this manager type, to distinguish from other schedulers.
- * These keys end up being persisted, so they must be considered permanently immutable.
- *
- * @return Job manager key.
- */
- public abstract String getUniqueKey();
-
- /**
- * Submits a job to the manager. The job may be submitted to the job runner before this method
- * returns or at any point in the future. This method will return false if the manager will not
- * execute the job.
- *
- * @param config The job to schedule.
- * @return {@code true} If the manager accepted the job, {@code false} otherwise.
- * @throws ScheduleException If there is a problem with scheduling the job.
- */
- public abstract boolean receiveJob(SanitizedConfiguration config) throws ScheduleException;
-
- /**
- * Fetches the configured jobs that this manager is storing.
- *
- * @return Jobs stored by this job manager.
- */
- // TODO(ksweeney): Consider adding a Map<JobKey, JobConfiguration> to complement this.
- public Iterable<IJobConfiguration> getJobs() {
- return Collections.emptyList();
- }
-
- /**
- * Checks whether this manager is storing a job with the given key.
- *
- * @param jobKey Job key.
- * @return {@code true} if the manager has a matching job, {@code false} otherwise.
- */
- public abstract boolean hasJob(IJobKey jobKey);
-
- /**
- * Instructs the manager to delete any jobs with the given key.
- *
- * @param jobKey Job key.
- * @return {@code true} if a matching job was deleted.
- */
- public boolean deleteJob(IJobKey jobKey) {
- // Optionally overridden by implementing class.
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/LockManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/LockManager.java b/src/main/java/com/twitter/aurora/scheduler/state/LockManager.java
deleted file mode 100644
index 58d7b0e..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/LockManager.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import com.google.common.base.Optional;
-
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.ILockKey;
-
-/**
- * Defines all {@link ILock} primitives like: acquire, release, validate.
- */
-public interface LockManager {
- /**
- * Creates, saves and returns a new {@link ILock} with the specified {@link ILockKey}.
- * This method is not re-entrant, i.e. attempting to acquire a lock with the
- * same key would throw a {@link LockException}.
- *
- * @param lockKey A key uniquely identify the lock to be created.
- * @param user Name of the user requesting a lock.
- * @return A new ILock instance.
- * @throws LockException In case the lock with specified key already exists.
- */
- ILock acquireLock(ILockKey lockKey, String user) throws LockException;
-
- /**
- * Releases (removes) the specified {@link ILock} from the system.
- *
- * @param lock {@link ILock} to remove from the system.
- */
- void releaseLock(ILock lock);
-
- /**
- * Verifies if the provided lock instance is identical to the one stored in the scheduler
- * ONLY if the operation context represented by the {@link ILockKey} is in fact locked.
- * No validation will be performed in case there is no correspondent scheduler lock
- * found for the provided context.
- *
- * @param context Operation context to validate with the provided lock.
- * @param heldLock Lock to validate.
- * @throws LockException If provided lock does not exist or not identical to the stored one.
- */
- void validateIfLocked(ILockKey context, Optional<ILock> heldLock) throws LockException;
-
- /**
- * Thrown when {@link ILock} related operation failed.
- */
- class LockException extends Exception {
- public LockException(String msg) {
- super(msg);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java b/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java
deleted file mode 100644
index 4667f9f..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/LockManagerImpl.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Date;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Optional;
-
-import com.twitter.aurora.gen.Lock;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.storage.LockStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.ILockKey;
-import com.twitter.common.util.Clock;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Implements lock-related primitives required to provide mutual exclusion guarantees
- * to the critical Scheduler state-mutating operations.
- */
-class LockManagerImpl implements LockManager {
- private final Storage storage;
- private final Clock clock;
- private final UUIDGenerator tokenGenerator;
-
- @Inject
- LockManagerImpl(Storage storage, Clock clock, UUIDGenerator tokenGenerator) {
- this.storage = checkNotNull(storage);
- this.clock = checkNotNull(clock);
- this.tokenGenerator = checkNotNull(tokenGenerator);
- }
-
- @Override
- public ILock acquireLock(final ILockKey lockKey, final String user) throws LockException {
- return storage.write(new MutateWork<ILock, LockException>() {
- @Override public ILock apply(Storage.MutableStoreProvider storeProvider)
- throws LockException {
-
- LockStore.Mutable lockStore = storeProvider.getLockStore();
- Optional<ILock> existingLock = lockStore.fetchLock(lockKey);
-
- if (existingLock.isPresent()) {
- throw new LockException(String.format(
- "Operation for: %s is already in progress. Started at: %s. Current owner: %s.",
- formatLockKey(lockKey),
- new Date(existingLock.get().getTimestampMs()).toString(),
- existingLock.get().getUser()));
- }
-
- ILock lock = ILock.build(new Lock()
- .setKey(lockKey.newBuilder())
- .setToken(tokenGenerator.createNew().toString())
- .setTimestampMs(clock.nowMillis())
- .setUser(user));
-
- lockStore.saveLock(lock);
- return lock;
- }
- });
- }
-
- @Override
- public void releaseLock(final ILock lock) {
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override public void execute(MutableStoreProvider storeProvider) {
- storeProvider.getLockStore().removeLock(lock.getKey());
- }
- });
- }
-
- @Override
- public synchronized void validateIfLocked(final ILockKey context, Optional<ILock> heldLock)
- throws LockException {
-
- Optional<ILock> stored = storage.consistentRead(new Work.Quiet<Optional<ILock>>() {
- @Override public Optional<ILock> apply(StoreProvider storeProvider) {
- return storeProvider.getLockStore().fetchLock(context);
- }
- });
-
- // The implementation below assumes the following use cases:
- // +-----------+-----------------+----------+
- // | eq | held | not held |
- // +-----------+-----------------+----------+
- // |stored |(stored == held)?| invalid |
- // +-----------+-----------------+----------+
- // |not stored | invalid | valid |
- // +-----------+-----------------+----------+
- if (!stored.equals(heldLock)) {
- if (stored.isPresent()) {
- throw new LockException(String.format(
- "Unable to perform operation for: %s. Use override/cancel option.",
- formatLockKey(context)));
- } else if (heldLock.isPresent()) {
- throw new LockException(
- String.format("Invalid operation context: %s", formatLockKey(context)));
- }
- }
- }
-
- private static String formatLockKey(ILockKey lockKey) {
- switch (lockKey.getSetField()) {
- case JOB:
- return JobKeys.toPath(lockKey.getJob());
- default:
- return "Unknown lock key type: " + lockKey.getSetField();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java b/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java
deleted file mode 100644
index d447f52..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Set;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
-import com.google.common.eventbus.Subscribe;
-
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.HostStatus;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.events.PubsubEvent;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import com.twitter.aurora.scheduler.storage.AttributeStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.common.base.Closure;
-import com.twitter.common.base.Closures;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
-
-/**
- * Logic that puts hosts into maintenance mode, and triggers draining of hosts upon request.
- * All state-changing functions return their results. Additionally, all state-changing functions
- * will ignore requests to change state of unknown hosts and subsequently omit these hosts from
- * return values.
- */
-public interface MaintenanceController {
-
- /**
- * Places hosts in maintenance mode.
- * Hosts in maintenance mode are less-preferred for scheduling.
- * No change will be made for hosts that are not recognized, and unrecognized hosts will not be
- * included in the result.
- *
- * @param hosts Hosts to put into maintenance mode.
- * @return The adjusted state of the hosts.
- */
- Set<HostStatus> startMaintenance(Set<String> hosts);
-
- /**
- * Initiate a drain of all active tasks on {@code hosts}.
- *
- * @param hosts Hosts to drain.
- * @return The adjusted state of the hosts. Hosts without any active tasks will be immediately
- * moved to DRAINED.
- */
- Set<HostStatus> drain(Set<String> hosts);
-
- /**
- * Fetches the current maintenance mode of {$code host}.
- *
- * @param host Host to fetch state for.
- * @return Maintenance mode of host, {@link MaintenanceMode#NONE} if the host is not known.
- */
- MaintenanceMode getMode(String host);
-
- /**
- * Fetches the current state of {@code hosts}.
- *
- * @param hosts Hosts to fetch state for.
- * @return The state of the hosts.
- */
- Set<HostStatus> getStatus(Set<String> hosts);
-
- /**
- * Moves {@code hosts} out of maintenance mode, returning them to mode NONE.
- *
- * @param hosts Hosts to move out of maintenance mode.
- * @return The adjusted state of the hosts.
- */
- Set<HostStatus> endMaintenance(Set<String> hosts);
-
- class MaintenanceControllerImpl implements MaintenanceController, EventSubscriber {
- private final Storage storage;
- private final StateManager stateManager;
- private final Closure<PubsubEvent> eventSink;
-
- @Inject
- public MaintenanceControllerImpl(
- Storage storage,
- StateManager stateManager,
- Closure<PubsubEvent> eventSink) {
-
- this.storage = checkNotNull(storage);
- this.stateManager = checkNotNull(stateManager);
- this.eventSink = checkNotNull(eventSink);
- }
-
- private Set<HostStatus> watchDrainingTasks(
- MutableStoreProvider store,
- Set<String> hosts,
- Closure<Query.Builder> callback) {
-
- Set<String> emptyHosts = Sets.newHashSet();
- for (String host : hosts) {
- // If there are no tasks on the host, immediately transition to DRAINED.
- Query.Builder query = Query.slaveScoped(host).active();
- Set<String> activeTasks = FluentIterable.from(store.getTaskStore().fetchTasks(query))
- .transform(Tasks.SCHEDULED_TO_ID)
- .toSet();
- if (activeTasks.isEmpty()) {
- emptyHosts.add(host);
- } else {
- callback.execute(query);
- }
- }
-
- return ImmutableSet.<HostStatus>builder()
- .addAll(setMaintenanceMode(store, emptyHosts, DRAINED))
- .addAll(setMaintenanceMode(store, Sets.difference(hosts, emptyHosts), DRAINING))
- .build();
- }
-
- private Set<HostStatus> watchDrainingTasks(MutableStoreProvider store, Set<String> hosts) {
- return watchDrainingTasks(store, hosts, Closures.<Query.Builder>noop());
- }
-
- private static final Predicate<HostAttributes> IS_DRAINING = new Predicate<HostAttributes>() {
- @Override public boolean apply(HostAttributes attributes) {
- return DRAINING == attributes.getMode();
- }
- };
-
- /**
- * Notifies the MaintenanceController that storage has started, and maintenance statuses are
- * ready to be loaded.
- *
- * @param started Event.
- */
- @Subscribe
- public void storageStarted(StorageStarted started) {
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- Set<String> drainingHosts =
- FluentIterable.from(storeProvider.getAttributeStore().getHostAttributes())
- .filter(IS_DRAINING)
- .transform(HOST_NAME)
- .toSet();
- watchDrainingTasks(storeProvider, drainingHosts);
- }
- });
- }
-
- /**
- * Notifies the MaintenanceController that a task has changed state
- *
- * @param change Event
- */
- @Subscribe
- public void taskChangedState(final TaskStateChange change) {
- if (Tasks.isTerminated(change.getNewState())) {
- final String host = change.getTask().getAssignedTask().getSlaveHost();
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override public void execute(MutableStoreProvider store) {
- // If the task _was_ associated with a draining host, and it was the last task on the
- // host.
- Optional<HostAttributes> attributes = store.getAttributeStore().getHostAttributes(host);
- if (attributes.isPresent() && attributes.get().getMode() == DRAINING) {
- Query.Builder builder = Query.slaveScoped(host).active();
- if (store.getTaskStore().fetchTasks(builder).isEmpty()) {
- setMaintenanceMode(store, ImmutableSet.of(host), DRAINED);
- }
- }
- }
- });
- }
- }
-
- @Override
- public Set<HostStatus> startMaintenance(final Set<String> hosts) {
- return storage.write(new MutateWork.Quiet<Set<HostStatus>>() {
- @Override public Set<HostStatus> apply(MutableStoreProvider storeProvider) {
- return setMaintenanceMode(storeProvider, hosts, MaintenanceMode.SCHEDULED);
- }
- });
- }
-
- @VisibleForTesting
- static final Optional<String> DRAINING_MESSAGE =
- Optional.of("Draining machine for maintenance.");
-
- @Override
- public Set<HostStatus> drain(final Set<String> hosts) {
- return storage.write(new MutateWork.Quiet<Set<HostStatus>>() {
- @Override public Set<HostStatus> apply(MutableStoreProvider store) {
- return watchDrainingTasks(store, hosts, new Closure<Query.Builder>() {
- @Override public void execute(Query.Builder query) {
- stateManager.changeState(query, ScheduleStatus.RESTARTING, DRAINING_MESSAGE);
- }
- });
- }
- });
- }
-
- private static final Function<HostAttributes, String> HOST_NAME =
- new Function<HostAttributes, String>() {
- @Override public String apply(HostAttributes attributes) {
- return attributes.getHost();
- }
- };
-
- private static final Function<HostAttributes, HostStatus> ATTRS_TO_STATUS =
- new Function<HostAttributes, HostStatus>() {
- @Override public HostStatus apply(HostAttributes attributes) {
- return new HostStatus().setHost(attributes.getHost()).setMode(attributes.getMode());
- }
- };
-
- private static final Function<HostStatus, MaintenanceMode> GET_MODE =
- new Function<HostStatus, MaintenanceMode>() {
- @Override public MaintenanceMode apply(HostStatus status) {
- return status.getMode();
- }
- };
-
- @Override
- public MaintenanceMode getMode(final String host) {
- return storage.weaklyConsistentRead(new Work.Quiet<MaintenanceMode>() {
- @Override public MaintenanceMode apply(StoreProvider storeProvider) {
- return storeProvider.getAttributeStore().getHostAttributes(host)
- .transform(ATTRS_TO_STATUS)
- .transform(GET_MODE)
- .or(MaintenanceMode.NONE);
- }
- });
- }
-
- @Override
- public Set<HostStatus> getStatus(final Set<String> hosts) {
- return storage.weaklyConsistentRead(new Work.Quiet<Set<HostStatus>>() {
- @Override public Set<HostStatus> apply(StoreProvider storeProvider) {
- // Warning - this is filtering _all_ host attributes. If using this to frequently query
- // for a small set of hosts, a getHostAttributes variant should be added.
- return FluentIterable.from(storeProvider.getAttributeStore().getHostAttributes())
- .filter(Predicates.compose(Predicates.in(hosts), HOST_NAME))
- .transform(ATTRS_TO_STATUS).toSet();
- }
- });
- }
-
- @Override
- public Set<HostStatus> endMaintenance(final Set<String> hosts) {
- return storage.write(new MutateWork.Quiet<Set<HostStatus>>() {
- @Override public Set<HostStatus> apply(MutableStoreProvider storeProvider) {
- return setMaintenanceMode(storeProvider, hosts, MaintenanceMode.NONE);
- }
- });
- }
-
- private Set<HostStatus> setMaintenanceMode(
- MutableStoreProvider storeProvider,
- Set<String> hosts,
- MaintenanceMode mode) {
-
- AttributeStore.Mutable store = storeProvider.getAttributeStore();
- ImmutableSet.Builder<HostStatus> statuses = ImmutableSet.builder();
- for (String host : hosts) {
- if (store.setMaintenanceMode(host, mode)) {
- HostStatus status = new HostStatus().setHost(host).setMode(mode);
- eventSink.execute(new PubsubEvent.HostMaintenanceStateChange(status.deepCopy()));
- statuses.add(status);
- }
- }
- return statuses.build();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCore.java b/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCore.java
deleted file mode 100644
index 2db01c0..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/state/SchedulerCore.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.state;
-
-import java.util.Set;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.ScheduleException;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
-import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * Scheduling core, stores scheduler state and makes decisions about which tasks to schedule when
- * a resource offer is made.
- *
- * When a job is submitted to the scheduler core, it will store the job configuration and offer
- * the job to all configured scheduler modules, which are responsible for triggering execution of
- * the job. Until a job is triggered by a scheduler module, it is retained in the scheduler core
- * in the PENDING state.
- */
-public interface SchedulerCore {
-
- /**
- * Creates a new job, whose tasks will become candidates for scheduling.
- *
- * @param sanitizedConfiguration The configuration of the job to create tasks for.
- * @throws ScheduleException If there was an error scheduling a cron job.
- * @throws TaskDescriptionException If an invalid task description was given.
- */
- void createJob(SanitizedConfiguration sanitizedConfiguration)
- throws ScheduleException, TaskDescriptionException;
-
- /**
- * Adds new instances specified by the instances set.
- * <p>
- * Provided instance IDs should be disjoint from the instance IDs active in the job.
- *
- * @param jobKey IJobKey identifying the parent job.
- * @param instanceIds Set of instance IDs to be added to the job.
- * @param config ITaskConfig to use with new instances.
- * @throws ScheduleException If any of the existing instance IDs already exist.
- */
- void addInstances(IJobKey jobKey, ImmutableSet<Integer> instanceIds, ITaskConfig config)
- throws ScheduleException;
-
- /**
- * Validates the new job configuration passes resource filters.
- *
- * @param sanitizedConfiguration Job configuration to validate.
- * @throws ScheduleException If job resources do not pass filters.
- */
- void validateJobResources(SanitizedConfiguration sanitizedConfiguration) throws ScheduleException;
-
- /**
- * Starts a cron job immediately.
- *
- * @param jobKey Job key.
- * @throws ScheduleException If the specified job does not exist, or is not a cron job.
- * @throws TaskDescriptionException If the parsing of the job failed.
- */
- void startCronJob(IJobKey jobKey) throws ScheduleException, TaskDescriptionException;
-
- /**
- * Assigns a new state to tasks.
- *
- * @param query Builder for a query to identify tasks
- * @param status The new state of the tasks.
- * @param message Additional information about the state transition.
- */
- void setTaskStatus(Query.Builder query, ScheduleStatus status, Optional<String> message);
-
- /**
- * Kills a specific set of tasks.
- *
- * @param query Builder for a query to identify tasks
- * @param user Name of the user performing the kill.
- * @throws ScheduleException If a problem occurs with the kill request.
- */
- void killTasks(Query.Builder query, String user) throws ScheduleException;
-
- /**
- * Initiates a restart of shards within an active job.
- *
- * @param jobKey Key of job to be restarted.
- * @param shards Shards to be restarted.
- * @param requestingUser User performing the restart action.
- * @throws ScheduleException If there are no matching active shards.
- */
- void restartShards(IJobKey jobKey, Set<Integer> shards, String requestingUser)
- throws ScheduleException;
-
- /**
- * Preempts a task in favor of another.
- *
- * @param task Task being preempted.
- * @param preemptingTask Task we are preempting in favor of.
- * @throws ScheduleException If a problem occurs while trying to perform the preemption.
- */
- void preemptTask(IAssignedTask task, IAssignedTask preemptingTask) throws ScheduleException;
-
- /**
- * Indicates to the scheduler that tasks were deleted on the assigned host.
- *
- * @param taskIds IDs of tasks that were deleted.
- */
- void tasksDeleted(Set<String> taskIds);
-}
[31/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/TaskStateMachine.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/TaskStateMachine.java b/src/main/java/org/apache/aurora/scheduler/state/TaskStateMachine.java
new file mode 100644
index 0000000..f32fd14
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/TaskStateMachine.java
@@ -0,0 +1,622 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.base.Throwables;
+
+import org.apache.commons.lang.builder.HashCodeBuilder;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.ScheduledTask;
+import com.twitter.aurora.gen.TaskEvent;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.base.Closure;
+import com.twitter.common.base.Closures;
+import com.twitter.common.base.Command;
+import com.twitter.common.base.MorePreconditions;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+import com.twitter.common.util.StateMachine;
+import com.twitter.common.util.StateMachine.Rule;
+import com.twitter.common.util.StateMachine.Transition;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * State machine for a task.
+ * <p>
+ * This enforces the lifecycle of a task, and triggers the actions that should be taken in response
+ * to different state transitions. These responses are externally communicated by populating a
+ * provided work queue.
+ * <p>
+ * TODO(William Farner): Introduce an interface to allow state machines to be dealt with
+ * abstractly from the consumption side.
+ */
+class TaskStateMachine {
+ private static final Logger LOG = Logger.getLogger(TaskStateMachine.class.getName());
+
+ private static final AtomicLong ILLEGAL_TRANSITIONS =
+ Stats.exportLong("scheduler_illegal_task_state_transitions");
+
+ // Re-declarations of statuses as wrapped state objects.
+ private static final State ASSIGNED = State.create(ScheduleStatus.ASSIGNED);
+ private static final State FAILED = State.create(ScheduleStatus.FAILED);
+ private static final State FINISHED = State.create(ScheduleStatus.FINISHED);
+ private static final State INIT = State.create(ScheduleStatus.INIT);
+ private static final State KILLED = State.create(ScheduleStatus.KILLED);
+ private static final State KILLING = State.create(ScheduleStatus.KILLING);
+ private static final State LOST = State.create(ScheduleStatus.LOST);
+ private static final State PENDING = State.create(ScheduleStatus.PENDING);
+ private static final State PREEMPTING = State.create(ScheduleStatus.PREEMPTING);
+ private static final State RESTARTING = State.create(ScheduleStatus.RESTARTING);
+ private static final State RUNNING = State.create(ScheduleStatus.RUNNING);
+ private static final State STARTING = State.create(ScheduleStatus.STARTING);
+ private static final State UNKNOWN = State.create(ScheduleStatus.UNKNOWN);
+
+ @VisibleForTesting
+ static final Supplier<String> LOCAL_HOST_SUPPLIER = Suppliers.memoize(
+ new Supplier<String>() {
+ @Override public String get() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ LOG.log(Level.SEVERE, "Failed to get self hostname.");
+ throw Throwables.propagate(e);
+ }
+ }
+ });
+
+ private final String taskId;
+ private final WorkSink workSink;
+ private final StateMachine<State> stateMachine;
+ private ScheduleStatus previousState = null;
+ private final Clock clock;
+
+ /**
+ * Composes a schedule status and a state change argument. Only the ScheduleStatuses in two
+ * States must be equal for them to be considered equal.
+ */
+ private static class State {
+ private final ScheduleStatus state;
+ private final Function<IScheduledTask, IScheduledTask> mutation;
+
+ State(ScheduleStatus state, Function<IScheduledTask, IScheduledTask> mutation) {
+ this.state = state;
+ this.mutation = mutation;
+ }
+
+ static State create(ScheduleStatus status) {
+ return create(status, Functions.<IScheduledTask>identity());
+ }
+
+ static State create(
+ ScheduleStatus status,
+ Function<IScheduledTask, IScheduledTask> mutation) {
+
+ return new State(status, mutation);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof State)) {
+ return false;
+ }
+
+ if (o == this) {
+ return true;
+ }
+
+ State other = (State) o;
+ return state == other.state;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder()
+ .append(state)
+ .toHashCode();
+ }
+
+ @Override
+ public String toString() {
+ return state.toString();
+ }
+
+ private ScheduleStatus getState() {
+ return state;
+ }
+
+ private Function<IScheduledTask, IScheduledTask> getMutation() {
+ return mutation;
+ }
+ }
+
+ /**
+ * A write-only work acceptor.
+ */
+ public interface WorkSink {
+ /**
+ * Appends external work that must be performed for a state machine transition to be fully
+ * complete.
+ *
+ * @param work Description of the work to be performed.
+ * @param stateMachine The state machine that the work is associated with.
+ * @param mutation Mutate operation to perform along with the state transition.
+ */
+ void addWork(
+ WorkCommand work,
+ TaskStateMachine stateMachine,
+ Function<IScheduledTask, IScheduledTask> mutation);
+ }
+
+ /**
+ * Creates a new task state machine.
+ *
+ * @param taskId ID of the task managed by this state machine.
+ * @param task Read-only task that this state machine manages.
+ * @param workSink Work sink to receive transition response actions
+ * @param clock Clock to use for reading the current time.
+ * @param initialState The state to begin the state machine at. All legal transitions will be
+ * added, but this allows the state machine to 'skip' states, for instance when a task is
+ * loaded from a persistent store.
+ */
+ public TaskStateMachine(
+ final String taskId,
+ final IScheduledTask task,
+ final WorkSink workSink,
+ final Clock clock,
+ final ScheduleStatus initialState) {
+
+ this.taskId = MorePreconditions.checkNotBlank(taskId);
+ this.workSink = checkNotNull(workSink);
+ this.clock = checkNotNull(clock);
+ checkNotNull(initialState);
+
+ @SuppressWarnings("unchecked")
+ Closure<Transition<State>> manageTerminatedTasks = Closures.combine(
+ /* Kill a task that we believe to be terminated when an attempt is made to revive. */
+ Closures.filter(Transition.to(ASSIGNED, STARTING, RUNNING),
+ addWorkClosure(WorkCommand.KILL)),
+ /* Remove a terminated task that is remotely removed. */
+ Closures.filter(Transition.to(UNKNOWN), addWorkClosure(WorkCommand.DELETE)));
+
+ final Closure<Transition<State>> manageRestartingTask = new Closure<Transition<State>>() {
+ @SuppressWarnings("fallthrough")
+ @Override public void execute(Transition<State> transition) {
+ switch (transition.getTo().getState()) {
+ case ASSIGNED:
+ case STARTING:
+ case RUNNING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case LOST:
+ addWork(WorkCommand.KILL);
+ // fall through
+
+ case FINISHED:
+ case FAILED:
+ case KILLED:
+ addWork(WorkCommand.RESCHEDULE, transition.getTo().getMutation());
+ break;
+
+ case UNKNOWN:
+ updateState(ScheduleStatus.LOST);
+ break;
+
+ default:
+ // No-op.
+ }
+ }
+ };
+
+ // To be called on a task transitioning into the FINISHED state.
+ final Command rescheduleIfService = new Command() {
+ @Override public void execute() {
+ if (task.getAssignedTask().getTask().isIsService()) {
+ addWork(WorkCommand.RESCHEDULE);
+ }
+ }
+ };
+
+ // To be called on a task transitioning into the FAILED state.
+ final Command incrementFailuresMaybeReschedule = new Command() {
+ @Override public void execute() {
+ addWork(WorkCommand.INCREMENT_FAILURES);
+
+ // Max failures is ignored for service task.
+ boolean isService = task.getAssignedTask().getTask().isIsService();
+
+ // Max failures is ignored when set to -1.
+ int maxFailures = task.getAssignedTask().getTask().getMaxTaskFailures();
+ if (isService || (maxFailures == -1) || (task.getFailureCount() < (maxFailures - 1))) {
+ addWork(WorkCommand.RESCHEDULE);
+ } else {
+ LOG.info("Task " + getTaskId() + " reached failure limit, not rescheduling");
+ }
+ }
+ };
+
+ stateMachine = StateMachine.<State>builder(taskId)
+ .logTransitions()
+ .initialState(State.create(initialState))
+ .addState(
+ Rule.from(INIT)
+ .to(PENDING, UNKNOWN))
+ .addState(
+ Rule.from(PENDING)
+ .to(ASSIGNED, KILLING)
+ .withCallback(
+ new Closure<Transition<State>>() {
+ @Override public void execute(Transition<State> transition) {
+ switch (transition.getTo().getState()) {
+ case KILLING:
+ addWork(WorkCommand.DELETE);
+ break;
+
+ default:
+ // No-op.
+ }
+ }
+ }
+ ))
+ .addState(
+ Rule.from(ASSIGNED)
+ .to(STARTING, RUNNING, FINISHED, FAILED, RESTARTING, KILLED,
+ KILLING, LOST, PREEMPTING)
+ .withCallback(
+ new Closure<Transition<State>>() {
+ @SuppressWarnings("fallthrough")
+ @Override public void execute(Transition<State> transition) {
+ switch (transition.getTo().getState()) {
+ case FINISHED:
+ rescheduleIfService.execute();
+ break;
+
+ case PREEMPTING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case FAILED:
+ incrementFailuresMaybeReschedule.execute();
+ break;
+
+ case RESTARTING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case KILLED:
+ addWork(WorkCommand.RESCHEDULE);
+ break;
+
+ case LOST:
+ addWork(WorkCommand.RESCHEDULE);
+ // fall through
+ case KILLING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case UNKNOWN:
+ break;
+
+ default:
+ // No-op.
+ }
+ }
+ }
+ ))
+ .addState(
+ Rule.from(STARTING)
+ .to(RUNNING, FINISHED, FAILED, RESTARTING, KILLING, KILLED, LOST, PREEMPTING)
+ .withCallback(
+ new Closure<Transition<State>>() {
+ @SuppressWarnings("fallthrough")
+ @Override public void execute(Transition<State> transition) {
+ switch (transition.getTo().getState()) {
+ case FINISHED:
+ rescheduleIfService.execute();
+ break;
+
+ case RESTARTING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case PREEMPTING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case FAILED:
+ incrementFailuresMaybeReschedule.execute();
+ break;
+
+ case KILLED:
+ addWork(WorkCommand.RESCHEDULE);
+ break;
+
+ case KILLING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case LOST:
+ addWork(WorkCommand.RESCHEDULE);
+ break;
+
+ case UNKNOWN:
+ // The slave previously acknowledged that it had the task, and now
+ // stopped reporting it.
+ updateState(ScheduleStatus.LOST);
+ break;
+
+ default:
+ // No-op.
+ }
+ }
+ }
+ ))
+ .addState(
+ Rule.from(RUNNING)
+ .to(FINISHED, RESTARTING, FAILED, KILLING, KILLED, LOST, PREEMPTING)
+ .withCallback(
+ new Closure<Transition<State>>() {
+ @SuppressWarnings("fallthrough")
+ @Override public void execute(Transition<State> transition) {
+ switch (transition.getTo().getState()) {
+ case FINISHED:
+ rescheduleIfService.execute();
+ break;
+
+ case PREEMPTING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case RESTARTING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case FAILED:
+ incrementFailuresMaybeReschedule.execute();
+ break;
+
+ case KILLED:
+ addWork(WorkCommand.RESCHEDULE);
+ break;
+
+ case KILLING:
+ addWork(WorkCommand.KILL);
+ break;
+
+ case LOST:
+ addWork(WorkCommand.RESCHEDULE);
+ break;
+
+ case UNKNOWN:
+ updateState(ScheduleStatus.LOST);
+ break;
+
+ default:
+ // No-op.
+ }
+ }
+ }
+ ))
+ .addState(
+ Rule.from(FINISHED)
+ .to(UNKNOWN)
+ .withCallback(manageTerminatedTasks))
+ .addState(
+ Rule.from(PREEMPTING)
+ .to(FINISHED, FAILED, KILLING, KILLED, LOST)
+ .withCallback(manageRestartingTask))
+ .addState(
+ Rule.from(RESTARTING)
+ .to(FINISHED, FAILED, KILLING, KILLED, LOST)
+ .withCallback(manageRestartingTask))
+ .addState(
+ Rule.from(FAILED)
+ .to(UNKNOWN)
+ .withCallback(manageTerminatedTasks))
+ .addState(
+ Rule.from(KILLED)
+ .to(UNKNOWN)
+ .withCallback(manageTerminatedTasks))
+ .addState(
+ Rule.from(KILLING)
+ .to(FINISHED, FAILED, KILLED, LOST, UNKNOWN)
+ .withCallback(manageTerminatedTasks))
+ .addState(
+ Rule.from(LOST)
+ .to(UNKNOWN)
+ .withCallback(manageTerminatedTasks))
+ .addState(
+ Rule.from(UNKNOWN)
+ .noTransitions()
+ .withCallback(manageTerminatedTasks))
+ // Since we want this action to be performed last in the transition sequence, the callback
+ // must be the last chained transition callback.
+ .onAnyTransition(
+ new Closure<Transition<State>>() {
+ @Override public void execute(final Transition<State> transition) {
+ ScheduleStatus from = transition.getFrom().getState();
+ ScheduleStatus to = transition.getTo().getState();
+
+ if (transition.isValidStateChange() && (to != ScheduleStatus.UNKNOWN)
+ // Prevent an update when killing a pending task, since the task is deleted
+ // prior to the update.
+ && !((from == ScheduleStatus.PENDING) && (to == ScheduleStatus.KILLING))) {
+ addWork(WorkCommand.UPDATE_STATE, transition.getTo().getMutation());
+ } else if (!transition.isAllowed()) {
+ LOG.log(Level.SEVERE, "Illegal state transition attempted: " + transition);
+ ILLEGAL_TRANSITIONS.incrementAndGet();
+ }
+
+ if (transition.isValidStateChange()) {
+ previousState = from;
+ }
+ }
+ }
+ )
+ // TODO(wfarner): Consider alternatives to allow exceptions to surface. This would allow
+ // the state machine to surface illegal state transitions and propagate better information
+ // to the caller. As it stands, the caller must implement logic that really belongs in
+ // the state machine. For example, preventing RESTARTING->UPDATING transitions
+ // (or for that matter, almost any user-initiated state transition) is awkward.
+ .throwOnBadTransition(false)
+ .build();
+ }
+
+ private Closure<Transition<State>> addWorkClosure(final WorkCommand work) {
+ return new Closure<Transition<State>>() {
+ @Override public void execute(Transition<State> item) {
+ addWork(work);
+ }
+ };
+ }
+
+ private void addWork(WorkCommand work) {
+ addWork(work, Functions.<IScheduledTask>identity());
+ }
+
+ private void addWork(WorkCommand work, Function<IScheduledTask, IScheduledTask> mutation) {
+ LOG.info("Adding work command " + work + " for " + this);
+ workSink.addWork(work, TaskStateMachine.this, mutation);
+ }
+
+ /**
+ * Same as {@link #updateState(ScheduleStatus, Function)}, but uses a noop mutation.
+ *
+ * @param status Status to apply to the task.
+ * @return {@code true} if the state change was allowed, {@code false} otherwise.
+ */
+ public synchronized boolean updateState(ScheduleStatus status) {
+ return updateState(status, Functions.<IScheduledTask>identity());
+ }
+
+ /**
+ * Same as {@link #updateState(ScheduleStatus, Function, Optional)}, but uses a noop mutation.
+ *
+ * @param status Status to apply to the task.
+ * @param auditMessage The (optional) audit message to associate with the transition.
+ * @return {@code true} if the state change was allowed, {@code false} otherwise.
+ */
+ public synchronized boolean updateState(ScheduleStatus status, Optional<String> auditMessage) {
+ return updateState(status, Functions.<IScheduledTask>identity(), auditMessage);
+ }
+
+ /**
+ * Same as {@link #updateState(ScheduleStatus, Function, Optional)}, but omits the audit message.
+ *
+ * @param status Status to apply to the task.
+ * @param mutation Mutate operation to perform while updating the task.
+ * @return {@code true} if the state change was allowed, {@code false} otherwise.
+ */
+ public synchronized boolean updateState(
+ ScheduleStatus status,
+ Function<IScheduledTask, IScheduledTask> mutation) {
+
+ return updateState(status, mutation, Optional.<String>absent());
+ }
+
+ /**
+ * Attempt to transition the state machine to the provided state.
+ * At the time this method returns, any work commands required to satisfy the state transition
+ * will be appended to the work queue.
+ *
+ * @param status Status to apply to the task.
+ * @param auditMessage The audit message to associate with the transition.
+ * @param mutation Mutate operation to perform while updating the task.
+ * @return {@code true} if the state change was allowed, {@code false} otherwise.
+ */
+ public synchronized boolean updateState(
+ final ScheduleStatus status,
+ Function<IScheduledTask, IScheduledTask> mutation,
+ final Optional<String> auditMessage) {
+
+ checkNotNull(status);
+ checkNotNull(mutation);
+ checkNotNull(auditMessage);
+
+ /**
+ * Don't bother applying noop state changes. If we end up modifying task state without a
+ * state transition (e.g. storing resource consumption of a running task), we need to find
+ * a different way to suppress noop transitions.
+ */
+ if (stateMachine.getState().getState() != status) {
+ Function<IScheduledTask, IScheduledTask> operation = Functions.compose(mutation,
+ new Function<IScheduledTask, IScheduledTask>() {
+ @Override public IScheduledTask apply(IScheduledTask task) {
+ ScheduledTask builder = task.newBuilder();
+ builder.addToTaskEvents(new TaskEvent()
+ .setTimestamp(clock.nowMillis())
+ .setStatus(status)
+ .setMessage(auditMessage.orNull())
+ .setScheduler(LOCAL_HOST_SUPPLIER.get()));
+ return IScheduledTask.build(builder);
+ }
+ });
+ return stateMachine.transition(State.create(status, operation));
+ }
+
+ return false;
+ }
+
+ /**
+ * Fetch the current state from the state machine.
+ *
+ * @return The current state.
+ */
+ public synchronized ScheduleStatus getState() {
+ return stateMachine.getState().getState();
+ }
+
+ /**
+ * Gets the ID for the task that this state machine manages.
+ *
+ * @return The state machine's task ID.
+ */
+ public String getTaskId() {
+ return taskId;
+ }
+
+ /**
+ * Gets the previous state of this state machine.
+ *
+ * @return The state machine's previous state, or {@code null} if the state machine has not
+ * transitioned since being created.
+ */
+ @Nullable
+ ScheduleStatus getPreviousState() {
+ return previousState;
+ }
+
+ @Override
+ public String toString() {
+ return getTaskId();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/UUIDGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/UUIDGenerator.java b/src/main/java/org/apache/aurora/scheduler/state/UUIDGenerator.java
new file mode 100644
index 0000000..d8de19c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/UUIDGenerator.java
@@ -0,0 +1,17 @@
+package com.twitter.aurora.scheduler.state;
+
+import java.util.UUID;
+
+/**
+ * Wraps {@link java.util.UUID#randomUUID()} to facilitate unit testing.
+ */
+interface UUIDGenerator {
+ UUID createNew();
+
+ class UUIDGeneratorImpl implements UUIDGenerator {
+ @Override
+ public UUID createNew() {
+ return UUID.randomUUID();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/WorkCommand.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/WorkCommand.java b/src/main/java/org/apache/aurora/scheduler/state/WorkCommand.java
new file mode 100644
index 0000000..6c5637d
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/WorkCommand.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+/**
+ * Descriptions of the different types of external work commands that task state machines may
+ * trigger.
+ */
+enum WorkCommand {
+ // Send an instruction for the runner of this task to kill the task.
+ KILL,
+ // Create a new state machine with a copy of this task.
+ RESCHEDULE,
+ // Update the task's state (schedule status) in the persistent store to match the state machine.
+ UPDATE_STATE,
+ // Delete this task from the persistent store.
+ DELETE,
+ // Increment the failure count for this task.
+ INCREMENT_FAILURES
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java b/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java
new file mode 100644
index 0000000..a52c5c5
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/stats/AsyncStatsModule.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.stats;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.common.base.Function;
+import com.google.common.collect.FluentIterable;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.AbstractModule;
+import com.google.inject.BindingAnnotation;
+
+import org.apache.mesos.Protos.Offer;
+
+import com.twitter.aurora.gen.Quota;
+import com.twitter.aurora.scheduler.async.OfferQueue;
+import com.twitter.aurora.scheduler.configuration.Resources;
+import com.twitter.aurora.scheduler.stats.SlotSizeCounter.ResourceSlotProvider;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.common.application.modules.LifecycleModule;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.base.Command;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+import com.twitter.common.quantity.Time;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Module to configure export of cluster-wide resource allocation and consumption statistics.
+ */
+public class AsyncStatsModule extends AbstractModule {
+
+ @CmdLine(name = "async_task_stat_update_interval",
+ help = "Interval on which to try to update resource consumption stats.")
+ private static final Arg<Amount<Long, Time>> TASK_STAT_INTERVAL =
+ Arg.create(Amount.of(1L, Time.HOURS));
+
+ @CmdLine(name = "async_slot_stat_update_interval",
+ help = "Interval on which to try to update open slot stats.")
+ private static final Arg<Amount<Long, Time>> SLOT_STAT_INTERVAL =
+ Arg.create(Amount.of(1L, Time.MINUTES));
+
+ @BindingAnnotation
+ @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+ private @interface StatExecutor { }
+
+ @Override
+ protected void configure() {
+ final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder().setNameFormat("AsyncStat-%d").setDaemon(true).build());
+
+ bind(TaskStatCalculator.class).in(Singleton.class);
+ bind(CachedCounters.class).in(Singleton.class);
+ bind(ResourceSlotProvider.class).to(OfferAdapter.class);
+ bind(SlotSizeCounter.class).in(Singleton.class);
+
+ bind(ScheduledExecutorService.class).annotatedWith(StatExecutor.class).toInstance(executor);
+ LifecycleModule.bindStartupAction(binder(), StatUpdater.class);
+ }
+
+ static class StatUpdater implements Command {
+ private final ScheduledExecutorService executor;
+ private final TaskStatCalculator taskStats;
+ private final SlotSizeCounter slotCounter;
+
+ @Inject
+ StatUpdater(
+ @StatExecutor ScheduledExecutorService executor,
+ TaskStatCalculator taskStats,
+ SlotSizeCounter slotCounter) {
+
+ this.executor = checkNotNull(executor);
+ this.taskStats = checkNotNull(taskStats);
+ this.slotCounter = checkNotNull(slotCounter);
+ }
+
+ @Override
+ public void execute() {
+ long taskInterval = TASK_STAT_INTERVAL.get().as(Time.SECONDS);
+ executor.scheduleAtFixedRate(taskStats, taskInterval, taskInterval, TimeUnit.SECONDS);
+ long slotInterval = SLOT_STAT_INTERVAL.get().as(Time.SECONDS);
+ executor.scheduleAtFixedRate(slotCounter, slotInterval, slotInterval, TimeUnit.SECONDS);
+ }
+ }
+
+ static class OfferAdapter implements ResourceSlotProvider {
+ private static final Function<Offer, IQuota> TO_QUOTA = new Function<Offer, IQuota>() {
+ @Override public IQuota apply(Offer offer) {
+ Resources resources = Resources.from(offer);
+ return IQuota.build(new Quota()
+ .setNumCpus(resources.getNumCpus())
+ .setRamMb(resources.getRam().as(Data.MB))
+ .setDiskMb(resources.getDisk().as(Data.MB)));
+ }
+ };
+
+ private final OfferQueue offerQueue;
+
+ @Inject
+ OfferAdapter(OfferQueue offerQueue) {
+ this.offerQueue = checkNotNull(offerQueue);
+ }
+
+ @Override
+ public Iterable<IQuota> get() {
+ Iterable<Offer> offers = offerQueue.getOffers();
+ return FluentIterable.from(offers).transform(TO_QUOTA);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/stats/CachedCounters.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/stats/CachedCounters.java b/src/main/java/org/apache/aurora/scheduler/stats/CachedCounters.java
new file mode 100644
index 0000000..81d6811
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/stats/CachedCounters.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.stats;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.inject.Inject;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import com.twitter.common.stats.StatsProvider;
+
+/**
+ * A cache of stats, allowing counters to be fetched and reused based on their names.
+ */
+class CachedCounters {
+ private final LoadingCache<String, AtomicLong> cache;
+
+ @Inject
+ CachedCounters(final StatsProvider stats) {
+ cache = CacheBuilder.newBuilder().build(
+ new CacheLoader<String, AtomicLong>() {
+ @Override public AtomicLong load(String key) {
+ return stats.makeCounter(key);
+ }
+ }
+ );
+ }
+
+ AtomicLong get(String name) {
+ return cache.getUnchecked(name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java b/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java
new file mode 100644
index 0000000..7b96e86
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/stats/ResourceCounter.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.stats;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Iterables;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StorageException;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * Computes aggregate metrics about resource allocation and consumption in the scheduler.
+ */
+public class ResourceCounter {
+ private final Storage storage;
+
+ @Inject
+ ResourceCounter(Storage storage) {
+ this.storage = Preconditions.checkNotNull(storage);
+ }
+
+ private Iterable<ITaskConfig> getTasks(Query.Builder query) throws StorageException {
+ return Iterables.transform(
+ Storage.Util.consistentFetchTasks(storage, query),
+ Tasks.SCHEDULED_TO_INFO);
+ }
+
+ /**
+ * Computes totals for each of the {@link MetricType}s.
+ *
+ * @return aggregates for each global metric type.
+ * @throws StorageException if there was a problem fetching tasks from storage.
+ */
+ public List<GlobalMetric> computeConsumptionTotals() throws StorageException {
+ List<GlobalMetric> counts = Arrays.asList(
+ new GlobalMetric(MetricType.TOTAL_CONSUMED),
+ new GlobalMetric(MetricType.DEDICATED_CONSUMED),
+ new GlobalMetric(MetricType.QUOTA_CONSUMED),
+ new GlobalMetric(MetricType.FREE_POOL_CONSUMED));
+
+ for (ITaskConfig task : getTasks(Query.unscoped().active())) {
+ for (GlobalMetric count : counts) {
+ count.accumulate(task);
+ }
+ }
+ return counts;
+ }
+
+ /**
+ * Computes total quota allocations.
+ *
+ * @return Total allocated quota.
+ * @throws StorageException if there was a problem fetching quotas from storage.
+ */
+ public Metric computeQuotaAllocationTotals() throws StorageException {
+ return storage.weaklyConsistentRead(new Work.Quiet<Metric>() {
+ @Override public Metric apply(StoreProvider storeProvider) {
+ Metric allocation = new Metric();
+ for (IQuota quota : storeProvider.getQuotaStore().fetchQuotas().values()) {
+ allocation.accumulate(quota);
+ }
+ return allocation;
+ }
+ });
+ }
+
+ /**
+ * Computes arbitrary resource aggregates based on a query, a filter, and a grouping function.
+ *
+ * @param query Query to select tasks for aggregation.
+ * @param filter Filter to apply on query result tasks.
+ * @param keyFunction Function to define aggregation groupings.
+ * @param <K> Key type.
+ * @return A map from the keys to their aggregates based on the tasks fetched.
+ * @throws StorageException if there was a problem fetching tasks from storage.
+ */
+ public <K> Map<K, Metric> computeAggregates(
+ Query.Builder query,
+ Predicate<ITaskConfig> filter,
+ Function<ITaskConfig, K> keyFunction) throws StorageException {
+
+ LoadingCache<K, Metric> metrics = CacheBuilder.newBuilder()
+ .build(new CacheLoader<K, Metric>() {
+ @Override public Metric load(K key) {
+ return new Metric();
+ }
+ });
+ for (ITaskConfig task : Iterables.filter(getTasks(query), filter)) {
+ metrics.getUnchecked(keyFunction.apply(task)).accumulate(task);
+ }
+ return metrics.asMap();
+ }
+
+ public enum MetricType {
+ TOTAL_CONSUMED(Predicates.<ITaskConfig>alwaysTrue()),
+ DEDICATED_CONSUMED(new Predicate<ITaskConfig>() {
+ @Override public boolean apply(ITaskConfig task) {
+ return ConfigurationManager.isDedicated(task);
+ }
+ }),
+ QUOTA_CONSUMED(new Predicate<ITaskConfig>() {
+ @Override public boolean apply(ITaskConfig task) {
+ return task.isProduction();
+ }
+ }),
+ FREE_POOL_CONSUMED(new Predicate<ITaskConfig>() {
+ @Override public boolean apply(ITaskConfig task) {
+ return !ConfigurationManager.isDedicated(task) && !task.isProduction();
+ }
+ });
+
+ public final Predicate<ITaskConfig> filter;
+
+ MetricType(Predicate<ITaskConfig> filter) {
+ this.filter = filter;
+ }
+ }
+
+ public static class GlobalMetric extends Metric {
+ public final MetricType type;
+
+ public GlobalMetric(MetricType type) {
+ this.type = type;
+ }
+
+ @Override
+ protected void accumulate(ITaskConfig task) {
+ if (type.filter.apply(task)) {
+ super.accumulate(task);
+ }
+ }
+ }
+
+ public static class Metric {
+ private long cpu = 0;
+ private long ramMb = 0;
+ private long diskMb = 0;
+
+ public Metric() {
+ this.cpu = 0;
+ this.ramMb = 0;
+ this.diskMb = 0;
+ }
+
+ public Metric(Metric copy) {
+ this.cpu = copy.cpu;
+ this.ramMb = copy.ramMb;
+ this.diskMb = copy.diskMb;
+ }
+
+ protected void accumulate(ITaskConfig task) {
+ cpu += task.getNumCpus();
+ ramMb += task.getRamMb();
+ diskMb += task.getDiskMb();
+ }
+
+ protected void accumulate(IQuota quota) {
+ cpu += quota.getNumCpus();
+ ramMb += quota.getRamMb();
+ diskMb += quota.getDiskMb();
+ }
+
+ public long getCpu() {
+ return cpu;
+ }
+
+ public long getRamGb() {
+ return ramMb / 1024;
+ }
+
+ public long getDiskGb() {
+ return diskMb / 1024;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/stats/SlotSizeCounter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/stats/SlotSizeCounter.java b/src/main/java/org/apache/aurora/scheduler/stats/SlotSizeCounter.java
new file mode 100644
index 0000000..6c0bf48
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/stats/SlotSizeCounter.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.stats;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMap;
+
+import com.twitter.aurora.gen.Quota;
+import com.twitter.aurora.scheduler.quota.Quotas;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A stat computer that aggregates the number of 'slots' available at different pre-determined
+ * slot sizes.
+ */
+class SlotSizeCounter implements Runnable {
+ private static final Map<String, IQuota> SLOT_SIZES = ImmutableMap.of(
+ "small", IQuota.build(new Quota(1.0, 1024, 4096)),
+ "medium", IQuota.build(new Quota(4.0, 8192, 16384)),
+ "large", IQuota.build(new Quota(8.0, 16384, 32768)),
+ "xlarge", IQuota.build(new Quota(16.0, 32768, 65536)));
+
+ private final Map<String, IQuota> slotSizes;
+ private final ResourceSlotProvider resourceSlotProvider;
+ private final CachedCounters cachedCounters;
+
+ @VisibleForTesting
+ SlotSizeCounter(
+ final Map<String, IQuota> slotSizes,
+ ResourceSlotProvider resourceSlotProvider,
+ CachedCounters cachedCounters) {
+
+ this.slotSizes = checkNotNull(slotSizes);
+ this.resourceSlotProvider = checkNotNull(resourceSlotProvider);
+ this.cachedCounters = checkNotNull(cachedCounters);
+ }
+
+ interface ResourceSlotProvider {
+ Iterable<IQuota> get();
+ }
+
+ @Inject
+ SlotSizeCounter(ResourceSlotProvider resourceSlotProvider, CachedCounters cachedCounters) {
+ this(SLOT_SIZES, resourceSlotProvider, cachedCounters);
+ }
+
+ @VisibleForTesting
+ static String getStatName(String slotName) {
+ return "empty_slots_" + slotName;
+ }
+
+ private int countSlots(Iterable<IQuota> slots, final IQuota slotSize) {
+ Function<IQuota, Integer> counter = new Function<IQuota, Integer>() {
+ @Override public Integer apply(IQuota machineSlack) {
+ return Quotas.divide(machineSlack, slotSize);
+ }
+ };
+
+ int sum = 0;
+ for (int slotCount : FluentIterable.from(slots).transform(counter)) {
+ sum += slotCount;
+ }
+ return sum;
+ }
+
+ @Override
+ public void run() {
+ Iterable<IQuota> slots = resourceSlotProvider.get();
+ for (Map.Entry<String, IQuota> entry : slotSizes.entrySet()) {
+ cachedCounters.get(getStatName(entry.getKey())).set(countSlots(slots, entry.getValue()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/stats/TaskStatCalculator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/stats/TaskStatCalculator.java b/src/main/java/org/apache/aurora/scheduler/stats/TaskStatCalculator.java
new file mode 100644
index 0000000..db34125
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/stats/TaskStatCalculator.java
@@ -0,0 +1,46 @@
+package com.twitter.aurora.scheduler.stats;
+
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.twitter.aurora.scheduler.stats.AsyncStatsModule.StatUpdater;
+import com.twitter.aurora.scheduler.stats.ResourceCounter.GlobalMetric;
+import com.twitter.aurora.scheduler.stats.ResourceCounter.Metric;
+import com.twitter.aurora.scheduler.storage.Storage.StorageException;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Calculates and exports aggregate stats about resources consumed by active tasks.
+ */
+class TaskStatCalculator implements Runnable {
+ private static final Logger LOG = Logger.getLogger(StatUpdater.class.getName());
+
+ private final CachedCounters counters;
+ private final ResourceCounter resourceCounter;
+
+ @Inject
+ TaskStatCalculator(ResourceCounter resourceCounter, CachedCounters counters) {
+ this.resourceCounter = checkNotNull(resourceCounter);
+ this.counters = checkNotNull(counters);
+ }
+
+ private void update(String prefix, Metric metric) {
+ counters.get(prefix + "_cpu").set(metric.getCpu());
+ counters.get(prefix + "_ram_gb").set(metric.getRamGb());
+ counters.get(prefix + "_disk_gb").set(metric.getDiskGb());
+ }
+
+ @Override
+ public void run() {
+ try {
+ for (GlobalMetric metric : resourceCounter.computeConsumptionTotals()) {
+ update("resources_" + metric.type.name().toLowerCase(), metric);
+ }
+ update("resources_allocated_quota", resourceCounter.computeQuotaAllocationTotals());
+ } catch (StorageException e) {
+ LOG.fine("Unable to fetch metrics, storage is likely not ready.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/AttributeStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/AttributeStore.java b/src/main/java/org/apache/aurora/scheduler/storage/AttributeStore.java
new file mode 100644
index 0000000..35c666c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/AttributeStore.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.Set;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+
+/**
+ * Storage interface for host attributes.
+ */
+public interface AttributeStore {
+ /**
+ * Fetches all host attributes given by the host.
+ *
+ * @param host host name.
+ * @return attributes associated with {@code host}, if the host is known.
+ */
+ Optional<HostAttributes> getHostAttributes(String host);
+
+ /**
+ * Fetches all attributes in the store.
+ *
+ * @return All host attributes.
+ */
+ Set<HostAttributes> getHostAttributes();
+
+ /**
+ * Attributes are considered mostly ephemeral and extremely low risk when inconsistency
+ * is present.
+ */
+ public interface Mutable extends AttributeStore {
+
+ /**
+ * Deletes all attributes in the store.
+ */
+ void deleteHostAttributes();
+
+ /**
+ * Save a host attribute in the attribute store.
+ *
+ * @param hostAttributes The attribute we are going to save.
+ */
+ void saveHostAttributes(HostAttributes hostAttributes);
+
+ /**
+ * Adjusts the maintenance mode for a host.
+ * No adjustment will be made if the host is unknown.
+ *
+ * @param host Host to adjust.
+ * @param mode Mode to place the host in.
+ * @return {@code true} if the host is known and the state was adjusted,
+ * {@code false} if the host is unrecognized.
+ */
+ boolean setMaintenanceMode(String host, MaintenanceMode mode);
+ }
+
+ public static final class Util {
+ private Util() {
+ }
+
+ /**
+ * Fetches attributes about a {@code host}.
+ *
+ * @param store Store to fetch host attributes from.
+ * @param host Host to fetch attributes about.
+ * @return Attributes associated with {@code host}, or an empty iterable if the host is
+ * unknown.
+ */
+ public static Iterable<Attribute> attributesOrNone(StoreProvider store, String host) {
+ Optional<HostAttributes> attributes = store.getAttributeStore().getHostAttributes(host);
+ return attributes.isPresent()
+ ? attributes.get().getAttributes() : ImmutableList.<Attribute>of();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/CallOrderEnforcingStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/CallOrderEnforcingStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/CallOrderEnforcingStorage.java
new file mode 100644
index 0000000..c71a114
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/CallOrderEnforcingStorage.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.inject.BindingAnnotation;
+import com.google.inject.Module;
+import com.google.inject.PrivateModule;
+
+import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.Event;
+import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork.NoResult.Quiet;
+import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
+import com.twitter.common.util.StateMachine;
+
+/**
+ * A non-volatile storage wrapper that enforces method call ordering.
+ */
+public class CallOrderEnforcingStorage implements NonVolatileStorage {
+
+ /**
+ * Identifies a storage whose call order should be enforced.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ ElementType.PARAMETER, ElementType.METHOD })
+ @BindingAnnotation
+ private @interface EnforceOrderOn { }
+
+ private final NonVolatileStorage wrapped;
+
+ private enum State {
+ CONSTRUCTED,
+ PREPARED,
+ READY,
+ STOPPED
+ }
+
+ private final StateMachine<State> stateMachine = StateMachine.<State>builder("storage")
+ .logTransitions()
+ .initialState(State.CONSTRUCTED)
+ .addState(State.CONSTRUCTED, State.PREPARED)
+ .addState(State.PREPARED, State.READY)
+ .addState(State.READY, State.STOPPED)
+ .build();
+
+ @Inject
+ CallOrderEnforcingStorage(@EnforceOrderOn NonVolatileStorage wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ private void checkInState(State state) throws StorageException {
+ if (stateMachine.getState() != state) {
+ throw new StorageException("Storage is not " + state);
+ }
+ }
+
+ @Override
+ public void prepare() throws StorageException {
+ checkInState(State.CONSTRUCTED);
+ wrapped.prepare();
+ stateMachine.transition(State.PREPARED);
+ }
+
+ @SendNotification(after = Event.StorageStarted)
+ @Override
+ public void start(Quiet initializationLogic) throws StorageException {
+ checkInState(State.PREPARED);
+ wrapped.start(initializationLogic);
+ stateMachine.transition(State.READY);
+ }
+
+ @Override
+ public void stop() {
+ wrapped.stop();
+ stateMachine.transition(State.STOPPED);
+ }
+
+ @Override
+ public <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
+ checkInState(State.READY);
+ return wrapped.consistentRead(work);
+ }
+
+ @Override
+ public <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work)
+ throws StorageException, E {
+
+ checkInState(State.READY);
+ return wrapped.weaklyConsistentRead(work);
+ }
+
+ @Override
+ public <T, E extends Exception> T write(MutateWork<T, E> work)
+ throws StorageException, E {
+ checkInState(State.READY);
+ return wrapped.write(work);
+ }
+
+ @Override
+ public void snapshot() throws StorageException {
+ checkInState(State.READY);
+ wrapped.snapshot();
+ }
+
+ /**
+ * Creates a binding module that will wrap a storage class with {@link CallOrderEnforcingStorage},
+ * exposing the order-enforced storage as {@link Storage} and {@link NonVolatileStorage}.
+ *
+ * @param storageClass Non-volatile storage implementation class.
+ * @return Binding module.
+ */
+ public static Module wrappingModule(final Class<? extends NonVolatileStorage> storageClass) {
+ return new PrivateModule() {
+ @Override protected void configure() {
+ bind(Storage.class).to(CallOrderEnforcingStorage.class);
+ bind(NonVolatileStorage.class).to(CallOrderEnforcingStorage.class);
+ bind(CallOrderEnforcingStorage.class).in(Singleton.class);
+ bind(NonVolatileStorage.class).annotatedWith(EnforceOrderOn.class).to(storageClass);
+ expose(Storage.class);
+ expose(NonVolatileStorage.class);
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/DistributedSnapshotStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/DistributedSnapshotStore.java b/src/main/java/org/apache/aurora/scheduler/storage/DistributedSnapshotStore.java
new file mode 100644
index 0000000..cba6303
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/DistributedSnapshotStore.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.storage.Snapshot;
+
+/**
+ * A distributed snapshot store that supports persisting globally-visible snapshots.
+ */
+public interface DistributedSnapshotStore {
+ /**
+ * Writes a snapshot to the distributed storage system.
+ * TODO(William Farner): Currently we're hiding some exceptions (which happen to be
+ * RuntimeExceptions). Clean these up to be checked, and throw another exception type here.
+ *
+ * @param snapshot Snapshot to write.
+ * @throws CodingException If the snapshot could not be serialized.
+ */
+ void persist(Snapshot snapshot) throws CodingException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/ForwardingStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/ForwardingStore.java b/src/main/java/org/apache/aurora/scheduler/storage/ForwardingStore.java
new file mode 100644
index 0000000..1a6a849
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/ForwardingStore.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.ILockKey;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A store that forwards all its operations to underlying storage systems. Useful for decorating
+ * an existing storage system.
+ */
+public class ForwardingStore implements
+ Storage,
+ SchedulerStore.Mutable,
+ JobStore.Mutable,
+ TaskStore.Mutable,
+ LockStore.Mutable,
+ QuotaStore.Mutable,
+ AttributeStore.Mutable {
+
+ private final Storage storage;
+ private final SchedulerStore.Mutable schedulerStore;
+ private final JobStore.Mutable jobStore;
+ private final TaskStore.Mutable taskStore;
+ private final LockStore.Mutable lockStore;
+ private final QuotaStore.Mutable quotaStore;
+ private final AttributeStore.Mutable attributeStore;
+
+ /**
+ * Creats a new forwarding store that delegates to the providing default stores.
+ *
+ * @param storage Delegate.
+ * @param schedulerStore Delegate.
+ * @param jobStore Delegate.
+ * @param taskStore Delegate.
+ * @param lockStore Delegate.
+ * @param quotaStore Delegate.
+ * @param attributeStore Delegate.
+ */
+ public ForwardingStore(
+ Storage storage,
+ SchedulerStore.Mutable schedulerStore,
+ JobStore.Mutable jobStore,
+ TaskStore.Mutable taskStore,
+ LockStore.Mutable lockStore,
+ QuotaStore.Mutable quotaStore,
+ AttributeStore.Mutable attributeStore) {
+
+ this.storage = checkNotNull(storage);
+ this.schedulerStore = checkNotNull(schedulerStore);
+ this.jobStore = checkNotNull(jobStore);
+ this.taskStore = checkNotNull(taskStore);
+ this.lockStore = checkNotNull(lockStore);
+ this.quotaStore = checkNotNull(quotaStore);
+ this.attributeStore = checkNotNull(attributeStore);
+ }
+
+ @Override
+ public <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
+ return storage.consistentRead(work);
+ }
+
+ @Override
+ public <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work)
+ throws StorageException, E {
+ return storage.weaklyConsistentRead(work);
+ }
+
+ @Override
+ public <T, E extends Exception> T write(MutateWork<T, E> work)
+ throws StorageException, E {
+
+ return storage.write(work);
+ }
+
+ @Override
+ public void snapshot() throws StorageException {
+ storage.snapshot();
+ }
+
+ @Override
+ public void saveFrameworkId(String frameworkId) {
+ schedulerStore.saveFrameworkId(frameworkId);
+ }
+
+ @Override
+ @Nullable
+ public String fetchFrameworkId() {
+ return schedulerStore.fetchFrameworkId();
+ }
+
+ @Override
+ public Iterable<IJobConfiguration> fetchJobs(String managerId) {
+ return jobStore.fetchJobs(managerId);
+ }
+
+ @Override
+ public Optional<IJobConfiguration> fetchJob(String managerId, IJobKey jobKey) {
+ return jobStore.fetchJob(managerId, jobKey);
+ }
+
+ @Override
+ public void saveAcceptedJob(String managerId, IJobConfiguration jobConfig) {
+ jobStore.saveAcceptedJob(managerId, jobConfig);
+ }
+
+ @Override
+ public void removeJob(IJobKey jobKey) {
+ jobStore.removeJob(jobKey);
+ }
+
+ @Override
+ public void deleteJobs() {
+ jobStore.deleteJobs();
+ }
+
+ @Override
+ public Set<String> fetchManagerIds() {
+ return jobStore.fetchManagerIds();
+ }
+
+ @Override
+ public void saveTasks(Set<IScheduledTask> tasks) throws IllegalStateException {
+ taskStore.saveTasks(tasks);
+ }
+
+ @Override
+ public void deleteAllTasks() {
+ taskStore.deleteAllTasks();
+ }
+
+ @Override
+ public void deleteTasks(Set<String> taskIds) {
+ taskStore.deleteTasks(taskIds);
+ }
+
+ @Override
+ public ImmutableSet<IScheduledTask> mutateTasks(
+ Query.Builder query,
+ Function<IScheduledTask, IScheduledTask> mutator) {
+
+ return taskStore.mutateTasks(query, mutator);
+ }
+
+ @Override
+ public boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration) {
+ return taskStore.unsafeModifyInPlace(taskId, taskConfiguration);
+ }
+
+ @Override
+ public ImmutableSet<IScheduledTask> fetchTasks(Query.Builder querySupplier) {
+ return taskStore.fetchTasks(querySupplier);
+ }
+
+ @Override
+ public Set<ILock> fetchLocks() {
+ return lockStore.fetchLocks();
+ }
+
+ @Override
+ public Optional<ILock> fetchLock(ILockKey lockKey) {
+ return lockStore.fetchLock(lockKey);
+ }
+
+ @Override
+ public void saveLock(ILock lock) {
+ lockStore.saveLock(lock);
+ }
+
+ @Override
+ public void removeLock(ILockKey lockKey) {
+ lockStore.removeLock(lockKey);
+ }
+
+ @Override
+ public void deleteLocks() {
+ lockStore.deleteLocks();
+ }
+
+ @Override
+ public Map<String, IQuota> fetchQuotas() {
+ return quotaStore.fetchQuotas();
+ }
+
+ @Override
+ public void removeQuota(String role) {
+ quotaStore.removeQuota(role);
+ }
+
+ @Override
+ public void deleteQuotas() {
+ quotaStore.deleteQuotas();
+ }
+
+ @Override
+ public void saveQuota(String role, IQuota quota) {
+ quotaStore.saveQuota(role, quota);
+ }
+
+ @Override
+ public Optional<IQuota> fetchQuota(String role) {
+ return quotaStore.fetchQuota(role);
+ }
+
+ @Override
+ public void saveHostAttributes(HostAttributes hostAttribute) {
+ attributeStore.saveHostAttributes(hostAttribute);
+ }
+
+ @Override
+ public Optional<HostAttributes> getHostAttributes(String host) {
+ return attributeStore.getHostAttributes(host);
+ }
+
+ @Override
+ public Set<HostAttributes> getHostAttributes() {
+ return attributeStore.getHostAttributes();
+ }
+
+ @Override
+ public void deleteHostAttributes() {
+ attributeStore.deleteHostAttributes();
+ }
+
+ @Override
+ public boolean setMaintenanceMode(String host, MaintenanceMode mode) {
+ return attributeStore.setMaintenanceMode(host, mode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/JobStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/JobStore.java b/src/main/java/org/apache/aurora/scheduler/storage/JobStore.java
new file mode 100644
index 0000000..7092e91
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/JobStore.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.Set;
+
+import com.google.common.base.Optional;
+
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+
+/**
+ * Stores job configuration data.
+ */
+public interface JobStore {
+
+ /**
+ * Fetches all {@code JobConfiguration}s for jobs owned by the manager identified by
+ * {@code managerId}; if there are none then an empty set is returned.
+ *
+ * @param managerId The unique identifier of the manager to find registered jobs for.
+ * @return the set of job configurations owned by the specififed job manager
+ */
+ Iterable<IJobConfiguration> fetchJobs(String managerId);
+
+ /**
+ * Fetches the {@code JobConfiguration} for the specified {@code jobKey} if it exists.
+ *
+ * @param managerId The unique identifier of the manager that accepted the job.
+ * @param jobKey The jobKey identifying the job to be fetched.
+ * @return the job configuration for the given {@code jobKey} or absent if none is found.
+ */
+ Optional<IJobConfiguration> fetchJob(String managerId, IJobKey jobKey);
+
+ /**
+ * Fetches all the unique manager ids that are present in the job store.
+ *
+ * @return The IDs of all stored job managers.
+ */
+ Set<String> fetchManagerIds();
+
+ public interface Mutable extends JobStore {
+ /**
+ * Saves the job configuration for a job that has been accepted by the scheduler. Acts as an
+ * update if the managerId already exists.
+ * TODO(William Farner): Consider accepting SanitizedConfiguration here to require that
+ * validation always happens for things entering storage.
+ *
+ * @param managerId The unique id of the manager that accepted the job.
+ * @param jobConfig The configuration of the accepted job.
+ */
+ void saveAcceptedJob(String managerId, IJobConfiguration jobConfig);
+
+ /**
+ * Removes the job configuration for the job identified by {@code jobKey}.
+ * If there is no stored configuration for the identified job, this method returns silently.
+ *
+ * @param jobKey the key identifying the job to delete.
+ */
+ void removeJob(IJobKey jobKey);
+
+ /**
+ * Deletes all jobs.
+ */
+ void deleteJobs();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/LockStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/LockStore.java b/src/main/java/org/apache/aurora/scheduler/storage/LockStore.java
new file mode 100644
index 0000000..13d7317
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/LockStore.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.Set;
+
+import com.google.common.base.Optional;
+
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.ILockKey;
+
+/**
+ * Stores all lock-related data and defines methods for saving, deleting and fetching locks.
+ */
+public interface LockStore {
+ /**
+ * Fetches all locks available in the store.
+ *
+ * @return All locks in the store.
+ */
+ Set<ILock> fetchLocks();
+
+ /**
+ * Fetches a lock by its key.
+ *
+ * @param lockKey Key of the lock to fetch.
+ * @return Optional lock.
+ */
+ Optional<ILock> fetchLock(ILockKey lockKey);
+
+ public interface Mutable extends LockStore {
+ /**
+ * Saves a new lock or overwrites the existing one with same LockKey.
+ *
+ * @param lock ILock to save.
+ */
+ void saveLock(ILock lock);
+
+ /**
+ * Removes the lock from the store.
+ *
+ * @param lockKey Key of the lock to remove.
+ */
+ void removeLock(ILockKey lockKey);
+
+ /**
+ * Deletes all locks from the store.
+ */
+ void deleteLocks();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/QuotaStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/QuotaStore.java b/src/main/java/org/apache/aurora/scheduler/storage/QuotaStore.java
new file mode 100644
index 0000000..ba8116b
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/QuotaStore.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.Map;
+
+import com.google.common.base.Optional;
+
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+
+/**
+ * Point of storage for quota records.
+ */
+public interface QuotaStore {
+ /**
+ * Fetches the existing quota record for a role.
+ *
+ * @param role Role to fetch quota for.
+ * @return Optional quota associated with {@code role}.
+ */
+ Optional<IQuota> fetchQuota(String role);
+
+ /**
+ * Fetches all allocated quotas.
+ *
+ * @return All allocated quotas.
+ */
+ Map<String, IQuota> fetchQuotas();
+
+ public interface Mutable extends QuotaStore {
+
+ /**
+ * Deletes all quotas.
+ */
+ void deleteQuotas();
+
+ /**
+ * Deletes quota for a role.
+ *
+ * @param role Role to remove quota record for.
+ */
+ void removeQuota(String role);
+
+ /**
+ * Saves a quota record for a role.
+ *
+ * @param role Role to create or update a quota record for.
+ * @param quota Quota to save.
+ */
+ void saveQuota(String role, IQuota quota);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/ReadWriteLockManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/ReadWriteLockManager.java b/src/main/java/org/apache/aurora/scheduler/storage/ReadWriteLockManager.java
new file mode 100644
index 0000000..58d7a68
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/ReadWriteLockManager.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A lock manager that wraps a ReadWriteLock and detects ill-fated attempts to upgrade
+ * a read-locked thread to a write-locked thread, which would otherwise deadlock.
+ */
+public class ReadWriteLockManager {
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ enum LockMode {
+ NONE,
+ READ,
+ WRITE
+ }
+
+ private static class LockState {
+ private LockMode initialLockMode = LockMode.NONE;
+ private int lockCount = 0;
+
+ private boolean lockAcquired(LockMode mode) {
+ boolean stateChanged = false;
+ if (initialLockMode == LockMode.NONE) {
+ initialLockMode = mode;
+ stateChanged = true;
+ }
+ if (initialLockMode == mode) {
+ lockCount++;
+ }
+ return stateChanged;
+ }
+
+ private void lockReleased(LockMode mode) {
+ if (initialLockMode == mode) {
+ lockCount--;
+ if (lockCount == 0) {
+ initialLockMode = LockMode.NONE;
+ }
+ }
+ }
+ }
+
+ private final ThreadLocal<LockState> lockState = new ThreadLocal<LockState>() {
+ @Override protected LockState initialValue() {
+ return new LockState();
+ }
+ };
+
+ /**
+ * Blocks until this thread has acquired a read lock.
+ *
+ * @return {@code true} if the lock was newly-acquired, or {@code false} if this thread previously
+ * secured the write lock and has yet to release it.
+ */
+ public boolean readLock() {
+ lock.readLock().lock();
+ return lockState.get().lockAcquired(LockMode.READ);
+ }
+
+ /**
+ * Releases this thread's read lock.
+ */
+ public void readUnlock() {
+ lock.readLock().unlock();
+ lockState.get().lockReleased(LockMode.READ);
+ }
+
+ /**
+ * Blocks until this thread has acquired a write lock.
+ *
+ * @return {@code true} if the lock was newly-acquired, or {@code false} if this thread previously
+ * secured the write lock and has yet to release it.
+ */
+ public boolean writeLock() {
+ Preconditions.checkState(lockState.get().initialLockMode != LockMode.READ,
+ "A read operation may not be upgraded to a write operation.");
+
+ lock.writeLock().lock();
+ return lockState.get().lockAcquired(LockMode.WRITE);
+ }
+
+ /**
+ * Releases this thread's write lock.
+ */
+ public void writeUnlock() {
+ lock.writeLock().unlock();
+ lockState.get().lockReleased(LockMode.WRITE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/SchedulerStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/SchedulerStore.java b/src/main/java/org/apache/aurora/scheduler/storage/SchedulerStore.java
new file mode 100644
index 0000000..504b90b
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/SchedulerStore.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import javax.annotation.Nullable;
+
+/**
+ * Stores data specific to the scheduler itself.
+ */
+public interface SchedulerStore {
+
+ /**
+ * Fetches the last saved framework id. If none is saved, null can be returned.
+ *
+ * @return the last saved framework id
+ */
+ @Nullable String fetchFrameworkId();
+
+ public interface Mutable extends SchedulerStore {
+ /**
+ * Stores the given framework id overwriting any previously saved id.
+ *
+ * @param frameworkId The framework id to store.
+ */
+ void saveFrameworkId(String frameworkId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/SnapshotStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/SnapshotStore.java b/src/main/java/org/apache/aurora/scheduler/storage/SnapshotStore.java
new file mode 100644
index 0000000..c775207
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/SnapshotStore.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+/**
+ * Storage mechanism that is able to create complete snapshots of the local storage system state
+ * and apply these to restore local storage from a snapshotted baseline.
+ */
+public interface SnapshotStore<T> {
+
+ /**
+ * Creates a consistent snapshot of the local storage system.
+ *
+ * @return A blob that can be used to recover local storage via {@link #applySnapshot(Object)}.
+ */
+ T createSnapshot();
+
+ /**
+ * Applies a snapshot blob to the local storage system, wiping out all existing data and
+ * resetting with the contents of the snapshot.
+ *
+ * @param snapshot A snapshot blob created by {@link #createSnapshot()}.
+ */
+ void applySnapshot(T snapshot);
+}
[19/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/assets/jquery.js
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/assets/jquery.js b/src/main/python/apache/thermos/observer/http/assets/jquery.js
new file mode 100644
index 0000000..3774ff9
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/assets/jquery.js
@@ -0,0 +1,9404 @@
+/*!
+ * jQuery JavaScript Library v1.7.2
+ * http://jquery.com/
+ *
+ * Copyright 2011, John Resig
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * Includes Sizzle.js
+ * http://sizzlejs.com/
+ * Copyright 2011, The Dojo Foundation
+ * Released under the MIT, BSD, and GPL Licenses.
+ *
+ * Date: Wed Mar 21 12:46:34 2012 -0700
+ */
+(function( window, undefined ) {
+
+// Use the correct document accordingly with window argument (sandbox)
+var document = window.document,
+ navigator = window.navigator,
+ location = window.location;
+var jQuery = (function() {
+
+// Define a local copy of jQuery
+var jQuery = function( selector, context ) {
+ // The jQuery object is actually just the init constructor 'enhanced'
+ return new jQuery.fn.init( selector, context, rootjQuery );
+ },
+
+ // Map over jQuery in case of overwrite
+ _jQuery = window.jQuery,
+
+ // Map over the $ in case of overwrite
+ _$ = window.$,
+
+ // A central reference to the root jQuery(document)
+ rootjQuery,
+
+ // A simple way to check for HTML strings or ID strings
+ // Prioritize #id over <tag> to avoid XSS via location.hash (#9521)
+ quickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,
+
+ // Check if a string has a non-whitespace character in it
+ rnotwhite = /\S/,
+
+ // Used for trimming whitespace
+ trimLeft = /^\s+/,
+ trimRight = /\s+$/,
+
+ // Match a standalone tag
+ rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>)?$/,
+
+ // JSON RegExp
+ rvalidchars = /^[\],:{}\s]*$/,
+ rvalidescape = /\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,
+ rvalidtokens = /"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,
+ rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g,
+
+ // Useragent RegExp
+ rwebkit = /(webkit)[ \/]([\w.]+)/,
+ ropera = /(opera)(?:.*version)?[ \/]([\w.]+)/,
+ rmsie = /(msie) ([\w.]+)/,
+ rmozilla = /(mozilla)(?:.*? rv:([\w.]+))?/,
+
+ // Matches dashed string for camelizing
+ rdashAlpha = /-([a-z]|[0-9])/ig,
+ rmsPrefix = /^-ms-/,
+
+ // Used by jQuery.camelCase as callback to replace()
+ fcamelCase = function( all, letter ) {
+ return ( letter + "" ).toUpperCase();
+ },
+
+ // Keep a UserAgent string for use with jQuery.browser
+ userAgent = navigator.userAgent,
+
+ // For matching the engine and version of the browser
+ browserMatch,
+
+ // The deferred used on DOM ready
+ readyList,
+
+ // The ready event handler
+ DOMContentLoaded,
+
+ // Save a reference to some core methods
+ toString = Object.prototype.toString,
+ hasOwn = Object.prototype.hasOwnProperty,
+ push = Array.prototype.push,
+ slice = Array.prototype.slice,
+ trim = String.prototype.trim,
+ indexOf = Array.prototype.indexOf,
+
+ // [[Class]] -> type pairs
+ class2type = {};
+
+jQuery.fn = jQuery.prototype = {
+ constructor: jQuery,
+ init: function( selector, context, rootjQuery ) {
+ var match, elem, ret, doc;
+
+ // Handle $(""), $(null), or $(undefined)
+ if ( !selector ) {
+ return this;
+ }
+
+ // Handle $(DOMElement)
+ if ( selector.nodeType ) {
+ this.context = this[0] = selector;
+ this.length = 1;
+ return this;
+ }
+
+ // The body element only exists once, optimize finding it
+ if ( selector === "body" && !context && document.body ) {
+ this.context = document;
+ this[0] = document.body;
+ this.selector = selector;
+ this.length = 1;
+ return this;
+ }
+
+ // Handle HTML strings
+ if ( typeof selector === "string" ) {
+ // Are we dealing with HTML string or an ID?
+ if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) {
+ // Assume that strings that start and end with <> are HTML and skip the regex check
+ match = [ null, selector, null ];
+
+ } else {
+ match = quickExpr.exec( selector );
+ }
+
+ // Verify a match, and that no context was specified for #id
+ if ( match && (match[1] || !context) ) {
+
+ // HANDLE: $(html) -> $(array)
+ if ( match[1] ) {
+ context = context instanceof jQuery ? context[0] : context;
+ doc = ( context ? context.ownerDocument || context : document );
+
+ // If a single string is passed in and it's a single tag
+ // just do a createElement and skip the rest
+ ret = rsingleTag.exec( selector );
+
+ if ( ret ) {
+ if ( jQuery.isPlainObject( context ) ) {
+ selector = [ document.createElement( ret[1] ) ];
+ jQuery.fn.attr.call( selector, context, true );
+
+ } else {
+ selector = [ doc.createElement( ret[1] ) ];
+ }
+
+ } else {
+ ret = jQuery.buildFragment( [ match[1] ], [ doc ] );
+ selector = ( ret.cacheable ? jQuery.clone(ret.fragment) : ret.fragment ).childNodes;
+ }
+
+ return jQuery.merge( this, selector );
+
+ // HANDLE: $("#id")
+ } else {
+ elem = document.getElementById( match[2] );
+
+ // Check parentNode to catch when Blackberry 4.6 returns
+ // nodes that are no longer in the document #6963
+ if ( elem && elem.parentNode ) {
+ // Handle the case where IE and Opera return items
+ // by name instead of ID
+ if ( elem.id !== match[2] ) {
+ return rootjQuery.find( selector );
+ }
+
+ // Otherwise, we inject the element directly into the jQuery object
+ this.length = 1;
+ this[0] = elem;
+ }
+
+ this.context = document;
+ this.selector = selector;
+ return this;
+ }
+
+ // HANDLE: $(expr, $(...))
+ } else if ( !context || context.jquery ) {
+ return ( context || rootjQuery ).find( selector );
+
+ // HANDLE: $(expr, context)
+ // (which is just equivalent to: $(context).find(expr)
+ } else {
+ return this.constructor( context ).find( selector );
+ }
+
+ // HANDLE: $(function)
+ // Shortcut for document ready
+ } else if ( jQuery.isFunction( selector ) ) {
+ return rootjQuery.ready( selector );
+ }
+
+ if ( selector.selector !== undefined ) {
+ this.selector = selector.selector;
+ this.context = selector.context;
+ }
+
+ return jQuery.makeArray( selector, this );
+ },
+
+ // Start with an empty selector
+ selector: "",
+
+ // The current version of jQuery being used
+ jquery: "1.7.2",
+
+ // The default length of a jQuery object is 0
+ length: 0,
+
+ // The number of elements contained in the matched element set
+ size: function() {
+ return this.length;
+ },
+
+ toArray: function() {
+ return slice.call( this, 0 );
+ },
+
+ // Get the Nth element in the matched element set OR
+ // Get the whole matched element set as a clean array
+ get: function( num ) {
+ return num == null ?
+
+ // Return a 'clean' array
+ this.toArray() :
+
+ // Return just the object
+ ( num < 0 ? this[ this.length + num ] : this[ num ] );
+ },
+
+ // Take an array of elements and push it onto the stack
+ // (returning the new matched element set)
+ pushStack: function( elems, name, selector ) {
+ // Build a new jQuery matched element set
+ var ret = this.constructor();
+
+ if ( jQuery.isArray( elems ) ) {
+ push.apply( ret, elems );
+
+ } else {
+ jQuery.merge( ret, elems );
+ }
+
+ // Add the old object onto the stack (as a reference)
+ ret.prevObject = this;
+
+ ret.context = this.context;
+
+ if ( name === "find" ) {
+ ret.selector = this.selector + ( this.selector ? " " : "" ) + selector;
+ } else if ( name ) {
+ ret.selector = this.selector + "." + name + "(" + selector + ")";
+ }
+
+ // Return the newly-formed element set
+ return ret;
+ },
+
+ // Execute a callback for every element in the matched set.
+ // (You can seed the arguments with an array of args, but this is
+ // only used internally.)
+ each: function( callback, args ) {
+ return jQuery.each( this, callback, args );
+ },
+
+ ready: function( fn ) {
+ // Attach the listeners
+ jQuery.bindReady();
+
+ // Add the callback
+ readyList.add( fn );
+
+ return this;
+ },
+
+ eq: function( i ) {
+ i = +i;
+ return i === -1 ?
+ this.slice( i ) :
+ this.slice( i, i + 1 );
+ },
+
+ first: function() {
+ return this.eq( 0 );
+ },
+
+ last: function() {
+ return this.eq( -1 );
+ },
+
+ slice: function() {
+ return this.pushStack( slice.apply( this, arguments ),
+ "slice", slice.call(arguments).join(",") );
+ },
+
+ map: function( callback ) {
+ return this.pushStack( jQuery.map(this, function( elem, i ) {
+ return callback.call( elem, i, elem );
+ }));
+ },
+
+ end: function() {
+ return this.prevObject || this.constructor(null);
+ },
+
+ // For internal use only.
+ // Behaves like an Array's method, not like a jQuery method.
+ push: push,
+ sort: [].sort,
+ splice: [].splice
+};
+
+// Give the init function the jQuery prototype for later instantiation
+jQuery.fn.init.prototype = jQuery.fn;
+
+jQuery.extend = jQuery.fn.extend = function() {
+ var options, name, src, copy, copyIsArray, clone,
+ target = arguments[0] || {},
+ i = 1,
+ length = arguments.length,
+ deep = false;
+
+ // Handle a deep copy situation
+ if ( typeof target === "boolean" ) {
+ deep = target;
+ target = arguments[1] || {};
+ // skip the boolean and the target
+ i = 2;
+ }
+
+ // Handle case when target is a string or something (possible in deep copy)
+ if ( typeof target !== "object" && !jQuery.isFunction(target) ) {
+ target = {};
+ }
+
+ // extend jQuery itself if only one argument is passed
+ if ( length === i ) {
+ target = this;
+ --i;
+ }
+
+ for ( ; i < length; i++ ) {
+ // Only deal with non-null/undefined values
+ if ( (options = arguments[ i ]) != null ) {
+ // Extend the base object
+ for ( name in options ) {
+ src = target[ name ];
+ copy = options[ name ];
+
+ // Prevent never-ending loop
+ if ( target === copy ) {
+ continue;
+ }
+
+ // Recurse if we're merging plain objects or arrays
+ if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) {
+ if ( copyIsArray ) {
+ copyIsArray = false;
+ clone = src && jQuery.isArray(src) ? src : [];
+
+ } else {
+ clone = src && jQuery.isPlainObject(src) ? src : {};
+ }
+
+ // Never move original objects, clone them
+ target[ name ] = jQuery.extend( deep, clone, copy );
+
+ // Don't bring in undefined values
+ } else if ( copy !== undefined ) {
+ target[ name ] = copy;
+ }
+ }
+ }
+ }
+
+ // Return the modified object
+ return target;
+};
+
+jQuery.extend({
+ noConflict: function( deep ) {
+ if ( window.$ === jQuery ) {
+ window.$ = _$;
+ }
+
+ if ( deep && window.jQuery === jQuery ) {
+ window.jQuery = _jQuery;
+ }
+
+ return jQuery;
+ },
+
+ // Is the DOM ready to be used? Set to true once it occurs.
+ isReady: false,
+
+ // A counter to track how many items to wait for before
+ // the ready event fires. See #6781
+ readyWait: 1,
+
+ // Hold (or release) the ready event
+ holdReady: function( hold ) {
+ if ( hold ) {
+ jQuery.readyWait++;
+ } else {
+ jQuery.ready( true );
+ }
+ },
+
+ // Handle when the DOM is ready
+ ready: function( wait ) {
+ // Either a released hold or an DOMready/load event and not yet ready
+ if ( (wait === true && !--jQuery.readyWait) || (wait !== true && !jQuery.isReady) ) {
+ // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443).
+ if ( !document.body ) {
+ return setTimeout( jQuery.ready, 1 );
+ }
+
+ // Remember that the DOM is ready
+ jQuery.isReady = true;
+
+ // If a normal DOM Ready event fired, decrement, and wait if need be
+ if ( wait !== true && --jQuery.readyWait > 0 ) {
+ return;
+ }
+
+ // If there are functions bound, to execute
+ readyList.fireWith( document, [ jQuery ] );
+
+ // Trigger any bound ready events
+ if ( jQuery.fn.trigger ) {
+ jQuery( document ).trigger( "ready" ).off( "ready" );
+ }
+ }
+ },
+
+ bindReady: function() {
+ if ( readyList ) {
+ return;
+ }
+
+ readyList = jQuery.Callbacks( "once memory" );
+
+ // Catch cases where $(document).ready() is called after the
+ // browser event has already occurred.
+ if ( document.readyState === "complete" ) {
+ // Handle it asynchronously to allow scripts the opportunity to delay ready
+ return setTimeout( jQuery.ready, 1 );
+ }
+
+ // Mozilla, Opera and webkit nightlies currently support this event
+ if ( document.addEventListener ) {
+ // Use the handy event callback
+ document.addEventListener( "DOMContentLoaded", DOMContentLoaded, false );
+
+ // A fallback to window.onload, that will always work
+ window.addEventListener( "load", jQuery.ready, false );
+
+ // If IE event model is used
+ } else if ( document.attachEvent ) {
+ // ensure firing before onload,
+ // maybe late but safe also for iframes
+ document.attachEvent( "onreadystatechange", DOMContentLoaded );
+
+ // A fallback to window.onload, that will always work
+ window.attachEvent( "onload", jQuery.ready );
+
+ // If IE and not a frame
+ // continually check to see if the document is ready
+ var toplevel = false;
+
+ try {
+ toplevel = window.frameElement == null;
+ } catch(e) {}
+
+ if ( document.documentElement.doScroll && toplevel ) {
+ doScrollCheck();
+ }
+ }
+ },
+
+ // See test/unit/core.js for details concerning isFunction.
+ // Since version 1.3, DOM methods and functions like alert
+ // aren't supported. They return false on IE (#2968).
+ isFunction: function( obj ) {
+ return jQuery.type(obj) === "function";
+ },
+
+ isArray: Array.isArray || function( obj ) {
+ return jQuery.type(obj) === "array";
+ },
+
+ isWindow: function( obj ) {
+ return obj != null && obj == obj.window;
+ },
+
+ isNumeric: function( obj ) {
+ return !isNaN( parseFloat(obj) ) && isFinite( obj );
+ },
+
+ type: function( obj ) {
+ return obj == null ?
+ String( obj ) :
+ class2type[ toString.call(obj) ] || "object";
+ },
+
+ isPlainObject: function( obj ) {
+ // Must be an Object.
+ // Because of IE, we also have to check the presence of the constructor property.
+ // Make sure that DOM nodes and window objects don't pass through, as well
+ if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) {
+ return false;
+ }
+
+ try {
+ // Not own constructor property must be Object
+ if ( obj.constructor &&
+ !hasOwn.call(obj, "constructor") &&
+ !hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) {
+ return false;
+ }
+ } catch ( e ) {
+ // IE8,9 Will throw exceptions on certain host objects #9897
+ return false;
+ }
+
+ // Own properties are enumerated firstly, so to speed up,
+ // if last one is own, then all properties are own.
+
+ var key;
+ for ( key in obj ) {}
+
+ return key === undefined || hasOwn.call( obj, key );
+ },
+
+ isEmptyObject: function( obj ) {
+ for ( var name in obj ) {
+ return false;
+ }
+ return true;
+ },
+
+ error: function( msg ) {
+ throw new Error( msg );
+ },
+
+ parseJSON: function( data ) {
+ if ( typeof data !== "string" || !data ) {
+ return null;
+ }
+
+ // Make sure leading/trailing whitespace is removed (IE can't handle it)
+ data = jQuery.trim( data );
+
+ // Attempt to parse using the native JSON parser first
+ if ( window.JSON && window.JSON.parse ) {
+ return window.JSON.parse( data );
+ }
+
+ // Make sure the incoming data is actual JSON
+ // Logic borrowed from http://json.org/json2.js
+ if ( rvalidchars.test( data.replace( rvalidescape, "@" )
+ .replace( rvalidtokens, "]" )
+ .replace( rvalidbraces, "")) ) {
+
+ return ( new Function( "return " + data ) )();
+
+ }
+ jQuery.error( "Invalid JSON: " + data );
+ },
+
+ // Cross-browser xml parsing
+ parseXML: function( data ) {
+ if ( typeof data !== "string" || !data ) {
+ return null;
+ }
+ var xml, tmp;
+ try {
+ if ( window.DOMParser ) { // Standard
+ tmp = new DOMParser();
+ xml = tmp.parseFromString( data , "text/xml" );
+ } else { // IE
+ xml = new ActiveXObject( "Microsoft.XMLDOM" );
+ xml.async = "false";
+ xml.loadXML( data );
+ }
+ } catch( e ) {
+ xml = undefined;
+ }
+ if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) {
+ jQuery.error( "Invalid XML: " + data );
+ }
+ return xml;
+ },
+
+ noop: function() {},
+
+ // Evaluates a script in a global context
+ // Workarounds based on findings by Jim Driscoll
+ // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context
+ globalEval: function( data ) {
+ if ( data && rnotwhite.test( data ) ) {
+ // We use execScript on Internet Explorer
+ // We use an anonymous function so that context is window
+ // rather than jQuery in Firefox
+ ( window.execScript || function( data ) {
+ window[ "eval" ].call( window, data );
+ } )( data );
+ }
+ },
+
+ // Convert dashed to camelCase; used by the css and data modules
+ // Microsoft forgot to hump their vendor prefix (#9572)
+ camelCase: function( string ) {
+ return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase );
+ },
+
+ nodeName: function( elem, name ) {
+ return elem.nodeName && elem.nodeName.toUpperCase() === name.toUpperCase();
+ },
+
+ // args is for internal usage only
+ each: function( object, callback, args ) {
+ var name, i = 0,
+ length = object.length,
+ isObj = length === undefined || jQuery.isFunction( object );
+
+ if ( args ) {
+ if ( isObj ) {
+ for ( name in object ) {
+ if ( callback.apply( object[ name ], args ) === false ) {
+ break;
+ }
+ }
+ } else {
+ for ( ; i < length; ) {
+ if ( callback.apply( object[ i++ ], args ) === false ) {
+ break;
+ }
+ }
+ }
+
+ // A special, fast, case for the most common use of each
+ } else {
+ if ( isObj ) {
+ for ( name in object ) {
+ if ( callback.call( object[ name ], name, object[ name ] ) === false ) {
+ break;
+ }
+ }
+ } else {
+ for ( ; i < length; ) {
+ if ( callback.call( object[ i ], i, object[ i++ ] ) === false ) {
+ break;
+ }
+ }
+ }
+ }
+
+ return object;
+ },
+
+ // Use native String.trim function wherever possible
+ trim: trim ?
+ function( text ) {
+ return text == null ?
+ "" :
+ trim.call( text );
+ } :
+
+ // Otherwise use our own trimming functionality
+ function( text ) {
+ return text == null ?
+ "" :
+ text.toString().replace( trimLeft, "" ).replace( trimRight, "" );
+ },
+
+ // results is for internal usage only
+ makeArray: function( array, results ) {
+ var ret = results || [];
+
+ if ( array != null ) {
+ // The window, strings (and functions) also have 'length'
+ // Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930
+ var type = jQuery.type( array );
+
+ if ( array.length == null || type === "string" || type === "function" || type === "regexp" || jQuery.isWindow( array ) ) {
+ push.call( ret, array );
+ } else {
+ jQuery.merge( ret, array );
+ }
+ }
+
+ return ret;
+ },
+
+ inArray: function( elem, array, i ) {
+ var len;
+
+ if ( array ) {
+ if ( indexOf ) {
+ return indexOf.call( array, elem, i );
+ }
+
+ len = array.length;
+ i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0;
+
+ for ( ; i < len; i++ ) {
+ // Skip accessing in sparse arrays
+ if ( i in array && array[ i ] === elem ) {
+ return i;
+ }
+ }
+ }
+
+ return -1;
+ },
+
+ merge: function( first, second ) {
+ var i = first.length,
+ j = 0;
+
+ if ( typeof second.length === "number" ) {
+ for ( var l = second.length; j < l; j++ ) {
+ first[ i++ ] = second[ j ];
+ }
+
+ } else {
+ while ( second[j] !== undefined ) {
+ first[ i++ ] = second[ j++ ];
+ }
+ }
+
+ first.length = i;
+
+ return first;
+ },
+
+ grep: function( elems, callback, inv ) {
+ var ret = [], retVal;
+ inv = !!inv;
+
+ // Go through the array, only saving the items
+ // that pass the validator function
+ for ( var i = 0, length = elems.length; i < length; i++ ) {
+ retVal = !!callback( elems[ i ], i );
+ if ( inv !== retVal ) {
+ ret.push( elems[ i ] );
+ }
+ }
+
+ return ret;
+ },
+
+ // arg is for internal usage only
+ map: function( elems, callback, arg ) {
+ var value, key, ret = [],
+ i = 0,
+ length = elems.length,
+ // jquery objects are treated as arrays
+ isArray = elems instanceof jQuery || length !== undefined && typeof length === "number" && ( ( length > 0 && elems[ 0 ] && elems[ length -1 ] ) || length === 0 || jQuery.isArray( elems ) ) ;
+
+ // Go through the array, translating each of the items to their
+ if ( isArray ) {
+ for ( ; i < length; i++ ) {
+ value = callback( elems[ i ], i, arg );
+
+ if ( value != null ) {
+ ret[ ret.length ] = value;
+ }
+ }
+
+ // Go through every key on the object,
+ } else {
+ for ( key in elems ) {
+ value = callback( elems[ key ], key, arg );
+
+ if ( value != null ) {
+ ret[ ret.length ] = value;
+ }
+ }
+ }
+
+ // Flatten any nested arrays
+ return ret.concat.apply( [], ret );
+ },
+
+ // A global GUID counter for objects
+ guid: 1,
+
+ // Bind a function to a context, optionally partially applying any
+ // arguments.
+ proxy: function( fn, context ) {
+ if ( typeof context === "string" ) {
+ var tmp = fn[ context ];
+ context = fn;
+ fn = tmp;
+ }
+
+ // Quick check to determine if target is callable, in the spec
+ // this throws a TypeError, but we will just return undefined.
+ if ( !jQuery.isFunction( fn ) ) {
+ return undefined;
+ }
+
+ // Simulated bind
+ var args = slice.call( arguments, 2 ),
+ proxy = function() {
+ return fn.apply( context, args.concat( slice.call( arguments ) ) );
+ };
+
+ // Set the guid of unique handler to the same of original handler, so it can be removed
+ proxy.guid = fn.guid = fn.guid || proxy.guid || jQuery.guid++;
+
+ return proxy;
+ },
+
+ // Mutifunctional method to get and set values to a collection
+ // The value/s can optionally be executed if it's a function
+ access: function( elems, fn, key, value, chainable, emptyGet, pass ) {
+ var exec,
+ bulk = key == null,
+ i = 0,
+ length = elems.length;
+
+ // Sets many values
+ if ( key && typeof key === "object" ) {
+ for ( i in key ) {
+ jQuery.access( elems, fn, i, key[i], 1, emptyGet, value );
+ }
+ chainable = 1;
+
+ // Sets one value
+ } else if ( value !== undefined ) {
+ // Optionally, function values get executed if exec is true
+ exec = pass === undefined && jQuery.isFunction( value );
+
+ if ( bulk ) {
+ // Bulk operations only iterate when executing function values
+ if ( exec ) {
+ exec = fn;
+ fn = function( elem, key, value ) {
+ return exec.call( jQuery( elem ), value );
+ };
+
+ // Otherwise they run against the entire set
+ } else {
+ fn.call( elems, value );
+ fn = null;
+ }
+ }
+
+ if ( fn ) {
+ for (; i < length; i++ ) {
+ fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass );
+ }
+ }
+
+ chainable = 1;
+ }
+
+ return chainable ?
+ elems :
+
+ // Gets
+ bulk ?
+ fn.call( elems ) :
+ length ? fn( elems[0], key ) : emptyGet;
+ },
+
+ now: function() {
+ return ( new Date() ).getTime();
+ },
+
+ // Use of jQuery.browser is frowned upon.
+ // More details: http://docs.jquery.com/Utilities/jQuery.browser
+ uaMatch: function( ua ) {
+ ua = ua.toLowerCase();
+
+ var match = rwebkit.exec( ua ) ||
+ ropera.exec( ua ) ||
+ rmsie.exec( ua ) ||
+ ua.indexOf("compatible") < 0 && rmozilla.exec( ua ) ||
+ [];
+
+ return { browser: match[1] || "", version: match[2] || "0" };
+ },
+
+ sub: function() {
+ function jQuerySub( selector, context ) {
+ return new jQuerySub.fn.init( selector, context );
+ }
+ jQuery.extend( true, jQuerySub, this );
+ jQuerySub.superclass = this;
+ jQuerySub.fn = jQuerySub.prototype = this();
+ jQuerySub.fn.constructor = jQuerySub;
+ jQuerySub.sub = this.sub;
+ jQuerySub.fn.init = function init( selector, context ) {
+ if ( context && context instanceof jQuery && !(context instanceof jQuerySub) ) {
+ context = jQuerySub( context );
+ }
+
+ return jQuery.fn.init.call( this, selector, context, rootjQuerySub );
+ };
+ jQuerySub.fn.init.prototype = jQuerySub.fn;
+ var rootjQuerySub = jQuerySub(document);
+ return jQuerySub;
+ },
+
+ browser: {}
+});
+
+// Populate the class2type map
+jQuery.each("Boolean Number String Function Array Date RegExp Object".split(" "), function(i, name) {
+ class2type[ "[object " + name + "]" ] = name.toLowerCase();
+});
+
+browserMatch = jQuery.uaMatch( userAgent );
+if ( browserMatch.browser ) {
+ jQuery.browser[ browserMatch.browser ] = true;
+ jQuery.browser.version = browserMatch.version;
+}
+
+// Deprecated, use jQuery.browser.webkit instead
+if ( jQuery.browser.webkit ) {
+ jQuery.browser.safari = true;
+}
+
+// IE doesn't match non-breaking spaces with \s
+if ( rnotwhite.test( "\xA0" ) ) {
+ trimLeft = /^[\s\xA0]+/;
+ trimRight = /[\s\xA0]+$/;
+}
+
+// All jQuery objects should point back to these
+rootjQuery = jQuery(document);
+
+// Cleanup functions for the document ready method
+if ( document.addEventListener ) {
+ DOMContentLoaded = function() {
+ document.removeEventListener( "DOMContentLoaded", DOMContentLoaded, false );
+ jQuery.ready();
+ };
+
+} else if ( document.attachEvent ) {
+ DOMContentLoaded = function() {
+ // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443).
+ if ( document.readyState === "complete" ) {
+ document.detachEvent( "onreadystatechange", DOMContentLoaded );
+ jQuery.ready();
+ }
+ };
+}
+
+// The DOM ready check for Internet Explorer
+function doScrollCheck() {
+ if ( jQuery.isReady ) {
+ return;
+ }
+
+ try {
+ // If IE is used, use the trick by Diego Perini
+ // http://javascript.nwbox.com/IEContentLoaded/
+ document.documentElement.doScroll("left");
+ } catch(e) {
+ setTimeout( doScrollCheck, 1 );
+ return;
+ }
+
+ // and execute any waiting functions
+ jQuery.ready();
+}
+
+return jQuery;
+
+})();
+
+
+// String to Object flags format cache
+var flagsCache = {};
+
+// Convert String-formatted flags into Object-formatted ones and store in cache
+function createFlags( flags ) {
+ var object = flagsCache[ flags ] = {},
+ i, length;
+ flags = flags.split( /\s+/ );
+ for ( i = 0, length = flags.length; i < length; i++ ) {
+ object[ flags[i] ] = true;
+ }
+ return object;
+}
+
+/*
+ * Create a callback list using the following parameters:
+ *
+ * flags: an optional list of space-separated flags that will change how
+ * the callback list behaves
+ *
+ * By default a callback list will act like an event callback list and can be
+ * "fired" multiple times.
+ *
+ * Possible flags:
+ *
+ * once: will ensure the callback list can only be fired once (like a Deferred)
+ *
+ * memory: will keep track of previous values and will call any callback added
+ * after the list has been fired right away with the latest "memorized"
+ * values (like a Deferred)
+ *
+ * unique: will ensure a callback can only be added once (no duplicate in the list)
+ *
+ * stopOnFalse: interrupt callings when a callback returns false
+ *
+ */
+jQuery.Callbacks = function( flags ) {
+
+ // Convert flags from String-formatted to Object-formatted
+ // (we check in cache first)
+ flags = flags ? ( flagsCache[ flags ] || createFlags( flags ) ) : {};
+
+ var // Actual callback list
+ list = [],
+ // Stack of fire calls for repeatable lists
+ stack = [],
+ // Last fire value (for non-forgettable lists)
+ memory,
+ // Flag to know if list was already fired
+ fired,
+ // Flag to know if list is currently firing
+ firing,
+ // First callback to fire (used internally by add and fireWith)
+ firingStart,
+ // End of the loop when firing
+ firingLength,
+ // Index of currently firing callback (modified by remove if needed)
+ firingIndex,
+ // Add one or several callbacks to the list
+ add = function( args ) {
+ var i,
+ length,
+ elem,
+ type,
+ actual;
+ for ( i = 0, length = args.length; i < length; i++ ) {
+ elem = args[ i ];
+ type = jQuery.type( elem );
+ if ( type === "array" ) {
+ // Inspect recursively
+ add( elem );
+ } else if ( type === "function" ) {
+ // Add if not in unique mode and callback is not in
+ if ( !flags.unique || !self.has( elem ) ) {
+ list.push( elem );
+ }
+ }
+ }
+ },
+ // Fire callbacks
+ fire = function( context, args ) {
+ args = args || [];
+ memory = !flags.memory || [ context, args ];
+ fired = true;
+ firing = true;
+ firingIndex = firingStart || 0;
+ firingStart = 0;
+ firingLength = list.length;
+ for ( ; list && firingIndex < firingLength; firingIndex++ ) {
+ if ( list[ firingIndex ].apply( context, args ) === false && flags.stopOnFalse ) {
+ memory = true; // Mark as halted
+ break;
+ }
+ }
+ firing = false;
+ if ( list ) {
+ if ( !flags.once ) {
+ if ( stack && stack.length ) {
+ memory = stack.shift();
+ self.fireWith( memory[ 0 ], memory[ 1 ] );
+ }
+ } else if ( memory === true ) {
+ self.disable();
+ } else {
+ list = [];
+ }
+ }
+ },
+ // Actual Callbacks object
+ self = {
+ // Add a callback or a collection of callbacks to the list
+ add: function() {
+ if ( list ) {
+ var length = list.length;
+ add( arguments );
+ // Do we need to add the callbacks to the
+ // current firing batch?
+ if ( firing ) {
+ firingLength = list.length;
+ // With memory, if we're not firing then
+ // we should call right away, unless previous
+ // firing was halted (stopOnFalse)
+ } else if ( memory && memory !== true ) {
+ firingStart = length;
+ fire( memory[ 0 ], memory[ 1 ] );
+ }
+ }
+ return this;
+ },
+ // Remove a callback from the list
+ remove: function() {
+ if ( list ) {
+ var args = arguments,
+ argIndex = 0,
+ argLength = args.length;
+ for ( ; argIndex < argLength ; argIndex++ ) {
+ for ( var i = 0; i < list.length; i++ ) {
+ if ( args[ argIndex ] === list[ i ] ) {
+ // Handle firingIndex and firingLength
+ if ( firing ) {
+ if ( i <= firingLength ) {
+ firingLength--;
+ if ( i <= firingIndex ) {
+ firingIndex--;
+ }
+ }
+ }
+ // Remove the element
+ list.splice( i--, 1 );
+ // If we have some unicity property then
+ // we only need to do this once
+ if ( flags.unique ) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ return this;
+ },
+ // Control if a given callback is in the list
+ has: function( fn ) {
+ if ( list ) {
+ var i = 0,
+ length = list.length;
+ for ( ; i < length; i++ ) {
+ if ( fn === list[ i ] ) {
+ return true;
+ }
+ }
+ }
+ return false;
+ },
+ // Remove all callbacks from the list
+ empty: function() {
+ list = [];
+ return this;
+ },
+ // Have the list do nothing anymore
+ disable: function() {
+ list = stack = memory = undefined;
+ return this;
+ },
+ // Is it disabled?
+ disabled: function() {
+ return !list;
+ },
+ // Lock the list in its current state
+ lock: function() {
+ stack = undefined;
+ if ( !memory || memory === true ) {
+ self.disable();
+ }
+ return this;
+ },
+ // Is it locked?
+ locked: function() {
+ return !stack;
+ },
+ // Call all callbacks with the given context and arguments
+ fireWith: function( context, args ) {
+ if ( stack ) {
+ if ( firing ) {
+ if ( !flags.once ) {
+ stack.push( [ context, args ] );
+ }
+ } else if ( !( flags.once && memory ) ) {
+ fire( context, args );
+ }
+ }
+ return this;
+ },
+ // Call all the callbacks with the given arguments
+ fire: function() {
+ self.fireWith( this, arguments );
+ return this;
+ },
+ // To know if the callbacks have already been called at least once
+ fired: function() {
+ return !!fired;
+ }
+ };
+
+ return self;
+};
+
+
+
+
+var // Static reference to slice
+ sliceDeferred = [].slice;
+
+jQuery.extend({
+
+ Deferred: function( func ) {
+ var doneList = jQuery.Callbacks( "once memory" ),
+ failList = jQuery.Callbacks( "once memory" ),
+ progressList = jQuery.Callbacks( "memory" ),
+ state = "pending",
+ lists = {
+ resolve: doneList,
+ reject: failList,
+ notify: progressList
+ },
+ promise = {
+ done: doneList.add,
+ fail: failList.add,
+ progress: progressList.add,
+
+ state: function() {
+ return state;
+ },
+
+ // Deprecated
+ isResolved: doneList.fired,
+ isRejected: failList.fired,
+
+ then: function( doneCallbacks, failCallbacks, progressCallbacks ) {
+ deferred.done( doneCallbacks ).fail( failCallbacks ).progress( progressCallbacks );
+ return this;
+ },
+ always: function() {
+ deferred.done.apply( deferred, arguments ).fail.apply( deferred, arguments );
+ return this;
+ },
+ pipe: function( fnDone, fnFail, fnProgress ) {
+ return jQuery.Deferred(function( newDefer ) {
+ jQuery.each( {
+ done: [ fnDone, "resolve" ],
+ fail: [ fnFail, "reject" ],
+ progress: [ fnProgress, "notify" ]
+ }, function( handler, data ) {
+ var fn = data[ 0 ],
+ action = data[ 1 ],
+ returned;
+ if ( jQuery.isFunction( fn ) ) {
+ deferred[ handler ](function() {
+ returned = fn.apply( this, arguments );
+ if ( returned && jQuery.isFunction( returned.promise ) ) {
+ returned.promise().then( newDefer.resolve, newDefer.reject, newDefer.notify );
+ } else {
+ newDefer[ action + "With" ]( this === deferred ? newDefer : this, [ returned ] );
+ }
+ });
+ } else {
+ deferred[ handler ]( newDefer[ action ] );
+ }
+ });
+ }).promise();
+ },
+ // Get a promise for this deferred
+ // If obj is provided, the promise aspect is added to the object
+ promise: function( obj ) {
+ if ( obj == null ) {
+ obj = promise;
+ } else {
+ for ( var key in promise ) {
+ obj[ key ] = promise[ key ];
+ }
+ }
+ return obj;
+ }
+ },
+ deferred = promise.promise({}),
+ key;
+
+ for ( key in lists ) {
+ deferred[ key ] = lists[ key ].fire;
+ deferred[ key + "With" ] = lists[ key ].fireWith;
+ }
+
+ // Handle state
+ deferred.done( function() {
+ state = "resolved";
+ }, failList.disable, progressList.lock ).fail( function() {
+ state = "rejected";
+ }, doneList.disable, progressList.lock );
+
+ // Call given func if any
+ if ( func ) {
+ func.call( deferred, deferred );
+ }
+
+ // All done!
+ return deferred;
+ },
+
+ // Deferred helper
+ when: function( firstParam ) {
+ var args = sliceDeferred.call( arguments, 0 ),
+ i = 0,
+ length = args.length,
+ pValues = new Array( length ),
+ count = length,
+ pCount = length,
+ deferred = length <= 1 && firstParam && jQuery.isFunction( firstParam.promise ) ?
+ firstParam :
+ jQuery.Deferred(),
+ promise = deferred.promise();
+ function resolveFunc( i ) {
+ return function( value ) {
+ args[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value;
+ if ( !( --count ) ) {
+ deferred.resolveWith( deferred, args );
+ }
+ };
+ }
+ function progressFunc( i ) {
+ return function( value ) {
+ pValues[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value;
+ deferred.notifyWith( promise, pValues );
+ };
+ }
+ if ( length > 1 ) {
+ for ( ; i < length; i++ ) {
+ if ( args[ i ] && args[ i ].promise && jQuery.isFunction( args[ i ].promise ) ) {
+ args[ i ].promise().then( resolveFunc(i), deferred.reject, progressFunc(i) );
+ } else {
+ --count;
+ }
+ }
+ if ( !count ) {
+ deferred.resolveWith( deferred, args );
+ }
+ } else if ( deferred !== firstParam ) {
+ deferred.resolveWith( deferred, length ? [ firstParam ] : [] );
+ }
+ return promise;
+ }
+});
+
+
+
+
+jQuery.support = (function() {
+
+ var support,
+ all,
+ a,
+ select,
+ opt,
+ input,
+ fragment,
+ tds,
+ events,
+ eventName,
+ i,
+ isSupported,
+ div = document.createElement( "div" ),
+ documentElement = document.documentElement;
+
+ // Preliminary tests
+ div.setAttribute("className", "t");
+ div.innerHTML = " <link/><table></table><a href='/a' style='top:1px;float:left;opacity:.55;'>a</a><input type='checkbox'/>";
+
+ all = div.getElementsByTagName( "*" );
+ a = div.getElementsByTagName( "a" )[ 0 ];
+
+ // Can't get basic test support
+ if ( !all || !all.length || !a ) {
+ return {};
+ }
+
+ // First batch of supports tests
+ select = document.createElement( "select" );
+ opt = select.appendChild( document.createElement("option") );
+ input = div.getElementsByTagName( "input" )[ 0 ];
+
+ support = {
+ // IE strips leading whitespace when .innerHTML is used
+ leadingWhitespace: ( div.firstChild.nodeType === 3 ),
+
+ // Make sure that tbody elements aren't automatically inserted
+ // IE will insert them into empty tables
+ tbody: !div.getElementsByTagName("tbody").length,
+
+ // Make sure that link elements get serialized correctly by innerHTML
+ // This requires a wrapper element in IE
+ htmlSerialize: !!div.getElementsByTagName("link").length,
+
+ // Get the style information from getAttribute
+ // (IE uses .cssText instead)
+ style: /top/.test( a.getAttribute("style") ),
+
+ // Make sure that URLs aren't manipulated
+ // (IE normalizes it by default)
+ hrefNormalized: ( a.getAttribute("href") === "/a" ),
+
+ // Make sure that element opacity exists
+ // (IE uses filter instead)
+ // Use a regex to work around a WebKit issue. See #5145
+ opacity: /^0.55/.test( a.style.opacity ),
+
+ // Verify style float existence
+ // (IE uses styleFloat instead of cssFloat)
+ cssFloat: !!a.style.cssFloat,
+
+ // Make sure that if no value is specified for a checkbox
+ // that it defaults to "on".
+ // (WebKit defaults to "" instead)
+ checkOn: ( input.value === "on" ),
+
+ // Make sure that a selected-by-default option has a working selected property.
+ // (WebKit defaults to false instead of true, IE too, if it's in an optgroup)
+ optSelected: opt.selected,
+
+ // Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7)
+ getSetAttribute: div.className !== "t",
+
+ // Tests for enctype support on a form(#6743)
+ enctype: !!document.createElement("form").enctype,
+
+ // Makes sure cloning an html5 element does not cause problems
+ // Where outerHTML is undefined, this still works
+ html5Clone: document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav></:nav>",
+
+ // Will be defined later
+ submitBubbles: true,
+ changeBubbles: true,
+ focusinBubbles: false,
+ deleteExpando: true,
+ noCloneEvent: true,
+ inlineBlockNeedsLayout: false,
+ shrinkWrapBlocks: false,
+ reliableMarginRight: true,
+ pixelMargin: true
+ };
+
+ // jQuery.boxModel DEPRECATED in 1.3, use jQuery.support.boxModel instead
+ jQuery.boxModel = support.boxModel = (document.compatMode === "CSS1Compat");
+
+ // Make sure checked status is properly cloned
+ input.checked = true;
+ support.noCloneChecked = input.cloneNode( true ).checked;
+
+ // Make sure that the options inside disabled selects aren't marked as disabled
+ // (WebKit marks them as disabled)
+ select.disabled = true;
+ support.optDisabled = !opt.disabled;
+
+ // Test to see if it's possible to delete an expando from an element
+ // Fails in Internet Explorer
+ try {
+ delete div.test;
+ } catch( e ) {
+ support.deleteExpando = false;
+ }
+
+ if ( !div.addEventListener && div.attachEvent && div.fireEvent ) {
+ div.attachEvent( "onclick", function() {
+ // Cloning a node shouldn't copy over any
+ // bound event handlers (IE does this)
+ support.noCloneEvent = false;
+ });
+ div.cloneNode( true ).fireEvent( "onclick" );
+ }
+
+ // Check if a radio maintains its value
+ // after being appended to the DOM
+ input = document.createElement("input");
+ input.value = "t";
+ input.setAttribute("type", "radio");
+ support.radioValue = input.value === "t";
+
+ input.setAttribute("checked", "checked");
+
+ // #11217 - WebKit loses check when the name is after the checked attribute
+ input.setAttribute( "name", "t" );
+
+ div.appendChild( input );
+ fragment = document.createDocumentFragment();
+ fragment.appendChild( div.lastChild );
+
+ // WebKit doesn't clone checked state correctly in fragments
+ support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked;
+
+ // Check if a disconnected checkbox will retain its checked
+ // value of true after appended to the DOM (IE6/7)
+ support.appendChecked = input.checked;
+
+ fragment.removeChild( input );
+ fragment.appendChild( div );
+
+ // Technique from Juriy Zaytsev
+ // http://perfectionkills.com/detecting-event-support-without-browser-sniffing/
+ // We only care about the case where non-standard event systems
+ // are used, namely in IE. Short-circuiting here helps us to
+ // avoid an eval call (in setAttribute) which can cause CSP
+ // to go haywire. See: https://developer.mozilla.org/en/Security/CSP
+ if ( div.attachEvent ) {
+ for ( i in {
+ submit: 1,
+ change: 1,
+ focusin: 1
+ }) {
+ eventName = "on" + i;
+ isSupported = ( eventName in div );
+ if ( !isSupported ) {
+ div.setAttribute( eventName, "return;" );
+ isSupported = ( typeof div[ eventName ] === "function" );
+ }
+ support[ i + "Bubbles" ] = isSupported;
+ }
+ }
+
+ fragment.removeChild( div );
+
+ // Null elements to avoid leaks in IE
+ fragment = select = opt = div = input = null;
+
+ // Run tests that need a body at doc ready
+ jQuery(function() {
+ var container, outer, inner, table, td, offsetSupport,
+ marginDiv, conMarginTop, style, html, positionTopLeftWidthHeight,
+ paddingMarginBorderVisibility, paddingMarginBorder,
+ body = document.getElementsByTagName("body")[0];
+
+ if ( !body ) {
+ // Return for frameset docs that don't have a body
+ return;
+ }
+
+ conMarginTop = 1;
+ paddingMarginBorder = "padding:0;margin:0;border:";
+ positionTopLeftWidthHeight = "position:absolute;top:0;left:0;width:1px;height:1px;";
+ paddingMarginBorderVisibility = paddingMarginBorder + "0;visibility:hidden;";
+ style = "style='" + positionTopLeftWidthHeight + paddingMarginBorder + "5px solid #000;";
+ html = "<div " + style + "display:block;'><div style='" + paddingMarginBorder + "0;display:block;overflow:hidden;'></div></div>" +
+ "<table " + style + "' cellpadding='0' cellspacing='0'>" +
+ "<tr><td></td></tr></table>";
+
+ container = document.createElement("div");
+ container.style.cssText = paddingMarginBorderVisibility + "width:0;height:0;position:static;top:0;margin-top:" + conMarginTop + "px";
+ body.insertBefore( container, body.firstChild );
+
+ // Construct the test element
+ div = document.createElement("div");
+ container.appendChild( div );
+
+ // Check if table cells still have offsetWidth/Height when they are set
+ // to display:none and there are still other visible table cells in a
+ // table row; if so, offsetWidth/Height are not reliable for use when
+ // determining if an element has been hidden directly using
+ // display:none (it is still safe to use offsets if a parent element is
+ // hidden; don safety goggles and see bug #4512 for more information).
+ // (only IE 8 fails this test)
+ div.innerHTML = "<table><tr><td style='" + paddingMarginBorder + "0;display:none'></td><td>t</td></tr></table>";
+ tds = div.getElementsByTagName( "td" );
+ isSupported = ( tds[ 0 ].offsetHeight === 0 );
+
+ tds[ 0 ].style.display = "";
+ tds[ 1 ].style.display = "none";
+
+ // Check if empty table cells still have offsetWidth/Height
+ // (IE <= 8 fail this test)
+ support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 );
+
+ // Check if div with explicit width and no margin-right incorrectly
+ // gets computed margin-right based on width of container. For more
+ // info see bug #3333
+ // Fails in WebKit before Feb 2011 nightlies
+ // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
+ if ( window.getComputedStyle ) {
+ div.innerHTML = "";
+ marginDiv = document.createElement( "div" );
+ marginDiv.style.width = "0";
+ marginDiv.style.marginRight = "0";
+ div.style.width = "2px";
+ div.appendChild( marginDiv );
+ support.reliableMarginRight =
+ ( parseInt( ( window.getComputedStyle( marginDiv, null ) || { marginRight: 0 } ).marginRight, 10 ) || 0 ) === 0;
+ }
+
+ if ( typeof div.style.zoom !== "undefined" ) {
+ // Check if natively block-level elements act like inline-block
+ // elements when setting their display to 'inline' and giving
+ // them layout
+ // (IE < 8 does this)
+ div.innerHTML = "";
+ div.style.width = div.style.padding = "1px";
+ div.style.border = 0;
+ div.style.overflow = "hidden";
+ div.style.display = "inline";
+ div.style.zoom = 1;
+ support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 );
+
+ // Check if elements with layout shrink-wrap their children
+ // (IE 6 does this)
+ div.style.display = "block";
+ div.style.overflow = "visible";
+ div.innerHTML = "<div style='width:5px;'></div>";
+ support.shrinkWrapBlocks = ( div.offsetWidth !== 3 );
+ }
+
+ div.style.cssText = positionTopLeftWidthHeight + paddingMarginBorderVisibility;
+ div.innerHTML = html;
+
+ outer = div.firstChild;
+ inner = outer.firstChild;
+ td = outer.nextSibling.firstChild.firstChild;
+
+ offsetSupport = {
+ doesNotAddBorder: ( inner.offsetTop !== 5 ),
+ doesAddBorderForTableAndCells: ( td.offsetTop === 5 )
+ };
+
+ inner.style.position = "fixed";
+ inner.style.top = "20px";
+
+ // safari subtracts parent border width here which is 5px
+ offsetSupport.fixedPosition = ( inner.offsetTop === 20 || inner.offsetTop === 15 );
+ inner.style.position = inner.style.top = "";
+
+ outer.style.overflow = "hidden";
+ outer.style.position = "relative";
+
+ offsetSupport.subtractsBorderForOverflowNotVisible = ( inner.offsetTop === -5 );
+ offsetSupport.doesNotIncludeMarginInBodyOffset = ( body.offsetTop !== conMarginTop );
+
+ if ( window.getComputedStyle ) {
+ div.style.marginTop = "1%";
+ support.pixelMargin = ( window.getComputedStyle( div, null ) || { marginTop: 0 } ).marginTop !== "1%";
+ }
+
+ if ( typeof container.style.zoom !== "undefined" ) {
+ container.style.zoom = 1;
+ }
+
+ body.removeChild( container );
+ marginDiv = div = container = null;
+
+ jQuery.extend( support, offsetSupport );
+ });
+
+ return support;
+})();
+
+
+
+
+var rbrace = /^(?:\{.*\}|\[.*\])$/,
+ rmultiDash = /([A-Z])/g;
+
+jQuery.extend({
+ cache: {},
+
+ // Please use with caution
+ uuid: 0,
+
+ // Unique for each copy of jQuery on the page
+ // Non-digits removed to match rinlinejQuery
+ expando: "jQuery" + ( jQuery.fn.jquery + Math.random() ).replace( /\D/g, "" ),
+
+ // The following elements throw uncatchable exceptions if you
+ // attempt to add expando properties to them.
+ noData: {
+ "embed": true,
+ // Ban all objects except for Flash (which handle expandos)
+ "object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",
+ "applet": true
+ },
+
+ hasData: function( elem ) {
+ elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ];
+ return !!elem && !isEmptyDataObject( elem );
+ },
+
+ data: function( elem, name, data, pvt /* Internal Use Only */ ) {
+ if ( !jQuery.acceptData( elem ) ) {
+ return;
+ }
+
+ var privateCache, thisCache, ret,
+ internalKey = jQuery.expando,
+ getByName = typeof name === "string",
+
+ // We have to handle DOM nodes and JS objects differently because IE6-7
+ // can't GC object references properly across the DOM-JS boundary
+ isNode = elem.nodeType,
+
+ // Only DOM nodes need the global jQuery cache; JS object data is
+ // attached directly to the object so GC can occur automatically
+ cache = isNode ? jQuery.cache : elem,
+
+ // Only defining an ID for JS objects if its cache already exists allows
+ // the code to shortcut on the same path as a DOM node with no cache
+ id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey,
+ isEvents = name === "events";
+
+ // Avoid doing any more work than we need to when trying to get data on an
+ // object that has no data at all
+ if ( (!id || !cache[id] || (!isEvents && !pvt && !cache[id].data)) && getByName && data === undefined ) {
+ return;
+ }
+
+ if ( !id ) {
+ // Only DOM nodes need a new unique ID for each element since their data
+ // ends up in the global cache
+ if ( isNode ) {
+ elem[ internalKey ] = id = ++jQuery.uuid;
+ } else {
+ id = internalKey;
+ }
+ }
+
+ if ( !cache[ id ] ) {
+ cache[ id ] = {};
+
+ // Avoids exposing jQuery metadata on plain JS objects when the object
+ // is serialized using JSON.stringify
+ if ( !isNode ) {
+ cache[ id ].toJSON = jQuery.noop;
+ }
+ }
+
+ // An object can be passed to jQuery.data instead of a key/value pair; this gets
+ // shallow copied over onto the existing cache
+ if ( typeof name === "object" || typeof name === "function" ) {
+ if ( pvt ) {
+ cache[ id ] = jQuery.extend( cache[ id ], name );
+ } else {
+ cache[ id ].data = jQuery.extend( cache[ id ].data, name );
+ }
+ }
+
+ privateCache = thisCache = cache[ id ];
+
+ // jQuery data() is stored in a separate object inside the object's internal data
+ // cache in order to avoid key collisions between internal data and user-defined
+ // data.
+ if ( !pvt ) {
+ if ( !thisCache.data ) {
+ thisCache.data = {};
+ }
+
+ thisCache = thisCache.data;
+ }
+
+ if ( data !== undefined ) {
+ thisCache[ jQuery.camelCase( name ) ] = data;
+ }
+
+ // Users should not attempt to inspect the internal events object using jQuery.data,
+ // it is undocumented and subject to change. But does anyone listen? No.
+ if ( isEvents && !thisCache[ name ] ) {
+ return privateCache.events;
+ }
+
+ // Check for both converted-to-camel and non-converted data property names
+ // If a data property was specified
+ if ( getByName ) {
+
+ // First Try to find as-is property data
+ ret = thisCache[ name ];
+
+ // Test for null|undefined property data
+ if ( ret == null ) {
+
+ // Try to find the camelCased property
+ ret = thisCache[ jQuery.camelCase( name ) ];
+ }
+ } else {
+ ret = thisCache;
+ }
+
+ return ret;
+ },
+
+ removeData: function( elem, name, pvt /* Internal Use Only */ ) {
+ if ( !jQuery.acceptData( elem ) ) {
+ return;
+ }
+
+ var thisCache, i, l,
+
+ // Reference to internal data cache key
+ internalKey = jQuery.expando,
+
+ isNode = elem.nodeType,
+
+ // See jQuery.data for more information
+ cache = isNode ? jQuery.cache : elem,
+
+ // See jQuery.data for more information
+ id = isNode ? elem[ internalKey ] : internalKey;
+
+ // If there is already no cache entry for this object, there is no
+ // purpose in continuing
+ if ( !cache[ id ] ) {
+ return;
+ }
+
+ if ( name ) {
+
+ thisCache = pvt ? cache[ id ] : cache[ id ].data;
+
+ if ( thisCache ) {
+
+ // Support array or space separated string names for data keys
+ if ( !jQuery.isArray( name ) ) {
+
+ // try the string as a key before any manipulation
+ if ( name in thisCache ) {
+ name = [ name ];
+ } else {
+
+ // split the camel cased version by spaces unless a key with the spaces exists
+ name = jQuery.camelCase( name );
+ if ( name in thisCache ) {
+ name = [ name ];
+ } else {
+ name = name.split( " " );
+ }
+ }
+ }
+
+ for ( i = 0, l = name.length; i < l; i++ ) {
+ delete thisCache[ name[i] ];
+ }
+
+ // If there is no data left in the cache, we want to continue
+ // and let the cache object itself get destroyed
+ if ( !( pvt ? isEmptyDataObject : jQuery.isEmptyObject )( thisCache ) ) {
+ return;
+ }
+ }
+ }
+
+ // See jQuery.data for more information
+ if ( !pvt ) {
+ delete cache[ id ].data;
+
+ // Don't destroy the parent cache unless the internal data object
+ // had been the only thing left in it
+ if ( !isEmptyDataObject(cache[ id ]) ) {
+ return;
+ }
+ }
+
+ // Browsers that fail expando deletion also refuse to delete expandos on
+ // the window, but it will allow it on all other JS objects; other browsers
+ // don't care
+ // Ensure that `cache` is not a window object #10080
+ if ( jQuery.support.deleteExpando || !cache.setInterval ) {
+ delete cache[ id ];
+ } else {
+ cache[ id ] = null;
+ }
+
+ // We destroyed the cache and need to eliminate the expando on the node to avoid
+ // false lookups in the cache for entries that no longer exist
+ if ( isNode ) {
+ // IE does not allow us to delete expando properties from nodes,
+ // nor does it have a removeAttribute function on Document nodes;
+ // we must handle all of these cases
+ if ( jQuery.support.deleteExpando ) {
+ delete elem[ internalKey ];
+ } else if ( elem.removeAttribute ) {
+ elem.removeAttribute( internalKey );
+ } else {
+ elem[ internalKey ] = null;
+ }
+ }
+ },
+
+ // For internal use only.
+ _data: function( elem, name, data ) {
+ return jQuery.data( elem, name, data, true );
+ },
+
+ // A method for determining if a DOM node can handle the data expando
+ acceptData: function( elem ) {
+ if ( elem.nodeName ) {
+ var match = jQuery.noData[ elem.nodeName.toLowerCase() ];
+
+ if ( match ) {
+ return !(match === true || elem.getAttribute("classid") !== match);
+ }
+ }
+
+ return true;
+ }
+});
+
+jQuery.fn.extend({
+ data: function( key, value ) {
+ var parts, part, attr, name, l,
+ elem = this[0],
+ i = 0,
+ data = null;
+
+ // Gets all values
+ if ( key === undefined ) {
+ if ( this.length ) {
+ data = jQuery.data( elem );
+
+ if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) {
+ attr = elem.attributes;
+ for ( l = attr.length; i < l; i++ ) {
+ name = attr[i].name;
+
+ if ( name.indexOf( "data-" ) === 0 ) {
+ name = jQuery.camelCase( name.substring(5) );
+
+ dataAttr( elem, name, data[ name ] );
+ }
+ }
+ jQuery._data( elem, "parsedAttrs", true );
+ }
+ }
+
+ return data;
+ }
+
+ // Sets multiple values
+ if ( typeof key === "object" ) {
+ return this.each(function() {
+ jQuery.data( this, key );
+ });
+ }
+
+ parts = key.split( ".", 2 );
+ parts[1] = parts[1] ? "." + parts[1] : "";
+ part = parts[1] + "!";
+
+ return jQuery.access( this, function( value ) {
+
+ if ( value === undefined ) {
+ data = this.triggerHandler( "getData" + part, [ parts[0] ] );
+
+ // Try to fetch any internally stored data first
+ if ( data === undefined && elem ) {
+ data = jQuery.data( elem, key );
+ data = dataAttr( elem, key, data );
+ }
+
+ return data === undefined && parts[1] ?
+ this.data( parts[0] ) :
+ data;
+ }
+
+ parts[1] = value;
+ this.each(function() {
+ var self = jQuery( this );
+
+ self.triggerHandler( "setData" + part, parts );
+ jQuery.data( this, key, value );
+ self.triggerHandler( "changeData" + part, parts );
+ });
+ }, null, value, arguments.length > 1, null, false );
+ },
+
+ removeData: function( key ) {
+ return this.each(function() {
+ jQuery.removeData( this, key );
+ });
+ }
+});
+
+function dataAttr( elem, key, data ) {
+ // If nothing was found internally, try to fetch any
+ // data from the HTML5 data-* attribute
+ if ( data === undefined && elem.nodeType === 1 ) {
+
+ var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase();
+
+ data = elem.getAttribute( name );
+
+ if ( typeof data === "string" ) {
+ try {
+ data = data === "true" ? true :
+ data === "false" ? false :
+ data === "null" ? null :
+ jQuery.isNumeric( data ) ? +data :
+ rbrace.test( data ) ? jQuery.parseJSON( data ) :
+ data;
+ } catch( e ) {}
+
+ // Make sure we set the data so it isn't changed later
+ jQuery.data( elem, key, data );
+
+ } else {
+ data = undefined;
+ }
+ }
+
+ return data;
+}
+
+// checks a cache object for emptiness
+function isEmptyDataObject( obj ) {
+ for ( var name in obj ) {
+
+ // if the public data object is empty, the private is still empty
+ if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) {
+ continue;
+ }
+ if ( name !== "toJSON" ) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+
+
+
+function handleQueueMarkDefer( elem, type, src ) {
+ var deferDataKey = type + "defer",
+ queueDataKey = type + "queue",
+ markDataKey = type + "mark",
+ defer = jQuery._data( elem, deferDataKey );
+ if ( defer &&
+ ( src === "queue" || !jQuery._data(elem, queueDataKey) ) &&
+ ( src === "mark" || !jQuery._data(elem, markDataKey) ) ) {
+ // Give room for hard-coded callbacks to fire first
+ // and eventually mark/queue something else on the element
+ setTimeout( function() {
+ if ( !jQuery._data( elem, queueDataKey ) &&
+ !jQuery._data( elem, markDataKey ) ) {
+ jQuery.removeData( elem, deferDataKey, true );
+ defer.fire();
+ }
+ }, 0 );
+ }
+}
+
+jQuery.extend({
+
+ _mark: function( elem, type ) {
+ if ( elem ) {
+ type = ( type || "fx" ) + "mark";
+ jQuery._data( elem, type, (jQuery._data( elem, type ) || 0) + 1 );
+ }
+ },
+
+ _unmark: function( force, elem, type ) {
+ if ( force !== true ) {
+ type = elem;
+ elem = force;
+ force = false;
+ }
+ if ( elem ) {
+ type = type || "fx";
+ var key = type + "mark",
+ count = force ? 0 : ( (jQuery._data( elem, key ) || 1) - 1 );
+ if ( count ) {
+ jQuery._data( elem, key, count );
+ } else {
+ jQuery.removeData( elem, key, true );
+ handleQueueMarkDefer( elem, type, "mark" );
+ }
+ }
+ },
+
+ queue: function( elem, type, data ) {
+ var q;
+ if ( elem ) {
+ type = ( type || "fx" ) + "queue";
+ q = jQuery._data( elem, type );
+
+ // Speed up dequeue by getting out quickly if this is just a lookup
+ if ( data ) {
+ if ( !q || jQuery.isArray(data) ) {
+ q = jQuery._data( elem, type, jQuery.makeArray(data) );
+ } else {
+ q.push( data );
+ }
+ }
+ return q || [];
+ }
+ },
+
+ dequeue: function( elem, type ) {
+ type = type || "fx";
+
+ var queue = jQuery.queue( elem, type ),
+ fn = queue.shift(),
+ hooks = {};
+
+ // If the fx queue is dequeued, always remove the progress sentinel
+ if ( fn === "inprogress" ) {
+ fn = queue.shift();
+ }
+
+ if ( fn ) {
+ // Add a progress sentinel to prevent the fx queue from being
+ // automatically dequeued
+ if ( type === "fx" ) {
+ queue.unshift( "inprogress" );
+ }
+
+ jQuery._data( elem, type + ".run", hooks );
+ fn.call( elem, function() {
+ jQuery.dequeue( elem, type );
+ }, hooks );
+ }
+
+ if ( !queue.length ) {
+ jQuery.removeData( elem, type + "queue " + type + ".run", true );
+ handleQueueMarkDefer( elem, type, "queue" );
+ }
+ }
+});
+
+jQuery.fn.extend({
+ queue: function( type, data ) {
+ var setter = 2;
+
+ if ( typeof type !== "string" ) {
+ data = type;
+ type = "fx";
+ setter--;
+ }
+
+ if ( arguments.length < setter ) {
+ return jQuery.queue( this[0], type );
+ }
+
+ return data === undefined ?
+ this :
+ this.each(function() {
+ var queue = jQuery.queue( this, type, data );
+
+ if ( type === "fx" && queue[0] !== "inprogress" ) {
+ jQuery.dequeue( this, type );
+ }
+ });
+ },
+ dequeue: function( type ) {
+ return this.each(function() {
+ jQuery.dequeue( this, type );
+ });
+ },
+ // Based off of the plugin by Clint Helfers, with permission.
+ // http://blindsignals.com/index.php/2009/07/jquery-delay/
+ delay: function( time, type ) {
+ time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time;
+ type = type || "fx";
+
+ return this.queue( type, function( next, hooks ) {
+ var timeout = setTimeout( next, time );
+ hooks.stop = function() {
+ clearTimeout( timeout );
+ };
+ });
+ },
+ clearQueue: function( type ) {
+ return this.queue( type || "fx", [] );
+ },
+ // Get a promise resolved when queues of a certain type
+ // are emptied (fx is the type by default)
+ promise: function( type, object ) {
+ if ( typeof type !== "string" ) {
+ object = type;
+ type = undefined;
+ }
+ type = type || "fx";
+ var defer = jQuery.Deferred(),
+ elements = this,
+ i = elements.length,
+ count = 1,
+ deferDataKey = type + "defer",
+ queueDataKey = type + "queue",
+ markDataKey = type + "mark",
+ tmp;
+ function resolve() {
+ if ( !( --count ) ) {
+ defer.resolveWith( elements, [ elements ] );
+ }
+ }
+ while( i-- ) {
+ if (( tmp = jQuery.data( elements[ i ], deferDataKey, undefined, true ) ||
+ ( jQuery.data( elements[ i ], queueDataKey, undefined, true ) ||
+ jQuery.data( elements[ i ], markDataKey, undefined, true ) ) &&
+ jQuery.data( elements[ i ], deferDataKey, jQuery.Callbacks( "once memory" ), true ) )) {
+ count++;
+ tmp.add( resolve );
+ }
+ }
+ resolve();
+ return defer.promise( object );
+ }
+});
+
+
+
+
+var rclass = /[\n\t\r]/g,
+ rspace = /\s+/,
+ rreturn = /\r/g,
+ rtype = /^(?:button|input)$/i,
+ rfocusable = /^(?:button|input|object|select|textarea)$/i,
+ rclickable = /^a(?:rea)?$/i,
+ rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i,
+ getSetAttribute = jQuery.support.getSetAttribute,
+ nodeHook, boolHook, fixSpecified;
+
+jQuery.fn.extend({
+ attr: function( name, value ) {
+ return jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 );
+ },
+
+ removeAttr: function( name ) {
+ return this.each(function() {
+ jQuery.removeAttr( this, name );
+ });
+ },
+
+ prop: function( name, value ) {
+ return jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 );
+ },
+
+ removeProp: function( name ) {
+ name = jQuery.propFix[ name ] || name;
+ return this.each(function() {
+ // try/catch handles cases where IE balks (such as removing a property on window)
+ try {
+ this[ name ] = undefined;
+ delete this[ name ];
+ } catch( e ) {}
+ });
+ },
+
+ addClass: function( value ) {
+ var classNames, i, l, elem,
+ setClass, c, cl;
+
+ if ( jQuery.isFunction( value ) ) {
+ return this.each(function( j ) {
+ jQuery( this ).addClass( value.call(this, j, this.className) );
+ });
+ }
+
+ if ( value && typeof value === "string" ) {
+ classNames = value.split( rspace );
+
+ for ( i = 0, l = this.length; i < l; i++ ) {
+ elem = this[ i ];
+
+ if ( elem.nodeType === 1 ) {
+ if ( !elem.className && classNames.length === 1 ) {
+ elem.className = value;
+
+ } else {
+ setClass = " " + elem.className + " ";
+
+ for ( c = 0, cl = classNames.length; c < cl; c++ ) {
+ if ( !~setClass.indexOf( " " + classNames[ c ] + " " ) ) {
+ setClass += classNames[ c ] + " ";
+ }
+ }
+ elem.className = jQuery.trim( setClass );
+ }
+ }
+ }
+ }
+
+ return this;
+ },
+
+ removeClass: function( value ) {
+ var classNames, i, l, elem, className, c, cl;
+
+ if ( jQuery.isFunction( value ) ) {
+ return this.each(function( j ) {
+ jQuery( this ).removeClass( value.call(this, j, this.className) );
+ });
+ }
+
+ if ( (value && typeof value === "string") || value === undefined ) {
+ classNames = ( value || "" ).split( rspace );
+
+ for ( i = 0, l = this.length; i < l; i++ ) {
+ elem = this[ i ];
+
+ if ( elem.nodeType === 1 && elem.className ) {
+ if ( value ) {
+ className = (" " + elem.className + " ").replace( rclass, " " );
+ for ( c = 0, cl = classNames.length; c < cl; c++ ) {
+ className = className.replace(" " + classNames[ c ] + " ", " ");
+ }
+ elem.className = jQuery.trim( className );
+
+ } else {
+ elem.className = "";
+ }
+ }
+ }
+ }
+
+ return this;
+ },
+
+ toggleClass: function( value, stateVal ) {
+ var type = typeof value,
+ isBool = typeof stateVal === "boolean";
+
+ if ( jQuery.isFunction( value ) ) {
+ return this.each(function( i ) {
+ jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal );
+ });
+ }
+
+ return this.each(function() {
+ if ( type === "string" ) {
+ // toggle individual class names
+ var className,
+ i = 0,
+ self = jQuery( this ),
+ state = stateVal,
+ classNames = value.split( rspace );
+
+ while ( (className = classNames[ i++ ]) ) {
+ // check each className given, space seperated list
+ state = isBool ? state : !self.hasClass( className );
+ self[ state ? "addClass" : "removeClass" ]( className );
+ }
+
+ } else if ( type === "undefined" || type === "boolean" ) {
+ if ( this.className ) {
+ // store className if set
+ jQuery._data( this, "__className__", this.className );
+ }
+
+ // toggle whole className
+ this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || "";
+ }
+ });
+ },
+
+ hasClass: function( selector ) {
+ var className = " " + selector + " ",
+ i = 0,
+ l = this.length;
+ for ( ; i < l; i++ ) {
+ if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) {
+ return true;
+ }
+ }
+
+ return false;
+ },
+
+ val: function( value ) {
+ var hooks, ret, isFunction,
+ elem = this[0];
+
+ if ( !arguments.length ) {
+ if ( elem ) {
+ hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ];
+
+ if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) {
+ return ret;
+ }
+
+ ret = elem.value;
+
+ return typeof ret === "string" ?
+ // handle most common string cases
+ ret.replace(rreturn, "") :
+ // handle cases where value is null/undef or number
+ ret == null ? "" : ret;
+ }
+
+ return;
+ }
+
+ isFunction = jQuery.isFunction( value );
+
+ return this.each(function( i ) {
+ var self = jQuery(this), val;
+
+ if ( this.nodeType !== 1 ) {
+ return;
+ }
+
+ if ( isFunction ) {
+ val = value.call( this, i, self.val() );
+ } else {
+ val = value;
+ }
+
+ // Treat null/undefined as ""; convert numbers to string
+ if ( val == null ) {
+ val = "";
+ } else if ( typeof val === "number" ) {
+ val += "";
+ } else if ( jQuery.isArray( val ) ) {
+ val = jQuery.map(val, function ( value ) {
+ return value == null ? "" : value + "";
+ });
+ }
+
+ hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ];
+
+ // If set returns undefined, fall back to normal setting
+ if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) {
+ this.value = val;
+ }
+ });
+ }
+});
+
+jQuery.extend({
+ valHooks: {
+ option: {
+ get: function( elem ) {
+ // attributes.value is undefined in Blackberry 4.7 but
+ // uses .value. See #6932
+ var val = elem.attributes.value;
+ return !val || val.specified ? elem.value : elem.text;
+ }
+ },
+ select: {
+ get: function( elem ) {
+ var value, i, max, option,
+ index = elem.selectedIndex,
+ values = [],
+ options = elem.options,
+ one = elem.type === "select-one";
+
+ // Nothing was selected
+ if ( index < 0 ) {
+ return null;
+ }
+
+ // Loop through all the selected options
+ i = one ? index : 0;
+ max = one ? index + 1 : options.length;
+ for ( ; i < max; i++ ) {
+ option = options[ i ];
+
+ // Don't return options that are disabled or in a disabled optgroup
+ if ( option.selected && (jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null) &&
+ (!option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" )) ) {
+
+ // Get the specific value for the option
+ value = jQuery( option ).val();
+
+ // We don't need an array for one selects
+ if ( one ) {
+ return value;
+ }
+
+ // Multi-Selects return an array
+ values.push( value );
+ }
+ }
+
+ // Fixes Bug #2551 -- select.val() broken in IE after form.reset()
+ if ( one && !values.length && options.length ) {
+ return jQuery( options[ index ] ).val();
+ }
+
+ return values;
+ },
+
+ set: function( elem, value ) {
+ var values = jQuery.makeArray( value );
+
+ jQuery(elem).find("option").each(function() {
+ this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0;
+ });
+
+ if ( !values.length ) {
+ elem.selectedIndex = -1;
+ }
+ return values;
+ }
+ }
+ },
+
+ attrFn: {
+ val: true,
+ css: true,
+ html: true,
+ text: true,
+ data: true,
+ width: true,
+ height: true,
+ offset: true
+ },
+
+ attr: function( elem, name, value, pass ) {
+ var ret, hooks, notxml,
+ nType = elem.nodeType;
+
+ // don't get/set attributes on text, comment and attribute nodes
+ if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
+ return;
+ }
+
+ if ( pass && name in jQuery.attrFn ) {
+ return jQuery( elem )[ name ]( value );
+ }
+
+ // Fallback to prop when attributes are not supported
+ if ( typeof elem.getAttribute === "undefined" ) {
+ return jQuery.prop( elem, name, value );
+ }
+
+ notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+
+ // All attributes are lowercase
+ // Grab necessary hook if one is defined
+ if ( notxml ) {
+ name = name.toLowerCase();
+ hooks = jQuery.attrHooks[ name ] || ( rboolean.test( name ) ? boolHook : nodeHook );
+ }
+
+ if ( value !== undefined ) {
+
+ if ( value === null ) {
+ jQuery.removeAttr( elem, name );
+ return;
+
+ } else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) {
+ return ret;
+
+ } else {
+ elem.setAttribute( name, "" + value );
+ return value;
+ }
+
+ } else if ( hooks && "get" in hooks && notxml && (ret = hooks.get( elem, name )) !== null ) {
+ return ret;
+
+ } else {
+
+ ret = elem.getAttribute( name );
+
+ // Non-existent attributes return null, we normalize to undefined
+ return ret === null ?
+ undefined :
+ ret;
+ }
+ },
+
+ removeAttr: function( elem, value ) {
+ var propName, attrNames, name, l, isBool,
+ i = 0;
+
+ if ( value && elem.nodeType === 1 ) {
+ attrNames = value.toLowerCase().split( rspace );
+ l = attrNames.length;
+
+ for ( ; i < l; i++ ) {
+ name = attrNames[ i ];
+
+ if ( name ) {
+ propName = jQuery.propFix[ name ] || name;
+ isBool = rboolean.test( name );
+
+ // See #9699 for explanation of this approach (setting first, then removal)
+ // Do not do this for boolean attributes (see #10870)
+ if ( !isBool ) {
+ jQuery.attr( elem, name, "" );
+ }
+ elem.removeAttribute( getSetAttribute ? name : propName );
+
+ // Set corresponding property to false for boolean attributes
+ if ( isBool && propName in elem ) {
+ elem[ propName ] = false;
+ }
+ }
+ }
+ }
+ },
+
+ attrHooks: {
+ type: {
+ set: function( elem, value ) {
+ // We can't allow the type property to be changed (since it causes problems in IE)
+ if ( rtype.test( elem.nodeName ) && elem.parentNode ) {
+ jQuery.error( "type property can't be changed" );
+ } else if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) {
+ // Setting the type on a radio button after the value resets the value in IE6-9
+ // Reset value to it's default in case type is set after value
+ // This is for element creation
+ var val = elem.value;
+ elem.setAttribute( "type", value );
+ if ( val ) {
+ elem.value = val;
+ }
+ return value;
+ }
+ }
+ },
+ // Use the value property for back compat
+ // Use the nodeHook for button elements in IE6/7 (#1954)
+ value: {
+ get: function( elem, name ) {
+ if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
+ return nodeHook.get( elem, name );
+ }
+ return name in elem ?
+ elem.value :
+ null;
+ },
+ set: function( elem, value, name ) {
+ if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
+ return nodeHook.set( elem, value, name );
+ }
+ // Does not return so that setAttribute is also used
+ elem.value = value;
+ }
+ }
+ },
+
+ propFix: {
+ tabindex: "tabIndex",
+ readonly: "readOnly",
+ "for": "htmlFor",
+ "class": "className",
+ maxlength: "maxLength",
+ cellspacing: "cellSpacing",
+ cellpadding: "cellPadding",
+ rowspan: "rowSpan",
+ colspan: "colSpan",
+ usemap: "useMap",
+ frameborder: "frameBorder",
+ contenteditable: "contentEditable"
+ },
+
+ prop: function( elem, name, value ) {
+ var ret, hooks, notxml,
+ nType = elem.nodeType;
+
+ // don't get/set properties on text, comment and attribute nodes
+ if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
+ return;
+ }
+
+ notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+
+ if ( notxml ) {
+ // Fix name and attach hooks
+ name = jQuery.propFix[ name ] || name;
+ hooks = jQuery.propHooks[ name ];
+ }
+
+ if ( value !== undefined ) {
+ if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) {
+ return ret;
+
+ } else {
+ return ( elem[ name ] = value );
+ }
+
+ } else {
+ if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) {
+ return ret;
+
+ } else {
+ return elem[ name ];
+ }
+ }
+ },
+
+ propHooks: {
+ tabIndex: {
+ get: function( elem ) {
+ // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set
+ // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/
+ var attributeNode = elem.getAttributeNode("tabindex");
+
+ return attributeNode && attributeNode.specified ?
+ parseInt( attributeNode.value, 10 ) :
+ rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ?
+ 0 :
+ undefined;
+ }
+ }
+ }
+});
+
+// Add the tabIndex propHook to attrHooks for back-compat (different case is intentional)
+jQuery.attrHooks.tabindex = jQuery.propHooks.tabIndex;
+
+// Hook for boolean attributes
+boolHook = {
+ get: function( elem, name ) {
+ // Align boolean attributes with corresponding properties
+ // Fall back to attribute presence where some booleans are not supported
+ var attrNode,
+ property = jQuery.prop( elem, name );
+ return property === true || typeof property !== "boolean" && ( attrNode = elem.getAttributeNode(name) ) && attrNode.nodeValue !== false ?
+ name.toLowerCase() :
+ undefined;
+ },
+ set: function( elem, value, name ) {
+ var propName;
+ if ( value === false ) {
+ // Remove boolean attributes when set to false
+ jQuery.removeAttr( elem, name );
+ } else {
+ // value is true since we know at this point it's type boolean and not false
+ // Set boolean attributes to the same name and set the DOM property
+ propName = jQuery.propFix[ name ] || name;
+ if ( propName in elem ) {
+ // Only set the IDL specifically if it already exists on the element
+ elem[ propName ] = true;
+ }
+
+ elem.setAttribute( name, name.toLowerCase() );
+ }
+ return name;
+ }
+};
+
+// IE6/7 do not support getting/setting some attributes with get/setAttribute
+if ( !getSetAttribute ) {
+
+ fixSpecified = {
+ name: true,
+ id: true,
+ coords: true
+ };
+
+ // Use this for any attribute in IE6/7
+ // This fixes almost every IE6/7 issue
+ nodeHook = jQuery.valHooks.button = {
+ get: function( elem, name ) {
+ var ret;
+ ret = elem.getAttributeNode( name );
+ return ret && ( fixSpecified[ name ] ? ret.nodeValue !== "" : ret.specified ) ?
+ ret.nodeValue :
+ undefined;
+ },
+ set: function( elem, value, name ) {
+ // Set the existing or create a new attribute node
+ var ret = elem.getAttributeNode( name );
+ if ( !ret ) {
+ ret = document.createAttribute( name );
+ elem.setAttributeNode( ret );
+ }
+ return ( ret.nodeValue = value + "" );
+ }
+ };
+
+ // Apply the nodeHook to tabindex
+ jQuery.attrHooks.tabindex.set = nodeHook.set;
+
+ // Set width and height to auto instead of 0 on empty string( Bug #8150 )
+ // This is for removals
+ jQuery.each([ "width", "height" ], function( i, name ) {
+ jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
+ set: function( elem, value ) {
+ if ( value === "" ) {
+ elem.setAttribute( name, "auto" );
+ return value;
+ }
+ }
+ });
+ });
+
+ // Set contenteditable to false on removals(#10429)
+ // Setting to empty string throws an error as an invalid value
+ jQuery.attrHooks.contenteditable = {
+ get: nodeHook.get,
+ set: function( elem, value, name ) {
+ if ( value === "" ) {
+ value = "false";
+ }
+ nodeHook.set( elem, value, name );
+ }
+ };
+}
+
+
+// Some attributes require a special call on IE
+if ( !jQuery.support.hrefNormalized ) {
+ jQuery.each([ "href", "src", "width", "height" ], function( i, name ) {
+ jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
+ get: function( elem ) {
+ var ret = elem.getAttribute( name, 2 );
+ return ret === null ? undefined : ret;
+ }
+ });
+ });
+}
+
+if ( !jQuery.support.style ) {
+ jQuery.attrHooks.style = {
+ get: function( elem ) {
+ // Return undefined in the case of empty string
+ // Normalize to lowercase since IE uppercases css property names
+ return elem.style.cssText.toLowerCase() || undefined;
+ },
+ set: function( elem, value ) {
+ return ( elem.style.cssText = "" + value );
+ }
+ };
+}
+
+// Safari mis-reports the default selected property of an option
+// Accessing the parent's selectedIndex property fixes it
+if ( !jQuery.support.optSelected ) {
+ jQuery.propHooks.selected = jQuery.extend( jQuery.propHooks.selected, {
+ get: function( elem ) {
+ var parent = elem.parentNode;
+
+ if ( parent ) {
+ parent.selectedIndex;
+
+ // Make sure that it also works with optgroups, see #5701
+ if ( parent.parentNode ) {
+ parent.parentNode.selectedIndex;
+ }
+ }
+ return null;
+ }
+ });
+}
+
+// IE6/7 call enctype encoding
+if ( !jQuery.support.enctype ) {
+ jQuery.propFix.enctype = "encoding";
+}
+
+// Radios and checkboxes getter/setter
+if ( !jQuery.support.checkOn ) {
+ jQuery.each([ "radio", "checkbox" ], function() {
+ jQuery.valHooks[ this ] = {
+ get: function( elem ) {
+ // Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified
+ return elem.getAttribute("value") === null ? "on" : elem.value;
+ }
+ };
+ });
+}
+jQuery.each([ "radio", "checkbox" ], function() {
+ jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], {
+ set: function( elem, value ) {
+ if ( jQuery.isArray( value ) ) {
+ return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 );
+ }
+ }
+ });
+});
+
+
+
+
+var rformElems = /^(?:textarea|input|select)$/i,
+ rtypenamespace = /^([^\.]*)?(?:\.(.+))?$/,
+ rhoverHack = /(?:^|\s)hover(\.\S+)?\b/,
+ rkeyEvent = /^key/,
+ rmouseEvent = /^(?:mouse|contextmenu)|click/,
+ rfocusMorph = /^(?:focusinfocus|focusoutblur)$/,
+ rquickIs = /^(\w*)(?:#([\w\-]+))?(?:\.([\w\-]+))?$/,
+ quickParse = function( selector ) {
+ var quick = rquickIs.exec( selector );
+ if ( quick ) {
+ // 0 1 2 3
+ // [ _, tag, id, class ]
+ quick[1] = ( quick[1] || "" ).toLowerCase();
+ quick[3] = quick[3] && new RegExp( "(?:^|\\s)" + quick[3] + "(?:\\s|$)" );
+ }
+ return quick;
+ },
+ quickIs = function( elem, m ) {
+ var attrs = elem.attributes || {};
+ return (
+ (!m[1] || elem.nodeName.toLowerCase() === m[1]) &&
+ (!m[2] || (attrs.id || {}).value === m[2]) &&
+ (!m[3] || m[3].test( (attrs[ "class" ] || {}).value ))
+ );
+ },
+ hoverHack = function( events ) {
+ return jQuery.event.special.hover ? events : events.replace( rhoverHack, "mouseenter$1 mouseleave$1" );
+ };
+
+/*
+ * Helper functions for managing events -- not part of the public interface.
+ * Props to Dean Edwards' addEvent library for many of the ideas.
+ */
+jQuery.event = {
+
+ add: function( elem, types, handler, data, selector ) {
+
+ var elemData, eventHandle, events,
+ t, tns, type, namespaces, handleObj,
+ handleObjIn, quick, handlers, special;
+
+ // Don't attach events to noData or text/comment nodes (allow plain objects tho)
+ if ( elem.nodeType === 3 || elem.nodeType === 8 || !types || !handler || !(elemData = jQuery._data( elem )) ) {
+ return;
+ }
+
+ // Caller can pass in an object of custom data in lieu of the handler
+ if ( handler.handler ) {
+ handleObjIn = handler;
+ handler = handleObjIn.handler;
+ selector = handleObjIn.selector;
+ }
+
+ // Make sure that the handler has a unique ID, used to find/remove it later
+ if ( !handler.guid ) {
+ handler.guid = jQuery.guid++;
+ }
+
+ // Init the element's event structure and main handler, if this is the first
+ events = elemData.events;
+ if ( !events ) {
+ elemData.events = events = {};
+ }
+ eventHandle = elemData.handle;
+ if ( !eventHandle ) {
+ elemData.handle = eventHandle = function( e ) {
+ // Discard the second event of a jQuery.event.trigger() and
+ // when an event is called after a page has unloaded
+ return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ?
+ jQuery.event.dispatch.apply( eventHandle.elem, arguments ) :
+ undefined;
+ };
+ // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events
+ eventHandle.elem = elem;
+ }
+
+ // Handle multiple events separated by a space
+ // jQuery(...).bind("mouseover mouseout", fn);
+ types = jQuery.trim( hoverHack(types) ).split( " " );
+ for ( t = 0; t < types.length; t++ ) {
+
+ tns = rtypenamespace.exec( types[t] ) || [];
+ type = tns[1];
+ namespaces = ( tns[2] || "" ).split( "." ).sort();
+
+ // If event changes its type, use the special event handlers for the changed type
+ special = jQuery.event.special[ type ] || {};
+
+ // If selector defined, determine special event api type, otherwise given type
+ type = ( selector ? special.delegateType : special.bindType ) || type;
+
+ // Update special based on newly reset type
+ special = jQuery.event.special[ type ] || {};
+
+ // handleObj is passed to all event handlers
+ handleObj = jQuery.extend({
+ type: type,
+ origType: tns[1],
+ data: data,
+ handler: handler,
+ guid: handler.guid,
+ selector: selector,
+ quick: selector && quickParse( selector ),
+ namespace: namespaces.join(".")
+ }, handleObjIn );
+
+ // Init the event handler queue if we're the first
+ handlers = events[ type ];
+ if ( !handlers ) {
+ handlers = events[ type ] = [];
+ handlers.delegateCount = 0;
+
+ // Only use addEventListener/attachEvent if the special events handler returns false
+ if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) {
+ // Bind the global event handler to the element
+ if ( elem.addEventListener ) {
+ elem.addEventListener( type, eventHandle, false );
+
+ } else if ( elem.attachEvent ) {
+ elem.attachEvent( "on" + type, eventHandle );
+ }
+ }
+ }
+
+ if ( special.add ) {
+ special.add.call( elem, handleObj );
+
+ if ( !handleObj.handler.guid ) {
+ handleObj.handler.guid = handler.guid;
+ }
+ }
+
+ // Add to the element's handler list, delegates in front
+ if ( selector ) {
+ handlers.splice( handlers.delegateCount++, 0, handleObj );
+ } else {
+ handlers.push( handleObj );
+ }
+
+ // Keep track of which events have ever been used, for event optimization
+ jQuery.event.global[ type ] = true;
+ }
+
+ // Nullify elem to prevent memory leaks in IE
+ elem = null;
+ },
+
+ global: {},
+
+ // Detach an event or set of events from an element
+ remove: function( elem, types, handler, selector, mappedTypes ) {
+
+ var elemData = jQuery.hasData( elem ) && jQuery._data( elem ),
+ t, tns, type, origType, namespaces, origCount,
+ j, events, special, handle, eventType, handleObj;
+
+ if ( !elemData || !(events = elemData.events) ) {
+ return;
+ }
+
+ // Once for each type.namespace in types; type may be omitted
+ types = jQuery.trim( hoverHack( types || "" ) ).split(" ");
+ for ( t = 0; t < types.length; t++ ) {
+ tns = rtypenamespace.exec( types[t] ) || [];
+ type = origType = tns[1];
+ namespaces = tns[2];
+
+ // Unbind all events (on this namespace, if provided) for the element
+ if ( !type ) {
+ for ( type in events ) {
+ jQuery.event.remove( elem, type + types[ t ], handler, selector, true );
+ }
+ continue;
+ }
+
+ special = jQuery.event.special[ type ] || {};
+ type = ( selector? special.delegateType : special.bindType ) || type;
+ eventType = events[ type ] || [];
+ origCount = eventType.length;
+ namespaces = namespaces ? new RegExp("(^|\\.)" + namespaces.split(".").sort().join("\\.(?:.*\\.)?") + "(\\.|$)") : null;
+
+ // Remove matching events
+ for ( j = 0; j < eventType.length; j++ ) {
+ handleObj = eventType[ j ];
+
+ if ( ( mappedTypes || origType === handleObj.origType ) &&
+ ( !handler || handler.guid === handleObj.guid ) &&
+ ( !namespaces || namespaces.test( handleObj.namespace ) ) &&
+ ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) {
+ eventType.splice( j--, 1 );
+
+ if ( handleObj.selector ) {
+ eventType.delegateCount--;
+ }
+ if ( special.remove ) {
+ special.remove.call( elem, handleObj );
+ }
+ }
+ }
+
+ // Remove generic event handler if we removed something and no more handlers exist
+ // (avoids potential for endless recursion during removal of special event handlers)
+ if ( eventType.length === 0 && origCount !== eventType.length ) {
+ if ( !special.teardown || special.teardown.call( elem, namespaces ) === false ) {
+ jQuery.removeEvent( elem, type, elemData.handle );
+ }
+
+ delete events[ type ];
+ }
+ }
+
+ // Remove the expando if it's no longer used
+ if ( jQuery.isEmptyObject( events ) ) {
+ handle = elemData.handle;
+ if ( handle ) {
+ handle.elem = null;
+ }
+
+ // removeData also checks for emptiness and clears the expando if empty
+ // so use it instead of delete
+ jQuery.removeData( elem, [ "events", "handle" ], true );
+ }
+ },
+
+ // Events that are safe to short-circuit if no handlers are attached.
+ // Native DOM events should not be added, they may have inline handlers.
+ customEvent: {
+ "getData": true,
+ "setData": true,
+ "changeData": true
+ },
+
+ trigger: function( event, data, elem, onlyHandlers ) {
+ // Don't do events on text and comment nodes
+ if ( elem && (elem.nodeType === 3 || elem.nodeType === 8) ) {
+ return;
+ }
+
+ // Event object or event type
+ var type = event.type || event,
+ namespaces = [],
+ cache, exclusive, i, cur, old, ontype, special, handle, eventPath, bubbleType;
+
+ // focus/blur morphs to focusin/out; ensure we're not firing them right now
+ if ( rfocusMorph.test( type + jQuery.event.triggered ) ) {
+ return;
+ }
+
+ if ( type.indexOf( "!" ) >= 0 ) {
+ // Exclusive events trigger only for the exact event (no namespaces)
+ type = type.slice(0, -1);
+ exclusive = true;
+ }
+
+ if ( type.indexOf( "." ) >= 0 ) {
+ // Namespaced trigger; create a regexp to match event type in handle()
+ namespaces = type.split(".");
+ type = namespaces.shift();
+ namespaces.sort();
+ }
+
+ if ( (!elem || jQuery.event.customEvent[ type ]) && !jQuery.event.global[ type ] ) {
+ // No jQuery handlers for this event type, and it can't have inline handlers
+ return;
+ }
+
+ // Caller can pass in an Event, Object, or just an event type string
+ event = typeof event === "object" ?
+ // jQuery.Event object
+ event[ jQuery.expando ] ? event :
+ // Object literal
+ new jQuery.Event( type, event ) :
+ // Just the event type (string)
+ new jQuery.Event( type );
+
+ event.type = type;
+ event.isTrigger = true;
+ event.exclusive = exclusive;
+ event.namespace = namespaces.join( "." );
+ event.namespace_re = event.namespace? new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.)?") + "(\\.|$)") : null;
+ ontype = type.indexOf( ":" ) < 0 ? "on" + type : "";
+
+ // Handle a global trigger
+ if ( !elem ) {
+
+ // TODO: Stop taunting the data cache; remove global events and always attach to document
+ cache = jQuery.cache;
+ for ( i in cache ) {
+ if ( cache[ i ].events && cache[ i ].events[ type ] ) {
+ jQuery.event.trigger( event, data, cache[ i ].handle.elem, true
<TRUNCATED>
[34/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/SchedulerzHome.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/SchedulerzHome.java b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzHome.java
new file mode 100644
index 0000000..07a3259
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzHome.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.base.Function;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+import org.antlr.stringtemplate.StringTemplate;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.state.CronJobManager;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.base.Closure;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * HTTP interface to serve as a HUD for the aurora scheduler.
+ */
+@Path("/scheduler")
+public class SchedulerzHome extends JerseyTemplateServlet {
+
+ private static final Function<String, Role> CREATE_ROLE = new Function<String, Role>() {
+ @Override public Role apply(String ownerRole) {
+ Role role = new Role();
+ role.role = ownerRole;
+ return role;
+ }
+ };
+
+ private final Storage storage;
+ private final CronJobManager cronScheduler;
+ private final String clusterName;
+
+ /**
+ * Creates a new scheduler home servlet.
+ *
+ * @param storage Backing store to fetch tasks from.
+ * @param cronScheduler Cron scheduler.
+ * @param clusterName Name of the serving cluster.
+ */
+ @Inject
+ public SchedulerzHome(
+ Storage storage,
+ CronJobManager cronScheduler,
+ @ClusterName String clusterName) {
+
+ super("schedulerzhome");
+ this.storage = checkNotNull(storage);
+ this.cronScheduler = checkNotNull(cronScheduler);
+ this.clusterName = checkNotBlank(clusterName);
+ }
+
+ /**
+ * Fetches the scheduler landing page.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response get() {
+ return fillTemplate(new Closure<StringTemplate>() {
+ @Override public void execute(StringTemplate template) {
+ template.setAttribute("cluster_name", clusterName);
+
+ LoadingCache<String, Role> owners =
+ CacheBuilder.newBuilder().build(CacheLoader.from(CREATE_ROLE));
+
+ // TODO(William Farner): Render this page without an expensive query.
+ Set<IScheduledTask> tasks =
+ Storage.Util.weaklyConsistentFetchTasks(storage, Query.unscoped());
+ for (ITaskConfig task : Iterables.transform(tasks, Tasks.SCHEDULED_TO_INFO)) {
+ owners.getUnchecked(task.getOwner().getRole()).accumulate(task);
+ }
+
+ // Add cron job counts for each role.
+ for (IJobConfiguration job : cronScheduler.getJobs()) {
+ owners.getUnchecked(job.getOwner().getRole()).accumulate(job);
+ }
+
+ template.setAttribute(
+ "owners",
+ DisplayUtils.ROLE_ORDERING.sortedCopy(owners.asMap().values()));
+ }
+ });
+ }
+
+ /**
+ * Template object to represent a role.
+ */
+ static class Role {
+ private String role;
+ private Set<String> jobs = Sets.newHashSet();
+ private Set<String> cronJobs = Sets.newHashSet();
+
+ private void accumulate(ITaskConfig task) {
+ jobs.add(task.getJobName());
+ }
+
+ private void accumulate(IJobConfiguration job) {
+ cronJobs.add(job.getKey().getName());
+ }
+
+ public String getRole() {
+ return role;
+ }
+
+ public int getJobCount() {
+ return jobs.size();
+ }
+
+ public int getCronJobCount() {
+ return cronJobs.size();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java
new file mode 100644
index 0000000..12b0bec
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java
@@ -0,0 +1,432 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.HashBiMap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Ordering;
+
+import org.antlr.stringtemplate.StringTemplate;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.apiConstants;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
+import com.twitter.aurora.scheduler.metadata.NearestFit;
+import com.twitter.aurora.scheduler.state.CronJobManager;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IConstraint;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConstraint;
+import com.twitter.aurora.scheduler.storage.entities.ITaskEvent;
+import com.twitter.common.base.Closure;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.ASSIGNED;
+import static com.twitter.aurora.gen.ScheduleStatus.FAILED;
+import static com.twitter.aurora.gen.ScheduleStatus.FINISHED;
+import static com.twitter.aurora.gen.ScheduleStatus.KILLED;
+import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
+import static com.twitter.aurora.gen.ScheduleStatus.LOST;
+import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
+import static com.twitter.aurora.gen.ScheduleStatus.RUNNING;
+import static com.twitter.aurora.gen.ScheduleStatus.STARTING;
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * HTTP interface to view information about a job in the aurora scheduler.
+ */
+@Path("/scheduler/{role}/{environment}/{job}")
+public class SchedulerzJob extends JerseyTemplateServlet {
+ private static final String STATUS_FILTER_PARAM = "status";
+ private static final String ADMIN_VIEW_PARAM = "admin";
+
+ // Pagination controls.
+ private static final String OFFSET_PARAM = "o";
+ private static final int PAGE_SIZE = 50;
+
+ private static final Ordering<IScheduledTask> INSTANCE_ID_COMPARATOR =
+ Ordering.natural().onResultOf(Tasks.SCHEDULED_TO_INSTANCE_ID);
+
+ private static final Map<ScheduleStatus, Set<ScheduleStatus>> FILTER_MAP =
+ ImmutableMap.<ScheduleStatus, Set<ScheduleStatus>>builder()
+ .put(PENDING, EnumSet.of(PENDING))
+ .put(RUNNING, EnumSet.of(ASSIGNED, STARTING, RUNNING, KILLING))
+ .put(FINISHED, EnumSet.of(KILLED, FINISHED))
+ .put(FAILED, EnumSet.of(LOST, FAILED))
+ .build();
+
+ private static final Comparator<IScheduledTask> REVERSE_CHRON_COMPARATOR =
+ new Comparator<IScheduledTask>() {
+ @Override public int compare(IScheduledTask taskA, IScheduledTask taskB) {
+ // Sort in reverse chronological order.
+ Iterable<ITaskEvent> taskAEvents = taskA.getTaskEvents();
+ Iterable<ITaskEvent> taskBEvents = taskB.getTaskEvents();
+
+ boolean taskAHasEvents = taskAEvents != null && !Iterables.isEmpty(taskAEvents);
+ boolean taskBHasEvents = taskBEvents != null && !Iterables.isEmpty(taskBEvents);
+ if (taskAHasEvents && taskBHasEvents) {
+ return Long.signum(Iterables.getLast(taskBEvents).getTimestamp()
+ - Iterables.getLast(taskAEvents).getTimestamp());
+ } else {
+ return 0;
+ }
+ }
+ };
+
+ private static final Function<Veto, String> GET_REASON = new Function<Veto, String>() {
+ @Override public String apply(Veto veto) {
+ return veto.getReason();
+ }
+ };
+
+ // Double percents to escape formatting sequence.
+ private static final String PORT_FORMAT = "%%port:%s%%";
+ // TODO(William Farner): Search for usage of this, figure out a deprecation strategy to switch
+ // to %instance_id%.
+ private static final String INSTANCE_ID_REGEXP = "%shard_id%";
+ private static final String TASK_ID_REGEXP = "%task_id%";
+ private static final String HOST_REGEXP = "%host%";
+
+ private static String expandText(String value, IAssignedTask task) {
+ String expanded = value
+ .replaceAll(INSTANCE_ID_REGEXP, String.valueOf(task.getInstanceId()))
+ .replaceAll(TASK_ID_REGEXP, task.getTaskId());
+
+ if (task.isSetSlaveHost()) {
+ expanded = expanded.replaceAll(HOST_REGEXP, task.getSlaveHost());
+ }
+
+ // Expand ports.
+ if (task.isSetAssignedPorts()) {
+ for (Map.Entry<String, Integer> portEntry : task.getAssignedPorts().entrySet()) {
+ expanded = expanded.replaceAll(
+ String.format(PORT_FORMAT, portEntry.getKey()),
+ String.valueOf(portEntry.getValue()));
+ }
+ }
+
+ return expanded;
+ }
+
+ private final Function<IScheduledTask, Map<String, Object>> taskToStringMap =
+ new Function<IScheduledTask, Map<String, Object>>() {
+ @Override public Map<String, Object> apply(IScheduledTask scheduledTask) {
+ final IAssignedTask task = scheduledTask.getAssignedTask();
+ ImmutableMap.Builder<String, Object> builder = ImmutableMap.<String, Object>builder()
+ .put("taskId", task.getTaskId())
+ .put("instanceId", task.getInstanceId())
+ .put("slaveHost", task.isSetSlaveHost() ? task.getSlaveHost() : "")
+ .put("status", scheduledTask.getStatus())
+ .put("statusTimestamp", Iterables.getLast(scheduledTask.getTaskEvents()).getTimestamp())
+ .put("taskEvents", scheduledTask.getTaskEvents());
+
+ if (scheduledTask.getStatus() == ScheduleStatus.PENDING) {
+ String pendingReason;
+ Set<Veto> vetoes = nearestFit.getNearestFit(task.getTaskId());
+ if (vetoes.isEmpty()) {
+ pendingReason = "No matching hosts.";
+ } else {
+ pendingReason = Joiner.on(",").join(Iterables.transform(vetoes, GET_REASON));
+ }
+ builder.put("pendingReason", pendingReason);
+ }
+
+ Function<String, String> expander = new Function<String, String>() {
+ @Override public String apply(String input) {
+ return expandText(input, task);
+ }
+ };
+
+ Map<String, String> links = ImmutableMap.of();
+ if (apiConstants.LIVE_STATES.contains(scheduledTask.getStatus())) {
+ links =
+ ImmutableMap.copyOf(Maps.transformValues(task.getTask().getTaskLinks(), expander));
+ }
+ builder.put("links", links);
+ builder.put("executorPort", 1338);
+ if (task.isSetSlaveHost()) {
+ builder.put("executorUri",
+ "http://" + task.getSlaveHost() + ":1338/task/" + task.getTaskId());
+ }
+ return builder.build();
+ }
+ };
+
+ private final Storage storage;
+ private final String clusterName;
+ private final NearestFit nearestFit;
+ private final CronJobManager cronJobManager;
+
+ /**
+ * Creates a new job servlet.
+ *
+ * @param storage Backing store to fetch tasks from.
+ * @param clusterName Name of the serving cluster.
+ */
+ @Inject
+ public SchedulerzJob(
+ Storage storage,
+ CronJobManager cronJobManager,
+ @ClusterName String clusterName,
+ NearestFit nearestFit) {
+
+ super("schedulerzjob");
+ this.storage = checkNotNull(storage);
+ this.clusterName = checkNotBlank(clusterName);
+ this.nearestFit = checkNotNull(nearestFit);
+ this.cronJobManager = checkNotNull(cronJobManager);
+ }
+
+ private static <T> Iterable<T> offsetAndLimit(Iterable<T> iterable, int offset) {
+ return ImmutableList.copyOf(Iterables.limit(Iterables.skip(iterable, offset), PAGE_SIZE));
+ }
+
+ private static String scaleMb(long mb) {
+ return (mb >= 1024) ? ((mb / 1024) + " GiB") : (mb + " MiB");
+ }
+
+ private static final Function<IConstraint, String> DISPLAY_CONSTRAINT =
+ new Function<IConstraint, String>() {
+ @Override public String apply(IConstraint constraint) {
+ StringBuilder sb = new StringBuilder().append(constraint.getName()).append(": ");
+ ITaskConstraint taskConstraint = constraint.getConstraint();
+ switch (taskConstraint.getSetField()) {
+ case VALUE:
+ if (taskConstraint.getValue().isNegated()) {
+ sb.append("not ");
+ }
+ sb.append(Joiner.on(", ").join(taskConstraint.getValue().getValues()));
+ break;
+
+ case LIMIT:
+ sb.append("limit ").append(taskConstraint.getLimit().getLimit());
+ break;
+
+ default:
+ sb.append("Unhandled constraint type " + taskConstraint.getSetField());
+ }
+
+ return sb.toString();
+ }
+ };
+
+ private static final Function<ITaskConfig, SchedulingDetails> CONFIG_TO_DETAILS =
+ new Function<ITaskConfig, SchedulingDetails>() {
+ @Override public SchedulingDetails apply(ITaskConfig task) {
+ String resources = Joiner.on(", ").join(
+ "cpu: " + task.getNumCpus(),
+ "ram: " + scaleMb(task.getRamMb()),
+ "disk: " + scaleMb(task.getDiskMb()));
+ ImmutableMap.Builder<String, Object> details = ImmutableMap.<String, Object>builder()
+ .put("resources", resources);
+ if (!task.getConstraints().isEmpty()) {
+ Iterable<String> displayConstraints = FluentIterable.from(task.getConstraints())
+ .transform(DISPLAY_CONSTRAINT)
+ .toSortedList(Ordering.<String>natural());
+ details.put("constraints", Joiner.on(", ").join(displayConstraints));
+ }
+ if (task.isIsService()) {
+ details.put("service", "true");
+ }
+ if (task.isProduction()) {
+ details.put("production", "true");
+ }
+ if (!task.getRequestedPorts().isEmpty()) {
+ details.put("ports",
+ Joiner.on(", ").join(ImmutableSortedSet.copyOf(task.getRequestedPorts())));
+ }
+ if (!task.getPackages().isEmpty()) {
+ List<String> packages = Ordering.natural().sortedCopy(
+ Iterables.transform(task.getPackages(), TransformationUtils.PACKAGE_TOSTRING));
+ details.put(
+ "packages",
+ Joiner.on(',').join(packages));
+ }
+ details.put("contact", task.isSetContactEmail() ? task.getContactEmail() : "none");
+ return new SchedulingDetails(details.build());
+ }
+ };
+
+ static class SchedulingDetails {
+ private final Map<String, Object> details;
+
+ SchedulingDetails(ImmutableMap<String, Object> details) {
+ this.details = details;
+ }
+
+ public Map<String, Object> getDetails() {
+ return details;
+ }
+
+ @Override
+ public int hashCode() {
+ return details.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof SchedulingDetails)) {
+ return false;
+ }
+
+ SchedulingDetails other = (SchedulingDetails) o;
+ return other.details.equals(details);
+ }
+ }
+
+
+
+ private static Map<String, SchedulingDetails> buildSchedulingTable(
+ Iterable<IAssignedTask> tasks) {
+
+ Map<Integer, ITaskConfig> byInstance = Maps.transformValues(
+ Maps.uniqueIndex(tasks, Tasks.ASSIGNED_TO_INSTANCE_ID),
+ Tasks.ASSIGNED_TO_INFO);
+ Map<Integer, SchedulingDetails> detailsByInstance =
+ Maps.transformValues(byInstance, CONFIG_TO_DETAILS);
+ Multimap<SchedulingDetails, Integer> instancesByDetails = Multimaps.invertFrom(
+ Multimaps.forMap(detailsByInstance), HashMultimap.<SchedulingDetails, Integer>create());
+ Map<SchedulingDetails, String> instanceStringsByDetails =
+ Maps.transformValues(instancesByDetails.asMap(), TransformationUtils.INSTANCES_TOSTRING);
+ return HashBiMap.create(instanceStringsByDetails).inverse();
+ }
+
+ /**
+ * Fetches the landing page for a job within a role.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response get(
+ @PathParam("role") final String role,
+ @PathParam("environment") final String environment,
+ @PathParam("job") final String job,
+ @QueryParam(OFFSET_PARAM) final int offset,
+ @QueryParam(STATUS_FILTER_PARAM) final String filterArg,
+ @QueryParam(ADMIN_VIEW_PARAM) final String adminView) {
+
+ return fillTemplate(new Closure<StringTemplate>() {
+ @Override public void execute(StringTemplate template) {
+ template.setAttribute("cluster_name", clusterName);
+ template.setAttribute(ADMIN_VIEW_PARAM, adminView != null);
+ IJobKey jobKey = JobKeys.from(role, environment, job);
+
+ boolean isCron = cronJobManager.hasJob(jobKey);
+ template.setAttribute("is_cron", isCron);
+
+ ScheduleStatus statusFilter = null;
+ if (filterArg != null) {
+ template.setAttribute(STATUS_FILTER_PARAM, filterArg);
+
+ try {
+ statusFilter = ScheduleStatus.valueOf(filterArg.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ template.setAttribute("exception", "Invalid status type: " + filterArg);
+ return;
+ }
+ }
+
+ template.setAttribute("role", role);
+ template.setAttribute("environment", environment);
+ template.setAttribute("job", job);
+ template.setAttribute("statsUrl", DisplayUtils.getJobDashboardUrl(jobKey));
+ boolean hasMore = false;
+
+ Query.Builder builder = Query.jobScoped(JobKeys.from(role, environment, job));
+
+ Optional<Query.Builder> activeQuery = Optional.absent();
+ Optional<Query.Builder> completedQuery = Optional.absent();
+ if (statusFilter != null) {
+ Collection<ScheduleStatus> queryStatuses = FILTER_MAP.get(statusFilter);
+ if (Tasks.isActive(statusFilter)) {
+ activeQuery = Optional.of(builder.byStatus(queryStatuses));
+ } else {
+ completedQuery = Optional.of(builder.byStatus(queryStatuses));
+ }
+ } else {
+ activeQuery = Optional.of(builder.active());
+ completedQuery = Optional.of(builder.terminal());
+ }
+
+ if (activeQuery.isPresent()) {
+ Set<IScheduledTask> activeTasks =
+ Storage.Util.weaklyConsistentFetchTasks(storage, activeQuery.get());
+ List<IScheduledTask> liveTasks = INSTANCE_ID_COMPARATOR.sortedCopy(activeTasks);
+ template.setAttribute("activeTasks",
+ ImmutableList.copyOf(
+ Iterables.transform(offsetAndLimit(liveTasks, offset), taskToStringMap)));
+ hasMore = hasMore || (liveTasks.size() > (offset + PAGE_SIZE));
+ template.setAttribute("schedulingDetails",
+ buildSchedulingTable(Iterables.transform(liveTasks, Tasks.SCHEDULED_TO_ASSIGNED)));
+ }
+ if (completedQuery.isPresent()) {
+ List<IScheduledTask> completedTasks = Lists.newArrayList(
+ Storage.Util.weaklyConsistentFetchTasks(storage, completedQuery.get()));
+ Collections.sort(completedTasks, REVERSE_CHRON_COMPARATOR);
+ template.setAttribute("completedTasks",
+ ImmutableList.copyOf(
+ Iterables.transform(offsetAndLimit(completedTasks, offset), taskToStringMap)));
+ hasMore = hasMore || (completedTasks.size() > (offset + PAGE_SIZE));
+ }
+
+ template.setAttribute("offset", offset);
+ if (offset > 0) {
+ template.setAttribute("prevOffset", Math.max(0, offset - PAGE_SIZE));
+ }
+ if (hasMore) {
+ template.setAttribute("nextOffset", offset + PAGE_SIZE);
+ }
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java
new file mode 100644
index 0000000..756c672
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java
@@ -0,0 +1,393 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
+
+import org.antlr.stringtemplate.StringTemplate;
+
+import com.twitter.aurora.gen.CronCollisionPolicy;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.apiConstants;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.cron.CronPredictor;
+import com.twitter.aurora.scheduler.quota.QuotaManager;
+import com.twitter.aurora.scheduler.quota.Quotas;
+import com.twitter.aurora.scheduler.state.CronJobManager;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.base.Closure;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.scheduler.base.Tasks.GET_STATUS;
+import static com.twitter.aurora.scheduler.base.Tasks.LATEST_ACTIVITY;
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * HTTP interface to provide information about jobs for a specific role.
+ */
+@Path("/scheduler/{role}")
+public class SchedulerzRole extends JerseyTemplateServlet {
+
+ private static final List<ScheduleStatus> STATUSES = ImmutableList.<ScheduleStatus>builder()
+ .addAll(apiConstants.TERMINAL_STATES)
+ .addAll(apiConstants.ACTIVE_STATES)
+ .build();
+
+ // The freshest task is the latest active task
+ // or the latest inactive task if no active task exists.
+ private static final Ordering<IScheduledTask> FRESH_TASK_ORDER =
+ Ordering.explicit(STATUSES).onResultOf(GET_STATUS).compound(LATEST_ACTIVITY);
+
+ @VisibleForTesting
+ static IScheduledTask getFreshestTask(Iterable<IScheduledTask> tasks) {
+ return FRESH_TASK_ORDER.max(tasks);
+ }
+
+ private final Storage storage;
+ private final CronJobManager cronJobManager;
+ private final CronPredictor cronPredictor;
+ private final String clusterName;
+ private final QuotaManager quotaManager;
+
+ @Inject
+ SchedulerzRole(
+ Storage storage,
+ CronJobManager cronJobManager,
+ CronPredictor cronPredictor,
+ @ClusterName String clusterName,
+ QuotaManager quotaManager) {
+
+ super("schedulerzrole");
+ this.storage = checkNotNull(storage);
+ this.cronJobManager = checkNotNull(cronJobManager);
+ this.cronPredictor = checkNotNull(cronPredictor);
+ this.clusterName = checkNotBlank(clusterName);
+ this.quotaManager = checkNotNull(quotaManager);
+ }
+
+ /**
+ * Fetches the landing page for a role.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response get(@PathParam("role") final String role) {
+ return processRequest(Optional.of(role), Optional.<String>absent());
+ }
+
+ private Response processRequest(final Optional<String> role, final Optional<String> environment) {
+ return fillTemplate(new Closure<StringTemplate>() {
+ @Override public void execute(StringTemplate template) {
+
+ if (!role.isPresent()) {
+ template.setAttribute("exception", "Please specify a user.");
+ return;
+ }
+
+ Map<IJobKey, Map<?, ?>> cronJobs = fetchCronJobsBy(role.get(), environment);
+ List<Job> jobs = fetchJobsBy(role.get(), environment, cronJobs);
+ if (jobs.isEmpty() && cronJobs.isEmpty()) {
+ String msg = "No jobs found for role " + role.get()
+ + (environment.isPresent() ? (" and environment " + environment.get()) : "");
+ throw new WebApplicationException(Response.status(Status.NOT_FOUND).entity(msg).build());
+ }
+
+ template.setAttribute("cluster_name", clusterName);
+ template.setAttribute("role", role.get());
+ template.setAttribute("environment", environment.orNull());
+ template.setAttribute("jobs", jobs);
+ template.setAttribute("cronJobs", cronJobs.values());
+
+ // TODO(Suman Karumuri): In future compute consumption for role and environment.
+ template.setAttribute("prodResourcesUsed", quotaManager.getConsumption(role.get()));
+ template.setAttribute("nonProdResourcesUsed", getNonProdConsumption(role.get()));
+ template.setAttribute("resourceQuota", getQuota(role.get()));
+ }
+ });
+ }
+
+ private IQuota getQuota(final String role) {
+ return Storage.Util.consistentFetchQuota(storage, role).or(Quotas.noQuota());
+ }
+
+ private IQuota getNonProdConsumption(String role) {
+ FluentIterable<ITaskConfig> tasks = FluentIterable
+ .from(Storage.Util.weaklyConsistentFetchTasks(storage, Query.roleScoped(role).active()))
+ .transform(Tasks.SCHEDULED_TO_INFO)
+ .filter(Predicates.not(Tasks.IS_PRODUCTION));
+
+ return Quotas.fromTasks(tasks);
+ }
+
+ /**
+ * Display jobs for a role and environment.
+ */
+ @Path("/{environment}")
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response get(
+ @PathParam("role") final String role,
+ @PathParam("environment") final String environment) {
+
+ Optional<String> env = Optional.of(environment);
+ if (env.isPresent() && env.get().isEmpty()) {
+ env = Optional.absent();
+ }
+
+ return processRequest(Optional.of(role), env);
+ }
+
+ private Map<IJobKey, Map<?, ?>> fetchCronJobsBy(
+ final String role,
+ final Optional<String> environment) {
+
+ Predicate<IJobConfiguration> byRoleEnv = new Predicate<IJobConfiguration>() {
+ @Override public boolean apply(IJobConfiguration job) {
+ boolean roleMatch = job.getOwner().getRole().equals(role);
+ boolean envMatch = !environment.isPresent()
+ || job.getKey().getEnvironment().equals(environment.get());
+ return roleMatch && envMatch;
+ }
+ };
+
+ Iterable<IJobConfiguration> jobs = FluentIterable
+ .from(cronJobManager.getJobs())
+ .filter(byRoleEnv);
+
+ return Maps.transformValues(Maps.uniqueIndex(jobs, JobKeys.FROM_CONFIG),
+ new Function<IJobConfiguration, Map<?, ?>>() {
+ @Override public Map<?, ?> apply(IJobConfiguration job) {
+ return ImmutableMap.<Object, Object>builder()
+ .put("jobKey", job.getKey())
+ .put("name", job.getKey().getName())
+ .put("environment", job.getKey().getEnvironment())
+ .put("pendingTaskCount", job.getInstanceCount())
+ .put("cronSchedule", job.getCronSchedule())
+ .put("nextRun", cronPredictor.predictNextRun(job.getCronSchedule()).getTime())
+ .put("cronCollisionPolicy", cronCollisionPolicy(job))
+ .put("packages", getPackages(job))
+ .build();
+ }
+ });
+ }
+
+ private static CronCollisionPolicy cronCollisionPolicy(IJobConfiguration jobConfiguration) {
+ return CronJobManager.orDefault(jobConfiguration.getCronCollisionPolicy());
+ }
+
+ private static String getPackages(IJobConfiguration job) {
+ Set<String> packages = Sets.newHashSet();
+
+ // Insert all packages for all tasks in the set to eliminate duplicates
+ ITaskConfig task = job.getTaskConfig();
+ if (!task.getPackages().isEmpty()) {
+ packages.addAll(Lists.newArrayList(
+ Iterables.transform(task.getPackages(), TransformationUtils.PACKAGE_TOSTRING)));
+ }
+ return Joiner.on(',').join(packages);
+ }
+
+ private List<Job> fetchJobsBy(
+ final String role,
+ final Optional<String> environment,
+ final Map<IJobKey, Map<?, ?>> cronJobs) {
+
+ final Function<Map.Entry<IJobKey, Collection<IScheduledTask>>, Job> toJob =
+ new Function<Map.Entry<IJobKey, Collection<IScheduledTask>>, Job>() {
+ @Override public Job apply(Map.Entry<IJobKey, Collection<IScheduledTask>> tasksByJobKey) {
+ IJobKey jobKey = tasksByJobKey.getKey();
+ Collection<IScheduledTask> tasks = tasksByJobKey.getValue();
+
+ Job job = new Job();
+ job.environment = jobKey.getEnvironment();
+ job.name = jobKey.getName();
+
+ // Pick the freshest task's config and associate it with the job.
+ ITaskConfig freshestConfig = getFreshestTask(tasks).getAssignedTask().getTask();
+ job.production = freshestConfig.isProduction();
+
+ // TODO(Suman Karumuri): Add a source/job type to TaskConfig and replace logic below
+ if (freshestConfig.isIsService()) {
+ job.type = JobType.SERVICE;
+ } else if (cronJobs.containsKey(jobKey)) {
+ job.type = JobType.CRON;
+ } else {
+ job.type = JobType.ADHOC;
+ }
+
+ for (IScheduledTask task : tasks) {
+ switch (task.getStatus()) {
+ case INIT:
+ case PENDING:
+ job.pendingTaskCount++;
+ break;
+
+ case ASSIGNED:
+ case STARTING:
+ case RESTARTING:
+ case RUNNING:
+ case KILLING:
+ case PREEMPTING:
+ job.activeTaskCount++;
+ break;
+
+ case KILLED:
+ case FINISHED:
+ job.finishedTaskCount++;
+ break;
+
+ case LOST:
+ case FAILED:
+ case UNKNOWN:
+ job.failedTaskCount++;
+ Date now = new Date();
+ long elapsedMillis = now.getTime()
+ - Iterables.getLast(task.getTaskEvents()).getTimestamp();
+
+ if (Amount.of(elapsedMillis, Time.MILLISECONDS).as(Time.HOURS) < 6) {
+ job.recentlyFailedTaskCount++;
+ }
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unsupported status: " + task.getStatus());
+ }
+ }
+
+ return job;
+ }
+ };
+
+ Query.Builder query = environment.isPresent()
+ ? Query.envScoped(role, environment.get())
+ : Query.roleScoped(role);
+
+ Multimap<IJobKey, IScheduledTask> tasks =
+ Tasks.byJobKey(Storage.Util.weaklyConsistentFetchTasks(storage, query));
+
+ Iterable<Job> jobs = FluentIterable
+ .from(tasks.asMap().entrySet())
+ .transform(toJob);
+
+ return DisplayUtils.JOB_ORDERING.sortedCopy(jobs);
+ }
+
+ /**
+ * Template object to represent a job.
+ */
+ static class Job {
+ private String name;
+ private String environment;
+ private int pendingTaskCount = 0;
+ private int activeTaskCount = 0;
+ private int finishedTaskCount = 0;
+ private int failedTaskCount = 0;
+ private int recentlyFailedTaskCount = 0;
+ private boolean production = false;
+ private JobType type;
+
+ public String getName() {
+ return name;
+ }
+
+ public String getEnvironment() {
+ return environment;
+ }
+
+ public int getPendingTaskCount() {
+ return pendingTaskCount;
+ }
+
+ public int getActiveTaskCount() {
+ return activeTaskCount;
+ }
+
+ public int getFinishedTaskCount() {
+ return finishedTaskCount;
+ }
+
+ public int getFailedTaskCount() {
+ return failedTaskCount;
+ }
+
+ public int getRecentlyFailedTaskCount() {
+ return recentlyFailedTaskCount;
+ }
+
+ public boolean getProduction() {
+ return production;
+ }
+
+ public String getType() {
+ return type.toString();
+ }
+ }
+
+ static enum JobType {
+ ADHOC("adhoc"), CRON("cron"), SERVICE("service");
+
+ private String jobType;
+
+ private JobType(String jobType) {
+ this.jobType = jobType;
+ }
+
+ public String toString() {
+ return jobType;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/ServletModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/ServletModule.java b/src/main/java/org/apache/aurora/scheduler/http/ServletModule.java
new file mode 100644
index 0000000..825e2e1
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/ServletModule.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.Resources;
+import com.google.common.net.MediaType;
+import com.google.inject.AbstractModule;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+import com.google.inject.servlet.GuiceFilter;
+import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
+import com.sun.jersey.guice.JerseyServletModule;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+
+import com.twitter.aurora.scheduler.quota.QuotaManager;
+import com.twitter.aurora.scheduler.state.CronJobManager;
+import com.twitter.aurora.scheduler.state.SchedulerCore;
+import com.twitter.common.application.http.Registration;
+import com.twitter.common.application.modules.LifecycleModule;
+import com.twitter.common.application.modules.LocalServiceRegistry;
+import com.twitter.common.base.ExceptionalCommand;
+import com.twitter.common.net.pool.DynamicHostSet;
+import com.twitter.common.net.pool.DynamicHostSet.MonitorException;
+import com.twitter.thrift.ServiceInstance;
+
+import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS;
+import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_RESPONSE_FILTERS;
+import static com.sun.jersey.api.json.JSONConfiguration.FEATURE_POJO_MAPPING;
+
+/**
+ * Binding module for scheduler HTTP servlets.
+ */
+public class ServletModule extends AbstractModule {
+
+ private static final Map<String, String> CONTAINER_PARAMS = ImmutableMap.of(
+ FEATURE_POJO_MAPPING, Boolean.TRUE.toString(),
+ PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName(),
+ PROPERTY_CONTAINER_RESPONSE_FILTERS, GZIPContentEncodingFilter.class.getName());
+
+ @Override
+ protected void configure() {
+ requireBinding(SchedulerCore.class);
+ requireBinding(CronJobManager.class);
+ requireBinding(Key.get(String.class, ClusterName.class));
+ requireBinding(QuotaManager.class);
+
+ // Bindings required for the leader redirector.
+ requireBinding(LocalServiceRegistry.class);
+ requireBinding(Key.get(new TypeLiteral<DynamicHostSet<ServiceInstance>>() { }));
+ Registration.registerServletFilter(binder(), GuiceFilter.class, "/*");
+ install(new JerseyServletModule() {
+ private void registerJerseyEndpoint(String indexPath, Class<?>... servlets) {
+ filter(indexPath + "*").through(LeaderRedirectFilter.class);
+ filter(indexPath + "*").through(GuiceContainer.class, CONTAINER_PARAMS);
+ Registration.registerEndpoint(binder(), indexPath);
+ for (Class<?> servlet : servlets) {
+ bind(servlet);
+ }
+ }
+
+ @Override protected void configureServlets() {
+ bind(HttpStatsFilter.class).in(Singleton.class);
+ filter("/scheduler*").through(HttpStatsFilter.class);
+ bind(LeaderRedirectFilter.class).in(Singleton.class);
+ registerJerseyEndpoint("/cron", Cron.class);
+ registerJerseyEndpoint("/maintenance", Maintenance.class);
+ registerJerseyEndpoint("/mname", Mname.class);
+ registerJerseyEndpoint("/offers", Offers.class);
+ registerJerseyEndpoint("/pendingtasks", PendingTasks.class);
+ registerJerseyEndpoint("/quotas", Quotas.class);
+ registerJerseyEndpoint(
+ "/scheduler",
+ SchedulerzHome.class,
+ SchedulerzRole.class,
+ SchedulerzJob.class);
+ registerJerseyEndpoint("/slaves", Slaves.class);
+ registerJerseyEndpoint("/structdump", StructDump.class);
+ registerJerseyEndpoint("/utilization", Utilization.class);
+ }
+ });
+
+ // Static assets.
+ registerJQueryAssets();
+ registerBootstrapAssets();
+
+ registerAsset("assets/util.js", "/js/util.js");
+ registerAsset("assets/dictionary.js", "/js/dictionary.js");
+ registerAsset("assets/images/viz.png", "/images/viz.png");
+ registerAsset("assets/images/aurora.png", "/images/aurora.png");
+
+ // Register datatables
+ registerAsset("assets/datatables/css/jquery.dataTables.css", "/css/jquery.dataTables.css");
+ registerAsset("assets/datatables/images/back_disabled.png", "/images/back_disabled.png");
+ registerAsset(
+ "assets/datatables/images/back_enabled_hover.png",
+ "/images/back_enabled_hover.png");
+ registerAsset("assets/datatables/images/back_enabled.png", "/images/back_enabled.png");
+ registerAsset(
+ "assets/datatables/images/forward_disabled.png",
+ "/images/forward_disabled.png");
+ registerAsset(
+ "assets/datatables/images/forward_enabled_hover.png",
+ "/images/forward_enabled_hover.png");
+ registerAsset(
+ "assets/datatables/images/forward_enabled.png",
+ "/images/forward_enabled.png");
+ registerAsset(
+ "assets/datatables/images/sort_asc_disabled.png",
+ "/images/sort_asc_disabled.png");
+ registerAsset("assets/datatables/images/sort_asc.png", "/images/sort_asc.png");
+ registerAsset("assets/datatables/images/sort_both.png", "/images/sort_both.png");
+ registerAsset(
+ "assets/datatables/images/sort_desc_disabled.png",
+ "/images/sort_desc_disabled.png");
+ registerAsset("assets/datatables/images/sort_desc.png", "/images/sort_desc.png");
+ registerAsset(
+ "assets/datatables/js/jquery.dataTables.min.js",
+ "/js/jquery.dataTables.min.js");
+ registerAsset(
+ "assets/datatables/js/dataTables.bootstrap.js",
+ "/js/dataTables.bootstrap.js");
+ registerAsset(
+ "assets/datatables/js/dataTables.localstorage.js",
+ "/js/dataTables.localstorage.js");
+ registerAsset(
+ "assets/datatables/js/dataTables.htmlNumberType.js",
+ "/js/dataTables.htmlNumberType.js");
+
+ bind(LeaderRedirect.class).in(Singleton.class);
+ LifecycleModule.bindStartupAction(binder(), RedirectMonitor.class);
+ }
+
+ private void registerJQueryAssets() {
+ registerAsset("bower_components/jquery/jquery.js", "/js/jquery.min.js", false);
+ }
+
+ private void registerBootstrapAssets() {
+ final String BOOTSTRAP_PATH = "bower_components/bootstrap.css/";
+
+ registerAsset(BOOTSTRAP_PATH + "js/bootstrap.min.js", "/js/bootstrap.min.js", false);
+ registerAsset(BOOTSTRAP_PATH + "css/bootstrap.min.css", "/css/bootstrap.min.css", false);
+ registerAsset(BOOTSTRAP_PATH + "css/bootstrap-responsive.min.css",
+ "/css/bootstrap-responsive.min.css",
+ false);
+ registerAsset(BOOTSTRAP_PATH + "img/glyphicons-halflings-white.png",
+ "/img/glyphicons-halflings-white.png",
+ false);
+ registerAsset(BOOTSTRAP_PATH + "img/glyphicons-halflings.png",
+ "/img/glyphicons-halflings.png",
+ false);
+ }
+
+ private void registerAsset(String resourceLocation, String registerLocation) {
+ registerAsset(resourceLocation, registerLocation, true);
+ }
+
+ private void registerAsset(String resourceLocation, String registerLocation, boolean isRelative) {
+ String mediaType = getMediaType(registerLocation).toString();
+
+ if (isRelative) {
+ Registration.registerHttpAsset(
+ binder(),
+ registerLocation,
+ ServletModule.class,
+ resourceLocation,
+ mediaType,
+ true);
+ } else {
+ Registration.registerHttpAsset(
+ binder(),
+ registerLocation,
+ Resources.getResource(resourceLocation),
+ mediaType,
+ true);
+ }
+ }
+
+ private MediaType getMediaType(String filePath) {
+ if (filePath.endsWith(".png")) {
+ return MediaType.PNG;
+ } else if (filePath.endsWith(".js")) {
+ return MediaType.JAVASCRIPT_UTF_8;
+ } else if (filePath.endsWith(".html")) {
+ return MediaType.HTML_UTF_8;
+ } else if (filePath.endsWith(".css")) {
+ return MediaType.CSS_UTF_8;
+ } else {
+ throw new IllegalArgumentException("Could not determine media type for " + filePath);
+ }
+ }
+
+ static class RedirectMonitor implements ExceptionalCommand<MonitorException> {
+
+ private final LeaderRedirect redirector;
+
+ @Inject
+ RedirectMonitor(LeaderRedirect redirector) {
+ this.redirector = Preconditions.checkNotNull(redirector);
+ }
+
+ @Override public void execute() throws MonitorException {
+ redirector.monitor();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/Slaves.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Slaves.java b/src/main/java/org/apache/aurora/scheduler/http/Slaves.java
new file mode 100644
index 0000000..401ee30
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/Slaves.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+
+import org.antlr.stringtemplate.StringTemplate;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.common.base.Closure;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import static com.twitter.aurora.scheduler.storage.Storage.Work;
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * HTTP interface to serve as a HUD for the mesos slaves tracked in the scheduler.
+ */
+@Path("/slaves")
+public class Slaves extends JerseyTemplateServlet {
+ private final String clusterName;
+ private Storage storage;
+
+ /**
+ * Injected constructor.
+ *
+ * @param clusterName cluster name
+ * @param storage store to fetch the host attributes from
+ */
+ @Inject
+ public Slaves(@ClusterName String clusterName, Storage storage) {
+ super("slaves");
+ this.clusterName = checkNotBlank(clusterName);
+ this.storage = checkNotNull(storage);
+ }
+
+ private Iterable<HostAttributes> getHostAttributes() {
+ return storage.weaklyConsistentRead(new Work.Quiet<Iterable<HostAttributes>>() {
+ @Override public Iterable<HostAttributes> apply(StoreProvider storeProvider) {
+ return storeProvider.getAttributeStore().getHostAttributes();
+ }
+ });
+ }
+
+ private static final Function<HostAttributes, Slave> TO_SLAVE =
+ new Function<HostAttributes, Slave>() {
+ @Override public Slave apply(HostAttributes attributes) {
+ return new Slave(attributes);
+ }
+ };
+
+ /**
+ * Fetches the listing of known slaves.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response get() {
+ return fillTemplate(new Closure<StringTemplate>() {
+ @Override public void execute(StringTemplate template) {
+ template.setAttribute("cluster_name", clusterName);
+
+ template.setAttribute("slaves",
+ FluentIterable.from(getHostAttributes()).transform(TO_SLAVE).toList());
+ }
+ });
+ }
+
+ private static final Ordering<Attribute> ATTR_ORDER = Ordering.natural().onResultOf(
+ new Function<Attribute, String>() {
+ @Override public String apply(Attribute attr) {
+ return attr .getName();
+ }
+ });
+
+ /**
+ * Template object to represent a slave.
+ */
+ private static class Slave {
+ private final HostAttributes attributes;
+
+ Slave(HostAttributes attributes) {
+ this.attributes = attributes;
+ }
+
+ public String getHost() {
+ return attributes.getHost();
+ }
+
+ public String getId() {
+ return attributes.getSlaveId();
+ }
+
+ public MaintenanceMode getMode() {
+ return attributes.getMode();
+ }
+
+ private static final Function<Attribute, String> ATTR_TO_STRING =
+ new Function<Attribute, String>() {
+ @Override public String apply(Attribute attr) {
+ return attr.getName() + "=[" + Joiner.on(",").join(attr.getValues()) + "]";
+ }
+ };
+
+ public String getAttributes() {
+ return Joiner.on(", ").join(
+ Iterables.transform(ATTR_ORDER.sortedCopy(attributes.getAttributes()), ATTR_TO_STRING));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/StructDump.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/StructDump.java b/src/main/java/org/apache/aurora/scheduler/http/StructDump.java
new file mode 100644
index 0000000..8da8b80
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/StructDump.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.thrift.TBase;
+
+import com.twitter.aurora.gen.JobConfiguration;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.state.CronJobManager;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.Storage.Work.Quiet;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.common.base.Closure;
+import com.twitter.common.thrift.Util;
+
+/**
+ * Servlet that prints out the raw configuration for a specified struct.
+ */
+@Path("/structdump")
+public class StructDump extends JerseyTemplateServlet {
+
+ private final Storage storage;
+
+ @Inject
+ public StructDump(Storage storage) {
+ super("structdump");
+ this.storage = Preconditions.checkNotNull(storage);
+ }
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response getUsage() {
+ return Response
+ .status(Status.BAD_REQUEST)
+ .entity("<html>Usage: /structdump/{task_id} or /structdump/job/{role}/{env}/{job}</html>")
+ .build();
+ }
+
+ /**
+ * Dumps a task struct.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Path("/task/{task}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response dumpJob(
+ @PathParam("task") final String taskId) {
+
+ return dumpEntity("Task " + taskId, new Work.Quiet<Optional<? extends TBase<?, ?>>>() {
+ @Override public Optional<? extends TBase<?, ?>> apply(StoreProvider storeProvider) {
+ // Deep copy the struct to sidestep any subclass trickery inside the storage system.
+ return Optional.fromNullable(Iterables.getOnlyElement(
+ storeProvider.getTaskStore().fetchTasks(Query.taskScoped(taskId)),
+ null)
+ .newBuilder());
+ }
+ });
+ }
+
+ /**
+ * Dumps a cron job struct.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Path("/cron/{role}/{environment}/{job}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response dump(
+ @PathParam("role") final String role,
+ @PathParam("environment") final String environment,
+ @PathParam("job") final String job) {
+
+ final IJobKey jobKey = JobKeys.from(role, environment, job);
+ return dumpEntity("Cron job " + JobKeys.toPath(jobKey),
+ new Work.Quiet<Optional<? extends TBase<?, ?>>>() {
+ @Override public Optional<JobConfiguration> apply(StoreProvider storeProvider) {
+ return storeProvider.getJobStore().fetchJob(CronJobManager.MANAGER_KEY, jobKey)
+ .transform(IJobConfiguration.TO_BUILDER);
+ }
+ });
+ }
+
+ private Response dumpEntity(final String id, final Quiet<Optional<? extends TBase<?, ?>>> work) {
+ return fillTemplate(new Closure<StringTemplate>() {
+ @Override public void execute(StringTemplate template) {
+ template.setAttribute("id", id);
+ Optional<? extends TBase<?, ?>> struct = storage.weaklyConsistentRead(work);
+ if (!struct.isPresent()) {
+ template.setAttribute("exception", "Entity not found");
+ } else {
+ template.setAttribute("structPretty", Util.prettyPrint(struct.get()));
+ }
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java b/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java
new file mode 100644
index 0000000..2f531a7
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Collection;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Range;
+
+import com.twitter.aurora.scheduler.base.Numbers;
+import com.twitter.aurora.scheduler.storage.entities.IPackage;
+
+/**
+ * Utility class to hold common object to string transformation helper functions.
+ */
+final class TransformationUtils {
+ public static final Function<IPackage, String> PACKAGE_TOSTRING =
+ new Function<IPackage, String>() {
+ @Override public String apply(IPackage pkg) {
+ return pkg.getRole() + "/" + pkg.getName() + " v" + pkg.getVersion();
+ }
+ };
+
+ public static final Function<Range<Integer>, String> RANGE_TOSTRING =
+ new Function<Range<Integer>, String>() {
+ @Override public String apply(Range<Integer> range) {
+ int lower = range.lowerEndpoint();
+ int upper = range.upperEndpoint();
+ return (lower == upper) ? String.valueOf(lower) : (lower + " - " + upper);
+ }
+ };
+
+ public static final Function<Collection<Integer>, String> INSTANCES_TOSTRING =
+ new Function<Collection<Integer>, String>() {
+ @Override public String apply(Collection<Integer> instances) {
+ return Joiner.on(", ")
+ .join(Iterables.transform(Numbers.toRanges(instances), RANGE_TOSTRING));
+ }
+ };
+
+ private TransformationUtils() {
+ // Utility class
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Utilization.java b/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
new file mode 100644
index 0000000..60a7d95
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.FluentIterable;
+
+import org.antlr.stringtemplate.StringTemplate;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.stats.ResourceCounter;
+import com.twitter.aurora.scheduler.stats.ResourceCounter.GlobalMetric;
+import com.twitter.aurora.scheduler.stats.ResourceCounter.Metric;
+import com.twitter.aurora.scheduler.stats.ResourceCounter.MetricType;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.base.Closure;
+import com.twitter.common.base.MorePreconditions;
+import com.twitter.common.util.templating.StringTemplateHelper;
+import com.twitter.common.util.templating.StringTemplateHelper.TemplateException;
+
+/**
+ * A servlet to give an aggregate view of cluster resources consumed, grouped by category.
+ */
+@Path("/utilization")
+public class Utilization {
+
+ private final String clusterName;
+ private final ResourceCounter counter;
+ private final StringTemplateHelper templateHelper;
+
+ @Inject
+ Utilization(ResourceCounter counter, @ClusterName String clusterName) {
+ templateHelper = new StringTemplateHelper(getClass(), "utilization", true);
+ this.counter = Preconditions.checkNotNull(counter);
+ this.clusterName = MorePreconditions.checkNotBlank(clusterName);
+ }
+
+ private String fillTemplate(Map<Display, Metric> metrics) {
+ Function<Entry<Display, Metric>, DisplayMetric> transform =
+ new Function<Entry<Display, Metric>, DisplayMetric>() {
+ @Override public DisplayMetric apply(Entry<Display, Metric> entry) {
+ return new DisplayMetric(entry.getKey(), entry.getValue());
+ }
+ };
+ return fillTemplate(FluentIterable.from(metrics.entrySet()).transform(transform).toList());
+ }
+
+ private String fillTemplate(final Iterable<DisplayMetric> metrics) {
+ StringWriter output = new StringWriter();
+ try {
+ templateHelper.writeTemplate(output, new Closure<StringTemplate>() {
+ @Override public void execute(StringTemplate template) {
+ template.setAttribute("cluster_name", clusterName);
+ template.setAttribute("metrics", metrics);
+ }
+ });
+ } catch (TemplateException e) {
+ throw new WebApplicationException(e);
+ }
+ return output.toString();
+ }
+
+ private static class Display {
+ private final String title;
+ @Nullable
+ private final String link;
+
+ Display(String title, @Nullable String link) {
+ this.title = title;
+ this.link = link;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(title, link);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Display)) {
+ return false;
+ }
+
+ Display other = (Display) o;
+ return Objects.equal(title, other.title) && Objects.equal(link, other.link);
+ }
+ }
+
+ private static class DisplayMetric extends Metric {
+ private final Display display;
+
+ DisplayMetric(Display display, Metric wrapped) {
+ super(wrapped);
+ this.display = display;
+ }
+
+ public String getTitle() {
+ return display.title;
+ }
+
+ @Nullable
+ public String getLink() {
+ return display.link;
+ }
+ }
+
+ private static final Function<GlobalMetric, DisplayMetric> TO_DISPLAY =
+ new Function<GlobalMetric, DisplayMetric>() {
+ @Override public DisplayMetric apply(GlobalMetric count) {
+ return new DisplayMetric(
+ new Display(
+ count.type.name().replace('_', ' ').toLowerCase(),
+ count.type.name().toLowerCase()),
+ count);
+ }
+ };
+
+ /**
+ * Displays the aggregate utilization for the entire cluster.
+ *
+ * @return HTML-formatted cluster utilization.
+ */
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response aggregateCluster() {
+ Iterable<DisplayMetric> metrics =
+ FluentIterable.from(counter.computeConsumptionTotals()).transform(TO_DISPLAY).toList();
+ return Response.ok(fillTemplate(metrics)).build();
+ }
+
+ private MetricType getTypeByName(String name) throws WebApplicationException {
+ MetricType type = MetricType.valueOf(name.toUpperCase());
+ if (type == null) {
+ throw new WebApplicationException(
+ Response.status(Status.BAD_REQUEST).entity("Invalid metric type.").build());
+ }
+ return type;
+ }
+
+ /**
+ * Displays the aggregate utilization for roles within a metric type.
+ *
+ * @param metric Metric id.
+ * @return HTML-formatted utilization within the metric type.
+ */
+ @GET
+ @Path("/{metric}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response aggregateRoles(@PathParam("metric") final String metric) {
+ final MetricType type = getTypeByName(metric);
+
+ Function<ITaskConfig, Display> toKey = new Function<ITaskConfig, Display>() {
+ @Override public Display apply(ITaskConfig task) {
+ String role = task.getOwner().getRole();
+ return new Display(role, metric + "/" + role);
+ }
+ };
+ Map<Display, Metric> byRole =
+ counter.computeAggregates(Query.unscoped().active(), type.filter, toKey);
+ return Response.ok(fillTemplate(byRole)).build();
+ }
+
+ /**
+ * Displays the aggregate utilization for jobs within a role.
+ *
+ * @param metric Metric id.
+ * @param role Role for jobs to aggregate.
+ * @return HTML-formatted utilization within the metric/role.
+ */
+ @GET
+ @Path("/{metric}/{role}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response aggregateJobs(
+ @PathParam("metric") String metric,
+ @PathParam("role") String role) {
+
+ MetricType type = getTypeByName(metric);
+ Function<ITaskConfig, Display> toKey = new Function<ITaskConfig, Display>() {
+ @Override public Display apply(ITaskConfig task) {
+ return new Display(task.getJobName(), null);
+ }
+ };
+ Map<Display, Metric> byJob =
+ counter.computeAggregates(Query.roleScoped(role).active(), type.filter, toKey);
+ return Response.ok(fillTemplate(byJob)).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/local/FakeDriverFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/local/FakeDriverFactory.java b/src/main/java/org/apache/aurora/scheduler/local/FakeDriverFactory.java
new file mode 100644
index 0000000..ae5062e
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/local/FakeDriverFactory.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.local;
+
+import java.util.Collection;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Provider;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+import org.apache.mesos.Protos.ExecutorID;
+import org.apache.mesos.Protos.Filters;
+import org.apache.mesos.Protos.FrameworkID;
+import org.apache.mesos.Protos.MasterInfo;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.Request;
+import org.apache.mesos.Protos.SlaveID;
+import org.apache.mesos.Protos.Status;
+import org.apache.mesos.Protos.TaskID;
+import org.apache.mesos.Protos.TaskInfo;
+import org.apache.mesos.Protos.TaskStatus;
+import org.apache.mesos.Scheduler;
+import org.apache.mesos.SchedulerDriver;
+
+import com.twitter.aurora.scheduler.DriverFactory;
+import com.twitter.common.application.Lifecycle;
+
+/**
+ * A factory for fake scheduler driver instances.
+ */
+class FakeDriverFactory implements DriverFactory {
+
+ private final Provider<Scheduler> scheduler;
+ private final Lifecycle lifecycle;
+
+ @Inject
+ FakeDriverFactory(Provider<Scheduler> scheduler, Lifecycle lifecycle) {
+ this.scheduler = Preconditions.checkNotNull(scheduler);
+ this.lifecycle = Preconditions.checkNotNull(lifecycle);
+ }
+
+ @Override
+ public SchedulerDriver apply(@Nullable final String frameworkId) {
+ return new FakeSchedulerDriver() {
+ @Override public Status run() {
+ scheduler.get().registered(
+ this,
+ FrameworkID.newBuilder().setValue(
+ Optional.fromNullable(frameworkId).or("new-framework-id")).build(),
+ MasterInfo.newBuilder().setId("master-id").setIp(100).setPort(200).build());
+ lifecycle.awaitShutdown();
+ return null;
+ }
+ };
+ }
+
+ static class FakeSchedulerDriver implements SchedulerDriver {
+ @Override public Status start() {
+ return null;
+ }
+
+ @Override public Status stop(boolean b) {
+ return null;
+ }
+
+ @Override public Status stop() {
+ return null;
+ }
+
+ @Override public Status abort() {
+ return null;
+ }
+
+ @Override public Status join() {
+ return run();
+ }
+
+ @Override public Status run() {
+ return null;
+ }
+
+ @Override public Status requestResources(Collection<Request> requests) {
+ return null;
+ }
+
+ @Override public Status launchTasks(OfferID offerID, Collection<TaskInfo> taskInfos,
+ Filters filters) {
+ return null;
+ }
+
+ @Override public Status launchTasks(OfferID offerID, Collection<TaskInfo> taskInfos) {
+ return null;
+ }
+
+ @Override public Status killTask(TaskID taskID) {
+ return null;
+ }
+
+ @Override public Status declineOffer(OfferID offerID, Filters filters) {
+ return null;
+ }
+
+ @Override public Status declineOffer(OfferID offerID) {
+ return null;
+ }
+
+ @Override public Status reviveOffers() {
+ return null;
+ }
+
+ @Override public Status sendFrameworkMessage(ExecutorID executorID, SlaveID slaveID,
+ byte[] bytes) {
+ return null;
+ }
+
+ @Override
+ public Status reconcileTasks(Collection<TaskStatus> statuses) {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java b/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java
new file mode 100644
index 0000000..c07359e
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java
@@ -0,0 +1,330 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.local;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Provider;
+import javax.inject.Singleton;
+
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.eventbus.Subscribe;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.AbstractModule;
+
+import org.apache.mesos.Protos.Attribute;
+import org.apache.mesos.Protos.FrameworkID;
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.Resource;
+import org.apache.mesos.Protos.SlaveID;
+import org.apache.mesos.Protos.TaskID;
+import org.apache.mesos.Protos.TaskState;
+import org.apache.mesos.Protos.TaskStatus;
+import org.apache.mesos.Protos.Value.Scalar;
+import org.apache.mesos.Protos.Value.Text;
+import org.apache.mesos.Protos.Value.Type;
+import org.apache.mesos.Scheduler;
+import org.apache.mesos.SchedulerDriver;
+import org.apache.thrift.TException;
+
+import com.twitter.aurora.gen.AuroraAdmin;
+import com.twitter.aurora.gen.ExecutorConfig;
+import com.twitter.aurora.gen.Identity;
+import com.twitter.aurora.gen.JobConfiguration;
+import com.twitter.aurora.gen.Package;
+import com.twitter.aurora.gen.Quota;
+import com.twitter.aurora.gen.Response;
+import com.twitter.aurora.gen.SessionKey;
+import com.twitter.aurora.gen.TaskConfig;
+import com.twitter.aurora.scheduler.DriverFactory;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
+import com.twitter.aurora.scheduler.configuration.Resources;
+import com.twitter.aurora.scheduler.events.PubsubEvent.DriverRegistered;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import com.twitter.aurora.scheduler.events.PubsubEventModule;
+import com.twitter.aurora.scheduler.local.FakeDriverFactory.FakeSchedulerDriver;
+import com.twitter.aurora.scheduler.log.testing.FileLogStreamModule;
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.base.Command;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.concurrent.ExecutorServiceShutdown;
+
+/**
+ * A module that binds a fake mesos driver factory and a local (non-replicated) storage system.
+ * <p>
+ * The easiest way to run the scheduler in local/isolated mode is by executing:
+ * <pre>
+ * $ ./pants goal bundle aurora:scheduler-local && ./aurora/scripts/scheduler.sh -c local
+ * </pre>
+ */
+public class IsolatedSchedulerModule extends AbstractModule {
+
+ private static final Logger LOG = Logger.getLogger(IsolatedSchedulerModule.class.getName());
+
+ @Override
+ protected void configure() {
+ bind(DriverFactory.class).to(FakeDriverFactory.class);
+ bind(FakeDriverFactory.class).in(Singleton.class);
+ PubsubEventModule.bindSubscriber(binder(), FakeClusterRunner.class);
+ install(new FileLogStreamModule());
+ }
+
+ static class FakeClusterRunner implements EventSubscriber {
+ private final FrameworkID frameworkId =
+ FrameworkID.newBuilder().setValue("framework-id").build();
+ private final List<FakeSlave> cluster = ImmutableList.of(
+ new FakeSlave(frameworkId, "fake-host1", "rack1", "slave-id1"),
+ new FakeSlave(frameworkId, "fake-host2", "rack2", "slave-id2")
+ );
+
+ private final AtomicLong offerId = new AtomicLong();
+ private final Function<FakeSlave, Offer> slaveToOffer = new Function<FakeSlave, Offer>() {
+ @Override public Offer apply(FakeSlave slave) {
+ return slave.makeOffer(offerId.incrementAndGet());
+ }
+ };
+
+ private final Provider<Scheduler> scheduler;
+ private final AuroraAdmin.Iface thrift;
+ private final ScheduledExecutorService executor;
+ private final SchedulerDriver driver;
+
+ @Inject
+ FakeClusterRunner(
+ Provider<Scheduler> scheduler,
+ AuroraAdmin.Iface thrift,
+ ShutdownRegistry shutdownRegistry) {
+
+ this.scheduler = scheduler;
+ this.thrift = thrift;
+ this.executor = createThreadPool(shutdownRegistry);
+ this.driver = new FakeSchedulerDriver();
+ }
+
+ private static ScheduledExecutorService createThreadPool(ShutdownRegistry shutdownRegistry) {
+ final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
+ 1,
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TaskScheduler-%d").build()) {
+
+ @Override protected void afterExecute(Runnable runnable, @Nullable Throwable throwable) {
+ if (throwable != null) {
+ LOG.log(Level.WARNING, "Error: " + throwable, throwable);
+ } else if (runnable instanceof Future) {
+ Future<?> future = (Future<?>) runnable;
+ try {
+ future.get();
+ } catch (InterruptedException | ExecutionException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ };
+ Stats.exportSize("schedule_queue_size", executor.getQueue());
+ shutdownRegistry.addAction(new Command() {
+ @Override public void execute() {
+ new ExecutorServiceShutdown(executor, Amount.of(1L, Time.SECONDS)).execute();
+ }
+ });
+ return executor;
+ }
+
+ private void offerClusterResources() {
+ executor.submit(new Runnable() {
+ @Override public void run() {
+ scheduler.get().resourceOffers(
+ driver,
+ FluentIterable.from(cluster).transform(slaveToOffer).toList());
+ }
+ });
+ }
+
+ private void setQuotas() {
+ executor.submit(new Runnable() {
+ @Override public void run() {
+ try {
+ thrift.setQuota(
+ "mesos",
+ new Quota(2.0 * 1000000, 100000000, 100000000),
+ new SessionKey());
+ } catch (TException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ });
+ }
+
+ @Subscribe
+ public void registered(DriverRegistered event) {
+ executor.submit(new Runnable() {
+ @Override public void run() {
+ Identity mesosUser = new Identity("mesos", "mesos");
+ for (int i = 0; i < 20; i++) {
+ JobConfiguration service = createJob("serviceJob" + i, mesosUser);
+ service.getTaskConfig().setProduction((i % 2) == 0);
+ service.getTaskConfig().setIsService(true);
+ submitJob(service);
+ }
+
+ for (int i = 0; i < 20; i++) {
+ JobConfiguration adhocJob = createJob("adhocJob" + i, mesosUser);
+ adhocJob.getTaskConfig().setProduction((i % 2) == 0);
+ adhocJob.getTaskConfig();
+ submitJob(adhocJob);
+ }
+
+ for (int i = 0; i < 20; i++) {
+ JobConfiguration cron = createJob("cronJob" + i, mesosUser);
+ cron.getTaskConfig().setProduction((i % 2) == 0);
+ cron.setCronSchedule("* * * * *");
+ submitJob(cron);
+ }
+ }
+ });
+
+ setQuotas();
+ offerClusterResources();
+ // Send the offers again, since the first batch of offers will be consumed by GC executors.
+ offerClusterResources();
+ }
+
+ private void moveTaskToState(final String taskId, final TaskState state, long delaySeconds) {
+ Runnable changeState = new Runnable() {
+ @Override public void run() {
+ scheduler.get().statusUpdate(
+ driver,
+ TaskStatus.newBuilder()
+ .setTaskId(TaskID.newBuilder().setValue(taskId))
+ .setState(state)
+ .build());
+ }
+ };
+ executor.schedule(changeState, delaySeconds, TimeUnit.SECONDS);
+ }
+
+ @Subscribe
+ public void stateChanged(TaskStateChange stateChange) {
+ String taskId = stateChange.getTaskId();
+ switch (stateChange.getNewState()) {
+ case ASSIGNED:
+ moveTaskToState(taskId, TaskState.TASK_STARTING, 1);
+ break;
+
+ case STARTING:
+ moveTaskToState(taskId, TaskState.TASK_RUNNING, 1);
+ break;
+
+ case RUNNING:
+ // Let the task finish some time randomly in the next 5 minutes.
+ moveTaskToState(taskId, TaskState.TASK_FINISHED, (long) (Math.random() * 300));
+ break;
+
+ case FINISHED:
+ offerClusterResources();
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ private JobConfiguration createJob(String jobName, Identity owner) {
+ return new JobConfiguration()
+ .setKey(JobKeys.from(owner.getRole(), "test", jobName).newBuilder())
+ .setOwner(owner)
+ .setInstanceCount(5)
+ .setTaskConfig(new TaskConfig()
+ .setOwner(owner)
+ .setJobName(jobName)
+ .setEnvironment("test")
+ .setNumCpus(1.0)
+ .setDiskMb(1024)
+ .setRamMb(1024)
+ .setPackages(ImmutableSet.of(new Package(owner.getRole(), "package", 15)))
+ .setExecutorConfig(new ExecutorConfig("aurora", "opaque")));
+ }
+
+ private void submitJob(JobConfiguration job) {
+ Response response;
+ try {
+ response = thrift.createJob(job, null, new SessionKey());
+ } catch (TException e) {
+ throw Throwables.propagate(e);
+ }
+ LOG.info("Create job response: " + response);
+ }
+ }
+
+ private static class FakeSlave {
+ private final FrameworkID framework;
+ private final String host;
+ private final String rack;
+ private final String slaveId;
+
+ FakeSlave(FrameworkID framework, String host, String rack, String slaveId) {
+ this.framework = framework;
+ this.host = host;
+ this.rack = rack;
+ this.slaveId = slaveId;
+ }
+
+ private static Resource.Builder scalar(String name, double value) {
+ return Resource.newBuilder()
+ .setName(name)
+ .setType(Type.SCALAR)
+ .setScalar(Scalar.newBuilder().setValue(value));
+ }
+
+ private static Attribute.Builder attribute(String name, String value) {
+ return Attribute.newBuilder()
+ .setName(name)
+ .setType(Type.TEXT)
+ .setText(Text.newBuilder().setValue(value));
+ }
+
+ Offer makeOffer(long offerId) {
+ return Offer.newBuilder()
+ .setId(OfferID.newBuilder().setValue("offer" + offerId))
+ .setFrameworkId(framework)
+ .setSlaveId(SlaveID.newBuilder().setValue(slaveId))
+ .setHostname(host)
+ .addResources(scalar(Resources.CPUS, 16))
+ .addResources(scalar(Resources.RAM_MB, 24576))
+ .addResources(scalar(Resources.DISK_MB, 102400))
+ .addAttributes(attribute(ConfigurationManager.RACK_CONSTRAINT, rack))
+ .addAttributes(attribute(ConfigurationManager.HOST_CONSTRAINT, host))
+ .build();
+ }
+ }
+}
[07/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/assets/jquery.js
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/assets/jquery.js b/src/main/python/twitter/thermos/observer/http/assets/jquery.js
deleted file mode 100644
index 3774ff9..0000000
--- a/src/main/python/twitter/thermos/observer/http/assets/jquery.js
+++ /dev/null
@@ -1,9404 +0,0 @@
-/*!
- * jQuery JavaScript Library v1.7.2
- * http://jquery.com/
- *
- * Copyright 2011, John Resig
- * Dual licensed under the MIT or GPL Version 2 licenses.
- * http://jquery.org/license
- *
- * Includes Sizzle.js
- * http://sizzlejs.com/
- * Copyright 2011, The Dojo Foundation
- * Released under the MIT, BSD, and GPL Licenses.
- *
- * Date: Wed Mar 21 12:46:34 2012 -0700
- */
-(function( window, undefined ) {
-
-// Use the correct document accordingly with window argument (sandbox)
-var document = window.document,
- navigator = window.navigator,
- location = window.location;
-var jQuery = (function() {
-
-// Define a local copy of jQuery
-var jQuery = function( selector, context ) {
- // The jQuery object is actually just the init constructor 'enhanced'
- return new jQuery.fn.init( selector, context, rootjQuery );
- },
-
- // Map over jQuery in case of overwrite
- _jQuery = window.jQuery,
-
- // Map over the $ in case of overwrite
- _$ = window.$,
-
- // A central reference to the root jQuery(document)
- rootjQuery,
-
- // A simple way to check for HTML strings or ID strings
- // Prioritize #id over <tag> to avoid XSS via location.hash (#9521)
- quickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,
-
- // Check if a string has a non-whitespace character in it
- rnotwhite = /\S/,
-
- // Used for trimming whitespace
- trimLeft = /^\s+/,
- trimRight = /\s+$/,
-
- // Match a standalone tag
- rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>)?$/,
-
- // JSON RegExp
- rvalidchars = /^[\],:{}\s]*$/,
- rvalidescape = /\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,
- rvalidtokens = /"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,
- rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g,
-
- // Useragent RegExp
- rwebkit = /(webkit)[ \/]([\w.]+)/,
- ropera = /(opera)(?:.*version)?[ \/]([\w.]+)/,
- rmsie = /(msie) ([\w.]+)/,
- rmozilla = /(mozilla)(?:.*? rv:([\w.]+))?/,
-
- // Matches dashed string for camelizing
- rdashAlpha = /-([a-z]|[0-9])/ig,
- rmsPrefix = /^-ms-/,
-
- // Used by jQuery.camelCase as callback to replace()
- fcamelCase = function( all, letter ) {
- return ( letter + "" ).toUpperCase();
- },
-
- // Keep a UserAgent string for use with jQuery.browser
- userAgent = navigator.userAgent,
-
- // For matching the engine and version of the browser
- browserMatch,
-
- // The deferred used on DOM ready
- readyList,
-
- // The ready event handler
- DOMContentLoaded,
-
- // Save a reference to some core methods
- toString = Object.prototype.toString,
- hasOwn = Object.prototype.hasOwnProperty,
- push = Array.prototype.push,
- slice = Array.prototype.slice,
- trim = String.prototype.trim,
- indexOf = Array.prototype.indexOf,
-
- // [[Class]] -> type pairs
- class2type = {};
-
-jQuery.fn = jQuery.prototype = {
- constructor: jQuery,
- init: function( selector, context, rootjQuery ) {
- var match, elem, ret, doc;
-
- // Handle $(""), $(null), or $(undefined)
- if ( !selector ) {
- return this;
- }
-
- // Handle $(DOMElement)
- if ( selector.nodeType ) {
- this.context = this[0] = selector;
- this.length = 1;
- return this;
- }
-
- // The body element only exists once, optimize finding it
- if ( selector === "body" && !context && document.body ) {
- this.context = document;
- this[0] = document.body;
- this.selector = selector;
- this.length = 1;
- return this;
- }
-
- // Handle HTML strings
- if ( typeof selector === "string" ) {
- // Are we dealing with HTML string or an ID?
- if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) {
- // Assume that strings that start and end with <> are HTML and skip the regex check
- match = [ null, selector, null ];
-
- } else {
- match = quickExpr.exec( selector );
- }
-
- // Verify a match, and that no context was specified for #id
- if ( match && (match[1] || !context) ) {
-
- // HANDLE: $(html) -> $(array)
- if ( match[1] ) {
- context = context instanceof jQuery ? context[0] : context;
- doc = ( context ? context.ownerDocument || context : document );
-
- // If a single string is passed in and it's a single tag
- // just do a createElement and skip the rest
- ret = rsingleTag.exec( selector );
-
- if ( ret ) {
- if ( jQuery.isPlainObject( context ) ) {
- selector = [ document.createElement( ret[1] ) ];
- jQuery.fn.attr.call( selector, context, true );
-
- } else {
- selector = [ doc.createElement( ret[1] ) ];
- }
-
- } else {
- ret = jQuery.buildFragment( [ match[1] ], [ doc ] );
- selector = ( ret.cacheable ? jQuery.clone(ret.fragment) : ret.fragment ).childNodes;
- }
-
- return jQuery.merge( this, selector );
-
- // HANDLE: $("#id")
- } else {
- elem = document.getElementById( match[2] );
-
- // Check parentNode to catch when Blackberry 4.6 returns
- // nodes that are no longer in the document #6963
- if ( elem && elem.parentNode ) {
- // Handle the case where IE and Opera return items
- // by name instead of ID
- if ( elem.id !== match[2] ) {
- return rootjQuery.find( selector );
- }
-
- // Otherwise, we inject the element directly into the jQuery object
- this.length = 1;
- this[0] = elem;
- }
-
- this.context = document;
- this.selector = selector;
- return this;
- }
-
- // HANDLE: $(expr, $(...))
- } else if ( !context || context.jquery ) {
- return ( context || rootjQuery ).find( selector );
-
- // HANDLE: $(expr, context)
- // (which is just equivalent to: $(context).find(expr)
- } else {
- return this.constructor( context ).find( selector );
- }
-
- // HANDLE: $(function)
- // Shortcut for document ready
- } else if ( jQuery.isFunction( selector ) ) {
- return rootjQuery.ready( selector );
- }
-
- if ( selector.selector !== undefined ) {
- this.selector = selector.selector;
- this.context = selector.context;
- }
-
- return jQuery.makeArray( selector, this );
- },
-
- // Start with an empty selector
- selector: "",
-
- // The current version of jQuery being used
- jquery: "1.7.2",
-
- // The default length of a jQuery object is 0
- length: 0,
-
- // The number of elements contained in the matched element set
- size: function() {
- return this.length;
- },
-
- toArray: function() {
- return slice.call( this, 0 );
- },
-
- // Get the Nth element in the matched element set OR
- // Get the whole matched element set as a clean array
- get: function( num ) {
- return num == null ?
-
- // Return a 'clean' array
- this.toArray() :
-
- // Return just the object
- ( num < 0 ? this[ this.length + num ] : this[ num ] );
- },
-
- // Take an array of elements and push it onto the stack
- // (returning the new matched element set)
- pushStack: function( elems, name, selector ) {
- // Build a new jQuery matched element set
- var ret = this.constructor();
-
- if ( jQuery.isArray( elems ) ) {
- push.apply( ret, elems );
-
- } else {
- jQuery.merge( ret, elems );
- }
-
- // Add the old object onto the stack (as a reference)
- ret.prevObject = this;
-
- ret.context = this.context;
-
- if ( name === "find" ) {
- ret.selector = this.selector + ( this.selector ? " " : "" ) + selector;
- } else if ( name ) {
- ret.selector = this.selector + "." + name + "(" + selector + ")";
- }
-
- // Return the newly-formed element set
- return ret;
- },
-
- // Execute a callback for every element in the matched set.
- // (You can seed the arguments with an array of args, but this is
- // only used internally.)
- each: function( callback, args ) {
- return jQuery.each( this, callback, args );
- },
-
- ready: function( fn ) {
- // Attach the listeners
- jQuery.bindReady();
-
- // Add the callback
- readyList.add( fn );
-
- return this;
- },
-
- eq: function( i ) {
- i = +i;
- return i === -1 ?
- this.slice( i ) :
- this.slice( i, i + 1 );
- },
-
- first: function() {
- return this.eq( 0 );
- },
-
- last: function() {
- return this.eq( -1 );
- },
-
- slice: function() {
- return this.pushStack( slice.apply( this, arguments ),
- "slice", slice.call(arguments).join(",") );
- },
-
- map: function( callback ) {
- return this.pushStack( jQuery.map(this, function( elem, i ) {
- return callback.call( elem, i, elem );
- }));
- },
-
- end: function() {
- return this.prevObject || this.constructor(null);
- },
-
- // For internal use only.
- // Behaves like an Array's method, not like a jQuery method.
- push: push,
- sort: [].sort,
- splice: [].splice
-};
-
-// Give the init function the jQuery prototype for later instantiation
-jQuery.fn.init.prototype = jQuery.fn;
-
-jQuery.extend = jQuery.fn.extend = function() {
- var options, name, src, copy, copyIsArray, clone,
- target = arguments[0] || {},
- i = 1,
- length = arguments.length,
- deep = false;
-
- // Handle a deep copy situation
- if ( typeof target === "boolean" ) {
- deep = target;
- target = arguments[1] || {};
- // skip the boolean and the target
- i = 2;
- }
-
- // Handle case when target is a string or something (possible in deep copy)
- if ( typeof target !== "object" && !jQuery.isFunction(target) ) {
- target = {};
- }
-
- // extend jQuery itself if only one argument is passed
- if ( length === i ) {
- target = this;
- --i;
- }
-
- for ( ; i < length; i++ ) {
- // Only deal with non-null/undefined values
- if ( (options = arguments[ i ]) != null ) {
- // Extend the base object
- for ( name in options ) {
- src = target[ name ];
- copy = options[ name ];
-
- // Prevent never-ending loop
- if ( target === copy ) {
- continue;
- }
-
- // Recurse if we're merging plain objects or arrays
- if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) {
- if ( copyIsArray ) {
- copyIsArray = false;
- clone = src && jQuery.isArray(src) ? src : [];
-
- } else {
- clone = src && jQuery.isPlainObject(src) ? src : {};
- }
-
- // Never move original objects, clone them
- target[ name ] = jQuery.extend( deep, clone, copy );
-
- // Don't bring in undefined values
- } else if ( copy !== undefined ) {
- target[ name ] = copy;
- }
- }
- }
- }
-
- // Return the modified object
- return target;
-};
-
-jQuery.extend({
- noConflict: function( deep ) {
- if ( window.$ === jQuery ) {
- window.$ = _$;
- }
-
- if ( deep && window.jQuery === jQuery ) {
- window.jQuery = _jQuery;
- }
-
- return jQuery;
- },
-
- // Is the DOM ready to be used? Set to true once it occurs.
- isReady: false,
-
- // A counter to track how many items to wait for before
- // the ready event fires. See #6781
- readyWait: 1,
-
- // Hold (or release) the ready event
- holdReady: function( hold ) {
- if ( hold ) {
- jQuery.readyWait++;
- } else {
- jQuery.ready( true );
- }
- },
-
- // Handle when the DOM is ready
- ready: function( wait ) {
- // Either a released hold or an DOMready/load event and not yet ready
- if ( (wait === true && !--jQuery.readyWait) || (wait !== true && !jQuery.isReady) ) {
- // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443).
- if ( !document.body ) {
- return setTimeout( jQuery.ready, 1 );
- }
-
- // Remember that the DOM is ready
- jQuery.isReady = true;
-
- // If a normal DOM Ready event fired, decrement, and wait if need be
- if ( wait !== true && --jQuery.readyWait > 0 ) {
- return;
- }
-
- // If there are functions bound, to execute
- readyList.fireWith( document, [ jQuery ] );
-
- // Trigger any bound ready events
- if ( jQuery.fn.trigger ) {
- jQuery( document ).trigger( "ready" ).off( "ready" );
- }
- }
- },
-
- bindReady: function() {
- if ( readyList ) {
- return;
- }
-
- readyList = jQuery.Callbacks( "once memory" );
-
- // Catch cases where $(document).ready() is called after the
- // browser event has already occurred.
- if ( document.readyState === "complete" ) {
- // Handle it asynchronously to allow scripts the opportunity to delay ready
- return setTimeout( jQuery.ready, 1 );
- }
-
- // Mozilla, Opera and webkit nightlies currently support this event
- if ( document.addEventListener ) {
- // Use the handy event callback
- document.addEventListener( "DOMContentLoaded", DOMContentLoaded, false );
-
- // A fallback to window.onload, that will always work
- window.addEventListener( "load", jQuery.ready, false );
-
- // If IE event model is used
- } else if ( document.attachEvent ) {
- // ensure firing before onload,
- // maybe late but safe also for iframes
- document.attachEvent( "onreadystatechange", DOMContentLoaded );
-
- // A fallback to window.onload, that will always work
- window.attachEvent( "onload", jQuery.ready );
-
- // If IE and not a frame
- // continually check to see if the document is ready
- var toplevel = false;
-
- try {
- toplevel = window.frameElement == null;
- } catch(e) {}
-
- if ( document.documentElement.doScroll && toplevel ) {
- doScrollCheck();
- }
- }
- },
-
- // See test/unit/core.js for details concerning isFunction.
- // Since version 1.3, DOM methods and functions like alert
- // aren't supported. They return false on IE (#2968).
- isFunction: function( obj ) {
- return jQuery.type(obj) === "function";
- },
-
- isArray: Array.isArray || function( obj ) {
- return jQuery.type(obj) === "array";
- },
-
- isWindow: function( obj ) {
- return obj != null && obj == obj.window;
- },
-
- isNumeric: function( obj ) {
- return !isNaN( parseFloat(obj) ) && isFinite( obj );
- },
-
- type: function( obj ) {
- return obj == null ?
- String( obj ) :
- class2type[ toString.call(obj) ] || "object";
- },
-
- isPlainObject: function( obj ) {
- // Must be an Object.
- // Because of IE, we also have to check the presence of the constructor property.
- // Make sure that DOM nodes and window objects don't pass through, as well
- if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) {
- return false;
- }
-
- try {
- // Not own constructor property must be Object
- if ( obj.constructor &&
- !hasOwn.call(obj, "constructor") &&
- !hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) {
- return false;
- }
- } catch ( e ) {
- // IE8,9 Will throw exceptions on certain host objects #9897
- return false;
- }
-
- // Own properties are enumerated firstly, so to speed up,
- // if last one is own, then all properties are own.
-
- var key;
- for ( key in obj ) {}
-
- return key === undefined || hasOwn.call( obj, key );
- },
-
- isEmptyObject: function( obj ) {
- for ( var name in obj ) {
- return false;
- }
- return true;
- },
-
- error: function( msg ) {
- throw new Error( msg );
- },
-
- parseJSON: function( data ) {
- if ( typeof data !== "string" || !data ) {
- return null;
- }
-
- // Make sure leading/trailing whitespace is removed (IE can't handle it)
- data = jQuery.trim( data );
-
- // Attempt to parse using the native JSON parser first
- if ( window.JSON && window.JSON.parse ) {
- return window.JSON.parse( data );
- }
-
- // Make sure the incoming data is actual JSON
- // Logic borrowed from http://json.org/json2.js
- if ( rvalidchars.test( data.replace( rvalidescape, "@" )
- .replace( rvalidtokens, "]" )
- .replace( rvalidbraces, "")) ) {
-
- return ( new Function( "return " + data ) )();
-
- }
- jQuery.error( "Invalid JSON: " + data );
- },
-
- // Cross-browser xml parsing
- parseXML: function( data ) {
- if ( typeof data !== "string" || !data ) {
- return null;
- }
- var xml, tmp;
- try {
- if ( window.DOMParser ) { // Standard
- tmp = new DOMParser();
- xml = tmp.parseFromString( data , "text/xml" );
- } else { // IE
- xml = new ActiveXObject( "Microsoft.XMLDOM" );
- xml.async = "false";
- xml.loadXML( data );
- }
- } catch( e ) {
- xml = undefined;
- }
- if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) {
- jQuery.error( "Invalid XML: " + data );
- }
- return xml;
- },
-
- noop: function() {},
-
- // Evaluates a script in a global context
- // Workarounds based on findings by Jim Driscoll
- // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context
- globalEval: function( data ) {
- if ( data && rnotwhite.test( data ) ) {
- // We use execScript on Internet Explorer
- // We use an anonymous function so that context is window
- // rather than jQuery in Firefox
- ( window.execScript || function( data ) {
- window[ "eval" ].call( window, data );
- } )( data );
- }
- },
-
- // Convert dashed to camelCase; used by the css and data modules
- // Microsoft forgot to hump their vendor prefix (#9572)
- camelCase: function( string ) {
- return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase );
- },
-
- nodeName: function( elem, name ) {
- return elem.nodeName && elem.nodeName.toUpperCase() === name.toUpperCase();
- },
-
- // args is for internal usage only
- each: function( object, callback, args ) {
- var name, i = 0,
- length = object.length,
- isObj = length === undefined || jQuery.isFunction( object );
-
- if ( args ) {
- if ( isObj ) {
- for ( name in object ) {
- if ( callback.apply( object[ name ], args ) === false ) {
- break;
- }
- }
- } else {
- for ( ; i < length; ) {
- if ( callback.apply( object[ i++ ], args ) === false ) {
- break;
- }
- }
- }
-
- // A special, fast, case for the most common use of each
- } else {
- if ( isObj ) {
- for ( name in object ) {
- if ( callback.call( object[ name ], name, object[ name ] ) === false ) {
- break;
- }
- }
- } else {
- for ( ; i < length; ) {
- if ( callback.call( object[ i ], i, object[ i++ ] ) === false ) {
- break;
- }
- }
- }
- }
-
- return object;
- },
-
- // Use native String.trim function wherever possible
- trim: trim ?
- function( text ) {
- return text == null ?
- "" :
- trim.call( text );
- } :
-
- // Otherwise use our own trimming functionality
- function( text ) {
- return text == null ?
- "" :
- text.toString().replace( trimLeft, "" ).replace( trimRight, "" );
- },
-
- // results is for internal usage only
- makeArray: function( array, results ) {
- var ret = results || [];
-
- if ( array != null ) {
- // The window, strings (and functions) also have 'length'
- // Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930
- var type = jQuery.type( array );
-
- if ( array.length == null || type === "string" || type === "function" || type === "regexp" || jQuery.isWindow( array ) ) {
- push.call( ret, array );
- } else {
- jQuery.merge( ret, array );
- }
- }
-
- return ret;
- },
-
- inArray: function( elem, array, i ) {
- var len;
-
- if ( array ) {
- if ( indexOf ) {
- return indexOf.call( array, elem, i );
- }
-
- len = array.length;
- i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0;
-
- for ( ; i < len; i++ ) {
- // Skip accessing in sparse arrays
- if ( i in array && array[ i ] === elem ) {
- return i;
- }
- }
- }
-
- return -1;
- },
-
- merge: function( first, second ) {
- var i = first.length,
- j = 0;
-
- if ( typeof second.length === "number" ) {
- for ( var l = second.length; j < l; j++ ) {
- first[ i++ ] = second[ j ];
- }
-
- } else {
- while ( second[j] !== undefined ) {
- first[ i++ ] = second[ j++ ];
- }
- }
-
- first.length = i;
-
- return first;
- },
-
- grep: function( elems, callback, inv ) {
- var ret = [], retVal;
- inv = !!inv;
-
- // Go through the array, only saving the items
- // that pass the validator function
- for ( var i = 0, length = elems.length; i < length; i++ ) {
- retVal = !!callback( elems[ i ], i );
- if ( inv !== retVal ) {
- ret.push( elems[ i ] );
- }
- }
-
- return ret;
- },
-
- // arg is for internal usage only
- map: function( elems, callback, arg ) {
- var value, key, ret = [],
- i = 0,
- length = elems.length,
- // jquery objects are treated as arrays
- isArray = elems instanceof jQuery || length !== undefined && typeof length === "number" && ( ( length > 0 && elems[ 0 ] && elems[ length -1 ] ) || length === 0 || jQuery.isArray( elems ) ) ;
-
- // Go through the array, translating each of the items to their
- if ( isArray ) {
- for ( ; i < length; i++ ) {
- value = callback( elems[ i ], i, arg );
-
- if ( value != null ) {
- ret[ ret.length ] = value;
- }
- }
-
- // Go through every key on the object,
- } else {
- for ( key in elems ) {
- value = callback( elems[ key ], key, arg );
-
- if ( value != null ) {
- ret[ ret.length ] = value;
- }
- }
- }
-
- // Flatten any nested arrays
- return ret.concat.apply( [], ret );
- },
-
- // A global GUID counter for objects
- guid: 1,
-
- // Bind a function to a context, optionally partially applying any
- // arguments.
- proxy: function( fn, context ) {
- if ( typeof context === "string" ) {
- var tmp = fn[ context ];
- context = fn;
- fn = tmp;
- }
-
- // Quick check to determine if target is callable, in the spec
- // this throws a TypeError, but we will just return undefined.
- if ( !jQuery.isFunction( fn ) ) {
- return undefined;
- }
-
- // Simulated bind
- var args = slice.call( arguments, 2 ),
- proxy = function() {
- return fn.apply( context, args.concat( slice.call( arguments ) ) );
- };
-
- // Set the guid of unique handler to the same of original handler, so it can be removed
- proxy.guid = fn.guid = fn.guid || proxy.guid || jQuery.guid++;
-
- return proxy;
- },
-
- // Mutifunctional method to get and set values to a collection
- // The value/s can optionally be executed if it's a function
- access: function( elems, fn, key, value, chainable, emptyGet, pass ) {
- var exec,
- bulk = key == null,
- i = 0,
- length = elems.length;
-
- // Sets many values
- if ( key && typeof key === "object" ) {
- for ( i in key ) {
- jQuery.access( elems, fn, i, key[i], 1, emptyGet, value );
- }
- chainable = 1;
-
- // Sets one value
- } else if ( value !== undefined ) {
- // Optionally, function values get executed if exec is true
- exec = pass === undefined && jQuery.isFunction( value );
-
- if ( bulk ) {
- // Bulk operations only iterate when executing function values
- if ( exec ) {
- exec = fn;
- fn = function( elem, key, value ) {
- return exec.call( jQuery( elem ), value );
- };
-
- // Otherwise they run against the entire set
- } else {
- fn.call( elems, value );
- fn = null;
- }
- }
-
- if ( fn ) {
- for (; i < length; i++ ) {
- fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass );
- }
- }
-
- chainable = 1;
- }
-
- return chainable ?
- elems :
-
- // Gets
- bulk ?
- fn.call( elems ) :
- length ? fn( elems[0], key ) : emptyGet;
- },
-
- now: function() {
- return ( new Date() ).getTime();
- },
-
- // Use of jQuery.browser is frowned upon.
- // More details: http://docs.jquery.com/Utilities/jQuery.browser
- uaMatch: function( ua ) {
- ua = ua.toLowerCase();
-
- var match = rwebkit.exec( ua ) ||
- ropera.exec( ua ) ||
- rmsie.exec( ua ) ||
- ua.indexOf("compatible") < 0 && rmozilla.exec( ua ) ||
- [];
-
- return { browser: match[1] || "", version: match[2] || "0" };
- },
-
- sub: function() {
- function jQuerySub( selector, context ) {
- return new jQuerySub.fn.init( selector, context );
- }
- jQuery.extend( true, jQuerySub, this );
- jQuerySub.superclass = this;
- jQuerySub.fn = jQuerySub.prototype = this();
- jQuerySub.fn.constructor = jQuerySub;
- jQuerySub.sub = this.sub;
- jQuerySub.fn.init = function init( selector, context ) {
- if ( context && context instanceof jQuery && !(context instanceof jQuerySub) ) {
- context = jQuerySub( context );
- }
-
- return jQuery.fn.init.call( this, selector, context, rootjQuerySub );
- };
- jQuerySub.fn.init.prototype = jQuerySub.fn;
- var rootjQuerySub = jQuerySub(document);
- return jQuerySub;
- },
-
- browser: {}
-});
-
-// Populate the class2type map
-jQuery.each("Boolean Number String Function Array Date RegExp Object".split(" "), function(i, name) {
- class2type[ "[object " + name + "]" ] = name.toLowerCase();
-});
-
-browserMatch = jQuery.uaMatch( userAgent );
-if ( browserMatch.browser ) {
- jQuery.browser[ browserMatch.browser ] = true;
- jQuery.browser.version = browserMatch.version;
-}
-
-// Deprecated, use jQuery.browser.webkit instead
-if ( jQuery.browser.webkit ) {
- jQuery.browser.safari = true;
-}
-
-// IE doesn't match non-breaking spaces with \s
-if ( rnotwhite.test( "\xA0" ) ) {
- trimLeft = /^[\s\xA0]+/;
- trimRight = /[\s\xA0]+$/;
-}
-
-// All jQuery objects should point back to these
-rootjQuery = jQuery(document);
-
-// Cleanup functions for the document ready method
-if ( document.addEventListener ) {
- DOMContentLoaded = function() {
- document.removeEventListener( "DOMContentLoaded", DOMContentLoaded, false );
- jQuery.ready();
- };
-
-} else if ( document.attachEvent ) {
- DOMContentLoaded = function() {
- // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443).
- if ( document.readyState === "complete" ) {
- document.detachEvent( "onreadystatechange", DOMContentLoaded );
- jQuery.ready();
- }
- };
-}
-
-// The DOM ready check for Internet Explorer
-function doScrollCheck() {
- if ( jQuery.isReady ) {
- return;
- }
-
- try {
- // If IE is used, use the trick by Diego Perini
- // http://javascript.nwbox.com/IEContentLoaded/
- document.documentElement.doScroll("left");
- } catch(e) {
- setTimeout( doScrollCheck, 1 );
- return;
- }
-
- // and execute any waiting functions
- jQuery.ready();
-}
-
-return jQuery;
-
-})();
-
-
-// String to Object flags format cache
-var flagsCache = {};
-
-// Convert String-formatted flags into Object-formatted ones and store in cache
-function createFlags( flags ) {
- var object = flagsCache[ flags ] = {},
- i, length;
- flags = flags.split( /\s+/ );
- for ( i = 0, length = flags.length; i < length; i++ ) {
- object[ flags[i] ] = true;
- }
- return object;
-}
-
-/*
- * Create a callback list using the following parameters:
- *
- * flags: an optional list of space-separated flags that will change how
- * the callback list behaves
- *
- * By default a callback list will act like an event callback list and can be
- * "fired" multiple times.
- *
- * Possible flags:
- *
- * once: will ensure the callback list can only be fired once (like a Deferred)
- *
- * memory: will keep track of previous values and will call any callback added
- * after the list has been fired right away with the latest "memorized"
- * values (like a Deferred)
- *
- * unique: will ensure a callback can only be added once (no duplicate in the list)
- *
- * stopOnFalse: interrupt callings when a callback returns false
- *
- */
-jQuery.Callbacks = function( flags ) {
-
- // Convert flags from String-formatted to Object-formatted
- // (we check in cache first)
- flags = flags ? ( flagsCache[ flags ] || createFlags( flags ) ) : {};
-
- var // Actual callback list
- list = [],
- // Stack of fire calls for repeatable lists
- stack = [],
- // Last fire value (for non-forgettable lists)
- memory,
- // Flag to know if list was already fired
- fired,
- // Flag to know if list is currently firing
- firing,
- // First callback to fire (used internally by add and fireWith)
- firingStart,
- // End of the loop when firing
- firingLength,
- // Index of currently firing callback (modified by remove if needed)
- firingIndex,
- // Add one or several callbacks to the list
- add = function( args ) {
- var i,
- length,
- elem,
- type,
- actual;
- for ( i = 0, length = args.length; i < length; i++ ) {
- elem = args[ i ];
- type = jQuery.type( elem );
- if ( type === "array" ) {
- // Inspect recursively
- add( elem );
- } else if ( type === "function" ) {
- // Add if not in unique mode and callback is not in
- if ( !flags.unique || !self.has( elem ) ) {
- list.push( elem );
- }
- }
- }
- },
- // Fire callbacks
- fire = function( context, args ) {
- args = args || [];
- memory = !flags.memory || [ context, args ];
- fired = true;
- firing = true;
- firingIndex = firingStart || 0;
- firingStart = 0;
- firingLength = list.length;
- for ( ; list && firingIndex < firingLength; firingIndex++ ) {
- if ( list[ firingIndex ].apply( context, args ) === false && flags.stopOnFalse ) {
- memory = true; // Mark as halted
- break;
- }
- }
- firing = false;
- if ( list ) {
- if ( !flags.once ) {
- if ( stack && stack.length ) {
- memory = stack.shift();
- self.fireWith( memory[ 0 ], memory[ 1 ] );
- }
- } else if ( memory === true ) {
- self.disable();
- } else {
- list = [];
- }
- }
- },
- // Actual Callbacks object
- self = {
- // Add a callback or a collection of callbacks to the list
- add: function() {
- if ( list ) {
- var length = list.length;
- add( arguments );
- // Do we need to add the callbacks to the
- // current firing batch?
- if ( firing ) {
- firingLength = list.length;
- // With memory, if we're not firing then
- // we should call right away, unless previous
- // firing was halted (stopOnFalse)
- } else if ( memory && memory !== true ) {
- firingStart = length;
- fire( memory[ 0 ], memory[ 1 ] );
- }
- }
- return this;
- },
- // Remove a callback from the list
- remove: function() {
- if ( list ) {
- var args = arguments,
- argIndex = 0,
- argLength = args.length;
- for ( ; argIndex < argLength ; argIndex++ ) {
- for ( var i = 0; i < list.length; i++ ) {
- if ( args[ argIndex ] === list[ i ] ) {
- // Handle firingIndex and firingLength
- if ( firing ) {
- if ( i <= firingLength ) {
- firingLength--;
- if ( i <= firingIndex ) {
- firingIndex--;
- }
- }
- }
- // Remove the element
- list.splice( i--, 1 );
- // If we have some unicity property then
- // we only need to do this once
- if ( flags.unique ) {
- break;
- }
- }
- }
- }
- }
- return this;
- },
- // Control if a given callback is in the list
- has: function( fn ) {
- if ( list ) {
- var i = 0,
- length = list.length;
- for ( ; i < length; i++ ) {
- if ( fn === list[ i ] ) {
- return true;
- }
- }
- }
- return false;
- },
- // Remove all callbacks from the list
- empty: function() {
- list = [];
- return this;
- },
- // Have the list do nothing anymore
- disable: function() {
- list = stack = memory = undefined;
- return this;
- },
- // Is it disabled?
- disabled: function() {
- return !list;
- },
- // Lock the list in its current state
- lock: function() {
- stack = undefined;
- if ( !memory || memory === true ) {
- self.disable();
- }
- return this;
- },
- // Is it locked?
- locked: function() {
- return !stack;
- },
- // Call all callbacks with the given context and arguments
- fireWith: function( context, args ) {
- if ( stack ) {
- if ( firing ) {
- if ( !flags.once ) {
- stack.push( [ context, args ] );
- }
- } else if ( !( flags.once && memory ) ) {
- fire( context, args );
- }
- }
- return this;
- },
- // Call all the callbacks with the given arguments
- fire: function() {
- self.fireWith( this, arguments );
- return this;
- },
- // To know if the callbacks have already been called at least once
- fired: function() {
- return !!fired;
- }
- };
-
- return self;
-};
-
-
-
-
-var // Static reference to slice
- sliceDeferred = [].slice;
-
-jQuery.extend({
-
- Deferred: function( func ) {
- var doneList = jQuery.Callbacks( "once memory" ),
- failList = jQuery.Callbacks( "once memory" ),
- progressList = jQuery.Callbacks( "memory" ),
- state = "pending",
- lists = {
- resolve: doneList,
- reject: failList,
- notify: progressList
- },
- promise = {
- done: doneList.add,
- fail: failList.add,
- progress: progressList.add,
-
- state: function() {
- return state;
- },
-
- // Deprecated
- isResolved: doneList.fired,
- isRejected: failList.fired,
-
- then: function( doneCallbacks, failCallbacks, progressCallbacks ) {
- deferred.done( doneCallbacks ).fail( failCallbacks ).progress( progressCallbacks );
- return this;
- },
- always: function() {
- deferred.done.apply( deferred, arguments ).fail.apply( deferred, arguments );
- return this;
- },
- pipe: function( fnDone, fnFail, fnProgress ) {
- return jQuery.Deferred(function( newDefer ) {
- jQuery.each( {
- done: [ fnDone, "resolve" ],
- fail: [ fnFail, "reject" ],
- progress: [ fnProgress, "notify" ]
- }, function( handler, data ) {
- var fn = data[ 0 ],
- action = data[ 1 ],
- returned;
- if ( jQuery.isFunction( fn ) ) {
- deferred[ handler ](function() {
- returned = fn.apply( this, arguments );
- if ( returned && jQuery.isFunction( returned.promise ) ) {
- returned.promise().then( newDefer.resolve, newDefer.reject, newDefer.notify );
- } else {
- newDefer[ action + "With" ]( this === deferred ? newDefer : this, [ returned ] );
- }
- });
- } else {
- deferred[ handler ]( newDefer[ action ] );
- }
- });
- }).promise();
- },
- // Get a promise for this deferred
- // If obj is provided, the promise aspect is added to the object
- promise: function( obj ) {
- if ( obj == null ) {
- obj = promise;
- } else {
- for ( var key in promise ) {
- obj[ key ] = promise[ key ];
- }
- }
- return obj;
- }
- },
- deferred = promise.promise({}),
- key;
-
- for ( key in lists ) {
- deferred[ key ] = lists[ key ].fire;
- deferred[ key + "With" ] = lists[ key ].fireWith;
- }
-
- // Handle state
- deferred.done( function() {
- state = "resolved";
- }, failList.disable, progressList.lock ).fail( function() {
- state = "rejected";
- }, doneList.disable, progressList.lock );
-
- // Call given func if any
- if ( func ) {
- func.call( deferred, deferred );
- }
-
- // All done!
- return deferred;
- },
-
- // Deferred helper
- when: function( firstParam ) {
- var args = sliceDeferred.call( arguments, 0 ),
- i = 0,
- length = args.length,
- pValues = new Array( length ),
- count = length,
- pCount = length,
- deferred = length <= 1 && firstParam && jQuery.isFunction( firstParam.promise ) ?
- firstParam :
- jQuery.Deferred(),
- promise = deferred.promise();
- function resolveFunc( i ) {
- return function( value ) {
- args[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value;
- if ( !( --count ) ) {
- deferred.resolveWith( deferred, args );
- }
- };
- }
- function progressFunc( i ) {
- return function( value ) {
- pValues[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value;
- deferred.notifyWith( promise, pValues );
- };
- }
- if ( length > 1 ) {
- for ( ; i < length; i++ ) {
- if ( args[ i ] && args[ i ].promise && jQuery.isFunction( args[ i ].promise ) ) {
- args[ i ].promise().then( resolveFunc(i), deferred.reject, progressFunc(i) );
- } else {
- --count;
- }
- }
- if ( !count ) {
- deferred.resolveWith( deferred, args );
- }
- } else if ( deferred !== firstParam ) {
- deferred.resolveWith( deferred, length ? [ firstParam ] : [] );
- }
- return promise;
- }
-});
-
-
-
-
-jQuery.support = (function() {
-
- var support,
- all,
- a,
- select,
- opt,
- input,
- fragment,
- tds,
- events,
- eventName,
- i,
- isSupported,
- div = document.createElement( "div" ),
- documentElement = document.documentElement;
-
- // Preliminary tests
- div.setAttribute("className", "t");
- div.innerHTML = " <link/><table></table><a href='/a' style='top:1px;float:left;opacity:.55;'>a</a><input type='checkbox'/>";
-
- all = div.getElementsByTagName( "*" );
- a = div.getElementsByTagName( "a" )[ 0 ];
-
- // Can't get basic test support
- if ( !all || !all.length || !a ) {
- return {};
- }
-
- // First batch of supports tests
- select = document.createElement( "select" );
- opt = select.appendChild( document.createElement("option") );
- input = div.getElementsByTagName( "input" )[ 0 ];
-
- support = {
- // IE strips leading whitespace when .innerHTML is used
- leadingWhitespace: ( div.firstChild.nodeType === 3 ),
-
- // Make sure that tbody elements aren't automatically inserted
- // IE will insert them into empty tables
- tbody: !div.getElementsByTagName("tbody").length,
-
- // Make sure that link elements get serialized correctly by innerHTML
- // This requires a wrapper element in IE
- htmlSerialize: !!div.getElementsByTagName("link").length,
-
- // Get the style information from getAttribute
- // (IE uses .cssText instead)
- style: /top/.test( a.getAttribute("style") ),
-
- // Make sure that URLs aren't manipulated
- // (IE normalizes it by default)
- hrefNormalized: ( a.getAttribute("href") === "/a" ),
-
- // Make sure that element opacity exists
- // (IE uses filter instead)
- // Use a regex to work around a WebKit issue. See #5145
- opacity: /^0.55/.test( a.style.opacity ),
-
- // Verify style float existence
- // (IE uses styleFloat instead of cssFloat)
- cssFloat: !!a.style.cssFloat,
-
- // Make sure that if no value is specified for a checkbox
- // that it defaults to "on".
- // (WebKit defaults to "" instead)
- checkOn: ( input.value === "on" ),
-
- // Make sure that a selected-by-default option has a working selected property.
- // (WebKit defaults to false instead of true, IE too, if it's in an optgroup)
- optSelected: opt.selected,
-
- // Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7)
- getSetAttribute: div.className !== "t",
-
- // Tests for enctype support on a form(#6743)
- enctype: !!document.createElement("form").enctype,
-
- // Makes sure cloning an html5 element does not cause problems
- // Where outerHTML is undefined, this still works
- html5Clone: document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav></:nav>",
-
- // Will be defined later
- submitBubbles: true,
- changeBubbles: true,
- focusinBubbles: false,
- deleteExpando: true,
- noCloneEvent: true,
- inlineBlockNeedsLayout: false,
- shrinkWrapBlocks: false,
- reliableMarginRight: true,
- pixelMargin: true
- };
-
- // jQuery.boxModel DEPRECATED in 1.3, use jQuery.support.boxModel instead
- jQuery.boxModel = support.boxModel = (document.compatMode === "CSS1Compat");
-
- // Make sure checked status is properly cloned
- input.checked = true;
- support.noCloneChecked = input.cloneNode( true ).checked;
-
- // Make sure that the options inside disabled selects aren't marked as disabled
- // (WebKit marks them as disabled)
- select.disabled = true;
- support.optDisabled = !opt.disabled;
-
- // Test to see if it's possible to delete an expando from an element
- // Fails in Internet Explorer
- try {
- delete div.test;
- } catch( e ) {
- support.deleteExpando = false;
- }
-
- if ( !div.addEventListener && div.attachEvent && div.fireEvent ) {
- div.attachEvent( "onclick", function() {
- // Cloning a node shouldn't copy over any
- // bound event handlers (IE does this)
- support.noCloneEvent = false;
- });
- div.cloneNode( true ).fireEvent( "onclick" );
- }
-
- // Check if a radio maintains its value
- // after being appended to the DOM
- input = document.createElement("input");
- input.value = "t";
- input.setAttribute("type", "radio");
- support.radioValue = input.value === "t";
-
- input.setAttribute("checked", "checked");
-
- // #11217 - WebKit loses check when the name is after the checked attribute
- input.setAttribute( "name", "t" );
-
- div.appendChild( input );
- fragment = document.createDocumentFragment();
- fragment.appendChild( div.lastChild );
-
- // WebKit doesn't clone checked state correctly in fragments
- support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked;
-
- // Check if a disconnected checkbox will retain its checked
- // value of true after appended to the DOM (IE6/7)
- support.appendChecked = input.checked;
-
- fragment.removeChild( input );
- fragment.appendChild( div );
-
- // Technique from Juriy Zaytsev
- // http://perfectionkills.com/detecting-event-support-without-browser-sniffing/
- // We only care about the case where non-standard event systems
- // are used, namely in IE. Short-circuiting here helps us to
- // avoid an eval call (in setAttribute) which can cause CSP
- // to go haywire. See: https://developer.mozilla.org/en/Security/CSP
- if ( div.attachEvent ) {
- for ( i in {
- submit: 1,
- change: 1,
- focusin: 1
- }) {
- eventName = "on" + i;
- isSupported = ( eventName in div );
- if ( !isSupported ) {
- div.setAttribute( eventName, "return;" );
- isSupported = ( typeof div[ eventName ] === "function" );
- }
- support[ i + "Bubbles" ] = isSupported;
- }
- }
-
- fragment.removeChild( div );
-
- // Null elements to avoid leaks in IE
- fragment = select = opt = div = input = null;
-
- // Run tests that need a body at doc ready
- jQuery(function() {
- var container, outer, inner, table, td, offsetSupport,
- marginDiv, conMarginTop, style, html, positionTopLeftWidthHeight,
- paddingMarginBorderVisibility, paddingMarginBorder,
- body = document.getElementsByTagName("body")[0];
-
- if ( !body ) {
- // Return for frameset docs that don't have a body
- return;
- }
-
- conMarginTop = 1;
- paddingMarginBorder = "padding:0;margin:0;border:";
- positionTopLeftWidthHeight = "position:absolute;top:0;left:0;width:1px;height:1px;";
- paddingMarginBorderVisibility = paddingMarginBorder + "0;visibility:hidden;";
- style = "style='" + positionTopLeftWidthHeight + paddingMarginBorder + "5px solid #000;";
- html = "<div " + style + "display:block;'><div style='" + paddingMarginBorder + "0;display:block;overflow:hidden;'></div></div>" +
- "<table " + style + "' cellpadding='0' cellspacing='0'>" +
- "<tr><td></td></tr></table>";
-
- container = document.createElement("div");
- container.style.cssText = paddingMarginBorderVisibility + "width:0;height:0;position:static;top:0;margin-top:" + conMarginTop + "px";
- body.insertBefore( container, body.firstChild );
-
- // Construct the test element
- div = document.createElement("div");
- container.appendChild( div );
-
- // Check if table cells still have offsetWidth/Height when they are set
- // to display:none and there are still other visible table cells in a
- // table row; if so, offsetWidth/Height are not reliable for use when
- // determining if an element has been hidden directly using
- // display:none (it is still safe to use offsets if a parent element is
- // hidden; don safety goggles and see bug #4512 for more information).
- // (only IE 8 fails this test)
- div.innerHTML = "<table><tr><td style='" + paddingMarginBorder + "0;display:none'></td><td>t</td></tr></table>";
- tds = div.getElementsByTagName( "td" );
- isSupported = ( tds[ 0 ].offsetHeight === 0 );
-
- tds[ 0 ].style.display = "";
- tds[ 1 ].style.display = "none";
-
- // Check if empty table cells still have offsetWidth/Height
- // (IE <= 8 fail this test)
- support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 );
-
- // Check if div with explicit width and no margin-right incorrectly
- // gets computed margin-right based on width of container. For more
- // info see bug #3333
- // Fails in WebKit before Feb 2011 nightlies
- // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
- if ( window.getComputedStyle ) {
- div.innerHTML = "";
- marginDiv = document.createElement( "div" );
- marginDiv.style.width = "0";
- marginDiv.style.marginRight = "0";
- div.style.width = "2px";
- div.appendChild( marginDiv );
- support.reliableMarginRight =
- ( parseInt( ( window.getComputedStyle( marginDiv, null ) || { marginRight: 0 } ).marginRight, 10 ) || 0 ) === 0;
- }
-
- if ( typeof div.style.zoom !== "undefined" ) {
- // Check if natively block-level elements act like inline-block
- // elements when setting their display to 'inline' and giving
- // them layout
- // (IE < 8 does this)
- div.innerHTML = "";
- div.style.width = div.style.padding = "1px";
- div.style.border = 0;
- div.style.overflow = "hidden";
- div.style.display = "inline";
- div.style.zoom = 1;
- support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 );
-
- // Check if elements with layout shrink-wrap their children
- // (IE 6 does this)
- div.style.display = "block";
- div.style.overflow = "visible";
- div.innerHTML = "<div style='width:5px;'></div>";
- support.shrinkWrapBlocks = ( div.offsetWidth !== 3 );
- }
-
- div.style.cssText = positionTopLeftWidthHeight + paddingMarginBorderVisibility;
- div.innerHTML = html;
-
- outer = div.firstChild;
- inner = outer.firstChild;
- td = outer.nextSibling.firstChild.firstChild;
-
- offsetSupport = {
- doesNotAddBorder: ( inner.offsetTop !== 5 ),
- doesAddBorderForTableAndCells: ( td.offsetTop === 5 )
- };
-
- inner.style.position = "fixed";
- inner.style.top = "20px";
-
- // safari subtracts parent border width here which is 5px
- offsetSupport.fixedPosition = ( inner.offsetTop === 20 || inner.offsetTop === 15 );
- inner.style.position = inner.style.top = "";
-
- outer.style.overflow = "hidden";
- outer.style.position = "relative";
-
- offsetSupport.subtractsBorderForOverflowNotVisible = ( inner.offsetTop === -5 );
- offsetSupport.doesNotIncludeMarginInBodyOffset = ( body.offsetTop !== conMarginTop );
-
- if ( window.getComputedStyle ) {
- div.style.marginTop = "1%";
- support.pixelMargin = ( window.getComputedStyle( div, null ) || { marginTop: 0 } ).marginTop !== "1%";
- }
-
- if ( typeof container.style.zoom !== "undefined" ) {
- container.style.zoom = 1;
- }
-
- body.removeChild( container );
- marginDiv = div = container = null;
-
- jQuery.extend( support, offsetSupport );
- });
-
- return support;
-})();
-
-
-
-
-var rbrace = /^(?:\{.*\}|\[.*\])$/,
- rmultiDash = /([A-Z])/g;
-
-jQuery.extend({
- cache: {},
-
- // Please use with caution
- uuid: 0,
-
- // Unique for each copy of jQuery on the page
- // Non-digits removed to match rinlinejQuery
- expando: "jQuery" + ( jQuery.fn.jquery + Math.random() ).replace( /\D/g, "" ),
-
- // The following elements throw uncatchable exceptions if you
- // attempt to add expando properties to them.
- noData: {
- "embed": true,
- // Ban all objects except for Flash (which handle expandos)
- "object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",
- "applet": true
- },
-
- hasData: function( elem ) {
- elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ];
- return !!elem && !isEmptyDataObject( elem );
- },
-
- data: function( elem, name, data, pvt /* Internal Use Only */ ) {
- if ( !jQuery.acceptData( elem ) ) {
- return;
- }
-
- var privateCache, thisCache, ret,
- internalKey = jQuery.expando,
- getByName = typeof name === "string",
-
- // We have to handle DOM nodes and JS objects differently because IE6-7
- // can't GC object references properly across the DOM-JS boundary
- isNode = elem.nodeType,
-
- // Only DOM nodes need the global jQuery cache; JS object data is
- // attached directly to the object so GC can occur automatically
- cache = isNode ? jQuery.cache : elem,
-
- // Only defining an ID for JS objects if its cache already exists allows
- // the code to shortcut on the same path as a DOM node with no cache
- id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey,
- isEvents = name === "events";
-
- // Avoid doing any more work than we need to when trying to get data on an
- // object that has no data at all
- if ( (!id || !cache[id] || (!isEvents && !pvt && !cache[id].data)) && getByName && data === undefined ) {
- return;
- }
-
- if ( !id ) {
- // Only DOM nodes need a new unique ID for each element since their data
- // ends up in the global cache
- if ( isNode ) {
- elem[ internalKey ] = id = ++jQuery.uuid;
- } else {
- id = internalKey;
- }
- }
-
- if ( !cache[ id ] ) {
- cache[ id ] = {};
-
- // Avoids exposing jQuery metadata on plain JS objects when the object
- // is serialized using JSON.stringify
- if ( !isNode ) {
- cache[ id ].toJSON = jQuery.noop;
- }
- }
-
- // An object can be passed to jQuery.data instead of a key/value pair; this gets
- // shallow copied over onto the existing cache
- if ( typeof name === "object" || typeof name === "function" ) {
- if ( pvt ) {
- cache[ id ] = jQuery.extend( cache[ id ], name );
- } else {
- cache[ id ].data = jQuery.extend( cache[ id ].data, name );
- }
- }
-
- privateCache = thisCache = cache[ id ];
-
- // jQuery data() is stored in a separate object inside the object's internal data
- // cache in order to avoid key collisions between internal data and user-defined
- // data.
- if ( !pvt ) {
- if ( !thisCache.data ) {
- thisCache.data = {};
- }
-
- thisCache = thisCache.data;
- }
-
- if ( data !== undefined ) {
- thisCache[ jQuery.camelCase( name ) ] = data;
- }
-
- // Users should not attempt to inspect the internal events object using jQuery.data,
- // it is undocumented and subject to change. But does anyone listen? No.
- if ( isEvents && !thisCache[ name ] ) {
- return privateCache.events;
- }
-
- // Check for both converted-to-camel and non-converted data property names
- // If a data property was specified
- if ( getByName ) {
-
- // First Try to find as-is property data
- ret = thisCache[ name ];
-
- // Test for null|undefined property data
- if ( ret == null ) {
-
- // Try to find the camelCased property
- ret = thisCache[ jQuery.camelCase( name ) ];
- }
- } else {
- ret = thisCache;
- }
-
- return ret;
- },
-
- removeData: function( elem, name, pvt /* Internal Use Only */ ) {
- if ( !jQuery.acceptData( elem ) ) {
- return;
- }
-
- var thisCache, i, l,
-
- // Reference to internal data cache key
- internalKey = jQuery.expando,
-
- isNode = elem.nodeType,
-
- // See jQuery.data for more information
- cache = isNode ? jQuery.cache : elem,
-
- // See jQuery.data for more information
- id = isNode ? elem[ internalKey ] : internalKey;
-
- // If there is already no cache entry for this object, there is no
- // purpose in continuing
- if ( !cache[ id ] ) {
- return;
- }
-
- if ( name ) {
-
- thisCache = pvt ? cache[ id ] : cache[ id ].data;
-
- if ( thisCache ) {
-
- // Support array or space separated string names for data keys
- if ( !jQuery.isArray( name ) ) {
-
- // try the string as a key before any manipulation
- if ( name in thisCache ) {
- name = [ name ];
- } else {
-
- // split the camel cased version by spaces unless a key with the spaces exists
- name = jQuery.camelCase( name );
- if ( name in thisCache ) {
- name = [ name ];
- } else {
- name = name.split( " " );
- }
- }
- }
-
- for ( i = 0, l = name.length; i < l; i++ ) {
- delete thisCache[ name[i] ];
- }
-
- // If there is no data left in the cache, we want to continue
- // and let the cache object itself get destroyed
- if ( !( pvt ? isEmptyDataObject : jQuery.isEmptyObject )( thisCache ) ) {
- return;
- }
- }
- }
-
- // See jQuery.data for more information
- if ( !pvt ) {
- delete cache[ id ].data;
-
- // Don't destroy the parent cache unless the internal data object
- // had been the only thing left in it
- if ( !isEmptyDataObject(cache[ id ]) ) {
- return;
- }
- }
-
- // Browsers that fail expando deletion also refuse to delete expandos on
- // the window, but it will allow it on all other JS objects; other browsers
- // don't care
- // Ensure that `cache` is not a window object #10080
- if ( jQuery.support.deleteExpando || !cache.setInterval ) {
- delete cache[ id ];
- } else {
- cache[ id ] = null;
- }
-
- // We destroyed the cache and need to eliminate the expando on the node to avoid
- // false lookups in the cache for entries that no longer exist
- if ( isNode ) {
- // IE does not allow us to delete expando properties from nodes,
- // nor does it have a removeAttribute function on Document nodes;
- // we must handle all of these cases
- if ( jQuery.support.deleteExpando ) {
- delete elem[ internalKey ];
- } else if ( elem.removeAttribute ) {
- elem.removeAttribute( internalKey );
- } else {
- elem[ internalKey ] = null;
- }
- }
- },
-
- // For internal use only.
- _data: function( elem, name, data ) {
- return jQuery.data( elem, name, data, true );
- },
-
- // A method for determining if a DOM node can handle the data expando
- acceptData: function( elem ) {
- if ( elem.nodeName ) {
- var match = jQuery.noData[ elem.nodeName.toLowerCase() ];
-
- if ( match ) {
- return !(match === true || elem.getAttribute("classid") !== match);
- }
- }
-
- return true;
- }
-});
-
-jQuery.fn.extend({
- data: function( key, value ) {
- var parts, part, attr, name, l,
- elem = this[0],
- i = 0,
- data = null;
-
- // Gets all values
- if ( key === undefined ) {
- if ( this.length ) {
- data = jQuery.data( elem );
-
- if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) {
- attr = elem.attributes;
- for ( l = attr.length; i < l; i++ ) {
- name = attr[i].name;
-
- if ( name.indexOf( "data-" ) === 0 ) {
- name = jQuery.camelCase( name.substring(5) );
-
- dataAttr( elem, name, data[ name ] );
- }
- }
- jQuery._data( elem, "parsedAttrs", true );
- }
- }
-
- return data;
- }
-
- // Sets multiple values
- if ( typeof key === "object" ) {
- return this.each(function() {
- jQuery.data( this, key );
- });
- }
-
- parts = key.split( ".", 2 );
- parts[1] = parts[1] ? "." + parts[1] : "";
- part = parts[1] + "!";
-
- return jQuery.access( this, function( value ) {
-
- if ( value === undefined ) {
- data = this.triggerHandler( "getData" + part, [ parts[0] ] );
-
- // Try to fetch any internally stored data first
- if ( data === undefined && elem ) {
- data = jQuery.data( elem, key );
- data = dataAttr( elem, key, data );
- }
-
- return data === undefined && parts[1] ?
- this.data( parts[0] ) :
- data;
- }
-
- parts[1] = value;
- this.each(function() {
- var self = jQuery( this );
-
- self.triggerHandler( "setData" + part, parts );
- jQuery.data( this, key, value );
- self.triggerHandler( "changeData" + part, parts );
- });
- }, null, value, arguments.length > 1, null, false );
- },
-
- removeData: function( key ) {
- return this.each(function() {
- jQuery.removeData( this, key );
- });
- }
-});
-
-function dataAttr( elem, key, data ) {
- // If nothing was found internally, try to fetch any
- // data from the HTML5 data-* attribute
- if ( data === undefined && elem.nodeType === 1 ) {
-
- var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase();
-
- data = elem.getAttribute( name );
-
- if ( typeof data === "string" ) {
- try {
- data = data === "true" ? true :
- data === "false" ? false :
- data === "null" ? null :
- jQuery.isNumeric( data ) ? +data :
- rbrace.test( data ) ? jQuery.parseJSON( data ) :
- data;
- } catch( e ) {}
-
- // Make sure we set the data so it isn't changed later
- jQuery.data( elem, key, data );
-
- } else {
- data = undefined;
- }
- }
-
- return data;
-}
-
-// checks a cache object for emptiness
-function isEmptyDataObject( obj ) {
- for ( var name in obj ) {
-
- // if the public data object is empty, the private is still empty
- if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) {
- continue;
- }
- if ( name !== "toJSON" ) {
- return false;
- }
- }
-
- return true;
-}
-
-
-
-
-function handleQueueMarkDefer( elem, type, src ) {
- var deferDataKey = type + "defer",
- queueDataKey = type + "queue",
- markDataKey = type + "mark",
- defer = jQuery._data( elem, deferDataKey );
- if ( defer &&
- ( src === "queue" || !jQuery._data(elem, queueDataKey) ) &&
- ( src === "mark" || !jQuery._data(elem, markDataKey) ) ) {
- // Give room for hard-coded callbacks to fire first
- // and eventually mark/queue something else on the element
- setTimeout( function() {
- if ( !jQuery._data( elem, queueDataKey ) &&
- !jQuery._data( elem, markDataKey ) ) {
- jQuery.removeData( elem, deferDataKey, true );
- defer.fire();
- }
- }, 0 );
- }
-}
-
-jQuery.extend({
-
- _mark: function( elem, type ) {
- if ( elem ) {
- type = ( type || "fx" ) + "mark";
- jQuery._data( elem, type, (jQuery._data( elem, type ) || 0) + 1 );
- }
- },
-
- _unmark: function( force, elem, type ) {
- if ( force !== true ) {
- type = elem;
- elem = force;
- force = false;
- }
- if ( elem ) {
- type = type || "fx";
- var key = type + "mark",
- count = force ? 0 : ( (jQuery._data( elem, key ) || 1) - 1 );
- if ( count ) {
- jQuery._data( elem, key, count );
- } else {
- jQuery.removeData( elem, key, true );
- handleQueueMarkDefer( elem, type, "mark" );
- }
- }
- },
-
- queue: function( elem, type, data ) {
- var q;
- if ( elem ) {
- type = ( type || "fx" ) + "queue";
- q = jQuery._data( elem, type );
-
- // Speed up dequeue by getting out quickly if this is just a lookup
- if ( data ) {
- if ( !q || jQuery.isArray(data) ) {
- q = jQuery._data( elem, type, jQuery.makeArray(data) );
- } else {
- q.push( data );
- }
- }
- return q || [];
- }
- },
-
- dequeue: function( elem, type ) {
- type = type || "fx";
-
- var queue = jQuery.queue( elem, type ),
- fn = queue.shift(),
- hooks = {};
-
- // If the fx queue is dequeued, always remove the progress sentinel
- if ( fn === "inprogress" ) {
- fn = queue.shift();
- }
-
- if ( fn ) {
- // Add a progress sentinel to prevent the fx queue from being
- // automatically dequeued
- if ( type === "fx" ) {
- queue.unshift( "inprogress" );
- }
-
- jQuery._data( elem, type + ".run", hooks );
- fn.call( elem, function() {
- jQuery.dequeue( elem, type );
- }, hooks );
- }
-
- if ( !queue.length ) {
- jQuery.removeData( elem, type + "queue " + type + ".run", true );
- handleQueueMarkDefer( elem, type, "queue" );
- }
- }
-});
-
-jQuery.fn.extend({
- queue: function( type, data ) {
- var setter = 2;
-
- if ( typeof type !== "string" ) {
- data = type;
- type = "fx";
- setter--;
- }
-
- if ( arguments.length < setter ) {
- return jQuery.queue( this[0], type );
- }
-
- return data === undefined ?
- this :
- this.each(function() {
- var queue = jQuery.queue( this, type, data );
-
- if ( type === "fx" && queue[0] !== "inprogress" ) {
- jQuery.dequeue( this, type );
- }
- });
- },
- dequeue: function( type ) {
- return this.each(function() {
- jQuery.dequeue( this, type );
- });
- },
- // Based off of the plugin by Clint Helfers, with permission.
- // http://blindsignals.com/index.php/2009/07/jquery-delay/
- delay: function( time, type ) {
- time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time;
- type = type || "fx";
-
- return this.queue( type, function( next, hooks ) {
- var timeout = setTimeout( next, time );
- hooks.stop = function() {
- clearTimeout( timeout );
- };
- });
- },
- clearQueue: function( type ) {
- return this.queue( type || "fx", [] );
- },
- // Get a promise resolved when queues of a certain type
- // are emptied (fx is the type by default)
- promise: function( type, object ) {
- if ( typeof type !== "string" ) {
- object = type;
- type = undefined;
- }
- type = type || "fx";
- var defer = jQuery.Deferred(),
- elements = this,
- i = elements.length,
- count = 1,
- deferDataKey = type + "defer",
- queueDataKey = type + "queue",
- markDataKey = type + "mark",
- tmp;
- function resolve() {
- if ( !( --count ) ) {
- defer.resolveWith( elements, [ elements ] );
- }
- }
- while( i-- ) {
- if (( tmp = jQuery.data( elements[ i ], deferDataKey, undefined, true ) ||
- ( jQuery.data( elements[ i ], queueDataKey, undefined, true ) ||
- jQuery.data( elements[ i ], markDataKey, undefined, true ) ) &&
- jQuery.data( elements[ i ], deferDataKey, jQuery.Callbacks( "once memory" ), true ) )) {
- count++;
- tmp.add( resolve );
- }
- }
- resolve();
- return defer.promise( object );
- }
-});
-
-
-
-
-var rclass = /[\n\t\r]/g,
- rspace = /\s+/,
- rreturn = /\r/g,
- rtype = /^(?:button|input)$/i,
- rfocusable = /^(?:button|input|object|select|textarea)$/i,
- rclickable = /^a(?:rea)?$/i,
- rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i,
- getSetAttribute = jQuery.support.getSetAttribute,
- nodeHook, boolHook, fixSpecified;
-
-jQuery.fn.extend({
- attr: function( name, value ) {
- return jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 );
- },
-
- removeAttr: function( name ) {
- return this.each(function() {
- jQuery.removeAttr( this, name );
- });
- },
-
- prop: function( name, value ) {
- return jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 );
- },
-
- removeProp: function( name ) {
- name = jQuery.propFix[ name ] || name;
- return this.each(function() {
- // try/catch handles cases where IE balks (such as removing a property on window)
- try {
- this[ name ] = undefined;
- delete this[ name ];
- } catch( e ) {}
- });
- },
-
- addClass: function( value ) {
- var classNames, i, l, elem,
- setClass, c, cl;
-
- if ( jQuery.isFunction( value ) ) {
- return this.each(function( j ) {
- jQuery( this ).addClass( value.call(this, j, this.className) );
- });
- }
-
- if ( value && typeof value === "string" ) {
- classNames = value.split( rspace );
-
- for ( i = 0, l = this.length; i < l; i++ ) {
- elem = this[ i ];
-
- if ( elem.nodeType === 1 ) {
- if ( !elem.className && classNames.length === 1 ) {
- elem.className = value;
-
- } else {
- setClass = " " + elem.className + " ";
-
- for ( c = 0, cl = classNames.length; c < cl; c++ ) {
- if ( !~setClass.indexOf( " " + classNames[ c ] + " " ) ) {
- setClass += classNames[ c ] + " ";
- }
- }
- elem.className = jQuery.trim( setClass );
- }
- }
- }
- }
-
- return this;
- },
-
- removeClass: function( value ) {
- var classNames, i, l, elem, className, c, cl;
-
- if ( jQuery.isFunction( value ) ) {
- return this.each(function( j ) {
- jQuery( this ).removeClass( value.call(this, j, this.className) );
- });
- }
-
- if ( (value && typeof value === "string") || value === undefined ) {
- classNames = ( value || "" ).split( rspace );
-
- for ( i = 0, l = this.length; i < l; i++ ) {
- elem = this[ i ];
-
- if ( elem.nodeType === 1 && elem.className ) {
- if ( value ) {
- className = (" " + elem.className + " ").replace( rclass, " " );
- for ( c = 0, cl = classNames.length; c < cl; c++ ) {
- className = className.replace(" " + classNames[ c ] + " ", " ");
- }
- elem.className = jQuery.trim( className );
-
- } else {
- elem.className = "";
- }
- }
- }
- }
-
- return this;
- },
-
- toggleClass: function( value, stateVal ) {
- var type = typeof value,
- isBool = typeof stateVal === "boolean";
-
- if ( jQuery.isFunction( value ) ) {
- return this.each(function( i ) {
- jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal );
- });
- }
-
- return this.each(function() {
- if ( type === "string" ) {
- // toggle individual class names
- var className,
- i = 0,
- self = jQuery( this ),
- state = stateVal,
- classNames = value.split( rspace );
-
- while ( (className = classNames[ i++ ]) ) {
- // check each className given, space seperated list
- state = isBool ? state : !self.hasClass( className );
- self[ state ? "addClass" : "removeClass" ]( className );
- }
-
- } else if ( type === "undefined" || type === "boolean" ) {
- if ( this.className ) {
- // store className if set
- jQuery._data( this, "__className__", this.className );
- }
-
- // toggle whole className
- this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || "";
- }
- });
- },
-
- hasClass: function( selector ) {
- var className = " " + selector + " ",
- i = 0,
- l = this.length;
- for ( ; i < l; i++ ) {
- if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) {
- return true;
- }
- }
-
- return false;
- },
-
- val: function( value ) {
- var hooks, ret, isFunction,
- elem = this[0];
-
- if ( !arguments.length ) {
- if ( elem ) {
- hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ];
-
- if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) {
- return ret;
- }
-
- ret = elem.value;
-
- return typeof ret === "string" ?
- // handle most common string cases
- ret.replace(rreturn, "") :
- // handle cases where value is null/undef or number
- ret == null ? "" : ret;
- }
-
- return;
- }
-
- isFunction = jQuery.isFunction( value );
-
- return this.each(function( i ) {
- var self = jQuery(this), val;
-
- if ( this.nodeType !== 1 ) {
- return;
- }
-
- if ( isFunction ) {
- val = value.call( this, i, self.val() );
- } else {
- val = value;
- }
-
- // Treat null/undefined as ""; convert numbers to string
- if ( val == null ) {
- val = "";
- } else if ( typeof val === "number" ) {
- val += "";
- } else if ( jQuery.isArray( val ) ) {
- val = jQuery.map(val, function ( value ) {
- return value == null ? "" : value + "";
- });
- }
-
- hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ];
-
- // If set returns undefined, fall back to normal setting
- if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) {
- this.value = val;
- }
- });
- }
-});
-
-jQuery.extend({
- valHooks: {
- option: {
- get: function( elem ) {
- // attributes.value is undefined in Blackberry 4.7 but
- // uses .value. See #6932
- var val = elem.attributes.value;
- return !val || val.specified ? elem.value : elem.text;
- }
- },
- select: {
- get: function( elem ) {
- var value, i, max, option,
- index = elem.selectedIndex,
- values = [],
- options = elem.options,
- one = elem.type === "select-one";
-
- // Nothing was selected
- if ( index < 0 ) {
- return null;
- }
-
- // Loop through all the selected options
- i = one ? index : 0;
- max = one ? index + 1 : options.length;
- for ( ; i < max; i++ ) {
- option = options[ i ];
-
- // Don't return options that are disabled or in a disabled optgroup
- if ( option.selected && (jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null) &&
- (!option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" )) ) {
-
- // Get the specific value for the option
- value = jQuery( option ).val();
-
- // We don't need an array for one selects
- if ( one ) {
- return value;
- }
-
- // Multi-Selects return an array
- values.push( value );
- }
- }
-
- // Fixes Bug #2551 -- select.val() broken in IE after form.reset()
- if ( one && !values.length && options.length ) {
- return jQuery( options[ index ] ).val();
- }
-
- return values;
- },
-
- set: function( elem, value ) {
- var values = jQuery.makeArray( value );
-
- jQuery(elem).find("option").each(function() {
- this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0;
- });
-
- if ( !values.length ) {
- elem.selectedIndex = -1;
- }
- return values;
- }
- }
- },
-
- attrFn: {
- val: true,
- css: true,
- html: true,
- text: true,
- data: true,
- width: true,
- height: true,
- offset: true
- },
-
- attr: function( elem, name, value, pass ) {
- var ret, hooks, notxml,
- nType = elem.nodeType;
-
- // don't get/set attributes on text, comment and attribute nodes
- if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
- return;
- }
-
- if ( pass && name in jQuery.attrFn ) {
- return jQuery( elem )[ name ]( value );
- }
-
- // Fallback to prop when attributes are not supported
- if ( typeof elem.getAttribute === "undefined" ) {
- return jQuery.prop( elem, name, value );
- }
-
- notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
-
- // All attributes are lowercase
- // Grab necessary hook if one is defined
- if ( notxml ) {
- name = name.toLowerCase();
- hooks = jQuery.attrHooks[ name ] || ( rboolean.test( name ) ? boolHook : nodeHook );
- }
-
- if ( value !== undefined ) {
-
- if ( value === null ) {
- jQuery.removeAttr( elem, name );
- return;
-
- } else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) {
- return ret;
-
- } else {
- elem.setAttribute( name, "" + value );
- return value;
- }
-
- } else if ( hooks && "get" in hooks && notxml && (ret = hooks.get( elem, name )) !== null ) {
- return ret;
-
- } else {
-
- ret = elem.getAttribute( name );
-
- // Non-existent attributes return null, we normalize to undefined
- return ret === null ?
- undefined :
- ret;
- }
- },
-
- removeAttr: function( elem, value ) {
- var propName, attrNames, name, l, isBool,
- i = 0;
-
- if ( value && elem.nodeType === 1 ) {
- attrNames = value.toLowerCase().split( rspace );
- l = attrNames.length;
-
- for ( ; i < l; i++ ) {
- name = attrNames[ i ];
-
- if ( name ) {
- propName = jQuery.propFix[ name ] || name;
- isBool = rboolean.test( name );
-
- // See #9699 for explanation of this approach (setting first, then removal)
- // Do not do this for boolean attributes (see #10870)
- if ( !isBool ) {
- jQuery.attr( elem, name, "" );
- }
- elem.removeAttribute( getSetAttribute ? name : propName );
-
- // Set corresponding property to false for boolean attributes
- if ( isBool && propName in elem ) {
- elem[ propName ] = false;
- }
- }
- }
- }
- },
-
- attrHooks: {
- type: {
- set: function( elem, value ) {
- // We can't allow the type property to be changed (since it causes problems in IE)
- if ( rtype.test( elem.nodeName ) && elem.parentNode ) {
- jQuery.error( "type property can't be changed" );
- } else if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) {
- // Setting the type on a radio button after the value resets the value in IE6-9
- // Reset value to it's default in case type is set after value
- // This is for element creation
- var val = elem.value;
- elem.setAttribute( "type", value );
- if ( val ) {
- elem.value = val;
- }
- return value;
- }
- }
- },
- // Use the value property for back compat
- // Use the nodeHook for button elements in IE6/7 (#1954)
- value: {
- get: function( elem, name ) {
- if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
- return nodeHook.get( elem, name );
- }
- return name in elem ?
- elem.value :
- null;
- },
- set: function( elem, value, name ) {
- if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
- return nodeHook.set( elem, value, name );
- }
- // Does not return so that setAttribute is also used
- elem.value = value;
- }
- }
- },
-
- propFix: {
- tabindex: "tabIndex",
- readonly: "readOnly",
- "for": "htmlFor",
- "class": "className",
- maxlength: "maxLength",
- cellspacing: "cellSpacing",
- cellpadding: "cellPadding",
- rowspan: "rowSpan",
- colspan: "colSpan",
- usemap: "useMap",
- frameborder: "frameBorder",
- contenteditable: "contentEditable"
- },
-
- prop: function( elem, name, value ) {
- var ret, hooks, notxml,
- nType = elem.nodeType;
-
- // don't get/set properties on text, comment and attribute nodes
- if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
- return;
- }
-
- notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
-
- if ( notxml ) {
- // Fix name and attach hooks
- name = jQuery.propFix[ name ] || name;
- hooks = jQuery.propHooks[ name ];
- }
-
- if ( value !== undefined ) {
- if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) {
- return ret;
-
- } else {
- return ( elem[ name ] = value );
- }
-
- } else {
- if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) {
- return ret;
-
- } else {
- return elem[ name ];
- }
- }
- },
-
- propHooks: {
- tabIndex: {
- get: function( elem ) {
- // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set
- // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/
- var attributeNode = elem.getAttributeNode("tabindex");
-
- return attributeNode && attributeNode.specified ?
- parseInt( attributeNode.value, 10 ) :
- rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ?
- 0 :
- undefined;
- }
- }
- }
-});
-
-// Add the tabIndex propHook to attrHooks for back-compat (different case is intentional)
-jQuery.attrHooks.tabindex = jQuery.propHooks.tabIndex;
-
-// Hook for boolean attributes
-boolHook = {
- get: function( elem, name ) {
- // Align boolean attributes with corresponding properties
- // Fall back to attribute presence where some booleans are not supported
- var attrNode,
- property = jQuery.prop( elem, name );
- return property === true || typeof property !== "boolean" && ( attrNode = elem.getAttributeNode(name) ) && attrNode.nodeValue !== false ?
- name.toLowerCase() :
- undefined;
- },
- set: function( elem, value, name ) {
- var propName;
- if ( value === false ) {
- // Remove boolean attributes when set to false
- jQuery.removeAttr( elem, name );
- } else {
- // value is true since we know at this point it's type boolean and not false
- // Set boolean attributes to the same name and set the DOM property
- propName = jQuery.propFix[ name ] || name;
- if ( propName in elem ) {
- // Only set the IDL specifically if it already exists on the element
- elem[ propName ] = true;
- }
-
- elem.setAttribute( name, name.toLowerCase() );
- }
- return name;
- }
-};
-
-// IE6/7 do not support getting/setting some attributes with get/setAttribute
-if ( !getSetAttribute ) {
-
- fixSpecified = {
- name: true,
- id: true,
- coords: true
- };
-
- // Use this for any attribute in IE6/7
- // This fixes almost every IE6/7 issue
- nodeHook = jQuery.valHooks.button = {
- get: function( elem, name ) {
- var ret;
- ret = elem.getAttributeNode( name );
- return ret && ( fixSpecified[ name ] ? ret.nodeValue !== "" : ret.specified ) ?
- ret.nodeValue :
- undefined;
- },
- set: function( elem, value, name ) {
- // Set the existing or create a new attribute node
- var ret = elem.getAttributeNode( name );
- if ( !ret ) {
- ret = document.createAttribute( name );
- elem.setAttributeNode( ret );
- }
- return ( ret.nodeValue = value + "" );
- }
- };
-
- // Apply the nodeHook to tabindex
- jQuery.attrHooks.tabindex.set = nodeHook.set;
-
- // Set width and height to auto instead of 0 on empty string( Bug #8150 )
- // This is for removals
- jQuery.each([ "width", "height" ], function( i, name ) {
- jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
- set: function( elem, value ) {
- if ( value === "" ) {
- elem.setAttribute( name, "auto" );
- return value;
- }
- }
- });
- });
-
- // Set contenteditable to false on removals(#10429)
- // Setting to empty string throws an error as an invalid value
- jQuery.attrHooks.contenteditable = {
- get: nodeHook.get,
- set: function( elem, value, name ) {
- if ( value === "" ) {
- value = "false";
- }
- nodeHook.set( elem, value, name );
- }
- };
-}
-
-
-// Some attributes require a special call on IE
-if ( !jQuery.support.hrefNormalized ) {
- jQuery.each([ "href", "src", "width", "height" ], function( i, name ) {
- jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
- get: function( elem ) {
- var ret = elem.getAttribute( name, 2 );
- return ret === null ? undefined : ret;
- }
- });
- });
-}
-
-if ( !jQuery.support.style ) {
- jQuery.attrHooks.style = {
- get: function( elem ) {
- // Return undefined in the case of empty string
- // Normalize to lowercase since IE uppercases css property names
- return elem.style.cssText.toLowerCase() || undefined;
- },
- set: function( elem, value ) {
- return ( elem.style.cssText = "" + value );
- }
- };
-}
-
-// Safari mis-reports the default selected property of an option
-// Accessing the parent's selectedIndex property fixes it
-if ( !jQuery.support.optSelected ) {
- jQuery.propHooks.selected = jQuery.extend( jQuery.propHooks.selected, {
- get: function( elem ) {
- var parent = elem.parentNode;
-
- if ( parent ) {
- parent.selectedIndex;
-
- // Make sure that it also works with optgroups, see #5701
- if ( parent.parentNode ) {
- parent.parentNode.selectedIndex;
- }
- }
- return null;
- }
- });
-}
-
-// IE6/7 call enctype encoding
-if ( !jQuery.support.enctype ) {
- jQuery.propFix.enctype = "encoding";
-}
-
-// Radios and checkboxes getter/setter
-if ( !jQuery.support.checkOn ) {
- jQuery.each([ "radio", "checkbox" ], function() {
- jQuery.valHooks[ this ] = {
- get: function( elem ) {
- // Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified
- return elem.getAttribute("value") === null ? "on" : elem.value;
- }
- };
- });
-}
-jQuery.each([ "radio", "checkbox" ], function() {
- jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], {
- set: function( elem, value ) {
- if ( jQuery.isArray( value ) ) {
- return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 );
- }
- }
- });
-});
-
-
-
-
-var rformElems = /^(?:textarea|input|select)$/i,
- rtypenamespace = /^([^\.]*)?(?:\.(.+))?$/,
- rhoverHack = /(?:^|\s)hover(\.\S+)?\b/,
- rkeyEvent = /^key/,
- rmouseEvent = /^(?:mouse|contextmenu)|click/,
- rfocusMorph = /^(?:focusinfocus|focusoutblur)$/,
- rquickIs = /^(\w*)(?:#([\w\-]+))?(?:\.([\w\-]+))?$/,
- quickParse = function( selector ) {
- var quick = rquickIs.exec( selector );
- if ( quick ) {
- // 0 1 2 3
- // [ _, tag, id, class ]
- quick[1] = ( quick[1] || "" ).toLowerCase();
- quick[3] = quick[3] && new RegExp( "(?:^|\\s)" + quick[3] + "(?:\\s|$)" );
- }
- return quick;
- },
- quickIs = function( elem, m ) {
- var attrs = elem.attributes || {};
- return (
- (!m[1] || elem.nodeName.toLowerCase() === m[1]) &&
- (!m[2] || (attrs.id || {}).value === m[2]) &&
- (!m[3] || m[3].test( (attrs[ "class" ] || {}).value ))
- );
- },
- hoverHack = function( events ) {
- return jQuery.event.special.hover ? events : events.replace( rhoverHack, "mouseenter$1 mouseleave$1" );
- };
-
-/*
- * Helper functions for managing events -- not part of the public interface.
- * Props to Dean Edwards' addEvent library for many of the ideas.
- */
-jQuery.event = {
-
- add: function( elem, types, handler, data, selector ) {
-
- var elemData, eventHandle, events,
- t, tns, type, namespaces, handleObj,
- handleObjIn, quick, handlers, special;
-
- // Don't attach events to noData or text/comment nodes (allow plain objects tho)
- if ( elem.nodeType === 3 || elem.nodeType === 8 || !types || !handler || !(elemData = jQuery._data( elem )) ) {
- return;
- }
-
- // Caller can pass in an object of custom data in lieu of the handler
- if ( handler.handler ) {
- handleObjIn = handler;
- handler = handleObjIn.handler;
- selector = handleObjIn.selector;
- }
-
- // Make sure that the handler has a unique ID, used to find/remove it later
- if ( !handler.guid ) {
- handler.guid = jQuery.guid++;
- }
-
- // Init the element's event structure and main handler, if this is the first
- events = elemData.events;
- if ( !events ) {
- elemData.events = events = {};
- }
- eventHandle = elemData.handle;
- if ( !eventHandle ) {
- elemData.handle = eventHandle = function( e ) {
- // Discard the second event of a jQuery.event.trigger() and
- // when an event is called after a page has unloaded
- return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ?
- jQuery.event.dispatch.apply( eventHandle.elem, arguments ) :
- undefined;
- };
- // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events
- eventHandle.elem = elem;
- }
-
- // Handle multiple events separated by a space
- // jQuery(...).bind("mouseover mouseout", fn);
- types = jQuery.trim( hoverHack(types) ).split( " " );
- for ( t = 0; t < types.length; t++ ) {
-
- tns = rtypenamespace.exec( types[t] ) || [];
- type = tns[1];
- namespaces = ( tns[2] || "" ).split( "." ).sort();
-
- // If event changes its type, use the special event handlers for the changed type
- special = jQuery.event.special[ type ] || {};
-
- // If selector defined, determine special event api type, otherwise given type
- type = ( selector ? special.delegateType : special.bindType ) || type;
-
- // Update special based on newly reset type
- special = jQuery.event.special[ type ] || {};
-
- // handleObj is passed to all event handlers
- handleObj = jQuery.extend({
- type: type,
- origType: tns[1],
- data: data,
- handler: handler,
- guid: handler.guid,
- selector: selector,
- quick: selector && quickParse( selector ),
- namespace: namespaces.join(".")
- }, handleObjIn );
-
- // Init the event handler queue if we're the first
- handlers = events[ type ];
- if ( !handlers ) {
- handlers = events[ type ] = [];
- handlers.delegateCount = 0;
-
- // Only use addEventListener/attachEvent if the special events handler returns false
- if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) {
- // Bind the global event handler to the element
- if ( elem.addEventListener ) {
- elem.addEventListener( type, eventHandle, false );
-
- } else if ( elem.attachEvent ) {
- elem.attachEvent( "on" + type, eventHandle );
- }
- }
- }
-
- if ( special.add ) {
- special.add.call( elem, handleObj );
-
- if ( !handleObj.handler.guid ) {
- handleObj.handler.guid = handler.guid;
- }
- }
-
- // Add to the element's handler list, delegates in front
- if ( selector ) {
- handlers.splice( handlers.delegateCount++, 0, handleObj );
- } else {
- handlers.push( handleObj );
- }
-
- // Keep track of which events have ever been used, for event optimization
- jQuery.event.global[ type ] = true;
- }
-
- // Nullify elem to prevent memory leaks in IE
- elem = null;
- },
-
- global: {},
-
- // Detach an event or set of events from an element
- remove: function( elem, types, handler, selector, mappedTypes ) {
-
- var elemData = jQuery.hasData( elem ) && jQuery._data( elem ),
- t, tns, type, origType, namespaces, origCount,
- j, events, special, handle, eventType, handleObj;
-
- if ( !elemData || !(events = elemData.events) ) {
- return;
- }
-
- // Once for each type.namespace in types; type may be omitted
- types = jQuery.trim( hoverHack( types || "" ) ).split(" ");
- for ( t = 0; t < types.length; t++ ) {
- tns = rtypenamespace.exec( types[t] ) || [];
- type = origType = tns[1];
- namespaces = tns[2];
-
- // Unbind all events (on this namespace, if provided) for the element
- if ( !type ) {
- for ( type in events ) {
- jQuery.event.remove( elem, type + types[ t ], handler, selector, true );
- }
- continue;
- }
-
- special = jQuery.event.special[ type ] || {};
- type = ( selector? special.delegateType : special.bindType ) || type;
- eventType = events[ type ] || [];
- origCount = eventType.length;
- namespaces = namespaces ? new RegExp("(^|\\.)" + namespaces.split(".").sort().join("\\.(?:.*\\.)?") + "(\\.|$)") : null;
-
- // Remove matching events
- for ( j = 0; j < eventType.length; j++ ) {
- handleObj = eventType[ j ];
-
- if ( ( mappedTypes || origType === handleObj.origType ) &&
- ( !handler || handler.guid === handleObj.guid ) &&
- ( !namespaces || namespaces.test( handleObj.namespace ) ) &&
- ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) {
- eventType.splice( j--, 1 );
-
- if ( handleObj.selector ) {
- eventType.delegateCount--;
- }
- if ( special.remove ) {
- special.remove.call( elem, handleObj );
- }
- }
- }
-
- // Remove generic event handler if we removed something and no more handlers exist
- // (avoids potential for endless recursion during removal of special event handlers)
- if ( eventType.length === 0 && origCount !== eventType.length ) {
- if ( !special.teardown || special.teardown.call( elem, namespaces ) === false ) {
- jQuery.removeEvent( elem, type, elemData.handle );
- }
-
- delete events[ type ];
- }
- }
-
- // Remove the expando if it's no longer used
- if ( jQuery.isEmptyObject( events ) ) {
- handle = elemData.handle;
- if ( handle ) {
- handle.elem = null;
- }
-
- // removeData also checks for emptiness and clears the expando if empty
- // so use it instead of delete
- jQuery.removeData( elem, [ "events", "handle" ], true );
- }
- },
-
- // Events that are safe to short-circuit if no handlers are attached.
- // Native DOM events should not be added, they may have inline handlers.
- customEvent: {
- "getData": true,
- "setData": true,
- "changeData": true
- },
-
- trigger: function( event, data, elem, onlyHandlers ) {
- // Don't do events on text and comment nodes
- if ( elem && (elem.nodeType === 3 || elem.nodeType === 8) ) {
- return;
- }
-
- // Event object or event type
- var type = event.type || event,
- namespaces = [],
- cache, exclusive, i, cur, old, ontype, special, handle, eventPath, bubbleType;
-
- // focus/blur morphs to focusin/out; ensure we're not firing them right now
- if ( rfocusMorph.test( type + jQuery.event.triggered ) ) {
- return;
- }
-
- if ( type.indexOf( "!" ) >= 0 ) {
- // Exclusive events trigger only for the exact event (no namespaces)
- type = type.slice(0, -1);
- exclusive = true;
- }
-
- if ( type.indexOf( "." ) >= 0 ) {
- // Namespaced trigger; create a regexp to match event type in handle()
- namespaces = type.split(".");
- type = namespaces.shift();
- namespaces.sort();
- }
-
- if ( (!elem || jQuery.event.customEvent[ type ]) && !jQuery.event.global[ type ] ) {
- // No jQuery handlers for this event type, and it can't have inline handlers
- return;
- }
-
- // Caller can pass in an Event, Object, or just an event type string
- event = typeof event === "object" ?
- // jQuery.Event object
- event[ jQuery.expando ] ? event :
- // Object literal
- new jQuery.Event( type, event ) :
- // Just the event type (string)
- new jQuery.Event( type );
-
- event.type = type;
- event.isTrigger = true;
- event.exclusive = exclusive;
- event.namespace = namespaces.join( "." );
- event.namespace_re = event.namespace? new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.)?") + "(\\.|$)") : null;
- ontype = type.indexOf( ":" ) < 0 ? "on" + type : "";
-
- // Handle a global trigger
- if ( !elem ) {
-
- // TODO: Stop taunting the data cache; remove global events and always attach to document
- cache = jQuery.cache;
- for ( i in cache ) {
- if ( cache[ i ].events && cache[ i ].events[ type ] ) {
- jQuery.event.trigger( event, data, cache[ i ].handle.el
<TRUNCATED>
[48/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/TaskScheduler.java b/src/main/java/com/twitter/aurora/scheduler/async/TaskScheduler.java
deleted file mode 100644
index 0ad9e13..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/TaskScheduler.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Ticker;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.collect.Iterables;
-import com.google.common.eventbus.Subscribe;
-import com.google.inject.BindingAnnotation;
-
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.SlaveID;
-import org.apache.mesos.Protos.TaskInfo;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import com.twitter.aurora.scheduler.state.StateManager;
-import com.twitter.aurora.scheduler.state.TaskAssigner;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.StatImpl;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.LOST;
-import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
-
-/**
- * Enables scheduling and preemption of tasks.
- */
-interface TaskScheduler extends EventSubscriber {
-
- /**
- * Attempts to schedule a task, possibly performing irreversible actions.
- *
- * @param taskId The task to attempt to schedule.
- * @return SUCCESS if the task was scheduled, TRY_AGAIN otherwise. The caller should call schedule
- * again if TRY_AGAIN is returned.
- */
- TaskSchedulerResult schedule(String taskId);
-
- enum TaskSchedulerResult {
- SUCCESS,
- TRY_AGAIN
- }
-
- /**
- * An asynchronous task scheduler. Scheduling of tasks is performed on a delay, where each task
- * backs off after a failed scheduling attempt.
- * <p>
- * Pending tasks are advertised to the scheduler via internal pubsub notifications.
- */
- class TaskSchedulerImpl implements TaskScheduler {
- /**
- * Binding annotation for the time duration of reservations
- */
- @BindingAnnotation
- @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
- @interface ReservationDuration { }
-
- private static final Logger LOG = Logger.getLogger(TaskSchedulerImpl.class.getName());
-
- private final Storage storage;
- private final StateManager stateManager;
- private final TaskAssigner assigner;
- private final OfferQueue offerQueue;
- private final Preemptor preemptor;
- private final Reservations reservations;
-
- private final AtomicLong scheduleAttemptsFired = Stats.exportLong("schedule_attempts_fired");
- private final AtomicLong scheduleAttemptsFailed = Stats.exportLong("schedule_attempts_failed");
-
- @Inject
- TaskSchedulerImpl(
- Storage storage,
- StateManager stateManager,
- TaskAssigner assigner,
- OfferQueue offerQueue,
- Preemptor preemptor,
- @ReservationDuration Amount<Long, Time> reservationDuration,
- final Clock clock) {
-
- this.storage = checkNotNull(storage);
- this.stateManager = checkNotNull(stateManager);
- this.assigner = checkNotNull(assigner);
- this.offerQueue = checkNotNull(offerQueue);
- this.preemptor = checkNotNull(preemptor);
- this.reservations = new Reservations(reservationDuration, clock);
- }
-
- private Function<Offer, Optional<TaskInfo>> getAssignerFunction(
- final String taskId,
- final IScheduledTask task) {
-
- return new Function<Offer, Optional<TaskInfo>>() {
- @Override public Optional<TaskInfo> apply(Offer offer) {
- Optional<String> reservedTaskId = reservations.getSlaveReservation(offer.getSlaveId());
- if (reservedTaskId.isPresent()) {
- if (taskId.equals(reservedTaskId.get())) {
- // Slave is reserved to satisfy this task.
- return assigner.maybeAssign(offer, task);
- } else {
- // Slave is reserved for another task.
- return Optional.absent();
- }
- } else {
- // Slave is not reserved.
- return assigner.maybeAssign(offer, task);
- }
- }
- };
- }
-
- @VisibleForTesting
- static final Optional<String> LAUNCH_FAILED_MSG =
- Optional.of("Unknown exception attempting to schedule task.");
-
- @Timed("task_schedule_attempt")
- @Override
- public TaskSchedulerResult schedule(final String taskId) {
- scheduleAttemptsFired.incrementAndGet();
- try {
- return storage.write(new MutateWork.Quiet<TaskSchedulerResult>() {
- @Override public TaskSchedulerResult apply(MutableStoreProvider store) {
- LOG.fine("Attempting to schedule task " + taskId);
- Query.Builder pendingTaskQuery = Query.taskScoped(taskId).byStatus(PENDING);
- final IScheduledTask task =
- Iterables.getOnlyElement(store.getTaskStore().fetchTasks(pendingTaskQuery), null);
- if (task == null) {
- LOG.warning("Failed to look up task " + taskId + ", it may have been deleted.");
- } else {
- try {
- if (!offerQueue.launchFirst(getAssignerFunction(taskId, task))) {
- // Task could not be scheduled.
- maybePreemptFor(taskId);
- return TaskSchedulerResult.TRY_AGAIN;
- }
- } catch (OfferQueue.LaunchException e) {
- LOG.log(Level.WARNING, "Failed to launch task.", e);
- scheduleAttemptsFailed.incrementAndGet();
-
- // The attempt to schedule the task failed, so we need to backpedal on the
- // assignment.
- // It is in the LOST state and a new task will move to PENDING to replace it.
- // Should the state change fail due to storage issues, that's okay. The task will
- // time out in the ASSIGNED state and be moved to LOST.
- stateManager.changeState(pendingTaskQuery, LOST, LAUNCH_FAILED_MSG);
- }
- }
-
- return TaskSchedulerResult.SUCCESS;
- }
- });
- } catch (RuntimeException e) {
- // We catch the generic unchecked exception here to ensure tasks are not abandoned
- // if there is a transient issue resulting in an unchecked exception.
- LOG.log(Level.WARNING, "Task scheduling unexpectedly failed, will be retried", e);
- scheduleAttemptsFailed.incrementAndGet();
- return TaskSchedulerResult.TRY_AGAIN;
- }
- }
-
- private void maybePreemptFor(String taskId) {
- if (reservations.hasReservationForTask(taskId)) {
- return;
- }
- Optional<String> slaveId = preemptor.findPreemptionSlotFor(taskId);
- if (slaveId.isPresent()) {
- this.reservations.add(SlaveID.newBuilder().setValue(slaveId.get()).build(), taskId);
- }
- }
-
- @Subscribe
- public void taskChanged(final TaskStateChange stateChangeEvent) {
- if (stateChangeEvent.getOldState() == PENDING) {
- reservations.invalidateTask(stateChangeEvent.getTaskId());
- }
- }
-
- private static class Reservations {
- private final Cache<SlaveID, String> reservations;
-
- Reservations(final Amount<Long, Time> duration, final Clock clock) {
- checkNotNull(duration);
- checkNotNull(clock);
- this.reservations = CacheBuilder.newBuilder()
- .expireAfterWrite(duration.as(Time.MINUTES), TimeUnit.MINUTES)
- .ticker(new Ticker() {
- @Override public long read() {
- return clock.nowNanos();
- }
- })
- .build();
- Stats.export(new StatImpl<Long>("reservation_cache_size") {
- @Override public Long read() {
- return reservations.size();
- }
- });
- }
-
- private synchronized void add(SlaveID slaveId, String taskId) {
- reservations.put(slaveId, taskId);
- }
-
- private synchronized boolean hasReservationForTask(String taskId) {
- return reservations.asMap().containsValue(taskId);
- }
-
- private synchronized Optional<String> getSlaveReservation(SlaveID slaveID) {
- return Optional.fromNullable(reservations.getIfPresent(slaveID));
- }
-
- private synchronized void invalidateTask(String taskId) {
- reservations.asMap().values().remove(taskId);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/TaskTimeout.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/TaskTimeout.java b/src/main/java/com/twitter/aurora/scheduler/async/TaskTimeout.java
deleted file mode 100644
index 19848c7..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/TaskTimeout.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Ordering;
-import com.google.common.eventbus.Subscribe;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import com.twitter.aurora.scheduler.state.StateManager;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.StatsProvider;
-import com.twitter.common.util.Clock;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Observes task transitions and identifies tasks that are 'stuck' in a transient state. Stuck
- * tasks will be transitioned to the LOST state.
- */
-class TaskTimeout implements EventSubscriber {
- private static final Logger LOG = Logger.getLogger(TaskTimeout.class.getName());
-
- @VisibleForTesting
- static final String TIMED_OUT_TASKS_COUNTER = "timed_out_tasks";
-
- @VisibleForTesting
- static final String TRANSIENT_COUNT_STAT_NAME = "transient_states";
-
- @VisibleForTesting
- static final Optional<String> TIMEOUT_MESSAGE = Optional.of("Task timed out");
-
- @VisibleForTesting
- static final Set<ScheduleStatus> TRANSIENT_STATES = EnumSet.of(
- ScheduleStatus.ASSIGNED,
- ScheduleStatus.PREEMPTING,
- ScheduleStatus.RESTARTING,
- ScheduleStatus.KILLING);
-
- @VisibleForTesting
- static final Query.Builder TRANSIENT_QUERY = Query.unscoped().byStatus(TRANSIENT_STATES);
-
- private final Map<TimeoutKey, Context> futures = Maps.newConcurrentMap();
-
- private static final class TimeoutKey {
- private final String taskId;
- private final ScheduleStatus status;
-
- private TimeoutKey(String taskId, ScheduleStatus status) {
- this.taskId = taskId;
- this.status = status;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(taskId, status);
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof TimeoutKey)) {
- return false;
- }
- TimeoutKey key = (TimeoutKey) o;
- return Objects.equal(taskId, key.taskId)
- && (status == key.status);
- }
-
- @Override
- public String toString() {
- return taskId + ":" + status;
- }
- }
-
- private final Storage storage;
- private final ScheduledExecutorService executor;
- private final StateManager stateManager;
- private final long timeoutMillis;
- private final Clock clock;
- private final AtomicLong timedOutTasks;
-
- @Inject
- TaskTimeout(
- Storage storage,
- ScheduledExecutorService executor,
- StateManager stateManager,
- final Clock clock,
- Amount<Long, Time> timeout,
- StatsProvider statsProvider) {
-
- this.storage = checkNotNull(storage);
- this.executor = checkNotNull(executor);
- this.stateManager = checkNotNull(stateManager);
- this.timeoutMillis = timeout.as(Time.MILLISECONDS);
- this.clock = checkNotNull(clock);
- this.timedOutTasks = statsProvider.makeCounter(TIMED_OUT_TASKS_COUNTER);
-
- exportStats(statsProvider);
- }
-
- private void registerTimeout(TimeoutKey key) {
- // This is an obvious check-then-act, but:
- // - there isn't much of a better option, given that we have to get the Future before
- // inserting into the map
- // - a key collision only happens in practice if something is wrong externally to this class
- // (double event for the same state)
- // - the outcome is low-risk, we would wind up with a redundant Future that will eventually
- // no-op
- if (!futures.containsKey(key)) {
- Future<?> timeoutHandler = executor.schedule(
- new TimedOutTaskHandler(key),
- timeoutMillis,
- TimeUnit.MILLISECONDS);
- futures.put(key, new Context(clock.nowMillis(), timeoutHandler));
- }
- }
-
- private static boolean isTransient(ScheduleStatus status) {
- return TRANSIENT_STATES.contains(status);
- }
-
- @Subscribe
- public void recordStateChange(TaskStateChange change) {
- String taskId = change.getTaskId();
- ScheduleStatus newState = change.getNewState();
- if (isTransient(change.getOldState())) {
- TimeoutKey oldKey = new TimeoutKey(taskId, change.getOldState());
- Context context = futures.remove(oldKey);
- if (context != null) {
- LOG.fine("Canceling state timeout for task " + oldKey);
- context.future.cancel(false);
- }
- }
-
- if (isTransient(newState)) {
- registerTimeout(new TimeoutKey(taskId, change.getNewState()));
- }
- }
-
- @Subscribe
- public void storageStarted(StorageStarted event) {
- for (IScheduledTask task : Storage.Util.consistentFetchTasks(storage, TRANSIENT_QUERY)) {
- registerTimeout(new TimeoutKey(Tasks.id(task), task.getStatus()));
- }
- }
-
- private class TimedOutTaskHandler implements Runnable {
- private final TimeoutKey key;
-
- TimedOutTaskHandler(TimeoutKey key) {
- this.key = key;
- }
-
- @Override public void run() {
- Context context = futures.get(key);
- try {
- if (context == null) {
- LOG.warning("Timeout context not found for " + key);
- return;
- }
-
- LOG.info("Timeout reached for task " + key);
- // This query acts as a CAS by including the state that we expect the task to be in if the
- // timeout is still valid. Ideally, the future would have already been canceled, but in the
- // event of a state transition race, including transientState prevents an unintended
- // task timeout.
- Query.Builder query = Query.taskScoped(key.taskId).byStatus(key.status);
- // Note: This requires LOST transitions trigger Driver.killTask.
- if (stateManager.changeState(query, ScheduleStatus.LOST, TIMEOUT_MESSAGE) > 0) {
- timedOutTasks.incrementAndGet();
- } else {
- LOG.warning("Task " + key + " does not exist, or was not in the expected state.");
- }
- } finally {
- futures.remove(key);
- }
- }
- }
-
- private class Context {
- private final long timestampMillis;
- private final Future<?> future;
-
- Context(long timestampMillis, Future<?> future) {
- this.timestampMillis = timestampMillis;
- this.future = future;
- }
- }
-
- private static final Function<Context, Long> CONTEXT_TIMESTAMP = new Function<Context, Long>() {
- @Override public Long apply(Context context) {
- return context.timestampMillis;
- }
- };
-
- private static final Ordering<Context> TIMESTAMP_ORDER =
- Ordering.natural().onResultOf(CONTEXT_TIMESTAMP);
-
- @VisibleForTesting
- static String waitingTimeStatName(ScheduleStatus status) {
- return "scheduler_max_" + status + "_waiting_ms";
- }
-
- private void exportStats(StatsProvider statsProvider) {
- statsProvider.makeGauge(TRANSIENT_COUNT_STAT_NAME, new Supplier<Number>() {
- @Override public Number get() {
- return futures.size();
- }
- });
-
- for (final ScheduleStatus status : TRANSIENT_STATES) {
- statsProvider.makeGauge(waitingTimeStatName(status), new Supplier<Number>() {
- private final Predicate<TimeoutKey> statusMatcher = new Predicate<TimeoutKey>() {
- @Override public boolean apply(TimeoutKey key) {
- return key.status == status;
- }
- };
-
- @Override public Number get() {
- Iterable<Context> matches = Maps.filterKeys(futures, statusMatcher).values();
- if (Iterables.isEmpty(matches)) {
- return 0L;
- } else {
- return clock.nowMillis() - TIMESTAMP_ORDER.min(matches).timestampMillis;
- }
- }
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/CommandUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/CommandUtil.java b/src/main/java/com/twitter/aurora/scheduler/base/CommandUtil.java
deleted file mode 100644
index b11c683..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/CommandUtil.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-import org.apache.mesos.Protos.CommandInfo;
-import org.apache.mesos.Protos.CommandInfo.URI;
-
-import com.twitter.common.base.MorePreconditions;
-
-/**
- * Utility class for constructing {@link CommandInfo} objects given an executor URI.
- */
-public final class CommandUtil {
-
- private CommandUtil() {
- // Utility class.
- }
-
- private static String uriBasename(String uri) {
- int lastSlash = uri.lastIndexOf("/");
- if (lastSlash == -1) {
- return uri;
- } else {
- String basename = uri.substring(lastSlash + 1);
- MorePreconditions.checkNotBlank(basename, "URI must not end with a slash.");
-
- return basename;
- }
- }
-
- /**
- * Creates a description of a command that will fetch and execute the given URI to an executor
- * binary.
- *
- * @param executorUri URI to the executor.
- * @return A command that will fetch and execute the executor.
- */
- public static CommandInfo create(String executorUri) {
- MorePreconditions.checkNotBlank(executorUri);
-
- return CommandInfo.newBuilder()
- .addUris(URI.newBuilder().setValue(executorUri).setExecutable(true))
- .setValue("./" + uriBasename(executorUri))
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/Conversions.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/Conversions.java b/src/main/java/com/twitter/aurora/scheduler/base/Conversions.java
deleted file mode 100644
index 2f84b5c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/Conversions.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-import java.util.Collection;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.logging.Logger;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-
-import org.apache.mesos.Protos;
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.TaskState;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.ScheduleStatus;
-
-/**
- * Collection of utility functions to convert mesos protobuf types to internal thrift types.
- */
-public final class Conversions {
-
- private static final Logger LOG = Logger.getLogger(Conversions.class.getName());
-
- private Conversions() {
- // Utility class.
- }
-
- // Maps from mesos state to scheduler interface state.
- private static final Map<TaskState, ScheduleStatus> STATE_TRANSLATION =
- new ImmutableMap.Builder<TaskState, ScheduleStatus>()
- .put(TaskState.TASK_STARTING, ScheduleStatus.STARTING)
- .put(TaskState.TASK_RUNNING, ScheduleStatus.RUNNING)
- .put(TaskState.TASK_FINISHED, ScheduleStatus.FINISHED)
- .put(TaskState.TASK_FAILED, ScheduleStatus.FAILED)
- .put(TaskState.TASK_KILLED, ScheduleStatus.KILLED)
- .put(TaskState.TASK_LOST, ScheduleStatus.LOST)
- .build();
-
- /**
- * Converts a protobuf state to an internal schedule status.
- *
- * @param taskState Protobuf state.
- * @return Equivalent thrift-generated state.
- */
- public static ScheduleStatus convertProtoState(TaskState taskState) {
- ScheduleStatus status = STATE_TRANSLATION.get(taskState);
- Preconditions.checkArgument(status != null, "Unrecognized task state " + taskState);
- return status;
- }
-
- private static final Function<Protos.Attribute, String> ATTRIBUTE_NAME =
- new Function<Protos.Attribute, String>() {
- @Override public String apply(Protos.Attribute attr) {
- return attr.getName();
- }
- };
-
- /**
- * Typedef to make anonymous implementation more concise.
- */
- private abstract static class AttributeConverter
- implements Function<Entry<String, Collection<Protos.Attribute>>, Attribute> {
- }
-
- private static final Function<Protos.Attribute, String> VALUE_CONVERTER =
- new Function<Protos.Attribute, String>() {
- @Override public String apply(Protos.Attribute attribute) {
- switch (attribute.getType()) {
- case SCALAR:
- return String.valueOf(attribute.getScalar().getValue());
-
- case TEXT:
- return attribute.getText().getValue();
-
- default:
- LOG.finest("Unrecognized attribute type:" + attribute.getType() + " , ignoring.");
- return null;
- }
- }
- };
-
- private static final AttributeConverter ATTRIBUTE_CONVERTER = new AttributeConverter() {
- @Override public Attribute apply(Entry<String, Collection<Protos.Attribute>> entry) {
- // Convert values and filter any that were ignored.
- return new Attribute(
- entry.getKey(),
- FluentIterable.from(entry.getValue())
- .transform(VALUE_CONVERTER)
- .filter(Predicates.notNull())
- .toSet());
- }
- };
-
- /**
- * Converts protobuf attributes into thrift-generated attributes.
- *
- * @param offer Resource offer.
- * @return Equivalent thrift host attributes.
- */
- public static HostAttributes getAttributes(Offer offer) {
- // Group by attribute name.
- Multimap<String, Protos.Attribute> valuesByName =
- Multimaps.index(offer.getAttributesList(), ATTRIBUTE_NAME);
-
- // TODO(William Farner): Include slave id.
- return new HostAttributes(
- offer.getHostname(),
- FluentIterable.from(valuesByName.asMap().entrySet())
- .transform(ATTRIBUTE_CONVERTER)
- .toSet())
- .setSlaveId(offer.getSlaveId().getValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/JobKeys.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/JobKeys.java b/src/main/java/com/twitter/aurora/scheduler/base/JobKeys.java
deleted file mode 100644
index 008e1cb..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/JobKeys.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Function;
-import com.google.common.base.Functions;
-import com.google.common.base.Optional;
-import com.google.common.base.Strings;
-
-import com.twitter.aurora.gen.JobKey;
-import com.twitter.aurora.gen.TaskQuery;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-/**
- * Utility class providing convenience functions relating to JobKeys.
- */
-public final class JobKeys {
- private JobKeys() {
- // Utility class.
- }
-
- public static final Function<IJobConfiguration, IJobKey> FROM_CONFIG =
- new Function<IJobConfiguration, IJobKey>() {
- @Override public IJobKey apply(IJobConfiguration job) {
- return job.getKey();
- }
- };
-
- public static final Function<IJobKey, String> TO_ROLE =
- new Function<IJobKey, String>() {
- @Override public String apply(IJobKey jobKey) {
- return jobKey.getRole();
- }
- };
-
- public static final Function<IJobKey, String> TO_ENVIRONMENT =
- new Function<IJobKey, String>() {
- @Override public String apply(IJobKey jobKey) {
- return jobKey.getEnvironment();
- }
- };
-
- public static final Function<IJobKey, String> TO_JOB_NAME =
- new Function<IJobKey, String>() {
- @Override public String apply(IJobKey jobKey) {
- return jobKey.getName();
- }
- };
-
- public static final Function<IJobConfiguration, String> CONFIG_TO_ROLE =
- Functions.compose(TO_ROLE, FROM_CONFIG);
-
- /**
- * Check that a jobKey struct is valid.
- *
- * @param jobKey The jobKey to validate.
- * @return {@code true} if the jobKey validates.
- */
- public static boolean isValid(@Nullable IJobKey jobKey) {
- return jobKey != null
- && !Strings.isNullOrEmpty(jobKey.getRole())
- && !Strings.isNullOrEmpty(jobKey.getEnvironment())
- && !Strings.isNullOrEmpty(jobKey.getName());
- }
-
- /**
- * Assert that a jobKey struct is valid.
- *
- * @param jobKey The key struct to validate.
- * @return The validated jobKey argument.
- * @throws IllegalArgumentException if the key struct fails to validate.
- */
- public static IJobKey assertValid(IJobKey jobKey) throws IllegalArgumentException {
- checkArgument(isValid(jobKey));
-
- return jobKey;
- }
-
- /**
- * Attempt to create a valid JobKey from the given (role, environment, name) triple.
- *
- * @param role The job's role.
- * @param environment The job's environment.
- * @param name The job's name.
- * @return A valid JobKey if it can be created.
- * @throws IllegalArgumentException if the key fails to validate.
- */
- public static IJobKey from(String role, String environment, String name)
- throws IllegalArgumentException {
-
- IJobKey job = IJobKey.build(new JobKey()
- .setRole(role)
- .setEnvironment(environment)
- .setName(name));
- return assertValid(job);
- }
-
- /**
- * Attempts to create a valid JobKey from the given task.
- *
- * @param task The task to create job key from.
- * @return A valid JobKey if it can be created.
- * @throws IllegalArgumentException if the key fails to validate.
- */
- public static IJobKey from(ITaskConfig task) throws IllegalArgumentException {
- return from(task.getOwner().getRole(), task.getEnvironment(), task.getJobName());
- }
-
- /**
- * Create a "/"-delimited String representation of a job key, suitable for logging but not
- * necessarily suitable for use as a unique identifier.
- *
- * @param jobKey Key to represent.
- * @return "/"-delimited representation of the key.
- */
- public static String toPath(IJobKey jobKey) {
- return jobKey.getRole() + "/" + jobKey.getEnvironment() + "/" + jobKey.getName();
- }
-
- /**
- * Create a "/"-delimited String representation of job key, suitable for logging but not
- * necessarily suitable for use as a unique identifier.
- *
- * @param job Job to represent.
- * @return "/"-delimited representation of the job's key.
- */
- public static String toPath(IJobConfiguration job) {
- return toPath(job.getKey());
- }
-
- /**
- * Attempt to extract a job key from the given query if it is scoped to a single job.
- *
- * @param query Query to extract the key from.
- * @return A present if one can be extracted, absent otherwise.
- */
- public static Optional<IJobKey> from(Query.Builder query) {
- if (Query.isJobScoped(query)) {
- TaskQuery taskQuery = query.get();
- return Optional.of(
- from(taskQuery.getOwner().getRole(), taskQuery.getEnvironment(), taskQuery.getJobName()));
-
- } else {
- return Optional.absent();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/Numbers.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/Numbers.java b/src/main/java/com/twitter/aurora/scheduler/base/Numbers.java
deleted file mode 100644
index 74b5e0b..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/Numbers.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-import java.util.Set;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.PeekingIterator;
-import com.google.common.collect.Range;
-import com.google.common.collect.Sets;
-
-/**
- * Utility class for working with numbers.
- */
-public final class Numbers {
-
- private Numbers() {
- // Utility class.
- }
-
- /**
- * Converts a set of integers into a set of contiguous closed ranges that equally represent the
- * input integers.
- * <p>
- * The resulting ranges will be in ascending order.
- *
- * @param values Values to transform to ranges.
- * @return Closed ranges with identical members to the input set.
- */
- public static Set<Range<Integer>> toRanges(Iterable<Integer> values) {
- ImmutableSet.Builder<Range<Integer>> builder = ImmutableSet.builder();
-
- PeekingIterator<Integer> iterator =
- Iterators.peekingIterator(Sets.newTreeSet(values).iterator());
-
- // Build ranges until there are no numbers left.
- while (iterator.hasNext()) {
- // Start a new range.
- int start = iterator.next();
- int end = start;
- // Increment the end until the range is non-contiguous.
- while (iterator.hasNext() && (iterator.peek() == (end + 1))) {
- end++;
- iterator.next();
- }
-
- builder.add(Range.closed(start, end));
- }
-
- return builder.build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/Query.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/Query.java b/src/main/java/com/twitter/aurora/scheduler/base/Query.java
deleted file mode 100644
index d02ef87..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/Query.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-import java.util.EnumSet;
-
-import com.google.common.base.Objects;
-import com.google.common.base.Optional;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.primitives.Ints;
-
-import com.twitter.aurora.gen.Identity;
-import com.twitter.aurora.gen.InstanceKey;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.TaskQuery;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static org.apache.commons.lang.StringUtils.isEmpty;
-
-/**
- * A utility class to construct storage queries.
- * TODO(Sathya): Add some basic unit tests for isJobScoped and isOnlyJobScoped.
- */
-public final class Query {
-
- private Query() {
- // Utility.
- }
-
- /**
- * Checks whether a query is scoped to a specific job.
- * A query scoped to a job specifies a role and job name.
- *
- * @param taskQuery Query to test.
- * @return {@code true} if the query specifies at least a role and job name,
- * otherwise {@code false}.
- */
- public static boolean isJobScoped(Builder taskQuery) {
- TaskQuery query = taskQuery.get();
- return (query.getOwner() != null)
- && !isEmpty(query.getOwner().getRole())
- && !isEmpty(query.getEnvironment())
- && !isEmpty(query.getJobName());
- }
-
- /**
- * Checks whether a query is strictly scoped to a specific job. A query is strictly job scoped,
- * iff it has the role, environment and jobName set.
- *
- * @param query Query to test.
- * @return {@code true} if the query is strictly job scoped, otherwise {@code false}.
- */
- public static boolean isOnlyJobScoped(Builder query) {
- Optional<IJobKey> jobKey = JobKeys.from(query);
- return jobKey.isPresent() && Query.jobScoped(jobKey.get()).equals(query);
- }
-
- public static Builder arbitrary(TaskQuery query) {
- return new Builder(query.deepCopy());
- }
-
- public static Builder unscoped() {
- return new Builder();
- }
-
- public static Builder roleScoped(String role) {
- return unscoped().byRole(role);
- }
-
- public static Builder envScoped(String role, String environment) {
- return unscoped().byEnv(role, environment);
- }
-
- public static Builder jobScoped(IJobKey jobKey) {
- return unscoped().byJob(jobKey);
- }
-
- public static Builder instanceScoped(InstanceKey instanceKey) {
- return instanceScoped(IJobKey.build(instanceKey.getJobKey()), instanceKey.getInstanceId());
- }
-
- public static Builder instanceScoped(IJobKey jobKey, int instanceId, int... instanceIds) {
- return unscoped().byInstances(jobKey, instanceId, instanceIds);
- }
-
- public static Builder instanceScoped(IJobKey jobKey, Iterable<Integer> instanceIds) {
- return unscoped().byInstances(jobKey, instanceIds);
- }
-
- public static Builder taskScoped(String taskId, String... taskIds) {
- return unscoped().byId(taskId, taskIds);
- }
-
- public static Builder taskScoped(Iterable<String> taskIds) {
- return unscoped().byId(taskIds);
- }
-
- public static Builder slaveScoped(String slaveHost) {
- return unscoped().bySlave(slaveHost);
- }
-
- public static Builder statusScoped(ScheduleStatus status, ScheduleStatus... statuses) {
- return unscoped().byStatus(status, statuses);
- }
-
- public static Builder statusScoped(Iterable<ScheduleStatus> statuses) {
- return unscoped().byStatus(statuses);
- }
-
- /**
- * A Builder of TaskQueries. Builders are immutable and provide access to a set of convenience
- * methods to return a new builder of another scope. Available scope filters include slave,
- * taskId, role, jobs of a role, and instances of a job.
- *
- * <p>
- * This class does not expose the full functionality of TaskQuery but rather subsets of it that
- * can be efficiently executed and make sense in the context of the scheduler datastores. This
- * builder should be preferred over constructing TaskQueries directly.
- * </p>
- *
- * TODO(ksweeney): Add an environment scope.
- */
- public static final class Builder implements Supplier<TaskQuery> {
- private final TaskQuery query;
-
- private Builder() {
- this.query = new TaskQuery();
- }
-
- private Builder(final TaskQuery query) {
- this.query = checkNotNull(query); // It is expected that the caller calls deepCopy.
- }
-
- /**
- * Build a query that is the combination of all the filters applied to a Builder. Mutating the
- * returned object will not affect the state of the builder. Can be called any number of times
- * and will return a new {@code TaskQuery} each time.
- *
- * @return A new TaskQuery satisfying this builder's constraints.
- */
- @Override
- public TaskQuery get() {
- return query.deepCopy();
- }
-
- @Override
- public boolean equals(Object that) {
- return that != null
- && that instanceof Builder
- && Objects.equal(query, ((Builder) that).query);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(query);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("query", query)
- .toString();
- }
-
- /**
- * Create a builder scoped to tasks.
- *
- * @param taskId An ID of a task to scope the builder to.
- * @param taskIds Additional IDs of tasks to scope the builder to (they are ORed together).
- * @return A new Builder scoped to the given tasks.
- */
- public Builder byId(String taskId, String... taskIds) {
- checkNotNull(taskId);
-
- return new Builder(
- query.deepCopy()
- .setTaskIds(ImmutableSet.<String>builder().add(taskId).add(taskIds).build()));
- }
-
- /**
- * Create a builder scoped to tasks.
- *
- * @see #byId(String, String...)
- *
- * @param taskIds The IDs of the tasks to scope the query to (ORed together).
- * @return A new Builder scoped to the given tasks.
- */
- public Builder byId(Iterable<String> taskIds) {
- checkNotNull(taskIds);
-
- return new Builder(
- query.deepCopy().setTaskIds(ImmutableSet.copyOf(taskIds)));
- }
-
- /**
- * Create a builder scoped to a role. A role scope conflicts with job and instance scopes.
- *
- * @param role The role to scope the query to.
- * @return A new Builder scoped to the given role.
- */
- public Builder byRole(String role) {
- checkNotNull(role);
-
- return new Builder(
- query.deepCopy().setOwner(new Identity().setRole(role)));
- }
-
- /**
- * Create a builder scoped to an environment. An environment scope conflicts with role, job,
- * and instance scopes.
- *
- * @param role The role to scope the query to.
- * @param environment The environment to scope the query to.
- * @return A new Builder scoped to the given environment.
- */
- public Builder byEnv(String role, String environment) {
- checkNotNull(role);
- checkNotNull(environment);
-
- return new Builder(
- query.deepCopy()
- .setOwner(new Identity().setRole(role))
- .setEnvironment(environment));
- }
-
- /**
- * Returns a new builder scoped to the job uniquely identified by the given key. A job scope
- * conflicts with role and instance scopes.
- *
- * @param jobKey The key of the job to scope the query to.
- * @return A new Builder scoped to the given jobKey.
- */
- public Builder byJob(IJobKey jobKey) {
- JobKeys.assertValid(jobKey);
-
- return new Builder(
- query.deepCopy()
- .setOwner(new Identity().setRole(jobKey.getRole()))
- .setEnvironment(jobKey.getEnvironment())
- .setJobName(jobKey.getName()));
- }
-
- /**
- * Returns a new builder scoped to the slave uniquely identified by the given slaveHost. A
- * builder can only be scoped to slaves once.
- *
- * @param slaveHost The hostname of the slave to scope the query to.
- * @return A new Builder scoped to the given slave.
- */
- public Builder bySlave(String slaveHost) {
- checkNotNull(slaveHost);
-
- return new Builder(query.deepCopy().setSlaveHost(slaveHost));
- }
-
- /**
- * Returns a new builder scoped to the given statuses. A builder can only be scoped to statuses
- * once.
- *
- * @param status The status to scope this Builder to.
- * @param statuses Additional statuses to scope this Builder to (they are ORed together).
- * @return A new Builder scoped to the given statuses.
- */
- public Builder byStatus(ScheduleStatus status, ScheduleStatus... statuses) {
- checkNotNull(status);
-
- return new Builder(
- query.deepCopy().setStatuses(EnumSet.of(status, statuses)));
- }
-
- /**
- * Create a new Builder scoped to statuses.
- *
- * @see Builder#byStatus(ScheduleStatus, ScheduleStatus...)
- *
- * @param statuses The statuses to scope this Builder to.
- * @return A new Builder scoped to the given statuses.
- */
- public Builder byStatus(Iterable<ScheduleStatus> statuses) {
- checkNotNull(statuses);
-
- return new Builder(
- query.deepCopy().setStatuses(EnumSet.copyOf(ImmutableSet.copyOf(statuses))));
- }
-
- /**
- * Returns a new Builder scoped to the given instances of the given job. A builder can only
- * be scoped to a set of instances, a job, or a role once.
- *
- * @param jobKey The key identifying the job.
- * @param instanceId An instance id of the target job.
- * @param instanceIds Additional instance ids of the target job.
- * @return A new Builder scoped to the given instance ids.
- */
- public Builder byInstances(IJobKey jobKey, int instanceId, int... instanceIds) {
- JobKeys.assertValid(jobKey);
-
- return new Builder(
- query.deepCopy()
- .setOwner(new Identity().setRole(jobKey.getRole()))
- .setEnvironment(jobKey.getEnvironment())
- .setJobName(jobKey.getName())
- .setInstanceIds(ImmutableSet.<Integer>builder()
- .add(instanceId)
- .addAll(Ints.asList(instanceIds))
- .build()));
- }
-
- /**
- * Create a new Builder scoped to instances.
- *
- * @see Builder#byInstances
- *
- * @param jobKey The key identifying the job.
- * @param instanceIds Instances of the target job.
- * @return A new Builder scoped to the given instance ids.
- */
- public Builder byInstances(IJobKey jobKey, Iterable<Integer> instanceIds) {
- JobKeys.assertValid(jobKey);
- checkNotNull(instanceIds);
-
- return new Builder(
- query.deepCopy()
- .setOwner(new Identity().setRole(jobKey.getRole()))
- .setEnvironment(jobKey.getEnvironment())
- .setJobName(jobKey.getName())
- .setInstanceIds(ImmutableSet.copyOf(instanceIds)));
- }
-
- /**
- * A convenience method to scope this builder to {@link Tasks#ACTIVE_STATES}.
- *
- * @return A new Builder scoped to Tasks#ACTIVE_STATES.
- */
- public Builder active() {
- return byStatus(Tasks.ACTIVE_STATES);
- }
-
- /**
- * A convenience method to scope this builder to {@link Tasks#TERMINAL_STATES}.
- *
- * @return A new Builder scoped to Tasks#TERMINAL_STATES.
- */
- public Builder terminal() {
- return byStatus(Tasks.TERMINAL_STATES);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/ScheduleException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/ScheduleException.java b/src/main/java/com/twitter/aurora/scheduler/base/ScheduleException.java
deleted file mode 100644
index 0420ee9..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/ScheduleException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-/**
- * Exception class to signal a failure to schedule a task or job.
- */
-public class ScheduleException extends Exception {
- public ScheduleException(String msg) {
- super(msg);
- }
-
- public ScheduleException(String msg, Throwable t) {
- super(msg, t);
- }
-
- public ScheduleException(Throwable t) {
- super(t);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/SchedulerException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/SchedulerException.java b/src/main/java/com/twitter/aurora/scheduler/base/SchedulerException.java
deleted file mode 100644
index a51c4e0..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/SchedulerException.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-/**
- * Indicates some form of unexpected scheduler exception.
- */
-public class SchedulerException extends RuntimeException {
- public SchedulerException(String message) {
- super(message);
- }
- public SchedulerException(String message, Throwable cause) {
- super(message, cause);
- }
- public SchedulerException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java b/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
deleted file mode 100644
index d98da3f..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/base/Tasks.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.base;
-
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.base.Function;
-import com.google.common.base.Functions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-import com.google.common.collect.Ordering;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.ScheduledTask;
-import com.twitter.aurora.gen.apiConstants;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * Utility class providing convenience functions relating to tasks.
- */
-public final class Tasks {
-
- public static final Function<IScheduledTask, IAssignedTask> SCHEDULED_TO_ASSIGNED =
- new Function<IScheduledTask, IAssignedTask>() {
- @Override public IAssignedTask apply(IScheduledTask task) {
- return task.getAssignedTask();
- }
- };
-
- public static final Function<IAssignedTask, ITaskConfig> ASSIGNED_TO_INFO =
- new Function<IAssignedTask, ITaskConfig>() {
- @Override public ITaskConfig apply(IAssignedTask task) {
- return task.getTask();
- }
- };
-
- public static final Function<IScheduledTask, ITaskConfig> SCHEDULED_TO_INFO =
- Functions.compose(ASSIGNED_TO_INFO, SCHEDULED_TO_ASSIGNED);
-
- public static final Function<IAssignedTask, String> ASSIGNED_TO_ID =
- new Function<IAssignedTask, String>() {
- @Override public String apply(IAssignedTask task) {
- return task.getTaskId();
- }
- };
-
- public static final Function<IScheduledTask, String> SCHEDULED_TO_ID =
- Functions.compose(ASSIGNED_TO_ID, SCHEDULED_TO_ASSIGNED);
-
- public static final Function<IAssignedTask, Integer> ASSIGNED_TO_INSTANCE_ID =
- new Function<IAssignedTask, Integer>() {
- @Override public Integer apply(IAssignedTask task) {
- return task.getInstanceId();
- }
- };
-
- public static final Function<IScheduledTask, Integer> SCHEDULED_TO_INSTANCE_ID =
- Functions.compose(ASSIGNED_TO_INSTANCE_ID, SCHEDULED_TO_ASSIGNED);
-
- public static final Function<ITaskConfig, IJobKey> INFO_TO_JOB_KEY =
- new Function<ITaskConfig, IJobKey>() {
- @Override public IJobKey apply(ITaskConfig task) {
- return JobKeys.from(task);
- }
- };
-
- public static final Function<IAssignedTask, IJobKey> ASSIGNED_TO_JOB_KEY =
- Functions.compose(INFO_TO_JOB_KEY, ASSIGNED_TO_INFO);
-
- public static final Function<IScheduledTask, IJobKey> SCHEDULED_TO_JOB_KEY =
- Functions.compose(ASSIGNED_TO_JOB_KEY, SCHEDULED_TO_ASSIGNED);
-
- /**
- * Different states that an active task may be in.
- */
- public static final EnumSet<ScheduleStatus> ACTIVE_STATES =
- EnumSet.copyOf(apiConstants.ACTIVE_STATES);
-
- /**
- * Terminal states, which a task should not move from.
- */
- public static final Set<ScheduleStatus> TERMINAL_STATES =
- EnumSet.copyOf(apiConstants.TERMINAL_STATES);
-
- public static final Predicate<ITaskConfig> IS_PRODUCTION =
- new Predicate<ITaskConfig>() {
- @Override public boolean apply(ITaskConfig task) {
- return task.isProduction();
- }
- };
-
- public static final Function<IScheduledTask, ScheduleStatus> GET_STATUS =
- new Function<IScheduledTask, ScheduleStatus>() {
- @Override public ScheduleStatus apply(IScheduledTask task) {
- return task.getStatus();
- }
- };
-
- /**
- * Order by production flag (true, then false), subsorting by task ID.
- */
- public static final Ordering<IAssignedTask> SCHEDULING_ORDER =
- Ordering.explicit(true, false)
- .onResultOf(Functions.compose(Functions.forPredicate(IS_PRODUCTION), ASSIGNED_TO_INFO))
- .compound(Ordering.natural().onResultOf(ASSIGNED_TO_ID));
-
- private Tasks() {
- // Utility class.
- }
-
- /**
- * A utility method that returns a multi-map of tasks keyed by IJobKey.
- * @param tasks A list of tasks to be keyed by map
- * @return A multi-map of tasks keyed by job key.
- */
- public static Multimap<IJobKey, IScheduledTask> byJobKey(Iterable<IScheduledTask> tasks) {
- return Multimaps.index(tasks, Tasks.SCHEDULED_TO_JOB_KEY);
- }
-
- public static boolean isActive(ScheduleStatus status) {
- return ACTIVE_STATES.contains(status);
- }
-
- public static boolean isTerminated(ScheduleStatus status) {
- return TERMINAL_STATES.contains(status);
- }
-
- public static String id(IScheduledTask task) {
- return task.getAssignedTask().getTaskId();
- }
-
- // TODO(William Farner: Remove this once the code base is switched to IScheduledTask.
- public static String id(ScheduledTask task) {
- return task.getAssignedTask().getTaskId();
- }
-
- public static Set<String> ids(Iterable<IScheduledTask> tasks) {
- return ImmutableSet.copyOf(Iterables.transform(tasks, SCHEDULED_TO_ID));
- }
-
- public static Set<String> ids(IScheduledTask... tasks) {
- return ids(ImmutableList.copyOf(tasks));
- }
-
- public static Map<String, IScheduledTask> mapById(Iterable<IScheduledTask> tasks) {
- return Maps.uniqueIndex(tasks, SCHEDULED_TO_ID);
- }
-
- public static String getRole(IScheduledTask task) {
- return task.getAssignedTask().getTask().getOwner().getRole();
- }
-
- public static String getJob(IScheduledTask task) {
- return task.getAssignedTask().getTask().getJobName();
- }
-
- public static final Ordering<IScheduledTask> LATEST_ACTIVITY = Ordering.natural()
- .onResultOf(new Function<IScheduledTask, Long>() {
- @Override public Long apply(IScheduledTask task) {
- return Iterables.getLast(task.getTaskEvents()).getTimestamp();
- }
- });
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
deleted file mode 100644
index 4839d0f..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/configuration/ConfigurationManager.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.configuration;
-
-import java.util.regex.Pattern;
-
-import javax.annotation.Nullable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.twitter.aurora.gen.Constraint;
-import com.twitter.aurora.gen.JobConfiguration;
-import com.twitter.aurora.gen.LimitConstraint;
-import com.twitter.aurora.gen.TaskConfig;
-import com.twitter.aurora.gen.TaskConfig._Fields;
-import com.twitter.aurora.gen.TaskConstraint;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.storage.entities.IConstraint;
-import com.twitter.aurora.scheduler.storage.entities.IIdentity;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConstraint;
-import com.twitter.aurora.scheduler.storage.entities.IValueConstraint;
-import com.twitter.common.base.Closure;
-import com.twitter.common.base.MorePreconditions;
-
-import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
-import static com.twitter.aurora.gen.apiConstants.GOOD_IDENTIFIER_PATTERN_JVM;
-
-/**
- * Manages translation from a string-mapped configuration to a concrete configuration type, and
- * defaults for optional values.
- *
- * TODO(William Farner): Add input validation to all fields (strings not empty, positive ints, etc).
- */
-public final class ConfigurationManager {
-
- public static final String DEDICATED_ATTRIBUTE = "dedicated";
-
- @VisibleForTesting public static final String HOST_CONSTRAINT = "host";
- @VisibleForTesting public static final String RACK_CONSTRAINT = "rack";
-
- private static final Pattern GOOD_IDENTIFIER = Pattern.compile(GOOD_IDENTIFIER_PATTERN_JVM);
-
- private static final int MAX_IDENTIFIER_LENGTH = 255;
-
- private static class DefaultField implements Closure<TaskConfig> {
- private final _Fields field;
- private final Object defaultValue;
-
- DefaultField(_Fields field, Object defaultValue) {
- this.field = field;
- this.defaultValue = defaultValue;
- }
-
- @Override public void execute(TaskConfig task) {
- if (!task.isSet(field)) {
- task.setFieldValue(field, defaultValue);
- }
- }
- }
-
- private interface Validator<T> {
- void validate(T value) throws TaskDescriptionException;
- }
-
- private static class GreaterThan implements Validator<Number> {
- private final double min;
- private final String label;
-
- GreaterThan(double min, String label) {
- this.min = min;
- this.label = label;
- }
-
- @Override public void validate(Number value) throws TaskDescriptionException {
- if (this.min >= value.doubleValue()) {
- throw new TaskDescriptionException(label + " must be greater than " + this.min);
- }
- }
- }
-
- private static class RequiredFieldValidator<T> implements Validator<TaskConfig> {
- private final _Fields field;
- private final Validator<T> validator;
-
- RequiredFieldValidator(_Fields field, Validator<T> validator) {
- this.field = field;
- this.validator = validator;
- }
-
- public void validate(TaskConfig task) throws TaskDescriptionException {
- if (!task.isSet(field)) {
- throw new TaskDescriptionException("Field " + field.getFieldName() + " is required.");
- }
- @SuppressWarnings("unchecked")
- T value = (T) task.getFieldValue(field);
- validator.validate(value);
- }
- }
-
- private static final Iterable<Closure<TaskConfig>> DEFAULT_FIELD_POPULATORS =
- ImmutableList.of(
- new DefaultField(_Fields.IS_SERVICE, false),
- new DefaultField(_Fields.PRIORITY, 0),
- new DefaultField(_Fields.PRODUCTION, false),
- new DefaultField(_Fields.MAX_TASK_FAILURES, 1),
- new DefaultField(_Fields.TASK_LINKS, Maps.<String, String>newHashMap()),
- new DefaultField(_Fields.REQUESTED_PORTS, Sets.<String>newHashSet()),
- new DefaultField(_Fields.CONSTRAINTS, Sets.<Constraint>newHashSet()),
- new DefaultField(_Fields.ENVIRONMENT, DEFAULT_ENVIRONMENT),
- new Closure<TaskConfig>() {
- @Override public void execute(TaskConfig task) {
- if (!Iterables.any(task.getConstraints(), hasName(HOST_CONSTRAINT))) {
- task.addToConstraints(hostLimitConstraint(1));
- }
- }
- },
- new Closure<TaskConfig>() {
- @Override public void execute(TaskConfig task) {
- if (!isDedicated(ITaskConfig.build(task))
- && task.isProduction()
- && task.isIsService()
- && !Iterables.any(task.getConstraints(), hasName(RACK_CONSTRAINT))) {
-
- task.addToConstraints(rackLimitConstraint(1));
- }
- }
- });
-
- private static final Iterable<RequiredFieldValidator<?>> REQUIRED_FIELDS_VALIDATORS =
- ImmutableList.<RequiredFieldValidator<?>>of(
- new RequiredFieldValidator<>(_Fields.NUM_CPUS, new GreaterThan(0.0, "num_cpus")),
- new RequiredFieldValidator<>(_Fields.RAM_MB, new GreaterThan(0.0, "ram_mb")),
- new RequiredFieldValidator<>(_Fields.DISK_MB, new GreaterThan(0.0, "disk_mb")));
-
- private ConfigurationManager() {
- // Utility class.
- }
-
- @VisibleForTesting
- static boolean isGoodIdentifier(String identifier) {
- return GOOD_IDENTIFIER.matcher(identifier).matches()
- && (identifier.length() <= MAX_IDENTIFIER_LENGTH);
- }
-
- private static void checkNotNull(Object value, String error) throws TaskDescriptionException {
- if (value == null) {
- throw new TaskDescriptionException(error);
- }
- }
-
- private static void assertOwnerValidity(IIdentity jobOwner) throws TaskDescriptionException {
- checkNotNull(jobOwner, "No job owner specified!");
- checkNotNull(jobOwner.getRole(), "No job role specified!");
- checkNotNull(jobOwner.getUser(), "No job user specified!");
-
- if (!isGoodIdentifier(jobOwner.getRole())) {
- throw new TaskDescriptionException(
- "Job role contains illegal characters: " + jobOwner.getRole());
- }
-
- if (!isGoodIdentifier(jobOwner.getUser())) {
- throw new TaskDescriptionException(
- "Job user contains illegal characters: " + jobOwner.getUser());
- }
- }
-
- private static String getRole(IValueConstraint constraint) {
- return Iterables.getOnlyElement(constraint.getValues()).split("/")[0];
- }
-
- private static boolean isValueConstraint(ITaskConstraint taskConstraint) {
- return taskConstraint.getSetField() == TaskConstraint._Fields.VALUE;
- }
-
- public static boolean isDedicated(ITaskConfig task) {
- return Iterables.any(task.getConstraints(), getConstraintByName(DEDICATED_ATTRIBUTE));
- }
-
- @Nullable
- private static IConstraint getDedicatedConstraint(ITaskConfig task) {
- return Iterables.find(task.getConstraints(), getConstraintByName(DEDICATED_ATTRIBUTE), null);
- }
-
- /**
- * Check validity of and populates defaults in a job configuration. This will return a deep copy
- * of the provided job configuration with default configuration values applied, and configuration
- * map values sanitized and applied to their respective struct fields.
- *
- * @param job Job to validate and populate.
- * @return A deep copy of {@code job} that has been populated.
- * @throws TaskDescriptionException If the job configuration is invalid.
- */
- public static IJobConfiguration validateAndPopulate(IJobConfiguration job)
- throws TaskDescriptionException {
-
- Preconditions.checkNotNull(job);
-
- if (!job.isSetTaskConfig()) {
- throw new TaskDescriptionException("Job configuration must have taskConfig set.");
- }
-
- if (!job.isSetInstanceCount()) {
- throw new TaskDescriptionException("Job configuration does not have shardCount set.");
- }
-
- if (job.getInstanceCount() <= 0) {
- throw new TaskDescriptionException("Shard count must be positive.");
- }
-
- JobConfiguration builder = job.newBuilder();
-
- assertOwnerValidity(job.getOwner());
-
- if (!JobKeys.isValid(job.getKey())) {
- throw new TaskDescriptionException("Job key " + job.getKey() + " is invalid.");
- }
- if (!job.getKey().getRole().equals(job.getOwner().getRole())) {
- throw new TaskDescriptionException("Role in job key must match job owner.");
- }
- if (!isGoodIdentifier(job.getKey().getRole())) {
- throw new TaskDescriptionException(
- "Job role contains illegal characters: " + job.getKey().getRole());
- }
- if (!isGoodIdentifier(job.getKey().getEnvironment())) {
- throw new TaskDescriptionException(
- "Job environment contains illegal characters: " + job.getKey().getEnvironment());
- }
- if (!isGoodIdentifier(job.getKey().getName())) {
- throw new TaskDescriptionException(
- "Job name contains illegal characters: " + job.getKey().getName());
- }
-
- builder.setTaskConfig(
- validateAndPopulate(ITaskConfig.build(builder.getTaskConfig())).newBuilder());
-
- // Only one of [service=true, cron_schedule] may be set.
- if (!StringUtils.isEmpty(job.getCronSchedule()) && builder.getTaskConfig().isIsService()) {
- throw new TaskDescriptionException(
- "A service task may not be run on a cron schedule: " + builder);
- }
-
- return IJobConfiguration.build(builder);
- }
-
- /**
- * Check validity of and populates defaults in a task configuration. This will return a deep copy
- * of the provided task configuration with default configuration values applied, and configuration
- * map values sanitized and applied to their respective struct fields.
- *
- *
- * @param config Task config to validate and populate.
- * @return A reference to the modified {@code config} (for chaining).
- * @throws TaskDescriptionException If the task is invalid.
- */
- public static ITaskConfig validateAndPopulate(ITaskConfig config)
- throws TaskDescriptionException {
-
- TaskConfig builder = config.newBuilder();
-
- if (!builder.isSetRequestedPorts()) {
- builder.setRequestedPorts(ImmutableSet.<String>of());
- }
-
- maybeFillLinks(builder);
-
- assertOwnerValidity(config.getOwner());
-
- if (!isGoodIdentifier(config.getJobName())) {
- throw new TaskDescriptionException(
- "Job name contains illegal characters: " + config.getJobName());
- }
-
- if (!isGoodIdentifier(config.getEnvironment())) {
- throw new TaskDescriptionException(
- "Environment contains illegal characters: " + config.getEnvironment());
- }
-
- if (!builder.isSetExecutorConfig()) {
- throw new TaskDescriptionException("Configuration may not be null");
- }
-
- // Maximize the usefulness of any thrown error message by checking required fields first.
- for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
- validator.validate(builder);
- }
-
- IConstraint constraint = getDedicatedConstraint(config);
- if (constraint != null) {
- if (!isValueConstraint(constraint.getConstraint())) {
- throw new TaskDescriptionException("A dedicated constraint must be of value type.");
- }
-
- IValueConstraint valueConstraint = constraint.getConstraint().getValue();
-
- if (!(valueConstraint.getValues().size() == 1)) {
- throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
- }
-
- String dedicatedRole = getRole(valueConstraint);
- if (!config.getOwner().getRole().equals(dedicatedRole)) {
- throw new TaskDescriptionException(
- "Only " + dedicatedRole + " may use hosts dedicated for that role.");
- }
- }
-
- return ITaskConfig.build(applyDefaultsIfUnset(builder));
- }
-
- /**
- * Provides a filter for the given constraint name.
- *
- * @param name The name of the constraint.
- * @return A filter that matches the constraint.
- */
- public static Predicate<IConstraint> getConstraintByName(final String name) {
- return new Predicate<IConstraint>() {
- @Override public boolean apply(IConstraint constraint) {
- return constraint.getName().equals(name);
- }
- };
- }
-
- @VisibleForTesting
- public static Constraint hostLimitConstraint(int limit) {
- return new Constraint(HOST_CONSTRAINT, TaskConstraint.limit(new LimitConstraint(limit)));
- }
-
- @VisibleForTesting
- public static Constraint rackLimitConstraint(int limit) {
- return new Constraint(RACK_CONSTRAINT, TaskConstraint.limit(new LimitConstraint(limit)));
- }
-
- private static Predicate<Constraint> hasName(final String name) {
- MorePreconditions.checkNotBlank(name);
- return new Predicate<Constraint>() {
- @Override public boolean apply(Constraint constraint) {
- return name.equals(constraint.getName());
- }
- };
- }
-
- /**
- * Applies defaults to unset values in a task.
- *
- * @param task Task to apply defaults to.
- * @return A reference to the (modified) {@code task}.
- */
- @VisibleForTesting
- public static TaskConfig applyDefaultsIfUnset(TaskConfig task) {
- for (Closure<TaskConfig> populator : DEFAULT_FIELD_POPULATORS) {
- populator.execute(task);
- }
-
- return task;
- }
-
- /**
- * Applies defaults to unset values in a job and its tasks.
- *
- * @param job Job to apply defaults to.
- */
- @VisibleForTesting
- public static void applyDefaultsIfUnset(JobConfiguration job) {
- ConfigurationManager.applyDefaultsIfUnset(job.getTaskConfig());
- }
-
- private static void maybeFillLinks(TaskConfig task) {
- if (task.getTaskLinksSize() == 0) {
- ImmutableMap.Builder<String, String> links = ImmutableMap.builder();
- if (task.getRequestedPorts().contains("health")) {
- links.put("health", "http://%host%:%port:health%");
- }
- if (task.getRequestedPorts().contains("http")) {
- links.put("http", "http://%host%:%port:http%");
- }
- task.setTaskLinks(links.build());
- }
- }
-
- /**
- * Thrown when an invalid task or job configuration is encountered.
- */
- public static class TaskDescriptionException extends Exception {
- public TaskDescriptionException(String msg) {
- super(msg);
- }
- }
-}
[30/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/Storage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/Storage.java b/src/main/java/org/apache/aurora/scheduler/storage/Storage.java
new file mode 100644
index 0000000..fdc9ae7
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/Storage.java
@@ -0,0 +1,322 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.BindingAnnotation;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.SchedulerException;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+
+/**
+ * Manages scheduler storage operations providing an interface to perform atomic changes.
+ */
+public interface Storage {
+
+ interface StoreProvider {
+ SchedulerStore getSchedulerStore();
+ JobStore getJobStore();
+ TaskStore getTaskStore();
+ LockStore getLockStore();
+ QuotaStore getQuotaStore();
+ AttributeStore getAttributeStore();
+ }
+
+ interface MutableStoreProvider extends StoreProvider {
+ SchedulerStore.Mutable getSchedulerStore();
+ JobStore.Mutable getJobStore();
+
+ /**
+ * Gets access to the mutable task store.
+ * <p>
+ * This is labeled as unsafe, since it's rare that a caller should be using this. In most
+ * cases, mutations to the task store should be done through
+ * {@link com.twitter.aurora.scheduler.state.StateManager}.
+ * <p>
+ * TODO(William Farner): Come up with a way to restrict access to this interface. As it stands,
+ * it's trivial for an unsuspecting caller to modify the task store directly and subvert the
+ * state machine and side effect systems.
+ *
+ * @return The mutable task store.
+ */
+ TaskStore.Mutable getUnsafeTaskStore();
+
+ LockStore.Mutable getLockStore();
+ QuotaStore.Mutable getQuotaStore();
+ AttributeStore.Mutable getAttributeStore();
+ }
+
+ /**
+ * Encapsulates a read-only storage operation.
+ *
+ * @param <T> The type of result this unit of work produces.
+ * @param <E> The type of exception this unit of work can throw.
+ */
+ interface Work<T, E extends Exception> {
+
+ /**
+ * Abstracts a unit of work that has a result, but may also throw a specific exception.
+ *
+ * @param storeProvider A provider to give access to different available stores.
+ * @return the result of the successfully completed unit of work
+ * @throws E if the unit of work could not be completed
+ */
+ T apply(StoreProvider storeProvider) throws E;
+
+ /**
+ * A convenient typedef for Work that throws no checked exceptions - it runs quietly.
+ *
+ * @param <T> The type of result this unit of work produces.
+ */
+ interface Quiet<T> extends Work<T, RuntimeException> {
+ // typedef
+ }
+ }
+
+ /**
+ * Encapsulates a storage operation, which has mutable storage access.
+ *
+ * @param <T> The type of result this unit of work produces.
+ * @param <E> The type of exception this unit of work can throw.
+ */
+ interface MutateWork<T, E extends Exception> {
+
+ NoResult.Quiet NOOP = new NoResult.Quiet() {
+ @Override protected void execute(Storage.MutableStoreProvider storeProvider) {
+ // No-op.
+ }
+ };
+
+ /**
+ * Abstracts a unit of work that should either commit a set of changes to storage as a side
+ * effect of successful completion or else commit no changes at all when an exception is thrown.
+ *
+ * @param storeProvider A provider to give access to different available stores.
+ * @return the result of the successfully completed unit of work
+ * @throws E if the unit of work could not be completed
+ */
+ T apply(MutableStoreProvider storeProvider) throws E;
+
+ /**
+ * A convenient typedef for Work that throws no checked exceptions - it runs quietly.
+ *
+ * @param <T> The type of result this unit of work produces.
+ */
+ interface Quiet<T> extends MutateWork<T, RuntimeException> {
+ // typedef
+ }
+
+ /**
+ * Encapsulates work that returns no result.
+ *
+ * @param <E> The type of exception this unit of work can throw.
+ */
+ abstract class NoResult<E extends Exception> implements MutateWork<Void, E> {
+
+ @Override public final Void apply(MutableStoreProvider storeProvider) throws E {
+ execute(storeProvider);
+ return null;
+ }
+
+ /**
+ * Similar to {@link #apply(MutableStoreProvider)} except that no result is
+ * returned.
+ *
+ * @param storeProvider A provider to give access to different available stores.
+ * @throws E if the unit of work could not be completed
+ */
+ protected abstract void execute(MutableStoreProvider storeProvider) throws E;
+
+ /**
+ * A convenient typedef for Work with no result that throws no checked exceptions - it runs
+ * quitely.
+ */
+ public abstract static class Quiet extends NoResult<RuntimeException> {
+ // typedef
+ }
+ }
+ }
+
+ /**
+ * Indicates a problem reading from or writing to stable storage.
+ */
+ class StorageException extends SchedulerException {
+ public StorageException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public StorageException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Executes the unit of read-only {@code work}. All data in the stores may be expected to be
+ * consistent, as the invocation is mutually exclusive of any writes.
+ *
+ * @param work The unit of work to execute.
+ * @param <T> The type of result this unit of work produces.
+ * @param <E> The type of exception this unit of work can throw.
+ * @return the result when the unit of work completes successfully
+ * @throws StorageException if there was a problem reading from stable storage.
+ * @throws E bubbled transparently when the unit of work throws
+ */
+ <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E;
+
+ /**
+ * Executes a unit of read-only {@code work}. This is functionally identical to
+ * {@link #consistentRead(Work)} with the exception that data in the stores may not be fully
+ * consistent.
+ *
+ * @param work The unit of work to execute.
+ * @param <T> The type of result this unit of work produces.
+ * @param <E> The type of exception this unit of work can throw.
+ * @return the result when the unit of work completes successfully
+ * @throws StorageException if there was a problem reading from stable storage.
+ * @throws E bubbled transparently when the unit of work throws
+ */
+ <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work) throws StorageException, E;
+
+ /**
+ * Executes the unit of mutating {@code work}.
+ *
+ * @param work The unit of work to execute.
+ * @param <T> The type of result this unit of work produces.
+ * @param <E> The type of exception this unit of work can throw.
+ * @return the result when the unit of work completes successfully
+ * @throws StorageException if there was a problem reading from or writing to stable storage.
+ * @throws E bubbled transparently when the unit of work throws
+ */
+ <T, E extends Exception> T write(MutateWork<T, E> work) throws StorageException, E;
+
+ /**
+ * Clean up the underlying storage by optimizing internal data structures. Does not change
+ * externally-visible state but might not run concurrently with write operations.
+ */
+ void snapshot() throws StorageException;
+
+ /**
+ * A non-volatile storage that has additional methods to control its lifecycle.
+ */
+ interface NonVolatileStorage extends Storage {
+ /**
+ * Requests the underlying storage prepare its data set; ie: initialize schemas, begin syncing
+ * out of date data, etc. This method should not block.
+ *
+ * @throws StorageException if there was a problem preparing storage.
+ */
+ void prepare() throws StorageException;
+
+ /**
+ * Prepares the underlying storage for serving traffic.
+ *
+ * @param initializationLogic work to perform after this storage system is ready but before
+ * allowing general use of
+ * {@link #consistentRead}.
+ * @throws StorageException if there was a starting storage.
+ */
+ void start(MutateWork.NoResult.Quiet initializationLogic) throws StorageException;
+
+ /**
+ * Prepares the underlying storage system for clean shutdown.
+ */
+ void stop();
+ }
+
+ /**
+ * Identifies a storage layer that is in-memory only.
+ * This generally should only be used when the storage is first starting up, to perform queries
+ * related to initially load the storage.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ ElementType.PARAMETER, ElementType.METHOD })
+ @BindingAnnotation
+ public @interface Volatile { }
+
+ /**
+ * Utility functions for interacting with a Storage instance.
+ */
+ public final class Util {
+
+ private Util() {
+ // Utility class.
+ }
+
+ /**
+ * Fetch tasks matching the query returned by {@code query} from {@code storage} in a
+ * read operation.
+ *
+ * @see #consistentFetchTasks
+ * @param storage Storage instance to query from.
+ * @param query Builder of the query to perform.
+ * @return Tasks returned from the query.
+ */
+ public static ImmutableSet<IScheduledTask> consistentFetchTasks(
+ Storage storage,
+ final Query.Builder query) {
+
+ return storage.consistentRead(new Work.Quiet<ImmutableSet<IScheduledTask>>() {
+ @Override public ImmutableSet<IScheduledTask> apply(StoreProvider storeProvider) {
+ return storeProvider.getTaskStore().fetchTasks(query);
+ }
+ });
+ }
+
+ /**
+ * Identical to {@link #consistentFetchTasks(Storage, Query.Builder)}, but fetches tasks using a
+ * weakly-consistent read operation.
+ *
+ * @see #consistentFetchTasks
+ * @param storage Storage instance to query from.
+ * @param query Builder of the query to perform.
+ * @return Tasks returned from the query.
+ */
+ public static ImmutableSet<IScheduledTask> weaklyConsistentFetchTasks(
+ Storage storage,
+ final Query.Builder query) {
+
+ return storage.weaklyConsistentRead(new Work.Quiet<ImmutableSet<IScheduledTask>>() {
+ @Override public ImmutableSet<IScheduledTask> apply(StoreProvider storeProvider) {
+ return storeProvider.getTaskStore().fetchTasks(query);
+ }
+ });
+ }
+
+ /**
+ * Fetch quota for {@code role} from {@code storage} in a consistent read operation.
+ *
+ * @param storage Storage instance to fetch quota from.
+ * @param role Role to fetch quota for.
+ * @return Quota returned from the fetch operation.
+ * @see QuotaStore#fetchQuota(String)
+ */
+ public static Optional<IQuota> consistentFetchQuota(Storage storage, final String role) {
+ return storage.consistentRead(new Work.Quiet<Optional<IQuota>>() {
+ @Override public Optional<IQuota> apply(StoreProvider storeProvider) {
+ return storeProvider.getQuotaStore().fetchQuota(role);
+ }
+ });
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java b/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java
new file mode 100644
index 0000000..df5b603
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+import com.twitter.aurora.gen.JobConfiguration;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.ScheduledTask;
+import com.twitter.aurora.gen.TaskConfig;
+import com.twitter.aurora.gen.TaskEvent;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.TaskStore.Mutable.TaskMutation;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+
+/**
+ * Utility class to contain and perform storage backfill operations.
+ */
+public final class StorageBackfill {
+
+ private static final Logger LOG = Logger.getLogger(StorageBackfill.class.getName());
+
+ private static final AtomicLong SHARD_SANITY_CHECK_FAILS =
+ Stats.exportLong("shard_sanity_check_failures");
+
+ private StorageBackfill() {
+ // Utility class.
+ }
+
+ private static void backfillJobDefaults(JobStore.Mutable jobStore) {
+ for (String id : jobStore.fetchManagerIds()) {
+ for (JobConfiguration job : IJobConfiguration.toBuildersList(jobStore.fetchJobs(id))) {
+ ConfigurationManager.applyDefaultsIfUnset(job);
+ jobStore.saveAcceptedJob(id, IJobConfiguration.build(job));
+ }
+ }
+ }
+
+ private static void guaranteeShardUniqueness(
+ ScheduledTask task,
+ TaskStore.Mutable taskStore,
+ Clock clock) {
+
+ if (Tasks.isActive(task.getStatus())) {
+ // Perform a sanity check on the number of active shards.
+ TaskConfig config = task.getAssignedTask().getTask();
+ Query.Builder query = Query.instanceScoped(
+ JobKeys.from(config.getOwner().getRole(), config.getEnvironment(), config.getJobName()),
+ task.getAssignedTask().getInstanceId())
+ .active();
+ Set<String> activeTasksInShard = FluentIterable.from(taskStore.fetchTasks(query))
+ .transform(Tasks.SCHEDULED_TO_ID)
+ .toSet();
+
+ if (activeTasksInShard.size() > 1) {
+ SHARD_SANITY_CHECK_FAILS.incrementAndGet();
+ LOG.severe("Active shard sanity check failed when loading " + Tasks.id(task)
+ + ", active tasks found: " + activeTasksInShard);
+
+ // We want to keep exactly one task from this shard, so sort the IDs and keep the
+ // highest (newest) in the hopes that it is legitimately running.
+ String newestTask = Iterables.getLast(Sets.newTreeSet(activeTasksInShard));
+ if (!Tasks.id(task).equals(newestTask)) {
+ task.setStatus(ScheduleStatus.KILLED);
+ task.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.KILLED)
+ .setMessage("Killed duplicate shard."));
+ // TODO(wfarner); Circle back if this is necessary. Currently there's a race
+ // condition between the time the scheduler is actually available without hitting
+ // IllegalStateException (see DriverImpl).
+ // driver.killTask(Tasks.id(task));
+ } else {
+ LOG.info("Retaining task " + Tasks.id(task));
+ }
+ }
+ }
+ }
+
+ private static final AtomicLong BOTH_FIELDS_SET = Stats.exportLong("both_instance_ids_set");
+ private static final AtomicLong OLD_FIELD_SET = Stats.exportLong("old_instance_id_set");
+ private static final AtomicLong NEW_FIELD_SET = Stats.exportLong("new_instance_id_set");
+ private static final AtomicLong FIELDS_INCONSISTENT =
+ Stats.exportLong("instance_ids_inconsistent");
+
+ /**
+ * Ensures backwards-compatibility of the throttled state, which exists in this version but is
+ * not handled.
+ *
+ * @param task Task to possibly rewrite.
+ */
+ private static void rewriteThrottledState(ScheduledTask task) {
+ if (ScheduleStatus.THROTTLED == task.getStatus()) {
+ task.setStatus(ScheduleStatus.PENDING);
+ }
+ }
+
+ /**
+ * Backfills the storage to make it match any assumptions that may have changed since
+ * the structs were first written.
+ *
+ * @param storeProvider Storage provider.
+ * @param clock Clock, used for timestamping backfilled task events.
+ */
+ public static void backfill(final MutableStoreProvider storeProvider, final Clock clock) {
+ backfillJobDefaults(storeProvider.getJobStore());
+
+ LOG.info("Performing shard uniqueness sanity check.");
+ storeProvider.getUnsafeTaskStore().mutateTasks(Query.unscoped(), new TaskMutation() {
+ @Override public IScheduledTask apply(final IScheduledTask task) {
+ ScheduledTask builder = task.newBuilder();
+ ConfigurationManager.applyDefaultsIfUnset(builder.getAssignedTask().getTask());
+ // TODO(ksweeney): Guarantee tasks pass current validation code here and quarantine if they
+ // don't.
+ guaranteeShardUniqueness(builder, storeProvider.getUnsafeTaskStore(), clock);
+ rewriteThrottledState(builder);
+ return IScheduledTask.build(builder);
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java
new file mode 100644
index 0000000..02e5096
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/TaskStore.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage;
+
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * Stores all tasks configured with the scheduler.
+ */
+public interface TaskStore {
+
+ /**
+ * Fetches a read-only view of tasks matching a query and filters. Intended for use with a
+ * {@link com.twitter.aurora.scheduler.base.Query.Builder}.
+ *
+ * @param query Builder of the query to identify tasks with.
+ * @return A read-only view of matching tasks.
+ */
+ ImmutableSet<IScheduledTask> fetchTasks(Query.Builder query);
+
+ public interface Mutable extends TaskStore {
+
+ /**
+ * A convenience interface to allow callers to more concisely implement a task mutation.
+ */
+ public interface TaskMutation extends Function<IScheduledTask, IScheduledTask> {
+ }
+
+ /**
+ * Saves tasks to the store. Tasks are copied internally, meaning that the tasks are stored in
+ * the state they were in when the method is called, and further object modifications will not
+ * affect the tasks. If any of the tasks already exist in the store, they will be overwritten
+ * by the supplied {@code newTasks}.
+ *
+ * @param tasks Tasks to add.
+ */
+ void saveTasks(Set<IScheduledTask> tasks);
+
+ /**
+ * Removes all tasks from the store.
+ */
+ void deleteAllTasks();
+
+ /**
+ * Deletes specific tasks from the store.
+ *
+ * @param taskIds IDs of tasks to delete.
+ */
+ void deleteTasks(Set<String> taskIds);
+
+ /**
+ * Offers temporary mutable access to tasks. If a task ID is not found, it will be silently
+ * skipped, and no corresponding task will be returned.
+ *
+ * @param query Query to match tasks against.
+ * @param mutator The mutate operation.
+ * @return Immutable copies of only the tasks that were mutated.
+ */
+ ImmutableSet<IScheduledTask> mutateTasks(
+ Query.Builder query,
+ Function<IScheduledTask, IScheduledTask> mutator);
+
+ /**
+ * Rewrites a task's configuration in-place.
+ * <p>
+ * <b>WARNING</b>: this is a dangerous operation, and should not be used without exercising
+ * great care. This feature should be used as a last-ditch effort to rewrite things that
+ * the scheduler otherwise can't (e.g. {@link ITaskConfig#executorConfig}) rewrite in a
+ * controlled/tested backfill operation.
+ *
+ * @param taskId ID of the task to alter.
+ * @param taskConfiguration Configuration object to swap with the existing task's
+ * configuration.
+ * @return {@code true} if the modification took effect, or {@code false} if the task does not
+ * exist in the store.
+ */
+ boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java b/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java
new file mode 100644
index 0000000..b6beba3
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/backup/BackupModule.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.backup;
+
+import java.io.File;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.inject.PrivateModule;
+import com.google.inject.Provides;
+import com.google.inject.TypeLiteral;
+
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.scheduler.storage.SnapshotStore;
+import com.twitter.aurora.scheduler.storage.backup.Recovery.RecoveryImpl;
+import com.twitter.aurora.scheduler.storage.backup.StorageBackup.StorageBackupImpl;
+import com.twitter.aurora.scheduler.storage.backup.StorageBackup.StorageBackupImpl.BackupConfig;
+import com.twitter.aurora.scheduler.storage.backup.TemporaryStorage.TemporaryStorageFactory;
+import com.twitter.common.application.Lifecycle;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.args.constraints.NotNull;
+import com.twitter.common.base.Command;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A module that will periodically save full storage backups to local disk and makes those backups
+ * available for on-line recovery.
+ */
+public class BackupModule extends PrivateModule {
+ private static final Logger LOG = Logger.getLogger(BackupModule.class.getName());
+
+ @CmdLine(name = "backup_interval", help = "Minimum interval on which to write a storage backup.")
+ private static final Arg<Amount<Long, Time>> BACKUP_INTERVAL =
+ Arg.create(Amount.of(1L, Time.HOURS));
+
+ @CmdLine(name = "max_saved_backups",
+ help = "Maximum number of backups to retain before deleting the oldest backups.")
+ private static final Arg<Integer> MAX_SAVED_BACKUPS = Arg.create(48);
+
+ @NotNull
+ @CmdLine(name = "backup_dir",
+ help = "Directory to store backups under. Will be created if it does not exist.")
+ private static final Arg<File> BACKUP_DIR = Arg.create();
+
+ private final Class<? extends SnapshotStore<Snapshot>> snapshotStore;
+ private final File unvalidatedBackupDir;
+
+ /**
+ * Creates a new backup module.
+ *
+ * @param snapshotStore Snapshot store implementation class.
+ */
+ public BackupModule(Class<? extends SnapshotStore<Snapshot>> snapshotStore) {
+ this(BACKUP_DIR.get(), snapshotStore);
+ }
+
+ /**
+ * Creates a new backup module using a given backupDir instead of a flagged one.
+ *
+ * @param backupDir Directory to write backups to.
+ * @param snapshotStore Snapshot store implementation class.
+ */
+ @VisibleForTesting
+ public BackupModule(File backupDir, Class<? extends SnapshotStore<Snapshot>> snapshotStore) {
+ this.unvalidatedBackupDir = checkNotNull(backupDir);
+ this.snapshotStore = checkNotNull(snapshotStore);
+ }
+
+ @Override
+ protected void configure() {
+ TypeLiteral<SnapshotStore<Snapshot>> type = new TypeLiteral<SnapshotStore<Snapshot>>() { };
+ bind(type).annotatedWith(StorageBackupImpl.SnapshotDelegate.class).to(snapshotStore);
+
+ bind(type).to(StorageBackupImpl.class);
+ bind(StorageBackup.class).to(StorageBackupImpl.class);
+ bind(StorageBackupImpl.class).in(Singleton.class);
+ expose(type);
+ expose(StorageBackup.class);
+
+ bind(new TypeLiteral<Function<Snapshot, TemporaryStorage>>() { })
+ .to(TemporaryStorageFactory.class);
+
+ bind(Command.class).to(LifecycleHook.class);
+ bind(Recovery.class).to(RecoveryImpl.class);
+ bind(RecoveryImpl.class).in(Singleton.class);
+ expose(Recovery.class);
+ }
+
+ static class LifecycleHook implements Command {
+ private final Lifecycle lifecycle;
+
+ @Inject LifecycleHook(Lifecycle lifecycle) {
+ this.lifecycle = checkNotNull(lifecycle);
+ }
+
+ @Override public void execute() {
+ lifecycle.shutdown();
+ }
+ }
+
+ @Provides
+ private File provideBackupDir() {
+ if (!unvalidatedBackupDir.exists()) {
+ if (!unvalidatedBackupDir.mkdirs()) {
+ throw new IllegalArgumentException(
+ "Unable to create backup dir " + unvalidatedBackupDir.getPath() + ".");
+ } else {
+ LOG.info("Created backup dir " + unvalidatedBackupDir.getPath() + ".");
+ }
+ }
+
+ if (!unvalidatedBackupDir.canWrite()) {
+ throw new IllegalArgumentException(
+ "Backup dir " + unvalidatedBackupDir.getPath() + " is not writable.");
+ }
+
+ return unvalidatedBackupDir;
+ }
+
+ @Provides
+ private BackupConfig provideBackupConfig(File backupDir) {
+ return new BackupConfig(backupDir, MAX_SAVED_BACKUPS.get(), BACKUP_INTERVAL.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java b/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java
new file mode 100644
index 0000000..4e91342
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.backup;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Files;
+import com.google.common.util.concurrent.Atomics;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.storage.DistributedSnapshotStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.base.Command;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A recovery mechanism that works with {@link StorageBackup} to provide a two-step storage
+ * recovery process.
+ */
+public interface Recovery {
+
+ /**
+ * List backups available for recovery.
+ *
+ * @return Available backup IDs.
+ */
+ Set<String> listBackups();
+
+ /**
+ * Loads a backup in 'staging' so that it may be queried and modified prior to committing.
+ *
+ * @param backupName Name of the backup to load.
+ * @throws RecoveryException If the backup could not be found or loaded.
+ */
+ void stage(String backupName) throws RecoveryException;
+
+ /**
+ * Queries a staged backup.
+ *
+ * @param query Builder of query to perform.
+ * @return Tasks matching the query.
+ * @throws RecoveryException If a backup is not staged, or could not be queried.
+ */
+ Set<IScheduledTask> query(Query.Builder query) throws RecoveryException;
+
+ /**
+ * Deletes tasks from a staged backup.
+ *
+ * @param query Query builder for tasks to delete.
+ * @throws RecoveryException If a backup is not staged, or tasks could not be deleted.
+ */
+ void deleteTasks(Query.Builder query) throws RecoveryException;
+
+ /**
+ * Unloads a staged backup.
+ */
+ void unload();
+
+ /**
+ * Commits a staged backup the main storage system.
+ *
+ * @throws RecoveryException If a backup is not staged, or the commit failed.
+ */
+ void commit() throws RecoveryException;
+
+ /**
+ * Thrown when a recovery operation could not be completed due to internal errors or improper
+ * invocation order.
+ */
+ public static class RecoveryException extends Exception {
+ RecoveryException(String message) {
+ super(message);
+ }
+
+ RecoveryException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
+ class RecoveryImpl implements Recovery {
+ private final File backupDir;
+ private final Function<Snapshot, TemporaryStorage> tempStorageFactory;
+ private final AtomicReference<PendingRecovery> recovery;
+ private final Storage primaryStorage;
+ private final DistributedSnapshotStore distributedStore;
+ private final Command shutDownNow;
+
+ @Inject
+ RecoveryImpl(
+ File backupDir,
+ Function<Snapshot, TemporaryStorage> tempStorageFactory,
+ Storage primaryStorage,
+ DistributedSnapshotStore distributedStore,
+ Command shutDownNow) {
+
+ this.backupDir = checkNotNull(backupDir);
+ this.tempStorageFactory = checkNotNull(tempStorageFactory);
+ this.recovery = Atomics.newReference();
+ this.primaryStorage = checkNotNull(primaryStorage);
+ this.distributedStore = checkNotNull(distributedStore);
+ this.shutDownNow = checkNotNull(shutDownNow);
+ }
+
+ @Override public Set<String> listBackups() {
+ return ImmutableSet.<String>builder().add(backupDir.list()).build();
+ }
+
+ @Override public void stage(String backupName) throws RecoveryException {
+ File backupFile = new File(backupDir, backupName);
+ if (!backupFile.exists()) {
+ throw new RecoveryException("Backup " + backupName + " does not exist.");
+ }
+
+ Snapshot snapshot;
+ try {
+ snapshot = ThriftBinaryCodec.decode(Snapshot.class, Files.toByteArray(backupFile));
+ } catch (CodingException e) {
+ throw new RecoveryException("Failed to decode backup " + e, e);
+ } catch (IOException e) {
+ throw new RecoveryException("Failed to read backup " + e, e);
+ }
+ boolean applied =
+ recovery.compareAndSet(null, new PendingRecovery(tempStorageFactory.apply(snapshot)));
+ if (!applied) {
+ throw new RecoveryException("Another backup is already loaded.");
+ }
+ }
+
+ private PendingRecovery getLoadedRecovery() throws RecoveryException {
+ @Nullable PendingRecovery loaded = this.recovery.get();
+ if (loaded == null) {
+ throw new RecoveryException("No backup loaded.");
+ }
+ return loaded;
+ }
+
+ @Override public Set<IScheduledTask> query(Query.Builder query) throws RecoveryException {
+ return getLoadedRecovery().query(query);
+ }
+
+ @Override public void deleteTasks(Query.Builder query) throws RecoveryException {
+ getLoadedRecovery().delete(query);
+ }
+
+ @Override public void unload() {
+ recovery.set(null);
+ }
+
+ @Override public void commit() throws RecoveryException {
+ getLoadedRecovery().commit();
+ }
+
+ private class PendingRecovery {
+ private final TemporaryStorage tempStorage;
+
+ PendingRecovery(TemporaryStorage tempStorage) {
+ this.tempStorage = tempStorage;
+ }
+
+ void commit() {
+ primaryStorage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ try {
+ distributedStore.persist(tempStorage.toSnapshot());
+ shutDownNow.execute();
+ } catch (CodingException e) {
+ throw new IllegalStateException("Failed to encode snapshot.", e);
+ }
+ }
+ });
+ }
+
+ Set<IScheduledTask> query(final Query.Builder query) {
+ return tempStorage.fetchTasks(query);
+ }
+
+ void delete(final Query.Builder query) {
+ tempStorage.deleteTasks(query);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/backup/StorageBackup.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/backup/StorageBackup.java b/src/main/java/org/apache/aurora/scheduler/storage/backup/StorageBackup.java
new file mode 100644
index 0000000..3faeb1f
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/backup/StorageBackup.java
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.backup;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Ordering;
+import com.google.common.io.Files;
+import com.google.inject.BindingAnnotation;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.scheduler.storage.SnapshotStore;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A backup routine that layers over a snapshot store and periodically writes snapshots to
+ * local disk.
+ *
+ * TODO(William Farner): Perform backups asynchronously. As written, they are performed in a
+ * blocking write operation, which is asking for trouble.
+ */
+public interface StorageBackup {
+
+ /**
+ * Perform a storage backup immediately, blocking until it is complete.
+ */
+ void backupNow();
+
+ class StorageBackupImpl implements StorageBackup, SnapshotStore<Snapshot> {
+ private static final Logger LOG = Logger.getLogger(StorageBackup.class.getName());
+
+ private static final String FILE_PREFIX = "scheduler-backup-";
+ private final BackupConfig config;
+
+ static class BackupConfig {
+ private final File dir;
+ private final int maxBackups;
+ private final Amount<Long, Time> interval;
+
+ BackupConfig(File dir, int maxBackups, Amount<Long, Time> interval) {
+ this.dir = checkNotNull(dir);
+ this.maxBackups = maxBackups;
+ this.interval = checkNotNull(interval);
+ }
+
+ @VisibleForTesting
+ File getDir() {
+ return dir;
+ }
+ }
+
+ /**
+ * Binding annotation that the underlying {@link SnapshotStore} must be bound with.
+ */
+ @BindingAnnotation
+ @Target({FIELD, PARAMETER, METHOD}) @Retention(RUNTIME)
+ @interface SnapshotDelegate { }
+
+ private final SnapshotStore<Snapshot> delegate;
+ private final Clock clock;
+ private final long backupIntervalMs;
+ private volatile long lastBackupMs;
+ private final DateFormat backupDateFormat;
+
+ private final AtomicLong successes = Stats.exportLong("scheduler_backup_success");
+ @VisibleForTesting
+ AtomicLong getSuccesses() {
+ return successes;
+ }
+
+ private final AtomicLong failures = Stats.exportLong("scheduler_backup_failed");
+ @VisibleForTesting
+ AtomicLong getFailures() {
+ return failures;
+ }
+
+ @Inject
+ StorageBackupImpl(
+ @SnapshotDelegate SnapshotStore<Snapshot> delegate,
+ Clock clock,
+ BackupConfig config) {
+
+ this.delegate = checkNotNull(delegate);
+ this.clock = checkNotNull(clock);
+ this.config = checkNotNull(config);
+ backupDateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
+ backupIntervalMs = config.interval.as(Time.MILLISECONDS);
+ lastBackupMs = clock.nowMillis();
+ }
+
+ @Override public Snapshot createSnapshot() {
+ Snapshot snapshot = delegate.createSnapshot();
+ if (clock.nowMillis() >= (lastBackupMs + backupIntervalMs)) {
+ save(snapshot);
+ }
+ return snapshot;
+ }
+
+ @Override public void backupNow() {
+ save(delegate.createSnapshot());
+ }
+
+ @VisibleForTesting
+ String createBackupName() {
+ return FILE_PREFIX + backupDateFormat.format(new Date(clock.nowMillis()));
+ }
+
+ private void save(Snapshot snapshot) {
+ lastBackupMs = clock.nowMillis();
+
+ String backupName = createBackupName();
+ String tempBackupName = "temp_" + backupName;
+ File tempFile = new File(config.dir, tempBackupName);
+ LOG.info("Saving backup to " + tempFile);
+ try {
+ byte[] backup = ThriftBinaryCodec.encodeNonNull(snapshot);
+ Files.write(backup, tempFile);
+ Files.move(tempFile, new File(config.dir, backupName));
+ successes.incrementAndGet();
+ } catch (IOException e) {
+ failures.incrementAndGet();
+ LOG.log(Level.SEVERE, "Failed to prepare backup " + backupName + ": " + e, e);
+ } catch (CodingException e) {
+ LOG.log(Level.SEVERE, "Failed to encode backup " + backupName + ": " + e, e);
+ failures.incrementAndGet();
+ } finally {
+ if (tempFile.exists()) {
+ LOG.info("Deleting incomplete backup file " + tempFile);
+ tempFile.delete();
+ }
+ }
+
+ File[] backups = config.dir.listFiles(BACKUP_FILTER);
+ if (backups == null) {
+ LOG.severe("Failed to list backup dir " + config.dir);
+ } else {
+ int backupsToDelete = backups.length - config.maxBackups;
+ if (backupsToDelete > 0) {
+ List<File> toDelete = Ordering.natural()
+ .onResultOf(FILE_NAME)
+ .sortedCopy(ImmutableList.copyOf(backups)).subList(0, backupsToDelete);
+ LOG.info("Deleting " + backupsToDelete + " outdated backups: " + toDelete);
+ for (File outdated : toDelete) {
+ outdated.delete();
+ }
+ }
+ }
+ }
+
+ private static final FilenameFilter BACKUP_FILTER = new FilenameFilter() {
+ @Override public boolean accept(File file, String s) {
+ return s.startsWith(FILE_PREFIX);
+ }
+ };
+
+ @VisibleForTesting
+ static final Function<File, String> FILE_NAME = new Function<File, String>() {
+ @Override public String apply(File file) {
+ return file.getName();
+ }
+ };
+
+ @Override
+ public void applySnapshot(Snapshot snapshot) {
+ delegate.applySnapshot(snapshot);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
new file mode 100644
index 0000000..e0906fe
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.backup;
+
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.collect.FluentIterable;
+
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.SnapshotStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.log.SnapshotStoreImpl;
+import com.twitter.aurora.scheduler.storage.mem.MemStorage;
+import com.twitter.common.util.testing.FakeClock;
+
+/**
+ * A short-lived in-memory storage system that can be converted to a {@link Snapshot}.
+ */
+interface TemporaryStorage {
+
+ /**
+ * Deletes all tasks matching a query. Deleted tasks will not be reflected in the snapshot when
+ * {@link #toSnapshot()} is executed.
+ *
+ * @param query Query builder for tasks to delete.
+ */
+ void deleteTasks(Query.Builder query);
+
+ /**
+ * Fetches tasks matching a query.
+ *
+ * @param query Query builder for tasks to fetch.
+ * @return Matching tasks.
+ */
+ Set<IScheduledTask> fetchTasks(Query.Builder query);
+
+ /**
+ * Creates a snapshot of the contents of the temporary storage.
+ *
+ * @return Temporary storage snapshot.
+ */
+ Snapshot toSnapshot();
+
+ /**
+ * A factory that creates temporary storage instances, detached from the rest of the system.
+ */
+ class TemporaryStorageFactory implements Function<Snapshot, TemporaryStorage> {
+ @Override public TemporaryStorage apply(Snapshot snapshot) {
+ final Storage storage = MemStorage.newEmptyStorage();
+ FakeClock clock = new FakeClock();
+ clock.setNowMillis(snapshot.getTimestamp());
+ final SnapshotStore<Snapshot> snapshotStore = new SnapshotStoreImpl(clock, storage);
+ snapshotStore.applySnapshot(snapshot);
+
+ return new TemporaryStorage() {
+ @Override public void deleteTasks(final Query.Builder query) {
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ Set<String> ids = FluentIterable.from(storeProvider.getTaskStore().fetchTasks(query))
+ .transform(Tasks.SCHEDULED_TO_ID)
+ .toSet();
+ storeProvider.getUnsafeTaskStore().deleteTasks(ids);
+ }
+ });
+ }
+
+ @Override public Set<IScheduledTask> fetchTasks(final Query.Builder query) {
+ return storage.consistentRead(new Work.Quiet<Set<IScheduledTask>>() {
+ @Override public Set<IScheduledTask> apply(StoreProvider storeProvider) {
+ return storeProvider.getTaskStore().fetchTasks(query);
+ }
+ });
+ }
+
+ @Override public Snapshot toSnapshot() {
+ return snapshotStore.createSnapshot();
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
new file mode 100644
index 0000000..74e8c07
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.log;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+import java.util.zip.DeflaterOutputStream;
+import java.util.zip.InflaterInputStream;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.ByteStreams;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.storage.LogEntry;
+import com.twitter.aurora.gen.storage.LogEntry._Fields;
+import com.twitter.common.stats.Stats;
+
+/**
+ * Utility class for working with log entries.
+ */
+final class Entries {
+
+ private static final Logger LOG = Logger.getLogger(Entries.class.getName());
+
+ private static final AtomicLong COMPRESSION_BYTES_SAVED =
+ Stats.exportLong("log_compressed_entry_bytes_saved");
+
+ private Entries() {
+ // Utility class.
+ }
+
+ /**
+ * Deflates a log entry and wraps it in a deflated entry.
+ * <p>
+ * This will encode the entry using the thrift binary codec, and will apply deflate compression to
+ * the resulting encoded data.
+ * <p>
+ * This operation is symmetric with {@link #inflate(LogEntry)}.
+ *
+ * @param entry Entry to deflate.
+ * @return An entry with the {@code deflatedEntry} field set with the deflated serialized value
+ * of the original entry.
+ * @throws CodingException If the value could not be encoded or deflated.
+ */
+ static LogEntry deflate(LogEntry entry) throws CodingException {
+ byte[] data = thriftBinaryEncode(entry);
+ int initialLength = data.length;
+ LOG.info("Deflating log entry of size " + initialLength);
+ ByteArrayOutputStream deflated = new ByteArrayOutputStream();
+ DeflaterOutputStream deflater = new DeflaterOutputStream(deflated);
+ try {
+ deflater.write(data);
+ deflater.flush();
+ deflater.close();
+ byte[] deflatedData = deflated.toByteArray();
+ int bytesSaved = initialLength - deflatedData.length;
+ if (bytesSaved < 0) {
+ LOG.warning("Deflated entry is larger than original by " + (bytesSaved * -1) + " bytes");
+ } else {
+ LOG.info("Deflated log entry size: " + deflatedData.length + " (saved " + bytesSaved + ")");
+ }
+
+ COMPRESSION_BYTES_SAVED.addAndGet(bytesSaved);
+ return LogEntry.deflatedEntry(ByteBuffer.wrap(deflatedData));
+ } catch (IOException e) {
+ throw new CodingException("Failed to deflate snapshot: " + e, e);
+ }
+ }
+
+ /**
+ * Inflates and deserializes a deflated log entry.
+ * <p>
+ * This requires that the {@code deflatedEntry} field is set on the provided {@code entry}.
+ * The encoded value will be inflated and deserialized as a {@link LogEntry}.
+ *
+ * @param entry Entry to inflate, which must be a deflated entry.
+ * @return The inflated entry.
+ * @throws CodingException If the value could not be inflated or decoded.
+ */
+ static LogEntry inflate(LogEntry entry) throws CodingException {
+ Preconditions.checkArgument(entry.isSet(_Fields.DEFLATED_ENTRY));
+
+ ByteArrayOutputStream inflated = new ByteArrayOutputStream();
+ ByteBuffer data = entry.bufferForDeflatedEntry();
+ LOG.info("Inflating deflated log entry of size " + data.remaining());
+ InflaterInputStream inflater = new InflaterInputStream(
+ new ByteArrayInputStream(data.array(), data.position(), data.remaining()));
+ try {
+ ByteStreams.copy(inflater, inflated);
+ byte[] inflatedData = inflated.toByteArray();
+ LOG.info("Inflated log entry size: " + inflatedData.length);
+ return thriftBinaryDecode(inflatedData);
+ } catch (IOException e) {
+ throw new CodingException("Failed to inflate compressed log entry.", e);
+ }
+ }
+
+ /**
+ * Thrift binary-encodes a log entry.
+ *
+ * @param entry The entry to encode.
+ * @return The serialized entry value.
+ * @throws CodingException If the entry could not be encoded.
+ */
+ static byte[] thriftBinaryEncode(LogEntry entry) throws CodingException {
+ return ThriftBinaryCodec.encodeNonNull(entry);
+ }
+
+ /**
+ * Decodes a byte array containing thrift binary-encoded data.
+ *
+ * @param contents The data to decode.
+ * @return The deserialized entry.
+ * @throws CodingException If the entry could not be deserialized.
+ */
+ static LogEntry thriftBinaryDecode(byte[] contents) throws CodingException {
+ return ThriftBinaryCodec.decodeNonNull(LogEntry.class, contents);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
new file mode 100644
index 0000000..da29401
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
@@ -0,0 +1,516 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.log;
+
+import java.io.IOException;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.primitives.Bytes;
+import com.google.inject.BindingAnnotation;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.ScheduledTask;
+import com.twitter.aurora.gen.storage.Frame;
+import com.twitter.aurora.gen.storage.FrameChunk;
+import com.twitter.aurora.gen.storage.FrameHeader;
+import com.twitter.aurora.gen.storage.LogEntry;
+import com.twitter.aurora.gen.storage.LogEntry._Fields;
+import com.twitter.aurora.gen.storage.Op;
+import com.twitter.aurora.gen.storage.RemoveTasks;
+import com.twitter.aurora.gen.storage.SaveHostAttributes;
+import com.twitter.aurora.gen.storage.SaveTasks;
+import com.twitter.aurora.gen.storage.Snapshot;
+import com.twitter.aurora.gen.storage.Transaction;
+import com.twitter.aurora.gen.storage.storageConstants;
+import com.twitter.aurora.scheduler.log.Log;
+import com.twitter.aurora.scheduler.log.Log.Entry;
+import com.twitter.aurora.scheduler.log.Log.Position;
+import com.twitter.aurora.scheduler.log.Log.Stream;
+import com.twitter.aurora.scheduler.log.Log.Stream.InvalidPositionException;
+import com.twitter.aurora.scheduler.log.Log.Stream.StreamAccessException;
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.base.Closure;
+import com.twitter.common.base.ExceptionalCommand;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+import com.twitter.common.stats.Stats;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Manages opening, reading from and writing to a {@link Log}.
+ */
+public final class LogManager {
+
+ /**
+ * Identifies the maximum log entry size to permit before chunking entries into frames.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ ElementType.PARAMETER, ElementType.METHOD })
+ @BindingAnnotation
+ public @interface MaxEntrySize { }
+
+ /**
+ * Binding annotation for settings regarding the way snapshots are written.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target({ ElementType.PARAMETER, ElementType.METHOD })
+ @BindingAnnotation
+ public @interface SnapshotSetting { }
+
+ private static final Logger LOG = Logger.getLogger(LogManager.class.getName());
+
+ private final Log log;
+ private final Amount<Integer, Data> maxEntrySize;
+ private final boolean deflateSnapshots;
+ private final ShutdownRegistry shutdownRegistry;
+
+ @Inject
+ LogManager(
+ Log log,
+ @MaxEntrySize Amount<Integer, Data> maxEntrySize,
+ @SnapshotSetting boolean deflateSnapshots,
+ ShutdownRegistry shutdownRegistry) {
+
+ this.log = checkNotNull(log);
+ this.maxEntrySize = checkNotNull(maxEntrySize);
+ this.deflateSnapshots = deflateSnapshots;
+ this.shutdownRegistry = checkNotNull(shutdownRegistry);
+ }
+
+ /**
+ * Opens the log for reading and writing.
+ *
+ * @return A stream manager that can be used to manipulate the log stream.
+ * @throws IOException If there is a problem opening the log.
+ */
+ public StreamManager open() throws IOException {
+ final Stream stream = log.open();
+ shutdownRegistry.addAction(new ExceptionalCommand<IOException>() {
+ @Override public void execute() throws IOException {
+ stream.close();
+ }
+ });
+ return new StreamManager(stream, deflateSnapshots, maxEntrySize);
+ }
+
+ /**
+ * Manages interaction with the log stream. Log entries can be
+ * {@link #readFromBeginning(com.twitter.common.base.Closure) read from} the beginning,
+ * a {@link #startTransaction() transaction} consisting of one or more local storage
+ * operations can be committed atomically, or the log can be compacted by
+ * {@link #snapshot(com.twitter.aurora.gen.storage.Snapshot) snapshotting}.
+ */
+ public static class StreamManager {
+
+ private static MessageDigest createDigest() {
+ try {
+ return MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("Could not find provider for standard algorithm 'MD5'", e);
+ }
+ }
+
+ private static class Vars {
+ private final AtomicInteger unSnapshottedTransactions =
+ Stats.exportInt("scheduler_log_un_snapshotted_transactions");
+ private final AtomicLong bytesWritten = Stats.exportLong("scheduler_log_bytes_written");
+ private final AtomicLong entriesWritten = Stats.exportLong("scheduler_log_entries_written");
+ private final AtomicLong badFramesRead = Stats.exportLong("scheduler_log_bad_frames_read");
+ private final AtomicLong bytesRead = Stats.exportLong("scheduler_log_bytes_read");
+ private final AtomicLong entriesRead = Stats.exportLong("scheduler_log_entries_read");
+ private final AtomicLong deflatedEntriesRead =
+ Stats.exportLong("scheduler_log_deflated_entries_read");
+ private final AtomicLong snapshots = Stats.exportLong("scheduler_log_snapshots");
+ }
+ private final Vars vars = new Vars();
+
+ private final Object writeMutex = new Object();
+ private final Stream stream;
+ private final boolean deflateSnapshots;
+ private final MessageDigest digest;
+ private final EntrySerializer entrySerializer;
+
+ StreamManager(Stream stream, boolean deflateSnapshots, Amount<Integer, Data> maxEntrySize) {
+ this.stream = checkNotNull(stream);
+ this.deflateSnapshots = deflateSnapshots;
+ digest = createDigest();
+ entrySerializer = new EntrySerializer(digest, maxEntrySize);
+ }
+
+ /**
+ * Reads all entries in the log stream after the given position. If the position
+ * supplied is {@code null} then all log entries in the stream will be read.
+ *
+ * @param reader A reader that will be handed log entries decoded from the stream.
+ * @throws CodingException if there was a problem decoding a log entry from the stream.
+ * @throws InvalidPositionException if the given position is not found in the log.
+ * @throws StreamAccessException if there is a problem reading from the log.
+ */
+ public void readFromBeginning(Closure<LogEntry> reader)
+ throws CodingException, InvalidPositionException, StreamAccessException {
+
+ Iterator<Entry> entries = stream.readAll();
+
+ while (entries.hasNext()) {
+ LogEntry logEntry = decodeLogEntry(entries.next());
+ while (logEntry != null && isFrame(logEntry)) {
+ logEntry = tryDecodeFrame(logEntry.getFrame(), entries);
+ }
+ if (logEntry != null) {
+ if (logEntry.isSet(_Fields.DEFLATED_ENTRY)) {
+ logEntry = Entries.inflate(logEntry);
+ vars.deflatedEntriesRead.incrementAndGet();
+ }
+
+ reader.execute(logEntry);
+ vars.entriesRead.incrementAndGet();
+ }
+ }
+ }
+
+ @Nullable
+ private LogEntry tryDecodeFrame(Frame frame, Iterator<Entry> entries) throws CodingException {
+ if (!isHeader(frame)) {
+ LOG.warning("Found a frame with no preceding header, skipping.");
+ return null;
+ }
+ FrameHeader header = frame.getHeader();
+ byte[][] chunks = new byte[header.chunkCount][];
+
+ digest.reset();
+ for (int i = 0; i < header.chunkCount; i++) {
+ if (!entries.hasNext()) {
+ logBadFrame(header, i);
+ return null;
+ }
+ LogEntry logEntry = decodeLogEntry(entries.next());
+ if (!isFrame(logEntry)) {
+ logBadFrame(header, i);
+ return logEntry;
+ }
+ Frame chunkFrame = logEntry.getFrame();
+ if (!isChunk(chunkFrame)) {
+ logBadFrame(header, i);
+ return logEntry;
+ }
+ byte[] chunkData = chunkFrame.getChunk().getData();
+ digest.update(chunkData);
+ chunks[i] = chunkData;
+ }
+ if (!Arrays.equals(header.getChecksum(), digest.digest())) {
+ throw new CodingException("Read back a framed log entry that failed its checksum");
+ }
+ return Entries.thriftBinaryDecode(Bytes.concat(chunks));
+ }
+
+ private static boolean isFrame(LogEntry logEntry) {
+ return logEntry.getSetField() == LogEntry._Fields.FRAME;
+ }
+
+ private static boolean isChunk(Frame frame) {
+ return frame.getSetField() == Frame._Fields.CHUNK;
+ }
+
+ private static boolean isHeader(Frame frame) {
+ return frame.getSetField() == Frame._Fields.HEADER;
+ }
+
+ private void logBadFrame(FrameHeader header, int chunkIndex) {
+ LOG.info(String.format("Found an aborted transaction, required %d frames and found %d",
+ header.chunkCount, chunkIndex));
+ vars.badFramesRead.incrementAndGet();
+ }
+
+ private LogEntry decodeLogEntry(Entry entry) throws CodingException {
+ byte[] contents = entry.contents();
+ vars.bytesRead.addAndGet(contents.length);
+ return Entries.thriftBinaryDecode(contents);
+ }
+
+ /**
+ * Truncates all entries in the log stream occuring before the given position. The entry at the
+ * given position becomes the first entry in the stream when this call completes.
+ *
+ * @param position The last position to keep in the stream.
+ * @throws InvalidPositionException if the specified position does not exist in this log.
+ * @throws StreamAccessException if the stream could not be truncated.
+ */
+ void truncateBefore(Position position) {
+ stream.truncateBefore(position);
+ }
+
+ /**
+ * Starts a transaction that can be used to commit a series of {@link Op}s to the log stream
+ * atomically.
+ *
+ * @return StreamTransaction A transaction manager to handle batching up commits to the
+ * underlying stream.
+ */
+ StreamTransaction startTransaction() {
+ return new StreamTransaction();
+ }
+
+ /**
+ * Adds a snapshot to the log and if successful, truncates the log entries preceding the
+ * snapshot.
+ *
+ * @param snapshot The snapshot to add.
+ * @throws CodingException if the was a problem encoding the snapshot into a log entry.
+ * @throws InvalidPositionException if there was a problem truncating before the snapshot.
+ * @throws StreamAccessException if there was a problem appending the snapshot to the log.
+ */
+ @Timed("log_manager_snapshot")
+ void snapshot(Snapshot snapshot)
+ throws CodingException, InvalidPositionException, StreamAccessException {
+
+ LogEntry entry = LogEntry.snapshot(snapshot);
+ if (deflateSnapshots) {
+ entry = Entries.deflate(entry);
+ }
+
+ Position position = appendAndGetPosition(entry);
+ vars.snapshots.incrementAndGet();
+ vars.unSnapshottedTransactions.set(0);
+ stream.truncateBefore(position);
+ }
+
+ @Timed("log_manager_append")
+ private Position appendAndGetPosition(LogEntry logEntry) throws CodingException {
+ Position firstPosition = null;
+ byte[][] entries = entrySerializer.serialize(logEntry);
+ synchronized (writeMutex) { // ensure all sub-entries are written as a unit
+ for (byte[] entry : entries) {
+ Position position = stream.append(entry);
+ if (firstPosition == null) {
+ firstPosition = position;
+ }
+ vars.bytesWritten.addAndGet(entry.length);
+ }
+ }
+ vars.entriesWritten.incrementAndGet();
+ return firstPosition;
+ }
+
+ @VisibleForTesting
+ public static class EntrySerializer {
+ private final MessageDigest digest;
+ private final int maxEntrySizeBytes;
+
+ private EntrySerializer(MessageDigest digest, Amount<Integer, Data> maxEntrySize) {
+ this.digest = checkNotNull(digest);
+ maxEntrySizeBytes = maxEntrySize.as(Data.BYTES);
+ }
+
+ public EntrySerializer(Amount<Integer, Data> maxEntrySize) {
+ this(createDigest(), maxEntrySize);
+ }
+
+ /**
+ * Serializes a log entry and splits it into chunks no larger than {@code maxEntrySizeBytes}.
+ *
+ * @param logEntry The log entry to serialize.
+ * @return Serialized and chunked log entry.
+ * @throws CodingException If the entry could not be serialized.
+ */
+ @VisibleForTesting
+ public byte[][] serialize(LogEntry logEntry) throws CodingException {
+ byte[] entry = Entries.thriftBinaryEncode(logEntry);
+ if (entry.length <= maxEntrySizeBytes) {
+ return new byte[][] {entry};
+ }
+
+ int chunks = (int) Math.ceil(entry.length / (double) maxEntrySizeBytes);
+ byte[][] frames = new byte[chunks + 1][];
+
+ frames[0] = encode(Frame.header(new FrameHeader(chunks, ByteBuffer.wrap(checksum(entry)))));
+ for (int i = 0; i < chunks; i++) {
+ int offset = i * maxEntrySizeBytes;
+ ByteBuffer chunk =
+ ByteBuffer.wrap(entry, offset, Math.min(maxEntrySizeBytes, entry.length - offset));
+ frames[i + 1] = encode(Frame.chunk(new FrameChunk(chunk)));
+ }
+ return frames;
+ }
+
+ private byte[] checksum(byte[] data) {
+ digest.reset();
+ return digest.digest(data);
+ }
+
+ private static byte[] encode(Frame frame) throws CodingException {
+ return Entries.thriftBinaryEncode(LogEntry.frame(frame));
+ }
+ }
+
+ /**
+ * Manages a single log stream append transaction. Local storage ops can be added to the
+ * transaction and then later committed as an atomic unit.
+ */
+ final class StreamTransaction {
+ private final Transaction transaction =
+ new Transaction().setSchemaVersion(storageConstants.CURRENT_SCHEMA_VERSION);
+ private final AtomicBoolean committed = new AtomicBoolean(false);
+
+ private StreamTransaction() {
+ // supplied by factory method
+ }
+
+ /**
+ * Appends any ops that have been added to this transaction to the log stream in a single
+ * atomic record.
+ *
+ * @return The position of the log entry committed in this transaction, if any.
+ * @throws CodingException If there was a problem encoding a log entry for commit.
+ */
+ Position commit() throws CodingException {
+ Preconditions.checkState(!committed.getAndSet(true),
+ "Can only call commit once per transaction.");
+
+ if (!transaction.isSetOps()) {
+ return null;
+ }
+
+ Position position = appendAndGetPosition(LogEntry.transaction(transaction));
+ vars.unSnapshottedTransactions.incrementAndGet();
+ return position;
+ }
+
+ /**
+ * Adds a local storage operation to this transaction.
+ *
+ * @param op The local storage op to add.
+ */
+ void add(Op op) {
+ Preconditions.checkState(!committed.get());
+
+ Op prior = transaction.isSetOps() ? Iterables.getLast(transaction.getOps(), null) : null;
+ if (prior == null || !coalesce(prior, op)) {
+ transaction.addToOps(op);
+ }
+ }
+
+ /**
+ * Tries to coalesce a new op into the prior to compact the binary representation and increase
+ * batching.
+ *
+ * <p>Its recommended that as new {@code Op}s are added, they be treated here although they
+ * need not be</p>
+ *
+ * @param prior The previous op.
+ * @param next The next op to be added.
+ * @return {@code true} if the next op was coalesced into the prior, {@code false} otherwise.
+ */
+ private boolean coalesce(Op prior, Op next) {
+ if (!prior.isSet() && !next.isSet()) {
+ return false;
+ }
+
+ Op._Fields priorType = prior.getSetField();
+ if (!priorType.equals(next.getSetField())) {
+ return false;
+ }
+
+ switch (priorType) {
+ case SAVE_FRAMEWORK_ID:
+ prior.setSaveFrameworkId(next.getSaveFrameworkId());
+ return true;
+
+ case SAVE_ACCEPTED_JOB:
+ case REMOVE_JOB:
+ case SAVE_QUOTA:
+ case REMOVE_QUOTA:
+ return false;
+
+ case SAVE_TASKS:
+ coalesce(prior.getSaveTasks(), next.getSaveTasks());
+ return true;
+ case REMOVE_TASKS:
+ coalesce(prior.getRemoveTasks(), next.getRemoveTasks());
+ return true;
+ case SAVE_HOST_ATTRIBUTES:
+ return coalesce(prior.getSaveHostAttributes(), next.getSaveHostAttributes());
+ default:
+ LOG.warning("Unoptimized op: " + priorType);
+ return false;
+ }
+ }
+
+ private void coalesce(SaveTasks prior, SaveTasks next) {
+ if (next.isSetTasks()) {
+ if (prior.isSetTasks()) {
+ // It is an expected invariant that an operation may reference a task (identified by
+ // task ID) no more than one time. Therefore, to coalesce two SaveTasks operations,
+ // the most recent task definition overrides the prior operation.
+ Map<String, ScheduledTask> coalesced = Maps.newHashMap();
+ for (ScheduledTask task : prior.getTasks()) {
+ coalesced.put(task.getAssignedTask().getTaskId(), task);
+ }
+ for (ScheduledTask task : next.getTasks()) {
+ coalesced.put(task.getAssignedTask().getTaskId(), task);
+ }
+ prior.setTasks(ImmutableSet.copyOf(coalesced.values()));
+ } else {
+ prior.setTasks(next.getTasks());
+ }
+ }
+ }
+
+ private void coalesce(RemoveTasks prior, RemoveTasks next) {
+ if (next.isSetTaskIds()) {
+ if (prior.isSetTaskIds()) {
+ prior.setTaskIds(ImmutableSet.<String>builder()
+ .addAll(prior.getTaskIds())
+ .addAll(next.getTaskIds())
+ .build());
+ } else {
+ prior.setTaskIds(next.getTaskIds());
+ }
+ }
+ }
+
+ private boolean coalesce(SaveHostAttributes prior, SaveHostAttributes next) {
+ if (prior.getHostAttributes().getHost().equals(next.getHostAttributes().getHost())) {
+ prior.getHostAttributes().setAttributes(next.getHostAttributes().getAttributes());
+ return true;
+ }
+ return false;
+ }
+ }
+ }
+}
[36/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/Numbers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/Numbers.java b/src/main/java/org/apache/aurora/scheduler/base/Numbers.java
new file mode 100644
index 0000000..74b5e0b
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/Numbers.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
+
+/**
+ * Utility class for working with numbers.
+ */
+public final class Numbers {
+
+ private Numbers() {
+ // Utility class.
+ }
+
+ /**
+ * Converts a set of integers into a set of contiguous closed ranges that equally represent the
+ * input integers.
+ * <p>
+ * The resulting ranges will be in ascending order.
+ *
+ * @param values Values to transform to ranges.
+ * @return Closed ranges with identical members to the input set.
+ */
+ public static Set<Range<Integer>> toRanges(Iterable<Integer> values) {
+ ImmutableSet.Builder<Range<Integer>> builder = ImmutableSet.builder();
+
+ PeekingIterator<Integer> iterator =
+ Iterators.peekingIterator(Sets.newTreeSet(values).iterator());
+
+ // Build ranges until there are no numbers left.
+ while (iterator.hasNext()) {
+ // Start a new range.
+ int start = iterator.next();
+ int end = start;
+ // Increment the end until the range is non-contiguous.
+ while (iterator.hasNext() && (iterator.peek() == (end + 1))) {
+ end++;
+ iterator.next();
+ }
+
+ builder.add(Range.closed(start, end));
+ }
+
+ return builder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/Query.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/Query.java b/src/main/java/org/apache/aurora/scheduler/base/Query.java
new file mode 100644
index 0000000..d02ef87
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/Query.java
@@ -0,0 +1,364 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+import java.util.EnumSet;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.primitives.Ints;
+
+import com.twitter.aurora.gen.Identity;
+import com.twitter.aurora.gen.InstanceKey;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.TaskQuery;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static org.apache.commons.lang.StringUtils.isEmpty;
+
+/**
+ * A utility class to construct storage queries.
+ * TODO(Sathya): Add some basic unit tests for isJobScoped and isOnlyJobScoped.
+ */
+public final class Query {
+
+ private Query() {
+ // Utility.
+ }
+
+ /**
+ * Checks whether a query is scoped to a specific job.
+ * A query scoped to a job specifies a role and job name.
+ *
+ * @param taskQuery Query to test.
+ * @return {@code true} if the query specifies at least a role and job name,
+ * otherwise {@code false}.
+ */
+ public static boolean isJobScoped(Builder taskQuery) {
+ TaskQuery query = taskQuery.get();
+ return (query.getOwner() != null)
+ && !isEmpty(query.getOwner().getRole())
+ && !isEmpty(query.getEnvironment())
+ && !isEmpty(query.getJobName());
+ }
+
+ /**
+ * Checks whether a query is strictly scoped to a specific job. A query is strictly job scoped,
+ * iff it has the role, environment and jobName set.
+ *
+ * @param query Query to test.
+ * @return {@code true} if the query is strictly job scoped, otherwise {@code false}.
+ */
+ public static boolean isOnlyJobScoped(Builder query) {
+ Optional<IJobKey> jobKey = JobKeys.from(query);
+ return jobKey.isPresent() && Query.jobScoped(jobKey.get()).equals(query);
+ }
+
+ public static Builder arbitrary(TaskQuery query) {
+ return new Builder(query.deepCopy());
+ }
+
+ public static Builder unscoped() {
+ return new Builder();
+ }
+
+ public static Builder roleScoped(String role) {
+ return unscoped().byRole(role);
+ }
+
+ public static Builder envScoped(String role, String environment) {
+ return unscoped().byEnv(role, environment);
+ }
+
+ public static Builder jobScoped(IJobKey jobKey) {
+ return unscoped().byJob(jobKey);
+ }
+
+ public static Builder instanceScoped(InstanceKey instanceKey) {
+ return instanceScoped(IJobKey.build(instanceKey.getJobKey()), instanceKey.getInstanceId());
+ }
+
+ public static Builder instanceScoped(IJobKey jobKey, int instanceId, int... instanceIds) {
+ return unscoped().byInstances(jobKey, instanceId, instanceIds);
+ }
+
+ public static Builder instanceScoped(IJobKey jobKey, Iterable<Integer> instanceIds) {
+ return unscoped().byInstances(jobKey, instanceIds);
+ }
+
+ public static Builder taskScoped(String taskId, String... taskIds) {
+ return unscoped().byId(taskId, taskIds);
+ }
+
+ public static Builder taskScoped(Iterable<String> taskIds) {
+ return unscoped().byId(taskIds);
+ }
+
+ public static Builder slaveScoped(String slaveHost) {
+ return unscoped().bySlave(slaveHost);
+ }
+
+ public static Builder statusScoped(ScheduleStatus status, ScheduleStatus... statuses) {
+ return unscoped().byStatus(status, statuses);
+ }
+
+ public static Builder statusScoped(Iterable<ScheduleStatus> statuses) {
+ return unscoped().byStatus(statuses);
+ }
+
+ /**
+ * A Builder of TaskQueries. Builders are immutable and provide access to a set of convenience
+ * methods to return a new builder of another scope. Available scope filters include slave,
+ * taskId, role, jobs of a role, and instances of a job.
+ *
+ * <p>
+ * This class does not expose the full functionality of TaskQuery but rather subsets of it that
+ * can be efficiently executed and make sense in the context of the scheduler datastores. This
+ * builder should be preferred over constructing TaskQueries directly.
+ * </p>
+ *
+ * TODO(ksweeney): Add an environment scope.
+ */
+ public static final class Builder implements Supplier<TaskQuery> {
+ private final TaskQuery query;
+
+ private Builder() {
+ this.query = new TaskQuery();
+ }
+
+ private Builder(final TaskQuery query) {
+ this.query = checkNotNull(query); // It is expected that the caller calls deepCopy.
+ }
+
+ /**
+ * Build a query that is the combination of all the filters applied to a Builder. Mutating the
+ * returned object will not affect the state of the builder. Can be called any number of times
+ * and will return a new {@code TaskQuery} each time.
+ *
+ * @return A new TaskQuery satisfying this builder's constraints.
+ */
+ @Override
+ public TaskQuery get() {
+ return query.deepCopy();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ return that != null
+ && that instanceof Builder
+ && Objects.equal(query, ((Builder) that).query);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(query);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("query", query)
+ .toString();
+ }
+
+ /**
+ * Create a builder scoped to tasks.
+ *
+ * @param taskId An ID of a task to scope the builder to.
+ * @param taskIds Additional IDs of tasks to scope the builder to (they are ORed together).
+ * @return A new Builder scoped to the given tasks.
+ */
+ public Builder byId(String taskId, String... taskIds) {
+ checkNotNull(taskId);
+
+ return new Builder(
+ query.deepCopy()
+ .setTaskIds(ImmutableSet.<String>builder().add(taskId).add(taskIds).build()));
+ }
+
+ /**
+ * Create a builder scoped to tasks.
+ *
+ * @see #byId(String, String...)
+ *
+ * @param taskIds The IDs of the tasks to scope the query to (ORed together).
+ * @return A new Builder scoped to the given tasks.
+ */
+ public Builder byId(Iterable<String> taskIds) {
+ checkNotNull(taskIds);
+
+ return new Builder(
+ query.deepCopy().setTaskIds(ImmutableSet.copyOf(taskIds)));
+ }
+
+ /**
+ * Create a builder scoped to a role. A role scope conflicts with job and instance scopes.
+ *
+ * @param role The role to scope the query to.
+ * @return A new Builder scoped to the given role.
+ */
+ public Builder byRole(String role) {
+ checkNotNull(role);
+
+ return new Builder(
+ query.deepCopy().setOwner(new Identity().setRole(role)));
+ }
+
+ /**
+ * Create a builder scoped to an environment. An environment scope conflicts with role, job,
+ * and instance scopes.
+ *
+ * @param role The role to scope the query to.
+ * @param environment The environment to scope the query to.
+ * @return A new Builder scoped to the given environment.
+ */
+ public Builder byEnv(String role, String environment) {
+ checkNotNull(role);
+ checkNotNull(environment);
+
+ return new Builder(
+ query.deepCopy()
+ .setOwner(new Identity().setRole(role))
+ .setEnvironment(environment));
+ }
+
+ /**
+ * Returns a new builder scoped to the job uniquely identified by the given key. A job scope
+ * conflicts with role and instance scopes.
+ *
+ * @param jobKey The key of the job to scope the query to.
+ * @return A new Builder scoped to the given jobKey.
+ */
+ public Builder byJob(IJobKey jobKey) {
+ JobKeys.assertValid(jobKey);
+
+ return new Builder(
+ query.deepCopy()
+ .setOwner(new Identity().setRole(jobKey.getRole()))
+ .setEnvironment(jobKey.getEnvironment())
+ .setJobName(jobKey.getName()));
+ }
+
+ /**
+ * Returns a new builder scoped to the slave uniquely identified by the given slaveHost. A
+ * builder can only be scoped to slaves once.
+ *
+ * @param slaveHost The hostname of the slave to scope the query to.
+ * @return A new Builder scoped to the given slave.
+ */
+ public Builder bySlave(String slaveHost) {
+ checkNotNull(slaveHost);
+
+ return new Builder(query.deepCopy().setSlaveHost(slaveHost));
+ }
+
+ /**
+ * Returns a new builder scoped to the given statuses. A builder can only be scoped to statuses
+ * once.
+ *
+ * @param status The status to scope this Builder to.
+ * @param statuses Additional statuses to scope this Builder to (they are ORed together).
+ * @return A new Builder scoped to the given statuses.
+ */
+ public Builder byStatus(ScheduleStatus status, ScheduleStatus... statuses) {
+ checkNotNull(status);
+
+ return new Builder(
+ query.deepCopy().setStatuses(EnumSet.of(status, statuses)));
+ }
+
+ /**
+ * Create a new Builder scoped to statuses.
+ *
+ * @see Builder#byStatus(ScheduleStatus, ScheduleStatus...)
+ *
+ * @param statuses The statuses to scope this Builder to.
+ * @return A new Builder scoped to the given statuses.
+ */
+ public Builder byStatus(Iterable<ScheduleStatus> statuses) {
+ checkNotNull(statuses);
+
+ return new Builder(
+ query.deepCopy().setStatuses(EnumSet.copyOf(ImmutableSet.copyOf(statuses))));
+ }
+
+ /**
+ * Returns a new Builder scoped to the given instances of the given job. A builder can only
+ * be scoped to a set of instances, a job, or a role once.
+ *
+ * @param jobKey The key identifying the job.
+ * @param instanceId An instance id of the target job.
+ * @param instanceIds Additional instance ids of the target job.
+ * @return A new Builder scoped to the given instance ids.
+ */
+ public Builder byInstances(IJobKey jobKey, int instanceId, int... instanceIds) {
+ JobKeys.assertValid(jobKey);
+
+ return new Builder(
+ query.deepCopy()
+ .setOwner(new Identity().setRole(jobKey.getRole()))
+ .setEnvironment(jobKey.getEnvironment())
+ .setJobName(jobKey.getName())
+ .setInstanceIds(ImmutableSet.<Integer>builder()
+ .add(instanceId)
+ .addAll(Ints.asList(instanceIds))
+ .build()));
+ }
+
+ /**
+ * Create a new Builder scoped to instances.
+ *
+ * @see Builder#byInstances
+ *
+ * @param jobKey The key identifying the job.
+ * @param instanceIds Instances of the target job.
+ * @return A new Builder scoped to the given instance ids.
+ */
+ public Builder byInstances(IJobKey jobKey, Iterable<Integer> instanceIds) {
+ JobKeys.assertValid(jobKey);
+ checkNotNull(instanceIds);
+
+ return new Builder(
+ query.deepCopy()
+ .setOwner(new Identity().setRole(jobKey.getRole()))
+ .setEnvironment(jobKey.getEnvironment())
+ .setJobName(jobKey.getName())
+ .setInstanceIds(ImmutableSet.copyOf(instanceIds)));
+ }
+
+ /**
+ * A convenience method to scope this builder to {@link Tasks#ACTIVE_STATES}.
+ *
+ * @return A new Builder scoped to Tasks#ACTIVE_STATES.
+ */
+ public Builder active() {
+ return byStatus(Tasks.ACTIVE_STATES);
+ }
+
+ /**
+ * A convenience method to scope this builder to {@link Tasks#TERMINAL_STATES}.
+ *
+ * @return A new Builder scoped to Tasks#TERMINAL_STATES.
+ */
+ public Builder terminal() {
+ return byStatus(Tasks.TERMINAL_STATES);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/ScheduleException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/ScheduleException.java b/src/main/java/org/apache/aurora/scheduler/base/ScheduleException.java
new file mode 100644
index 0000000..0420ee9
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/ScheduleException.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+/**
+ * Exception class to signal a failure to schedule a task or job.
+ */
+public class ScheduleException extends Exception {
+ public ScheduleException(String msg) {
+ super(msg);
+ }
+
+ public ScheduleException(String msg, Throwable t) {
+ super(msg, t);
+ }
+
+ public ScheduleException(Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/SchedulerException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/SchedulerException.java b/src/main/java/org/apache/aurora/scheduler/base/SchedulerException.java
new file mode 100644
index 0000000..a51c4e0
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/SchedulerException.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+/**
+ * Indicates some form of unexpected scheduler exception.
+ */
+public class SchedulerException extends RuntimeException {
+ public SchedulerException(String message) {
+ super(message);
+ }
+ public SchedulerException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ public SchedulerException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/Tasks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/Tasks.java b/src/main/java/org/apache/aurora/scheduler/base/Tasks.java
new file mode 100644
index 0000000..d98da3f
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/Tasks.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Ordering;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.ScheduledTask;
+import com.twitter.aurora.gen.apiConstants;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * Utility class providing convenience functions relating to tasks.
+ */
+public final class Tasks {
+
+ public static final Function<IScheduledTask, IAssignedTask> SCHEDULED_TO_ASSIGNED =
+ new Function<IScheduledTask, IAssignedTask>() {
+ @Override public IAssignedTask apply(IScheduledTask task) {
+ return task.getAssignedTask();
+ }
+ };
+
+ public static final Function<IAssignedTask, ITaskConfig> ASSIGNED_TO_INFO =
+ new Function<IAssignedTask, ITaskConfig>() {
+ @Override public ITaskConfig apply(IAssignedTask task) {
+ return task.getTask();
+ }
+ };
+
+ public static final Function<IScheduledTask, ITaskConfig> SCHEDULED_TO_INFO =
+ Functions.compose(ASSIGNED_TO_INFO, SCHEDULED_TO_ASSIGNED);
+
+ public static final Function<IAssignedTask, String> ASSIGNED_TO_ID =
+ new Function<IAssignedTask, String>() {
+ @Override public String apply(IAssignedTask task) {
+ return task.getTaskId();
+ }
+ };
+
+ public static final Function<IScheduledTask, String> SCHEDULED_TO_ID =
+ Functions.compose(ASSIGNED_TO_ID, SCHEDULED_TO_ASSIGNED);
+
+ public static final Function<IAssignedTask, Integer> ASSIGNED_TO_INSTANCE_ID =
+ new Function<IAssignedTask, Integer>() {
+ @Override public Integer apply(IAssignedTask task) {
+ return task.getInstanceId();
+ }
+ };
+
+ public static final Function<IScheduledTask, Integer> SCHEDULED_TO_INSTANCE_ID =
+ Functions.compose(ASSIGNED_TO_INSTANCE_ID, SCHEDULED_TO_ASSIGNED);
+
+ public static final Function<ITaskConfig, IJobKey> INFO_TO_JOB_KEY =
+ new Function<ITaskConfig, IJobKey>() {
+ @Override public IJobKey apply(ITaskConfig task) {
+ return JobKeys.from(task);
+ }
+ };
+
+ public static final Function<IAssignedTask, IJobKey> ASSIGNED_TO_JOB_KEY =
+ Functions.compose(INFO_TO_JOB_KEY, ASSIGNED_TO_INFO);
+
+ public static final Function<IScheduledTask, IJobKey> SCHEDULED_TO_JOB_KEY =
+ Functions.compose(ASSIGNED_TO_JOB_KEY, SCHEDULED_TO_ASSIGNED);
+
+ /**
+ * Different states that an active task may be in.
+ */
+ public static final EnumSet<ScheduleStatus> ACTIVE_STATES =
+ EnumSet.copyOf(apiConstants.ACTIVE_STATES);
+
+ /**
+ * Terminal states, which a task should not move from.
+ */
+ public static final Set<ScheduleStatus> TERMINAL_STATES =
+ EnumSet.copyOf(apiConstants.TERMINAL_STATES);
+
+ public static final Predicate<ITaskConfig> IS_PRODUCTION =
+ new Predicate<ITaskConfig>() {
+ @Override public boolean apply(ITaskConfig task) {
+ return task.isProduction();
+ }
+ };
+
+ public static final Function<IScheduledTask, ScheduleStatus> GET_STATUS =
+ new Function<IScheduledTask, ScheduleStatus>() {
+ @Override public ScheduleStatus apply(IScheduledTask task) {
+ return task.getStatus();
+ }
+ };
+
+ /**
+ * Order by production flag (true, then false), subsorting by task ID.
+ */
+ public static final Ordering<IAssignedTask> SCHEDULING_ORDER =
+ Ordering.explicit(true, false)
+ .onResultOf(Functions.compose(Functions.forPredicate(IS_PRODUCTION), ASSIGNED_TO_INFO))
+ .compound(Ordering.natural().onResultOf(ASSIGNED_TO_ID));
+
+ private Tasks() {
+ // Utility class.
+ }
+
+ /**
+ * A utility method that returns a multi-map of tasks keyed by IJobKey.
+ * @param tasks A list of tasks to be keyed by map
+ * @return A multi-map of tasks keyed by job key.
+ */
+ public static Multimap<IJobKey, IScheduledTask> byJobKey(Iterable<IScheduledTask> tasks) {
+ return Multimaps.index(tasks, Tasks.SCHEDULED_TO_JOB_KEY);
+ }
+
+ public static boolean isActive(ScheduleStatus status) {
+ return ACTIVE_STATES.contains(status);
+ }
+
+ public static boolean isTerminated(ScheduleStatus status) {
+ return TERMINAL_STATES.contains(status);
+ }
+
+ public static String id(IScheduledTask task) {
+ return task.getAssignedTask().getTaskId();
+ }
+
+ // TODO(William Farner: Remove this once the code base is switched to IScheduledTask.
+ public static String id(ScheduledTask task) {
+ return task.getAssignedTask().getTaskId();
+ }
+
+ public static Set<String> ids(Iterable<IScheduledTask> tasks) {
+ return ImmutableSet.copyOf(Iterables.transform(tasks, SCHEDULED_TO_ID));
+ }
+
+ public static Set<String> ids(IScheduledTask... tasks) {
+ return ids(ImmutableList.copyOf(tasks));
+ }
+
+ public static Map<String, IScheduledTask> mapById(Iterable<IScheduledTask> tasks) {
+ return Maps.uniqueIndex(tasks, SCHEDULED_TO_ID);
+ }
+
+ public static String getRole(IScheduledTask task) {
+ return task.getAssignedTask().getTask().getOwner().getRole();
+ }
+
+ public static String getJob(IScheduledTask task) {
+ return task.getAssignedTask().getTask().getJobName();
+ }
+
+ public static final Ordering<IScheduledTask> LATEST_ACTIVITY = Ordering.natural()
+ .onResultOf(new Function<IScheduledTask, Long>() {
+ @Override public Long apply(IScheduledTask task) {
+ return Iterables.getLast(task.getTaskEvents()).getTimestamp();
+ }
+ });
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
new file mode 100644
index 0000000..4839d0f
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
@@ -0,0 +1,413 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.configuration;
+
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.twitter.aurora.gen.Constraint;
+import com.twitter.aurora.gen.JobConfiguration;
+import com.twitter.aurora.gen.LimitConstraint;
+import com.twitter.aurora.gen.TaskConfig;
+import com.twitter.aurora.gen.TaskConfig._Fields;
+import com.twitter.aurora.gen.TaskConstraint;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.storage.entities.IConstraint;
+import com.twitter.aurora.scheduler.storage.entities.IIdentity;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConstraint;
+import com.twitter.aurora.scheduler.storage.entities.IValueConstraint;
+import com.twitter.common.base.Closure;
+import com.twitter.common.base.MorePreconditions;
+
+import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT;
+import static com.twitter.aurora.gen.apiConstants.GOOD_IDENTIFIER_PATTERN_JVM;
+
+/**
+ * Manages translation from a string-mapped configuration to a concrete configuration type, and
+ * defaults for optional values.
+ *
+ * TODO(William Farner): Add input validation to all fields (strings not empty, positive ints, etc).
+ */
+public final class ConfigurationManager {
+
+ public static final String DEDICATED_ATTRIBUTE = "dedicated";
+
+ @VisibleForTesting public static final String HOST_CONSTRAINT = "host";
+ @VisibleForTesting public static final String RACK_CONSTRAINT = "rack";
+
+ private static final Pattern GOOD_IDENTIFIER = Pattern.compile(GOOD_IDENTIFIER_PATTERN_JVM);
+
+ private static final int MAX_IDENTIFIER_LENGTH = 255;
+
+ private static class DefaultField implements Closure<TaskConfig> {
+ private final _Fields field;
+ private final Object defaultValue;
+
+ DefaultField(_Fields field, Object defaultValue) {
+ this.field = field;
+ this.defaultValue = defaultValue;
+ }
+
+ @Override public void execute(TaskConfig task) {
+ if (!task.isSet(field)) {
+ task.setFieldValue(field, defaultValue);
+ }
+ }
+ }
+
+ private interface Validator<T> {
+ void validate(T value) throws TaskDescriptionException;
+ }
+
+ private static class GreaterThan implements Validator<Number> {
+ private final double min;
+ private final String label;
+
+ GreaterThan(double min, String label) {
+ this.min = min;
+ this.label = label;
+ }
+
+ @Override public void validate(Number value) throws TaskDescriptionException {
+ if (this.min >= value.doubleValue()) {
+ throw new TaskDescriptionException(label + " must be greater than " + this.min);
+ }
+ }
+ }
+
+ private static class RequiredFieldValidator<T> implements Validator<TaskConfig> {
+ private final _Fields field;
+ private final Validator<T> validator;
+
+ RequiredFieldValidator(_Fields field, Validator<T> validator) {
+ this.field = field;
+ this.validator = validator;
+ }
+
+ public void validate(TaskConfig task) throws TaskDescriptionException {
+ if (!task.isSet(field)) {
+ throw new TaskDescriptionException("Field " + field.getFieldName() + " is required.");
+ }
+ @SuppressWarnings("unchecked")
+ T value = (T) task.getFieldValue(field);
+ validator.validate(value);
+ }
+ }
+
+ private static final Iterable<Closure<TaskConfig>> DEFAULT_FIELD_POPULATORS =
+ ImmutableList.of(
+ new DefaultField(_Fields.IS_SERVICE, false),
+ new DefaultField(_Fields.PRIORITY, 0),
+ new DefaultField(_Fields.PRODUCTION, false),
+ new DefaultField(_Fields.MAX_TASK_FAILURES, 1),
+ new DefaultField(_Fields.TASK_LINKS, Maps.<String, String>newHashMap()),
+ new DefaultField(_Fields.REQUESTED_PORTS, Sets.<String>newHashSet()),
+ new DefaultField(_Fields.CONSTRAINTS, Sets.<Constraint>newHashSet()),
+ new DefaultField(_Fields.ENVIRONMENT, DEFAULT_ENVIRONMENT),
+ new Closure<TaskConfig>() {
+ @Override public void execute(TaskConfig task) {
+ if (!Iterables.any(task.getConstraints(), hasName(HOST_CONSTRAINT))) {
+ task.addToConstraints(hostLimitConstraint(1));
+ }
+ }
+ },
+ new Closure<TaskConfig>() {
+ @Override public void execute(TaskConfig task) {
+ if (!isDedicated(ITaskConfig.build(task))
+ && task.isProduction()
+ && task.isIsService()
+ && !Iterables.any(task.getConstraints(), hasName(RACK_CONSTRAINT))) {
+
+ task.addToConstraints(rackLimitConstraint(1));
+ }
+ }
+ });
+
+ private static final Iterable<RequiredFieldValidator<?>> REQUIRED_FIELDS_VALIDATORS =
+ ImmutableList.<RequiredFieldValidator<?>>of(
+ new RequiredFieldValidator<>(_Fields.NUM_CPUS, new GreaterThan(0.0, "num_cpus")),
+ new RequiredFieldValidator<>(_Fields.RAM_MB, new GreaterThan(0.0, "ram_mb")),
+ new RequiredFieldValidator<>(_Fields.DISK_MB, new GreaterThan(0.0, "disk_mb")));
+
+ private ConfigurationManager() {
+ // Utility class.
+ }
+
+ @VisibleForTesting
+ static boolean isGoodIdentifier(String identifier) {
+ return GOOD_IDENTIFIER.matcher(identifier).matches()
+ && (identifier.length() <= MAX_IDENTIFIER_LENGTH);
+ }
+
+ private static void checkNotNull(Object value, String error) throws TaskDescriptionException {
+ if (value == null) {
+ throw new TaskDescriptionException(error);
+ }
+ }
+
+ private static void assertOwnerValidity(IIdentity jobOwner) throws TaskDescriptionException {
+ checkNotNull(jobOwner, "No job owner specified!");
+ checkNotNull(jobOwner.getRole(), "No job role specified!");
+ checkNotNull(jobOwner.getUser(), "No job user specified!");
+
+ if (!isGoodIdentifier(jobOwner.getRole())) {
+ throw new TaskDescriptionException(
+ "Job role contains illegal characters: " + jobOwner.getRole());
+ }
+
+ if (!isGoodIdentifier(jobOwner.getUser())) {
+ throw new TaskDescriptionException(
+ "Job user contains illegal characters: " + jobOwner.getUser());
+ }
+ }
+
+ private static String getRole(IValueConstraint constraint) {
+ return Iterables.getOnlyElement(constraint.getValues()).split("/")[0];
+ }
+
+ private static boolean isValueConstraint(ITaskConstraint taskConstraint) {
+ return taskConstraint.getSetField() == TaskConstraint._Fields.VALUE;
+ }
+
+ public static boolean isDedicated(ITaskConfig task) {
+ return Iterables.any(task.getConstraints(), getConstraintByName(DEDICATED_ATTRIBUTE));
+ }
+
+ @Nullable
+ private static IConstraint getDedicatedConstraint(ITaskConfig task) {
+ return Iterables.find(task.getConstraints(), getConstraintByName(DEDICATED_ATTRIBUTE), null);
+ }
+
+ /**
+ * Check validity of and populates defaults in a job configuration. This will return a deep copy
+ * of the provided job configuration with default configuration values applied, and configuration
+ * map values sanitized and applied to their respective struct fields.
+ *
+ * @param job Job to validate and populate.
+ * @return A deep copy of {@code job} that has been populated.
+ * @throws TaskDescriptionException If the job configuration is invalid.
+ */
+ public static IJobConfiguration validateAndPopulate(IJobConfiguration job)
+ throws TaskDescriptionException {
+
+ Preconditions.checkNotNull(job);
+
+ if (!job.isSetTaskConfig()) {
+ throw new TaskDescriptionException("Job configuration must have taskConfig set.");
+ }
+
+ if (!job.isSetInstanceCount()) {
+ throw new TaskDescriptionException("Job configuration does not have shardCount set.");
+ }
+
+ if (job.getInstanceCount() <= 0) {
+ throw new TaskDescriptionException("Shard count must be positive.");
+ }
+
+ JobConfiguration builder = job.newBuilder();
+
+ assertOwnerValidity(job.getOwner());
+
+ if (!JobKeys.isValid(job.getKey())) {
+ throw new TaskDescriptionException("Job key " + job.getKey() + " is invalid.");
+ }
+ if (!job.getKey().getRole().equals(job.getOwner().getRole())) {
+ throw new TaskDescriptionException("Role in job key must match job owner.");
+ }
+ if (!isGoodIdentifier(job.getKey().getRole())) {
+ throw new TaskDescriptionException(
+ "Job role contains illegal characters: " + job.getKey().getRole());
+ }
+ if (!isGoodIdentifier(job.getKey().getEnvironment())) {
+ throw new TaskDescriptionException(
+ "Job environment contains illegal characters: " + job.getKey().getEnvironment());
+ }
+ if (!isGoodIdentifier(job.getKey().getName())) {
+ throw new TaskDescriptionException(
+ "Job name contains illegal characters: " + job.getKey().getName());
+ }
+
+ builder.setTaskConfig(
+ validateAndPopulate(ITaskConfig.build(builder.getTaskConfig())).newBuilder());
+
+ // Only one of [service=true, cron_schedule] may be set.
+ if (!StringUtils.isEmpty(job.getCronSchedule()) && builder.getTaskConfig().isIsService()) {
+ throw new TaskDescriptionException(
+ "A service task may not be run on a cron schedule: " + builder);
+ }
+
+ return IJobConfiguration.build(builder);
+ }
+
+ /**
+ * Check validity of and populates defaults in a task configuration. This will return a deep copy
+ * of the provided task configuration with default configuration values applied, and configuration
+ * map values sanitized and applied to their respective struct fields.
+ *
+ *
+ * @param config Task config to validate and populate.
+ * @return A reference to the modified {@code config} (for chaining).
+ * @throws TaskDescriptionException If the task is invalid.
+ */
+ public static ITaskConfig validateAndPopulate(ITaskConfig config)
+ throws TaskDescriptionException {
+
+ TaskConfig builder = config.newBuilder();
+
+ if (!builder.isSetRequestedPorts()) {
+ builder.setRequestedPorts(ImmutableSet.<String>of());
+ }
+
+ maybeFillLinks(builder);
+
+ assertOwnerValidity(config.getOwner());
+
+ if (!isGoodIdentifier(config.getJobName())) {
+ throw new TaskDescriptionException(
+ "Job name contains illegal characters: " + config.getJobName());
+ }
+
+ if (!isGoodIdentifier(config.getEnvironment())) {
+ throw new TaskDescriptionException(
+ "Environment contains illegal characters: " + config.getEnvironment());
+ }
+
+ if (!builder.isSetExecutorConfig()) {
+ throw new TaskDescriptionException("Configuration may not be null");
+ }
+
+ // Maximize the usefulness of any thrown error message by checking required fields first.
+ for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
+ validator.validate(builder);
+ }
+
+ IConstraint constraint = getDedicatedConstraint(config);
+ if (constraint != null) {
+ if (!isValueConstraint(constraint.getConstraint())) {
+ throw new TaskDescriptionException("A dedicated constraint must be of value type.");
+ }
+
+ IValueConstraint valueConstraint = constraint.getConstraint().getValue();
+
+ if (!(valueConstraint.getValues().size() == 1)) {
+ throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
+ }
+
+ String dedicatedRole = getRole(valueConstraint);
+ if (!config.getOwner().getRole().equals(dedicatedRole)) {
+ throw new TaskDescriptionException(
+ "Only " + dedicatedRole + " may use hosts dedicated for that role.");
+ }
+ }
+
+ return ITaskConfig.build(applyDefaultsIfUnset(builder));
+ }
+
+ /**
+ * Provides a filter for the given constraint name.
+ *
+ * @param name The name of the constraint.
+ * @return A filter that matches the constraint.
+ */
+ public static Predicate<IConstraint> getConstraintByName(final String name) {
+ return new Predicate<IConstraint>() {
+ @Override public boolean apply(IConstraint constraint) {
+ return constraint.getName().equals(name);
+ }
+ };
+ }
+
+ @VisibleForTesting
+ public static Constraint hostLimitConstraint(int limit) {
+ return new Constraint(HOST_CONSTRAINT, TaskConstraint.limit(new LimitConstraint(limit)));
+ }
+
+ @VisibleForTesting
+ public static Constraint rackLimitConstraint(int limit) {
+ return new Constraint(RACK_CONSTRAINT, TaskConstraint.limit(new LimitConstraint(limit)));
+ }
+
+ private static Predicate<Constraint> hasName(final String name) {
+ MorePreconditions.checkNotBlank(name);
+ return new Predicate<Constraint>() {
+ @Override public boolean apply(Constraint constraint) {
+ return name.equals(constraint.getName());
+ }
+ };
+ }
+
+ /**
+ * Applies defaults to unset values in a task.
+ *
+ * @param task Task to apply defaults to.
+ * @return A reference to the (modified) {@code task}.
+ */
+ @VisibleForTesting
+ public static TaskConfig applyDefaultsIfUnset(TaskConfig task) {
+ for (Closure<TaskConfig> populator : DEFAULT_FIELD_POPULATORS) {
+ populator.execute(task);
+ }
+
+ return task;
+ }
+
+ /**
+ * Applies defaults to unset values in a job and its tasks.
+ *
+ * @param job Job to apply defaults to.
+ */
+ @VisibleForTesting
+ public static void applyDefaultsIfUnset(JobConfiguration job) {
+ ConfigurationManager.applyDefaultsIfUnset(job.getTaskConfig());
+ }
+
+ private static void maybeFillLinks(TaskConfig task) {
+ if (task.getTaskLinksSize() == 0) {
+ ImmutableMap.Builder<String, String> links = ImmutableMap.builder();
+ if (task.getRequestedPorts().contains("health")) {
+ links.put("health", "http://%host%:%port:health%");
+ }
+ if (task.getRequestedPorts().contains("http")) {
+ links.put("http", "http://%host%:%port:http%");
+ }
+ task.setTaskLinks(links.build());
+ }
+ }
+
+ /**
+ * Thrown when an invalid task or job configuration is encountered.
+ */
+ public static class TaskDescriptionException extends Exception {
+ public TaskDescriptionException(String msg) {
+ super(msg);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java b/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java
new file mode 100644
index 0000000..51e9973
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/configuration/Resources.java
@@ -0,0 +1,447 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.configuration;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ContiguousSet;
+import com.google.common.collect.DiscreteDomain;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.Resource;
+import org.apache.mesos.Protos.Value.Range;
+import org.apache.mesos.Protos.Value.Ranges;
+import org.apache.mesos.Protos.Value.Scalar;
+import org.apache.mesos.Protos.Value.Type;
+
+import com.twitter.aurora.scheduler.base.Numbers;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A container for multiple resource vectors.
+ * TODO(wfarner): Collapse this in with Quotas.
+ */
+public class Resources {
+
+ public static final String CPUS = "cpus";
+ public static final String RAM_MB = "mem";
+ public static final String DISK_MB = "disk";
+ public static final String PORTS = "ports";
+
+ private static final Function<Range, Set<Integer>> RANGE_TO_MEMBERS =
+ new Function<Range, Set<Integer>>() {
+ @Override public Set<Integer> apply(Range range) {
+ return ContiguousSet.create(
+ com.google.common.collect.Range.closed((int) range.getBegin(), (int) range.getEnd()),
+ DiscreteDomain.integers());
+ }
+ };
+
+ private final double numCpus;
+ private final Amount<Long, Data> disk;
+ private final Amount<Long, Data> ram;
+ private final int numPorts;
+
+ /**
+ * Creates a new resources object.
+ *
+ * @param numCpus Number of CPUs.
+ * @param ram Amount of RAM.
+ * @param disk Amount of disk.
+ * @param numPorts Number of ports.
+ */
+ public Resources(double numCpus, Amount<Long, Data> ram, Amount<Long, Data> disk, int numPorts) {
+ this.numCpus = numCpus;
+ this.ram = checkNotNull(ram);
+ this.disk = checkNotNull(disk);
+ this.numPorts = numPorts;
+ }
+
+ /**
+ * Tests whether this bundle of resources is greater than or equal to another bundle of resources.
+ *
+ * @param other Resources being compared to.
+ * @return {@code true} if all resources in this bundle are greater than or equal to the
+ * equivalents from {@code other}, otherwise {@code false}.
+ */
+ public boolean greaterThanOrEqual(Resources other) {
+ return (numCpus >= other.numCpus)
+ && (disk.as(Data.MB) >= other.disk.as(Data.MB))
+ && (ram.as(Data.MB) >= other.ram.as(Data.MB))
+ && (numPorts >= other.numPorts);
+ }
+
+ /**
+ * Adapts this resources object to a list of mesos resources.
+ *
+ * @param selectedPorts The ports selected, to be applied as concrete task ranges.
+ * @return Mesos resources.
+ */
+ public List<Resource> toResourceList(Set<Integer> selectedPorts) {
+ ImmutableList.Builder<Resource> resourceBuilder =
+ ImmutableList.<Resource>builder()
+ .add(Resources.makeMesosResource(CPUS, numCpus))
+ .add(Resources.makeMesosResource(DISK_MB, disk.as(Data.MB)))
+ .add(Resources.makeMesosResource(RAM_MB, ram.as(Data.MB)));
+ if (selectedPorts.size() > 0) {
+ resourceBuilder.add(Resources.makeMesosRangeResource(Resources.PORTS, selectedPorts));
+ }
+
+ return resourceBuilder.build();
+ }
+
+ /**
+ * Convenience method for adapting to mesos resources without applying a port range.
+ *
+ * @see {@link #toResourceList(java.util.Set)}
+ * @return Mesos resources.
+ */
+ public List<Resource> toResourceList() {
+ return toResourceList(ImmutableSet.<Integer>of());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Resources)) {
+ return false;
+ }
+
+ Resources other = (Resources) o;
+ return new EqualsBuilder()
+ .append(numCpus, other.numCpus)
+ .append(ram, other.ram)
+ .append(disk, other.disk)
+ .append(numPorts, other.numPorts)
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(numCpus, ram, disk, numPorts);
+ }
+
+ /**
+ * Extracts the resources required from a task.
+ *
+ * @param task Task to get resources from.
+ * @return The resources required by the task.
+ */
+ public static Resources from(ITaskConfig task) {
+ checkNotNull(task);
+ return new Resources(
+ task.getNumCpus(),
+ Amount.of(task.getRamMb(), Data.MB),
+ Amount.of(task.getDiskMb(), Data.MB),
+ task.getRequestedPorts().size());
+ }
+
+ /**
+ * Extracts the resources specified in a list of resource objects.
+ *
+ * @param resources Resources to translate.
+ * @return The canonical resources.
+ */
+ public static Resources from(List<Resource> resources) {
+ checkNotNull(resources);
+ return new Resources(
+ getScalarValue(resources, CPUS),
+ Amount.of((long) getScalarValue(resources, RAM_MB), Data.MB),
+ Amount.of((long) getScalarValue(resources, DISK_MB), Data.MB),
+ getNumAvailablePorts(resources)
+ );
+ }
+
+ /**
+ * Extracts the resources available in a slave offer.
+ *
+ * @param offer Offer to get resources from.
+ * @return The resources available in the offer.
+ */
+ public static Resources from(Offer offer) {
+ checkNotNull(offer);
+ return new Resources(
+ getScalarValue(offer, CPUS),
+ Amount.of((long) getScalarValue(offer, RAM_MB), Data.MB),
+ Amount.of((long) getScalarValue(offer, DISK_MB), Data.MB),
+ getNumAvailablePorts(offer.getResourcesList()));
+ }
+
+ private static final Resources NO_RESOURCES =
+ new Resources(0, Amount.of(0L, Data.BITS), Amount.of(0L, Data.BITS), 0);
+
+ private static Resources none() {
+ return NO_RESOURCES;
+ }
+
+ /**
+ * a - b
+ */
+ public static Resources subtract(Resources a, Resources b) {
+ return new Resources(
+ a.getNumCpus() - b.getNumCpus(),
+ Amount.of(a.getRam().as(Data.MB) - b.getRam().as(Data.MB), Data.MB),
+ Amount.of(a.getDisk().as(Data.MB) - b.getDisk().as(Data.MB), Data.MB),
+ a.getNumPorts() - b.getNumPorts());
+ }
+
+ /**
+ * sum(a, b)
+ */
+ public static Resources sum(Resources a, Resources b) {
+ return sum(ImmutableList.of(a, b));
+ }
+
+ /**
+ * sum(rs)
+ */
+ public static Resources sum(Iterable<Resources> rs) {
+ Resources sum = none();
+
+ for (Resources r : rs) {
+ double numCpus = sum.getNumCpus() + r.getNumCpus();
+ Amount<Long, Data> disk =
+ Amount.of(sum.getDisk().as(Data.BYTES) + r.getDisk().as(Data.BYTES), Data.BYTES);
+ Amount<Long, Data> ram =
+ Amount.of(sum.getRam().as(Data.BYTES) + r.getRam().as(Data.BYTES), Data.BYTES);
+ int ports = sum.getNumPorts() + r.getNumPorts();
+ sum = new Resources(numCpus, ram, disk, ports);
+ }
+
+ return sum;
+ }
+
+ private static int getNumAvailablePorts(List<Resource> resource) {
+ int offeredPorts = 0;
+ for (Range range : getPortRanges(resource)) {
+ offeredPorts += 1 + (range.getEnd() - range.getBegin());
+ }
+ return offeredPorts;
+ }
+
+ private static double getScalarValue(Offer offer, String key) {
+ return getScalarValue(offer.getResourcesList(), key);
+ }
+
+ private static double getScalarValue(List<Resource> resources, String key) {
+ Resource resource = getResource(resources, key);
+ if (resource == null) {
+ return 0;
+ }
+
+ return resource.getScalar().getValue();
+ }
+
+ private static Resource getResource(List<Resource> resource, String key) {
+ return Iterables.find(resource, withName(key), null);
+ }
+
+ private static Predicate<Resource> withName(final String name) {
+ return new Predicate<Resource>() {
+ @Override public boolean apply(Resource resource) {
+ return resource.getName().equals(name);
+ }
+ };
+ }
+
+ private static Iterable<Range> getPortRanges(List<Resource> resources) {
+ Resource resource = getResource(resources, Resources.PORTS);
+ if (resource == null) {
+ return ImmutableList.of();
+ }
+
+ return resource.getRanges().getRangeList();
+ }
+
+ /**
+ * Creates a scalar mesos resource.
+ *
+ * @param name Name of the resource.
+ * @param value Value for the resource.
+ * @return A mesos resource.
+ */
+ public static Resource makeMesosResource(String name, double value) {
+ return Resource.newBuilder().setName(name).setType(Type.SCALAR)
+ .setScalar(Scalar.newBuilder().setValue(value)).build();
+ }
+
+ private static final Function<com.google.common.collect.Range<Integer>, Range> RANGE_TRANSFORM =
+ new Function<com.google.common.collect.Range<Integer>, Range>() {
+ @Override public Range apply(com.google.common.collect.Range<Integer> input) {
+ return Range.newBuilder()
+ .setBegin(input.lowerEndpoint())
+ .setEnd(input.upperEndpoint())
+ .build();
+ }
+ };
+
+ /**
+ * Creates a mesos resource of integer ranges.
+ *
+ * @param name Name of the resource
+ * @param values Values to translate into ranges.
+ * @return A mesos ranges resource.
+ */
+ static Resource makeMesosRangeResource(String name, Set<Integer> values) {
+ return Resource.newBuilder()
+ .setName(name)
+ .setType(Type.RANGES)
+ .setRanges(Ranges.newBuilder()
+ .addAllRange(Iterables.transform(Numbers.toRanges(values), RANGE_TRANSFORM)))
+ .build();
+ }
+
+ /**
+ * Number of CPUs.
+ *
+ * @return CPUs.
+ */
+ public double getNumCpus() {
+ return numCpus;
+ }
+
+ /**
+ * Disk amount.
+ *
+ * @return Disk.
+ */
+ public Amount<Long, Data> getDisk() {
+ return disk;
+ }
+
+ /**
+ * RAM amount.
+ *
+ * @return RAM.
+ */
+ public Amount<Long, Data> getRam() {
+ return ram;
+ }
+
+ /**
+ * Number of ports.
+ *
+ * @return Port count.
+ */
+ public int getNumPorts() {
+ return numPorts;
+ }
+
+ /**
+ * Thrown when there are insufficient resources to satisfy a request.
+ */
+ static class InsufficientResourcesException extends RuntimeException {
+ public InsufficientResourcesException(String message) {
+ super(message);
+ }
+ }
+
+ /**
+ * Attempts to grab {@code numPorts} from the given resource {@code offer}.
+ *
+ * @param offer The offer to grab ports from.
+ * @param numPorts The number of ports to grab.
+ * @return The set of ports grabbed.
+ * @throws InsufficientResourcesException if not enough ports were available.
+ */
+ public static Set<Integer> getPorts(Offer offer, int numPorts)
+ throws InsufficientResourcesException {
+
+ checkNotNull(offer);
+
+ if (numPorts == 0) {
+ return ImmutableSet.of();
+ }
+
+ List<Integer> availablePorts = Lists.newArrayList(Sets.newHashSet(
+ Iterables.concat(
+ Iterables.transform(getPortRanges(offer.getResourcesList()), RANGE_TO_MEMBERS))));
+
+ if (availablePorts.size() < numPorts) {
+ throw new InsufficientResourcesException(
+ String.format("Could not get %d ports from %s", numPorts, offer));
+ }
+
+ Collections.shuffle(availablePorts);
+ return ImmutableSet.copyOf(availablePorts.subList(0, numPorts));
+ }
+
+ /**
+ * A Resources object is greater than another iff _all_ of its resource components are greater
+ * or equal. A Resources object compares as equal if some but not all components are greater than
+ * or equal to the other.
+ */
+ public static final Ordering<Resources> RESOURCE_ORDER = new Ordering<Resources>() {
+ @Override public int compare(Resources left, Resources right) {
+ int diskC = left.getDisk().compareTo(right.getDisk());
+ int ramC = left.getRam().compareTo(right.getRam());
+ int portC = Integer.compare(left.getNumPorts(), right.getNumPorts());
+ int cpuC = Double.compare(left.getNumCpus(), right.getNumCpus());
+
+ FluentIterable<Integer> vector =
+ FluentIterable.from(ImmutableList.of(diskC, ramC, portC, cpuC));
+
+ if (vector.allMatch(IS_ZERO)) {
+ return 0;
+ }
+
+ if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_POSITIVE)) {
+ return 1;
+ }
+
+ if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_NEGATIVE)) {
+ return -1;
+ }
+
+ return 0;
+ }
+ };
+
+ private static final Predicate<Integer> IS_POSITIVE = new Predicate<Integer>() {
+ @Override public boolean apply(Integer input) {
+ return input > 0;
+ }
+ };
+
+ private static final Predicate<Integer> IS_NEGATIVE = new Predicate<Integer>() {
+ @Override public boolean apply(Integer input) {
+ return input < 0;
+ }
+ };
+
+ private static final Predicate<Integer> IS_ZERO = new Predicate<Integer>() {
+ @Override public boolean apply(Integer input) {
+ return input == 0;
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/configuration/SanitizedConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/SanitizedConfiguration.java b/src/main/java/org/apache/aurora/scheduler/configuration/SanitizedConfiguration.java
new file mode 100644
index 0000000..890acbb
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/configuration/SanitizedConfiguration.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.configuration;
+
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Functions;
+import com.google.common.base.Objects;
+import com.google.common.collect.ContiguousSet;
+import com.google.common.collect.DiscreteDomain;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * Wrapper for a configuration that has been fully-sanitized and populated with defaults.
+ */
+public final class SanitizedConfiguration {
+
+ private final IJobConfiguration sanitized;
+ private final Map<Integer, ITaskConfig> tasks;
+
+ /**
+ * Constructs a SanitizedConfiguration object and populates the set of {@link ITaskConfig}s for
+ * the provided config.
+ *
+ * @param sanitized A sanitized configuration.
+ */
+ @VisibleForTesting
+ public SanitizedConfiguration(IJobConfiguration sanitized) {
+ this.sanitized = sanitized;
+ this.tasks = Maps.toMap(
+ ContiguousSet.create(
+ Range.closedOpen(0, sanitized.getInstanceCount()),
+ DiscreteDomain.integers()),
+ Functions.constant(sanitized.getTaskConfig()));
+ }
+
+ /**
+ * Wraps an unsanitized job configuration.
+ *
+ * @param unsanitized Unsanitized configuration to sanitize/populate and wrap.
+ * @return A wrapper containing the sanitized configuration.
+ * @throws TaskDescriptionException If the configuration is invalid.
+ */
+ public static SanitizedConfiguration fromUnsanitized(IJobConfiguration unsanitized)
+ throws TaskDescriptionException {
+
+ return new SanitizedConfiguration(ConfigurationManager.validateAndPopulate(unsanitized));
+ }
+
+ public IJobConfiguration getJobConfig() {
+ return sanitized;
+ }
+
+ // TODO(William Farner): Rework this API now that all configs are identical.
+ public Map<Integer, ITaskConfig> getTaskConfigs() {
+ return tasks;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof SanitizedConfiguration)) {
+ return false;
+ }
+
+ SanitizedConfiguration other = (SanitizedConfiguration) o;
+
+ return Objects.equal(sanitized, other.sanitized);
+ }
+
+ @Override
+ public int hashCode() {
+ return sanitized.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return sanitized.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/cron/CronException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/cron/CronException.java b/src/main/java/org/apache/aurora/scheduler/cron/CronException.java
new file mode 100644
index 0000000..c29a578
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/cron/CronException.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.cron;
+
+/**
+ * Exception class to signal a failure in the underlying cron implementation.
+ */
+public class CronException extends Exception {
+ public CronException(String msg) {
+ super(msg);
+ }
+
+ public CronException(String msg, Throwable t) {
+ super(msg, t);
+ }
+
+ public CronException(Throwable t) {
+ super(t);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/cron/CronPredictor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/cron/CronPredictor.java b/src/main/java/org/apache/aurora/scheduler/cron/CronPredictor.java
new file mode 100644
index 0000000..d01e2f8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/cron/CronPredictor.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.cron;
+
+import java.util.Date;
+
+/**
+ * A utility function that predicts a cron run given a schedule.
+ */
+public interface CronPredictor {
+ /**
+ * Predicts the next date at which a cron schedule will trigger.
+ *
+ * @param schedule Cron schedule to predict the next time for.
+ * @return A prediction for the next time a cron will run.
+ */
+ Date predictNextRun(String schedule);
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/cron/CronScheduler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/cron/CronScheduler.java b/src/main/java/org/apache/aurora/scheduler/cron/CronScheduler.java
new file mode 100644
index 0000000..0ea9b65
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/cron/CronScheduler.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.cron;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Optional;
+
+/**
+ * An execution manager that executes work on a cron schedule.
+ */
+public interface CronScheduler {
+ /**
+ * Schedules a task on a cron schedule.
+ *
+ * @param schedule Cron-style schedule.
+ * @param task Work to run when on the cron schedule.
+ * @return A unique ID to identify the scheduled cron task.
+ * @throws CronException when there was a failure to schedule, for example if {@code schedule}
+ * is not a valid input.
+ * @throws IllegalStateException If the cron scheduler is not currently running.
+ */
+ String schedule(String schedule, Runnable task) throws CronException, IllegalStateException;
+
+ /**
+ * Removes a scheduled cron item.
+ *
+ * @param key Key previously returned from {@link #schedule(String, Runnable)}.
+ * @throws IllegalStateException If the cron scheduler is not currently running.
+ */
+ void deschedule(String key) throws IllegalStateException;
+
+ /**
+ * Gets the cron schedule associated with a scheduling key.
+ *
+ * @param key Key previously returned from {@link #schedule(String, Runnable)}.
+ * @return The task's cron schedule, if a matching task was found.
+ * @throws IllegalStateException If the cron scheduler is not currently running.
+ */
+ Optional<String> getSchedule(String key) throws IllegalStateException;
+
+ /**
+ * Block until fully initialized. It is an error to call start twice. Prior to calling start,
+ * all other methods of this interface may throw {@link IllegalStateException}. The underlying
+ * implementation should not spawn threads or connect to databases prior to invocation of
+ * {@link #start()}.
+ *
+ * @throws IllegalStateException If called twice.
+ */
+ void start() throws IllegalStateException;
+
+ /**
+ * Block until stopped. Generally this means that underlying resources are freed, threads are
+ * terminated, and any bookkeeping state is persisted. If {@link #stop()} has already been called
+ * by another thread, {@link #stop()} either blocks until completion or returns immediately.
+ *
+ * @throws CronException If there was a problem stopping the scheduler, for example if it was not
+ * started.
+ */
+ void stop() throws CronException;
+
+ /**
+ * Checks to see if the scheduler would be accepted by the underlying scheduler.
+ *
+ * @param schedule Cron scheduler to validate.
+ * @return {@code true} if the schedule is valid.
+ */
+ boolean isValidSchedule(@Nullable String schedule);
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronModule.java b/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronModule.java
new file mode 100644
index 0000000..d1c5419
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronModule.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.cron.noop;
+
+import javax.inject.Singleton;
+
+import com.google.inject.AbstractModule;
+
+import com.twitter.aurora.scheduler.cron.CronPredictor;
+import com.twitter.aurora.scheduler.cron.CronScheduler;
+
+/**
+ * A Module to wire up a cron scheduler that does not actually schedule cron jobs.
+ *
+ * This class exists as a short term hack to get around a license compatibility issue - Real
+ * Implementation (TM) coming soon.
+ */
+public class NoopCronModule extends AbstractModule {
+ @Override
+ protected void configure() {
+ bind(CronScheduler.class).to(NoopCronScheduler.class);
+ bind(NoopCronScheduler.class).in(Singleton.class);
+
+ bind(CronPredictor.class).to(NoopCronPredictor.class);
+ bind(NoopCronPredictor.class).in(Singleton.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronPredictor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronPredictor.java b/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronPredictor.java
new file mode 100644
index 0000000..a779d2b
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronPredictor.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.cron.noop;
+
+import java.util.Date;
+
+import com.twitter.aurora.scheduler.cron.CronPredictor;
+
+/**
+ * A cron predictor that always suggests that the next run is Unix epoch time.
+ *
+ * This class exists as a short term hack to get around a license compatibility issue - Real
+ * Implementation (TM) coming soon.
+ */
+class NoopCronPredictor implements CronPredictor {
+ @Override
+ public Date predictNextRun(String schedule) {
+ return new Date(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronScheduler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronScheduler.java b/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronScheduler.java
new file mode 100644
index 0000000..2893a6a
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/cron/noop/NoopCronScheduler.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.cron.noop;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Sets;
+
+import com.twitter.aurora.scheduler.cron.CronException;
+import com.twitter.aurora.scheduler.cron.CronScheduler;
+
+/**
+ * A cron scheduler that accepts cron jobs but never runs them. Useful if you want to hook up an
+ * external triggering mechanism (e.g. a system cron job that calls the startCronJob RPC manually
+ * on an interval).
+ *
+ * This class exists as a short term hack to get around a license compatibility issue - Real
+ * Implementation (TM) coming soon.
+ */
+class NoopCronScheduler implements CronScheduler {
+ private static final Logger LOG = Logger.getLogger(NoopCronScheduler.class.getName());
+
+ // Keep a list of schedules we've seen.
+ private final Set<String> schedules = Collections.synchronizedSet(Sets.<String>newHashSet());
+
+ @Override
+ public String schedule(String schedule, Runnable task) {
+ schedules.add(schedule);
+
+ LOG.warning(String.format(
+ "NO-OP cron scheduler is in use! %s with schedule %s WILL NOT be automatically triggered!",
+ task,
+ schedule));
+
+ return schedule;
+ }
+
+ @Override
+ public void deschedule(String key) throws IllegalStateException {
+ schedules.remove(key);
+ }
+
+ @Override
+ public Optional<String> getSchedule(String key) throws IllegalStateException {
+ return schedules.contains(key)
+ ? Optional.of(key)
+ : Optional.<String>absent();
+ }
+
+ @Override
+ public void start() throws IllegalStateException {
+ LOG.warning("NO-OP cron scheduler is in use. Cron jobs submitted will not be triggered!");
+ }
+
+ @Override
+ public void stop() throws CronException {
+ // No-op.
+ }
+
+ @Override
+ public boolean isValidSchedule(@Nullable String schedule) {
+ // Accept everything.
+ return schedule != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/cron/testing/AbstractCronIT.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/cron/testing/AbstractCronIT.java b/src/main/java/org/apache/aurora/scheduler/cron/testing/AbstractCronIT.java
new file mode 100644
index 0000000..6bfc909
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/cron/testing/AbstractCronIT.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.cron.testing;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.Test;
+
+import com.twitter.aurora.scheduler.cron.CronPredictor;
+import com.twitter.aurora.scheduler.cron.CronScheduler;
+import com.twitter.common.testing.easymock.EasyMockTest;
+
+import static org.junit.Assert.assertTrue;
+
+import static com.twitter.aurora.gen.test.testConstants.VALID_CRON_SCHEDULES;
+
+/**
+ * Abstract test to verify conformance with the {@link CronScheduler} interface.
+ */
+public abstract class AbstractCronIT extends EasyMockTest {
+ /**
+ * Child should return an instance of the {@link CronScheduler} test under test here.
+ */
+ protected abstract CronScheduler makeCronScheduler() throws Exception;
+
+ /**
+ * Child should configure expectations for a scheduler start.
+ */
+ protected abstract void expectStartCronScheduler();
+
+ /**
+ * Child should configure expectations for a scheduler stop.
+ */
+ protected abstract void expectStopCronScheduler();
+
+ /**
+ * Child should return an instance of the {@link CronPredictor} under test here.
+ */
+ protected abstract CronPredictor makeCronPredictor() throws Exception;
+
+ @Test
+ public void testCronSchedulerLifecycle() throws Exception {
+ CronScheduler scheduler = makeCronScheduler();
+
+ expectStartCronScheduler();
+ expectStopCronScheduler();
+
+ control.replay();
+
+ scheduler.start();
+ final CountDownLatch cronRan = new CountDownLatch(1);
+ scheduler.schedule("* * * * *", new Runnable() {
+ @Override public void run() {
+ cronRan.countDown();
+ }
+ });
+ cronRan.await();
+ scheduler.stop();
+ }
+
+ @Test
+ public void testCronPredictorAcceptsValidSchedules() throws Exception {
+ control.replay();
+
+ CronPredictor cronPredictor = makeCronPredictor();
+ for (String schedule : VALID_CRON_SCHEDULES) {
+ cronPredictor.predictNextRun(schedule);
+ }
+ }
+
+ @Test
+ public void testCronScheduleValidatorAcceptsValidSchedules() throws Exception {
+ CronScheduler cron = makeCronScheduler();
+
+ control.replay();
+
+ for (String schedule : VALID_CRON_SCHEDULES) {
+ assertTrue(String.format("Cron schedule %s should validate.", schedule),
+ cron.isValidSchedule(schedule));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/events/NotifyingMethodInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/events/NotifyingMethodInterceptor.java b/src/main/java/org/apache/aurora/scheduler/events/NotifyingMethodInterceptor.java
new file mode 100644
index 0000000..2656766
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/events/NotifyingMethodInterceptor.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.events;
+
+import java.lang.reflect.Method;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Preconditions;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+
+import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.Event;
+import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
+import com.twitter.common.base.Closure;
+
+/**
+ * A method interceptor that sends pubsub notifications before and/or after a method annotated
+ * with {@link SendNotification}
+ * is invoked.
+ */
+class NotifyingMethodInterceptor implements MethodInterceptor {
+ private static final Logger LOG = Logger.getLogger(NotifyingMethodInterceptor.class.getName());
+
+ @Inject
+ private Closure<PubsubEvent> eventSink;
+
+ private void maybeFire(Event event) {
+ if (event != Event.None) {
+ eventSink.execute(event.getEvent());
+ }
+ }
+
+ @Override
+ public Object invoke(MethodInvocation invocation) throws Throwable {
+ Preconditions.checkNotNull(eventSink, "Event sink has not yet been set.");
+
+ Method method = invocation.getMethod();
+ SendNotification sendNotification = method.getAnnotation(SendNotification.class);
+ if (sendNotification == null) {
+ LOG.warning("Interceptor should not match methods without @"
+ + SendNotification.class.getSimpleName());
+ return invocation.proceed();
+ }
+
+ maybeFire(sendNotification.before());
+ Object result = invocation.proceed();
+ maybeFire(sendNotification.after());
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java b/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java
new file mode 100644
index 0000000..ffb952c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/events/NotifyingSchedulingFilter.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.events;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.Set;
+
+import javax.inject.Inject;
+
+import com.google.inject.BindingAnnotation;
+
+import com.twitter.aurora.scheduler.ResourceSlot;
+import com.twitter.aurora.scheduler.events.PubsubEvent.Vetoed;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.base.Closure;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A decorating scheduling filter that sends an event when a scheduling assignment is vetoed.
+ */
+class NotifyingSchedulingFilter implements SchedulingFilter {
+
+ /**
+ * Binding annotation that the underlying {@link SchedulingFilter} must be bound with.
+ */
+ @BindingAnnotation
+ @Target({FIELD, PARAMETER, METHOD}) @Retention(RUNTIME)
+ public @interface NotifyDelegate { }
+
+ private final SchedulingFilter delegate;
+ private final Closure<PubsubEvent> eventSink;
+
+ @Inject
+ NotifyingSchedulingFilter(
+ @NotifyDelegate SchedulingFilter delegate,
+ Closure<PubsubEvent> eventSink) {
+
+ this.delegate = checkNotNull(delegate);
+ this.eventSink = checkNotNull(eventSink);
+ }
+
+ @Override
+ public Set<Veto> filter(ResourceSlot offer, String slaveHost, ITaskConfig task, String taskId) {
+ Set<Veto> vetoes = delegate.filter(offer, slaveHost, task, taskId);
+ if (!vetoes.isEmpty()) {
+ eventSink.execute(new Vetoed(taskId, vetoes));
+ }
+
+ return vetoes;
+ }
+}
[22/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/common/path.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/path.py b/src/main/python/apache/thermos/common/path.py
new file mode 100644
index 0000000..28e07ac
--- /dev/null
+++ b/src/main/python/apache/thermos/common/path.py
@@ -0,0 +1,101 @@
+import os
+
+
+class TaskPath(object):
+ """
+ Handle the resolution / detection of the path structure for thermos tasks.
+
+ This is used by the runner to determine where it should be dumping checkpoints and writing
+ stderr/stdout, and by the observer to determine how to detect the running tasks on the system.
+
+ Examples:
+ pathspec = TaskPath(root = "/var/run/thermos")
+ ^ substitution dictionary for DIR_TEMPLATE
+
+
+ which template to acquire
+ v
+ pathspec.given(task_id = "12345-thermos-wickman-23", state='active').getpath("task_path")
+ ^
+ further substitutions DIR_TEMPLATE
+
+
+ As a detection mechanism:
+ path_glob = pathspec.given(task_id = "*").getpath(task_type)
+ matching_paths = glob.glob(path_glob)
+
+ path_re = pathspec.given(task_id = "(\S+)").getpath(task_type)
+ path_re = re.compile(path_re)
+
+ ids = []
+ for path in matching_paths:
+ matched_blobs = path_re.match(path).groups()
+ ids.append(int(matched_blobs[0]))
+ return ids
+ """
+
+ class UnknownPath(Exception): pass
+ class UnderspecifiedPath(Exception): pass
+
+ DEFAULT_CHECKPOINT_ROOT = "/var/run/thermos"
+ KNOWN_KEYS = [ 'root', 'task_id', 'state', 'process', 'run', 'log_dir' ]
+ LEGACY_KNOWN_KEYS = KNOWN_KEYS[:-1]
+
+ DIR_TEMPLATE = {
+ 'task_path': ['%(root)s', 'tasks', '%(state)s', '%(task_id)s'],
+ 'checkpoint_path': ['%(root)s', 'checkpoints', '%(task_id)s'],
+ 'runner_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'runner'],
+ 'process_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'coordinator.%(process)s'],
+ 'process_logbase': ['%(log_dir)s'],
+ 'process_logdir': ['%(log_dir)s', '%(process)s', '%(run)s']
+ }
+
+ LEGACY_DIR_TEMPLATE = DIR_TEMPLATE.copy()
+ LEGACY_DIR_TEMPLATE.update(
+ process_logbase = ['%(root)s', 'logs', '%(task_id)s'],
+ process_logdir = ['%(root)s', 'logs', '%(task_id)s', '%(process)s', '%(run)s']
+ )
+
+ def __init__(self, **kw):
+ self._filename = None
+ # initialize with self-interpolating values
+ if kw.get('root') is None:
+ kw['root'] = self.DEFAULT_CHECKPOINT_ROOT
+ # Before log_dir was added explicitly to RunnerHeader, it resolved to %(root)s/logs
+ if kw.get('log_dir'):
+ self._template, keys = self.DIR_TEMPLATE, self.KNOWN_KEYS
+ else:
+ self._template, keys = self.LEGACY_DIR_TEMPLATE, self.LEGACY_KNOWN_KEYS
+ self._data = dict((key, '%%(%s)s' % key) for key in keys)
+ self._data.update(kw)
+
+ def given(self, **kw):
+ """ Perform further interpolation of the templates given the kwargs """
+ eval_dict = dict(self._data) # copy
+ eval_dict.update(kw)
+ tp = TaskPath(**eval_dict)
+ tp._filename = self._filename
+ return tp
+
+ def with_filename(self, filename):
+ """ Return a TaskPath with the specific filename appended to the end of the path """
+ wp = TaskPath(**self._data)
+ wp._filename = filename
+ return wp
+
+ def getpath(self, pathname):
+ if pathname not in self._template:
+ raise self.UnknownPath("Internal error, unknown id: %s" % pathname)
+ path = self._template[pathname][:]
+
+ if self._filename:
+ path += [self._filename]
+ path = os.path.join(*path)
+ interpolated_path = path % self._data
+ try:
+ _ = interpolated_path % {}
+ except KeyError:
+ raise TaskPath.UnderspecifiedPath(
+ "Tried to interpolate path with insufficient variables: %s as %s" % (
+ pathname, interpolated_path))
+ return interpolated_path
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/common/planner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/planner.py b/src/main/python/apache/thermos/common/planner.py
new file mode 100644
index 0000000..a886dad
--- /dev/null
+++ b/src/main/python/apache/thermos/common/planner.py
@@ -0,0 +1,304 @@
+"""Planners to schedule processes within Tasks.
+
+TaskPlanner:
+ - a daemon process can depend upon a regular process
+ - a regular process cannot depend upon a daemon process
+ - a non-ephemeral process cannot depend upon an ephemeral process
+"""
+
+from collections import defaultdict, namedtuple
+import copy
+from functools import partial
+import sys
+import time
+
+class Planner(object):
+ """
+ Given a set of process names and a graph of dependencies between them, determine
+ what can run predicated upon process completions.
+ """
+ class InvalidSchedule(Exception): pass
+
+ @staticmethod
+ def filter_runnable(processes, dependencies):
+ return set(process for process in processes if not dependencies.get(process))
+
+ @staticmethod
+ def filter_dependencies(dependencies, given=frozenset()):
+ """
+ Provided a map of process => list of process :dependencies, and a set of satisfied
+ prior processes in :given, return the new map of dependencies with priors removed.
+ """
+ dependencies = copy.deepcopy(dependencies)
+ for process_set in dependencies.values():
+ process_set -= given
+ return dependencies
+
+ @staticmethod
+ def satisfiable(processes, dependencies):
+ """
+ Given a set of processes and a dependency map, determine if this is a consistent
+ schedule without cycles.
+ """
+ processes = copy.copy(processes)
+ dependencies = copy.deepcopy(dependencies)
+
+ scheduling = True
+ while scheduling:
+ scheduling = False
+ runnables = Planner.filter_runnable(processes, dependencies)
+ if runnables:
+ scheduling = True
+ processes -= runnables
+ dependencies = Planner.filter_dependencies(dependencies, given=runnables)
+ return len(processes) == 0
+
+ def __init__(self, processes, dependencies):
+ self._processes = set(processes)
+ self._dependencies = dict((process, set(dependencies.get(process, [])))
+ for process in self._processes)
+ if not Planner.satisfiable(self._processes, self._dependencies):
+ raise Planner.InvalidSchedule("Cycles detected in the task schedule!")
+ self._running = set()
+ self._finished = set()
+ self._failed = set()
+
+ @property
+ def runnable(self):
+ return Planner.filter_runnable(self._processes - self._running - self._finished - self._failed,
+ Planner.filter_dependencies(self._dependencies, given=self._finished))
+
+ @property
+ def processes(self):
+ return set(self._processes)
+
+ @property
+ def running(self):
+ return set(self._running)
+
+ @property
+ def finished(self):
+ return set(self._finished)
+
+ @property
+ def failed(self):
+ return set(self._failed)
+
+ def reset(self, process):
+ assert process in self._running
+ assert process not in self._finished
+ assert process not in self._failed
+ self._running.discard(process)
+
+ def set_running(self, process):
+ assert process not in self._failed
+ assert process not in self._finished
+ assert process in self._running or process in self.runnable
+ self._running.add(process)
+
+ def set_finished(self, process):
+ assert process in self._running
+ assert process not in self._failed
+ self._running.discard(process)
+ self._finished.add(process)
+
+ def set_failed(self, process):
+ assert process in self._running
+ assert process not in self._finished
+ self._running.discard(process)
+ self._failed.add(process)
+
+ def is_complete(self):
+ return self._finished.union(self._failed) == self._processes
+
+
+TaskAttributes = namedtuple('TaskAttributes', 'min_duration is_daemon max_failures is_ephemeral')
+
+class TaskPlanner(object):
+ """
+ A planner for the processes part of a Thermos task, taking into account ephemeral and daemon
+ bits, in addition to duration restrictions [and failure limits?].
+
+ is_daemon
+ .----------------------------------------------------.
+ | |
+ | clock gate .----------------------. |
+ | .---------------> | runnable && !waiting | |
+ v | `----------------------' |
+ .----------. | |
+ | runnable | | set_running |
+ `----------' v |
+ ^ forget .---------. | !is_daemon .----------.
+ `------------------------| running |--------------+------------> | finished |
+ ^ `---------' add_success `----------'
+ | |
+ | under failure limit | add_failure
+ `-----------------------------+
+ | past failure limit
+ v
+ .--------.
+ | failed |
+ `--------'
+ """
+ InvalidSchedule = Planner.InvalidSchedule
+ INFINITY = sys.float_info.max
+ TOTAL_RUN_LIMIT = sys.maxsize
+
+ @staticmethod
+ def extract_dependencies(task, process_filter=None):
+ """
+ Construct a set of processes and the process dependencies from a Thermos Task.
+ """
+ process_map = dict((process.name().get(), process)
+ for process in filter(process_filter, task.processes()))
+ processes = set(process_map)
+ dependencies = defaultdict(set)
+ if task.has_constraints():
+ for constraint in task.constraints():
+ # handle process orders
+ process_names = constraint.order().get()
+ process_name_set = set(process_names)
+ # either all process_names must be in processes or none should be
+ if process_name_set.issubset(processes) == process_name_set.isdisjoint(processes):
+ raise TaskPlanner.InvalidSchedule('Invalid process dependencies!')
+ if not process_name_set.issubset(processes):
+ continue
+ for k in range(1, len(process_names)):
+ pnk, pnk1 = process_names[k], process_names[k-1]
+ if process_map[pnk1].daemon().get():
+ raise TaskPlanner.InvalidSchedule(
+ 'Process %s may not depend upon daemon process %s' % (pnk, pnk1))
+ if not process_map[pnk].ephemeral().get() and process_map[pnk1].ephemeral().get():
+ raise TaskPlanner.InvalidSchedule(
+ 'Non-ephemeral process %s may not depend upon ephemeral process %s' % (pnk, pnk1))
+ dependencies[pnk].add(pnk1)
+ return (processes, dependencies)
+
+ def __init__(self, task, clock=time, process_filter=None):
+ self._filter = process_filter
+ assert self._filter is None or callable(self._filter), (
+ 'TaskPlanner must be given callable process filter.')
+ self._planner = Planner(*self.extract_dependencies(task, self._filter))
+ self._clock = clock
+ self._last_terminal = {} # process => timestamp of last terminal state
+ self._failures = defaultdict(int)
+ self._successes = defaultdict(int)
+ self._attributes = {}
+ self._ephemerals = set(process.name().get() for process in task.processes()
+ if (self._filter is None or self._filter(process)) and process.ephemeral().get())
+
+ for process in filter(self._filter, task.processes()):
+ self._attributes[process.name().get()] = TaskAttributes(
+ is_daemon=bool(process.daemon().get()),
+ is_ephemeral=bool(process.ephemeral().get()),
+ max_failures=process.max_failures().get(),
+ min_duration=process.min_duration().get())
+
+ def get_wait(self, process, timestamp=None):
+ now = timestamp if timestamp is not None else self._clock.time()
+ if process not in self._last_terminal:
+ return 0
+ return self._attributes[process].min_duration - (now - self._last_terminal[process])
+
+ def is_ready(self, process, timestamp=None):
+ return self.get_wait(process, timestamp) <= 0
+
+ def is_waiting(self, process, timestamp=None):
+ return not self.is_ready(process, timestamp)
+
+ @property
+ def runnable(self):
+ """A list of processes that are runnable w/o duration restrictions."""
+ return self.runnable_at(self._clock.time())
+
+ @property
+ def waiting(self):
+ """A list of processes that are waiting w/o duration restrictions."""
+ return self.waiting_at(self._clock.time())
+
+ def runnable_at(self, timestamp):
+ return set(filter(partial(self.is_ready, timestamp=timestamp), self._planner.runnable))
+
+ def waiting_at(self, timestamp):
+ return set(filter(partial(self.is_waiting, timestamp=timestamp), self._planner.runnable))
+
+ def min_wait(self, timestamp=None):
+ """Return the current wait time for the next process to become runnable, 0 if something is ready
+ immediately, or sys.float.max if there are no waiters."""
+ if self.runnable_at(timestamp if timestamp is not None else self._clock.time()):
+ return 0
+ waits = [self.get_wait(waiter, timestamp) for waiter in self.waiting_at(timestamp)]
+ return min(waits) if waits else self.INFINITY
+
+ def set_running(self, process):
+ self._planner.set_running(process)
+
+ def add_failure(self, process, timestamp=None):
+ """Increment the failure count of a process, and reset it to runnable if maximum number of
+ failures has not been reached, or mark it as failed otherwise (ephemeral processes do not
+ count towards the success of a task, and are hence marked finished instead)"""
+ timestamp = timestamp if timestamp is not None else self._clock.time()
+ self._last_terminal[process] = timestamp
+ self._failures[process] += 1
+ self.failure_transition(process)
+
+ def has_reached_run_limit(self, process):
+ return (self._successes[process] + self._failures[process]) >= self.TOTAL_RUN_LIMIT
+
+ def failure_transition(self, process):
+ if self.has_reached_run_limit(process):
+ self._planner.set_failed(process)
+ return
+
+ if self._attributes[process].max_failures == 0 or (
+ self._failures[process] < self._attributes[process].max_failures):
+ self._planner.reset(process)
+ elif self._attributes[process].is_ephemeral:
+ self._planner.set_finished(process)
+ else:
+ self._planner.set_failed(process)
+
+ def add_success(self, process, timestamp=None):
+ """Reset a process to runnable if it is a daemon, or mark it as finished otherwise."""
+ timestamp = timestamp if timestamp is not None else self._clock.time()
+ self._last_terminal[process] = timestamp
+ self._successes[process] += 1
+ self.success_transition(process)
+
+ def success_transition(self, process):
+ if self.has_reached_run_limit(process):
+ self._planner.set_failed(process)
+ return
+
+ if not self._attributes[process].is_daemon:
+ self._planner.set_finished(process)
+ else:
+ self._planner.reset(process)
+
+ def set_failed(self, process):
+ """Force a process to be in failed state. E.g. kill -9 and you want it pinned failed."""
+ self._planner.set_failed(process)
+
+ def lost(self, process):
+ """Mark a process as lost. This sets its runnable state back to the previous runnable
+ state and does not increment its failure count."""
+ self._planner.reset(process)
+
+ def is_complete(self):
+ """A task is complete if all ordinary processes are finished or failed (there may still be
+ running ephemeral processes)"""
+ terminals = self._planner.finished.union(self._planner.failed).union(self._ephemerals)
+ return self._planner.processes == terminals
+
+ # TODO(wickman) Should we consider subclassing again?
+ @property
+ def failed(self):
+ return self._planner.failed
+
+ @property
+ def running(self):
+ return self._planner.running
+
+ @property
+ def finished(self):
+ return self._planner.finished
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/BUILD b/src/main/python/apache/thermos/config/BUILD
new file mode 100644
index 0000000..57e7f0d
--- /dev/null
+++ b/src/main/python/apache/thermos/config/BUILD
@@ -0,0 +1,24 @@
+import os
+
+python_library(
+ name = 'schema',
+ sources = globs('*.py'),
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('src/main/python/twitter/thermos:pystachio'),
+ pants('src/main/python/twitter/thermos/common:planner')
+ ],
+)
+
+python_library(
+ name = 'config',
+ dependencies = [
+ pants(':schema'),
+ pants('src/main/python/twitter/thermos/common'), # cover common:planner
+ ],
+ provides = setup_py(
+ name = 'twitter.thermos.config',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ description = 'Thermos configuration schema and loader.',
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/__init__.py b/src/main/python/apache/thermos/config/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/bin/config_load.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/bin/config_load.py b/src/main/python/apache/thermos/config/bin/config_load.py
new file mode 100644
index 0000000..c0680ec
--- /dev/null
+++ b/src/main/python/apache/thermos/config/bin/config_load.py
@@ -0,0 +1,47 @@
+import copy
+import sys
+import json
+import pprint
+
+from twitter.common import app
+from twitter.thermos.config.loader import ThermosConfigLoader
+
+def main(args):
+ """
+ Given .thermos configs, loads them and prints out information about them.
+ """
+
+ if len(args) == 0:
+ app.help()
+
+ for arg in args:
+ print '\nparsing %s\n' % arg
+ tc = ThermosConfigLoader.load(arg)
+
+ for task_wrapper in tc.tasks():
+ task = task_wrapper.task
+ if not task.has_name():
+ print 'Found unnamed task! Skipping...'
+ continue
+
+ print 'Task: %s [check: %s]' % (task.name(), task.check())
+ if not task.processes():
+ print ' No processes.'
+ else:
+ print ' Processes:'
+ for proc in task.processes():
+ print ' %s' % proc
+
+ ports = task_wrapper.ports()
+ if not ports:
+ print ' No unbound ports.'
+ else:
+ print ' Ports:'
+ for port in ports:
+ print ' %s' % port
+
+ print 'raw:'
+ pprint.pprint(json.loads(task_wrapper.to_json()))
+
+app.set_usage("%s config1 config2 ..." % app.name())
+app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/bin/config_repl.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/bin/config_repl.py b/src/main/python/apache/thermos/config/bin/config_repl.py
new file mode 100644
index 0000000..4a5c797
--- /dev/null
+++ b/src/main/python/apache/thermos/config/bin/config_repl.py
@@ -0,0 +1,3 @@
+from twitter.thermos.config.schema import *
+from code import interact
+interact('Thermos Config REPL', local=locals())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/dsl.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/dsl.py b/src/main/python/apache/thermos/config/dsl.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/loader.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/loader.py b/src/main/python/apache/thermos/config/loader.py
new file mode 100644
index 0000000..54721c3
--- /dev/null
+++ b/src/main/python/apache/thermos/config/loader.py
@@ -0,0 +1,179 @@
+import copy
+import json
+import os
+import re
+import textwrap
+
+from twitter.common.dirutil import safe_open
+from twitter.common.lang import Compatibility
+from twitter.thermos.common.planner import TaskPlanner
+from twitter.thermos.config.schema import Task
+
+from pystachio import Ref
+from pystachio.config import Config
+
+
+class PortExtractor(object):
+ class InvalidPorts(Exception): pass
+
+ @staticmethod
+ def extract(obj):
+ port_scope = Ref.from_address('thermos.ports')
+ _, uninterp = obj.interpolate()
+ ports = []
+ for ref in uninterp:
+ subscope = port_scope.scoped_to(ref)
+ if subscope is not None:
+ if not subscope.is_index():
+ raise PortExtractor.InvalidPorts(
+ 'Bad port specification "%s" (should be of form "thermos.ports[name]"' % ref.address())
+ ports.append(subscope.action().value)
+ return ports
+
+
+class ThermosProcessWrapper(object):
+ # >=1 characters && anything but NULL and '/'
+ VALID_PROCESS_NAME_RE = re.compile(r'^[^./][^/]*$')
+ class InvalidProcess(Exception): pass
+
+ def __init__(self, process):
+ self._process = process
+
+ def ports(self):
+ try:
+ return PortExtractor.extract(self._process)
+ except PortExtractor.InvalidPorts:
+ raise self.InvalidProcess('Process has invalid ports scoping!')
+
+ @staticmethod
+ def assert_valid_process_name(name):
+ if not ThermosProcessWrapper.VALID_PROCESS_NAME_RE.match(name):
+ raise ThermosProcessWrapper.InvalidProcess('Invalid process name: %s' % name)
+
+
+class ThermosTaskWrapper(object):
+ class InvalidTask(Exception): pass
+
+ def __init__(self, task, bindings=None, strict=True):
+ if bindings:
+ task = task.bind(*bindings)
+ if not task.check().ok() and strict:
+ raise ThermosTaskWrapper.InvalidTask(task.check().message())
+ self._task = task
+
+ @property
+ def task(self):
+ return self._task
+
+ def ports(self):
+ ti, _ = self._task.interpolate()
+ ports = set()
+ if ti.has_processes():
+ for process in ti.processes():
+ try:
+ ports.update(ThermosProcessWrapper(process).ports())
+ except ThermosProcessWrapper.InvalidProcess:
+ raise self.InvalidTask('Task has invalid process: %s' % process)
+ return ports
+
+ def to_json(self):
+ return json.dumps(self._task.get())
+
+ def to_file(self, filename):
+ ti, _ = self._task.interpolate()
+ with safe_open(filename, 'w') as fp:
+ json.dump(ti.get(), fp)
+
+ @staticmethod
+ def from_file(filename, **kw):
+ try:
+ with safe_open(filename) as fp:
+ task = Task.json_load(fp)
+ return ThermosTaskWrapper(task, **kw)
+ except Exception as e:
+ return None
+
+
+# TODO(wickman) These should be validators pushed onto ThermosConfigLoader.plugins
+class ThermosTaskValidator(object):
+ class InvalidTaskError(Exception): pass
+
+ @classmethod
+ def assert_valid_task(cls, task):
+ cls.assert_valid_names(task)
+ cls.assert_typecheck(task)
+ cls.assert_valid_plan(task)
+
+ @classmethod
+ def assert_valid_plan(cls, task):
+ try:
+ TaskPlanner(task, process_filter=lambda proc: proc.final().get() == False)
+ TaskPlanner(task, process_filter=lambda proc: proc.final().get() == True)
+ except TaskPlanner.InvalidSchedule as e:
+ raise cls.InvalidTaskError('Task has invalid plan: %s' % e)
+
+ @classmethod
+ def assert_valid_names(cls, task):
+ for process in task.processes():
+ name = process.name().get()
+ try:
+ ThermosProcessWrapper.assert_valid_process_name(name)
+ except ThermosProcessWrapper.InvalidProcess as e:
+ raise cls.InvalidTaskError('Task has invalid process: %s' % e)
+
+ @classmethod
+ def assert_typecheck(cls, task):
+ typecheck = task.check()
+ if not typecheck.ok():
+ raise cls.InvalidTaskError('Failed to fully evaluate task: %s' %
+ typecheck.message())
+
+ @classmethod
+ def assert_valid_ports(cls, task, portmap):
+ for port in ThermosTaskWrapper(task).ports():
+ if port not in portmap:
+ raise cls.InvalidTaskError('Task requires unbound port %s!' % port)
+
+ @classmethod
+ def assert_same_task(cls, spec, task):
+ active_task = spec.given(state='active').getpath('task_path')
+ if os.path.exists(active_task):
+ task_on_disk = ThermosTaskWrapper.from_file(active_task)
+ if not task_on_disk or task_on_disk.task != task:
+ raise cls.InvalidTaskError('Task differs from on disk copy: %r vs %r' % (
+ task_on_disk.task if task_on_disk else None, task))
+
+
+class ThermosConfigLoader(object):
+ SCHEMA = textwrap.dedent("""
+ from pystachio import *
+ from twitter.thermos.config.schema import *
+
+ __TASKS = []
+
+ def export(task):
+ __TASKS.append(Task(task) if isinstance(task, dict) else task)
+ """)
+
+ @classmethod
+ def load(cls, loadable, **kw):
+ config = Config(loadable, schema=cls.SCHEMA)
+ return cls(ThermosTaskWrapper(task, **kw) for task in config.environment['__TASKS'])
+
+ @classmethod
+ def load_json(cls, filename, **kw):
+ tc = cls()
+ task = ThermosTaskWrapper.from_file(filename, **kw)
+ if task:
+ ThermosTaskValidator.assert_valid_task(task.task())
+ tc.add_task(task)
+ return tc
+
+ def __init__(self, exported_tasks=None):
+ self._exported_tasks = exported_tasks or []
+
+ def add_task(self, task):
+ self._exported_tasks.append(task)
+
+ def tasks(self):
+ return self._exported_tasks
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/schema.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/schema.py b/src/main/python/apache/thermos/config/schema.py
new file mode 100644
index 0000000..2ee38a1
--- /dev/null
+++ b/src/main/python/apache/thermos/config/schema.py
@@ -0,0 +1,2 @@
+from .schema_base import *
+from .schema_helpers import *
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/schema_base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/schema_base.py b/src/main/python/apache/thermos/config/schema_base.py
new file mode 100644
index 0000000..a47cfdf
--- /dev/null
+++ b/src/main/python/apache/thermos/config/schema_base.py
@@ -0,0 +1,75 @@
+from pystachio import (
+ Boolean,
+ Default,
+ Empty,
+ Float,
+ Integer,
+ List,
+ Map,
+ Required,
+ String,
+ Struct
+)
+
+
+# Define constants for resources
+BYTES = 1
+KB = 1024 * BYTES
+MB = 1024 * KB
+GB = 1024 * MB
+TB = 1024 * GB
+
+
+class ThermosContext(Struct):
+ # TODO(wickman) Move the underlying replacement mechanism to %port% replacements
+ ports = Map(String, Integer)
+
+ # TODO(wickman) Move the underlying replacement mechanism to %task_id%
+ task_id = String
+
+ # TODO(wickman) Move underlying mechanism to %user%
+ user = String
+
+
+class Resources(Struct):
+ cpu = Required(Float)
+ ram = Required(Integer)
+ disk = Required(Integer)
+
+
+class Constraint(Struct):
+ order = List(String)
+
+
+class Process(Struct):
+ cmdline = Required(String)
+ name = Required(String)
+
+ # This is currently unused but reserved for future use by Thermos.
+ resources = Resources
+
+ # optionals
+ max_failures = Default(Integer, 1) # maximum number of failed process runs
+ # before process is failed.
+ daemon = Default(Boolean, False)
+ ephemeral = Default(Boolean, False)
+ min_duration = Default(Integer, 5) # integer seconds
+ final = Default(Boolean, False) # if this process should be a finalizing process
+ # that should always be run after regular processes
+
+
+class Task(Struct):
+ name = Default(String, '{{processes[0].name}}')
+ processes = List(Process)
+
+ # optionals
+ constraints = Default(List(Constraint), [])
+ resources = Resources
+ max_failures = Default(Integer, 1) # maximum number of failed processes before task is failed.
+ max_concurrency = Default(Integer, 0) # 0 is infinite concurrency.
+ # > 0 is max concurrent processes.
+ finalization_wait = Default(Integer, 30) # the amount of time in seconds we allocate to run the
+ # finalization schedule.
+
+ # TODO(jon): remove/replace with proper solution to MESOS-3546
+ user = String
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/config/schema_helpers.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/config/schema_helpers.py b/src/main/python/apache/thermos/config/schema_helpers.py
new file mode 100644
index 0000000..502de5a
--- /dev/null
+++ b/src/main/python/apache/thermos/config/schema_helpers.py
@@ -0,0 +1,247 @@
+"""Helpers for composing Thermos workflows."""
+import itertools
+
+from twitter.common.lang import Compatibility
+
+from .schema_base import (
+ Constraint,
+ GB,
+ Process,
+ Resources,
+ Task,
+)
+
+from pystachio import Empty, List
+
+
+__all__ = (
+ # shorthand for process ordering constraint
+ 'order',
+
+ # task combinators
+ 'combine_tasks', # merge N tasks in parallel
+ 'concat_tasks', # serialize N tasks
+
+ # options helpers
+ 'java_options',
+ 'python_options',
+
+ # the automatically-sequential version of a task
+ 'SequentialTask',
+
+ # create a simple task from a command line + name
+ 'SimpleTask',
+
+ # helper classes
+ 'Options',
+ 'Processes',
+ 'Tasks',
+ 'Units',
+)
+
+
+class Units(object):
+ """Helpers for base units of Tasks and Processes."""
+
+ @classmethod
+ def optional_resources(cls, resources):
+ return Resources() if resources is Empty else resources
+
+ @classmethod
+ def resources_sum(cls, *resources):
+ """Add two Resources objects together."""
+ def add_unit(f1, f2):
+ return (0 if f1 is Empty else f1.get()) + (0 if f2 is Empty else f2.get())
+
+ def add(r1, r2):
+ return Resources(cpu=add_unit(r1.cpu(), r2.cpu()),
+ ram=add_unit(r1.ram(), r2.ram()),
+ disk=add_unit(r1.disk(), r2.disk()))
+
+ return reduce(add, map(cls.optional_resources, resources), Resources(cpu=0, ram=0, disk=0))
+
+ @classmethod
+ def resources_max(cls, resources):
+ """Return a Resource object that is the maximum of the inputs along each
+ resource dimension."""
+ def max_unit(f1, f2):
+ return max(0 if f1 is Empty else f1.get(), 0 if f2 is Empty else f2.get())
+
+ def resource_max(r1, r2):
+ return Resources(cpu=max_unit(r1.cpu(), r2.cpu()),
+ ram=max_unit(r1.ram(), r2.ram()),
+ disk=max_unit(r1.disk(), r2.disk()))
+
+ return reduce(resource_max,
+ map(cls.optional_resources, resources), Resources(cpu=0, ram=0, disk=0))
+
+ @classmethod
+ def processes_merge(cls, tasks):
+ """Return a deduped list of the processes from all tasks."""
+ return list(set(itertools.chain.from_iterable(task.processes() for task in tasks)))
+
+ @classmethod
+ def constraints_merge(cls, tasks):
+ """Return a deduped list of the constraints from all tasks."""
+ return list(set(itertools.chain.from_iterable(task.constraints() for task in tasks)))
+
+
+class Processes(object):
+ """Helper class for Process objects."""
+
+ @classmethod
+ def _process_name(cls, process):
+ if isinstance(process, Process):
+ return process.name()
+ elif isinstance(process, Compatibility.string):
+ return process
+ raise ValueError("Unknown value for process order: %s" % repr(process))
+
+ @classmethod
+ def order(cls, *processes):
+ """Given a list of processes, return the list of constraints that keeps them in order, e.g.
+ order(p1, p2, p3) => [Constraint(order=[p1.name(), p2.name(), p3.name()])].
+
+ Similarly, concatenation operations are valid, i.e.
+ order(p1, p2) + order(p2, p3) <=> order(p1, p2, p3)
+ """
+ return [Constraint(order=[cls._process_name(p) for p in processes])]
+
+
+class Tasks(object):
+ """Helper class for Task objects."""
+
+ SIMPLE_CPU = 1.0
+ SIMPLE_RAM = 1 * GB
+ SIMPLE_DISK = 1 * GB
+
+ @classmethod
+ def _combine_processes(cls, *tasks):
+ """Given multiple tasks, merge their processes together, retaining the identity of the first
+ task."""
+ if len(tasks) == 0:
+ return Task()
+ head_task = tasks[-1]
+ return head_task(processes=Units.processes_merge(tasks))
+
+ @classmethod
+ def combine(cls, *tasks, **kw):
+ """Given multiple tasks, return a Task that runs all processes in parallel."""
+ if len(tasks) == 0:
+ return Task()
+ base = cls._combine_processes(*tasks)
+ return base(
+ resources=Units.resources_sum(*(task.resources() for task in tasks)),
+ constraints=Units.constraints_merge(tasks),
+ **kw
+ )
+
+ @classmethod
+ def concat(cls, *tasks, **kw):
+ """Given tasks T1...TN, return a single Task that runs all processes such that
+ all processes in Tk run before any process in Tk+1."""
+ if len(tasks) == 0:
+ return Task()
+ base = cls._combine_processes(*tasks)
+ base = base(resources=Units.resources_max(task.resources() for task in tasks))
+ base_constraints = Units.constraints_merge(tasks)
+ # TODO(wickman) be smarter about this in light of existing constraints
+ for (t1, t2) in zip(tasks[0:-1], tasks[1:]):
+ for p1 in t1.processes():
+ for p2 in t2.processes():
+ if p1 != p2:
+ base_constraints.extend(Processes.order(p1, p2))
+ return base(constraints=base_constraints, **kw)
+
+ @classmethod
+ def simple(cls, name, command):
+ """Create a usable Task from a provided name + command line and a default set of resources"""
+ return Task(
+ name=name,
+ processes=[Process(name=name, cmdline=command)],
+ resources=Resources(cpu=cls.SIMPLE_CPU,
+ ram=cls.SIMPLE_RAM,
+ disk=cls.SIMPLE_DISK))
+
+ @classmethod
+ def sequential(cls, task):
+ """Add a constraint that makes all processes within a task run sequentially."""
+ def maybe_constrain(task):
+ return {'constraints': order(*task.processes())} if task.processes() is not Empty else {}
+ if task.constraints() is Empty or task.constraints() == List(Constraint)([]):
+ return task(**maybe_constrain(task))
+ raise ValueError('Cannot turn a Task with existing constraints into a SequentialTask!')
+
+
+class Options(object):
+ """Helper class for constructing command-line arguments."""
+
+ @classmethod
+ def render_option(cls, short_prefix, long_prefix, option, value=None):
+ option = '%s%s' % (short_prefix if len(option) == 1 else long_prefix, option)
+ return '%s %s' % (option, value) if value else option
+
+ @classmethod
+ def render_options(cls, short_prefix, long_prefix, *options, **kw_options):
+ renders = []
+
+ for option in options:
+ if isinstance(option, Compatibility.string):
+ renders.append(cls.render_option(short_prefix, long_prefix, option))
+ elif isinstance(option, dict):
+ # preserve order in case option is an OrderedDict, rather than recursing with **option
+ for argument, value in option.items():
+ renders.append(cls.render_option(short_prefix, long_prefix, argument, value))
+ else:
+ raise ValueError('Got an unexpected argument to render_options: %s' % repr(option))
+
+ for argument, value in kw_options.items():
+ renders.append(cls.render_option(short_prefix, long_prefix, argument, value))
+
+ return renders
+
+ @classmethod
+ def java(cls, *options, **kw_options):
+ """
+ Given a set of arguments, keyword arguments or dictionaries, render
+ command-line parameters accordingly. For example:
+
+ java_options('a', 'b') == '-a -b'
+ java_options({
+ 'a': 23,
+ 'b': 'foo'
+ }) == '-a 23 -b foo'
+ java_options(a=23, b='foo') == '-a 23 -b foo'
+ """
+ return ' '.join(cls.render_options('-', '-', *options, **kw_options))
+
+ @classmethod
+ def python(cls, *options, **kw_options):
+ """
+ Given a set of arguments, keyword arguments or dictionaries, render
+ command-line parameters accordingly. Single letter parameters are
+ rendered with single '-'. For example:
+
+ python_options('a', 'boo') == '-a --boo'
+ python_options({
+ 'a': 23,
+ 'boo': 'foo'
+ }) == '-a 23 --boo foo'
+ python_options(a=23, boo='foo') == '-a 23 --boo foo'
+ """
+ return ' '.join(cls.render_options('-', '--', *options, **kw_options))
+
+
+def SimpleTask(name, command):
+ """A simple command-line Task with default resources"""
+ return Tasks.simple(name, command)
+
+def SequentialTask(*args, **kw):
+ """A Task whose processes are always sequential."""
+ return Tasks.sequential(Task(*args, **kw))
+
+python_options = Options.python
+java_options = Options.java
+combine_tasks = Tasks.combine
+concat_tasks = Tasks.concat
+order = Processes.order
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/core/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/BUILD b/src/main/python/apache/thermos/core/BUILD
new file mode 100644
index 0000000..d260112
--- /dev/null
+++ b/src/main/python/apache/thermos/core/BUILD
@@ -0,0 +1,88 @@
+import os
+
+python_library(
+ name = 'helper',
+ sources = ['helper.py'],
+ dependencies = [
+ pants('src/main/python/twitter/thermos:psutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'inspector',
+ sources = ['inspector.py'],
+ dependencies = [
+ pants(':muxer'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'muxer',
+ sources = ['muxer.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'process',
+ sources = ['process.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'runner',
+ sources = ['__init__.py', 'runner.py'],
+ dependencies = [
+ pants(':helper'),
+ pants(':muxer'),
+ pants(':process'),
+ pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/python/twitter/thermos:psutil'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/python/twitter/thermos/common:planner'),
+ pants('src/main/python/twitter/thermos/config:schema'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'core',
+ dependencies = [
+ pants(':inspector'),
+ pants(':runner'),
+
+ # covering libs
+ pants('src/main/python/twitter/thermos/common'),
+ pants('src/main/python/twitter/thermos/config'),
+ ],
+ provides = setup_py(
+ name = 'twitter.thermos.core',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ description = 'The Thermos core state machine.',
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/core/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/__init__.py b/src/main/python/apache/thermos/core/__init__.py
new file mode 100644
index 0000000..de40ea7
--- /dev/null
+++ b/src/main/python/apache/thermos/core/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/core/helper.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/helper.py b/src/main/python/apache/thermos/core/helper.py
new file mode 100644
index 0000000..3587a85
--- /dev/null
+++ b/src/main/python/apache/thermos/core/helper.py
@@ -0,0 +1,387 @@
+from contextlib import closing
+import errno
+import os
+import signal
+import time
+
+from twitter.common import log
+from twitter.common.dirutil import lock_file, safe_mkdir
+from twitter.common.quantity import Amount, Time
+from twitter.common.recordio import ThriftRecordWriter
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+from twitter.thermos.common.path import TaskPath
+
+from gen.twitter.thermos.ttypes import (
+ ProcessState,
+ ProcessStatus,
+ RunnerCkpt,
+ TaskState,
+ TaskStatus)
+
+import psutil
+
+
+class TaskKiller(object):
+ """
+ Task killing interface.
+ """
+
+ def __init__(self, task_id, checkpoint_root):
+ self._task_id = task_id
+ self._checkpoint_root = checkpoint_root
+
+ def kill(self, force=True):
+ TaskRunnerHelper.kill(self._task_id, self._checkpoint_root, force=force,
+ terminal_status=TaskState.KILLED)
+
+ def lose(self, force=True):
+ TaskRunnerHelper.kill(self._task_id, self._checkpoint_root, force=force,
+ terminal_status=TaskState.LOST)
+
+
+class TaskRunnerHelper(object):
+ """
+ TaskRunner helper methods that can be operated directly upon checkpoint
+ state. These operations do not require knowledge of the underlying
+ task.
+
+ TaskRunnerHelper is sort of a mishmash of "checkpoint-only" operations and
+ the "Process Platform" stuff that started to get pulled into process.py
+
+ This really needs some hard design thought to see if it can be extracted out
+ even further.
+ """
+ class Error(Exception): pass
+ class PermissionError(Error): pass
+
+ # Maximum drift between when the system says a task was forked and when we checkpointed
+ # its fork_time (used as a heuristic to determine a forked task is really ours instead of
+ # a task with coincidentally the same PID but just wrapped around.)
+ MAX_START_TIME_DRIFT = Amount(10, Time.SECONDS)
+
+ @staticmethod
+ def get_actual_user():
+ import getpass, pwd
+ try:
+ pwd_entry = pwd.getpwuid(os.getuid())
+ except KeyError:
+ return getpass.getuser()
+ return pwd_entry[0]
+
+ @staticmethod
+ def process_from_name(task, process_name):
+ if task.has_processes():
+ for process in task.processes():
+ if process.name().get() == process_name:
+ return process
+ return None
+
+ @classmethod
+ def this_is_really_our_pid(cls, process, current_user, start_time):
+ """
+ A heuristic to make sure that this is likely the pid that we own/forked. Necessary
+ because of pid-space wrapping. We don't want to go and kill processes we don't own,
+ especially if the killer is running as root.
+
+ process: psutil.Process representing the process to check
+ current_user: user expected to own the process
+ start_time: time at which it's expected the process has started
+
+ Raises:
+ psutil.NoSuchProcess - if the Process supplied no longer exists
+ """
+ if process.username != current_user:
+ log.info("Expected pid %s to be ours but the pid user is %s and we're %s" % (
+ process.pid, process.username, current_user))
+ return False
+
+ if abs(start_time - process.create_time) >= cls.MAX_START_TIME_DRIFT.as_(Time.SECONDS):
+ log.info("Expected pid %s start time to be %s but it's %s" % (
+ process.pid, start_time, process.create_time))
+ return False
+
+ return True
+
+ @classmethod
+ def scan_process(cls, state, process_name):
+ """
+ Given a RunnerState and a process_name, return the following:
+ (coordinator pid, process pid, process tree)
+ (int or None, int or None, set)
+
+ """
+ process_run = state.processes[process_name][-1]
+ process_owner = state.header.user
+
+ coordinator_pid, pid, tree = None, None, set()
+
+ if process_run.coordinator_pid:
+ try:
+ coordinator_process = psutil.Process(process_run.coordinator_pid)
+ if cls.this_is_really_our_pid(coordinator_process, process_owner, process_run.fork_time):
+ coordinator_pid = process_run.coordinator_pid
+ except psutil.NoSuchProcess:
+ log.info(' Coordinator %s [pid: %s] completed.' % (process_run.process,
+ process_run.coordinator_pid))
+ except psutil.Error as err:
+ log.warning(' Error gathering information on pid %s: %s' % (process_run.coordinator_pid,
+ err))
+
+ if process_run.pid:
+ try:
+ process = psutil.Process(process_run.pid)
+ if cls.this_is_really_our_pid(process, process_owner, process_run.start_time):
+ pid = process.pid
+ except psutil.NoSuchProcess:
+ log.info(' Process %s [pid: %s] completed.' % (process_run.process, process_run.pid))
+ except psutil.Error as err:
+ log.warning(' Error gathering information on pid %s: %s' % (process_run.pid, err))
+ else:
+ if pid:
+ try:
+ tree = set(proc.pid for proc in process.get_children(recursive=True))
+ except psutil.Error:
+ log.warning(' Error gathering information on children of pid %s' % pid)
+
+ return (coordinator_pid, pid, tree)
+
+ @classmethod
+ def scantree(cls, state):
+ """
+ Scan the process tree associated with the provided task state.
+
+ Returns a dictionary of process name => (coordinator pid, pid, pid children)
+ If the coordinator is no longer active, coordinator pid will be None. If the
+ forked process is no longer active, pid will be None and its children will be
+ an empty set.
+ """
+ return dict((process_name, cls.scan_process(state, process_name))
+ for process_name in state.processes)
+
+ @classmethod
+ def safe_signal(cls, pid, sig=signal.SIGTERM):
+ try:
+ os.kill(pid, sig)
+ except OSError as e:
+ if e.errno not in (errno.ESRCH, errno.EPERM):
+ log.error('Unexpected error in os.kill: %s' % e)
+ except Exception as e:
+ log.error('Unexpected error in os.kill: %s' % e)
+
+ @classmethod
+ def terminate_pid(cls, pid):
+ cls.safe_signal(pid, signal.SIGTERM)
+
+ @classmethod
+ def kill_pid(cls, pid):
+ cls.safe_signal(pid, signal.SIGKILL)
+
+ @classmethod
+ def kill_group(cls, pgrp):
+ cls.safe_signal(-pgrp, signal.SIGKILL)
+
+ @classmethod
+ def _get_process_tuple(cls, state, process_name):
+ assert process_name in state.processes and len(state.processes[process_name]) > 0
+ return cls.scan_process(state, process_name)
+
+ @classmethod
+ def _get_coordinator_group(cls, state, process_name):
+ assert process_name in state.processes and len(state.processes[process_name]) > 0
+ return state.processes[process_name][-1].coordinator_pid
+
+ @classmethod
+ def terminate_process(cls, state, process_name):
+ log.debug('TaskRunnerHelper.terminate_process(%s)' % process_name)
+ _, pid, _ = cls._get_process_tuple(state, process_name)
+ if pid:
+ log.debug(' => SIGTERM pid %s' % pid)
+ cls.terminate_pid(pid)
+ return bool(pid)
+
+ @classmethod
+ def kill_process(cls, state, process_name):
+ log.debug('TaskRunnerHelper.kill_process(%s)' % process_name)
+ coordinator_pgid = cls._get_coordinator_group(state, process_name)
+ coordinator_pid, pid, tree = cls._get_process_tuple(state, process_name)
+ # This is super dangerous. TODO(wickman) Add a heuristic that determines
+ # that 1) there are processes that currently belong to this process group
+ # and 2) those processes have inherited the coordinator checkpoint filehandle
+ # This way we validate that it is in fact the process group we expect.
+ if coordinator_pgid:
+ log.debug(' => SIGKILL coordinator group %s' % coordinator_pgid)
+ cls.kill_group(coordinator_pgid)
+ if coordinator_pid:
+ log.debug(' => SIGKILL coordinator %s' % coordinator_pid)
+ cls.kill_pid(coordinator_pid)
+ if pid:
+ log.debug(' => SIGKILL pid %s' % pid)
+ cls.kill_pid(pid)
+ for child in tree:
+ log.debug(' => SIGKILL child %s' % child)
+ cls.kill_pid(child)
+ return bool(coordinator_pid or pid or tree)
+
+ @classmethod
+ def kill_runner(cls, state):
+ log.debug('TaskRunnerHelper.kill_runner()')
+ if not state or not state.statuses:
+ raise cls.Error('Could not read state!')
+ pid = state.statuses[-1].runner_pid
+ if pid == os.getpid():
+ raise cls.Error('Unwilling to commit seppuku.')
+ try:
+ os.kill(pid, signal.SIGKILL)
+ return True
+ except OSError as e:
+ if e.errno == errno.EPERM:
+ # Permission denied
+ return False
+ elif e.errno == errno.ESRCH:
+ # pid no longer exists
+ return True
+ raise
+
+ @classmethod
+ def open_checkpoint(cls, filename, force=False, state=None):
+ """
+ Acquire a locked checkpoint stream.
+ """
+ safe_mkdir(os.path.dirname(filename))
+ fp = lock_file(filename, "a+")
+ if fp in (None, False):
+ if force:
+ log.info('Found existing runner, forcing leadership forfeit.')
+ state = state or CheckpointDispatcher.from_file(filename)
+ if cls.kill_runner(state):
+ log.info('Successfully killed leader.')
+ # TODO(wickman) Blocking may not be the best idea here. Perhaps block up to
+ # a maximum timeout. But blocking is necessary because os.kill does not immediately
+ # release the lock if we're in force mode.
+ fp = lock_file(filename, "a+", blocking=True)
+ else:
+ log.error('Found existing runner, cannot take control.')
+ if fp in (None, False):
+ raise cls.PermissionError('Could not open locked checkpoint: %s, lock_file = %s' %
+ (filename, fp))
+ ckpt = ThriftRecordWriter(fp)
+ ckpt.set_sync(True)
+ return ckpt
+
+ @classmethod
+ def kill(cls, task_id, checkpoint_root, force=False,
+ terminal_status=TaskState.KILLED, clock=time):
+ """
+ An implementation of Task killing that doesn't require a fully hydrated TaskRunner object.
+ Terminal status must be either KILLED or LOST state.
+ """
+ if terminal_status not in (TaskState.KILLED, TaskState.LOST):
+ raise cls.Error('terminal_status must be KILLED or LOST (got %s)' %
+ TaskState._VALUES_TO_NAMES.get(terminal_status) or terminal_status)
+ pathspec = TaskPath(root=checkpoint_root, task_id=task_id)
+ checkpoint = pathspec.getpath('runner_checkpoint')
+ state = CheckpointDispatcher.from_file(checkpoint)
+
+ if state is None or state.header is None or state.statuses is None:
+ if force:
+ log.error('Task has uninitialized TaskState - forcibly finalizing')
+ cls.finalize_task(pathspec)
+ return
+ else:
+ log.error('Cannot update states in uninitialized TaskState!')
+ return
+
+ ckpt = cls.open_checkpoint(checkpoint, force=force, state=state)
+
+ def write_task_state(state):
+ update = TaskStatus(state=state, timestamp_ms=int(clock.time() * 1000),
+ runner_pid=os.getpid(), runner_uid=os.getuid())
+ ckpt.write(RunnerCkpt(task_status=update))
+
+ def write_process_status(status):
+ ckpt.write(RunnerCkpt(process_status=status))
+
+ if cls.is_task_terminal(state.statuses[-1].state):
+ log.info('Task is already in terminal state! Finalizing.')
+ cls.finalize_task(pathspec)
+ return
+
+ with closing(ckpt):
+ write_task_state(TaskState.ACTIVE)
+ for process, history in state.processes.items():
+ process_status = history[-1]
+ if not cls.is_process_terminal(process_status.state):
+ if cls.kill_process(state, process):
+ write_process_status(ProcessStatus(process=process,
+ state=ProcessState.KILLED, seq=process_status.seq + 1, return_code=-9,
+ stop_time=clock.time()))
+ else:
+ if process_status.state is not ProcessState.WAITING:
+ write_process_status(ProcessStatus(process=process,
+ state=ProcessState.LOST, seq=process_status.seq + 1))
+ write_task_state(terminal_status)
+ cls.finalize_task(pathspec)
+
+ @classmethod
+ def reap_children(cls):
+ pids = set()
+
+ while True:
+ try:
+ pid, status, rusage = os.wait3(os.WNOHANG)
+ if pid == 0:
+ break
+ pids.add(pid)
+ log.debug('Detected terminated process: pid=%s, status=%s, rusage=%s' % (
+ pid, status, rusage))
+ except OSError as e:
+ if e.errno != errno.ECHILD:
+ log.warning('Unexpected error when calling waitpid: %s' % e)
+ break
+
+ return pids
+
+ TERMINAL_PROCESS_STATES = frozenset([
+ ProcessState.SUCCESS,
+ ProcessState.KILLED,
+ ProcessState.FAILED,
+ ProcessState.LOST])
+
+ TERMINAL_TASK_STATES = frozenset([
+ TaskState.SUCCESS,
+ TaskState.FAILED,
+ TaskState.KILLED,
+ TaskState.LOST])
+
+ @classmethod
+ def is_process_terminal(cls, process_status):
+ return process_status in cls.TERMINAL_PROCESS_STATES
+
+ @classmethod
+ def is_task_terminal(cls, task_status):
+ return task_status in cls.TERMINAL_TASK_STATES
+
+ @classmethod
+ def initialize_task(cls, spec, task):
+ active_task = spec.given(state='active').getpath('task_path')
+ finished_task = spec.given(state='finished').getpath('task_path')
+ is_active, is_finished = os.path.exists(active_task), os.path.exists(finished_task)
+ if is_finished:
+ raise cls.Error('Cannot initialize task with "finished" record!')
+ if not is_active:
+ safe_mkdir(os.path.dirname(active_task))
+ with open(active_task, 'w') as fp:
+ fp.write(task)
+
+ @classmethod
+ def finalize_task(cls, spec):
+ active_task = spec.given(state='active').getpath('task_path')
+ finished_task = spec.given(state='finished').getpath('task_path')
+ is_active, is_finished = os.path.exists(active_task), os.path.exists(finished_task)
+ if not is_active:
+ raise cls.Error('Cannot finalize task with no "active" record!')
+ elif is_finished:
+ raise cls.Error('Cannot finalize task with "finished" record!')
+ safe_mkdir(os.path.dirname(finished_task))
+ os.rename(active_task, finished_task)
+ os.utime(finished_task, None)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/core/inspector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/inspector.py b/src/main/python/apache/thermos/core/inspector.py
new file mode 100644
index 0000000..8c646a1
--- /dev/null
+++ b/src/main/python/apache/thermos/core/inspector.py
@@ -0,0 +1,105 @@
+from collections import namedtuple
+from contextlib import closing
+import pwd
+
+from twitter.common import log
+from twitter.common.recordio import RecordIO, ThriftRecordReader
+
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+from twitter.thermos.common.path import TaskPath
+
+from gen.twitter.thermos.ttypes import (
+ ProcessState,
+ RunnerCkpt,
+ RunnerState)
+
+from .muxer import ProcessMuxer
+
+
+CheckpointInspection = namedtuple('CheckpointInspection',
+ ['runner_latest_update',
+ 'process_latest_update',
+ 'runner_processes',
+ 'coordinator_processes',
+ 'processes'])
+
+
+class CheckpointInspector(object):
+ def __init__(self, checkpoint_root):
+ self._path = TaskPath(root=checkpoint_root)
+
+ @staticmethod
+ def get_timestamp(process_record):
+ if process_record :
+ for timestamp in ('fork_time', 'start_time', 'stop_time'):
+ stamp = getattr(process_record, timestamp, None)
+ if stamp:
+ return stamp
+ return 0
+
+ def inspect(self, task_id):
+ """
+ Reconstructs the checkpoint stream and returns a CheckpointInspection.
+ """
+ dispatcher = CheckpointDispatcher()
+ state = RunnerState(processes = {})
+ muxer = ProcessMuxer(self._path.given(task_id=task_id))
+
+ runner_processes = []
+ coordinator_processes = set()
+ processes = set()
+
+ def consume_process_record(record):
+ if not record.process_status:
+ return
+ try:
+ user_uid = pwd.getpwnam(state.header.user).pw_uid
+ except KeyError:
+ log.error('Could not find user: %s' % state.header.user)
+ return
+ if record.process_status.state == ProcessState.FORKED:
+ coordinator_processes.add((record.process_status.coordinator_pid, user_uid,
+ record.process_status.fork_time))
+ elif record.process_status.state == ProcessState.RUNNING:
+ processes.add((record.process_status.pid, user_uid,
+ record.process_status.start_time))
+
+ # replay runner checkpoint
+ runner_pid = None
+ runner_latest_update = 0
+ try:
+ with open(self._path.given(task_id=task_id).getpath('runner_checkpoint')) as fp:
+ with closing(ThriftRecordReader(fp, RunnerCkpt)) as ckpt:
+ for record in ckpt:
+ dispatcher.dispatch(state, record)
+ runner_latest_update = max(runner_latest_update,
+ self.get_timestamp(record.process_status))
+ # collect all bound runners
+ if record.task_status:
+ if record.task_status.runner_pid != runner_pid:
+ runner_processes.append((record.task_status.runner_pid,
+ record.task_status.runner_uid or 0,
+ record.task_status.timestamp_ms))
+ runner_pid = record.task_status.runner_pid
+ elif record.process_status:
+ consume_process_record(record)
+ except (IOError, OSError, RecordIO.Error) as err:
+ log.debug('Error inspecting task runner checkpoint: %s' % err)
+ return
+
+ # register existing processes in muxer
+ for process_name in state.processes:
+ muxer.register(process_name)
+
+ # read process checkpoints
+ process_latest_update = runner_latest_update
+ for record in muxer.select():
+ process_latest_update = max(process_latest_update, self.get_timestamp(record.process_status))
+ consume_process_record(record)
+
+ return CheckpointInspection(
+ runner_latest_update=runner_latest_update,
+ process_latest_update=process_latest_update,
+ runner_processes=runner_processes,
+ coordinator_processes=coordinator_processes,
+ processes=processes)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/core/muxer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/muxer.py b/src/main/python/apache/thermos/core/muxer.py
new file mode 100644
index 0000000..9c0b389
--- /dev/null
+++ b/src/main/python/apache/thermos/core/muxer.py
@@ -0,0 +1,142 @@
+import os
+import errno
+
+from twitter.common import log
+from twitter.common.recordio import ThriftRecordReader
+from gen.twitter.thermos.ttypes import RunnerCkpt
+
+
+class ProcessMuxer(object):
+ class ProcessExists(Exception): pass
+ class ProcessNotFound(Exception): pass
+ class CorruptCheckpoint(Exception): pass
+
+ def __init__(self, pathspec):
+ self._processes = {} # process_name => fp
+ self._watermarks = {} # process_name => sequence high watermark
+ self._pathspec = pathspec
+
+ def __del__(self):
+ for fp in filter(None, self._processes.values()):
+ fp.close()
+
+ def register(self, process_name, watermark=0):
+ log.debug('registering %s' % process_name)
+ if process_name in self._processes:
+ raise ProcessMuxer.ProcessExists("Process %s is already registered" % process_name)
+ self._processes[process_name] = None
+ self._watermarks[process_name] = watermark
+
+ def _bind_processes(self):
+ for process_name, fp in self._processes.items():
+ if fp is None:
+ process_ckpt = self._pathspec.given(process=process_name).getpath('process_checkpoint')
+ log.debug('ProcessMuxer binding %s => %s' % (process_name, process_ckpt))
+ try:
+ self._processes[process_name] = open(process_ckpt, 'r')
+ except IOError as e:
+ if e.errno == errno.ENOENT:
+ log.debug(' => bind failed, checkpoint not available yet.')
+ continue
+ else:
+ log.error("Unexpected inability to open %s! %s" % (process_ckpt, e))
+ except Exception as e:
+ log.error("Unexpected inability to open %s! %s" % (process_ckpt, e))
+ self._fast_forward_stream(process_name)
+
+ def _fast_forward_stream(self, process_name):
+ log.debug('Fast forwarding %s stream to seq=%s' % (process_name,
+ self._watermarks[process_name]))
+ assert self._processes.get(process_name) is not None
+ fp = self._processes[process_name]
+ rr = ThriftRecordReader(fp, RunnerCkpt)
+ current_watermark = -1
+ records = 0
+ while current_watermark < self._watermarks[process_name]:
+ last_pos = fp.tell()
+ record = rr.try_read()
+ if record is None:
+ break
+ new_watermark = record.process_status.seq
+ if new_watermark > self._watermarks[process_name]:
+ log.debug('Over-seeked %s [watermark = %s, high watermark = %s], rewinding.' % (
+ process_name, new_watermark, self._watermarks[process_name]))
+ fp.seek(last_pos)
+ break
+ current_watermark = new_watermark
+ records += 1
+
+ if current_watermark < self._watermarks[process_name]:
+ log.warning('Only able to fast forward to %s@sequence=%s, high watermark is %s' % (
+ process_name, current_watermark, self._watermarks[process_name]))
+
+ if records:
+ log.debug('Fast forwarded %s %s record(s) to seq=%s.' % (process_name, records,
+ current_watermark))
+
+ def unregister(self, process_name):
+ log.debug('unregistering %s' % process_name)
+ if process_name not in self._processes:
+ raise ProcessMuxer.ProcessNotFound("No trace of process: %s" % process_name)
+ else:
+ self._watermarks.pop(process_name)
+ fp = self._processes.pop(process_name)
+ if fp is not None:
+ fp.close()
+
+ def has_data(self, process):
+ """
+ Return true if we think that there are updates available from the supplied process.
+ """
+ self._bind_processes()
+ # TODO(wickman) Should this raise ProcessNotFound?
+ if process not in self._processes:
+ return False
+ fp = self._processes[process]
+ rr = ThriftRecordReader(fp, RunnerCkpt)
+ old_pos = fp.tell()
+ try:
+ expected_new_pos = os.fstat(fp.fileno()).st_size
+ except OSError as e:
+ log.debug('ProcessMuxer could not fstat for process %s' % process)
+ return False
+ update = rr.try_read()
+ if update:
+ fp.seek(old_pos)
+ return True
+ return False
+
+ def select(self):
+ """
+ Read and multiplex checkpoint records from all the forked off process coordinators.
+
+ Checkpoint records can come from one of two places:
+ in-process: checkpoint records synthesized for FORKED and LOST events
+ out-of-process: checkpoint records from from file descriptors of forked coordinators
+
+ Returns a list of RunnerCkpt objects that were successfully read, or an empty
+ list if none were read.
+ """
+ self._bind_processes()
+ updates = []
+ for handle in filter(None, self._processes.values()):
+ try:
+ fstat = os.fstat(handle.fileno())
+ except OSError as e:
+ log.error('Unable to fstat %s!' % handle.name)
+ continue
+ if handle.tell() > fstat.st_size:
+ log.error('Truncated checkpoint record detected on %s!' % handle.name)
+ elif handle.tell() < fstat.st_size:
+ rr = ThriftRecordReader(handle, RunnerCkpt)
+ while True:
+ process_update = rr.try_read()
+ if process_update:
+ updates.append(process_update)
+ else:
+ break
+ if len(updates) > 0:
+ log.debug('select() returning %s updates:' % len(updates))
+ for update in updates:
+ log.debug(' = %s' % update)
+ return updates
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/core/process.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/process.py b/src/main/python/apache/thermos/core/process.py
new file mode 100644
index 0000000..dd1d62a
--- /dev/null
+++ b/src/main/python/apache/thermos/core/process.py
@@ -0,0 +1,371 @@
+"""Run processes of a Thermos task.
+
+This module contains the Process class, used to manage the execution of the constituent processes of
+a Thermos task. Each process is represented by a "coordinator" process, which fires off the actual
+commandline in a subprocess of its own.
+
+"""
+
+from abc import abstractmethod
+import getpass
+import grp
+import os
+import pwd
+import signal
+import subprocess
+import sys
+import time
+
+from twitter.common import log
+from twitter.common.dirutil import (
+ lock_file,
+ safe_mkdir,
+ safe_open,
+)
+from twitter.common.lang import Interface
+from twitter.common.quantity import Amount, Time
+from twitter.common.recordio import ThriftRecordReader, ThriftRecordWriter
+
+from gen.twitter.thermos.ttypes import (
+ ProcessState,
+ ProcessStatus,
+ RunnerCkpt,
+)
+
+
+class Platform(Interface):
+ """Abstract representation of a platform encapsulating system-level functions"""
+ @abstractmethod
+ def clock(self):
+ pass
+
+ @abstractmethod
+ def fork(self):
+ pass
+
+ @abstractmethod
+ def getpid(self):
+ pass
+
+
+class ProcessBase(object):
+ """
+ Encapsulate a running process for a task.
+ """
+ class Error(Exception): pass
+ class UnknownUserError(Error): pass
+ class CheckpointError(Error): pass
+ class UnspecifiedSandbox(Error): pass
+ class PermissionError(Error): pass
+
+ CONTROL_WAIT_CHECK_INTERVAL = Amount(100, Time.MILLISECONDS)
+ MAXIMUM_CONTROL_WAIT = Amount(1, Time.MINUTES)
+
+ def __init__(self, name, cmdline, sequence, pathspec, sandbox_dir, user=None, platform=None):
+ """
+ required:
+ name = name of the process
+ cmdline = cmdline of the process
+ sequence = the next available sequence number for state updates
+ pathspec = TaskPath object for synthesizing path names
+ sandbox_dir = the sandbox in which to run the process
+ platform = Platform providing fork, clock, getpid
+
+ optional:
+ user = the user to run as (if unspecified, will default to current user.)
+ if specified to a user that is not the current user, you must have root access
+ """
+ self._name = name
+ self._cmdline = cmdline
+ self._pathspec = pathspec
+ self._seq = sequence
+ self._sandbox = sandbox_dir
+ if self._sandbox:
+ safe_mkdir(self._sandbox)
+ self._pid = None
+ self._fork_time = None
+ self._stdout = None
+ self._stderr = None
+ self._user = user
+ if self._user:
+ user, current_user = self._getpwuid() # may raise self.UnknownUserError
+ if user != current_user and os.geteuid() != 0:
+ raise self.PermissionError('Must be root to run processes as other users!')
+ self._ckpt = None
+ self._ckpt_head = -1
+ if platform is None:
+ raise ValueError("Platform must be specified")
+ self._platform = platform
+
+ def _log(self, msg):
+ log.debug('[process:%5s=%s]: %s' % (self._pid, self.name(), msg))
+
+ def _ckpt_write(self, msg):
+ self._init_ckpt_if_necessary()
+ self._log("child state transition [%s] <= %s" % (self.ckpt_file(), msg))
+ self._ckpt.write(msg)
+
+ def _write_process_update(self, **kw):
+ """Write a process update to the coordinator's checkpoint stream."""
+ process_status = ProcessStatus(**kw)
+ process_status.seq = self._seq
+ process_status.process = self.name()
+ self._ckpt_write(RunnerCkpt(process_status=process_status))
+ self._seq += 1
+
+ def _write_initial_update(self):
+ self._write_process_update(state=ProcessState.FORKED,
+ fork_time=self._fork_time,
+ coordinator_pid=self._pid)
+
+ def cmdline(self):
+ return self._cmdline
+
+ def name(self):
+ return self._name
+
+ def pid(self):
+ """pid of the coordinator"""
+ return self._pid
+
+ def rebind(self, pid, fork_time):
+ """rebind Process to an existing coordinator pid without forking"""
+ self._pid = pid
+ self._fork_time = fork_time
+
+ def ckpt_file(self):
+ return self._pathspec.getpath('process_checkpoint')
+
+ def _setup_ckpt(self):
+ """Set up the checkpoint: must be run on the parent."""
+ self._log('initializing checkpoint file: %s' % self.ckpt_file())
+ ckpt_fp = lock_file(self.ckpt_file(), "a+")
+ if ckpt_fp in (None, False):
+ raise self.CheckpointError('Could not acquire checkpoint permission or lock for %s!' %
+ self.ckpt_file())
+ self._ckpt_head = os.path.getsize(self.ckpt_file())
+ ckpt_fp.seek(self._ckpt_head)
+ self._ckpt = ThriftRecordWriter(ckpt_fp)
+ self._ckpt.set_sync(True)
+
+ def _init_ckpt_if_necessary(self):
+ if self._ckpt is None:
+ self._setup_ckpt()
+
+ def _wait_for_control(self):
+ """Wait for control of the checkpoint stream: must be run in the child."""
+ total_wait_time = Amount(0, Time.SECONDS)
+
+ with open(self.ckpt_file(), 'r') as fp:
+ fp.seek(self._ckpt_head)
+ rr = ThriftRecordReader(fp, RunnerCkpt)
+ while total_wait_time < self.MAXIMUM_CONTROL_WAIT:
+ ckpt_tail = os.path.getsize(self.ckpt_file())
+ if ckpt_tail == self._ckpt_head:
+ self._platform.clock().sleep(self.CONTROL_WAIT_CHECK_INTERVAL.as_(Time.SECONDS))
+ total_wait_time += self.CONTROL_WAIT_CHECK_INTERVAL
+ continue
+ checkpoint = rr.try_read()
+ if checkpoint:
+ if not checkpoint.process_status:
+ raise self.CheckpointError('No process status in checkpoint!')
+ if (checkpoint.process_status.process != self.name() or
+ checkpoint.process_status.state != ProcessState.FORKED or
+ checkpoint.process_status.fork_time != self._fork_time or
+ checkpoint.process_status.coordinator_pid != self._pid):
+ self._log('Losing control of the checkpoint stream:')
+ self._log(' fork_time [%s] vs self._fork_time [%s]' % (
+ checkpoint.process_status.fork_time, self._fork_time))
+ self._log(' coordinator_pid [%s] vs self._pid [%s]' % (
+ checkpoint.process_status.coordinator_pid, self._pid))
+ raise self.CheckpointError('Lost control of the checkpoint stream!')
+ self._log('Taking control of the checkpoint stream at record: %s' %
+ checkpoint.process_status)
+ self._seq = checkpoint.process_status.seq + 1
+ return True
+ raise self.CheckpointError('Timed out waiting for checkpoint stream!')
+
+ def _prepare_fork(self):
+ user, current_user = self._getpwuid()
+ uid, gid = user.pw_uid, user.pw_gid
+ self._fork_time = self._platform.clock().time()
+ self._setup_ckpt()
+ self._stdout = safe_open(self._pathspec.with_filename('stdout').getpath('process_logdir'), "w")
+ self._stderr = safe_open(self._pathspec.with_filename('stderr').getpath('process_logdir'), "w")
+ os.chown(self._stdout.name, user.pw_uid, user.pw_gid)
+ os.chown(self._stderr.name, user.pw_uid, user.pw_gid)
+
+ def _finalize_fork(self):
+ self._write_initial_update()
+ self._ckpt.close()
+ self._ckpt = None
+
+ def _getpwuid(self):
+ """Returns a tuple of the user (i.e. --user) and current user."""
+ try:
+ current_user = pwd.getpwuid(os.getuid())
+ except KeyError:
+ raise self.UnknownUserError('Unknown user %s!' % self._user)
+ try:
+ user = pwd.getpwnam(self._user) if self._user else current_user
+ except KeyError:
+ raise self.UnknownUserError('Unable to get pwent information!')
+ return user, current_user
+
+ def start(self):
+ """
+ This is the main call point from the runner, and forks a co-ordinator process to run the
+ target process (i.e. self.cmdline())
+
+ The parent returns immediately and populates information about the pid of the co-ordinator.
+ The child (co-ordinator) will launch the target process in a subprocess.
+ """
+ self._prepare_fork()
+ self._pid = self._platform.fork()
+ if self._pid == 0:
+ self._pid = self._platform.getpid()
+ self._wait_for_control()
+ try:
+ self.execute()
+ finally:
+ self._ckpt.close()
+ self.finish()
+ else:
+ self._finalize_fork()
+
+ def execute(self):
+ raise NotImplementedError
+
+ def finish(self):
+ pass
+
+
+class RealPlatform(Platform):
+ IGNORE_SIGNALS = (signal.SIGINT,)
+
+ def __init__(self, fork=os.fork):
+ self._fork = fork
+
+ def fork(self):
+ pid = self._fork()
+ if pid == 0:
+ self._sanitize()
+ return pid
+
+ def _sanitize(self):
+ for sig in self.IGNORE_SIGNALS:
+ signal.signal(sig, signal.SIG_IGN)
+
+ def getpid(self):
+ return os.getpid()
+
+ def clock(self):
+ return time
+
+
+class Process(ProcessBase):
+ """
+ Encapsulate a running process for a task.
+ """
+ RCFILE = '.thermos_profile'
+ FD_CLOEXEC = True
+
+ def __init__(self, *args, **kw):
+ """
+ See ProcessBase.__init__
+
+ Takes additional arguments:
+ fork: the fork function to use [default: os.fork]
+ chroot: whether or not to chroot into the sandbox [default: False]
+ """
+ fork = kw.pop('fork', os.fork)
+ self._use_chroot = bool(kw.pop('chroot', False))
+ self._rc = None
+ kw['platform'] = RealPlatform(fork=fork)
+ ProcessBase.__init__(self, *args, **kw)
+ if self._use_chroot and self._sandbox is None:
+ raise self.UnspecifiedSandbox('If using chroot, must specify sandbox!')
+
+ def _chroot(self):
+ """chdir and chroot to the sandbox directory."""
+ os.chdir(self._sandbox)
+ os.chroot(self._sandbox)
+
+ def _setuid(self):
+ """Drop privileges to the user supplied in Process creation (if necessary.)"""
+ user, current_user = self._getpwuid()
+ if user.pw_uid == current_user.pw_uid:
+ return
+
+ uid, gid = user.pw_uid, user.pw_gid
+ username = user.pw_name
+ group_ids = [group.gr_gid for group in grp.getgrall() if username in group.gr_mem]
+ os.setgroups(group_ids)
+ os.setgid(gid)
+ os.setuid(uid)
+
+ def execute(self):
+ """Perform final initialization and launch target process commandline in a subprocess."""
+ if not self._stderr:
+ raise RuntimeError('self._stderr not set up!')
+ if not self._stdout:
+ raise RuntimeError('self._stdout not set up!')
+
+ user, _ = self._getpwuid()
+ username, homedir = user.pw_name, user.pw_dir
+
+ # TODO(wickman) reconsider setsid now that we're invoking in a subshell
+ os.setsid()
+ if self._use_chroot:
+ self._chroot()
+ self._setuid()
+
+ # start process
+ start_time = self._platform.clock().time()
+
+ if not self._sandbox:
+ sandbox = os.getcwd()
+ else:
+ sandbox = self._sandbox if not self._use_chroot else '/'
+
+ thermos_profile = os.path.join(sandbox, self.RCFILE)
+ env = {
+ 'HOME': homedir if self._use_chroot else sandbox,
+ 'LOGNAME': username,
+ 'USER': username,
+ 'PATH': os.environ['PATH']
+ }
+
+ if os.path.exists(thermos_profile):
+ env.update(BASH_ENV=thermos_profile)
+
+ self._popen = subprocess.Popen(["/bin/bash", "-c", self.cmdline()],
+ stderr=self._stderr,
+ stdout=self._stdout,
+ close_fds=self.FD_CLOEXEC,
+ cwd=sandbox,
+ env=env)
+
+ self._write_process_update(state=ProcessState.RUNNING,
+ pid=self._popen.pid,
+ start_time=start_time)
+
+ # wait for job to finish
+ rc = self._popen.wait()
+
+ # indicate that we have finished/failed
+ if rc < 0:
+ state = ProcessState.KILLED
+ elif rc == 0:
+ state = ProcessState.SUCCESS
+ else:
+ state = ProcessState.FAILED
+
+ self._write_process_update(state=state,
+ return_code=rc,
+ stop_time=self._platform.clock().time())
+ self._rc = rc
+
+ def finish(self):
+ self._log('Coordinator exiting.')
+ sys.exit(0)
[17/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/assets/mootools-core.js
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/assets/mootools-core.js b/src/main/python/apache/thermos/observer/http/assets/mootools-core.js
new file mode 100644
index 0000000..959afa4
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/assets/mootools-core.js
@@ -0,0 +1,5789 @@
+/*
+---
+
+name: Core
+
+description: The heart of MooTools.
+
+license: MIT-style license.
+
+copyright: Copyright (c) 2006-2010 [Valerio Proietti](http://mad4milk.net/).
+
+authors: The MooTools production team (http://mootools.net/developers/)
+
+inspiration:
+ - Class implementation inspired by [Base.js](http://dean.edwards.name/weblog/2006/03/base/) Copyright (c) 2006 Dean Edwards, [GNU Lesser General Public License](http://opensource.org/licenses/lgpl-license.php)
+ - Some functionality inspired by [Prototype.js](http://prototypejs.org) Copyright (c) 2005-2007 Sam Stephenson, [MIT License](http://opensource.org/licenses/mit-license.php)
+
+provides: [Core, MooTools, Type, typeOf, instanceOf, Native]
+
+...
+*/
+
+(function(){
+
+this.MooTools = {
+ version: '1.3dev',
+ build: '29de8c7a2a562582a4ec2e42feb8cd667217b3fa'
+};
+
+// typeOf, instanceOf
+
+var typeOf = this.typeOf = function(item){
+ if (item == null) return 'null';
+ if (item.$family) return item.$family();
+
+ if (item.nodeName){
+ if (item.nodeType == 1) return 'element';
+ if (item.nodeType == 3) return (/\S/).test(item.nodeValue) ? 'textnode' : 'whitespace';
+ } else if (typeof item.length == 'number'){
+ if (item.callee) return 'arguments';
+ if ('item' in item) return 'collection';
+ }
+
+ return typeof item;
+};
+
+var instanceOf = this.instanceOf = function(item, object){
+ if (item == null) return false;
+ var constructor = item.$constructor || item.constructor;
+ while (constructor){
+ if (constructor === object) return true;
+ constructor = constructor.parent;
+ }
+ return item instanceof object;
+};
+
+// Function overloading
+
+var Function = this.Function;
+
+var enumerables = true;
+for (var i in {toString: 1}) enumerables = null;
+if (enumerables) enumerables = ['hasOwnProperty', 'valueOf', 'isPrototypeOf', 'propertyIsEnumerable', 'toLocaleString', 'toString', 'constructor'];
+
+Function.prototype.overloadSetter = function(usePlural){
+ var self = this;
+ return function(a, b){
+ if (a == null) return this;
+ if (usePlural || typeof a != 'string'){
+ for (var k in a) self.call(this, k, a[k]);
+ if (enumerables) for (var i = enumerables.length; i--;){
+ k = enumerables[i];
+ if (a.hasOwnProperty(k)) self.call(this, k, a[k]);
+ }
+ } else {
+ self.call(this, a, b);
+ }
+ return this;
+ };
+};
+
+Function.prototype.overloadGetter = function(usePlural){
+ var self = this;
+ return function(a){
+ var args, result;
+ if (usePlural || typeof a != 'string') args = a;
+ else if (arguments.length > 1) args = arguments;
+ if (args){
+ result = {};
+ for (var i = 0; i < args.length; i++) result[args[i]] = self.call(this, args[i]);
+ } else {
+ result = self.call(this, a);
+ }
+ return result;
+ };
+};
+
+Function.prototype.extend = function(key, value){
+ this[key] = value;
+}.overloadSetter();
+
+Function.prototype.implement = function(key, value){
+ this.prototype[key] = value;
+}.overloadSetter();
+
+// From
+
+var slice = Array.prototype.slice;
+
+Function.from = function(item){
+ return (typeOf(item) == 'function') ? item : function(){
+ return item;
+ };
+};
+
+Array.from = function(item){
+ if (item == null) return [];
+ return (Type.isEnumerable(item) && typeof item != 'string') ? (typeOf(item) == 'array') ? item : slice.call(item) : [item];
+};
+
+Number.from = function(item){
+ var number = parseFloat(item);
+ return isFinite(number) ? number : null;
+};
+
+String.from = function(item){
+ return item + '';
+};
+
+// hide, protect
+
+Function.implement({
+
+ hide: function(){
+ this.$hidden = true;
+ return this;
+ },
+
+ protect: function(){
+ this.$protected = true;
+ return this;
+ }
+
+});
+
+// Type
+
+var Type = this.Type = function(name, object){
+ if (name){
+ var lower = name.toLowerCase();
+ var typeCheck = function(item){
+ return (typeOf(item) == lower);
+ };
+
+ Type['is' + name] = typeCheck;
+ if (object != null){
+ object.prototype.$family = (function(){
+ return lower;
+ }).hide();
+ //<1.2compat>
+ object.type = typeCheck;
+ //</1.2compat>
+ }
+ }
+
+ if (object == null) return null;
+
+ object.extend(this);
+ object.$constructor = Type;
+ object.prototype.$constructor = object;
+
+ return object;
+};
+
+var toString = Object.prototype.toString;
+
+Type.isEnumerable = function(item){
+ return (item != null && typeof item.length == 'number' && toString.call(item) != '[object Function]' );
+};
+
+var hooks = {};
+
+var hooksOf = function(object){
+ var type = typeOf(object.prototype);
+ return hooks[type] || (hooks[type] = []);
+};
+
+var implement = function(name, method){
+ if (method && method.$hidden) return this;
+
+ var hooks = hooksOf(this);
+
+ for (var i = 0; i < hooks.length; i++){
+ var hook = hooks[i];
+ if (typeOf(hook) == 'type') implement.call(hook, name, method);
+ else hook.call(this, name, method);
+ }
+
+ var previous = this.prototype[name];
+ if (previous == null || !previous.$protected) this.prototype[name] = method;
+
+ if (this[name] == null && typeOf(method) == 'function') extend.call(this, name, function(item){
+ return method.apply(item, slice.call(arguments, 1));
+ });
+
+ return this;
+};
+
+var extend = function(name, method){
+ if (method && method.$hidden) return this;
+ var previous = this[name];
+ if (previous == null || !previous.$protected) this[name] = method;
+ return this;
+};
+
+Type.implement({
+
+ implement: implement.overloadSetter(),
+
+ extend: extend.overloadSetter(),
+
+ alias: function(name, existing){
+ implement.call(this, name, this.prototype[existing]);
+ }.overloadSetter(),
+
+ mirror: function(hook){
+ hooksOf(this).push(hook);
+ return this;
+ }
+
+});
+
+new Type('Type', Type);
+
+// Default Types
+
+var force = function(name, object, methods){
+ var isType = (object != Object),
+ prototype = object.prototype;
+
+ if (isType) object = new Type(name, object);
+
+ for (var i = 0, l = methods.length; i < l; i++){
+ var key = methods[i],
+ generic = object[key],
+ proto = prototype[key];
+
+ if (generic) generic.protect();
+
+ if (isType && proto){
+ delete prototype[key];
+ prototype[key] = proto.protect();
+ }
+ }
+
+ if (isType) object.implement(prototype);
+
+ return force;
+};
+
+force('String', String, [
+ 'charAt', 'charCodeAt', 'concat', 'indexOf', 'lastIndexOf', 'match', 'quote', 'replace', 'search',
+ 'slice', 'split', 'substr', 'substring', 'toLowerCase', 'toUpperCase'
+])('Array', Array, [
+ 'pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift', 'concat', 'join', 'slice',
+ 'indexOf', 'lastIndexOf', 'filter', 'forEach', 'every', 'map', 'some', 'reduce', 'reduceRight'
+])('Number', Number, [
+ 'toExponential', 'toFixed', 'toLocaleString', 'toPrecision'
+])('Function', Function, [
+ 'apply', 'call', 'bind'
+])('RegExp', RegExp, [
+ 'exec', 'test'
+])('Object', Object, [
+ 'create', 'defineProperty', 'defineProperties', 'keys',
+ 'getPrototypeOf', 'getOwnPropertyDescriptor', 'getOwnPropertyNames',
+ 'preventExtensions', 'isExtensible', 'seal', 'isSealed', 'freeze', 'isFrozen'
+])('Date', Date, ['now']);
+
+Object.extend = extend.overloadSetter();
+
+Date.extend('now', function(){
+ return +(new Date);
+});
+
+new Type('Boolean', Boolean);
+
+// fixes NaN returning as Number
+
+Number.prototype.$family = function(){
+ return isFinite(this) ? 'number' : 'null';
+}.hide();
+
+// Number.random
+
+Number.extend('random', function(min, max){
+ return Math.floor(Math.random() * (max - min + 1) + min);
+});
+
+// forEach, each
+
+Object.extend('forEach', function(object, fn, bind){
+ for (var key in object){
+ if (object.hasOwnProperty(key)) fn.call(bind, object[key], key, object);
+ }
+});
+
+Object.each = Object.forEach;
+
+Array.implement({
+
+ forEach: function(fn, bind){
+ for (var i = 0, l = this.length; i < l; i++){
+ if (i in this) fn.call(bind, this[i], i, this);
+ }
+ },
+
+ each: function(fn, bind){
+ Array.forEach(this, fn, bind);
+ return this;
+ }
+
+});
+
+// Array & Object cloning, Object merging and appending
+
+var cloneOf = function(item){
+ switch (typeOf(item)){
+ case 'array': return item.clone();
+ case 'object': return Object.clone(item);
+ default: return item;
+ }
+};
+
+Array.implement('clone', function(){
+ var i = this.length, clone = new Array(i);
+ while (i--) clone[i] = cloneOf(this[i]);
+ return clone;
+});
+
+var mergeOne = function(source, key, current){
+ switch (typeOf(current)){
+ case 'object':
+ if (typeOf(source[key]) == 'object') Object.merge(source[key], current);
+ else source[key] = Object.clone(current);
+ break;
+ case 'array': source[key] = current.clone(); break;
+ default: source[key] = current;
+ }
+ return source;
+};
+
+Object.extend({
+
+ merge: function(source, k, v){
+ if (typeOf(k) == 'string') return mergeOne(source, k, v);
+ for (var i = 1, l = arguments.length; i < l; i++){
+ var object = arguments[i];
+ for (var key in object) mergeOne(source, key, object[key]);
+ }
+ return source;
+ },
+
+ clone: function(object){
+ var clone = {};
+ for (var key in object) clone[key] = cloneOf(object[key]);
+ return clone;
+ },
+
+ append: function(original){
+ for (var i = 1, l = arguments.length; i < l; i++){
+ var extended = arguments[i] || {};
+ for (var key in extended) original[key] = extended[key];
+ }
+ return original;
+ }
+
+});
+
+// Object-less types
+
+['Object', 'WhiteSpace', 'TextNode', 'Collection', 'Arguments'].each(function(name){
+ new Type(name);
+});
+
+// Unique ID
+
+var UID = Date.now();
+
+String.extend('generateUID', function(){
+ return (UID++).toString(36);
+});
+
+//<1.2compat>
+
+var Hash = this.Hash = new Type('Hash', function(object){
+ if (typeOf(object) == 'hash') object = Object.clone(object.getClean());
+ for (var key in object) this[key] = object[key];
+ return this;
+});
+
+Hash.implement({
+
+ forEach: function(fn, bind){
+ Object.forEach(this, fn, bind);
+ },
+
+ getClean: function(){
+ var clean = {};
+ for (var key in this){
+ if (this.hasOwnProperty(key)) clean[key] = this[key];
+ }
+ return clean;
+ },
+
+ getLength: function(){
+ var length = 0;
+ for (var key in this){
+ if (this.hasOwnProperty(key)) length++;
+ }
+ return length;
+ }
+
+});
+
+Hash.alias('each', 'forEach');
+
+Object.type = Type.isObject;
+
+var Native = this.Native = function(properties){
+ return new Type(properties.name, properties.initialize);
+};
+
+Native.type = Type.type;
+
+Native.implement = function(objects, methods){
+ for (var i = 0; i < objects.length; i++) objects[i].implement(methods);
+ return Native;
+};
+
+var arrayType = Array.type;
+Array.type = function(item){
+ return instanceOf(item, Array) || arrayType(item);
+};
+
+this.$A = function(item){
+ return Array.from(item).slice();
+};
+
+this.$arguments = function(i){
+ return function(){
+ return arguments[i];
+ };
+};
+
+this.$chk = function(obj){
+ return !!(obj || obj === 0);
+};
+
+this.$clear = function(timer){
+ clearTimeout(timer);
+ clearInterval(timer);
+ return null;
+};
+
+this.$defined = function(obj){
+ return (obj != null);
+};
+
+this.$each = function(iterable, fn, bind){
+ var type = typeOf(iterable);
+ ((type == 'arguments' || type == 'collection' || type == 'array' || type == 'elements') ? Array : Object).each(iterable, fn, bind);
+};
+
+this.$empty = function(){};
+
+this.$extend = function(original, extended){
+ return Object.append(original, extended);
+};
+
+this.$H = function(object){
+ return new Hash(object);
+};
+
+this.$merge = function(){
+ var args = Array.slice(arguments);
+ args.unshift({});
+ return Object.merge.apply(null, args);
+};
+
+this.$lambda = Function.from;
+this.$mixin = Object.merge;
+this.$random = Number.random;
+this.$splat = Array.from;
+this.$time = Date.now;
+
+this.$type = function(object){
+ var type = typeOf(object);
+ if (type == 'elements') return 'array';
+ return (type == 'null') ? false : type;
+};
+
+this.$unlink = function(object){
+ switch (typeOf(object)){
+ case 'object': return Object.clone(object);
+ case 'array': return Array.clone(object);
+ case 'hash': return new Hash(object);
+ default: return object;
+ }
+};
+
+//</1.2compat>
+
+})();
+
+
+/*
+---
+
+name: Array
+
+description: Contains Array Prototypes like each, contains, and erase.
+
+license: MIT-style license.
+
+requires: Type
+
+provides: Array
+
+...
+*/
+
+Array.implement({
+
+ invoke: function(methodName){
+ var args = Array.slice(arguments, 1);
+ return this.map(function(item){
+ return item[methodName].apply(item, args);
+ });
+ },
+
+ every: function(fn, bind){
+ for (var i = 0, l = this.length; i < l; i++){
+ if ((i in this) && !fn.call(bind, this[i], i, this)) return false;
+ }
+ return true;
+ },
+
+ filter: function(fn, bind){
+ var results = [];
+ for (var i = 0, l = this.length; i < l; i++){
+ if ((i in this) && fn.call(bind, this[i], i, this)) results.push(this[i]);
+ }
+ return results;
+ },
+
+ clean: function(){
+ return this.filter(function(item){
+ return item != null;
+ });
+ },
+
+ indexOf: function(item, from){
+ var len = this.length;
+ for (var i = (from < 0) ? Math.max(0, len + from) : from || 0; i < len; i++){
+ if (this[i] === item) return i;
+ }
+ return -1;
+ },
+
+ map: function(fn, bind){
+ var results = [];
+ for (var i = 0, l = this.length; i < l; i++){
+ if (i in this) results[i] = fn.call(bind, this[i], i, this);
+ }
+ return results;
+ },
+
+ some: function(fn, bind){
+ for (var i = 0, l = this.length; i < l; i++){
+ if ((i in this) && fn.call(bind, this[i], i, this)) return true;
+ }
+ return false;
+ },
+
+ associate: function(keys){
+ var obj = {}, length = Math.min(this.length, keys.length);
+ for (var i = 0; i < length; i++) obj[keys[i]] = this[i];
+ return obj;
+ },
+
+ link: function(object){
+ var result = {};
+ for (var i = 0, l = this.length; i < l; i++){
+ for (var key in object){
+ if (object[key](this[i])){
+ result[key] = this[i];
+ delete object[key];
+ break;
+ }
+ }
+ }
+ return result;
+ },
+
+ contains: function(item, from){
+ return this.indexOf(item, from) != -1;
+ },
+
+ append: function(array){
+ this.push.apply(this, array);
+ return this;
+ },
+
+ getLast: function(){
+ return (this.length) ? this[this.length - 1] : null;
+ },
+
+ getRandom: function(){
+ return (this.length) ? this[Number.random(0, this.length - 1)] : null;
+ },
+
+ include: function(item){
+ if (!this.contains(item)) this.push(item);
+ return this;
+ },
+
+ combine: function(array){
+ for (var i = 0, l = array.length; i < l; i++) this.include(array[i]);
+ return this;
+ },
+
+ erase: function(item){
+ for (var i = this.length; i--;){
+ if (this[i] === item) this.splice(i, 1);
+ }
+ return this;
+ },
+
+ empty: function(){
+ this.length = 0;
+ return this;
+ },
+
+ flatten: function(){
+ var array = [];
+ for (var i = 0, l = this.length; i < l; i++){
+ var type = typeOf(this[i]);
+ if (type == 'null') continue;
+ array = array.concat((type == 'array' || type == 'collection' || type == 'arguments' || instanceOf(this[i], Array)) ? Array.flatten(this[i]) : this[i]);
+ }
+ return array;
+ },
+
+ pick: function(){
+ for (var i = 0, l = this.length; i < l; i++){
+ if (this[i] != null) return this[i];
+ }
+ return null;
+ },
+
+ hexToRgb: function(array){
+ if (this.length != 3) return null;
+ var rgb = this.map(function(value){
+ if (value.length == 1) value += value;
+ return value.toInt(16);
+ });
+ return (array) ? rgb : 'rgb(' + rgb + ')';
+ },
+
+ rgbToHex: function(array){
+ if (this.length < 3) return null;
+ if (this.length == 4 && this[3] == 0 && !array) return 'transparent';
+ var hex = [];
+ for (var i = 0; i < 3; i++){
+ var bit = (this[i] - 0).toString(16);
+ hex.push((bit.length == 1) ? '0' + bit : bit);
+ }
+ return (array) ? hex : '#' + hex.join('');
+ }
+
+});
+
+//<1.2compat>
+
+Array.alias('extend', 'append');
+
+var $pick = function(){
+ return Array.from(arguments).pick();
+};
+
+//</1.2compat>
+
+
+/*
+---
+
+name: String
+
+description: Contains String Prototypes like camelCase, capitalize, test, and toInt.
+
+license: MIT-style license.
+
+requires: Type
+
+provides: String
+
+...
+*/
+
+String.implement({
+
+ test: function(regex, params){
+ return ((typeOf(regex) == 'regexp') ? regex : new RegExp('' + regex, params)).test(this);
+ },
+
+ contains: function(string, separator){
+ return (separator) ? (separator + this + separator).indexOf(separator + string + separator) > -1 : this.indexOf(string) > -1;
+ },
+
+ trim: function(){
+ return this.replace(/^\s+|\s+$/g, '');
+ },
+
+ clean: function(){
+ return this.replace(/\s+/g, ' ').trim();
+ },
+
+ camelCase: function(){
+ return this.replace(/-\D/g, function(match){
+ return match.charAt(1).toUpperCase();
+ });
+ },
+
+ hyphenate: function(){
+ return this.replace(/[A-Z]/g, function(match){
+ return ('-' + match.charAt(0).toLowerCase());
+ });
+ },
+
+ capitalize: function(){
+ return this.replace(/\b[a-z]/g, function(match){
+ return match.toUpperCase();
+ });
+ },
+
+ escapeRegExp: function(){
+ return this.replace(/([-.*+?^${}()|[\]\/\\])/g, '\\$1');
+ },
+
+ toInt: function(base){
+ return parseInt(this, base || 10);
+ },
+
+ toFloat: function(){
+ return parseFloat(this);
+ },
+
+ hexToRgb: function(array){
+ var hex = this.match(/^#?(\w{1,2})(\w{1,2})(\w{1,2})$/);
+ return (hex) ? hex.slice(1).hexToRgb(array) : null;
+ },
+
+ rgbToHex: function(array){
+ var rgb = this.match(/\d{1,3}/g);
+ return (rgb) ? rgb.rgbToHex(array) : null;
+ },
+
+ substitute: function(object, regexp){
+ return this.replace(regexp || (/\\?\{([^{}]+)\}/g), function(match, name){
+ if (match.charAt(0) == '\\') return match.slice(1);
+ return (object[name] != null) ? object[name] : '';
+ });
+ }
+
+});
+
+
+/*
+---
+
+name: Number
+
+description: Contains Number Prototypes like limit, round, times, and ceil.
+
+license: MIT-style license.
+
+requires: Type
+
+provides: Number
+
+...
+*/
+
+Number.implement({
+
+ limit: function(min, max){
+ return Math.min(max, Math.max(min, this));
+ },
+
+ round: function(precision){
+ precision = Math.pow(10, precision || 0).toFixed(precision < 0 ? -precision : 0);
+ return Math.round(this * precision) / precision;
+ },
+
+ times: function(fn, bind){
+ for (var i = 0; i < this; i++) fn.call(bind, i, this);
+ },
+
+ toFloat: function(){
+ return parseFloat(this);
+ },
+
+ toInt: function(base){
+ return parseInt(this, base || 10);
+ }
+
+});
+
+Number.alias('each', 'times');
+
+(function(math){
+ var methods = {};
+ math.each(function(name){
+ if (!Number[name]) methods[name] = function(){
+ return Math[name].apply(null, [this].concat(Array.from(arguments)));
+ };
+ });
+ Number.implement(methods);
+})(['abs', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'exp', 'floor', 'log', 'max', 'min', 'pow', 'sin', 'sqrt', 'tan']);
+
+
+/*
+---
+
+name: Function
+
+description: Contains Function Prototypes like create, bind, pass, and delay.
+
+license: MIT-style license.
+
+requires: Type
+
+provides: Function
+
+...
+*/
+
+Function.extend({
+
+ attempt: function(){
+ for (var i = 0, l = arguments.length; i < l; i++){
+ try {
+ return arguments[i]();
+ } catch (e){}
+ }
+ return null;
+ }
+
+});
+
+Function.implement({
+
+ attempt: function(args, bind){
+ try {
+ return this.apply(bind, Array.from(args));
+ } catch (e){}
+
+ return null;
+ },
+
+ bind: function(bind){
+ var self = this,
+ args = (arguments.length > 1) ? Array.slice(arguments, 1) : null;
+
+ return function(){
+ if (!args && !arguments.length) return self.call(bind);
+ if (args && arguments.length) return self.apply(bind, args.concat(Array.from(arguments)));
+ return self.apply(bind, args || arguments);
+ };
+ },
+
+ pass: function(args, bind){
+ var self = this;
+ if (args != null) args = Array.from(args);
+ return function(){
+ return self.apply(bind, args || arguments);
+ };
+ },
+
+ delay: function(delay, bind, args){
+ return setTimeout(this.pass(args, bind), delay);
+ },
+
+ periodical: function(periodical, bind, args){
+ return setInterval(this.pass(args, bind), periodical);
+ }
+
+});
+
+//<1.2compat>
+
+delete Function.prototype.bind;
+
+Function.implement({
+
+ create: function(options){
+ var self = this;
+ options = options || {};
+ return function(event){
+ var args = options.arguments;
+ args = (args != null) ? Array.from(args) : Array.slice(arguments, (options.event) ? 1 : 0);
+ if (options.event) args = [event || window.event].extend(args);
+ var returns = function(){
+ return self.apply(options.bind || null, args);
+ };
+ if (options.delay) return setTimeout(returns, options.delay);
+ if (options.periodical) return setInterval(returns, options.periodical);
+ if (options.attempt) return Function.attempt(returns);
+ return returns();
+ };
+ },
+
+ bind: function(bind, args){
+ var self = this;
+ if (args != null) args = Array.from(args);
+ return function(){
+ return self.apply(bind, args || arguments);
+ };
+ },
+
+ bindWithEvent: function(bind, args){
+ var self = this;
+ if (args != null) args = Array.from(args);
+ return function(event){
+ return self.apply(bind, (args == null) ? arguments : [event].concat(args));
+ };
+ },
+
+ run: function(args, bind){
+ return this.apply(bind, Array.from(args));
+ }
+
+});
+
+var $try = Function.attempt;
+
+//</1.2compat>
+
+
+/*
+---
+
+name: Object
+
+description: Object generic methods
+
+license: MIT-style license.
+
+requires: Type
+
+provides: [Object, Hash]
+
+...
+*/
+
+
+Object.extend({
+
+ subset: function(object, keys){
+ var results = {};
+ for (var i = 0, l = keys.length; i < l; i++){
+ var k = keys[i];
+ results[k] = object[k];
+ }
+ return results;
+ },
+
+ map: function(object, fn, bind){
+ var results = {};
+ for (var key in object){
+ if (object.hasOwnProperty(key)) results[key] = fn.call(bind, object[key], key, object);
+ }
+ return results;
+ },
+
+ filter: function(object, fn, bind){
+ var results = {};
+ Object.each(object, function(value, key){
+ if (fn.call(bind, value, key, object)) results[key] = value;
+ });
+ return results;
+ },
+
+ every: function(object, fn, bind){
+ for (var key in object){
+ if (object.hasOwnProperty(key) && !fn.call(bind, object[key], key)) return false;
+ }
+ return true;
+ },
+
+ some: function(object, fn, bind){
+ for (var key in object){
+ if (object.hasOwnProperty(key) && fn.call(bind, object[key], key)) return true;
+ }
+ return false;
+ },
+
+ keys: function(object){
+ var keys = [];
+ for (var key in object){
+ if (object.hasOwnProperty(key)) keys.push(key);
+ }
+ return keys;
+ },
+
+ values: function(object){
+ var values = [];
+ for (var key in object){
+ if (object.hasOwnProperty(key)) values.push(object[key]);
+ }
+ return values;
+ },
+
+ getLength: function(object){
+ return Object.keys(object).length;
+ },
+
+ keyOf: function(object, value){
+ for (var key in object){
+ if (object.hasOwnProperty(key) && object[key] === value) return key;
+ }
+ return null;
+ },
+
+ contains: function(object, value){
+ return Object.keyOf(object, value) != null;
+ },
+
+ toQueryString: function(object, base){
+ var queryString = [];
+
+ Object.each(object, function(value, key){
+ if (base) key = base + '[' + key + ']';
+ var result;
+ switch (typeOf(value)){
+ case 'object': result = Object.toQueryString(value, key); break;
+ case 'array':
+ var qs = {};
+ value.each(function(val, i){
+ qs[i] = val;
+ });
+ result = Object.toQueryString(qs, key);
+ break;
+ default: result = key + '=' + encodeURIComponent(value);
+ }
+ if (value != null) queryString.push(result);
+ });
+
+ return queryString.join('&');
+ }
+
+});
+
+
+//<1.2compat>
+
+Hash.implement({
+
+ has: Object.prototype.hasOwnProperty,
+
+ keyOf: function(value){
+ return Object.keyOf(this, value);
+ },
+
+ hasValue: function(value){
+ return Object.contains(this, value);
+ },
+
+ extend: function(properties){
+ Hash.each(properties || {}, function(value, key){
+ Hash.set(this, key, value);
+ }, this);
+ return this;
+ },
+
+ combine: function(properties){
+ Hash.each(properties || {}, function(value, key){
+ Hash.include(this, key, value);
+ }, this);
+ return this;
+ },
+
+ erase: function(key){
+ if (this.hasOwnProperty(key)) delete this[key];
+ return this;
+ },
+
+ get: function(key){
+ return (this.hasOwnProperty(key)) ? this[key] : null;
+ },
+
+ set: function(key, value){
+ if (!this[key] || this.hasOwnProperty(key)) this[key] = value;
+ return this;
+ },
+
+ empty: function(){
+ Hash.each(this, function(value, key){
+ delete this[key];
+ }, this);
+ return this;
+ },
+
+ include: function(key, value){
+ if (this[key] == null) this[key] = value;
+ return this;
+ },
+
+ map: function(fn, bind){
+ return new Hash(Object.map(this, fn, bind));
+ },
+
+ filter: function(fn, bind){
+ return new Hash(Object.filter(this, fn, bind));
+ },
+
+ every: function(fn, bind){
+ return Object.every(this, fn, bind);
+ },
+
+ some: function(fn, bind){
+ return Object.some(this, fn, bind);
+ },
+
+ getKeys: function(){
+ return Object.keys(this);
+ },
+
+ getValues: function(){
+ return Object.values(this);
+ },
+
+ toQueryString: function(base){
+ return Object.toQueryString(this, base);
+ }
+
+});
+
+Hash.extend = Object.append;
+
+Hash.alias({indexOf: 'keyOf', contains: 'hasValue'});
+
+//</1.2compat>
+
+
+/*
+---
+
+name: Browser
+
+description: The Browser Object. Contains Browser initialization, Window and Document, and the Browser Hash.
+
+license: MIT-style license.
+
+requires: [Array, Function, Number, String]
+
+provides: [Browser, Window, Document]
+
+...
+*/
+
+(function(){
+
+var document = this.document;
+var window = document.window = this;
+
+var UID = 1;
+
+this.$uid = (window.ActiveXObject) ? function(item){
+ return (item.uid || (item.uid = [UID++]))[0];
+} : function(item){
+ return item.uid || (item.uid = UID++);
+};
+
+$uid(window);
+$uid(document);
+
+var ua = navigator.userAgent.toLowerCase(),
+ platform = navigator.platform.toLowerCase(),
+ UA = ua.match(/(opera|ie|firefox|chrome|version)[\s\/:]([\w\d\.]+)?.*?(safari|version[\s\/:]([\w\d\.]+)|$)/) || [null, 'unknown', 0];
+
+var Browser = this.Browser = {
+
+ extend: Function.prototype.extend,
+
+ name: (UA[1] == 'version') ? UA[3] : UA[1],
+
+ version: parseFloat((UA[1] == 'opera' && UA[4]) ? UA[4] : UA[2]),
+
+ Platform: {
+ name: ua.match(/ip(?:ad|od|hone)/) ? 'ios' : (ua.match(/(?:webos|android)/) || platform.match(/mac|win|linux/) || ['other'])[0]
+ },
+
+ Features: {
+ xpath: !!(document.evaluate),
+ air: !!(window.runtime),
+ query: !!(document.querySelector),
+ json: !!(window.JSON)
+ },
+
+ Plugins: {}
+
+};
+
+Browser[Browser.name] = true;
+Browser[Browser.name + parseInt(Browser.version, 10)] = true;
+Browser.Platform[Browser.Platform.name] = true;
+
+// Request
+
+Browser.Request = (function(){
+
+ var XMLHTTP = function(){
+ return new XMLHttpRequest();
+ };
+
+ var MSXML2 = function(){
+ return new ActiveXObject('MSXML2.XMLHTTP');
+ };
+
+ var MSXML = function(){
+ return new ActiveXObject('Microsoft.XMLHTTP');
+ };
+
+ return Function.attempt(function(){
+ XMLHTTP();
+ return XMLHTTP;
+ }, function(){
+ MSXML2();
+ return MSXML2;
+ }, function(){
+ MSXML();
+ return MSXML;
+ });
+
+})();
+
+Browser.Features.xhr = !!(Browser.Request);
+
+// Flash detection
+
+var version = (Function.attempt(function(){
+ return navigator.plugins['Shockwave Flash'].description;
+}, function(){
+ return new ActiveXObject('ShockwaveFlash.ShockwaveFlash').GetVariable('$version');
+}) || '0 r0').match(/\d+/g);
+
+Browser.Plugins.Flash = {
+ version: Number(version[0] || '0.' + version[1]) || 0,
+ build: Number(version[2]) || 0
+};
+
+// String scripts
+
+Browser.exec = function(text){
+ if (!text) return text;
+ if (window.execScript){
+ window.execScript(text);
+ } else {
+ var script = document.createElement('script');
+ script.setAttribute('type', 'text/javascript');
+ script.text = text;
+ document.head.appendChild(script);
+ document.head.removeChild(script);
+ }
+ return text;
+};
+
+String.implement('stripScripts', function(exec){
+ var scripts = '';
+ var text = this.replace(/<script[^>]*>([\s\S]*?)<\/script>/gi, function(all, code){
+ scripts += code + '\n';
+ return '';
+ });
+ if (exec === true) Browser.exec(scripts);
+ else if (typeOf(exec) == 'function') exec(scripts, text);
+ return text;
+});
+
+// Window, Document
+
+Browser.extend({
+ Document: this.Document,
+ Window: this.Window,
+ Element: this.Element,
+ Event: this.Event
+});
+
+this.Window = this.$constructor = new Type('Window', function(){});
+
+this.$family = Function.from('window').hide();
+
+Window.mirror(function(name, method){
+ window[name] = method;
+});
+
+this.Document = document.$constructor = new Type('Document', function(){});
+
+document.$family = Function.from('document').hide();
+
+Document.mirror(function(name, method){
+ document[name] = method;
+});
+
+document.html = document.documentElement;
+document.head = document.getElementsByTagName('head')[0];
+
+if (document.execCommand) try {
+ document.execCommand("BackgroundImageCache", false, true);
+} catch (e){}
+
+if (this.attachEvent && !this.addEventListener){
+ var unloadEvent = function(){
+ this.detachEvent('onunload', unloadEvent);
+ document.head = document.html = document.window = null;
+ };
+ this.attachEvent('onunload', unloadEvent);
+}
+
+// IE fails on collections and <select>.options (refers to <select>)
+var arrayFrom = Array.from;
+try {
+ arrayFrom(document.html.childNodes);
+} catch(e){
+ Array.from = function(item){
+ if (typeof item != 'string' && Type.isEnumerable(item) && typeOf(item) != 'array'){
+ var i = item.length, array = new Array(i);
+ while (i--) array[i] = item[i];
+ return array;
+ }
+ return arrayFrom(item);
+ };
+
+ var prototype = Array.prototype,
+ slice = prototype.slice;
+ ['pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift', 'concat', 'join', 'slice'].each(function(name){
+ var method = prototype[name];
+ Array[name] = function(item){
+ return method.apply(Array.from(item), slice.call(arguments, 1));
+ };
+ });
+}
+
+//<1.2compat>
+
+if (Browser.Platform.ios) Browser.Platform.ipod = true;
+
+Browser.Engine = {};
+
+var setEngine = function(name, version){
+ Browser.Engine.name = name;
+ Browser.Engine[name + version] = true;
+ Browser.Engine.version = version;
+};
+
+if (Browser.ie){
+ Browser.Engine.trident = true;
+
+ switch (Browser.version){
+ case 6: setEngine('trident', 4); break;
+ case 7: setEngine('trident', 5); break;
+ case 8: setEngine('trident', 6);
+ }
+}
+
+if (Browser.firefox){
+ Browser.Engine.gecko = true;
+
+ if (Browser.version >= 3) setEngine('gecko', 19);
+ else setEngine('gecko', 18);
+}
+
+if (Browser.safari || Browser.chrome){
+ Browser.Engine.webkit = true;
+
+ switch (Browser.version){
+ case 2: setEngine('webkit', 419); break;
+ case 3: setEngine('webkit', 420); break;
+ case 4: setEngine('webkit', 525);
+ }
+}
+
+if (Browser.opera){
+ Browser.Engine.presto = true;
+
+ if (Browser.version >= 9.6) setEngine('presto', 960);
+ else if (Browser.version >= 9.5) setEngine('presto', 950);
+ else setEngine('presto', 925);
+}
+
+if (Browser.name == 'unknown'){
+ switch ((ua.match(/(?:webkit|khtml|gecko)/) || [])[0]){
+ case 'webkit':
+ case 'khtml':
+ Browser.Engine.webkit = true;
+ break;
+ case 'gecko':
+ Browser.Engine.gecko = true;
+ }
+}
+
+this.$exec = Browser.exec;
+
+//</1.2compat>
+
+})();
+
+
+/*
+---
+
+name: Event
+
+description: Contains the Event Class, to make the event object cross-browser.
+
+license: MIT-style license.
+
+requires: [Window, Document, Array, Function, String, Object]
+
+provides: Event
+
+...
+*/
+
+var Event = new Type('Event', function(event, win){
+ if (!win) win = window;
+ var doc = win.document;
+ event = event || win.event;
+ if (event.$extended) return event;
+ this.$extended = true;
+ var type = event.type,
+ target = event.target || event.srcElement,
+ page = {},
+ client = {};
+ while (target && target.nodeType == 3) target = target.parentNode;
+
+ if (type.indexOf('key') != -1){
+ var code = event.which || event.keyCode;
+ var key = Object.keyOf(Event.Keys, code);
+ if (type == 'keydown'){
+ var fKey = code - 111;
+ if (fKey > 0 && fKey < 13) key = 'f' + fKey;
+ }
+ if (!key) key = String.fromCharCode(code).toLowerCase();
+ } else if (type.test(/click|mouse|menu/i)){
+ doc = (!doc.compatMode || doc.compatMode == 'CSS1Compat') ? doc.html : doc.body;
+ page = {
+ x: (event.pageX != null) ? event.pageX : event.clientX + doc.scrollLeft,
+ y: (event.pageY != null) ? event.pageY : event.clientY + doc.scrollTop
+ };
+ client = {
+ x: (event.pageX != null) ? event.pageX - win.pageXOffset : event.clientX,
+ y: (event.pageY != null) ? event.pageY - win.pageYOffset : event.clientY
+ };
+ if (type.test(/DOMMouseScroll|mousewheel/)){
+ var wheel = (event.wheelDelta) ? event.wheelDelta / 120 : -(event.detail || 0) / 3;
+ }
+ var rightClick = (event.which == 3) || (event.button == 2),
+ related = null;
+ if (type.test(/over|out/)){
+ related = event.relatedTarget || event[(type == 'mouseover' ? 'from' : 'to') + 'Element'];
+ var testRelated = function(){
+ while (related && related.nodeType == 3) related = related.parentNode;
+ return true;
+ };
+ var hasRelated = (Browser.firefox2) ? testRelated.attempt() : testRelated();
+ related = (hasRelated) ? related : null;
+ }
+ } else if (type.test(/gesture|touch/i)){
+ this.rotation = event.rotation;
+ this.scale = event.scale;
+ this.targetTouches = event.targetTouches;
+ this.changedTouches = event.changedTouches;
+ var touches = this.touches = event.touches;
+ if (touches && touches[0]){
+ var touch = touches[0];
+ page = {x: touch.pageX, y: touch.pageY};
+ client = {x: touch.clientX, y: touch.clientY};
+ }
+ }
+
+ return Object.append(this, {
+ event: event,
+ type: type,
+
+ page: page,
+ client: client,
+ rightClick: rightClick,
+
+ wheel: wheel,
+
+ relatedTarget: document.id(related),
+ target: document.id(target),
+
+ code: code,
+ key: key,
+
+ shift: event.shiftKey,
+ control: event.ctrlKey,
+ alt: event.altKey,
+ meta: event.metaKey
+ });
+});
+
+Event.Keys = {
+ 'enter': 13,
+ 'up': 38,
+ 'down': 40,
+ 'left': 37,
+ 'right': 39,
+ 'esc': 27,
+ 'space': 32,
+ 'backspace': 8,
+ 'tab': 9,
+ 'delete': 46
+};
+
+//<1.2compat>
+
+Event.Keys = new Hash(Event.Keys);
+
+//</1.2compat>
+
+Event.implement({
+
+ stop: function(){
+ return this.stopPropagation().preventDefault();
+ },
+
+ stopPropagation: function(){
+ if (this.event.stopPropagation) this.event.stopPropagation();
+ else this.event.cancelBubble = true;
+ return this;
+ },
+
+ preventDefault: function(){
+ if (this.event.preventDefault) this.event.preventDefault();
+ else this.event.returnValue = false;
+ return this;
+ }
+
+});
+
+
+/*
+---
+
+name: Class
+
+description: Contains the Class Function for easily creating, extending, and implementing reusable Classes.
+
+license: MIT-style license.
+
+requires: [Array, String, Function, Number]
+
+provides: Class
+
+...
+*/
+
+(function(){
+
+var Class = this.Class = new Type('Class', function(params){
+ if (instanceOf(params, Function)) params = {initialize: params};
+
+ var newClass = function(){
+ reset(this);
+ if (newClass.$prototyping) return this;
+ this.$caller = null;
+ var value = (this.initialize) ? this.initialize.apply(this, arguments) : this;
+ this.$caller = this.caller = null;
+ return value;
+ }.extend(this).implement(params);
+
+ newClass.$constructor = Class;
+ newClass.prototype.$constructor = newClass;
+ newClass.prototype.parent = parent;
+
+ return newClass;
+});
+
+var parent = function(){
+ if (!this.$caller) throw new Error('The method "parent" cannot be called.');
+ var name = this.$caller.$name,
+ parent = this.$caller.$owner.parent,
+ previous = (parent) ? parent.prototype[name] : null;
+ if (!previous) throw new Error('The method "' + name + '" has no parent.');
+ return previous.apply(this, arguments);
+};
+
+var reset = function(object){
+ for (var key in object){
+ var value = object[key];
+ switch (typeOf(value)){
+ case 'object':
+ var F = function(){};
+ F.prototype = value;
+ object[key] = reset(new F);
+ break;
+ case 'array': object[key] = value.clone(); break;
+ }
+ }
+ return object;
+};
+
+var wrap = function(self, key, method){
+ if (method.$origin) method = method.$origin;
+ var wrapper = function(){
+ if (method.$protected && this.$caller == null) throw new Error('The method "' + key + '" cannot be called.');
+ var caller = this.caller, current = this.$caller;
+ this.caller = current; this.$caller = wrapper;
+ var result = method.apply(this, arguments);
+ this.$caller = current; this.caller = caller;
+ return result;
+ }.extend({$owner: self, $origin: method, $name: key});
+ return wrapper;
+};
+
+var implement = function(key, value, retain){
+ if (Class.Mutators.hasOwnProperty(key)){
+ value = Class.Mutators[key].call(this, value);
+ if (value == null) return this;
+ }
+
+ if (typeOf(value) == 'function'){
+ if (value.$hidden) return this;
+ this.prototype[key] = (retain) ? value : wrap(this, key, value);
+ } else {
+ Object.merge(this.prototype, key, value);
+ }
+
+ return this;
+};
+
+var getInstance = function(klass){
+ klass.$prototyping = true;
+ var proto = new klass;
+ delete klass.$prototyping;
+ return proto;
+};
+
+Class.implement('implement', implement.overloadSetter());
+
+Class.Mutators = {
+
+ Extends: function(parent){
+ this.parent = parent;
+ this.prototype = getInstance(parent);
+ },
+
+ Implements: function(items){
+ Array.from(items).each(function(item){
+ var instance = new item;
+ for (var key in instance) implement.call(this, key, instance[key], true);
+ }, this);
+ }
+};
+
+})();
+
+
+/*
+---
+
+name: Class.Extras
+
+description: Contains Utility Classes that can be implemented into your own Classes to ease the execution of many common tasks.
+
+license: MIT-style license.
+
+requires: Class
+
+provides: [Class.Extras, Chain, Events, Options]
+
+...
+*/
+
+(function(){
+
+this.Chain = new Class({
+
+ $chain: [],
+
+ chain: function(){
+ this.$chain.append(Array.flatten(arguments));
+ return this;
+ },
+
+ callChain: function(){
+ return (this.$chain.length) ? this.$chain.shift().apply(this, arguments) : false;
+ },
+
+ clearChain: function(){
+ this.$chain.empty();
+ return this;
+ }
+
+});
+
+var removeOn = function(string){
+ return string.replace(/^on([A-Z])/, function(full, first){
+ return first.toLowerCase();
+ });
+};
+
+this.Events = new Class({
+
+ $events: {},
+
+ addEvent: function(type, fn, internal){
+ type = removeOn(type);
+
+ /*<1.2compat>*/
+ if (fn == $empty) return this;
+ /*</1.2compat>*/
+
+ this.$events[type] = (this.$events[type] || []).include(fn);
+ if (internal) fn.internal = true;
+ return this;
+ },
+
+ addEvents: function(events){
+ for (var type in events) this.addEvent(type, events[type]);
+ return this;
+ },
+
+ fireEvent: function(type, args, delay){
+ type = removeOn(type);
+ var events = this.$events[type];
+ if (!events) return this;
+ args = Array.from(args);
+ events.each(function(fn){
+ if (delay) fn.delay(delay, this, args);
+ else fn.apply(this, args);
+ }, this);
+ return this;
+ },
+
+ removeEvent: function(type, fn){
+ type = removeOn(type);
+ var events = this.$events[type];
+ if (events && !fn.internal){
+ var index = events.indexOf(fn);
+ if (index != -1) delete events[index];
+ }
+ return this;
+ },
+
+ removeEvents: function(events){
+ var type;
+ if (typeOf(events) == 'object'){
+ for (type in events) this.removeEvent(type, events[type]);
+ return this;
+ }
+ if (events) events = removeOn(events);
+ for (type in this.$events){
+ if (events && events != type) continue;
+ var fns = this.$events[type];
+ for (var i = fns.length; i--;) this.removeEvent(type, fns[i]);
+ }
+ return this;
+ }
+
+});
+
+this.Options = new Class({
+
+ setOptions: function(){
+ var options = this.options = Object.merge.apply(null, [{}, this.options].append(arguments));
+ if (!this.addEvent) return this;
+ for (var option in options){
+ if (typeOf(options[option]) != 'function' || !(/^on[A-Z]/).test(option)) continue;
+ this.addEvent(option, options[option]);
+ delete options[option];
+ }
+ return this;
+ }
+
+});
+
+})();
+
+
+/*
+---
+name: Slick.Parser
+description: Standalone CSS3 Selector parser
+provides: Slick.Parser
+...
+*/
+
+(function(){
+
+var parsed,
+ separatorIndex,
+ combinatorIndex,
+ reversed,
+ cache = {},
+ reverseCache = {},
+ reUnescape = /\\/g;
+
+var parse = function(expression, isReversed){
+ if (expression == null) return null;
+ if (expression.Slick === true) return expression;
+ expression = ('' + expression).replace(/^\s+|\s+$/g, '');
+ reversed = !!isReversed;
+ var currentCache = (reversed) ? reverseCache : cache;
+ if (currentCache[expression]) return currentCache[expression];
+ parsed = {Slick: true, expressions: [], raw: expression, reverse: function(){
+ return parse(this.raw, true);
+ }};
+ separatorIndex = -1;
+ while (expression != (expression = expression.replace(regexp, parser)));
+ parsed.length = parsed.expressions.length;
+ return currentCache[expression] = (reversed) ? reverse(parsed) : parsed;
+};
+
+var reverseCombinator = function(combinator){
+ if (combinator === '!') return ' ';
+ else if (combinator === ' ') return '!';
+ else if ((/^!/).test(combinator)) return combinator.replace(/^!/, '');
+ else return '!' + combinator;
+};
+
+var reverse = function(expression){
+ var expressions = expression.expressions;
+ for (var i = 0; i < expressions.length; i++){
+ var exp = expressions[i];
+ var last = {parts: [], tag: '*', combinator: reverseCombinator(exp[0].combinator)};
+
+ for (var j = 0; j < exp.length; j++){
+ var cexp = exp[j];
+ if (!cexp.reverseCombinator) cexp.reverseCombinator = ' ';
+ cexp.combinator = cexp.reverseCombinator;
+ delete cexp.reverseCombinator;
+ }
+
+ exp.reverse().push(last);
+ }
+ return expression;
+};
+
+var escapeRegExp = function(string){// Credit: XRegExp 0.6.1 (c) 2007-2008 Steven Levithan <http://stevenlevithan.com/regex/xregexp/> MIT License
+ return string.replace(/[-[\]{}()*+?.\\^$|,#\s]/g, "\\$&");
+};
+
+var regexp = new RegExp(
+/*
+#!/usr/bin/env ruby
+puts "\t\t" + DATA.read.gsub(/\(\?x\)|\s+#.*$|\s+|\\$|\\n/,'')
+__END__
+ "(?x)^(?:\
+ \\s* ( , ) \\s* # Separator \n\
+ | \\s* ( <combinator>+ ) \\s* # Combinator \n\
+ | ( \\s+ ) # CombinatorChildren \n\
+ | ( <unicode>+ | \\* ) # Tag \n\
+ | \\# ( <unicode>+ ) # ID \n\
+ | \\. ( <unicode>+ ) # ClassName \n\
+ | # Attribute \n\
+ \\[ \
+ \\s* (<unicode1>+) (?: \
+ \\s* ([*^$!~|]?=) (?: \
+ \\s* (?:\
+ ([\"']?)(.*?)\\9 \
+ )\
+ ) \
+ )? \\s* \
+ \\](?!\\]) \n\
+ | :+ ( <unicode>+ )(?:\
+ \\( (?:\
+ (?:([\"'])([^\\12]*)\\12)|((?:\\([^)]+\\)|[^()]*)+)\
+ ) \\)\
+ )?\
+ )"
+*/
+ "^(?:\\s*(,)\\s*|\\s*(<combinator>+)\\s*|(\\s+)|(<unicode>+|\\*)|\\#(<unicode>+)|\\.(<unicode>+)|\\[\\s*(<unicode1>+)(?:\\s*([*^$!~|]?=)(?:\\s*(?:([\"']?)(.*?)\\9)))?\\s*\\](?!\\])|:+(<unicode>+)(?:\\((?:(?:([\"'])([^\\12]*)\\12)|((?:\\([^)]+\\)|[^()]*)+))\\))?)"
+ .replace(/<combinator>/, '[' + escapeRegExp(">+~`!@$%^&={}\\;</") + ']')
+ .replace(/<unicode>/g, '(?:[\\w\\u00a1-\\uFFFF-]|\\\\[^\\s0-9a-f])')
+ .replace(/<unicode1>/g, '(?:[:\\w\\u00a1-\\uFFFF-]|\\\\[^\\s0-9a-f])')
+);
+
+function parser(
+ rawMatch,
+
+ separator,
+ combinator,
+ combinatorChildren,
+
+ tagName,
+ id,
+ className,
+
+ attributeKey,
+ attributeOperator,
+ attributeQuote,
+ attributeValue,
+
+ pseudoClass,
+ pseudoQuote,
+ pseudoClassQuotedValue,
+ pseudoClassValue
+){
+ if (separator || separatorIndex === -1){
+ parsed.expressions[++separatorIndex] = [];
+ combinatorIndex = -1;
+ if (separator) return '';
+ }
+
+ if (combinator || combinatorChildren || combinatorIndex === -1){
+ combinator = combinator || ' ';
+ var currentSeparator = parsed.expressions[separatorIndex];
+ if (reversed && currentSeparator[combinatorIndex])
+ currentSeparator[combinatorIndex].reverseCombinator = reverseCombinator(combinator);
+ currentSeparator[++combinatorIndex] = {combinator: combinator, tag: '*'};
+ }
+
+ var currentParsed = parsed.expressions[separatorIndex][combinatorIndex];
+
+ if (tagName){
+ currentParsed.tag = tagName.replace(reUnescape, '');
+
+ } else if (id){
+ currentParsed.id = id.replace(reUnescape, '');
+
+ } else if (className){
+ className = className.replace(reUnescape, '');
+
+ if (!currentParsed.classList) currentParsed.classList = [];
+ if (!currentParsed.classes) currentParsed.classes = [];
+ currentParsed.classList.push(className);
+ currentParsed.classes.push({
+ value: className,
+ regexp: new RegExp('(^|\\s)' + escapeRegExp(className) + '(\\s|$)')
+ });
+
+ } else if (pseudoClass){
+ pseudoClassValue = pseudoClassValue || pseudoClassQuotedValue;
+ pseudoClassValue = pseudoClassValue ? pseudoClassValue.replace(reUnescape, '') : null;
+
+ if (!currentParsed.pseudos) currentParsed.pseudos = [];
+ currentParsed.pseudos.push({
+ key: pseudoClass.replace(reUnescape, ''),
+ value: pseudoClassValue
+ });
+
+ } else if (attributeKey){
+ attributeKey = attributeKey.replace(reUnescape, '');
+ attributeValue = (attributeValue || '').replace(reUnescape, '');
+
+ var test, regexp;
+
+ switch (attributeOperator){
+ case '^=' : regexp = new RegExp( '^'+ escapeRegExp(attributeValue) ); break;
+ case '$=' : regexp = new RegExp( escapeRegExp(attributeValue) +'$' ); break;
+ case '~=' : regexp = new RegExp( '(^|\\s)'+ escapeRegExp(attributeValue) +'(\\s|$)' ); break;
+ case '|=' : regexp = new RegExp( '^'+ escapeRegExp(attributeValue) +'(-|$)' ); break;
+ case '=' : test = function(value){
+ return attributeValue == value;
+ }; break;
+ case '*=' : test = function(value){
+ return value && value.indexOf(attributeValue) > -1;
+ }; break;
+ case '!=' : test = function(value){
+ return attributeValue != value;
+ }; break;
+ default : test = function(value){
+ return !!value;
+ };
+ }
+
+ if (attributeValue == '' && (/^[*$^]=$/).test(attributeOperator)) test = function(){
+ return false;
+ };
+
+ if (!test) test = function(value){
+ return value && regexp.test(value);
+ };
+
+ if (!currentParsed.attributes) currentParsed.attributes = [];
+ currentParsed.attributes.push({
+ key: attributeKey,
+ operator: attributeOperator,
+ value: attributeValue,
+ test: test
+ });
+
+ }
+
+ return '';
+};
+
+// Slick NS
+
+var Slick = (this.Slick || {});
+
+Slick.parse = function(expression){
+ return parse(expression);
+};
+
+Slick.escapeRegExp = escapeRegExp;
+
+if (!this.Slick) this.Slick = Slick;
+
+}).apply(/*<CommonJS>*/(typeof exports != 'undefined') ? exports : /*</CommonJS>*/this);
+
+
+/*
+---
+name: Slick.Finder
+description: The new, superfast css selector engine.
+provides: Slick.Finder
+requires: Slick.Parser
+...
+*/
+
+(function(){
+
+var local = {};
+
+// Feature / Bug detection
+
+local.isNativeCode = function(fn){
+ return (/\{\s*\[native code\]\s*\}/).test('' + fn);
+};
+
+local.isXML = function(document){
+ return (!!document.xmlVersion) || (!!document.xml) || (Object.prototype.toString.call(document) === '[object XMLDocument]') ||
+ (document.nodeType === 9 && document.documentElement.nodeName !== 'HTML');
+};
+
+local.setDocument = function(document){
+
+ // convert elements / window arguments to document. if document cannot be extrapolated, the function returns.
+
+ if (document.nodeType === 9); // document
+ else if (document.ownerDocument) document = document.ownerDocument; // node
+ else if (document.navigator) document = document.document; // window
+ else return;
+
+ // check if it's the old document
+
+ if (this.document === document) return;
+ this.document = document;
+ var root = this.root = document.documentElement;
+
+ this.isXMLDocument = this.isXML(document);
+
+ this.brokenStarGEBTN
+ = this.starSelectsClosedQSA
+ = this.idGetsName
+ = this.brokenMixedCaseQSA
+ = this.brokenGEBCN
+ = this.brokenCheckedQSA
+ = this.brokenEmptyAttributeQSA
+ = this.isHTMLDocument
+ = false;
+
+ var starSelectsClosed, starSelectsComments,
+ brokenSecondClassNameGEBCN, cachedGetElementsByClassName;
+
+ var selected, id;
+ var testNode = document.createElement('div');
+ root.appendChild(testNode);
+
+ // on non-HTML documents innerHTML and getElementsById doesnt work properly
+ try {
+ id = 'slick_getbyid_test';
+ testNode.innerHTML = '<a id="'+id+'"></a>';
+ this.isHTMLDocument = !!document.getElementById(id);
+ } catch(e){};
+
+ if (this.isHTMLDocument){
+
+ testNode.style.display = 'none';
+
+ // IE returns comment nodes for getElementsByTagName('*') for some documents
+ testNode.appendChild(document.createComment(''));
+ starSelectsComments = (testNode.getElementsByTagName('*').length > 0);
+
+ // IE returns closed nodes (EG:"</foo>") for getElementsByTagName('*') for some documents
+ try {
+ testNode.innerHTML = 'foo</foo>';
+ selected = testNode.getElementsByTagName('*');
+ starSelectsClosed = (selected && selected.length && selected[0].nodeName.charAt(0) == '/');
+ } catch(e){};
+
+ this.brokenStarGEBTN = starSelectsComments || starSelectsClosed;
+
+ // IE 8 returns closed nodes (EG:"</foo>") for querySelectorAll('*') for some documents
+ if (testNode.querySelectorAll) try {
+ testNode.innerHTML = 'foo</foo>';
+ selected = testNode.querySelectorAll('*');
+ this.starSelectsClosedQSA = (selected && selected.length && selected[0].nodeName.charAt(0) == '/');
+ } catch(e){};
+
+ // IE returns elements with the name instead of just id for getElementsById for some documents
+ try {
+ id = 'slick_id_gets_name';
+ testNode.innerHTML = '<a name="'+id+'"></a><b id="'+id+'"></b>';
+ this.idGetsName = document.getElementById(id) === testNode.firstChild;
+ } catch(e){};
+
+ // Safari 3.2 querySelectorAll doesnt work with mixedcase on quirksmode
+ try {
+ testNode.innerHTML = '<a class="MiXedCaSe"></a>';
+ this.brokenMixedCaseQSA = !testNode.querySelectorAll('.MiXedCaSe').length;
+ } catch(e){};
+
+ try {
+ testNode.innerHTML = '<a class="f"></a><a class="b"></a>';
+ testNode.getElementsByClassName('b').length;
+ testNode.firstChild.className = 'b';
+ cachedGetElementsByClassName = (testNode.getElementsByClassName('b').length != 2);
+ } catch(e){};
+
+ // Opera 9.6 getElementsByClassName doesnt detects the class if its not the first one
+ try {
+ testNode.innerHTML = '<a class="a"></a><a class="f b a"></a>';
+ brokenSecondClassNameGEBCN = (testNode.getElementsByClassName('a').length != 2);
+ } catch(e){};
+
+ this.brokenGEBCN = cachedGetElementsByClassName || brokenSecondClassNameGEBCN;
+
+ // Webkit dont return selected options on querySelectorAll
+ try {
+ testNode.innerHTML = '<select><option selected="selected">a</option></select>';
+ this.brokenCheckedQSA = (testNode.querySelectorAll(':checked').length == 0);
+ } catch(e){};
+
+ // IE returns incorrect results for attr[*^$]="" selectors on querySelectorAll
+ try {
+ testNode.innerHTML = '<a class=""></a>';
+ this.brokenEmptyAttributeQSA = (testNode.querySelectorAll('[class*=""]').length != 0);
+ } catch(e){};
+
+ }
+
+ root.removeChild(testNode);
+ testNode = null;
+
+ // hasAttribute
+
+ this.hasAttribute = (root && this.isNativeCode(root.hasAttribute)) ? function(node, attribute) {
+ return node.hasAttribute(attribute);
+ } : function(node, attribute) {
+ node = node.getAttributeNode(attribute);
+ return !!(node && (node.specified || node.nodeValue));
+ };
+
+ // contains
+ // FIXME: Add specs: local.contains should be different for xml and html documents?
+ this.contains = (root && this.isNativeCode(root.contains)) ? function(context, node){
+ return context.contains(node);
+ } : (root && root.compareDocumentPosition) ? function(context, node){
+ return context === node || !!(context.compareDocumentPosition(node) & 16);
+ } : function(context, node){
+ if (node) do {
+ if (node === context) return true;
+ } while ((node = node.parentNode));
+ return false;
+ };
+
+ // document order sorting
+ // credits to Sizzle (http://sizzlejs.com/)
+
+ this.documentSorter = (root.compareDocumentPosition) ? function(a, b){
+ if (!a.compareDocumentPosition || !b.compareDocumentPosition) return 0;
+ return a.compareDocumentPosition(b) & 4 ? -1 : a === b ? 0 : 1;
+ } : ('sourceIndex' in root) ? function(a, b){
+ if (!a.sourceIndex || !b.sourceIndex) return 0;
+ return a.sourceIndex - b.sourceIndex;
+ } : (document.createRange) ? function(a, b){
+ if (!a.ownerDocument || !b.ownerDocument) return 0;
+ var aRange = a.ownerDocument.createRange(), bRange = b.ownerDocument.createRange();
+ aRange.setStart(a, 0);
+ aRange.setEnd(a, 0);
+ bRange.setStart(b, 0);
+ bRange.setEnd(b, 0);
+ return aRange.compareBoundaryPoints(Range.START_TO_END, bRange);
+ } : null ;
+
+ this.getUID = (this.isHTMLDocument) ? this.getUIDHTML : this.getUIDXML;
+
+};
+
+// Main Method
+
+local.search = function(context, expression, append, first){
+
+ var found = this.found = (first) ? null : (append || []);
+
+ // context checks
+
+ if (!context) return found; // No context
+ if (context.navigator) context = context.document; // Convert the node from a window to a document
+ else if (!context.nodeType) return found; // Reject misc junk input
+
+ // setup
+
+ var parsed, i;
+
+ var uniques = this.uniques = {};
+
+ if (this.document !== (context.ownerDocument || context)) this.setDocument(context);
+
+ // should sort if there are nodes in append and if you pass multiple expressions.
+ // should remove duplicates if append already has items
+ var shouldUniques = !!(append && append.length);
+
+ // avoid duplicating items already in the append array
+ if (shouldUniques) for (i = found.length; i--;) this.uniques[this.getUID(found[i])] = true;
+
+ // expression checks
+
+ if (typeof expression == 'string'){ // expression is a string
+
+ // Overrides
+
+ for (i = this.overrides.length; i--;){
+ var override = this.overrides[i];
+ if (override.regexp.test(expression)){
+ var result = override.method.call(context, expression, found, first);
+ if (result === false) continue;
+ if (result === true) return found;
+ return result;
+ }
+ }
+
+ parsed = this.Slick.parse(expression);
+ if (!parsed.length) return found;
+ } else if (expression == null){ // there is no expression
+ return found;
+ } else if (expression.Slick){ // expression is a parsed Slick object
+ parsed = expression;
+ } else if (this.contains(context.documentElement || context, expression)){ // expression is a node
+ (found) ? found.push(expression) : found = expression;
+ return found;
+ } else { // other junk
+ return found;
+ }
+
+ // cache elements for the nth selectors
+
+ /*<pseudo-selectors>*//*<nth-pseudo-selectors>*/
+
+ this.posNTH = {};
+ this.posNTHLast = {};
+ this.posNTHType = {};
+ this.posNTHTypeLast = {};
+
+ /*</nth-pseudo-selectors>*//*</pseudo-selectors>*/
+
+ // if append is null and there is only a single selector with one expression use pushArray, else use pushUID
+ this.push = (!shouldUniques && (first || (parsed.length == 1 && parsed.expressions[0].length == 1))) ? this.pushArray : this.pushUID;
+
+ if (found == null) found = [];
+
+ // default engine
+
+ var j, m, n;
+ var combinator, tag, id, classList, classes, attributes, pseudos;
+ var currentItems, currentExpression, currentBit, lastBit, expressions = parsed.expressions;
+
+ search: for (i = 0; (currentExpression = expressions[i]); i++) for (j = 0; (currentBit = currentExpression[j]); j++){
+
+ combinator = 'combinator:' + currentBit.combinator;
+ if (!this[combinator]) continue search;
+
+ tag = (this.isXMLDocument) ? currentBit.tag : currentBit.tag.toUpperCase();
+ id = currentBit.id;
+ classList = currentBit.classList;
+ classes = currentBit.classes;
+ attributes = currentBit.attributes;
+ pseudos = currentBit.pseudos;
+ lastBit = (j === (currentExpression.length - 1));
+
+ this.bitUniques = {};
+
+ if (lastBit){
+ this.uniques = uniques;
+ this.found = found;
+ } else {
+ this.uniques = {};
+ this.found = [];
+ }
+
+ if (j === 0){
+ this[combinator](context, tag, id, classes, attributes, pseudos, classList);
+ if (first && lastBit && found.length) break search;
+ } else {
+ if (first && lastBit) for (m = 0, n = currentItems.length; m < n; m++){
+ this[combinator](currentItems[m], tag, id, classes, attributes, pseudos, classList);
+ if (found.length) break search;
+ } else for (m = 0, n = currentItems.length; m < n; m++) this[combinator](currentItems[m], tag, id, classes, attributes, pseudos, classList);
+ }
+
+ currentItems = this.found;
+ }
+
+ if (shouldUniques || (parsed.expressions.length > 1)) this.sort(found);
+
+ return (first) ? (found[0] || null) : found;
+};
+
+// Utils
+
+local.uidx = 1;
+local.uidk = 'slick:uniqueid';
+
+local.getUIDXML = function(node){
+ var uid = node.getAttribute(this.uidk);
+ if (!uid){
+ uid = this.uidx++;
+ node.setAttribute(this.uidk, uid);
+ }
+ return uid;
+};
+
+local.getUIDHTML = function(node){
+ return node.uniqueNumber || (node.uniqueNumber = this.uidx++);
+};
+
+// sort based on the setDocument documentSorter method.
+
+local.sort = function(results){
+ if (!this.documentSorter) return results;
+ results.sort(this.documentSorter);
+ return results;
+};
+
+/*<pseudo-selectors>*//*<nth-pseudo-selectors>*/
+
+local.cacheNTH = {};
+
+local.matchNTH = /^([+-]?\d*)?([a-z]+)?([+-]\d+)?$/;
+
+local.parseNTHArgument = function(argument){
+ var parsed = argument.match(this.matchNTH);
+ if (!parsed) return false;
+ var special = parsed[2] || false;
+ var a = parsed[1] || 1;
+ if (a == '-') a = -1;
+ var b = +parsed[3] || 0;
+ parsed =
+ (special == 'n') ? {a: a, b: b} :
+ (special == 'odd') ? {a: 2, b: 1} :
+ (special == 'even') ? {a: 2, b: 0} : {a: 0, b: a};
+
+ return (this.cacheNTH[argument] = parsed);
+};
+
+local.createNTHPseudo = function(child, sibling, positions, ofType){
+ return function(node, argument){
+ var uid = this.getUID(node);
+ if (!this[positions][uid]){
+ var parent = node.parentNode;
+ if (!parent) return false;
+ var el = parent[child], count = 1;
+ if (ofType){
+ var nodeName = node.nodeName;
+ do {
+ if (el.nodeName !== nodeName) continue;
+ this[positions][this.getUID(el)] = count++;
+ } while ((el = el[sibling]));
+ } else {
+ do {
+ if (el.nodeType !== 1) continue;
+ this[positions][this.getUID(el)] = count++;
+ } while ((el = el[sibling]));
+ }
+ }
+ argument = argument || 'n';
+ var parsed = this.cacheNTH[argument] || this.parseNTHArgument(argument);
+ if (!parsed) return false;
+ var a = parsed.a, b = parsed.b, pos = this[positions][uid];
+ if (a == 0) return b == pos;
+ if (a > 0){
+ if (pos < b) return false;
+ } else {
+ if (b < pos) return false;
+ }
+ return ((pos - b) % a) == 0;
+ };
+};
+
+/*</nth-pseudo-selectors>*//*</pseudo-selectors>*/
+
+local.pushArray = function(node, tag, id, classes, attributes, pseudos){
+ if (this.matchSelector(node, tag, id, classes, attributes, pseudos)) this.found.push(node);
+};
+
+local.pushUID = function(node, tag, id, classes, attributes, pseudos){
+ var uid = this.getUID(node);
+ if (!this.uniques[uid] && this.matchSelector(node, tag, id, classes, attributes, pseudos)){
+ this.uniques[uid] = true;
+ this.found.push(node);
+ }
+};
+
+local.matchNode = function(node, selector){
+ var parsed = this.Slick.parse(selector);
+ if (!parsed) return true;
+
+ // simple (single) selectors
+ if(parsed.length == 1 && parsed.expressions[0].length == 1){
+ var exp = parsed.expressions[0][0];
+ return this.matchSelector(node, (this.isXMLDocument) ? exp.tag : exp.tag.toUpperCase(), exp.id, exp.classes, exp.attributes, exp.pseudos);
+ }
+
+ var nodes = this.search(this.document, parsed);
+ for (var i = 0, item; item = nodes[i++];){
+ if (item === node) return true;
+ }
+ return false;
+};
+
+local.matchPseudo = function(node, name, argument){
+ var pseudoName = 'pseudo:' + name;
+ if (this[pseudoName]) return this[pseudoName](node, argument);
+ var attribute = this.getAttribute(node, name);
+ return (argument) ? argument == attribute : !!attribute;
+};
+
+local.matchSelector = function(node, tag, id, classes, attributes, pseudos){
+ if (tag){
+ if (tag == '*'){
+ if (node.nodeName < '@') return false; // Fix for comment nodes and closed nodes
+ } else {
+ if (node.nodeName != tag) return false;
+ }
+ }
+
+ if (id && node.getAttribute('id') != id) return false;
+
+ var i, part, cls;
+ if (classes) for (i = classes.length; i--;){
+ cls = ('className' in node) ? node.className : node.getAttribute('class');
+ if (!(cls && classes[i].regexp.test(cls))) return false;
+ }
+ if (attributes) for (i = attributes.length; i--;){
+ part = attributes[i];
+ if (part.operator ? !part.test(this.getAttribute(node, part.key)) : !this.hasAttribute(node, part.key)) return false;
+ }
+ if (pseudos) for (i = pseudos.length; i--;){
+ part = pseudos[i];
+ if (!this.matchPseudo(node, part.key, part.value)) return false;
+ }
+ return true;
+};
+
+var combinators = {
+
+ ' ': function(node, tag, id, classes, attributes, pseudos, classList){ // all child nodes, any level
+
+ var i, item, children;
+
+ if (this.isHTMLDocument){
+ getById: if (id){
+ item = this.document.getElementById(id);
+ if ((!item && node.all) || (this.idGetsName && item && item.getAttributeNode('id').nodeValue != id)){
+ // all[id] returns all the elements with that name or id inside node
+ // if theres just one it will return the element, else it will be a collection
+ children = node.all[id];
+ if (!children) return;
+ if (!children[0]) children = [children];
+ for (i = 0; item = children[i++];) if (item.getAttributeNode('id').nodeValue == id){
+ this.push(item, tag, null, classes, attributes, pseudos);
+ break;
+ }
+ return;
+ }
+ if (!item){
+ // if the context is in the dom we return, else we will try GEBTN, breaking the getById label
+ if (this.contains(this.document.documentElement, node)) return;
+ else break getById;
+ } else if (this.document !== node && !this.contains(node, item)) return;
+ this.push(item, tag, null, classes, attributes, pseudos);
+ return;
+ }
+ getByClass: if (classes && node.getElementsByClassName && !this.brokenGEBCN){
+ children = node.getElementsByClassName(classList.join(' '));
+ if (!(children && children.length)) break getByClass;
+ for (i = 0; item = children[i++];) this.push(item, tag, id, null, attributes, pseudos);
+ return;
+ }
+ }
+ getByTag: {
+ children = node.getElementsByTagName(tag);
+ if (!(children && children.length)) break getByTag;
+ if (!this.brokenStarGEBTN) tag = null;
+ for (i = 0; item = children[i++];) this.push(item, tag, id, classes, attributes, pseudos);
+ }
+ },
+
+ '>': function(node, tag, id, classes, attributes, pseudos){ // direct children
+ if ((node = node.firstChild)) do {
+ if (node.nodeType === 1) this.push(node, tag, id, classes, attributes, pseudos);
+ } while ((node = node.nextSibling));
+ },
+
+ '+': function(node, tag, id, classes, attributes, pseudos){ // next sibling
+ while ((node = node.nextSibling)) if (node.nodeType === 1){
+ this.push(node, tag, id, classes, attributes, pseudos);
+ break;
+ }
+ },
+
+ '^': function(node, tag, id, classes, attributes, pseudos){ // first child
+ node = node.firstChild;
+ if (node){
+ if (node.nodeType === 1) this.push(node, tag, id, classes, attributes, pseudos);
+ else this['combinator:+'](node, tag, id, classes, attributes, pseudos);
+ }
+ },
+
+ '~': function(node, tag, id, classes, attributes, pseudos){ // next siblings
+ while ((node = node.nextSibling)){
+ if (node.nodeType !== 1) continue;
+ var uid = this.getUID(node);
+ if (this.bitUniques[uid]) break;
+ this.bitUniques[uid] = true;
+ this.push(node, tag, id, classes, attributes, pseudos);
+ }
+ },
+
+ '++': function(node, tag, id, classes, attributes, pseudos){ // next sibling and previous sibling
+ this['combinator:+'](node, tag, id, classes, attributes, pseudos);
+ this['combinator:!+'](node, tag, id, classes, attributes, pseudos);
+ },
+
+ '~~': function(node, tag, id, classes, attributes, pseudos){ // next siblings and previous siblings
+ this['combinator:~'](node, tag, id, classes, attributes, pseudos);
+ this['combinator:!~'](node, tag, id, classes, attributes, pseudos);
+ },
+
+ '!': function(node, tag, id, classes, attributes, pseudos){ // all parent nodes up to document
+ while ((node = node.parentNode)) if (node !== this.document) this.push(node, tag, id, classes, attributes, pseudos);
+ },
+
+ '!>': function(node, tag, id, classes, attributes, pseudos){ // direct parent (one level)
+ node = node.parentNode;
+ if (node !== this.document) this.push(node, tag, id, classes, attributes, pseudos);
+ },
+
+ '!+': function(node, tag, id, classes, attributes, pseudos){ // previous sibling
+ while ((node = node.previousSibling)) if (node.nodeType === 1){
+ this.push(node, tag, id, classes, attributes, pseudos);
+ break;
+ }
+ },
+
+ '!^': function(node, tag, id, classes, attributes, pseudos){ // last child
+ node = node.lastChild;
+ if (node){
+ if (node.nodeType === 1) this.push(node, tag, id, classes, attributes, pseudos);
+ else this['combinator:!+'](node, tag, id, classes, attributes, pseudos);
+ }
+ },
+
+ '!~': function(node, tag, id, classes, attributes, pseudos){ // previous siblings
+ while ((node = node.previousSibling)){
+ if (node.nodeType !== 1) continue;
+ var uid = this.getUID(node);
+ if (this.bitUniques[uid]) break;
+ this.bitUniques[uid] = true;
+ this.push(node, tag, id, classes, attributes, pseudos);
+ }
+ }
+
+};
+
+for (var c in combinators) local['combinator:' + c] = combinators[c];
+
+var pseudos = {
+
+ /*<pseudo-selectors>*/
+
+ 'empty': function(node){
+ var child = node.firstChild;
+ return !(child && child.nodeType == 1) && !(node.innerText || node.textContent || '').length;
+ },
+
+ 'not': function(node, expression){
+ return !this.matchNode(node, expression);
+ },
+
+ 'contains': function(node, text){
+ return (node.innerText || node.textContent || '').indexOf(text) > -1;
+ },
+
+ 'first-child': function(node){
+ while ((node = node.previousSibling)) if (node.nodeType === 1) return false;
+ return true;
+ },
+
+ 'last-child': function(node){
+ while ((node = node.nextSibling)) if (node.nodeType === 1) return false;
+ return true;
+ },
+
+ 'only-child': function(node){
+ var prev = node;
+ while ((prev = prev.previousSibling)) if (prev.nodeType === 1) return false;
+ var next = node;
+ while ((next = next.nextSibling)) if (next.nodeType === 1) return false;
+ return true;
+ },
+
+ /*<nth-pseudo-selectors>*/
+
+ 'nth-child': local.createNTHPseudo('firstChild', 'nextSibling', 'posNTH'),
+
+ 'nth-last-child': local.createNTHPseudo('lastChild', 'previousSibling', 'posNTHLast'),
+
+ 'nth-of-type': local.createNTHPseudo('firstChild', 'nextSibling', 'posNTHType', true),
+
+ 'nth-last-of-type': local.createNTHPseudo('lastChild', 'previousSibling', 'posNTHTypeLast', true),
+
+ 'index': function(node, index){
+ return this['pseudo:nth-child'](node, '' + index + 1);
+ },
+
+ 'even': function(node, argument){
+ return this['pseudo:nth-child'](node, '2n');
+ },
+
+ 'odd': function(node, argument){
+ return this['pseudo:nth-child'](node, '2n+1');
+ },
+
+ /*</nth-pseudo-selectors>*/
+
+ /*<of-type-pseudo-selectors>*/
+
+ 'first-of-type': function(node){
+ var nodeName = node.nodeName;
+ while ((node = node.previousSibling)) if (node.nodeName === nodeName) return false;
+ return true;
+ },
+
+ 'last-of-type': function(node){
+ var nodeName = node.nodeName;
+ while ((node = node.nextSibling)) if (node.nodeName === nodeName) return false;
+ return true;
+ },
+
+ 'only-of-type': function(node){
+ var prev = node, nodeName = node.nodeName;
+ while ((prev = prev.previousSibling)) if (prev.nodeName === nodeName) return false;
+ var next = node;
+ while ((next = next.nextSibling)) if (next.nodeName === nodeName) return false;
+ return true;
+ },
+
+ /*</of-type-pseudo-selectors>*/
+
+ // custom pseudos
+
+ 'enabled': function(node){
+ return (node.disabled === false);
+ },
+
+ 'disabled': function(node){
+ return (node.disabled === true);
+ },
+
+ 'checked': function(node){
+ return node.checked || node.selected;
+ },
+
+ 'focus': function(node){
+ return this.isHTMLDocument && this.document.activeElement === node && (node.href || node.type || this.hasAttribute(node, 'tabindex'));
+ },
+
+ 'root': function(node){
+ return (node === this.root);
+ },
+
+ 'selected': function(node){
+ return node.selected;
+ }
+
+ /*</pseudo-selectors>*/
+};
+
+for (var p in pseudos) local['pseudo:' + p] = pseudos[p];
+
+// attributes methods
+
+local.attributeGetters = {
+
+ 'class': function(){
+ return ('className' in this) ? this.className : this.getAttribute('class');
+ },
+
+ 'for': function(){
+ return ('htmlFor' in this) ? this.htmlFor : this.getAttribute('for');
+ },
+
+ 'href': function(){
+ return ('href' in this) ? this.getAttribute('href', 2) : this.getAttribute('href');
+ },
+
+ 'style': function(){
+ return (this.style) ? this.style.cssText : this.getAttribute('style');
+ }
+
+};
+
+local.getAttribute = function(node, name){
+ // FIXME: check if getAttribute() will get input elements on a form on this browser
+ // getAttribute is faster than getAttributeNode().nodeValue
+ var method = this.attributeGetters[name];
+ if (method) return method.call(node);
+ var attributeNode = node.getAttributeNode(name);
+ return attributeNode ? attributeNode.nodeValue : null;
+};
+
+// overrides
+
+local.overrides = [];
+
+local.override = function(regexp, method){
+ this.overrides.push({regexp: regexp, method: method});
+};
+
+/*<overrides>*/
+
+/*<query-selector-override>*/
+
+var reEmptyAttribute = /\[.*[*$^]=(?:["']{2})?\]/;
+
+local.override(/./, function(expression, found, first){ //querySelectorAll override
+
+ if (!this.querySelectorAll || this.nodeType != 9 || !local.isHTMLDocument || local.brokenMixedCaseQSA ||
+ (local.brokenCheckedQSA && expression.indexOf(':checked') > -1) ||
+ (local.brokenEmptyAttributeQSA && reEmptyAttribute.test(expression)) || Slick.disableQSA) return false;
+
+ var nodes, node;
+ try {
+ if (first) return this.querySelector(expression) || null;
+ else nodes = this.querySelectorAll(expression);
+ } catch(error){
+ return false;
+ }
+
+ var i, hasOthers = !!(found.length);
+
+ if (local.starSelectsClosedQSA) for (i = 0; node = nodes[i++];){
+ if (node.nodeName > '@' && (!hasOthers || !local.uniques[local.getUIDHTML(node)])) found.push(node);
+ } else for (i = 0; node = nodes[i++];){
+ if (!hasOthers || !local.uniques[local.getUIDHTML(node)]) found.push(node);
+ }
+
+ if (hasOthers) local.sort(found);
+
+ return true;
+
+});
+
+/*</query-selector-override>*/
+
+/*<tag-override>*/
+
+local.override(/^[\w-]+$|^\*$/, function(expression, found, first){ // tag override
+ var tag = expression;
+ if (tag == '*' && local.brokenStarGEBTN) return false;
+
+ var nodes = this.getElementsByTagName(tag);
+
+ if (first) return nodes[0] || null;
+ var i, node, hasOthers = !!(found.length);
+
+ for (i = 0; node = nodes[i++];){
+ if (!hasOthers || !local.uniques[local.getUID(node)]) found.push(node);
+ }
+
+ if (hasOthers) local.sort(found);
+
+ return true;
+});
+
+/*</tag-override>*/
+
+/*<class-override>*/
+
+local.override(/^\.[\w-]+$/, function(expression, found, first){ // class override
+ if (!local.isHTMLDocument || (!this.getElementsByClassName && this.querySelectorAll)) return false;
+
+ var nodes, node, i, hasOthers = !!(found && found.length), className = expression.substring(1);
+ if (this.getElementsByClassName && !local.brokenGEBCN){
+ nodes = this.getElementsByClassName(className);
+ if (first) return nodes[0] || null;
+ for (i = 0; node = nodes[i++];){
+ if (!hasOthers || !local.uniques[local.getUIDHTML(node)]) found.push(node);
+ }
+ } else {
+ var matchClass = new RegExp('(^|\\s)'+ Slick.escapeRegExp(className) +'(\\s|$)');
+ nodes = this.getElementsByTagName('*');
+ for (i = 0; node = nodes[i++];){
+ className = node.className;
+ if (!className || !matchClass.test(className)) continue;
+ if (first) return node;
+ if (!hasOthers || !local.uniques[local.getUIDHTML(node)]) found.push(node);
+ }
+ }
+ if (hasOthers) local.sort(found);
+ return (first) ? null : true;
+});
+
+/*</class-override>*/
+
+/*<id-override>*/
+
+local.override(/^#[\w-]+$/, function(expression, found, first){ // ID override
+ if (!local.isHTMLDocument || this.nodeType != 9) return false;
+
+ var id = expression.substring(1), el = this.getElementById(id);
+ if (!el) return found;
+ if (local.idGetsName && el.getAttributeNode('id').nodeValue != id) return false;
+ if (first) return el || null;
+ var hasOthers = !!(found.length);
+ if (!hasOthers || !local.uniques[local.getUIDHTML(el)]) found.push(el);
+ if (hasOthers) local.sort(found);
+ return true;
+});
+
+/*</id-override>*/
+
+/*</overrides>*/
+
+if (typeof document != 'undefined') local.setDocument(document);
+
+// Slick
+
+var Slick = local.Slick = (this.Slick || {});
+
+Slick.version = '0.9dev';
+
+// Slick finder
+
+Slick.search = function(context, expression, append){
+ return local.search(context, expression, append);
+};
+
+Slick.find = function(context, expression){
+ return local.search(context, expression, null, true);
+};
+
+// Slick containment checker
+
+Slick.contains = function(container, node){
+ local.setDocument(container);
+ return local.contains(container, node);
+};
+
+// Slick attribute getter
+
+Slick.getAttribute = function(node, name){
+ return local.getAttribute(node, name);
+};
+
+// Slick matcher
+
+Slick.match = function(node, selector){
+ if (!(node && selector)) return false;
+ if (!selector || selector === node) return true;
+ if (typeof selector != 'string') return false;
+ local.setDocument(node);
+ return local.matchNode(node, selector);
+};
+
+// Slick attribute accessor
+
+Slick.defineAttributeGetter = function(name, fn){
+ local.attributeGetters[name] = fn;
+ return this;
+};
+
+Slick.lookupAttributeGetter = function(name){
+ return local.attributeGetters[name];
+};
+
+// Slick pseudo accessor
+
+Slick.definePseudo = function(name, fn){
+ local['pseudo:' + name] = function(node, argument){
+ return fn.call(node, argument);
+ };
+ return this;
+};
+
+Slick.lookupPseudo = function(name){
+ var pseudo = local['pseudo:' + name];
+ if (pseudo) return function(argument){
+ return pseudo.call(this, argument);
+ };
+ return null;
+};
+
+// Slick overrides accessor
+
+Slick.override = function(regexp, fn){
+ local.override(regexp, fn);
+ return this;
+};
+
+Slick.isXML = local.isXML;
+
+Slick.uidOf = function(node){
+ return local.getUIDHTML(node);
+};
+
+if (!this.Slick) this.Slick = Slick;
+
+}).apply(/*<CommonJS>*/(typeof exports != 'undefined') ? exports : /*</CommonJS>*/this);
+
+
+/*
+---
+
+name: Element
+
+description: One of the most important items in MooTools. Contains the dollar function, the dollars function, and an handful of cross-browser, time-saver methods to let you easily work with HTML Elements.
+
+license: MIT-style license.
+
+requires: [Window, Document, Array, String, Function, Number, Slick.Parser, Slick.Finder]
+
+provides: [Element, Elements, $, $$, Iframe, Selectors]
+
+...
+*/
+
+var Element = function(tag, props){
+ var konstructor = Element.Constructors[tag];
+ if (konstructor) return konstructor(props);
+ if (typeof tag != 'string') return document.id(tag).set(props);
+
+ if (!props) props = {};
+
+ if (!tag.test(/^[\w-]+$/)){
+ var parsed = Slick.parse(tag).expressions[0][0];
+ tag = (parsed.tag == '*') ? 'div' : parsed.tag;
+ if (parsed.id && props.id == null) props.id = parsed.id;
+
+ var attributes = parsed.attributes;
+ if (attributes) for (var i = 0, l = attributes.length; i < l; i++){
+ var attr = attributes[i];
+ if (attr.value != null && attr.operator == '=' && props[attr.key] == null)
+ props[attr.key] = attr.value;
+ }
+
+ if (parsed.classList && props['class'] == null) props['class'] = parsed.classList.join(' ');
+ }
+
+ return document.newElement(tag, props);
+};
+
+if (Browser.Element) Element.prototype = Browser.Element.prototype;
+
+new Type('Element', Element).mirror(function(name){
+ if (Array.prototype[name]) return;
+
+ var obj = {};
+ obj[name] = function(){
+ var results = [], args = arguments, elements = true;
+ for (var i = 0, l = this.length; i < l; i++){
+ var element = this[i], result = results[i] = element[name].apply(element, args);
+ elements = (elements && typeOf(result) == 'element');
+ }
+ return (elements) ? new Elements(results) : results;
+ };
+
+ Elements.implement(obj);
+});
+
+if (!Browser.Element){
+ Element.parent = Object;
+
+ Element.Prototype = {'$family': Function.from('element').hide()};
+
+ Element.mirror(function(name, method){
+ Element.Prototype[name] = method;
+ });
+}
+
+Element.Constructors = {};
+
+//<1.2compat>
+
+Element.Constructors = new Hash;
+
+//</1.2compat>
+
+var IFrame = new Type('IFrame', function(){
+ var params = Array.link(arguments, {
+ properties: Type.isObject,
+ iframe: function(obj){
+ return (obj != null);
+ }
+ });
+
+ var props = params.properties || {}, iframe;
+ if (params.iframe) iframe = document.id(params.iframe);
+ var onload = props.onload || function(){};
+ delete props.onload;
+ props.id = props.name = [props.id, props.name, iframe ? (iframe.id || iframe.name) : 'IFrame_' + String.generateUID()].pick();
+ iframe = new Element(iframe || 'iframe', props);
+
+ var onLoad = function(){
+ onload.call(iframe.contentWindow);
+ };
+
+ if (window.frames[props.id]) onLoad();
+ else iframe.addListener('load', onLoad);
+ return iframe;
+});
+
+var Elements = this.Elements = function(nodes){
+ if (nodes && nodes.length){
+ var uniques = {}, node;
+ for (var i = 0; node = nodes[i++];){
+ var uid = Slick.uidOf(node);
+ if (!uniques[uid]){
+ uniques[uid] = true;
+ this.push(node);
+ }
+ }
+ }
+};
+
+Elements.prototype = {length: 0};
+Elements.parent = Array;
+
+new Type('Elements', Elements).implement({
+
+ filter: function(filter, bind){
+ if (!filter) return this;
+ return new Elements(Array.filter(this, (typeOf(filter) == 'string') ? function(item){
+ return item.match(filter);
+ } : filter, bind));
+ }.protect(),
+
+ push: function(){
+ var length = this.length;
+ for (var i = 0, l = arguments.length; i < l; i++){
+ var item = document.id(arguments[i]);
+ if (item) this[length++] = item;
+ }
+ return (this.length = length);
+ }.protect(),
+
+ concat: function(){
+ var newElements = new Elements(this);
+ for (var i = 0, l = arguments.length; i < l; i++){
+ var item = arguments[i];
+ if (Type.isEnumerable(item)) newElements.append(item);
+ else newElements.push(item);
+ }
+ return newElements;
+ }.protect(),
+
+ append: function(collection){
+ for (var i = 0, l = collection.length; i < l; i++) this.push(collection[i]);
+ return this;
+ }.protect(),
+
+ empty: function(){
+ while (this.length) delete this[--this.length];
+ return this;
+ }.protect()
+
+});
+
+(function(){
+
+// FF, IE
+var splice = Array.prototype.splice, object = {'0': 0, '1': 1, length: 2};
+
+splice.call(object, 1, 1);
+if (object[1] == 1) Elements.implement('splice', function(){
+ var length = this.length;
+ splice.apply(this, arguments);
+ while (length >= this.length) delete this[length--];
+ return this;
+}.protect());
+
+Elements.implement(Array.prototype);
+
+Array.mirror(Elements);
+
+/*<ltIE8>*/
+var createElementAcceptsHTML;
+try {
+ var x = document.createElement('<input name=x>');
+ createElementAcceptsHTML = (x.name == 'x');
+} catch(e){}
+
+var escapeQuotes = function(html){
+ return ('' + html).replace(/&/g, '&').replace(/"/g, '"');
+};
+/*</ltIE8>*/
+
+Document.implement({
+
+ newElement: function(tag, props){
+ if (props && props.checked != null) props.defaultChecked = props.checked;
+ /*<ltIE8>*/// Fix for readonly name and type properties in IE < 8
+ if (createElementAcceptsHTML && props){
+ tag = '<' + tag;
+ if (props.name) tag += ' name="' + escapeQuotes(props.name) + '"';
+ if (props.type) tag += ' type="' + escapeQuotes(props.type) + '"';
+ tag += '>';
+ delete props.name;
+ delete props.type;
+ }
+ /*</ltIE8>*/
+ return this.id(this.createElement(tag)).set(props);
+ }
+
+});
+
+})();
+
+Document.implement({
+
+ newTextNode: function(text){
+ return this.createTextNode(text);
+ },
+
+ getDocument: function(){
+ return this;
+ },
+
+ getWindow: function(){
+ return this.window;
+ },
+
+ id: (function(){
+
+ var types = {
+
+ string: function(id, nocash, doc){
+ id = Slick.find(doc, '#' + id.replace(/(\W)/g, '\\$1'));
+ return (id) ? types.element(id, nocash) : null;
+ },
+
+ element: function(el, nocash){
+ $uid(el);
+ if (!nocash && !el.$family && !(/^object|embed$/i).test(el.tagName)){
+ Object.append(el, Element.Prototype);
+ }
+ return el;
+ },
+
+ object: function(obj, nocash, doc){
+ if (obj.toElement) return types.element(obj.toElement(doc), nocash);
+ return null;
+ }
+
+ };
+
+ types.textnode = types.whitespace = types.window = types.document = function(zero){
+ return zero;
+ };
+
+ return function(el, nocash, doc){
+ if (el && el.$family && el.uid) return el;
+ var type = typeOf(el);
+ return (types[type]) ? types[type](el, nocash, doc || document) : null;
+ };
+
+ })()
+
+});
+
+if (window.$ == null) Window.implement('$', function(el, nc){
+ return document.id(el, nc, this.document);
+});
+
+Window.implement({
+
+ getDocument: function(){
+ return this.document;
+ },
+
+ getWindow: function(){
+ return this;
+ }
+
+});
+
+[Document, Element].invoke('implement', {
+
+ getElements: function(expression){
+ return Slick.search(this, expression, new Elements);
+ },
+
+ getElement: function(expression){
+ return document.id(Slick.find(this, expression));
+ }
+
+});
+
+//<1.2compat>
+
+(function(search, find, match){
+
+ this.Selectors = {};
+ var pseudos = this.Selectors.Pseudo = new Hash();
+
+ var addSlickPseudos = function(){
+ for (var name in pseudos) if (pseudos.hasOwnProperty(name)){
+ Slick.definePseudo(name, pseudos[name]);
+ delete pseudos[name];
+ }
+ };
+
+ Slick.search = function(context, expression, append){
+ addSlickPseudos();
+ return search.call(this, context, expression, append);
+ };
+
+ Slick.find = function(context, expression){
+ addSlickPseudos();
+ return find.call(this, context, expression);
+ };
+
+ Slick.match = function(node, selector){
+ addSlickPseudos();
+ return match.call(this, node, selector);
+ };
+
+})(Slick.search, Slick.find, Slick.match);
+
+if (window.$$ == null) Window.implement('$$', function(selector){
+ var elements = new Elements;
+ if (arguments.length == 1 && typeof selector == 'string') return Slick.search(this.document, selector, elements);
+ var args = Array.flatten(arguments);
+ for (var i = 0, l = args.length; i < l; i++){
+ var item = args[i];
+ switch (typeOf(item)){
+ case 'element': elements.push(item); break;
+ case 'string': Slick.search(this.document, item, elements);
+ }
+ }
+ return elements;
+});
+
+//</1.2compat>
+
+if (window.$$ == null) Window.implement('$$', function(selector){
+ if (arguments.length == 1){
+ if (typeof selector == 'string') return Slick.search(this.document, selector, new Elements);
+ else if (Type.isEnumerable(selector)) return new Elements(selector);
+ }
+ return new Elements(arguments);
+});
+
+(function(){
+
+var collected = {}, storage = {};
+var props = {input: 'checked', option: 'selected', textarea: 'value'};
+
+var get = function(uid){
+ return (storage[uid] || (storage[uid] = {}));
+};
+
+var clean = function(item){
+ if (item.removeEvents) item.removeEvents();
+ if (item.clearAttributes) item.clearAttributes();
+ var uid = item.uid;
+ if (uid != null){
+ delete collected[uid];
+ delete storage[uid];
+ }
+ return item;
+};
+
+var camels = ['defaultValue', 'accessKey', 'cellPadding', 'cellSpacing', 'colSpan', 'frameBorder', 'maxLength', 'readOnly',
+ 'rowSpan', 'tabIndex', 'useMap'
+];
+var bools = ['compact', 'nowrap', 'ismap', 'declare', 'noshade', 'checked', 'disabled', 'readOnly', 'multiple', 'selected',
+ 'noresize', 'defer'
+];
+ var attributes = {
+ 'html': 'innerHTML',
+ 'class': 'className',
+ 'for': 'htmlFor',
+ 'text': (function(){
+ var temp = document.createElement('div');
+ return (temp.innerText == null) ? 'textContent' : 'innerText';
+ })()
+};
+var readOnly = ['type'];
+var expandos = ['value', 'defaultValue'];
+var uriAttrs = /^(?:href|src|usemap)$/i;
+
+bools = bools.associate(bools);
+camels = camels.associate(camels.map(String.toLowerCase));
+readOnly = readOnly.associate(readOnly);
+
+Object.append(attributes, expandos.associate(expandos));
+
+var inserters = {
+
+ before: function(context, element){
+ var parent = element.parentNode;
+ if (parent) parent.insertBefore(context, element);
+ },
+
+ after: function(context, element){
+ var parent = element.parentNode;
+ if (parent) parent.insertBefore(context, element.nextSibling);
+ },
+
+ bottom: function(context, element){
+ element.appendChild(context);
+ },
+
+ top: function(context, element){
+ element.insertBefore(context, element.firstChild);
+ }
+
+};
+
+inserters.inside = inserters.bottom;
+
+//<1.2compat>
+
+Object.each(inserters, function(inserter, where){
+
+ where = where.capitalize();
+
+ var methods = {};
+
+ methods['inject' + where] = function(el){
+ inserter(this, document.id(el, true));
+ return this;
+ };
+
+ methods['grab' + where] = function(el){
+ inserter(document.id(el, true), this);
+ return this;
+ };
+
+ Element.implement(methods);
+
+});
+
+//</1.2compat>
+
+var injectCombinator = function(expression, combinator){
+ if (!expression) return combinator;
+
+ expression = Slick.parse(expression);
+
+ var expressions = expression.expressions;
+ for (var i = expressions.length; i--;)
+ expressions[i][0].combinator = combinator;
+
+ return expression;
+};
+
+Element.implement({
+
+ set: function(prop, value){
+ var property = Element.Properties[prop];
+ (property && property.set) ? property.set.call(this, value) : this.setProperty(prop, value);
+ }.overloadSetter(),
+
+ get: function(prop){
+ var property = Element.Properties[prop];
+ return (property && property.get) ? property.get.apply(this) : this.getProperty(prop);
+ }.overloadGetter(),
+
+ erase: function(prop){
+ var property = Element.Properties[prop];
+ (property && property.erase) ? property.erase.apply(this) : this.removeProperty(prop);
+ return this;
+ },
+
+ setProperty: function(attribute, value){
+ attribute = camels[attribute] || attribute;
+ if (value == null) return this.removeProperty(attribute);
+ var key = attributes[attribute];
+ (key) ? this[key] = value :
+ (bools[attribute]) ? this[attribute] = !!value : this.setAttribute(attribute, '' + value);
+ return this;
+ },
+
+ setProperties: function(attributes){
+ for (var attribute in attributes) this.setProperty(attribute, attributes[attribute]);
+ return this;
+ },
+
+ getProperty: function(attribute){
+ attribute = camels[attribute] || attribute;
+ var key = attributes[attribute] || readOnly[attribute];
+ return (key) ? this[key] :
+ (bools[attribute]) ? !!this[attribute] :
+ (uriAttrs.test(attribute) ? this.getAttribute(attribute, 2) :
+ (key = this.getAttributeNode(attribute)) ? key.nodeValue : null) || null;
+ },
+
+ getProperties: function(){
+ var args = Array.from(arguments);
+ return args.map(this.getProperty, this).associate(args);
+ },
+
+ removeProperty: function(attribute){
+ attribute = camels[attribute] || attribute;
+ var key = attributes[attribute];
+ (key) ? this[key] = '' :
+ (bools[attribute]) ? this[attribute] = false : this.removeAttribute(attribute);
+ return this;
+ },
+
+ removeProperties: function(){
+ Array.each(arguments, this.removeProperty, this);
+ return this;
+ },
+
+ hasClass: function(className){
+ return this.className.clean().contains(className, ' ');
+ },
+
+ addClass: function(className){
+ if (!this.hasClass(className)) this.classNa
<TRUNCATED>
[42/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/stats/SlotSizeCounter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/stats/SlotSizeCounter.java b/src/main/java/com/twitter/aurora/scheduler/stats/SlotSizeCounter.java
deleted file mode 100644
index 6c0bf48..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/stats/SlotSizeCounter.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.stats;
-
-import java.util.Map;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableMap;
-
-import com.twitter.aurora.gen.Quota;
-import com.twitter.aurora.scheduler.quota.Quotas;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A stat computer that aggregates the number of 'slots' available at different pre-determined
- * slot sizes.
- */
-class SlotSizeCounter implements Runnable {
- private static final Map<String, IQuota> SLOT_SIZES = ImmutableMap.of(
- "small", IQuota.build(new Quota(1.0, 1024, 4096)),
- "medium", IQuota.build(new Quota(4.0, 8192, 16384)),
- "large", IQuota.build(new Quota(8.0, 16384, 32768)),
- "xlarge", IQuota.build(new Quota(16.0, 32768, 65536)));
-
- private final Map<String, IQuota> slotSizes;
- private final ResourceSlotProvider resourceSlotProvider;
- private final CachedCounters cachedCounters;
-
- @VisibleForTesting
- SlotSizeCounter(
- final Map<String, IQuota> slotSizes,
- ResourceSlotProvider resourceSlotProvider,
- CachedCounters cachedCounters) {
-
- this.slotSizes = checkNotNull(slotSizes);
- this.resourceSlotProvider = checkNotNull(resourceSlotProvider);
- this.cachedCounters = checkNotNull(cachedCounters);
- }
-
- interface ResourceSlotProvider {
- Iterable<IQuota> get();
- }
-
- @Inject
- SlotSizeCounter(ResourceSlotProvider resourceSlotProvider, CachedCounters cachedCounters) {
- this(SLOT_SIZES, resourceSlotProvider, cachedCounters);
- }
-
- @VisibleForTesting
- static String getStatName(String slotName) {
- return "empty_slots_" + slotName;
- }
-
- private int countSlots(Iterable<IQuota> slots, final IQuota slotSize) {
- Function<IQuota, Integer> counter = new Function<IQuota, Integer>() {
- @Override public Integer apply(IQuota machineSlack) {
- return Quotas.divide(machineSlack, slotSize);
- }
- };
-
- int sum = 0;
- for (int slotCount : FluentIterable.from(slots).transform(counter)) {
- sum += slotCount;
- }
- return sum;
- }
-
- @Override
- public void run() {
- Iterable<IQuota> slots = resourceSlotProvider.get();
- for (Map.Entry<String, IQuota> entry : slotSizes.entrySet()) {
- cachedCounters.get(getStatName(entry.getKey())).set(countSlots(slots, entry.getValue()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/stats/TaskStatCalculator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/stats/TaskStatCalculator.java b/src/main/java/com/twitter/aurora/scheduler/stats/TaskStatCalculator.java
deleted file mode 100644
index db34125..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/stats/TaskStatCalculator.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package com.twitter.aurora.scheduler.stats;
-
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.twitter.aurora.scheduler.stats.AsyncStatsModule.StatUpdater;
-import com.twitter.aurora.scheduler.stats.ResourceCounter.GlobalMetric;
-import com.twitter.aurora.scheduler.stats.ResourceCounter.Metric;
-import com.twitter.aurora.scheduler.storage.Storage.StorageException;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Calculates and exports aggregate stats about resources consumed by active tasks.
- */
-class TaskStatCalculator implements Runnable {
- private static final Logger LOG = Logger.getLogger(StatUpdater.class.getName());
-
- private final CachedCounters counters;
- private final ResourceCounter resourceCounter;
-
- @Inject
- TaskStatCalculator(ResourceCounter resourceCounter, CachedCounters counters) {
- this.resourceCounter = checkNotNull(resourceCounter);
- this.counters = checkNotNull(counters);
- }
-
- private void update(String prefix, Metric metric) {
- counters.get(prefix + "_cpu").set(metric.getCpu());
- counters.get(prefix + "_ram_gb").set(metric.getRamGb());
- counters.get(prefix + "_disk_gb").set(metric.getDiskGb());
- }
-
- @Override
- public void run() {
- try {
- for (GlobalMetric metric : resourceCounter.computeConsumptionTotals()) {
- update("resources_" + metric.type.name().toLowerCase(), metric);
- }
- update("resources_allocated_quota", resourceCounter.computeQuotaAllocationTotals());
- } catch (StorageException e) {
- LOG.fine("Unable to fetch metrics, storage is likely not ready.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/AttributeStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/AttributeStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/AttributeStore.java
deleted file mode 100644
index 35c666c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/AttributeStore.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.Set;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-
-/**
- * Storage interface for host attributes.
- */
-public interface AttributeStore {
- /**
- * Fetches all host attributes given by the host.
- *
- * @param host host name.
- * @return attributes associated with {@code host}, if the host is known.
- */
- Optional<HostAttributes> getHostAttributes(String host);
-
- /**
- * Fetches all attributes in the store.
- *
- * @return All host attributes.
- */
- Set<HostAttributes> getHostAttributes();
-
- /**
- * Attributes are considered mostly ephemeral and extremely low risk when inconsistency
- * is present.
- */
- public interface Mutable extends AttributeStore {
-
- /**
- * Deletes all attributes in the store.
- */
- void deleteHostAttributes();
-
- /**
- * Save a host attribute in the attribute store.
- *
- * @param hostAttributes The attribute we are going to save.
- */
- void saveHostAttributes(HostAttributes hostAttributes);
-
- /**
- * Adjusts the maintenance mode for a host.
- * No adjustment will be made if the host is unknown.
- *
- * @param host Host to adjust.
- * @param mode Mode to place the host in.
- * @return {@code true} if the host is known and the state was adjusted,
- * {@code false} if the host is unrecognized.
- */
- boolean setMaintenanceMode(String host, MaintenanceMode mode);
- }
-
- public static final class Util {
- private Util() {
- }
-
- /**
- * Fetches attributes about a {@code host}.
- *
- * @param store Store to fetch host attributes from.
- * @param host Host to fetch attributes about.
- * @return Attributes associated with {@code host}, or an empty iterable if the host is
- * unknown.
- */
- public static Iterable<Attribute> attributesOrNone(StoreProvider store, String host) {
- Optional<HostAttributes> attributes = store.getAttributeStore().getHostAttributes(host);
- return attributes.isPresent()
- ? attributes.get().getAttributes() : ImmutableList.<Attribute>of();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/CallOrderEnforcingStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/CallOrderEnforcingStorage.java b/src/main/java/com/twitter/aurora/scheduler/storage/CallOrderEnforcingStorage.java
deleted file mode 100644
index c71a114..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/CallOrderEnforcingStorage.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.inject.BindingAnnotation;
-import com.google.inject.Module;
-import com.google.inject.PrivateModule;
-
-import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.Event;
-import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork.NoResult.Quiet;
-import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
-import com.twitter.common.util.StateMachine;
-
-/**
- * A non-volatile storage wrapper that enforces method call ordering.
- */
-public class CallOrderEnforcingStorage implements NonVolatileStorage {
-
- /**
- * Identifies a storage whose call order should be enforced.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ ElementType.PARAMETER, ElementType.METHOD })
- @BindingAnnotation
- private @interface EnforceOrderOn { }
-
- private final NonVolatileStorage wrapped;
-
- private enum State {
- CONSTRUCTED,
- PREPARED,
- READY,
- STOPPED
- }
-
- private final StateMachine<State> stateMachine = StateMachine.<State>builder("storage")
- .logTransitions()
- .initialState(State.CONSTRUCTED)
- .addState(State.CONSTRUCTED, State.PREPARED)
- .addState(State.PREPARED, State.READY)
- .addState(State.READY, State.STOPPED)
- .build();
-
- @Inject
- CallOrderEnforcingStorage(@EnforceOrderOn NonVolatileStorage wrapped) {
- this.wrapped = wrapped;
- }
-
- private void checkInState(State state) throws StorageException {
- if (stateMachine.getState() != state) {
- throw new StorageException("Storage is not " + state);
- }
- }
-
- @Override
- public void prepare() throws StorageException {
- checkInState(State.CONSTRUCTED);
- wrapped.prepare();
- stateMachine.transition(State.PREPARED);
- }
-
- @SendNotification(after = Event.StorageStarted)
- @Override
- public void start(Quiet initializationLogic) throws StorageException {
- checkInState(State.PREPARED);
- wrapped.start(initializationLogic);
- stateMachine.transition(State.READY);
- }
-
- @Override
- public void stop() {
- wrapped.stop();
- stateMachine.transition(State.STOPPED);
- }
-
- @Override
- public <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
- checkInState(State.READY);
- return wrapped.consistentRead(work);
- }
-
- @Override
- public <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work)
- throws StorageException, E {
-
- checkInState(State.READY);
- return wrapped.weaklyConsistentRead(work);
- }
-
- @Override
- public <T, E extends Exception> T write(MutateWork<T, E> work)
- throws StorageException, E {
- checkInState(State.READY);
- return wrapped.write(work);
- }
-
- @Override
- public void snapshot() throws StorageException {
- checkInState(State.READY);
- wrapped.snapshot();
- }
-
- /**
- * Creates a binding module that will wrap a storage class with {@link CallOrderEnforcingStorage},
- * exposing the order-enforced storage as {@link Storage} and {@link NonVolatileStorage}.
- *
- * @param storageClass Non-volatile storage implementation class.
- * @return Binding module.
- */
- public static Module wrappingModule(final Class<? extends NonVolatileStorage> storageClass) {
- return new PrivateModule() {
- @Override protected void configure() {
- bind(Storage.class).to(CallOrderEnforcingStorage.class);
- bind(NonVolatileStorage.class).to(CallOrderEnforcingStorage.class);
- bind(CallOrderEnforcingStorage.class).in(Singleton.class);
- bind(NonVolatileStorage.class).annotatedWith(EnforceOrderOn.class).to(storageClass);
- expose(Storage.class);
- expose(NonVolatileStorage.class);
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/DistributedSnapshotStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/DistributedSnapshotStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/DistributedSnapshotStore.java
deleted file mode 100644
index cba6303..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/DistributedSnapshotStore.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.storage.Snapshot;
-
-/**
- * A distributed snapshot store that supports persisting globally-visible snapshots.
- */
-public interface DistributedSnapshotStore {
- /**
- * Writes a snapshot to the distributed storage system.
- * TODO(William Farner): Currently we're hiding some exceptions (which happen to be
- * RuntimeExceptions). Clean these up to be checked, and throw another exception type here.
- *
- * @param snapshot Snapshot to write.
- * @throws CodingException If the snapshot could not be serialized.
- */
- void persist(Snapshot snapshot) throws CodingException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/ForwardingStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/ForwardingStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/ForwardingStore.java
deleted file mode 100644
index 1a6a849..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/ForwardingStore.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.ILockKey;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A store that forwards all its operations to underlying storage systems. Useful for decorating
- * an existing storage system.
- */
-public class ForwardingStore implements
- Storage,
- SchedulerStore.Mutable,
- JobStore.Mutable,
- TaskStore.Mutable,
- LockStore.Mutable,
- QuotaStore.Mutable,
- AttributeStore.Mutable {
-
- private final Storage storage;
- private final SchedulerStore.Mutable schedulerStore;
- private final JobStore.Mutable jobStore;
- private final TaskStore.Mutable taskStore;
- private final LockStore.Mutable lockStore;
- private final QuotaStore.Mutable quotaStore;
- private final AttributeStore.Mutable attributeStore;
-
- /**
- * Creats a new forwarding store that delegates to the providing default stores.
- *
- * @param storage Delegate.
- * @param schedulerStore Delegate.
- * @param jobStore Delegate.
- * @param taskStore Delegate.
- * @param lockStore Delegate.
- * @param quotaStore Delegate.
- * @param attributeStore Delegate.
- */
- public ForwardingStore(
- Storage storage,
- SchedulerStore.Mutable schedulerStore,
- JobStore.Mutable jobStore,
- TaskStore.Mutable taskStore,
- LockStore.Mutable lockStore,
- QuotaStore.Mutable quotaStore,
- AttributeStore.Mutable attributeStore) {
-
- this.storage = checkNotNull(storage);
- this.schedulerStore = checkNotNull(schedulerStore);
- this.jobStore = checkNotNull(jobStore);
- this.taskStore = checkNotNull(taskStore);
- this.lockStore = checkNotNull(lockStore);
- this.quotaStore = checkNotNull(quotaStore);
- this.attributeStore = checkNotNull(attributeStore);
- }
-
- @Override
- public <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
- return storage.consistentRead(work);
- }
-
- @Override
- public <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work)
- throws StorageException, E {
- return storage.weaklyConsistentRead(work);
- }
-
- @Override
- public <T, E extends Exception> T write(MutateWork<T, E> work)
- throws StorageException, E {
-
- return storage.write(work);
- }
-
- @Override
- public void snapshot() throws StorageException {
- storage.snapshot();
- }
-
- @Override
- public void saveFrameworkId(String frameworkId) {
- schedulerStore.saveFrameworkId(frameworkId);
- }
-
- @Override
- @Nullable
- public String fetchFrameworkId() {
- return schedulerStore.fetchFrameworkId();
- }
-
- @Override
- public Iterable<IJobConfiguration> fetchJobs(String managerId) {
- return jobStore.fetchJobs(managerId);
- }
-
- @Override
- public Optional<IJobConfiguration> fetchJob(String managerId, IJobKey jobKey) {
- return jobStore.fetchJob(managerId, jobKey);
- }
-
- @Override
- public void saveAcceptedJob(String managerId, IJobConfiguration jobConfig) {
- jobStore.saveAcceptedJob(managerId, jobConfig);
- }
-
- @Override
- public void removeJob(IJobKey jobKey) {
- jobStore.removeJob(jobKey);
- }
-
- @Override
- public void deleteJobs() {
- jobStore.deleteJobs();
- }
-
- @Override
- public Set<String> fetchManagerIds() {
- return jobStore.fetchManagerIds();
- }
-
- @Override
- public void saveTasks(Set<IScheduledTask> tasks) throws IllegalStateException {
- taskStore.saveTasks(tasks);
- }
-
- @Override
- public void deleteAllTasks() {
- taskStore.deleteAllTasks();
- }
-
- @Override
- public void deleteTasks(Set<String> taskIds) {
- taskStore.deleteTasks(taskIds);
- }
-
- @Override
- public ImmutableSet<IScheduledTask> mutateTasks(
- Query.Builder query,
- Function<IScheduledTask, IScheduledTask> mutator) {
-
- return taskStore.mutateTasks(query, mutator);
- }
-
- @Override
- public boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration) {
- return taskStore.unsafeModifyInPlace(taskId, taskConfiguration);
- }
-
- @Override
- public ImmutableSet<IScheduledTask> fetchTasks(Query.Builder querySupplier) {
- return taskStore.fetchTasks(querySupplier);
- }
-
- @Override
- public Set<ILock> fetchLocks() {
- return lockStore.fetchLocks();
- }
-
- @Override
- public Optional<ILock> fetchLock(ILockKey lockKey) {
- return lockStore.fetchLock(lockKey);
- }
-
- @Override
- public void saveLock(ILock lock) {
- lockStore.saveLock(lock);
- }
-
- @Override
- public void removeLock(ILockKey lockKey) {
- lockStore.removeLock(lockKey);
- }
-
- @Override
- public void deleteLocks() {
- lockStore.deleteLocks();
- }
-
- @Override
- public Map<String, IQuota> fetchQuotas() {
- return quotaStore.fetchQuotas();
- }
-
- @Override
- public void removeQuota(String role) {
- quotaStore.removeQuota(role);
- }
-
- @Override
- public void deleteQuotas() {
- quotaStore.deleteQuotas();
- }
-
- @Override
- public void saveQuota(String role, IQuota quota) {
- quotaStore.saveQuota(role, quota);
- }
-
- @Override
- public Optional<IQuota> fetchQuota(String role) {
- return quotaStore.fetchQuota(role);
- }
-
- @Override
- public void saveHostAttributes(HostAttributes hostAttribute) {
- attributeStore.saveHostAttributes(hostAttribute);
- }
-
- @Override
- public Optional<HostAttributes> getHostAttributes(String host) {
- return attributeStore.getHostAttributes(host);
- }
-
- @Override
- public Set<HostAttributes> getHostAttributes() {
- return attributeStore.getHostAttributes();
- }
-
- @Override
- public void deleteHostAttributes() {
- attributeStore.deleteHostAttributes();
- }
-
- @Override
- public boolean setMaintenanceMode(String host, MaintenanceMode mode) {
- return attributeStore.setMaintenanceMode(host, mode);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/JobStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/JobStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/JobStore.java
deleted file mode 100644
index 7092e91..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/JobStore.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.Set;
-
-import com.google.common.base.Optional;
-
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-
-/**
- * Stores job configuration data.
- */
-public interface JobStore {
-
- /**
- * Fetches all {@code JobConfiguration}s for jobs owned by the manager identified by
- * {@code managerId}; if there are none then an empty set is returned.
- *
- * @param managerId The unique identifier of the manager to find registered jobs for.
- * @return the set of job configurations owned by the specififed job manager
- */
- Iterable<IJobConfiguration> fetchJobs(String managerId);
-
- /**
- * Fetches the {@code JobConfiguration} for the specified {@code jobKey} if it exists.
- *
- * @param managerId The unique identifier of the manager that accepted the job.
- * @param jobKey The jobKey identifying the job to be fetched.
- * @return the job configuration for the given {@code jobKey} or absent if none is found.
- */
- Optional<IJobConfiguration> fetchJob(String managerId, IJobKey jobKey);
-
- /**
- * Fetches all the unique manager ids that are present in the job store.
- *
- * @return The IDs of all stored job managers.
- */
- Set<String> fetchManagerIds();
-
- public interface Mutable extends JobStore {
- /**
- * Saves the job configuration for a job that has been accepted by the scheduler. Acts as an
- * update if the managerId already exists.
- * TODO(William Farner): Consider accepting SanitizedConfiguration here to require that
- * validation always happens for things entering storage.
- *
- * @param managerId The unique id of the manager that accepted the job.
- * @param jobConfig The configuration of the accepted job.
- */
- void saveAcceptedJob(String managerId, IJobConfiguration jobConfig);
-
- /**
- * Removes the job configuration for the job identified by {@code jobKey}.
- * If there is no stored configuration for the identified job, this method returns silently.
- *
- * @param jobKey the key identifying the job to delete.
- */
- void removeJob(IJobKey jobKey);
-
- /**
- * Deletes all jobs.
- */
- void deleteJobs();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/LockStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/LockStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/LockStore.java
deleted file mode 100644
index 13d7317..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/LockStore.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.Set;
-
-import com.google.common.base.Optional;
-
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.ILockKey;
-
-/**
- * Stores all lock-related data and defines methods for saving, deleting and fetching locks.
- */
-public interface LockStore {
- /**
- * Fetches all locks available in the store.
- *
- * @return All locks in the store.
- */
- Set<ILock> fetchLocks();
-
- /**
- * Fetches a lock by its key.
- *
- * @param lockKey Key of the lock to fetch.
- * @return Optional lock.
- */
- Optional<ILock> fetchLock(ILockKey lockKey);
-
- public interface Mutable extends LockStore {
- /**
- * Saves a new lock or overwrites the existing one with same LockKey.
- *
- * @param lock ILock to save.
- */
- void saveLock(ILock lock);
-
- /**
- * Removes the lock from the store.
- *
- * @param lockKey Key of the lock to remove.
- */
- void removeLock(ILockKey lockKey);
-
- /**
- * Deletes all locks from the store.
- */
- void deleteLocks();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/QuotaStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/QuotaStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/QuotaStore.java
deleted file mode 100644
index ba8116b..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/QuotaStore.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.Map;
-
-import com.google.common.base.Optional;
-
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-
-/**
- * Point of storage for quota records.
- */
-public interface QuotaStore {
- /**
- * Fetches the existing quota record for a role.
- *
- * @param role Role to fetch quota for.
- * @return Optional quota associated with {@code role}.
- */
- Optional<IQuota> fetchQuota(String role);
-
- /**
- * Fetches all allocated quotas.
- *
- * @return All allocated quotas.
- */
- Map<String, IQuota> fetchQuotas();
-
- public interface Mutable extends QuotaStore {
-
- /**
- * Deletes all quotas.
- */
- void deleteQuotas();
-
- /**
- * Deletes quota for a role.
- *
- * @param role Role to remove quota record for.
- */
- void removeQuota(String role);
-
- /**
- * Saves a quota record for a role.
- *
- * @param role Role to create or update a quota record for.
- * @param quota Quota to save.
- */
- void saveQuota(String role, IQuota quota);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/ReadWriteLockManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/ReadWriteLockManager.java b/src/main/java/com/twitter/aurora/scheduler/storage/ReadWriteLockManager.java
deleted file mode 100644
index 58d7a68..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/ReadWriteLockManager.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import com.google.common.base.Preconditions;
-
-/**
- * A lock manager that wraps a ReadWriteLock and detects ill-fated attempts to upgrade
- * a read-locked thread to a write-locked thread, which would otherwise deadlock.
- */
-public class ReadWriteLockManager {
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- enum LockMode {
- NONE,
- READ,
- WRITE
- }
-
- private static class LockState {
- private LockMode initialLockMode = LockMode.NONE;
- private int lockCount = 0;
-
- private boolean lockAcquired(LockMode mode) {
- boolean stateChanged = false;
- if (initialLockMode == LockMode.NONE) {
- initialLockMode = mode;
- stateChanged = true;
- }
- if (initialLockMode == mode) {
- lockCount++;
- }
- return stateChanged;
- }
-
- private void lockReleased(LockMode mode) {
- if (initialLockMode == mode) {
- lockCount--;
- if (lockCount == 0) {
- initialLockMode = LockMode.NONE;
- }
- }
- }
- }
-
- private final ThreadLocal<LockState> lockState = new ThreadLocal<LockState>() {
- @Override protected LockState initialValue() {
- return new LockState();
- }
- };
-
- /**
- * Blocks until this thread has acquired a read lock.
- *
- * @return {@code true} if the lock was newly-acquired, or {@code false} if this thread previously
- * secured the write lock and has yet to release it.
- */
- public boolean readLock() {
- lock.readLock().lock();
- return lockState.get().lockAcquired(LockMode.READ);
- }
-
- /**
- * Releases this thread's read lock.
- */
- public void readUnlock() {
- lock.readLock().unlock();
- lockState.get().lockReleased(LockMode.READ);
- }
-
- /**
- * Blocks until this thread has acquired a write lock.
- *
- * @return {@code true} if the lock was newly-acquired, or {@code false} if this thread previously
- * secured the write lock and has yet to release it.
- */
- public boolean writeLock() {
- Preconditions.checkState(lockState.get().initialLockMode != LockMode.READ,
- "A read operation may not be upgraded to a write operation.");
-
- lock.writeLock().lock();
- return lockState.get().lockAcquired(LockMode.WRITE);
- }
-
- /**
- * Releases this thread's write lock.
- */
- public void writeUnlock() {
- lock.writeLock().unlock();
- lockState.get().lockReleased(LockMode.WRITE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/SchedulerStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/SchedulerStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/SchedulerStore.java
deleted file mode 100644
index 504b90b..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/SchedulerStore.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import javax.annotation.Nullable;
-
-/**
- * Stores data specific to the scheduler itself.
- */
-public interface SchedulerStore {
-
- /**
- * Fetches the last saved framework id. If none is saved, null can be returned.
- *
- * @return the last saved framework id
- */
- @Nullable String fetchFrameworkId();
-
- public interface Mutable extends SchedulerStore {
- /**
- * Stores the given framework id overwriting any previously saved id.
- *
- * @param frameworkId The framework id to store.
- */
- void saveFrameworkId(String frameworkId);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/SnapshotStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/SnapshotStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/SnapshotStore.java
deleted file mode 100644
index c775207..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/SnapshotStore.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-/**
- * Storage mechanism that is able to create complete snapshots of the local storage system state
- * and apply these to restore local storage from a snapshotted baseline.
- */
-public interface SnapshotStore<T> {
-
- /**
- * Creates a consistent snapshot of the local storage system.
- *
- * @return A blob that can be used to recover local storage via {@link #applySnapshot(Object)}.
- */
- T createSnapshot();
-
- /**
- * Applies a snapshot blob to the local storage system, wiping out all existing data and
- * resetting with the contents of the snapshot.
- *
- * @param snapshot A snapshot blob created by {@link #createSnapshot()}.
- */
- void applySnapshot(T snapshot);
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/Storage.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/Storage.java b/src/main/java/com/twitter/aurora/scheduler/storage/Storage.java
deleted file mode 100644
index fdc9ae7..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/Storage.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.BindingAnnotation;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.SchedulerException;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-
-/**
- * Manages scheduler storage operations providing an interface to perform atomic changes.
- */
-public interface Storage {
-
- interface StoreProvider {
- SchedulerStore getSchedulerStore();
- JobStore getJobStore();
- TaskStore getTaskStore();
- LockStore getLockStore();
- QuotaStore getQuotaStore();
- AttributeStore getAttributeStore();
- }
-
- interface MutableStoreProvider extends StoreProvider {
- SchedulerStore.Mutable getSchedulerStore();
- JobStore.Mutable getJobStore();
-
- /**
- * Gets access to the mutable task store.
- * <p>
- * This is labeled as unsafe, since it's rare that a caller should be using this. In most
- * cases, mutations to the task store should be done through
- * {@link com.twitter.aurora.scheduler.state.StateManager}.
- * <p>
- * TODO(William Farner): Come up with a way to restrict access to this interface. As it stands,
- * it's trivial for an unsuspecting caller to modify the task store directly and subvert the
- * state machine and side effect systems.
- *
- * @return The mutable task store.
- */
- TaskStore.Mutable getUnsafeTaskStore();
-
- LockStore.Mutable getLockStore();
- QuotaStore.Mutable getQuotaStore();
- AttributeStore.Mutable getAttributeStore();
- }
-
- /**
- * Encapsulates a read-only storage operation.
- *
- * @param <T> The type of result this unit of work produces.
- * @param <E> The type of exception this unit of work can throw.
- */
- interface Work<T, E extends Exception> {
-
- /**
- * Abstracts a unit of work that has a result, but may also throw a specific exception.
- *
- * @param storeProvider A provider to give access to different available stores.
- * @return the result of the successfully completed unit of work
- * @throws E if the unit of work could not be completed
- */
- T apply(StoreProvider storeProvider) throws E;
-
- /**
- * A convenient typedef for Work that throws no checked exceptions - it runs quietly.
- *
- * @param <T> The type of result this unit of work produces.
- */
- interface Quiet<T> extends Work<T, RuntimeException> {
- // typedef
- }
- }
-
- /**
- * Encapsulates a storage operation, which has mutable storage access.
- *
- * @param <T> The type of result this unit of work produces.
- * @param <E> The type of exception this unit of work can throw.
- */
- interface MutateWork<T, E extends Exception> {
-
- NoResult.Quiet NOOP = new NoResult.Quiet() {
- @Override protected void execute(Storage.MutableStoreProvider storeProvider) {
- // No-op.
- }
- };
-
- /**
- * Abstracts a unit of work that should either commit a set of changes to storage as a side
- * effect of successful completion or else commit no changes at all when an exception is thrown.
- *
- * @param storeProvider A provider to give access to different available stores.
- * @return the result of the successfully completed unit of work
- * @throws E if the unit of work could not be completed
- */
- T apply(MutableStoreProvider storeProvider) throws E;
-
- /**
- * A convenient typedef for Work that throws no checked exceptions - it runs quietly.
- *
- * @param <T> The type of result this unit of work produces.
- */
- interface Quiet<T> extends MutateWork<T, RuntimeException> {
- // typedef
- }
-
- /**
- * Encapsulates work that returns no result.
- *
- * @param <E> The type of exception this unit of work can throw.
- */
- abstract class NoResult<E extends Exception> implements MutateWork<Void, E> {
-
- @Override public final Void apply(MutableStoreProvider storeProvider) throws E {
- execute(storeProvider);
- return null;
- }
-
- /**
- * Similar to {@link #apply(MutableStoreProvider)} except that no result is
- * returned.
- *
- * @param storeProvider A provider to give access to different available stores.
- * @throws E if the unit of work could not be completed
- */
- protected abstract void execute(MutableStoreProvider storeProvider) throws E;
-
- /**
- * A convenient typedef for Work with no result that throws no checked exceptions - it runs
- * quitely.
- */
- public abstract static class Quiet extends NoResult<RuntimeException> {
- // typedef
- }
- }
- }
-
- /**
- * Indicates a problem reading from or writing to stable storage.
- */
- class StorageException extends SchedulerException {
- public StorageException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public StorageException(String message) {
- super(message);
- }
- }
-
- /**
- * Executes the unit of read-only {@code work}. All data in the stores may be expected to be
- * consistent, as the invocation is mutually exclusive of any writes.
- *
- * @param work The unit of work to execute.
- * @param <T> The type of result this unit of work produces.
- * @param <E> The type of exception this unit of work can throw.
- * @return the result when the unit of work completes successfully
- * @throws StorageException if there was a problem reading from stable storage.
- * @throws E bubbled transparently when the unit of work throws
- */
- <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E;
-
- /**
- * Executes a unit of read-only {@code work}. This is functionally identical to
- * {@link #consistentRead(Work)} with the exception that data in the stores may not be fully
- * consistent.
- *
- * @param work The unit of work to execute.
- * @param <T> The type of result this unit of work produces.
- * @param <E> The type of exception this unit of work can throw.
- * @return the result when the unit of work completes successfully
- * @throws StorageException if there was a problem reading from stable storage.
- * @throws E bubbled transparently when the unit of work throws
- */
- <T, E extends Exception> T weaklyConsistentRead(Work<T, E> work) throws StorageException, E;
-
- /**
- * Executes the unit of mutating {@code work}.
- *
- * @param work The unit of work to execute.
- * @param <T> The type of result this unit of work produces.
- * @param <E> The type of exception this unit of work can throw.
- * @return the result when the unit of work completes successfully
- * @throws StorageException if there was a problem reading from or writing to stable storage.
- * @throws E bubbled transparently when the unit of work throws
- */
- <T, E extends Exception> T write(MutateWork<T, E> work) throws StorageException, E;
-
- /**
- * Clean up the underlying storage by optimizing internal data structures. Does not change
- * externally-visible state but might not run concurrently with write operations.
- */
- void snapshot() throws StorageException;
-
- /**
- * A non-volatile storage that has additional methods to control its lifecycle.
- */
- interface NonVolatileStorage extends Storage {
- /**
- * Requests the underlying storage prepare its data set; ie: initialize schemas, begin syncing
- * out of date data, etc. This method should not block.
- *
- * @throws StorageException if there was a problem preparing storage.
- */
- void prepare() throws StorageException;
-
- /**
- * Prepares the underlying storage for serving traffic.
- *
- * @param initializationLogic work to perform after this storage system is ready but before
- * allowing general use of
- * {@link #consistentRead}.
- * @throws StorageException if there was a starting storage.
- */
- void start(MutateWork.NoResult.Quiet initializationLogic) throws StorageException;
-
- /**
- * Prepares the underlying storage system for clean shutdown.
- */
- void stop();
- }
-
- /**
- * Identifies a storage layer that is in-memory only.
- * This generally should only be used when the storage is first starting up, to perform queries
- * related to initially load the storage.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ ElementType.PARAMETER, ElementType.METHOD })
- @BindingAnnotation
- public @interface Volatile { }
-
- /**
- * Utility functions for interacting with a Storage instance.
- */
- public final class Util {
-
- private Util() {
- // Utility class.
- }
-
- /**
- * Fetch tasks matching the query returned by {@code query} from {@code storage} in a
- * read operation.
- *
- * @see #consistentFetchTasks
- * @param storage Storage instance to query from.
- * @param query Builder of the query to perform.
- * @return Tasks returned from the query.
- */
- public static ImmutableSet<IScheduledTask> consistentFetchTasks(
- Storage storage,
- final Query.Builder query) {
-
- return storage.consistentRead(new Work.Quiet<ImmutableSet<IScheduledTask>>() {
- @Override public ImmutableSet<IScheduledTask> apply(StoreProvider storeProvider) {
- return storeProvider.getTaskStore().fetchTasks(query);
- }
- });
- }
-
- /**
- * Identical to {@link #consistentFetchTasks(Storage, Query.Builder)}, but fetches tasks using a
- * weakly-consistent read operation.
- *
- * @see #consistentFetchTasks
- * @param storage Storage instance to query from.
- * @param query Builder of the query to perform.
- * @return Tasks returned from the query.
- */
- public static ImmutableSet<IScheduledTask> weaklyConsistentFetchTasks(
- Storage storage,
- final Query.Builder query) {
-
- return storage.weaklyConsistentRead(new Work.Quiet<ImmutableSet<IScheduledTask>>() {
- @Override public ImmutableSet<IScheduledTask> apply(StoreProvider storeProvider) {
- return storeProvider.getTaskStore().fetchTasks(query);
- }
- });
- }
-
- /**
- * Fetch quota for {@code role} from {@code storage} in a consistent read operation.
- *
- * @param storage Storage instance to fetch quota from.
- * @param role Role to fetch quota for.
- * @return Quota returned from the fetch operation.
- * @see QuotaStore#fetchQuota(String)
- */
- public static Optional<IQuota> consistentFetchQuota(Storage storage, final String role) {
- return storage.consistentRead(new Work.Quiet<Optional<IQuota>>() {
- @Override public Optional<IQuota> apply(StoreProvider storeProvider) {
- return storeProvider.getQuotaStore().fetchQuota(role);
- }
- });
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java b/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java
deleted file mode 100644
index df5b603..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/StorageBackfill.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-
-import com.twitter.aurora.gen.JobConfiguration;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.ScheduledTask;
-import com.twitter.aurora.gen.TaskConfig;
-import com.twitter.aurora.gen.TaskEvent;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.TaskStore.Mutable.TaskMutation;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-
-/**
- * Utility class to contain and perform storage backfill operations.
- */
-public final class StorageBackfill {
-
- private static final Logger LOG = Logger.getLogger(StorageBackfill.class.getName());
-
- private static final AtomicLong SHARD_SANITY_CHECK_FAILS =
- Stats.exportLong("shard_sanity_check_failures");
-
- private StorageBackfill() {
- // Utility class.
- }
-
- private static void backfillJobDefaults(JobStore.Mutable jobStore) {
- for (String id : jobStore.fetchManagerIds()) {
- for (JobConfiguration job : IJobConfiguration.toBuildersList(jobStore.fetchJobs(id))) {
- ConfigurationManager.applyDefaultsIfUnset(job);
- jobStore.saveAcceptedJob(id, IJobConfiguration.build(job));
- }
- }
- }
-
- private static void guaranteeShardUniqueness(
- ScheduledTask task,
- TaskStore.Mutable taskStore,
- Clock clock) {
-
- if (Tasks.isActive(task.getStatus())) {
- // Perform a sanity check on the number of active shards.
- TaskConfig config = task.getAssignedTask().getTask();
- Query.Builder query = Query.instanceScoped(
- JobKeys.from(config.getOwner().getRole(), config.getEnvironment(), config.getJobName()),
- task.getAssignedTask().getInstanceId())
- .active();
- Set<String> activeTasksInShard = FluentIterable.from(taskStore.fetchTasks(query))
- .transform(Tasks.SCHEDULED_TO_ID)
- .toSet();
-
- if (activeTasksInShard.size() > 1) {
- SHARD_SANITY_CHECK_FAILS.incrementAndGet();
- LOG.severe("Active shard sanity check failed when loading " + Tasks.id(task)
- + ", active tasks found: " + activeTasksInShard);
-
- // We want to keep exactly one task from this shard, so sort the IDs and keep the
- // highest (newest) in the hopes that it is legitimately running.
- String newestTask = Iterables.getLast(Sets.newTreeSet(activeTasksInShard));
- if (!Tasks.id(task).equals(newestTask)) {
- task.setStatus(ScheduleStatus.KILLED);
- task.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.KILLED)
- .setMessage("Killed duplicate shard."));
- // TODO(wfarner); Circle back if this is necessary. Currently there's a race
- // condition between the time the scheduler is actually available without hitting
- // IllegalStateException (see DriverImpl).
- // driver.killTask(Tasks.id(task));
- } else {
- LOG.info("Retaining task " + Tasks.id(task));
- }
- }
- }
- }
-
- private static final AtomicLong BOTH_FIELDS_SET = Stats.exportLong("both_instance_ids_set");
- private static final AtomicLong OLD_FIELD_SET = Stats.exportLong("old_instance_id_set");
- private static final AtomicLong NEW_FIELD_SET = Stats.exportLong("new_instance_id_set");
- private static final AtomicLong FIELDS_INCONSISTENT =
- Stats.exportLong("instance_ids_inconsistent");
-
- /**
- * Ensures backwards-compatibility of the throttled state, which exists in this version but is
- * not handled.
- *
- * @param task Task to possibly rewrite.
- */
- private static void rewriteThrottledState(ScheduledTask task) {
- if (ScheduleStatus.THROTTLED == task.getStatus()) {
- task.setStatus(ScheduleStatus.PENDING);
- }
- }
-
- /**
- * Backfills the storage to make it match any assumptions that may have changed since
- * the structs were first written.
- *
- * @param storeProvider Storage provider.
- * @param clock Clock, used for timestamping backfilled task events.
- */
- public static void backfill(final MutableStoreProvider storeProvider, final Clock clock) {
- backfillJobDefaults(storeProvider.getJobStore());
-
- LOG.info("Performing shard uniqueness sanity check.");
- storeProvider.getUnsafeTaskStore().mutateTasks(Query.unscoped(), new TaskMutation() {
- @Override public IScheduledTask apply(final IScheduledTask task) {
- ScheduledTask builder = task.newBuilder();
- ConfigurationManager.applyDefaultsIfUnset(builder.getAssignedTask().getTask());
- // TODO(ksweeney): Guarantee tasks pass current validation code here and quarantine if they
- // don't.
- guaranteeShardUniqueness(builder, storeProvider.getUnsafeTaskStore(), clock);
- rewriteThrottledState(builder);
- return IScheduledTask.build(builder);
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/TaskStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/TaskStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/TaskStore.java
deleted file mode 100644
index 02e5096..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/TaskStore.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage;
-
-import java.util.Set;
-
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableSet;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * Stores all tasks configured with the scheduler.
- */
-public interface TaskStore {
-
- /**
- * Fetches a read-only view of tasks matching a query and filters. Intended for use with a
- * {@link com.twitter.aurora.scheduler.base.Query.Builder}.
- *
- * @param query Builder of the query to identify tasks with.
- * @return A read-only view of matching tasks.
- */
- ImmutableSet<IScheduledTask> fetchTasks(Query.Builder query);
-
- public interface Mutable extends TaskStore {
-
- /**
- * A convenience interface to allow callers to more concisely implement a task mutation.
- */
- public interface TaskMutation extends Function<IScheduledTask, IScheduledTask> {
- }
-
- /**
- * Saves tasks to the store. Tasks are copied internally, meaning that the tasks are stored in
- * the state they were in when the method is called, and further object modifications will not
- * affect the tasks. If any of the tasks already exist in the store, they will be overwritten
- * by the supplied {@code newTasks}.
- *
- * @param tasks Tasks to add.
- */
- void saveTasks(Set<IScheduledTask> tasks);
-
- /**
- * Removes all tasks from the store.
- */
- void deleteAllTasks();
-
- /**
- * Deletes specific tasks from the store.
- *
- * @param taskIds IDs of tasks to delete.
- */
- void deleteTasks(Set<String> taskIds);
-
- /**
- * Offers temporary mutable access to tasks. If a task ID is not found, it will be silently
- * skipped, and no corresponding task will be returned.
- *
- * @param query Query to match tasks against.
- * @param mutator The mutate operation.
- * @return Immutable copies of only the tasks that were mutated.
- */
- ImmutableSet<IScheduledTask> mutateTasks(
- Query.Builder query,
- Function<IScheduledTask, IScheduledTask> mutator);
-
- /**
- * Rewrites a task's configuration in-place.
- * <p>
- * <b>WARNING</b>: this is a dangerous operation, and should not be used without exercising
- * great care. This feature should be used as a last-ditch effort to rewrite things that
- * the scheduler otherwise can't (e.g. {@link ITaskConfig#executorConfig}) rewrite in a
- * controlled/tested backfill operation.
- *
- * @param taskId ID of the task to alter.
- * @param taskConfiguration Configuration object to swap with the existing task's
- * configuration.
- * @return {@code true} if the modification took effect, or {@code false} if the task does not
- * exist in the store.
- */
- boolean unsafeModifyInPlace(String taskId, ITaskConfig taskConfiguration);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/backup/BackupModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/backup/BackupModule.java b/src/main/java/com/twitter/aurora/scheduler/storage/backup/BackupModule.java
deleted file mode 100644
index b6beba3..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/backup/BackupModule.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.backup;
-
-import java.io.File;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.inject.PrivateModule;
-import com.google.inject.Provides;
-import com.google.inject.TypeLiteral;
-
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.scheduler.storage.SnapshotStore;
-import com.twitter.aurora.scheduler.storage.backup.Recovery.RecoveryImpl;
-import com.twitter.aurora.scheduler.storage.backup.StorageBackup.StorageBackupImpl;
-import com.twitter.aurora.scheduler.storage.backup.StorageBackup.StorageBackupImpl.BackupConfig;
-import com.twitter.aurora.scheduler.storage.backup.TemporaryStorage.TemporaryStorageFactory;
-import com.twitter.common.application.Lifecycle;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.args.constraints.NotNull;
-import com.twitter.common.base.Command;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A module that will periodically save full storage backups to local disk and makes those backups
- * available for on-line recovery.
- */
-public class BackupModule extends PrivateModule {
- private static final Logger LOG = Logger.getLogger(BackupModule.class.getName());
-
- @CmdLine(name = "backup_interval", help = "Minimum interval on which to write a storage backup.")
- private static final Arg<Amount<Long, Time>> BACKUP_INTERVAL =
- Arg.create(Amount.of(1L, Time.HOURS));
-
- @CmdLine(name = "max_saved_backups",
- help = "Maximum number of backups to retain before deleting the oldest backups.")
- private static final Arg<Integer> MAX_SAVED_BACKUPS = Arg.create(48);
-
- @NotNull
- @CmdLine(name = "backup_dir",
- help = "Directory to store backups under. Will be created if it does not exist.")
- private static final Arg<File> BACKUP_DIR = Arg.create();
-
- private final Class<? extends SnapshotStore<Snapshot>> snapshotStore;
- private final File unvalidatedBackupDir;
-
- /**
- * Creates a new backup module.
- *
- * @param snapshotStore Snapshot store implementation class.
- */
- public BackupModule(Class<? extends SnapshotStore<Snapshot>> snapshotStore) {
- this(BACKUP_DIR.get(), snapshotStore);
- }
-
- /**
- * Creates a new backup module using a given backupDir instead of a flagged one.
- *
- * @param backupDir Directory to write backups to.
- * @param snapshotStore Snapshot store implementation class.
- */
- @VisibleForTesting
- public BackupModule(File backupDir, Class<? extends SnapshotStore<Snapshot>> snapshotStore) {
- this.unvalidatedBackupDir = checkNotNull(backupDir);
- this.snapshotStore = checkNotNull(snapshotStore);
- }
-
- @Override
- protected void configure() {
- TypeLiteral<SnapshotStore<Snapshot>> type = new TypeLiteral<SnapshotStore<Snapshot>>() { };
- bind(type).annotatedWith(StorageBackupImpl.SnapshotDelegate.class).to(snapshotStore);
-
- bind(type).to(StorageBackupImpl.class);
- bind(StorageBackup.class).to(StorageBackupImpl.class);
- bind(StorageBackupImpl.class).in(Singleton.class);
- expose(type);
- expose(StorageBackup.class);
-
- bind(new TypeLiteral<Function<Snapshot, TemporaryStorage>>() { })
- .to(TemporaryStorageFactory.class);
-
- bind(Command.class).to(LifecycleHook.class);
- bind(Recovery.class).to(RecoveryImpl.class);
- bind(RecoveryImpl.class).in(Singleton.class);
- expose(Recovery.class);
- }
-
- static class LifecycleHook implements Command {
- private final Lifecycle lifecycle;
-
- @Inject LifecycleHook(Lifecycle lifecycle) {
- this.lifecycle = checkNotNull(lifecycle);
- }
-
- @Override public void execute() {
- lifecycle.shutdown();
- }
- }
-
- @Provides
- private File provideBackupDir() {
- if (!unvalidatedBackupDir.exists()) {
- if (!unvalidatedBackupDir.mkdirs()) {
- throw new IllegalArgumentException(
- "Unable to create backup dir " + unvalidatedBackupDir.getPath() + ".");
- } else {
- LOG.info("Created backup dir " + unvalidatedBackupDir.getPath() + ".");
- }
- }
-
- if (!unvalidatedBackupDir.canWrite()) {
- throw new IllegalArgumentException(
- "Backup dir " + unvalidatedBackupDir.getPath() + " is not writable.");
- }
-
- return unvalidatedBackupDir;
- }
-
- @Provides
- private BackupConfig provideBackupConfig(File backupDir) {
- return new BackupConfig(backupDir, MAX_SAVED_BACKUPS.get(), BACKUP_INTERVAL.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/backup/Recovery.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/backup/Recovery.java b/src/main/java/com/twitter/aurora/scheduler/storage/backup/Recovery.java
deleted file mode 100644
index 4e91342..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/backup/Recovery.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.backup;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.io.Files;
-import com.google.common.util.concurrent.Atomics;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.storage.DistributedSnapshotStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.base.Command;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A recovery mechanism that works with {@link StorageBackup} to provide a two-step storage
- * recovery process.
- */
-public interface Recovery {
-
- /**
- * List backups available for recovery.
- *
- * @return Available backup IDs.
- */
- Set<String> listBackups();
-
- /**
- * Loads a backup in 'staging' so that it may be queried and modified prior to committing.
- *
- * @param backupName Name of the backup to load.
- * @throws RecoveryException If the backup could not be found or loaded.
- */
- void stage(String backupName) throws RecoveryException;
-
- /**
- * Queries a staged backup.
- *
- * @param query Builder of query to perform.
- * @return Tasks matching the query.
- * @throws RecoveryException If a backup is not staged, or could not be queried.
- */
- Set<IScheduledTask> query(Query.Builder query) throws RecoveryException;
-
- /**
- * Deletes tasks from a staged backup.
- *
- * @param query Query builder for tasks to delete.
- * @throws RecoveryException If a backup is not staged, or tasks could not be deleted.
- */
- void deleteTasks(Query.Builder query) throws RecoveryException;
-
- /**
- * Unloads a staged backup.
- */
- void unload();
-
- /**
- * Commits a staged backup the main storage system.
- *
- * @throws RecoveryException If a backup is not staged, or the commit failed.
- */
- void commit() throws RecoveryException;
-
- /**
- * Thrown when a recovery operation could not be completed due to internal errors or improper
- * invocation order.
- */
- public static class RecoveryException extends Exception {
- RecoveryException(String message) {
- super(message);
- }
-
- RecoveryException(String message, Throwable cause) {
- super(message, cause);
- }
- }
-
- class RecoveryImpl implements Recovery {
- private final File backupDir;
- private final Function<Snapshot, TemporaryStorage> tempStorageFactory;
- private final AtomicReference<PendingRecovery> recovery;
- private final Storage primaryStorage;
- private final DistributedSnapshotStore distributedStore;
- private final Command shutDownNow;
-
- @Inject
- RecoveryImpl(
- File backupDir,
- Function<Snapshot, TemporaryStorage> tempStorageFactory,
- Storage primaryStorage,
- DistributedSnapshotStore distributedStore,
- Command shutDownNow) {
-
- this.backupDir = checkNotNull(backupDir);
- this.tempStorageFactory = checkNotNull(tempStorageFactory);
- this.recovery = Atomics.newReference();
- this.primaryStorage = checkNotNull(primaryStorage);
- this.distributedStore = checkNotNull(distributedStore);
- this.shutDownNow = checkNotNull(shutDownNow);
- }
-
- @Override public Set<String> listBackups() {
- return ImmutableSet.<String>builder().add(backupDir.list()).build();
- }
-
- @Override public void stage(String backupName) throws RecoveryException {
- File backupFile = new File(backupDir, backupName);
- if (!backupFile.exists()) {
- throw new RecoveryException("Backup " + backupName + " does not exist.");
- }
-
- Snapshot snapshot;
- try {
- snapshot = ThriftBinaryCodec.decode(Snapshot.class, Files.toByteArray(backupFile));
- } catch (CodingException e) {
- throw new RecoveryException("Failed to decode backup " + e, e);
- } catch (IOException e) {
- throw new RecoveryException("Failed to read backup " + e, e);
- }
- boolean applied =
- recovery.compareAndSet(null, new PendingRecovery(tempStorageFactory.apply(snapshot)));
- if (!applied) {
- throw new RecoveryException("Another backup is already loaded.");
- }
- }
-
- private PendingRecovery getLoadedRecovery() throws RecoveryException {
- @Nullable PendingRecovery loaded = this.recovery.get();
- if (loaded == null) {
- throw new RecoveryException("No backup loaded.");
- }
- return loaded;
- }
-
- @Override public Set<IScheduledTask> query(Query.Builder query) throws RecoveryException {
- return getLoadedRecovery().query(query);
- }
-
- @Override public void deleteTasks(Query.Builder query) throws RecoveryException {
- getLoadedRecovery().delete(query);
- }
-
- @Override public void unload() {
- recovery.set(null);
- }
-
- @Override public void commit() throws RecoveryException {
- getLoadedRecovery().commit();
- }
-
- private class PendingRecovery {
- private final TemporaryStorage tempStorage;
-
- PendingRecovery(TemporaryStorage tempStorage) {
- this.tempStorage = tempStorage;
- }
-
- void commit() {
- primaryStorage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- try {
- distributedStore.persist(tempStorage.toSnapshot());
- shutDownNow.execute();
- } catch (CodingException e) {
- throw new IllegalStateException("Failed to encode snapshot.", e);
- }
- }
- });
- }
-
- Set<IScheduledTask> query(final Query.Builder query) {
- return tempStorage.fetchTasks(query);
- }
-
- void delete(final Query.Builder query) {
- tempStorage.deleteTasks(query);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/backup/StorageBackup.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/backup/StorageBackup.java b/src/main/java/com/twitter/aurora/scheduler/storage/backup/StorageBackup.java
deleted file mode 100644
index 3faeb1f..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/backup/StorageBackup.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.backup;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Ordering;
-import com.google.common.io.Files;
-import com.google.inject.BindingAnnotation;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.scheduler.storage.SnapshotStore;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A backup routine that layers over a snapshot store and periodically writes snapshots to
- * local disk.
- *
- * TODO(William Farner): Perform backups asynchronously. As written, they are performed in a
- * blocking write operation, which is asking for trouble.
- */
-public interface StorageBackup {
-
- /**
- * Perform a storage backup immediately, blocking until it is complete.
- */
- void backupNow();
-
- class StorageBackupImpl implements StorageBackup, SnapshotStore<Snapshot> {
- private static final Logger LOG = Logger.getLogger(StorageBackup.class.getName());
-
- private static final String FILE_PREFIX = "scheduler-backup-";
- private final BackupConfig config;
-
- static class BackupConfig {
- private final File dir;
- private final int maxBackups;
- private final Amount<Long, Time> interval;
-
- BackupConfig(File dir, int maxBackups, Amount<Long, Time> interval) {
- this.dir = checkNotNull(dir);
- this.maxBackups = maxBackups;
- this.interval = checkNotNull(interval);
- }
-
- @VisibleForTesting
- File getDir() {
- return dir;
- }
- }
-
- /**
- * Binding annotation that the underlying {@link SnapshotStore} must be bound with.
- */
- @BindingAnnotation
- @Target({FIELD, PARAMETER, METHOD}) @Retention(RUNTIME)
- @interface SnapshotDelegate { }
-
- private final SnapshotStore<Snapshot> delegate;
- private final Clock clock;
- private final long backupIntervalMs;
- private volatile long lastBackupMs;
- private final DateFormat backupDateFormat;
-
- private final AtomicLong successes = Stats.exportLong("scheduler_backup_success");
- @VisibleForTesting
- AtomicLong getSuccesses() {
- return successes;
- }
-
- private final AtomicLong failures = Stats.exportLong("scheduler_backup_failed");
- @VisibleForTesting
- AtomicLong getFailures() {
- return failures;
- }
-
- @Inject
- StorageBackupImpl(
- @SnapshotDelegate SnapshotStore<Snapshot> delegate,
- Clock clock,
- BackupConfig config) {
-
- this.delegate = checkNotNull(delegate);
- this.clock = checkNotNull(clock);
- this.config = checkNotNull(config);
- backupDateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
- backupIntervalMs = config.interval.as(Time.MILLISECONDS);
- lastBackupMs = clock.nowMillis();
- }
-
- @Override public Snapshot createSnapshot() {
- Snapshot snapshot = delegate.createSnapshot();
- if (clock.nowMillis() >= (lastBackupMs + backupIntervalMs)) {
- save(snapshot);
- }
- return snapshot;
- }
-
- @Override public void backupNow() {
- save(delegate.createSnapshot());
- }
-
- @VisibleForTesting
- String createBackupName() {
- return FILE_PREFIX + backupDateFormat.format(new Date(clock.nowMillis()));
- }
-
- private void save(Snapshot snapshot) {
- lastBackupMs = clock.nowMillis();
-
- String backupName = createBackupName();
- String tempBackupName = "temp_" + backupName;
- File tempFile = new File(config.dir, tempBackupName);
- LOG.info("Saving backup to " + tempFile);
- try {
- byte[] backup = ThriftBinaryCodec.encodeNonNull(snapshot);
- Files.write(backup, tempFile);
- Files.move(tempFile, new File(config.dir, backupName));
- successes.incrementAndGet();
- } catch (IOException e) {
- failures.incrementAndGet();
- LOG.log(Level.SEVERE, "Failed to prepare backup " + backupName + ": " + e, e);
- } catch (CodingException e) {
- LOG.log(Level.SEVERE, "Failed to encode backup " + backupName + ": " + e, e);
- failures.incrementAndGet();
- } finally {
- if (tempFile.exists()) {
- LOG.info("Deleting incomplete backup file " + tempFile);
- tempFile.delete();
- }
- }
-
- File[] backups = config.dir.listFiles(BACKUP_FILTER);
- if (backups == null) {
- LOG.severe("Failed to list backup dir " + config.dir);
- } else {
- int backupsToDelete = backups.length - config.maxBackups;
- if (backupsToDelete > 0) {
- List<File> toDelete = Ordering.natural()
- .onResultOf(FILE_NAME)
- .sortedCopy(ImmutableList.copyOf(backups)).subList(0, backupsToDelete);
- LOG.info("Deleting " + backupsToDelete + " outdated backups: " + toDelete);
- for (File outdated : toDelete) {
- outdated.delete();
- }
- }
- }
- }
-
- private static final FilenameFilter BACKUP_FILTER = new FilenameFilter() {
- @Override public boolean accept(File file, String s) {
- return s.startsWith(FILE_PREFIX);
- }
- };
-
- @VisibleForTesting
- static final Function<File, String> FILE_NAME = new Function<File, String>() {
- @Override public String apply(File file) {
- return file.getName();
- }
- };
-
- @Override
- public void applySnapshot(Snapshot snapshot) {
- delegate.applySnapshot(snapshot);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/backup/TemporaryStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/backup/TemporaryStorage.java b/src/main/java/com/twitter/aurora/scheduler/storage/backup/TemporaryStorage.java
deleted file mode 100644
index e0906fe..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/backup/TemporaryStorage.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.backup;
-
-import java.util.Set;
-
-import com.google.common.base.Function;
-import com.google.common.collect.FluentIterable;
-
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.storage.SnapshotStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.log.SnapshotStoreImpl;
-import com.twitter.aurora.scheduler.storage.mem.MemStorage;
-import com.twitter.common.util.testing.FakeClock;
-
-/**
- * A short-lived in-memory storage system that can be converted to a {@link Snapshot}.
- */
-interface TemporaryStorage {
-
- /**
- * Deletes all tasks matching a query. Deleted tasks will not be reflected in the snapshot when
- * {@link #toSnapshot()} is executed.
- *
- * @param query Query builder for tasks to delete.
- */
- void deleteTasks(Query.Builder query);
-
- /**
- * Fetches tasks matching a query.
- *
- * @param query Query builder for tasks to fetch.
- * @return Matching tasks.
- */
- Set<IScheduledTask> fetchTasks(Query.Builder query);
-
- /**
- * Creates a snapshot of the contents of the temporary storage.
- *
- * @return Temporary storage snapshot.
- */
- Snapshot toSnapshot();
-
- /**
- * A factory that creates temporary storage instances, detached from the rest of the system.
- */
- class TemporaryStorageFactory implements Function<Snapshot, TemporaryStorage> {
- @Override public TemporaryStorage apply(Snapshot snapshot) {
- final Storage storage = MemStorage.newEmptyStorage();
- FakeClock clock = new FakeClock();
- clock.setNowMillis(snapshot.getTimestamp());
- final SnapshotStore<Snapshot> snapshotStore = new SnapshotStoreImpl(clock, storage);
- snapshotStore.applySnapshot(snapshot);
-
- return new TemporaryStorage() {
- @Override public void deleteTasks(final Query.Builder query) {
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- Set<String> ids = FluentIterable.from(storeProvider.getTaskStore().fetchTasks(query))
- .transform(Tasks.SCHEDULED_TO_ID)
- .toSet();
- storeProvider.getUnsafeTaskStore().deleteTasks(ids);
- }
- });
- }
-
- @Override public Set<IScheduledTask> fetchTasks(final Query.Builder query) {
- return storage.consistentRead(new Work.Quiet<Set<IScheduledTask>>() {
- @Override public Set<IScheduledTask> apply(StoreProvider storeProvider) {
- return storeProvider.getTaskStore().fetchTasks(query);
- }
- });
- }
-
- @Override public Snapshot toSnapshot() {
- return snapshotStore.createSnapshot();
- }
- };
- }
- }
-}
[47/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/configuration/Resources.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/configuration/Resources.java b/src/main/java/com/twitter/aurora/scheduler/configuration/Resources.java
deleted file mode 100644
index 51e9973..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/configuration/Resources.java
+++ /dev/null
@@ -1,447 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.configuration;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ContiguousSet;
-import com.google.common.collect.DiscreteDomain;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.Sets;
-
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.Resource;
-import org.apache.mesos.Protos.Value.Range;
-import org.apache.mesos.Protos.Value.Ranges;
-import org.apache.mesos.Protos.Value.Scalar;
-import org.apache.mesos.Protos.Value.Type;
-
-import com.twitter.aurora.scheduler.base.Numbers;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Data;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A container for multiple resource vectors.
- * TODO(wfarner): Collapse this in with Quotas.
- */
-public class Resources {
-
- public static final String CPUS = "cpus";
- public static final String RAM_MB = "mem";
- public static final String DISK_MB = "disk";
- public static final String PORTS = "ports";
-
- private static final Function<Range, Set<Integer>> RANGE_TO_MEMBERS =
- new Function<Range, Set<Integer>>() {
- @Override public Set<Integer> apply(Range range) {
- return ContiguousSet.create(
- com.google.common.collect.Range.closed((int) range.getBegin(), (int) range.getEnd()),
- DiscreteDomain.integers());
- }
- };
-
- private final double numCpus;
- private final Amount<Long, Data> disk;
- private final Amount<Long, Data> ram;
- private final int numPorts;
-
- /**
- * Creates a new resources object.
- *
- * @param numCpus Number of CPUs.
- * @param ram Amount of RAM.
- * @param disk Amount of disk.
- * @param numPorts Number of ports.
- */
- public Resources(double numCpus, Amount<Long, Data> ram, Amount<Long, Data> disk, int numPorts) {
- this.numCpus = numCpus;
- this.ram = checkNotNull(ram);
- this.disk = checkNotNull(disk);
- this.numPorts = numPorts;
- }
-
- /**
- * Tests whether this bundle of resources is greater than or equal to another bundle of resources.
- *
- * @param other Resources being compared to.
- * @return {@code true} if all resources in this bundle are greater than or equal to the
- * equivalents from {@code other}, otherwise {@code false}.
- */
- public boolean greaterThanOrEqual(Resources other) {
- return (numCpus >= other.numCpus)
- && (disk.as(Data.MB) >= other.disk.as(Data.MB))
- && (ram.as(Data.MB) >= other.ram.as(Data.MB))
- && (numPorts >= other.numPorts);
- }
-
- /**
- * Adapts this resources object to a list of mesos resources.
- *
- * @param selectedPorts The ports selected, to be applied as concrete task ranges.
- * @return Mesos resources.
- */
- public List<Resource> toResourceList(Set<Integer> selectedPorts) {
- ImmutableList.Builder<Resource> resourceBuilder =
- ImmutableList.<Resource>builder()
- .add(Resources.makeMesosResource(CPUS, numCpus))
- .add(Resources.makeMesosResource(DISK_MB, disk.as(Data.MB)))
- .add(Resources.makeMesosResource(RAM_MB, ram.as(Data.MB)));
- if (selectedPorts.size() > 0) {
- resourceBuilder.add(Resources.makeMesosRangeResource(Resources.PORTS, selectedPorts));
- }
-
- return resourceBuilder.build();
- }
-
- /**
- * Convenience method for adapting to mesos resources without applying a port range.
- *
- * @see {@link #toResourceList(java.util.Set)}
- * @return Mesos resources.
- */
- public List<Resource> toResourceList() {
- return toResourceList(ImmutableSet.<Integer>of());
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof Resources)) {
- return false;
- }
-
- Resources other = (Resources) o;
- return new EqualsBuilder()
- .append(numCpus, other.numCpus)
- .append(ram, other.ram)
- .append(disk, other.disk)
- .append(numPorts, other.numPorts)
- .isEquals();
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(numCpus, ram, disk, numPorts);
- }
-
- /**
- * Extracts the resources required from a task.
- *
- * @param task Task to get resources from.
- * @return The resources required by the task.
- */
- public static Resources from(ITaskConfig task) {
- checkNotNull(task);
- return new Resources(
- task.getNumCpus(),
- Amount.of(task.getRamMb(), Data.MB),
- Amount.of(task.getDiskMb(), Data.MB),
- task.getRequestedPorts().size());
- }
-
- /**
- * Extracts the resources specified in a list of resource objects.
- *
- * @param resources Resources to translate.
- * @return The canonical resources.
- */
- public static Resources from(List<Resource> resources) {
- checkNotNull(resources);
- return new Resources(
- getScalarValue(resources, CPUS),
- Amount.of((long) getScalarValue(resources, RAM_MB), Data.MB),
- Amount.of((long) getScalarValue(resources, DISK_MB), Data.MB),
- getNumAvailablePorts(resources)
- );
- }
-
- /**
- * Extracts the resources available in a slave offer.
- *
- * @param offer Offer to get resources from.
- * @return The resources available in the offer.
- */
- public static Resources from(Offer offer) {
- checkNotNull(offer);
- return new Resources(
- getScalarValue(offer, CPUS),
- Amount.of((long) getScalarValue(offer, RAM_MB), Data.MB),
- Amount.of((long) getScalarValue(offer, DISK_MB), Data.MB),
- getNumAvailablePorts(offer.getResourcesList()));
- }
-
- private static final Resources NO_RESOURCES =
- new Resources(0, Amount.of(0L, Data.BITS), Amount.of(0L, Data.BITS), 0);
-
- private static Resources none() {
- return NO_RESOURCES;
- }
-
- /**
- * a - b
- */
- public static Resources subtract(Resources a, Resources b) {
- return new Resources(
- a.getNumCpus() - b.getNumCpus(),
- Amount.of(a.getRam().as(Data.MB) - b.getRam().as(Data.MB), Data.MB),
- Amount.of(a.getDisk().as(Data.MB) - b.getDisk().as(Data.MB), Data.MB),
- a.getNumPorts() - b.getNumPorts());
- }
-
- /**
- * sum(a, b)
- */
- public static Resources sum(Resources a, Resources b) {
- return sum(ImmutableList.of(a, b));
- }
-
- /**
- * sum(rs)
- */
- public static Resources sum(Iterable<Resources> rs) {
- Resources sum = none();
-
- for (Resources r : rs) {
- double numCpus = sum.getNumCpus() + r.getNumCpus();
- Amount<Long, Data> disk =
- Amount.of(sum.getDisk().as(Data.BYTES) + r.getDisk().as(Data.BYTES), Data.BYTES);
- Amount<Long, Data> ram =
- Amount.of(sum.getRam().as(Data.BYTES) + r.getRam().as(Data.BYTES), Data.BYTES);
- int ports = sum.getNumPorts() + r.getNumPorts();
- sum = new Resources(numCpus, ram, disk, ports);
- }
-
- return sum;
- }
-
- private static int getNumAvailablePorts(List<Resource> resource) {
- int offeredPorts = 0;
- for (Range range : getPortRanges(resource)) {
- offeredPorts += 1 + (range.getEnd() - range.getBegin());
- }
- return offeredPorts;
- }
-
- private static double getScalarValue(Offer offer, String key) {
- return getScalarValue(offer.getResourcesList(), key);
- }
-
- private static double getScalarValue(List<Resource> resources, String key) {
- Resource resource = getResource(resources, key);
- if (resource == null) {
- return 0;
- }
-
- return resource.getScalar().getValue();
- }
-
- private static Resource getResource(List<Resource> resource, String key) {
- return Iterables.find(resource, withName(key), null);
- }
-
- private static Predicate<Resource> withName(final String name) {
- return new Predicate<Resource>() {
- @Override public boolean apply(Resource resource) {
- return resource.getName().equals(name);
- }
- };
- }
-
- private static Iterable<Range> getPortRanges(List<Resource> resources) {
- Resource resource = getResource(resources, Resources.PORTS);
- if (resource == null) {
- return ImmutableList.of();
- }
-
- return resource.getRanges().getRangeList();
- }
-
- /**
- * Creates a scalar mesos resource.
- *
- * @param name Name of the resource.
- * @param value Value for the resource.
- * @return A mesos resource.
- */
- public static Resource makeMesosResource(String name, double value) {
- return Resource.newBuilder().setName(name).setType(Type.SCALAR)
- .setScalar(Scalar.newBuilder().setValue(value)).build();
- }
-
- private static final Function<com.google.common.collect.Range<Integer>, Range> RANGE_TRANSFORM =
- new Function<com.google.common.collect.Range<Integer>, Range>() {
- @Override public Range apply(com.google.common.collect.Range<Integer> input) {
- return Range.newBuilder()
- .setBegin(input.lowerEndpoint())
- .setEnd(input.upperEndpoint())
- .build();
- }
- };
-
- /**
- * Creates a mesos resource of integer ranges.
- *
- * @param name Name of the resource
- * @param values Values to translate into ranges.
- * @return A mesos ranges resource.
- */
- static Resource makeMesosRangeResource(String name, Set<Integer> values) {
- return Resource.newBuilder()
- .setName(name)
- .setType(Type.RANGES)
- .setRanges(Ranges.newBuilder()
- .addAllRange(Iterables.transform(Numbers.toRanges(values), RANGE_TRANSFORM)))
- .build();
- }
-
- /**
- * Number of CPUs.
- *
- * @return CPUs.
- */
- public double getNumCpus() {
- return numCpus;
- }
-
- /**
- * Disk amount.
- *
- * @return Disk.
- */
- public Amount<Long, Data> getDisk() {
- return disk;
- }
-
- /**
- * RAM amount.
- *
- * @return RAM.
- */
- public Amount<Long, Data> getRam() {
- return ram;
- }
-
- /**
- * Number of ports.
- *
- * @return Port count.
- */
- public int getNumPorts() {
- return numPorts;
- }
-
- /**
- * Thrown when there are insufficient resources to satisfy a request.
- */
- static class InsufficientResourcesException extends RuntimeException {
- public InsufficientResourcesException(String message) {
- super(message);
- }
- }
-
- /**
- * Attempts to grab {@code numPorts} from the given resource {@code offer}.
- *
- * @param offer The offer to grab ports from.
- * @param numPorts The number of ports to grab.
- * @return The set of ports grabbed.
- * @throws InsufficientResourcesException if not enough ports were available.
- */
- public static Set<Integer> getPorts(Offer offer, int numPorts)
- throws InsufficientResourcesException {
-
- checkNotNull(offer);
-
- if (numPorts == 0) {
- return ImmutableSet.of();
- }
-
- List<Integer> availablePorts = Lists.newArrayList(Sets.newHashSet(
- Iterables.concat(
- Iterables.transform(getPortRanges(offer.getResourcesList()), RANGE_TO_MEMBERS))));
-
- if (availablePorts.size() < numPorts) {
- throw new InsufficientResourcesException(
- String.format("Could not get %d ports from %s", numPorts, offer));
- }
-
- Collections.shuffle(availablePorts);
- return ImmutableSet.copyOf(availablePorts.subList(0, numPorts));
- }
-
- /**
- * A Resources object is greater than another iff _all_ of its resource components are greater
- * or equal. A Resources object compares as equal if some but not all components are greater than
- * or equal to the other.
- */
- public static final Ordering<Resources> RESOURCE_ORDER = new Ordering<Resources>() {
- @Override public int compare(Resources left, Resources right) {
- int diskC = left.getDisk().compareTo(right.getDisk());
- int ramC = left.getRam().compareTo(right.getRam());
- int portC = Integer.compare(left.getNumPorts(), right.getNumPorts());
- int cpuC = Double.compare(left.getNumCpus(), right.getNumCpus());
-
- FluentIterable<Integer> vector =
- FluentIterable.from(ImmutableList.of(diskC, ramC, portC, cpuC));
-
- if (vector.allMatch(IS_ZERO)) {
- return 0;
- }
-
- if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_POSITIVE)) {
- return 1;
- }
-
- if (vector.filter(Predicates.not(IS_ZERO)).allMatch(IS_NEGATIVE)) {
- return -1;
- }
-
- return 0;
- }
- };
-
- private static final Predicate<Integer> IS_POSITIVE = new Predicate<Integer>() {
- @Override public boolean apply(Integer input) {
- return input > 0;
- }
- };
-
- private static final Predicate<Integer> IS_NEGATIVE = new Predicate<Integer>() {
- @Override public boolean apply(Integer input) {
- return input < 0;
- }
- };
-
- private static final Predicate<Integer> IS_ZERO = new Predicate<Integer>() {
- @Override public boolean apply(Integer input) {
- return input == 0;
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/configuration/SanitizedConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/configuration/SanitizedConfiguration.java b/src/main/java/com/twitter/aurora/scheduler/configuration/SanitizedConfiguration.java
deleted file mode 100644
index 890acbb..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/configuration/SanitizedConfiguration.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.configuration;
-
-import java.util.Map;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Functions;
-import com.google.common.base.Objects;
-import com.google.common.collect.ContiguousSet;
-import com.google.common.collect.DiscreteDomain;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Range;
-
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * Wrapper for a configuration that has been fully-sanitized and populated with defaults.
- */
-public final class SanitizedConfiguration {
-
- private final IJobConfiguration sanitized;
- private final Map<Integer, ITaskConfig> tasks;
-
- /**
- * Constructs a SanitizedConfiguration object and populates the set of {@link ITaskConfig}s for
- * the provided config.
- *
- * @param sanitized A sanitized configuration.
- */
- @VisibleForTesting
- public SanitizedConfiguration(IJobConfiguration sanitized) {
- this.sanitized = sanitized;
- this.tasks = Maps.toMap(
- ContiguousSet.create(
- Range.closedOpen(0, sanitized.getInstanceCount()),
- DiscreteDomain.integers()),
- Functions.constant(sanitized.getTaskConfig()));
- }
-
- /**
- * Wraps an unsanitized job configuration.
- *
- * @param unsanitized Unsanitized configuration to sanitize/populate and wrap.
- * @return A wrapper containing the sanitized configuration.
- * @throws TaskDescriptionException If the configuration is invalid.
- */
- public static SanitizedConfiguration fromUnsanitized(IJobConfiguration unsanitized)
- throws TaskDescriptionException {
-
- return new SanitizedConfiguration(ConfigurationManager.validateAndPopulate(unsanitized));
- }
-
- public IJobConfiguration getJobConfig() {
- return sanitized;
- }
-
- // TODO(William Farner): Rework this API now that all configs are identical.
- public Map<Integer, ITaskConfig> getTaskConfigs() {
- return tasks;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof SanitizedConfiguration)) {
- return false;
- }
-
- SanitizedConfiguration other = (SanitizedConfiguration) o;
-
- return Objects.equal(sanitized, other.sanitized);
- }
-
- @Override
- public int hashCode() {
- return sanitized.hashCode();
- }
-
- @Override
- public String toString() {
- return sanitized.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/cron/CronException.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/CronException.java b/src/main/java/com/twitter/aurora/scheduler/cron/CronException.java
deleted file mode 100644
index c29a578..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/cron/CronException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.cron;
-
-/**
- * Exception class to signal a failure in the underlying cron implementation.
- */
-public class CronException extends Exception {
- public CronException(String msg) {
- super(msg);
- }
-
- public CronException(String msg, Throwable t) {
- super(msg, t);
- }
-
- public CronException(Throwable t) {
- super(t);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/cron/CronPredictor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/CronPredictor.java b/src/main/java/com/twitter/aurora/scheduler/cron/CronPredictor.java
deleted file mode 100644
index d01e2f8..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/cron/CronPredictor.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.cron;
-
-import java.util.Date;
-
-/**
- * A utility function that predicts a cron run given a schedule.
- */
-public interface CronPredictor {
- /**
- * Predicts the next date at which a cron schedule will trigger.
- *
- * @param schedule Cron schedule to predict the next time for.
- * @return A prediction for the next time a cron will run.
- */
- Date predictNextRun(String schedule);
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/cron/CronScheduler.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/CronScheduler.java b/src/main/java/com/twitter/aurora/scheduler/cron/CronScheduler.java
deleted file mode 100644
index 0ea9b65..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/cron/CronScheduler.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.cron;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Optional;
-
-/**
- * An execution manager that executes work on a cron schedule.
- */
-public interface CronScheduler {
- /**
- * Schedules a task on a cron schedule.
- *
- * @param schedule Cron-style schedule.
- * @param task Work to run when on the cron schedule.
- * @return A unique ID to identify the scheduled cron task.
- * @throws CronException when there was a failure to schedule, for example if {@code schedule}
- * is not a valid input.
- * @throws IllegalStateException If the cron scheduler is not currently running.
- */
- String schedule(String schedule, Runnable task) throws CronException, IllegalStateException;
-
- /**
- * Removes a scheduled cron item.
- *
- * @param key Key previously returned from {@link #schedule(String, Runnable)}.
- * @throws IllegalStateException If the cron scheduler is not currently running.
- */
- void deschedule(String key) throws IllegalStateException;
-
- /**
- * Gets the cron schedule associated with a scheduling key.
- *
- * @param key Key previously returned from {@link #schedule(String, Runnable)}.
- * @return The task's cron schedule, if a matching task was found.
- * @throws IllegalStateException If the cron scheduler is not currently running.
- */
- Optional<String> getSchedule(String key) throws IllegalStateException;
-
- /**
- * Block until fully initialized. It is an error to call start twice. Prior to calling start,
- * all other methods of this interface may throw {@link IllegalStateException}. The underlying
- * implementation should not spawn threads or connect to databases prior to invocation of
- * {@link #start()}.
- *
- * @throws IllegalStateException If called twice.
- */
- void start() throws IllegalStateException;
-
- /**
- * Block until stopped. Generally this means that underlying resources are freed, threads are
- * terminated, and any bookkeeping state is persisted. If {@link #stop()} has already been called
- * by another thread, {@link #stop()} either blocks until completion or returns immediately.
- *
- * @throws CronException If there was a problem stopping the scheduler, for example if it was not
- * started.
- */
- void stop() throws CronException;
-
- /**
- * Checks to see if the scheduler would be accepted by the underlying scheduler.
- *
- * @param schedule Cron scheduler to validate.
- * @return {@code true} if the schedule is valid.
- */
- boolean isValidSchedule(@Nullable String schedule);
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronModule.java b/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronModule.java
deleted file mode 100644
index d1c5419..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronModule.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.cron.noop;
-
-import javax.inject.Singleton;
-
-import com.google.inject.AbstractModule;
-
-import com.twitter.aurora.scheduler.cron.CronPredictor;
-import com.twitter.aurora.scheduler.cron.CronScheduler;
-
-/**
- * A Module to wire up a cron scheduler that does not actually schedule cron jobs.
- *
- * This class exists as a short term hack to get around a license compatibility issue - Real
- * Implementation (TM) coming soon.
- */
-public class NoopCronModule extends AbstractModule {
- @Override
- protected void configure() {
- bind(CronScheduler.class).to(NoopCronScheduler.class);
- bind(NoopCronScheduler.class).in(Singleton.class);
-
- bind(CronPredictor.class).to(NoopCronPredictor.class);
- bind(NoopCronPredictor.class).in(Singleton.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronPredictor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronPredictor.java b/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronPredictor.java
deleted file mode 100644
index a779d2b..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronPredictor.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.cron.noop;
-
-import java.util.Date;
-
-import com.twitter.aurora.scheduler.cron.CronPredictor;
-
-/**
- * A cron predictor that always suggests that the next run is Unix epoch time.
- *
- * This class exists as a short term hack to get around a license compatibility issue - Real
- * Implementation (TM) coming soon.
- */
-class NoopCronPredictor implements CronPredictor {
- @Override
- public Date predictNextRun(String schedule) {
- return new Date(0);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronScheduler.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronScheduler.java b/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronScheduler.java
deleted file mode 100644
index 2893a6a..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/cron/noop/NoopCronScheduler.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.cron.noop;
-
-import java.util.Collections;
-import java.util.Set;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.Sets;
-
-import com.twitter.aurora.scheduler.cron.CronException;
-import com.twitter.aurora.scheduler.cron.CronScheduler;
-
-/**
- * A cron scheduler that accepts cron jobs but never runs them. Useful if you want to hook up an
- * external triggering mechanism (e.g. a system cron job that calls the startCronJob RPC manually
- * on an interval).
- *
- * This class exists as a short term hack to get around a license compatibility issue - Real
- * Implementation (TM) coming soon.
- */
-class NoopCronScheduler implements CronScheduler {
- private static final Logger LOG = Logger.getLogger(NoopCronScheduler.class.getName());
-
- // Keep a list of schedules we've seen.
- private final Set<String> schedules = Collections.synchronizedSet(Sets.<String>newHashSet());
-
- @Override
- public String schedule(String schedule, Runnable task) {
- schedules.add(schedule);
-
- LOG.warning(String.format(
- "NO-OP cron scheduler is in use! %s with schedule %s WILL NOT be automatically triggered!",
- task,
- schedule));
-
- return schedule;
- }
-
- @Override
- public void deschedule(String key) throws IllegalStateException {
- schedules.remove(key);
- }
-
- @Override
- public Optional<String> getSchedule(String key) throws IllegalStateException {
- return schedules.contains(key)
- ? Optional.of(key)
- : Optional.<String>absent();
- }
-
- @Override
- public void start() throws IllegalStateException {
- LOG.warning("NO-OP cron scheduler is in use. Cron jobs submitted will not be triggered!");
- }
-
- @Override
- public void stop() throws CronException {
- // No-op.
- }
-
- @Override
- public boolean isValidSchedule(@Nullable String schedule) {
- // Accept everything.
- return schedule != null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java b/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java
deleted file mode 100644
index 6bfc909..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/cron/testing/AbstractCronIT.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.cron.testing;
-
-import java.util.concurrent.CountDownLatch;
-
-import org.junit.Test;
-
-import com.twitter.aurora.scheduler.cron.CronPredictor;
-import com.twitter.aurora.scheduler.cron.CronScheduler;
-import com.twitter.common.testing.easymock.EasyMockTest;
-
-import static org.junit.Assert.assertTrue;
-
-import static com.twitter.aurora.gen.test.testConstants.VALID_CRON_SCHEDULES;
-
-/**
- * Abstract test to verify conformance with the {@link CronScheduler} interface.
- */
-public abstract class AbstractCronIT extends EasyMockTest {
- /**
- * Child should return an instance of the {@link CronScheduler} test under test here.
- */
- protected abstract CronScheduler makeCronScheduler() throws Exception;
-
- /**
- * Child should configure expectations for a scheduler start.
- */
- protected abstract void expectStartCronScheduler();
-
- /**
- * Child should configure expectations for a scheduler stop.
- */
- protected abstract void expectStopCronScheduler();
-
- /**
- * Child should return an instance of the {@link CronPredictor} under test here.
- */
- protected abstract CronPredictor makeCronPredictor() throws Exception;
-
- @Test
- public void testCronSchedulerLifecycle() throws Exception {
- CronScheduler scheduler = makeCronScheduler();
-
- expectStartCronScheduler();
- expectStopCronScheduler();
-
- control.replay();
-
- scheduler.start();
- final CountDownLatch cronRan = new CountDownLatch(1);
- scheduler.schedule("* * * * *", new Runnable() {
- @Override public void run() {
- cronRan.countDown();
- }
- });
- cronRan.await();
- scheduler.stop();
- }
-
- @Test
- public void testCronPredictorAcceptsValidSchedules() throws Exception {
- control.replay();
-
- CronPredictor cronPredictor = makeCronPredictor();
- for (String schedule : VALID_CRON_SCHEDULES) {
- cronPredictor.predictNextRun(schedule);
- }
- }
-
- @Test
- public void testCronScheduleValidatorAcceptsValidSchedules() throws Exception {
- CronScheduler cron = makeCronScheduler();
-
- control.replay();
-
- for (String schedule : VALID_CRON_SCHEDULES) {
- assertTrue(String.format("Cron schedule %s should validate.", schedule),
- cron.isValidSchedule(schedule));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/events/NotifyingMethodInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/events/NotifyingMethodInterceptor.java b/src/main/java/com/twitter/aurora/scheduler/events/NotifyingMethodInterceptor.java
deleted file mode 100644
index 2656766..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/events/NotifyingMethodInterceptor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.events;
-
-import java.lang.reflect.Method;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Preconditions;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-
-import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.Event;
-import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
-import com.twitter.common.base.Closure;
-
-/**
- * A method interceptor that sends pubsub notifications before and/or after a method annotated
- * with {@link SendNotification}
- * is invoked.
- */
-class NotifyingMethodInterceptor implements MethodInterceptor {
- private static final Logger LOG = Logger.getLogger(NotifyingMethodInterceptor.class.getName());
-
- @Inject
- private Closure<PubsubEvent> eventSink;
-
- private void maybeFire(Event event) {
- if (event != Event.None) {
- eventSink.execute(event.getEvent());
- }
- }
-
- @Override
- public Object invoke(MethodInvocation invocation) throws Throwable {
- Preconditions.checkNotNull(eventSink, "Event sink has not yet been set.");
-
- Method method = invocation.getMethod();
- SendNotification sendNotification = method.getAnnotation(SendNotification.class);
- if (sendNotification == null) {
- LOG.warning("Interceptor should not match methods without @"
- + SendNotification.class.getSimpleName());
- return invocation.proceed();
- }
-
- maybeFire(sendNotification.before());
- Object result = invocation.proceed();
- maybeFire(sendNotification.after());
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/events/NotifyingSchedulingFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/events/NotifyingSchedulingFilter.java b/src/main/java/com/twitter/aurora/scheduler/events/NotifyingSchedulingFilter.java
deleted file mode 100644
index ffb952c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/events/NotifyingSchedulingFilter.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.events;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.Set;
-
-import javax.inject.Inject;
-
-import com.google.inject.BindingAnnotation;
-
-import com.twitter.aurora.scheduler.ResourceSlot;
-import com.twitter.aurora.scheduler.events.PubsubEvent.Vetoed;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.base.Closure;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A decorating scheduling filter that sends an event when a scheduling assignment is vetoed.
- */
-class NotifyingSchedulingFilter implements SchedulingFilter {
-
- /**
- * Binding annotation that the underlying {@link SchedulingFilter} must be bound with.
- */
- @BindingAnnotation
- @Target({FIELD, PARAMETER, METHOD}) @Retention(RUNTIME)
- public @interface NotifyDelegate { }
-
- private final SchedulingFilter delegate;
- private final Closure<PubsubEvent> eventSink;
-
- @Inject
- NotifyingSchedulingFilter(
- @NotifyDelegate SchedulingFilter delegate,
- Closure<PubsubEvent> eventSink) {
-
- this.delegate = checkNotNull(delegate);
- this.eventSink = checkNotNull(eventSink);
- }
-
- @Override
- public Set<Veto> filter(ResourceSlot offer, String slaveHost, ITaskConfig task, String taskId) {
- Set<Veto> vetoes = delegate.filter(offer, slaveHost, task, taskId);
- if (!vetoes.isEmpty()) {
- eventSink.execute(new Vetoed(taskId, vetoes));
- }
-
- return vetoes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/events/PubsubEvent.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/events/PubsubEvent.java b/src/main/java/com/twitter/aurora/scheduler/events/PubsubEvent.java
deleted file mode 100644
index 400d8b7..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/events/PubsubEvent.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.events;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.Set;
-
-import com.google.common.base.Objects;
-
-import com.twitter.aurora.gen.HostStatus;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Event notifications related to tasks.
- */
-public interface PubsubEvent {
-
- /**
- * Interface with no functionality, but identifies a class as supporting task pubsub events.
- */
- public interface EventSubscriber {
- }
-
- /**
- * Event sent when tasks were deleted.
- */
- public static class TasksDeleted implements PubsubEvent {
- private final Set<IScheduledTask> tasks;
-
- public TasksDeleted(Set<IScheduledTask> tasks) {
- this.tasks = checkNotNull(tasks);
- }
-
- public Set<IScheduledTask> getTasks() {
- return tasks;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof TasksDeleted)) {
- return false;
- }
-
- TasksDeleted other = (TasksDeleted) o;
- return Objects.equal(tasks, other.tasks);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(tasks);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("tasks", Tasks.ids(tasks))
- .toString();
- }
- }
-
- /**
- * Event sent when a task changed state.
- */
- public static class TaskStateChange implements PubsubEvent {
- private final IScheduledTask task;
- private final ScheduleStatus oldState;
-
- public TaskStateChange(IScheduledTask task, ScheduleStatus oldState) {
- this.task = checkNotNull(task);
- this.oldState = checkNotNull(oldState);
- }
-
- public String getTaskId() {
- return Tasks.id(task);
- }
-
- public ScheduleStatus getOldState() {
- return oldState;
- }
-
- public IScheduledTask getTask() {
- return task;
- }
-
- public ScheduleStatus getNewState() {
- return task.getStatus();
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof TaskStateChange)) {
- return false;
- }
-
- TaskStateChange other = (TaskStateChange) o;
- return Objects.equal(task, other.task)
- && Objects.equal(oldState, other.oldState);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(task, oldState);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("task", Tasks.id(task))
- .add("oldState", getOldState())
- .add("newState", getNewState())
- .toString();
- }
- }
-
- /**
- * Event sent when a host changed maintenance state.
- */
- public static class HostMaintenanceStateChange implements PubsubEvent {
- private final HostStatus status;
-
- public HostMaintenanceStateChange(HostStatus status) {
- this.status = checkNotNull(status);
- }
-
- public HostStatus getStatus() {
- return status;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof HostMaintenanceStateChange)) {
- return false;
- }
-
- HostMaintenanceStateChange other = (HostMaintenanceStateChange) o;
- return Objects.equal(status, other.status);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(status);
- }
- }
-
- /**
- * Event sent when a scheduling assignment was vetoed.
- */
- public static class Vetoed implements PubsubEvent {
- private final String taskId;
- private final Set<Veto> vetoes;
-
- public Vetoed(String taskId, Set<Veto> vetoes) {
- this.taskId = checkNotNull(taskId);
- this.vetoes = checkNotNull(vetoes);
- }
-
- public String getTaskId() {
- return taskId;
- }
-
- public Set<Veto> getVetoes() {
- return vetoes;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof Vetoed)) {
- return false;
- }
-
- Vetoed other = (Vetoed) o;
- return Objects.equal(taskId, other.taskId)
- && Objects.equal(vetoes, other.vetoes);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(taskId, vetoes);
- }
- }
-
- public static class TaskRescheduled implements PubsubEvent {
- private final String role;
- private final String job;
- private final int instance;
-
- public TaskRescheduled(String role, String job, int instance) {
- this.role = role;
- this.job = job;
- this.instance = instance;
- }
-
- public String getRole() {
- return role;
- }
-
- public String getJob() {
- return job;
- }
-
- public int getInstance() {
- return instance;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof TaskRescheduled)) {
- return false;
- }
-
- TaskRescheduled other = (TaskRescheduled) o;
- return Objects.equal(role, other.role)
- && Objects.equal(job, other.job)
- && Objects.equal(instance, other.instance);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(role, job, instance);
- }
- }
-
- public static class StorageStarted implements PubsubEvent {
- @Override
- public boolean equals(Object o) {
- return (o != null) && getClass().equals(o.getClass());
- }
-
- @Override
- public int hashCode() {
- return getClass().hashCode();
- }
- }
-
- public static class DriverRegistered implements PubsubEvent {
- @Override
- public boolean equals(Object o) {
- return (o != null) && getClass().equals(o.getClass());
- }
-
- @Override
- public int hashCode() {
- return getClass().hashCode();
- }
- }
-
- public static class DriverDisconnected implements PubsubEvent {
- @Override
- public boolean equals(Object o) {
- return (o != null) && getClass().equals(o.getClass());
- }
-
- @Override
- public int hashCode() {
- return getClass().hashCode();
- }
- }
-
- public static final class Interceptors {
- private Interceptors() {
- // Utility class.
- }
-
- public enum Event {
- None(null),
- StorageStarted(new StorageStarted()),
- DriverRegistered(new DriverRegistered()),
- DriverDisconnected(new DriverDisconnected());
-
- private final PubsubEvent event;
- private Event(PubsubEvent event) {
- this.event = event;
- }
-
- public PubsubEvent getEvent() {
- return event;
- }
- }
-
- /**
- * An annotation to place on methods of injected classes that which to fire events before
- * and/or after their invocation.
- */
- @Target(METHOD) @Retention(RUNTIME)
- public @interface SendNotification {
- /**
- * Event to fire prior to invocation.
- */
- Event before() default Event.None;
-
- /**
- * Event to fire after invocation.
- */
- Event after() default Event.None;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/events/PubsubEventModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/events/PubsubEventModule.java b/src/main/java/com/twitter/aurora/scheduler/events/PubsubEventModule.java
deleted file mode 100644
index a8b5633..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/events/PubsubEventModule.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.events;
-
-import java.util.Set;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.eventbus.DeadEvent;
-import com.google.common.eventbus.EventBus;
-import com.google.common.eventbus.Subscribe;
-import com.google.inject.AbstractModule;
-import com.google.inject.Binder;
-import com.google.inject.TypeLiteral;
-import com.google.inject.matcher.Matchers;
-import com.google.inject.multibindings.Multibinder;
-
-import org.aopalliance.intercept.MethodInterceptor;
-
-import com.twitter.aurora.scheduler.events.NotifyingSchedulingFilter.NotifyDelegate;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter;
-import com.twitter.common.application.modules.LifecycleModule;
-import com.twitter.common.base.Closure;
-import com.twitter.common.base.Command;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Binding module for plumbing event notifications.
- */
-public final class PubsubEventModule extends AbstractModule {
-
- private static final Logger LOG = Logger.getLogger(PubsubEventModule.class.getName());
-
- private PubsubEventModule() {
- // Must be constructed through factory.
- }
-
- @VisibleForTesting
- public static void installForTest(Binder binder) {
- binder.install(new PubsubEventModule());
- }
-
- @Override
- protected void configure() {
- final EventBus eventBus = new EventBus("TaskEvents");
- eventBus.register(new Object() {
- @Subscribe public void logDeadEvent(DeadEvent event) {
- LOG.warning("Captured dead event " + event.getEvent());
- }
- });
-
- bind(EventBus.class).toInstance(eventBus);
-
- Closure<PubsubEvent> eventPoster = new Closure<PubsubEvent>() {
- @Override public void execute(PubsubEvent event) {
- eventBus.post(event);
- }
- };
- bind(new TypeLiteral<Closure<PubsubEvent>>() { }).toInstance(eventPoster);
-
- // Ensure at least an empty binding is present.
- getSubscriberBinder(binder());
- LifecycleModule.bindStartupAction(binder(), RegisterSubscribers.class);
- bindNotifyingInterceptor(binder());
- }
-
- static class RegisterSubscribers implements Command {
- private final EventBus eventBus;
- private final Set<EventSubscriber> subscribers;
-
- @Inject
- RegisterSubscribers(EventBus eventBus, Set<EventSubscriber> subscribers) {
- this.eventBus = checkNotNull(eventBus);
- this.subscribers = checkNotNull(subscribers);
- }
-
- @Override
- public void execute() {
- for (EventSubscriber subscriber : subscribers) {
- eventBus.register(subscriber);
- }
- }
- }
-
- /**
- * Binds a task event module.
- *
- * @param binder Binder to bind against.
- * @param filterClass Delegate scheduling filter implementation class.
- */
- public static void bind(Binder binder, final Class<? extends SchedulingFilter> filterClass) {
- binder.bind(SchedulingFilter.class).annotatedWith(NotifyDelegate.class).to(filterClass);
- binder.bind(SchedulingFilter.class).to(NotifyingSchedulingFilter.class);
- binder.bind(NotifyingSchedulingFilter.class).in(Singleton.class);
- binder.install(new PubsubEventModule());
- }
-
- private static Multibinder<EventSubscriber> getSubscriberBinder(Binder binder) {
- return Multibinder.newSetBinder(binder, EventSubscriber.class);
- }
-
- /**
- * Binds a subscriber to receive task events.
- *
- * @param binder Binder to bind the subscriber with.
- * @param subscriber Subscriber implementation class to register for events.
- */
- public static void bindSubscriber(Binder binder, Class<? extends EventSubscriber> subscriber) {
- getSubscriberBinder(binder).addBinding().to(subscriber);
- }
-
- /**
- * Binds a method interceptor to all methods annotated with {@link SendNotification}.
- * <p>
- * The interceptor will send notifications before and/or after the wrapped method invocation.
- *
- * @param binder Guice binder.
- */
- @VisibleForTesting
- public static void bindNotifyingInterceptor(Binder binder) {
- MethodInterceptor interceptor = new NotifyingMethodInterceptor();
- binder.requestInjection(interceptor);
- binder.bindInterceptor(
- Matchers.any(),
- Matchers.annotatedWith(SendNotification.class),
- interceptor);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/filter/AttributeFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/filter/AttributeFilter.java b/src/main/java/com/twitter/aurora/scheduler/filter/AttributeFilter.java
deleted file mode 100644
index 66327fa..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/filter/AttributeFilter.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.filter;
-
-import java.util.Set;
-
-import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.AttributeLoader;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.IValueConstraint;
-
-/**
- * Utility class that matches attributes to constraints.
- */
-final class AttributeFilter {
-
- private static final Function<Attribute, Set<String>> GET_VALUES =
- new Function<Attribute, Set<String>>() {
- @Override public Set<String> apply(Attribute attribute) {
- return attribute.getValues();
- }
- };
-
- private AttributeFilter() {
- // Utility class.
- }
-
- /**
- * Tests whether a constraint is satisfied by attributes.
- *
- * @param attributes Host attributes.
- * @param constraint Constraint to match.
- * @return {@code true} if the attribute satisfies the constraint, {@code false} otherwise.
- */
- static boolean matches(Set<Attribute> attributes, IValueConstraint constraint) {
- Set<String> allAttributes =
- ImmutableSet.copyOf(Iterables.concat(Iterables.transform(attributes, GET_VALUES)));
- boolean match = Iterables.any(constraint.getValues(), Predicates.in(allAttributes));
- return constraint.isNegated() ^ match;
- }
-
- /**
- * Tests whether an attribute matches a limit constraint.
- *
- * @param attributes Attributes to match against.
- * @param jobKey Key of the job with the limited constraint.
- * @param limit Limit value.
- * @param activeTasks All active tasks in the system.
- * @param attributeFetcher Interface for fetching attributes for hosts in the system.
- * @return {@code true} if the limit constraint is satisfied, {@code false} otherwise.
- */
- static boolean matches(final Set<Attribute> attributes,
- final IJobKey jobKey,
- int limit,
- Iterable<IScheduledTask> activeTasks,
- final AttributeLoader attributeFetcher) {
-
- Predicate<IScheduledTask> sameJob =
- Predicates.compose(Predicates.equalTo(jobKey), Tasks.SCHEDULED_TO_JOB_KEY);
-
- Predicate<IScheduledTask> hasAttribute = new Predicate<IScheduledTask>() {
- @Override public boolean apply(IScheduledTask task) {
- Iterable<Attribute> hostAttributes =
- attributeFetcher.apply(task.getAssignedTask().getSlaveHost());
- return Iterables.any(hostAttributes, Predicates.in(attributes));
- }
- };
-
- return limit > Iterables.size(
- Iterables.filter(activeTasks, Predicates.and(sameJob, hasAttribute)));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/filter/ConstraintFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/filter/ConstraintFilter.java b/src/main/java/com/twitter/aurora/scheduler/filter/ConstraintFilter.java
deleted file mode 100644
index 2612cb8..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/filter/ConstraintFilter.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.filter;
-
-import java.util.Collection;
-import java.util.Set;
-import java.util.logging.Logger;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.scheduler.base.SchedulerException;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
-import com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.AttributeLoader;
-import com.twitter.aurora.scheduler.storage.entities.IConstraint;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConstraint;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Filter that determines whether a task's constraints are satisfied.
- */
-class ConstraintFilter implements Function<IConstraint, Optional<Veto>> {
-
- private static final Logger LOG = Logger.getLogger(ConstraintFilter.class.getName());
-
- private final IJobKey jobKey;
- private final Supplier<Collection<IScheduledTask>> activeTasksSupplier;
- private final AttributeLoader attributeLoader;
- private final Iterable<Attribute> hostAttributes;
-
- /**
- * Creates a new constraint filer for a given job.
- *
- * @param jobKey Key for the job.
- * @param activeTasksSupplier Supplier to fetch active tasks (if necessary).
- * @param attributeLoader Interface to fetch host attributes (if necessary).
- * @param hostAttributes The attributes of the host to test against.
- */
- ConstraintFilter(
- IJobKey jobKey,
- Supplier<Collection<IScheduledTask>> activeTasksSupplier,
- AttributeLoader attributeLoader,
- Iterable<Attribute> hostAttributes) {
-
- this.jobKey = checkNotNull(jobKey);
- this.activeTasksSupplier = checkNotNull(activeTasksSupplier);
- this.attributeLoader = checkNotNull(attributeLoader);
- this.hostAttributes = checkNotNull(hostAttributes);
- }
-
- @VisibleForTesting
- static Veto limitVeto(String limit) {
- return new Veto("Limit not satisfied: " + limit, Veto.MAX_SCORE);
- }
-
- @VisibleForTesting
- static Veto mismatchVeto(String constraint) {
- return Veto.constraintMismatch("Constraint not satisfied: " + constraint);
- }
-
- @VisibleForTesting
- static Veto maintenanceVeto(String reason) {
- return new Veto("Host " + reason + " for maintenance", Veto.MAX_SCORE);
- }
-
- @Override
- public Optional<Veto> apply(IConstraint constraint) {
- Set<Attribute> attributes =
- ImmutableSet.copyOf(Iterables.filter(hostAttributes, new NameFilter(constraint.getName())));
-
- ITaskConstraint taskConstraint = constraint.getConstraint();
- switch (taskConstraint.getSetField()) {
- case VALUE:
- boolean matches =
- AttributeFilter.matches(attributes, taskConstraint.getValue());
- return matches
- ? Optional.<Veto>absent()
- : Optional.of(mismatchVeto(constraint.getName()));
-
- case LIMIT:
- if (attributes.isEmpty()) {
- return Optional.of(mismatchVeto(constraint.getName()));
- }
-
- boolean satisfied = AttributeFilter.matches(
- attributes,
- jobKey,
- taskConstraint.getLimit().getLimit(),
- activeTasksSupplier.get(),
- attributeLoader);
- return satisfied
- ? Optional.<Veto>absent()
- : Optional.of(limitVeto(constraint.getName()));
-
- default:
- LOG.warning("Unrecognized constraint type: " + taskConstraint.getSetField());
- throw new SchedulerException("Failed to recognize the constraint type: "
- + taskConstraint.getSetField());
- }
- }
-
- /**
- * A filter to find attributes matching a name.
- */
- static class NameFilter implements Predicate<Attribute> {
- private final String attributeName;
-
- NameFilter(String attributeName) {
- this.attributeName = attributeName;
- }
-
- @Override public boolean apply(Attribute attribute) {
- return attributeName.equals(attribute.getName());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilter.java b/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilter.java
deleted file mode 100644
index 7e8fbdb..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilter.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.filter;
-
-import java.util.Set;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-
-import com.twitter.aurora.scheduler.ResourceSlot;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-
-/**
- * Determines whether a proposed scheduling assignment should be allowed.
- */
-public interface SchedulingFilter {
-
- /**
- * Reason for a proposed scheduling assignment to be filtered out.
- * A veto also contains a score, which is an opaque indicator as to how strong a veto is. This
- * is only intended to be used for relative ranking of vetoes for determining which veto against
- * a scheduling assignment is 'weakest'.
- */
- public static class Veto {
- public static final int MAX_SCORE = 1000;
-
- private final String reason;
- private final int score;
- private final boolean valueMismatch;
-
- private Veto(String reason, int score, boolean valueMismatch) {
- this.reason = reason;
- this.score = Math.min(MAX_SCORE, score);
- this.valueMismatch = valueMismatch;
- }
-
- @VisibleForTesting
- public Veto(String reason, int score) {
- this(reason, score, false);
- }
-
- /**
- * Creates a special veto that represents a mismatch between the server and task's configuration
- * for an attribute.
- *
- * @param reason Information about the value mismatch.
- * @return A constraint mismatch veto.
- */
- public static Veto constraintMismatch(String reason) {
- return new Veto(reason, MAX_SCORE, true);
- }
-
- public String getReason() {
- return reason;
- }
-
- public int getScore() {
- return score;
- }
-
- public boolean isConstraintMismatch() {
- return valueMismatch;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof Veto)) {
- return false;
- }
-
- Veto other = (Veto) o;
- return Objects.equal(reason, other.reason)
- && (score == other.score)
- && (valueMismatch == other.valueMismatch);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(reason, score, valueMismatch);
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("reason", reason)
- .add("score", score)
- .add("valueMismatch", valueMismatch)
- .toString();
- }
- }
-
- /**
- * Applies a task against the filter with the given resources, and on the host.
- *
- * @param offer Resources offered.
- * @param slaveHost Host that the resources are associated with.
- * @param task Task.
- * @param taskId Canonical ID of the task.
- * @return A set of vetoes indicating reasons the task cannot be scheduled. If the task may be
- * scheduled, the set will be empty.
- */
- Set<Veto> filter(ResourceSlot offer, String slaveHost, ITaskConfig task, String taskId);
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilterImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilterImpl.java b/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilterImpl.java
deleted file mode 100644
index 33272ce..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/filter/SchedulingFilterImpl.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.filter;
-
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.Set;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Supplier;
-import com.google.common.base.Suppliers;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.Sets;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.TaskConstraint;
-import com.twitter.aurora.scheduler.ResourceSlot;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
-import com.twitter.aurora.scheduler.state.MaintenanceController;
-import com.twitter.aurora.scheduler.storage.AttributeStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work.Quiet;
-import com.twitter.aurora.scheduler.storage.entities.IConstraint;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Data;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
-import static com.twitter.aurora.scheduler.configuration.ConfigurationManager.DEDICATED_ATTRIBUTE;
-import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.CPU;
-import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.DISK;
-import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.PORTS;
-import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.RAM;
-
-/**
- * Implementation of the scheduling filter that ensures resource requirements of tasks are
- * fulfilled, and that tasks are allowed to run on the given machine.
- *
- */
-public class SchedulingFilterImpl implements SchedulingFilter {
-
- @VisibleForTesting static final Veto DEDICATED_HOST_VETO =
- Veto.constraintMismatch("Host is dedicated");
-
- private static final Optional<Veto> NO_VETO = Optional.absent();
-
- private static final Set<MaintenanceMode> VETO_MODES = EnumSet.of(DRAINING, DRAINED);
-
- private final Storage storage;
- private final MaintenanceController maintenance;
-
- /**
- * Creates a new scheduling filter.
- *
- * @param storage Interface to accessing the task store.
- * @param maintenance Interface to accessing the maintenance controller
- */
- @Inject
- public SchedulingFilterImpl(Storage storage, MaintenanceController maintenance) {
- this.storage = checkNotNull(storage);
- this.maintenance = checkNotNull(maintenance);
- }
-
- /**
- * A function that fetches attributes associated with a given host.
- */
- public interface AttributeLoader extends Function<String, Iterable<Attribute>> { }
-
- /**
- * A function that may veto a task.
- */
- private interface FilterRule extends Function<ITaskConfig, Iterable<Veto>> { }
-
- /**
- * Convenience class for a rule that will only ever have a single veto.
- */
- private abstract static class SingleVetoRule implements FilterRule {
- @Override public final Iterable<Veto> apply(ITaskConfig task) {
- return doApply(task).asSet();
- }
-
- abstract Optional<Veto> doApply(ITaskConfig task);
- }
-
- // Scaling ranges to use for comparison of vetos. This has no real bearing besides trying to
- // determine if a veto along one resource vector is a 'stronger' veto than that of another vector.
- // The values below represent the maximum resources on a typical slave machine.
- @VisibleForTesting
- enum ResourceVector {
- CPU("CPU", 16),
- RAM("RAM", Amount.of(24, Data.GB).as(Data.MB)),
- DISK("disk", Amount.of(450, Data.GB).as(Data.MB)),
- PORTS("ports", 1000);
-
- private final String name;
- private final int range;
- @VisibleForTesting
- int getRange() {
- return range;
- }
-
- private ResourceVector(String name, int range) {
- this.name = name;
- this.range = range;
- }
-
- Optional<Veto> maybeVeto(double available, double requested) {
- double tooLarge = requested - available;
- if (tooLarge <= 0) {
- return NO_VETO;
- } else {
- return Optional.of(veto(tooLarge));
- }
- }
-
- private static int scale(double value, int range) {
- return Math.min(Veto.MAX_SCORE, (int) ((Veto.MAX_SCORE * value)) / range);
- }
-
- @VisibleForTesting
- Veto veto(double excess) {
- return new Veto("Insufficient " + name, scale(excess, range));
- }
- }
-
- private Iterable<FilterRule> rulesFromOffer(final ResourceSlot available) {
- return ImmutableList.<FilterRule>of(
- new SingleVetoRule() {
- @Override public Optional<Veto> doApply(ITaskConfig task) {
- return CPU.maybeVeto(
- available.getNumCpus(),
- ResourceSlot.from(task).getNumCpus());
- }
- },
- new SingleVetoRule() {
- @Override public Optional<Veto> doApply(ITaskConfig task) {
- return RAM.maybeVeto(
- available.getRam().as(Data.MB),
- ResourceSlot.from(task).getRam().as(Data.MB));
- }
- },
- new SingleVetoRule() {
- @Override public Optional<Veto> doApply(ITaskConfig task) {
- return DISK.maybeVeto(available.getDisk().as(Data.MB),
- ResourceSlot.from(task).getDisk().as(Data.MB));
- }
- },
- new SingleVetoRule() {
- @Override public Optional<Veto> doApply(ITaskConfig task) {
- return PORTS.maybeVeto(available.getNumPorts(),
- ResourceSlot.from(task).getNumPorts());
- }
- }
- );
- }
-
- private static boolean isValueConstraint(IConstraint constraint) {
- return constraint.getConstraint().getSetField() == TaskConstraint._Fields.VALUE;
- }
-
- private static final Ordering<IConstraint> VALUES_FIRST = Ordering.from(
- new Comparator<IConstraint>() {
- @Override public int compare(IConstraint a, IConstraint b) {
- if (a.getConstraint().getSetField() == b.getConstraint().getSetField()) {
- return 0;
- }
- return isValueConstraint(a) ? -1 : 1;
- }
- });
-
- private static final Iterable<ScheduleStatus> ACTIVE_NOT_PENDING_STATES =
- EnumSet.copyOf(Sets.difference(Tasks.ACTIVE_STATES, EnumSet.of(ScheduleStatus.PENDING)));
-
- private FilterRule getConstraintFilter(final String slaveHost) {
- return new FilterRule() {
- @Override public Iterable<Veto> apply(final ITaskConfig task) {
- if (!task.isSetConstraints()) {
- return ImmutableList.of();
- }
-
- // In the interest of performance, we perform a weakly consistent read here. The biggest
- // risk of this is that we might schedule against stale host attributes, or we might fail
- // to correctly satisfy a diversity constraint. Given that the likelihood is relatively low
- // for both of these, and the impact is also low, the weak consistency is acceptable.
- return storage.weaklyConsistentRead(new Quiet<Iterable<Veto>>() {
- @Override public Iterable<Veto> apply(final StoreProvider storeProvider) {
- AttributeLoader attributeLoader = new AttributeLoader() {
- @Override public Iterable<Attribute> apply(String host) {
- return AttributeStore.Util.attributesOrNone(storeProvider, host);
- }
- };
-
- Supplier<Collection<IScheduledTask>> activeTasksSupplier =
- Suppliers.memoize(new Supplier<Collection<IScheduledTask>>() {
- @Override public Collection<IScheduledTask> get() {
- return storeProvider.getTaskStore().fetchTasks(
- Query.jobScoped(Tasks.INFO_TO_JOB_KEY.apply(task))
- .byStatus(ACTIVE_NOT_PENDING_STATES));
- }
- });
-
- ConstraintFilter constraintFilter = new ConstraintFilter(
- Tasks.INFO_TO_JOB_KEY.apply(task),
- activeTasksSupplier,
- attributeLoader,
- attributeLoader.apply(slaveHost));
- ImmutableList.Builder<Veto> vetoes = ImmutableList.builder();
- for (IConstraint constraint : VALUES_FIRST.sortedCopy(task.getConstraints())) {
- Optional<Veto> veto = constraintFilter.apply(constraint);
- if (veto.isPresent()) {
- vetoes.add(veto.get());
- if (isValueConstraint(constraint)) {
- // Break when a value constraint mismatch is found to avoid other
- // potentially-expensive operations to satisfy other constraints.
- break;
- }
- }
- }
-
- return vetoes.build();
- }
- });
- }
- };
- }
-
- private Optional<Veto> getMaintenanceVeto(String slaveHost) {
- MaintenanceMode mode = maintenance.getMode(slaveHost);
- return VETO_MODES.contains(mode)
- ? Optional.of(ConstraintFilter.maintenanceVeto(mode.toString().toLowerCase()))
- : NO_VETO;
- }
-
- private Set<Veto> getResourceVetoes(ResourceSlot offer, ITaskConfig task) {
- ImmutableSet.Builder<Veto> builder = ImmutableSet.builder();
- for (FilterRule rule : rulesFromOffer(offer)) {
- builder.addAll(rule.apply(task));
- }
- return builder.build();
- }
-
- private boolean isDedicated(final String slaveHost) {
- Iterable<Attribute> slaveAttributes =
- storage.weaklyConsistentRead(new Quiet<Iterable<Attribute>>() {
- @Override public Iterable<Attribute> apply(final StoreProvider storeProvider) {
- return AttributeStore.Util.attributesOrNone(storeProvider, slaveHost);
- }
- });
-
- return Iterables.any(slaveAttributes, new ConstraintFilter.NameFilter(DEDICATED_ATTRIBUTE));
- }
-
- @Override
- public Set<Veto> filter(ResourceSlot offer, String slaveHost, ITaskConfig task, String taskId) {
- if (!ConfigurationManager.isDedicated(task) && isDedicated(slaveHost)) {
- return ImmutableSet.of(DEDICATED_HOST_VETO);
- }
- return ImmutableSet.<Veto>builder()
- .addAll(getConstraintFilter(slaveHost).apply(task))
- .addAll(getResourceVetoes(offer, task))
- .addAll(getMaintenanceVeto(slaveHost).asSet())
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/ClusterName.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/ClusterName.java b/src/main/java/com/twitter/aurora/scheduler/http/ClusterName.java
deleted file mode 100644
index 90c87a7..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/ClusterName.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import com.google.inject.BindingAnnotation;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Binding annotation for the cluster name.
- */
-@BindingAnnotation
-@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
-public @interface ClusterName { }
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/Cron.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Cron.java b/src/main/java/com/twitter/aurora/scheduler/http/Cron.java
deleted file mode 100644
index 8c97b5e..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/Cron.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.collect.ImmutableMap;
-
-import com.twitter.aurora.scheduler.state.CronJobManager;
-
-/**
- * HTTP interface to dump state of the internal cron scheduler.
- */
-@Path("/cron")
-public class Cron {
- private final CronJobManager cronManager;
-
- @Inject
- Cron(CronJobManager cronManager) {
- this.cronManager = cronManager;
- }
-
- /**
- * Dumps the state of the cron manager.
- *
- * @return An HTTP response containing the cron manager's state.
- */
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- public Response dumpContents() {
- Map<String, Object> response = ImmutableMap.<String, Object>builder()
- .put("scheduled", cronManager.getScheduledJobs())
- .put("pending", cronManager.getPendingRuns())
- .build();
-
- return Response.ok(response).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/DisplayUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/DisplayUtils.java b/src/main/java/com/twitter/aurora/scheduler/http/DisplayUtils.java
deleted file mode 100644
index b4a2762..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/DisplayUtils.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Ordering;
-
-import com.twitter.aurora.scheduler.MesosTaskFactory.MesosTaskFactoryImpl;
-import com.twitter.aurora.scheduler.http.SchedulerzHome.Role;
-import com.twitter.aurora.scheduler.http.SchedulerzRole.Job;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-
-/**
- * Utility class to hold common display helper functions.
- */
-public final class DisplayUtils {
-
- @CmdLine(name = "viz_job_url_prefix", help = "URL prefix for job container stats.")
- private static final Arg<String> VIZ_JOB_URL_PREFIX = Arg.create("");
-
- private DisplayUtils() {
- // Utility class.
- }
-
- static final Ordering<Role> ROLE_ORDERING = Ordering.natural().onResultOf(
- new Function<Role, String>() {
- @Override public String apply(Role role) {
- return role.getRole();
- }
- });
-
- static final Ordering<Job> JOB_ORDERING = Ordering.natural().onResultOf(
- new Function<Job, String>() {
- @Override public String apply(Job job) {
- return job.getName();
- }
- });
-
- static String getJobDashboardUrl(IJobKey jobKey) {
- return VIZ_JOB_URL_PREFIX.get() + MesosTaskFactoryImpl.getJobSourceName(jobKey);
- }
-}
[15/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/admin/mesos_maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/admin/mesos_maintenance.py b/src/main/python/twitter/aurora/admin/mesos_maintenance.py
deleted file mode 100644
index 4e14436..0000000
--- a/src/main/python/twitter/aurora/admin/mesos_maintenance.py
+++ /dev/null
@@ -1,113 +0,0 @@
-from collections import defaultdict
-import time
-
-from twitter.common import log
-from twitter.common.quantity import Amount, Time
-
-from twitter.aurora.client.api import AuroraClientAPI
-from twitter.aurora.client.base import check_and_log_response
-
-from gen.twitter.aurora.ttypes import Hosts, MaintenanceMode
-
-
-def group_by_host(hostname):
- return hostname
-
-
-class MesosMaintenance(object):
- """This class provides more methods to interact with the mesos cluster and perform
- maintenance.
- """
-
- DEFAULT_GROUPING = 'by_host'
- GROUPING_FUNCTIONS = {
- 'by_host': group_by_host,
- }
- START_MAINTENANCE_DELAY = Amount(30, Time.SECONDS)
-
- @classmethod
- def group_hosts(cls, hostnames, grouping_function=DEFAULT_GROUPING):
- try:
- grouping_function = cls.GROUPING_FUNCTIONS[grouping_function]
- except KeyError:
- raise ValueError('Unknown grouping function %s!' % grouping_function)
- groups = defaultdict(set)
- for hostname in hostnames:
- groups[grouping_function(hostname)].add(hostname)
- return groups
-
- @classmethod
- def iter_batches(cls, hostnames, batch_size, grouping_function=DEFAULT_GROUPING):
- if batch_size <= 0:
- raise ValueError('Batch size must be > 0!')
- groups = cls.group_hosts(hostnames, grouping_function)
- groups = sorted(groups.items(), key=lambda v: v[0])
- for k in range(0, len(groups), batch_size):
- yield Hosts(set.union(*(hostset for (key, hostset) in groups[k:k+batch_size])))
-
- def __init__(self, cluster, verbosity):
- self._client = AuroraClientAPI(cluster, verbosity == 'verbose')
-
- def _drain_hosts(self, drainable_hosts, clock=time):
- """This will actively turn down tasks running on hosts."""
- check_and_log_response(self._client.drain_hosts(drainable_hosts))
- not_ready_hosts = [hostname for hostname in drainable_hosts.hostNames]
- while not_ready_hosts:
- log.info("Sleeping for %s." % self.START_MAINTENANCE_DELAY)
- clock.sleep(self.START_MAINTENANCE_DELAY.as_(Time.SECONDS))
- resp = self._client.maintenance_status(Hosts(not_ready_hosts))
- #TODO(jsmith): Workaround until scheduler responds with unknown slaves in MESOS-3454
- if not resp.result.maintenanceStatusResult.statuses:
- not_ready_hosts = None
- for host_status in resp.result.maintenanceStatusResult.statuses:
- if host_status.mode != MaintenanceMode.DRAINED:
- log.warning('%s is currently in status %s' %
- (host_status.host, MaintenanceMode._VALUES_TO_NAMES[host_status.mode]))
- else:
- not_ready_hosts.remove(host_status.host)
-
- def _complete_maintenance(self, drained_hosts):
- """End the maintenance status for a give set of hosts."""
- check_and_log_response(self._client.end_maintenance(drained_hosts))
- resp = self._client.maintenance_status(drained_hosts)
- for host_status in resp.result.maintenanceStatusResult.statuses:
- if host_status.mode != MaintenanceMode.NONE:
- log.warning('%s is DRAINING or in DRAINED' % host_status.host)
-
- def _operate_on_hosts(self, drained_hosts, callback):
- """Perform a given operation on a list of hosts that are ready for maintenance."""
- for host in drained_hosts.hostNames:
- callback(host)
-
- def end_maintenance(self, hosts):
- """Pull a list of hosts out of maintenance mode."""
- self._complete_maintenance(Hosts(set(hosts)))
-
- def start_maintenance(self, hosts):
- """Put a list of hosts into maintenance mode, to de-prioritize scheduling."""
- check_and_log_response(self._client.start_maintenance(Hosts(set(hosts))))
-
- def perform_maintenance(self, hosts, batch_size=1, grouping_function=DEFAULT_GROUPING,
- callback=None):
- """The wrap a callback in between sending hosts into maintenance mode and back.
-
- Walk through the process of putting hosts into maintenance, draining them of tasks,
- performing an action on them once drained, then removing them from maintenance mode
- so tasks can schedule.
- """
- self._complete_maintenance(Hosts(set(hosts)))
- self.start_maintenance(hosts)
-
- for hosts in self.iter_batches(hosts, batch_size, grouping_function):
- self._drain_hosts(hosts)
- if callback:
- self._operate_on_hosts(hosts, callback)
- self._complete_maintenance(hosts)
-
- def check_status(self, hosts):
- resp = self._client.maintenance_status(Hosts(set(hosts)))
- check_and_log_response(resp)
- statuses = []
- for host_status in resp.result.maintenanceStatusResult.statuses:
- statuses.append((host_status.host, MaintenanceMode._VALUES_TO_NAMES[host_status.mode]))
- return statuses
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/BUILD b/src/main/python/twitter/aurora/client/BUILD
deleted file mode 100644
index 70eef2e..0000000
--- a/src/main/python/twitter/aurora/client/BUILD
+++ /dev/null
@@ -1,78 +0,0 @@
-import os
-
-# Create an alias for the previous target
-python_library(
- name = 'api',
- dependencies = [
- pants('src/main/python/twitter/aurora/client/api')
- ]
-)
-
-python_library(
- name = 'base',
- sources = ['base.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'binding_helper',
- sources = ['binding_helper.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- ]
-)
-
-python_library(
- name = 'config',
- sources = ['config.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- pants(':base'),
- pants(':binding_helper'),
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/config'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'factory',
- sources = ['factory.py'],
- dependencies = [
- pants(':base'),
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('src/main/python/twitter/aurora/client/hooks'),
- pants('src/main/python/twitter/aurora/common:cluster'),
- pants('src/main/python/twitter/aurora/common:clusters'),
- ]
-)
-
-python_library(
- name = 'options',
- sources = ['options.py'],
- dependencies = [
- pants('src/main/python/twitter/thermos/common:options'),
- pants('src/main/python/twitter/aurora/common:aurora_job_key'),
- ]
-)
-
-python_library(
- name = 'client-packaged',
- dependencies = [
- pants('src/main/python/twitter/aurora/common'),
- pants('src/main/python/twitter/aurora/config'),
- pants('src/main/python/twitter/thermos/common'),
- ],
- provides = setup_py(
- name = 'twitter.aurora.client',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- ).with_binaries(
- aurora_admin = pants('src/main/python/twitter/aurora/client/bin:aurora_admin'),
- aurora_client = pants('src/main/python/twitter/aurora/client/bin:aurora_client'),
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/__init__.py b/src/main/python/twitter/aurora/client/__init__.py
deleted file mode 100644
index b0d6433..0000000
--- a/src/main/python/twitter/aurora/client/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/BUILD b/src/main/python/twitter/aurora/client/api/BUILD
deleted file mode 100644
index 4935b8a..0000000
--- a/src/main/python/twitter/aurora/client/api/BUILD
+++ /dev/null
@@ -1,104 +0,0 @@
-python_library(
- name = 'api',
- sources = ['__init__.py'],
- dependencies = [
- pants(':restarter'),
- pants(':scheduler_client'),
- pants(':updater'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/common'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'command_runner',
- sources = ['command_runner.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- pants(':api'),
- pants('src/main/python/twitter/thermos/config:schema'),
- pants('src/main/python/twitter/aurora/common:cluster'),
- pants('src/main/python/twitter/aurora/config:schema'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'disambiguator',
- sources = ['disambiguator.py'],
- dependencies = [
- pants(':api'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/python/twitter/aurora/common'),
- ]
-)
-
-python_library(
- name = 'job_monitor',
- sources = ['job_monitor.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'restarter',
- sources = ['restarter.py'],
- dependencies = [
- pants(':instance_watcher'),
- pants(':updater_util'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'scheduler_client',
- sources = ['scheduler_client.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('aurora/twitterdeps/src/python/twitter/common/rpc/transports:tsslsocket'),
- pants('aurora/twitterdeps/src/python/twitter/common/zookeeper/serverset:kazoo_serverset'),
- pants('aurora/twitterdeps/src/python/twitter/common/zookeeper:kazoo_client'),
- pants('src/main/python/twitter/aurora/common/auth'),
- pants('src/main/python/twitter/aurora/common:cluster'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'instance_watcher',
- sources = ['instance_watcher.py', 'health_check.py'],
- dependencies = [
- pants(':scheduler_client'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/common:http_signaler'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'updater',
- sources = ['updater.py'],
- dependencies = [
- pants(':scheduler_client'),
- pants(':instance_watcher'),
- pants(':updater_util'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'updater_util',
- sources = ['updater_util.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/__init__.py b/src/main/python/twitter/aurora/client/api/__init__.py
deleted file mode 100644
index 60f4011..0000000
--- a/src/main/python/twitter/aurora/client/api/__init__.py
+++ /dev/null
@@ -1,190 +0,0 @@
-from twitter.common import log
-
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-from twitter.aurora.common.auth import make_session_key
-from twitter.aurora.common.cluster import Cluster
-
-from gen.twitter.aurora.constants import LIVE_STATES
-from gen.twitter.aurora.ttypes import (
- Response,
- Identity,
- Quota,
- ResponseCode,
- TaskQuery)
-
-from .restarter import Restarter
-from .scheduler_client import SchedulerProxy
-from .updater import Updater
-
-
-class AuroraClientAPI(object):
- """This class provides the API to talk to the twitter scheduler"""
-
- class Error(Exception): pass
- class TypeError(Error, TypeError): pass
- class ClusterMismatch(Error, ValueError): pass
-
- def __init__(self, cluster, verbose=False, session_key_factory=make_session_key):
- if not isinstance(cluster, Cluster):
- raise TypeError('AuroraClientAPI expects instance of Cluster for "cluster", got %s' %
- type(cluster))
- self._scheduler = SchedulerProxy(
- cluster, verbose=verbose, session_key_factory=session_key_factory)
- self._cluster = cluster
-
- @property
- def cluster(self):
- return self._cluster
-
- @property
- def scheduler(self):
- return self._scheduler
-
- def create_job(self, config, lock=None):
- log.info('Creating job %s' % config.name())
- log.debug('Full configuration: %s' % config.job())
- log.debug('Lock %s' % lock)
- return self._scheduler.createJob(config.job(), lock)
-
- def populate_job_config(self, config, validation=None):
- return self._scheduler.populateJobConfig(config.job(), validation)
-
- def start_cronjob(self, job_key):
- self._assert_valid_job_key(job_key)
-
- log.info("Starting cron job: %s" % job_key)
- return self._scheduler.startCronJob(job_key.to_thrift())
-
- def get_jobs(self, role):
- log.info("Retrieving jobs for role %s" % role)
- return self._scheduler.getJobs(role)
-
- def kill_job(self, job_key, instances=None, lock=None):
- log.info("Killing tasks for job: %s" % job_key)
- if not isinstance(job_key, AuroraJobKey):
- raise TypeError('Expected type of job_key %r to be %s but got %s instead'
- % (job_key, AuroraJobKey.__name__, job_key.__class__.__name__))
-
- # Leave query.owner.user unset so the query doesn't filter jobs only submitted by a particular
- # user.
- # TODO(wfarner): Refactor this when Identity is removed from TaskQuery.
- query = job_key.to_thrift_query()
- if instances is not None:
- log.info("Instances to be killed: %s" % instances)
- query.instanceIds = frozenset([int(s) for s in instances])
- return self._scheduler.killTasks(query, lock)
-
- def check_status(self, job_key):
- self._assert_valid_job_key(job_key)
-
- log.info("Checking status of %s" % job_key)
- return self.query(job_key.to_thrift_query())
-
- @classmethod
- def build_query(cls, role, job, instances=None, statuses=LIVE_STATES, env=None):
- return TaskQuery(owner=Identity(role=role),
- jobName=job,
- statuses=statuses,
- instanceIds=instances,
- environment=env)
-
- def query(self, query):
- return self._scheduler.getTasksStatus(query)
-
- def update_job(self, config, health_check_interval_seconds=3, instances=None):
- """Run a job update for a given config, for the specified instances. If
- instances is left unspecified, update all instances. Returns whether or not
- the update was successful."""
-
- log.info("Updating job: %s" % config.name())
- updater = Updater(config, health_check_interval_seconds, self._scheduler)
-
- return updater.update(instances)
-
- def cancel_update(self, job_key):
- """Cancel the update represented by job_key. Returns whether or not the cancellation was
- successful."""
- self._assert_valid_job_key(job_key)
-
- log.info("Canceling update on job %s" % job_key)
- resp = Updater.cancel_update(self._scheduler, job_key)
- if resp.responseCode != ResponseCode.OK:
- log.error('Error cancelling the update: %s' % resp.message)
- return resp
-
- def restart(self, job_key, instances, updater_config, health_check_interval_seconds):
- """Perform a rolling restart of the job. If instances is None or [], restart all instances. Returns
- the scheduler response for the last restarted batch of instances (which allows the client to
- show the job URL), or the status check response if no tasks were active.
- """
- self._assert_valid_job_key(job_key)
-
- return Restarter(job_key, updater_config, health_check_interval_seconds, self._scheduler
- ).restart(instances)
-
- def start_maintenance(self, hosts):
- log.info("Starting maintenance for: %s" % hosts.hostNames)
- return self._scheduler.startMaintenance(hosts)
-
- def drain_hosts(self, hosts):
- log.info("Draining tasks on: %s" % hosts.hostNames)
- return self._scheduler.drainHosts(hosts)
-
- def maintenance_status(self, hosts):
- log.info("Maintenance status for: %s" % hosts.hostNames)
- return self._scheduler.maintenanceStatus(hosts)
-
- def end_maintenance(self, hosts):
- log.info("Ending maintenance for: %s" % hosts.hostNames)
- return self._scheduler.endMaintenance(hosts)
-
- def get_quota(self, role):
- log.info("Getting quota for: %s" % role)
- return self._scheduler.getQuota(role)
-
- def set_quota(self, role, cpu, ram_mb, disk_mb):
- log.info("Setting quota for user:%s cpu:%f ram_mb:%d disk_mb: %d"
- % (role, cpu, ram_mb, disk_mb))
- return self._scheduler.setQuota(role, Quota(cpu, ram_mb, disk_mb))
-
- def force_task_state(self, task_id, status):
- log.info("Requesting that task %s transition to state %s" % (task_id, status))
- return self._scheduler.forceTaskState(task_id, status)
-
- def perform_backup(self):
- return self._scheduler.performBackup()
-
- def list_backups(self):
- return self._scheduler.listBackups()
-
- def stage_recovery(self, backup_id):
- return self._scheduler.stageRecovery(backup_id)
-
- def query_recovery(self, query):
- return self._scheduler.queryRecovery(query)
-
- def delete_recovery_tasks(self, query):
- return self._scheduler.deleteRecoveryTasks(query)
-
- def commit_recovery(self):
- return self._scheduler.commitRecovery()
-
- def unload_recovery(self):
- return self._scheduler.unloadRecovery()
-
- def get_job_updates(self):
- return self._scheduler.getJobUpdates()
-
- def snapshot(self):
- return self._scheduler.snapshot()
-
- def unsafe_rewrite_config(self, rewrite_request):
- return self._scheduler.rewriteConfigs(rewrite_request)
-
- def _assert_valid_job_key(self, job_key):
- if not isinstance(job_key, AuroraJobKey):
- raise self.TypeError('Invalid job_key %r: expected %s but got %s'
- % (job_key, AuroraJobKey.__name__, job_key.__class__.__name__))
- if job_key.cluster != self.cluster.name:
- raise self.ClusterMismatch('job %s does not belong to cluster %s' % (job_key,
- self.cluster.name))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/command_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/command_runner.py b/src/main/python/twitter/aurora/client/api/command_runner.py
deleted file mode 100644
index 0a8ad33..0000000
--- a/src/main/python/twitter/aurora/client/api/command_runner.py
+++ /dev/null
@@ -1,120 +0,0 @@
-from multiprocessing.pool import ThreadPool
-import posixpath
-import subprocess
-
-from twitter.common import log
-
-from twitter.aurora.client.api import AuroraClientAPI
-from twitter.aurora.config.schema.base import MesosContext
-from twitter.aurora.common.cluster import Cluster
-from twitter.thermos.config.schema import ThermosContext
-
-from gen.twitter.aurora.constants import LIVE_STATES
-from gen.twitter.aurora.ttypes import (
- Identity,
- ResponseCode,
- TaskQuery)
-
-from pystachio import Environment, Required, String
-
-
-class CommandRunnerTrait(Cluster.Trait):
- slave_root = Required(String)
- slave_run_directory = Required(String)
-
-
-class DistributedCommandRunner(object):
- @staticmethod
- def execute(args):
- hostname, role, command = args
- ssh_command = ['ssh', '-n', '-q', '%s@%s' % (role, hostname), command]
- po = subprocess.Popen(ssh_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- output = po.communicate()
- return '\n'.join('%s: %s' % (hostname, line) for line in output[0].splitlines())
-
- @classmethod
- def make_executor_path(cls, cluster, executor_name):
- parameters = cls.sandbox_args(cluster)
- parameters.update(executor_name=executor_name)
- return posixpath.join(
- '%(slave_root)s',
- 'slaves/*/frameworks/*/executors/%(executor_name)s/runs',
- '%(slave_run_directory)s'
- ) % parameters
-
- @classmethod
- def thermos_sandbox(cls, cluster, executor_sandbox=False):
- sandbox = cls.make_executor_path(cluster, 'thermos-{{thermos.task_id}}')
- return sandbox if executor_sandbox else posixpath.join(sandbox, 'sandbox')
-
- @classmethod
- def sandbox_args(cls, cluster):
- cluster = cluster.with_trait(CommandRunnerTrait)
- return {'slave_root': cluster.slave_root, 'slave_run_directory': cluster.slave_run_directory}
-
- @classmethod
- def substitute_thermos(cls, command, task, cluster, **kw):
- prefix_command = 'cd %s;' % cls.thermos_sandbox(cluster, **kw)
- thermos_namespace = ThermosContext(
- task_id=task.assignedTask.taskId,
- ports=task.assignedTask.assignedPorts)
- mesos_namespace = MesosContext(instance=task.assignedTask.instanceId)
- command = String(prefix_command + command) % Environment(
- thermos=thermos_namespace,
- mesos=mesos_namespace)
- return command.get()
-
- @classmethod
- def aurora_sandbox(cls, cluster, executor_sandbox=False):
- if executor_sandbox:
- return cls.make_executor_path(cluster, 'twitter')
- else:
- return '/var/run/nexus/%task_id%/sandbox'
-
- @classmethod
- def substitute_aurora(cls, command, task, cluster, **kw):
- command = ('cd %s;' % cls.aurora_sandbox(cluster, **kw)) + command
- command = command.replace('%shard_id%', str(task.assignedTask.instanceId))
- command = command.replace('%task_id%', task.assignedTask.taskId)
- for name, port in task.assignedTask.assignedPorts.items():
- command = command.replace('%port:' + name + '%', str(port))
- return command
-
- @classmethod
- def substitute(cls, command, task, cluster, **kw):
- if task.assignedTask.task.executorConfig:
- return cls.substitute_thermos(command, task, cluster, **kw)
- else:
- return cls.substitute_aurora(command, task, cluster, **kw)
-
- @classmethod
- def query_from(cls, role, env, job):
- return TaskQuery(statuses=LIVE_STATES, owner=Identity(role), jobName=job, environment=env)
-
- def __init__(self, cluster, role, env, jobs, ssh_user=None):
- self._cluster = cluster
- self._api = AuroraClientAPI(cluster=cluster)
- self._role = role
- self._env = env
- self._jobs = jobs
- self._ssh_user = ssh_user if ssh_user else self._role
-
- def resolve(self):
- for job in self._jobs:
- resp = self._api.query(self.query_from(self._role, self._env, job))
- if resp.responseCode != ResponseCode.OK:
- log.error('Failed to query job: %s' % job)
- continue
- for task in resp.result.scheduleStatusResult.tasks:
- yield task
-
- def process_arguments(self, command, **kw):
- for task in self.resolve():
- host = task.assignedTask.slaveHost
- role = task.assignedTask.task.owner.role
- yield (host, self._ssh_user, self.substitute(command, task, self._cluster, **kw))
-
- def run(self, command, parallelism=1, **kw):
- threadpool = ThreadPool(processes=parallelism)
- for result in threadpool.imap_unordered(self.execute, self.process_arguments(command, **kw)):
- print result
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/disambiguator.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/disambiguator.py b/src/main/python/twitter/aurora/client/api/disambiguator.py
deleted file mode 100644
index 4693574..0000000
--- a/src/main/python/twitter/aurora/client/api/disambiguator.py
+++ /dev/null
@@ -1,89 +0,0 @@
-from twitter.common import log
-
-from twitter.aurora.client.api import AuroraClientAPI
-from twitter.aurora.client.base import check_and_log_response, deprecation_warning, die
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-
-
-class LiveJobDisambiguator(object):
- """
- Disambiguates a job-specification into concrete AuroraJobKeys by querying the scheduler API.
- """
-
- def __init__(self, client, role, env, name):
- if not isinstance(client, AuroraClientAPI):
- raise TypeError("client must be a AuroraClientAPI")
- self._client = client
-
- if not role:
- raise ValueError("role is required")
- self._role = role
- if not name:
- raise ValueError("name is required")
- self._name = name
- self._env = env
-
- @property
- def ambiguous(self):
- return not all((self._role, self._env, self._name))
-
- def query_matches(self):
- resp = self._client.get_jobs(self._role)
- check_and_log_response(resp)
- return set(AuroraJobKey(self._client.cluster.name, j.key.role, j.key.environment, j.key.name)
- for j in resp.result.getJobsResult.configs if j.key.name == self._name)
-
- @classmethod
- def _disambiguate_or_die(cls, client, role, env, name):
- # Returns a single AuroraJobKey if one can be found given the args, potentially
- # querying the scheduler. Calls die() with an appropriate error message otherwise.
- try:
- disambiguator = cls(client, role, env, name)
- except ValueError as e:
- die(e)
-
- if not disambiguator.ambiguous:
- return AuroraJobKey(client.cluster.name, role, env, name)
-
- deprecation_warning("Job ambiguously specified - querying the scheduler to disambiguate")
- matches = disambiguator.query_matches()
- if len(matches) == 1:
- (match,) = matches
- log.info("Found job %s" % match)
- return match
- elif len(matches) == 0:
- die("No jobs found")
- else:
- die("Multiple jobs match (%s) - disambiguate by using the CLUSTER/ROLE/ENV/NAME form"
- % ",".join(str(m) for m in matches))
-
- @classmethod
- def disambiguate_args_or_die(cls, args, options, client_factory=AuroraClientAPI):
- """
- Returns a (AuroraClientAPI, AuroraJobKey, AuroraConfigFile:str) tuple
- if one can be found given the args, potentially querying the scheduler with the returned client.
- Calls die() with an appropriate error message otherwise.
-
- Arguments:
- args: args from app command invocation.
- options: options from app command invocation. must have env and cluster attributes.
- client_factory: a callable (cluster) -> AuroraClientAPI.
- """
- if not len(args) > 0:
- die('job path is required')
- try:
- job_key = AuroraJobKey.from_path(args[0])
- client = client_factory(job_key.cluster)
- config_file = args[1] if len(args) > 1 else None # the config for hooks
- return client, job_key, config_file
- except AuroraJobKey.Error:
- log.warning("Failed to parse job path, falling back to compatibility mode")
- role = args[0] if len(args) > 0 else None
- name = args[1] if len(args) > 1 else None
- env = None
- config_file = None # deprecated form does not support hooks functionality
- cluster = options.cluster
- if not cluster:
- die('cluster is required')
- client = client_factory(cluster)
- return client, cls._disambiguate_or_die(client, role, env, name), config_file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/health_check.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/health_check.py b/src/main/python/twitter/aurora/client/api/health_check.py
deleted file mode 100644
index b75ee6e..0000000
--- a/src/main/python/twitter/aurora/client/api/health_check.py
+++ /dev/null
@@ -1,123 +0,0 @@
-from abc import abstractmethod
-
-from twitter.common import log
-from twitter.common.lang import Interface
-
-from twitter.aurora.common.http_signaler import HttpSignaler
-
-from gen.twitter.aurora.ttypes import ScheduleStatus
-
-
-class HealthCheck(Interface):
- @abstractmethod
- def health(self, task):
- """Checks health of the task and returns a (healthy, retriable) pair."""
-
-
-class HealthStatus(object):
- @classmethod
- def alive(cls):
- return cls(True).health()
-
- @classmethod
- def dead(cls):
- return cls(False).health()
-
- def __init__(self, retry, health):
- self._retry = retry
- self._health = health
-
- def health(self):
- return (self._health, self._retry)
-
-
-class NotRetriable(HealthStatus):
- def __init__(self, health):
- super(NotRetriable, self).__init__(False, health)
-
-
-class Retriable(HealthStatus):
- def __init__(self, health):
- super(Retriable, self).__init__(True, health)
-
-
-class StatusHealthCheck(HealthCheck):
- """Verifies the health of a task based on the task status. A task is healthy iff,
- 1. A task is in state RUNNING
- 2. A task that satisfies (1) and is already known has the same task id.
- """
- def __init__(self):
- self._task_ids = {}
-
- def health(self, task):
- task_id = task.assignedTask.taskId
- instance_id = task.assignedTask.instanceId
- status = task.status
-
- if status == ScheduleStatus.RUNNING:
- if instance_id in self._task_ids:
- return Retriable.alive() if task_id == self._task_ids.get(instance_id) else NotRetriable.dead()
- else:
- log.info('Detected RUNNING instance %s' % instance_id)
- self._task_ids[instance_id] = task_id
- return Retriable.alive()
- else:
- return Retriable.dead()
-
-
-class HttpHealthCheck(HealthCheck):
- """Verifies the health of a task based on http health checks. A new http signaler is created for a
- task iff,
- 1. The instance id of the task is unknown.
- 2. The instance id is known but the (host, port) is different for the task.
- """
- def __init__(self, http_signaler_factory=HttpSignaler):
- self._http_signalers = {}
- self._http_signaler_factory = http_signaler_factory
-
- def health(self, task):
- assigned_task = task.assignedTask
- instance_id = assigned_task.instanceId
- host_port = (assigned_task.slaveHost, assigned_task.assignedPorts['health'])
- http_signaler = None
- if instance_id in self._http_signalers:
- checker_host_port, signaler = self._http_signalers.get(instance_id)
- # Only reuse the health checker if it is for the same destination.
- if checker_host_port == host_port:
- http_signaler = signaler
- if not http_signaler:
- http_signaler = self._http_signaler_factory(host_port[1], host_port[0])
- self._http_signalers[instance_id] = (host_port, http_signaler)
- return Retriable.alive() if http_signaler.health()[0] else Retriable.dead()
-
-
-class ChainedHealthCheck(HealthCheck):
- """Delegates health checks to configured health checkers."""
- def __init__(self, *health_checkers):
- self._health_checkers = health_checkers
-
- def health(self, task):
- for checker in self._health_checkers:
- healthy, retriable = checker.health(task)
- if not healthy:
- return (healthy, retriable)
- return Retriable.alive()
-
-
-class InstanceWatcherHealthCheck(HealthCheck):
- """Makes the decision: if a task has health port, then use Status+HTTP, else use only status.
- Caveat: Only works if either ALL tasks have a health port or none of them have a health port.
- """
- # TODO(atollenaere) Refactor the code to use the executor StatusChecker/HealthChecker instead
-
- def __init__(self, http_signaler_factory=HttpSignaler):
- self._has_health_port = False
- self._health_checker = StatusHealthCheck()
- self._http_signaler_factory = http_signaler_factory
-
- def health(self, task):
- if not self._has_health_port and 'health' in task.assignedTask.assignedPorts:
- log.debug('Health port detected, enabling HTTP checks')
- self._health_checker = ChainedHealthCheck(self._health_checker, HttpHealthCheck(self._http_signaler_factory))
- self._has_health_port = True
- return self._health_checker.health(task)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/instance_watcher.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/instance_watcher.py b/src/main/python/twitter/aurora/client/api/instance_watcher.py
deleted file mode 100644
index 5c69487..0000000
--- a/src/main/python/twitter/aurora/client/api/instance_watcher.py
+++ /dev/null
@@ -1,134 +0,0 @@
-import time
-
-from twitter.common import log
-from .health_check import InstanceWatcherHealthCheck
-
-from gen.twitter.aurora.ttypes import (
- Identity,
- ResponseCode,
- ScheduleStatus,
- TaskQuery,
-)
-
-class Instance(object):
- def __init__(self, birthday=None, finished=False):
- self.birthday = birthday
- self.finished = finished
- self.healthy = False
-
- def set_healthy(self, value):
- self.healthy = value
- self.finished = True
-
- def __str__(self):
- return ('[birthday=%s, healthy=%s, finished=%s]' % (self.birthday, self.healthy, self.finished))
-
-
-class InstanceWatcher(object):
- def __init__(self,
- scheduler,
- job_key,
- restart_threshold,
- watch_secs,
- health_check_interval_seconds,
- clock=time):
-
- self._scheduler = scheduler
- self._job_key = job_key
- self._restart_threshold = restart_threshold
- self._watch_secs = watch_secs
- self._health_check_interval_seconds = health_check_interval_seconds
- self._clock = clock
-
- def watch(self, instance_ids, health_check=None):
- """Watches a set of instances and detects failures based on a delegated health check.
-
- Arguments:
- instance_ids -- set of instances to watch.
-
- Returns a set of instances that are considered failed.
- """
- log.info('Watching instances: %s' % instance_ids)
- instance_ids = set(instance_ids)
- health_check = health_check or InstanceWatcherHealthCheck()
- now = self._clock.time()
- expected_healthy_by = now + self._restart_threshold
- max_time = now + self._restart_threshold + self._watch_secs
-
- instance_states = {}
-
- def finished_instances():
- return dict((s_id, s) for s_id, s in instance_states.items() if s.finished)
-
- def set_instance_healthy(instance_id, now):
- if instance_id not in instance_states:
- instance_states[instance_id] = Instance(now)
- instance = instance_states.get(instance_id)
- if now > (instance.birthday + self._watch_secs):
- log.info('Instance %s has been up and healthy for at least %d seconds' % (
- instance_id, self._watch_secs))
- instance.set_healthy(True)
-
- def maybe_set_instance_unhealthy(instance_id, retriable):
- # An instance that was previously healthy and currently unhealthy has failed.
- if instance_id in instance_states:
- log.info('Instance %s is unhealthy' % instance_id)
- instance_states[instance_id].set_healthy(False)
- # If the restart threshold has expired or if the instance cannot be retried it is unhealthy.
- elif now > expected_healthy_by or not retriable:
- log.info('Instance %s was not reported healthy within %d seconds' % (
- instance_id, self._restart_threshold))
- instance_states[instance_id] = Instance(finished=True)
-
- while True:
- running_tasks = self._get_tasks_by_instance_id(instance_ids)
- now = self._clock.time()
- tasks_by_instance = dict((task.assignedTask.instanceId, task) for task in running_tasks)
- for instance_id in instance_ids:
- if instance_id not in finished_instances():
- running_task = tasks_by_instance.get(instance_id)
- if running_task is not None:
- task_healthy, retriable = health_check.health(running_task)
- if task_healthy:
- set_instance_healthy(instance_id, now)
- else:
- maybe_set_instance_unhealthy(instance_id, retriable)
- else:
- # Set retriable=True since an instance should be retried if it has not been healthy.
- maybe_set_instance_unhealthy(instance_id, retriable=True)
-
- log.debug('Instances health: %s' % ['%s: %s' % val for val in instance_states.items()])
-
- # Return if all tasks are finished.
- if set(finished_instances().keys()) == instance_ids:
- return set([s_id for s_id, s in instance_states.items() if not s.healthy])
-
- # Return if time is up.
- if now > max_time:
- return set([s_id for s_id in instance_ids if s_id not in instance_states
- or not instance_states[s_id].healthy])
-
- self._clock.sleep(self._health_check_interval_seconds)
-
- def _get_tasks_by_instance_id(self, instance_ids):
- log.debug('Querying instance statuses.')
- query = TaskQuery()
- query.owner = Identity(role=self._job_key.role)
- query.environment = self._job_key.environment
- query.jobName = self._job_key.name
- query.statuses = set([ScheduleStatus.RUNNING])
-
- query.instanceIds = instance_ids
- try:
- resp = self._scheduler.getTasksStatus(query)
- except IOError as e:
- log.error('IO Exception during scheduler call: %s' % e)
- return []
-
- tasks = []
- if resp.responseCode == ResponseCode.OK:
- tasks = resp.result.scheduleStatusResult.tasks
-
- log.debug('Response from scheduler: %s (message: %s)'
- % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
- return tasks
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/job_monitor.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/job_monitor.py b/src/main/python/twitter/aurora/client/api/job_monitor.py
deleted file mode 100644
index 45edd1a..0000000
--- a/src/main/python/twitter/aurora/client/api/job_monitor.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import time
-
-from twitter.common.quantity import Amount, Time
-
-from gen.twitter.aurora.constants import (
- LIVE_STATES,
- TERMINAL_STATES
-)
-from gen.twitter.aurora.ttypes import (
- Identity,
- TaskQuery
-)
-
-from thrift.transport import TTransport
-
-
-class JobMonitor(object):
- MIN_POLL_INTERVAL = Amount(10, Time.SECONDS)
- MAX_POLL_INTERVAL = Amount(2, Time.MINUTES)
-
- @classmethod
- def running_or_finished(cls, status):
- return status in (LIVE_STATES | TERMINAL_STATES)
-
- @classmethod
- def terminal(cls, status):
- return status in TERMINAL_STATES
-
- # TODO(ksweeney): Make this use the AuroraJobKey
- def __init__(self, client, role, env, jobname):
- self._client = client
- self._query = TaskQuery(owner=Identity(role=role), environment=env, jobName=jobname)
- self._initial_tasks = set()
- self._initial_tasks = set(task.assignedTask.taskId for task in self.iter_query())
-
- def iter_query(self):
- try:
- res = self._client.scheduler.getTasksStatus(self._query)
- except TTransport.TTransportException as e:
- print('Failed to query slaves from scheduler: %s' % e)
- return
- if res is None or res.result is None:
- return
- for task in res.result.scheduleStatusResult.tasks:
- if task.assignedTask.taskId not in self._initial_tasks:
- yield task
-
- def states(self):
- states = {}
- for task in self.iter_query():
- status, instance_id = task.status, task.assignedTask.instanceId
- first_timestamp = task.taskEvents[0].timestamp
- if instance_id not in states or first_timestamp > states[instance_id][0]:
- states[instance_id] = (first_timestamp, status)
- return dict((instance_id, status[1]) for (instance_id, status) in states.items())
-
- def wait_until(self, predicate):
- """Given a predicate (from ScheduleStatus => Boolean), return once all tasks
- return true for that predicate."""
- poll_interval = self.MIN_POLL_INTERVAL
- while not all(predicate(state) for state in self.states().values()):
- time.sleep(poll_interval.as_(Time.SECONDS))
- poll_interval = min(self.MAX_POLL_INTERVAL, 2 * poll_interval)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/restarter.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/restarter.py b/src/main/python/twitter/aurora/client/api/restarter.py
deleted file mode 100644
index bf196e1..0000000
--- a/src/main/python/twitter/aurora/client/api/restarter.py
+++ /dev/null
@@ -1,73 +0,0 @@
-from twitter.common import log
-
-from gen.twitter.aurora.constants import ACTIVE_STATES
-from gen.twitter.aurora.ttypes import ResponseCode
-
-from .instance_watcher import InstanceWatcher
-from .updater_util import FailureThreshold
-
-
-class Restarter(object):
- def __init__(self,
- job_key,
- update_config,
- health_check_interval_seconds,
- scheduler,
- instance_watcher=None,
- lock=None):
- self._job_key = job_key
- self._update_config = update_config
- self.health_check_interval_seconds = health_check_interval_seconds
- self._scheduler = scheduler
- self._lock = lock
- self._instance_watcher = instance_watcher or InstanceWatcher(
- scheduler,
- job_key.to_thrift(),
- update_config.restart_threshold,
- update_config.watch_secs,
- health_check_interval_seconds)
-
- def restart(self, instances):
- failure_threshold = FailureThreshold(
- self._update_config.max_per_instance_failures,
- self._update_config.max_total_failures)
-
- if not instances:
- query = self._job_key.to_thrift_query()
- query.statuses = ACTIVE_STATES
- status = self._scheduler.getTasksStatus(query)
-
- if status.responseCode != ResponseCode.OK:
- return status
-
- tasks = status.result.scheduleStatusResult.tasks
-
- instances = sorted(task.assignedTask.instanceId for task in tasks)
- if not instances:
- log.info("No instances specified, and no active instances found in job %s" % self._job_key)
- log.info("Nothing to do.")
- return status
-
- log.info("Performing rolling restart of job %s (instances: %s)" % (self._job_key, instances))
-
- while instances and not failure_threshold.is_failed_update():
- batch = instances[:self._update_config.batch_size]
- instances = instances[self._update_config.batch_size:]
-
- log.info("Restarting instances: %s", batch)
-
- resp = self._scheduler.restartShards(self._job_key.to_thrift(), batch, self._lock)
- if resp.responseCode != ResponseCode.OK:
- log.error('Error restarting instances: %s', resp.message)
- return resp
-
- failed_instances = self._instance_watcher.watch(batch)
- instances += failed_instances
- failure_threshold.update_failure_counts(failed_instances)
-
- if failure_threshold.is_failed_update():
- log.info("Restart failures threshold reached. Aborting")
- else:
- log.info("All instances were restarted successfully")
-
- return resp
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/scheduler_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/scheduler_client.py b/src/main/python/twitter/aurora/client/api/scheduler_client.py
deleted file mode 100644
index ffec604..0000000
--- a/src/main/python/twitter/aurora/client/api/scheduler_client.py
+++ /dev/null
@@ -1,257 +0,0 @@
-import functools
-import threading
-import time
-
-from twitter.aurora.common.auth import make_session_key, SessionKeyError
-from twitter.aurora.common.cluster import Cluster
-from twitter.common import log
-from twitter.common.quantity import Amount, Time
-from twitter.common.rpc.transports.tsslsocket import DelayedHandshakeTSSLSocket
-from twitter.common.zookeeper.kazoo_client import TwitterKazooClient
-from twitter.common.zookeeper.serverset import ServerSet
-
-from gen.twitter.aurora import AuroraAdmin
-from gen.twitter.aurora.constants import CURRENT_API_VERSION
-
-from thrift.protocol import TBinaryProtocol
-from thrift.transport import TSocket, TTransport
-from pystachio import Boolean, Default, Integer, String
-
-
-class SchedulerClientTrait(Cluster.Trait):
- zk = String
- zk_port = Default(Integer, 2181)
- scheduler_zk_path = String
- scheduler_uri = String
- proxy_url = String
- auth_mechanism = Default(String, 'UNAUTHENTICATED')
- use_thrift_ssl = Default(Boolean, False)
-
-
-class SchedulerClient(object):
- THRIFT_RETRIES = 5
- RETRY_TIMEOUT = Amount(1, Time.SECONDS)
-
- class CouldNotConnect(Exception): pass
-
- # TODO(wickman) Refactor per MESOS-3005 into two separate classes with separate traits:
- # ZookeeperClientTrait
- # DirectClientTrait
- @classmethod
- def get(cls, cluster, **kwargs):
- if not isinstance(cluster, Cluster):
- raise TypeError('"cluster" must be an instance of Cluster, got %s' % type(cluster))
- cluster = cluster.with_trait(SchedulerClientTrait)
- if cluster.zk:
- return ZookeeperSchedulerClient(
- cluster, port=cluster.zk_port, ssl=cluster.use_thrift_ssl, **kwargs)
- elif cluster.scheduler_uri:
- try:
- host, port = cluster.scheduler_uri.split(':', 2)
- port = int(port)
- except ValueError:
- raise ValueError('Malformed Cluster scheduler_uri: %s' % cluster.scheduler_uri)
- return DirectSchedulerClient(host, port, ssl=cluster.use_thrift_ssl)
- else:
- raise ValueError('"cluster" does not specify zk or scheduler_uri')
-
- def __init__(self, verbose=False, ssl=False):
- self._client = None
- self._verbose = verbose
- self._ssl = ssl
-
- def get_thrift_client(self):
- if self._client is None:
- self._client = self._connect()
- return self._client
-
- # per-class implementation -- mostly meant to set up a valid host/port
- # pair and then delegate the opening to SchedulerClient._connect_scheduler
- def _connect(self):
- return None
-
- @staticmethod
- def _connect_scheduler(host, port, with_ssl=False):
- if with_ssl:
- socket = DelayedHandshakeTSSLSocket(host, port, delay_handshake=True, validate=False)
- else:
- socket = TSocket.TSocket(host, port)
- transport = TTransport.TBufferedTransport(socket)
- protocol = TBinaryProtocol.TBinaryProtocol(transport)
- schedulerClient = AuroraAdmin.Client(protocol)
- for _ in range(SchedulerClient.THRIFT_RETRIES):
- try:
- transport.open()
- return schedulerClient
- except TTransport.TTransportException:
- time.sleep(SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS))
- continue
- except Exception as e:
- # Monkey-patched proxies, like socks, can generate a proxy error here.
- # without adding a dependency, we can't catch those in a more specific way.
- raise SchedulerClient.CouldNotConnect('Connection to scheduler failed: %s' % e)
- raise SchedulerClient.CouldNotConnect('Could not connect to %s:%s' % (host, port))
-
-
-class ZookeeperSchedulerClient(SchedulerClient):
- SERVERSET_TIMEOUT = Amount(10, Time.SECONDS)
-
- @classmethod
- def get_scheduler_serverset(cls, cluster, port=2181, verbose=False, **kw):
- if cluster.zk is None:
- raise ValueError('Cluster has no associated zookeeper ensemble!')
- if cluster.scheduler_zk_path is None:
- raise ValueError('Cluster has no defined scheduler path, must specify scheduler_zk_path '
- 'in your cluster config!')
- zk = TwitterKazooClient.make(str('%s:%s' % (cluster.zk, port)), verbose=verbose)
- return zk, ServerSet(zk, cluster.scheduler_zk_path, **kw)
-
- def __init__(self, cluster, port=2181, ssl=False, verbose=False):
- SchedulerClient.__init__(self, verbose=verbose, ssl=ssl)
- self._cluster = cluster
- self._zkport = port
- self._endpoint = None
-
- def _connect(self):
- joined = threading.Event()
- def on_join(elements):
- joined.set()
- zk, serverset = self.get_scheduler_serverset(self._cluster, verbose=self._verbose,
- port=self._zkport, on_join=on_join)
- joined.wait(timeout=self.SERVERSET_TIMEOUT.as_(Time.SECONDS))
- serverset_endpoints = list(serverset)
- if len(serverset_endpoints) == 0:
- raise self.CouldNotConnect('No schedulers detected in %s!' % self._cluster.name)
- instance = serverset_endpoints[0]
- self._endpoint = instance.service_endpoint
- self._http = instance.additional_endpoints.get('http')
- zk.stop()
- return self._connect_scheduler(self._endpoint.host, self._endpoint.port, self._ssl)
-
- @property
- def url(self):
- proxy_url = self._cluster.proxy_url
- if proxy_url:
- return proxy_url
- if self._http:
- return 'http://%s:%s' % (self._http.host, self._http.port)
-
-
-class DirectSchedulerClient(SchedulerClient):
- def __init__(self, host, port, ssl=False):
- SchedulerClient.__init__(self, verbose=True, ssl=ssl)
- self._host = host
- self._port = port
-
- def _connect(self):
- return self._connect_scheduler(self._host, self._port, with_ssl=self._ssl)
-
- @property
- def url(self):
- # TODO(wickman) This is broken -- make this tunable in MESOS-3005
- return 'http://%s:8081' % self._host
-
-
-class SchedulerProxy(object):
- """
- This class is responsible for creating a reliable thrift client to the
- twitter scheduler. Basically all the dirty work needed by the
- AuroraClientAPI.
- """
- CONNECT_MAXIMUM_WAIT = Amount(1, Time.MINUTES)
- RPC_RETRY_INTERVAL = Amount(5, Time.SECONDS)
- RPC_MAXIMUM_WAIT = Amount(10, Time.MINUTES)
- UNAUTHENTICATED_RPCS = frozenset([
- 'populateJobConfig',
- 'getTasksStatus',
- 'getJobs',
- 'getQuota',
- 'getVersion',
- ])
-
- class Error(Exception): pass
- class TimeoutError(Error): pass
- class AuthenticationError(Error): pass
- class APIVersionError(Error): pass
-
- def __init__(self, cluster, verbose=False, session_key_factory=make_session_key):
- """A callable session_key_factory should be provided for authentication"""
- self.cluster = cluster
- # TODO(Sathya): Make this a part of cluster trait when authentication is pushed to the transport
- # layer.
- self._session_key_factory = session_key_factory
- self._client = self._scheduler = None
- self.verbose = verbose
-
- def with_scheduler(method):
- """Decorator magic to make sure a connection is made to the scheduler"""
- def _wrapper(self, *args, **kwargs):
- if not self._scheduler:
- self._construct_scheduler()
- return method(self, *args, **kwargs)
- return _wrapper
-
- def invalidate(self):
- self._client = self._scheduler = None
-
- @with_scheduler
- def client(self):
- return self._client
-
- @with_scheduler
- def scheduler(self):
- return self._scheduler
-
- def session_key(self):
- try:
- return self._session_key_factory(self.cluster.auth_mechanism)
- except SessionKeyError as e:
- raise self.AuthenticationError('Unable to create session key %s' % e)
-
- def _construct_scheduler(self):
- """
- Populates:
- self._scheduler
- self._client
- """
- self._scheduler = SchedulerClient.get(self.cluster, verbose=self.verbose)
- assert self._scheduler, "Could not find scheduler (cluster = %s)" % self.cluster.name
- start = time.time()
- while (time.time() - start) < self.CONNECT_MAXIMUM_WAIT.as_(Time.SECONDS):
- try:
- self._client = self._scheduler.get_thrift_client()
- break
- except SchedulerClient.CouldNotConnect as e:
- log.warning('Could not connect to scheduler: %s' % e)
- if not self._client:
- raise self.TimeoutError('Timed out trying to connect to scheduler at %s' % self.cluster.name)
-
- server_version = self._client.getVersion().result.getVersionResult
- if server_version != CURRENT_API_VERSION:
- raise self.APIVersionError("Client Version: %s, Server Version: %s" %
- (CURRENT_API_VERSION, server_version))
-
- def __getattr__(self, method_name):
- # If the method does not exist, getattr will return AttributeError for us.
- method = getattr(AuroraAdmin.Client, method_name)
- if not callable(method):
- return method
-
- @functools.wraps(method)
- def method_wrapper(*args):
- start = time.time()
- while (time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(Time.SECONDS):
- auth_args = () if method_name in self.UNAUTHENTICATED_RPCS else (self.session_key(),)
- try:
- method = getattr(self.client(), method_name)
- if not callable(method):
- return method
- return method(*(args + auth_args))
- except (TTransport.TTransportException, self.TimeoutError) as e:
- log.warning('Connection error with scheduler: %s, reconnecting...' % e)
- self.invalidate()
- time.sleep(self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
- raise self.TimeoutError('Timed out attempting to issue %s to %s' % (
- method_name, self.cluster.name))
-
- return method_wrapper
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/updater.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/updater.py b/src/main/python/twitter/aurora/client/api/updater.py
deleted file mode 100644
index 6e8e8a9..0000000
--- a/src/main/python/twitter/aurora/client/api/updater.py
+++ /dev/null
@@ -1,410 +0,0 @@
-from collections import namedtuple
-from difflib import unified_diff
-
-from twitter.common import log
-
-from gen.twitter.aurora.constants import ACTIVE_STATES
-
-from gen.twitter.aurora.ttypes import (
- AddInstancesConfig,
- JobConfigValidation,
- JobKey,
- Identity,
- Lock,
- LockKey,
- LockValidation,
- Response,
- ResponseCode,
- TaskQuery,
-)
-from .updater_util import FailureThreshold, UpdaterConfig
-from .instance_watcher import InstanceWatcher
-from .scheduler_client import SchedulerProxy
-
-
-class Updater(object):
- """Update the instances of a job in batches."""
-
- class Error(Exception): pass
-
- InstanceState = namedtuple('InstanceState', ['instance_id', 'is_updated'])
- OperationConfigs = namedtuple('OperationConfigs', ['from_config', 'to_config'])
- InstanceConfigs = namedtuple(
- 'InstanceConfigs',
- ['remote_config_map', 'local_config_map', 'instances_to_process']
- )
-
- def __init__(self, config, health_check_interval_seconds, scheduler=None, instance_watcher=None):
- self._config = config
- self._job_key = JobKey(role=config.role(), environment=config.environment(), name=config.name())
- self._health_check_interval_seconds = health_check_interval_seconds
- self._scheduler = scheduler or SchedulerProxy(config.cluster())
- try:
- self._update_config = UpdaterConfig(**config.update_config().get())
- except ValueError as e:
- raise self.Error(str(e))
- self._lock = None
- self._watcher = instance_watcher or InstanceWatcher(
- self._scheduler,
- self._job_key,
- self._update_config.restart_threshold,
- self._update_config.watch_secs,
- self._health_check_interval_seconds)
-
- def _start(self):
- """Starts an update by applying an exclusive lock on a job being updated.
-
- Returns:
- Response instance from the scheduler call.
- """
- resp = self._scheduler.acquireLock(LockKey(job=self._job_key))
- if resp.responseCode == ResponseCode.OK:
- self._lock = resp.result.acquireLockResult.lock
- return resp
-
- def _finish(self):
- """Finishes an update by removing an exclusive lock on an updated job.
-
- Returns:
- Response instance from the scheduler call.
- """
- resp = self._scheduler.releaseLock(self._lock, LockValidation.CHECKED)
-
- if resp.responseCode == ResponseCode.OK:
- self._lock = None
- else:
- log.error('There was an error finalizing the update: %s' % resp.message)
- return resp
-
- def _update(self, instance_configs):
- """Drives execution of the update logic. Performs a batched update/rollback for all instances
- affected by the current update request.
-
- Arguments:
- instance_configs -- list of instance update configurations to go through.
-
- Returns the set of instances that failed to update.
- """
- failure_threshold = FailureThreshold(
- self._update_config.max_per_instance_failures,
- self._update_config.max_total_failures
- )
-
- instance_operation = self.OperationConfigs(
- from_config=instance_configs.remote_config_map,
- to_config=instance_configs.local_config_map
- )
-
- remaining_instances = [
- self.InstanceState(instance_id, is_updated=False)
- for instance_id in instance_configs.instances_to_process
- ]
-
- log.info('Starting job update.')
- while remaining_instances and not failure_threshold.is_failed_update():
- batch_instances = remaining_instances[0 : self._update_config.batch_size]
- remaining_instances = list(set(remaining_instances) - set(batch_instances))
- instances_to_restart = [s.instance_id for s in batch_instances if s.is_updated]
- instances_to_update = [s.instance_id for s in batch_instances if not s.is_updated]
-
- instances_to_watch = []
- if instances_to_restart:
- instances_to_watch += self._restart_instances(instances_to_restart)
-
- if instances_to_update:
- instances_to_watch += self._update_instances(instances_to_update, instance_operation)
-
- failed_instances = self._watcher.watch(instances_to_watch) if instances_to_watch else set()
-
- if failed_instances:
- log.error('Failed instances: %s' % failed_instances)
-
- unretryable_instances = failure_threshold.update_failure_counts(failed_instances)
- if unretryable_instances:
- log.warn('Not restarting failed instances %s, which exceeded '
- 'maximum allowed instance failure limit of %s' %
- (unretryable_instances, self._update_config.max_per_instance_failures))
- retryable_instances = list(set(failed_instances) - set(unretryable_instances))
- remaining_instances += [
- self.InstanceState(instance_id, is_updated=True) for instance_id in retryable_instances
- ]
- remaining_instances.sort(key=lambda tup: tup.instance_id)
-
- if failure_threshold.is_failed_update():
- untouched_instances = [s.instance_id for s in remaining_instances if not s.is_updated]
- instances_to_rollback = list(
- set(instance_configs.instances_to_process) - set(untouched_instances)
- )
- self._rollback(instances_to_rollback, instance_configs)
-
- return not failure_threshold.is_failed_update()
-
- def _rollback(self, instances_to_rollback, instance_configs):
- """Performs a rollback operation for the failed instances.
-
- Arguments:
- instances_to_rollback -- instance ids to rollback.
- instance_configs -- instance configuration to use for rollback.
- """
- log.info('Reverting update for %s' % instances_to_rollback)
- instance_operation = self.OperationConfigs(
- from_config=instance_configs.local_config_map,
- to_config=instance_configs.remote_config_map
- )
- instances_to_rollback.sort(reverse=True)
- failed_instances = []
- while instances_to_rollback:
- batch_instances = instances_to_rollback[0 : self._update_config.batch_size]
- instances_to_rollback = list(set(instances_to_rollback) - set(batch_instances))
- instances_to_rollback.sort(reverse=True)
- instances_to_watch = self._update_instances(batch_instances, instance_operation)
- failed_instances += self._watcher.watch(instances_to_watch)
-
- if failed_instances:
- log.error('Rollback failed for instances: %s' % failed_instances)
-
- def _create_kill_add_lists(self, instance_ids, operation_configs):
- """Determines a particular action (kill or add) to use for every instance in instance_ids.
-
- Arguments:
- instance_ids -- current batch of IDs to process.
- operation_configs -- OperationConfigs with update details.
-
- Returns lists of instances to kill and to add.
- """
- to_kill = []
- to_add = []
- for instance_id in instance_ids:
- from_config = operation_configs.from_config.get(instance_id)
- to_config = operation_configs.to_config.get(instance_id)
-
- if from_config and to_config:
- # Sort internal dicts before comparing to rule out differences due to hashing.
- diff_output = ''.join(unified_diff(
- str(sorted(from_config.__dict__.items(), key=lambda x: x[0])),
- str(sorted(to_config.__dict__.items(), key=lambda x: x[0]))))
- if diff_output:
- log.debug('Task configuration changed for instance [%s]:\n%s' % (instance_id, diff_output))
- to_kill.append(instance_id)
- to_add.append(instance_id)
- elif from_config and not to_config:
- to_kill.append(instance_id)
- elif not from_config and to_config:
- to_add.append(instance_id)
- else:
- raise self.Error('Instance %s is outside of supported range' % instance_id)
-
- return to_kill, to_add
-
- def _update_instances(self, instance_ids, operation_configs):
- """Applies kill/add actions for the specified batch instances.
-
- Arguments:
- instance_ids -- current batch of IDs to process.
- operation_configs -- OperationConfigs with update details.
-
- Returns a list of added instances.
- """
- log.info('Examining instances: %s' % instance_ids)
-
- to_kill, to_add = self._create_kill_add_lists(instance_ids, operation_configs)
-
- unchanged = list(set(instance_ids) - set(to_kill + to_add))
- if unchanged:
- log.info('Skipping unchanged instances: %s' % unchanged)
-
- # Kill is a blocking call in scheduler -> no need to watch it yet.
- self._kill_instances(to_kill)
- self._add_instances(to_add, operation_configs.to_config)
- return to_add
-
- def _kill_instances(self, instance_ids):
- """Instructs the scheduler to kill instances and waits for completion.
-
- Arguments:
- instance_ids -- list of IDs to kill.
- """
- if instance_ids:
- log.info('Killing instances: %s' % instance_ids)
- query = self._create_task_query(instanceIds=frozenset(int(s) for s in instance_ids))
- self._check_and_log_response(self._scheduler.killTasks(query, self._lock))
- log.info('Instances killed')
-
- def _add_instances(self, instance_ids, to_config):
- """Instructs the scheduler to add instances.
-
- Arguments:
- instance_ids -- list of IDs to add.
- to_config -- OperationConfigs with update details.
- """
- if instance_ids:
- log.info('Adding instances: %s' % instance_ids)
- add_config = AddInstancesConfig(
- key=self._job_key,
- taskConfig=to_config[instance_ids[0]], # instance_ids will always have at least 1 item.
- instanceIds=frozenset(int(s) for s in instance_ids))
- self._check_and_log_response(self._scheduler.addInstances(add_config, self._lock))
- log.info('Instances added')
-
- def _restart_instances(self, instance_ids):
- """Instructs the scheduler to restart instances.
-
- Arguments:
- instance_ids -- set of instances to be restarted by the scheduler.
- """
- log.info('Restarting instances: %s' % instance_ids)
- resp = self._scheduler.restartShards(self._job_key, instance_ids, self._lock)
- self._check_and_log_response(resp)
- return instance_ids
-
- def _get_update_instructions(self, instances=None):
- """Loads, validates and populates update working set.
-
- Arguments:
- instances -- (optional) set of instances to update.
-
- Returns:
- InstanceConfigs with the following data:
- remote_config_map -- dictionary of {key:instance_id, value:task_config} from scheduler.
- local_config_map -- dictionary of {key:instance_id, value:task_config} with local
- task configs validated and populated with default values.
- instances_to_process -- list of instance IDs to go over in update.
- """
- # Load existing tasks and populate remote config map and instance list.
- assigned_tasks = self._get_existing_tasks()
- remote_config_map = {}
- remote_instances = []
- for assigned_task in assigned_tasks:
- remote_config_map[assigned_task.instanceId] = assigned_task.task
- remote_instances.append(assigned_task.instanceId)
-
- # Validate local job config and populate local task config.
- local_task_config = self._validate_and_populate_local_config()
-
- # Union of local and remote instance IDs.
- job_config_instances = list(range(self._config.instances()))
- instance_superset = sorted(list(set(remote_instances) | set(job_config_instances)))
-
- # Calculate the update working set.
- if instances is None:
- # Full job update -> union of remote and local instances
- instances_to_process = instance_superset
- else:
- # Partial job update -> validate all instances are recognized
- instances_to_process = instances
- unrecognized = list(set(instances) - set(instance_superset))
- if unrecognized:
- raise self.Error('Instances %s are outside of supported range' % unrecognized)
-
- # Populate local config map
- local_config_map = dict.fromkeys(job_config_instances, local_task_config)
-
- return self.InstanceConfigs(remote_config_map, local_config_map, instances_to_process)
-
- def _get_existing_tasks(self):
- """Loads all existing tasks from the scheduler.
-
- Returns a list of AssignedTasks.
- """
- resp = self._scheduler.getTasksStatus(self._create_task_query())
- self._check_and_log_response(resp)
- return [t.assignedTask for t in resp.result.scheduleStatusResult.tasks]
-
- def _validate_and_populate_local_config(self):
- """Validates local job configuration and populates local task config with default values.
-
- Returns a TaskConfig populated with default values.
- """
- resp = self._scheduler.populateJobConfig(self._config.job(), JobConfigValidation.RUN_FILTERS)
- self._check_and_log_response(resp)
-
- # Safe to take the first element as Scheduler would throw in case zero instances provided.
- return list(resp.result.populateJobResult.populated)[0]
-
- def _replace_template_if_cron(self):
- """Checks if the provided job config represents a cron job and if so, replaces it.
-
- Returns True if job is cron and False otherwise.
- """
- if self._config.job().cronSchedule:
- resp = self._scheduler.replaceCronTemplate(self._config.job(), self._lock)
- self._check_and_log_response(resp)
- return True
- else:
- return False
-
- def _create_task_query(self, instanceIds=None):
- return TaskQuery(
- owner=Identity(role=self._job_key.role),
- environment=self._job_key.environment,
- jobName=self._job_key.name,
- statuses=ACTIVE_STATES,
- instanceIds=instanceIds)
-
- def _failed_response(self, message):
- return Response(responseCode=ResponseCode.ERROR, message=message)
-
- def update(self, instances=None):
- """Performs the job update, blocking until it completes.
- A rollback will be performed if the update was considered a failure based on the
- update configuration.
-
- Arguments:
- instances -- (optional) instances to update. If not specified, all instances will be updated.
-
- Returns a response object with update result status.
- """
- resp = self._start()
- if resp.responseCode != ResponseCode.OK:
- return resp
-
- try:
- # Handle cron jobs separately from other jobs.
- if self._replace_template_if_cron():
- log.info('Cron template updated, next run will reflect changes')
- return self._finish()
- else:
- try:
- instance_configs = self._get_update_instructions(instances)
- except self.Error as e:
- # Safe to release the lock acquired above as no job mutation has happened yet.
- self._finish()
- return self._failed_response('Unable to start job update: %s' % e)
-
- if not self._update(instance_configs):
- log.warn('Update failures threshold reached')
- self._finish()
- return self._failed_response('Update reverted')
- else:
- log.info('Update successful')
- return self._finish()
- except self.Error as e:
- return self._failed_response('Aborting update without rollback! Fatal error: %s' % e)
-
- @classmethod
- def cancel_update(cls, scheduler, job_key):
- """Cancels an update process by removing an exclusive lock on a provided job.
-
- Arguments:
- scheduler -- scheduler instance to use.
- job_key -- job key to cancel update for.
-
- Returns a response object with cancel update result status.
- """
- return scheduler.releaseLock(
- Lock(key=LockKey(job=job_key.to_thrift())),
- LockValidation.UNCHECKED)
-
- def _check_and_log_response(self, resp):
- """Checks scheduler return status, raises Error in case of unexpected response.
-
- Arguments:
- resp -- scheduler response object.
-
- Raises Error in case of unexpected response status.
- """
- name, message = ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message
- if resp.responseCode == ResponseCode.OK:
- log.debug('Response from scheduler: %s (message: %s)' % (name, message))
- else:
- raise self.Error(message)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/api/updater_util.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/api/updater_util.py b/src/main/python/twitter/aurora/client/api/updater_util.py
deleted file mode 100644
index db9f053..0000000
--- a/src/main/python/twitter/aurora/client/api/updater_util.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import collections
-
-from twitter.common import log
-
-class UpdaterConfig(object):
- """
- For updates involving a health check,
-
- UPDATE INSTANCE HEALTHY REMAIN HEALTHY
- ----------------------------------------|-----------------------|
- \--------------------------------------/ \----------------------/
- restart_thresold watch_secs
-
- When an update is initiated, an instance is expected to be "healthy" before restart_threshold.
- An instance is also expected to remain healthy for at least watch_secs. If these conditions are
- not satisfied, the instance is deemed unhealthy.
- """
- def __init__(self,
- batch_size,
- restart_threshold,
- watch_secs,
- max_per_shard_failures,
- max_total_failures):
-
- if batch_size <= 0:
- raise ValueError('Batch size should be greater than 0')
- if restart_threshold <= 0:
- raise ValueError('Restart Threshold should be greater than 0')
- if watch_secs <= 0:
- raise ValueError('Watch seconds should be greater than 0')
- self.batch_size = batch_size
- self.restart_threshold = restart_threshold
- self.watch_secs = watch_secs
- self.max_total_failures = max_total_failures
- self.max_per_instance_failures = max_per_shard_failures
-
-
-class FailureThreshold(object):
- def __init__(self, max_per_instance_failures, max_total_failures):
- self._max_per_instance_failures = max_per_instance_failures
- self._max_total_failures = max_total_failures
- self._failures_by_instance = collections.defaultdict(int)
-
-
- def update_failure_counts(self, failed_instances):
- """Update the failure counts metrics based upon a batch of failed instances.
-
- Arguments:
- failed_instances - list of failed instances
-
- Returns a list of instances with failure counts exceeding _max_per_instance_failures.
- """
- exceeded_failure_count_instances = []
- for instance in failed_instances:
- self._failures_by_instance[instance] += 1
- if self._failures_by_instance[instance] > self._max_per_instance_failures:
- exceeded_failure_count_instances.append(instance)
-
- return exceeded_failure_count_instances
-
- def is_failed_update(self):
- total_failed_instances = self._exceeded_instance_fail_count()
- is_failed = total_failed_instances > self._max_total_failures
-
- if is_failed:
- log.error('%s failed instances observed, maximum allowed is %s' % (total_failed_instances,
- self._max_total_failures))
- for instance, failure_count in self._failures_by_instance.items():
- if failure_count > self._max_per_instance_failures:
- log.error('%s instance failures for instance %s, maximum allowed is %s' %
- (failure_count, instance, self._max_per_instance_failures))
- return is_failed
-
- def _exceeded_instance_fail_count(self):
- """Checks if the per instance failure is greater than a threshold."""
- return sum(count > self._max_per_instance_failures
- for count in self._failures_by_instance.values())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/base.py b/src/main/python/twitter/aurora/client/base.py
deleted file mode 100644
index 0936743..0000000
--- a/src/main/python/twitter/aurora/client/base.py
+++ /dev/null
@@ -1,151 +0,0 @@
-from collections import defaultdict
-import functools
-import sys
-from urlparse import urljoin
-
-from twitter.common import app, log
-
-from gen.twitter.aurora.ttypes import ResponseCode
-
-LOCKED_WARNING = """
-Note: if the scheduler detects that a job update is in progress (or was not
-properly completed) it will reject subsequent updates. This is because your
-job is likely in a partially-updated state. You should only begin another
-update if you are confident that nobody is updating this job, and that
-the job is in a state suitable for an update.
-
-After checking on the above, you may release the update lock on the job by
-invoking cancel_update.
-"""
-
-def die(msg):
- log.fatal(msg)
- sys.exit(1)
-
-def check_and_log_response(resp):
- log.info('Response from scheduler: %s (message: %s)'
- % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
- if resp.responseCode != ResponseCode.OK:
- check_and_log_locked_response(resp)
- sys.exit(1)
-
-def check_and_log_locked_response(resp):
- if resp.responseCode == ResponseCode.LOCK_ERROR:
- log.info(LOCKED_WARNING)
-
-def deprecation_warning(text):
- log.warning('')
- log.warning('*' * 80)
- log.warning('* The command you ran is deprecated and will soon break!')
- for line in text.split('\n'):
- log.warning('* %s' % line)
- log.warning('*' * 80)
- log.warning('')
-
-
-class requires(object):
- @staticmethod
- def wrap_function(fn, fnargs, comparator):
- @functools.wraps(fn)
- def wrapped_function(args):
- if not comparator(args, fnargs):
- help = 'Incorrect parameters for %s' % fn.__name__
- if fn.__doc__:
- help = '%s\n\nsee the help subcommand for more details.' % fn.__doc__.split('\n')[0]
- die(help)
- return fn(*args)
- return wrapped_function
-
- @staticmethod
- def exactly(*args):
- def wrap(fn):
- return requires.wrap_function(fn, args, (lambda want, got: len(want) == len(got)))
- return wrap
-
- @staticmethod
- def at_least(*args):
- def wrap(fn):
- return requires.wrap_function(fn, args, (lambda want, got: len(want) >= len(got)))
- return wrap
-
- @staticmethod
- def nothing(fn):
- @functools.wraps(fn)
- def real_fn(line):
- return fn(*line)
- return real_fn
-
-
-def synthesize_url(scheduler_url, role=None, env=None, job=None):
- if not scheduler_url:
- log.warning("Unable to find scheduler web UI!")
- return None
-
- if env and not role:
- die('If env specified, must specify role')
- if job and not (role and env):
- die('If job specified, must specify role and env')
-
- scheduler_url = urljoin(scheduler_url, 'scheduler')
- if role:
- scheduler_url += '/' + role
- if env:
- scheduler_url += '/' + env
- if job:
- scheduler_url += '/' + job
- return scheduler_url
-
-
-def handle_open(scheduler_url, role, env, job):
- url = synthesize_url(scheduler_url, role, env, job)
- if url:
- log.info('Job url: %s' % url)
- if app.get_options().open_browser:
- import webbrowser
- webbrowser.open_new_tab(url)
-
-
-def make_commands_str(command_aliases):
- """Format a string representation of a number of command aliases."""
- commands = command_aliases[:]
- commands.sort()
- if len(commands) == 1:
- return str(commands[0])
- elif len(commands) == 2:
- return '%s (or %s)' % (str(commands[0]), str(commands[1]))
- else:
- return '%s (or any of: %s)' % (str(commands[0]), ' '.join(map(str, commands[1:])))
-
-
-# TODO(wickman) This likely belongs in twitter.common.app (or split out as
-# part of a possible twitter.common.cli)
-def generate_full_usage():
- """Generate verbose application usage from all registered
- twitter.common.app commands and return as a string."""
- docs_to_commands = defaultdict(list)
- for (command, doc) in app.get_commands_and_docstrings():
- docs_to_commands[doc].append(command)
- def make_docstring(item):
- (doc_text, commands) = item
- def format_line(line):
- return ' %s\n' % line.lstrip()
- stripped = ''.join(map(format_line, doc_text.splitlines()))
- return '%s\n%s' % (make_commands_str(commands), stripped)
- usage = sorted(map(make_docstring, docs_to_commands.items()))
- return 'Available commands:\n\n' + '\n'.join(usage)
-
-
-def generate_terse_usage():
- """Generate minimal application usage from all registered
- twitter.common.app commands and return as a string."""
- docs_to_commands = defaultdict(list)
- for (command, doc) in app.get_commands_and_docstrings():
- docs_to_commands[doc].append(command)
- usage = '\n '.join(sorted(map(make_commands_str, docs_to_commands.values())))
- return """
-Available commands:
- %s
-
-For more help on an individual command:
- %s help <command>
-""" % (usage, app.name())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/bin/BUILD b/src/main/python/twitter/aurora/client/bin/BUILD
deleted file mode 100644
index 7802245..0000000
--- a/src/main/python/twitter/aurora/client/bin/BUILD
+++ /dev/null
@@ -1,25 +0,0 @@
-python_binary(
- name = 'aurora_client',
- source = 'aurora_client.py',
- entry_point = 'twitter.aurora.client.bin.aurora_client:proxy_main',
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/client/commands:all'),
- pants('src/main/python/twitter/aurora/client:base'),
- ]
-)
-
-python_binary(
- name = 'aurora_admin',
- source = 'aurora_admin.py',
- entry_point = 'twitter.aurora.client.bin.aurora_admin:proxy_main',
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/client/commands:admin'),
- pants('src/main/python/twitter/aurora/client/commands:help'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/python/twitter/aurora/client:options'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/bin/__init__.py b/src/main/python/twitter/aurora/client/bin/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/bin/aurora_admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/bin/aurora_admin.py b/src/main/python/twitter/aurora/client/bin/aurora_admin.py
deleted file mode 100644
index 53fd882..0000000
--- a/src/main/python/twitter/aurora/client/bin/aurora_admin.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from twitter.aurora.client.base import generate_terse_usage
-from twitter.aurora.client.commands import admin, help
-from twitter.aurora.client.options import add_verbosity_options
-from twitter.common import app
-from twitter.common.log.options import LogOptions
-
-
-app.register_commands_from(admin, help)
-add_verbosity_options()
-
-
-def main():
- app.help()
-
-
-LogOptions.set_stderr_log_level('INFO')
-LogOptions.disable_disk_logging()
-app.set_name('aurora-admin')
-app.set_usage(generate_terse_usage())
-
-
-def proxy_main():
- app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/bin/aurora_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/bin/aurora_client.py b/src/main/python/twitter/aurora/client/bin/aurora_client.py
deleted file mode 100644
index 8228b88..0000000
--- a/src/main/python/twitter/aurora/client/bin/aurora_client.py
+++ /dev/null
@@ -1,33 +0,0 @@
-from twitter.aurora.client.base import generate_terse_usage
-from twitter.common import app
-from twitter.common.log.options import LogOptions
-
-# These are are side-effecting imports in that they register commands via
-# app.command. This is a poor code practice and should be fixed long-term
-# with the creation of twitter.common.cli that allows for argparse-style CLI
-# composition.
-from twitter.aurora.client.commands import (
- core,
- help,
- run,
- ssh,
-)
-from twitter.aurora.client.options import add_verbosity_options
-
-app.register_commands_from(core, run, ssh)
-app.register_commands_from(help)
-add_verbosity_options()
-
-
-def main():
- app.help()
-
-
-LogOptions.set_stderr_log_level('INFO')
-LogOptions.disable_disk_logging()
-app.set_name('aurora-client')
-app.set_usage(generate_terse_usage())
-
-
-def proxy_main():
- app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/binding_helper.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/binding_helper.py b/src/main/python/twitter/aurora/client/binding_helper.py
deleted file mode 100644
index fa03b17..0000000
--- a/src/main/python/twitter/aurora/client/binding_helper.py
+++ /dev/null
@@ -1,115 +0,0 @@
-from abc import abstractmethod, abstractproperty
-import inspect
-import os
-import sys
-
-from twitter.common.lang import Interface
-
-__all__ = (
- 'BindingHelper',
- 'CachingBindingHelper',
- 'apply_all',
- 'clear_binding_caches',
- 'unregister_all',
-)
-
-
-# The registry for binding helpers.
-_BINDING_HELPERS = []
-
-
-# TODO(wickman) Update the pydocs to remove references to common_internal components.
-class BindingHelper(Interface):
- """A component which resolves some set of pseudo-bindings in a config.
-
- Many bindings are too complex to resolve with bindings using the standard mechanisms,
- because they require some python computation to determine how to bind them. For example,
- for references like {{packer[role][pkg][version]}}, we need to talk to the packer to figure
- out the correct packer call for the desired cluster.
-
- A BindingHelper is responsible for resolving one of these types of pseudo-bindings.
- PackerBindingHelper will resolve "packer" bindings; BuildBindingHelper will resolve "build"
- bindings, JenkinsBindingHelper will resolve "jenkins" bindings, etc.
-
- A BindingHelper can be registered by calling "BindingHelper.register(Helper)". Instead of
- explicitly calling "inject" methods in populate_namespaces, it will compute the set of open
- bindings, and then call the appropriate helpers for each.
-
- The bindings can be computed either from scratch, or from a binding dictionary. A binding
- dictionary can be computed from live data, and then passed over an RPC connection, so that
- the bindings can be recomputed on the server.
-
- Each helper is responsible for computing its own binding dict. The data in the dict should
- meet two requirements: it should be enough data to allow it to produce exactly the same
- result as the scratch binding, and the data should provide information that makes the
- binding comprehensible for a human debugging a job.
-
- For example, a packer helper's binding dict should provide enough information to identify
- the HDFS file that should be used, but also the version number of the binary in packer,
- (because a human reader wants to know the version of the package, not the meaningless
- HDFS URL.
- """
- @classmethod
- def register(cls):
- _BINDING_HELPERS.append(cls())
-
- def apply(self, config, env=None, binding_dict=None):
- for match in self.matcher.match(config.raw()):
- self.bind(config, match, env, binding_dict or config.binding_dicts[self.name])
-
- @abstractproperty
- def name(self):
- """Returns the name of this BindingHelper. Typically it is the first component of
- the matcher, e.g. if the matcher matches {{git[sha]}}, return "git"."""
-
- @abstractproperty
- def matcher(self):
- """Returns the pystachio matcher for refs that this binding helper binds."""
-
- @abstractmethod
- def bind(self, config, match, env, binding_dict):
- """Resolves a ref, adding a binding to the config."""
-
-
-class CachingBindingHelper(BindingHelper):
- """A binding helper implementation that caches binding results"""
- def __init__(self):
- self.cache = {}
-
- def flush_cache(self):
- self.cache = {}
-
- def bind(self, config, match, env, binding_dict):
- if match not in self.cache:
- self.cache[match] = self.uncached_bind(config, match, env, binding_dict)
- config.bind(self.cache[match])
-
- @abstractmethod
- def uncached_bind(self, config, match, env, binding_dict):
- """Compute the binding for a ref that hasn't been seen before."""
-
-
-def unregister_all():
- _BINDING_HELPERS[:] = []
-
-
-def apply_all(config, env=None, binding_dict=None):
- """Computes a set of bindings and applies them to the config.
-
- :param config: the config whose bindings need to be computed.
- :param env: the python environment where the configuration was evaluated.
- :param binding_dict: an optional dictionary containing data to be used to compute the
- bindings. If this is provided, then data from the dictionary should be used in
- preference over live data.
- :return: a binding dictionary with data that can be used to recompute the bindings. The
- config is updated in-place.
- """
- for helper in _BINDING_HELPERS:
- helper.apply(config, env, binding_dict or config.binding_dicts[helper.name])
-
-
-def clear_binding_caches():
- """Clear the binding helper's caches for testing."""
- for helper in _BINDING_HELPERS:
- if isinstance(helper, CachingBindingHelper):
- helper.flush_cache()
[12/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/status_checker.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/status_checker.py b/src/main/python/twitter/aurora/executor/common/status_checker.py
deleted file mode 100644
index efffa8f..0000000
--- a/src/main/python/twitter/aurora/executor/common/status_checker.py
+++ /dev/null
@@ -1,111 +0,0 @@
-from abc import abstractmethod, abstractproperty
-
-from twitter.common import log
-from twitter.common.lang import Interface
-from twitter.common.metrics import NamedGauge, Observable
-
-
-# This mirrors mesos_pb2 TaskStatus without explicitly depending upon it.
-#
-# The dependency is a 30MB egg, so for smaller applications that just need
-# the status text, we proxy them. The actual conversion betwen ExitState
-# and TaskStatus is done in the StatusManager.
-class ExitState(object):
- FAILED = object()
- FINISHED = object()
- KILLED = object()
- LOST = object()
-
- ALL_STATES = {
- FAILED: 'FAILED',
- FINISHED: 'FINISHED',
- KILLED: 'KILLED',
- LOST: 'LOST',
- }
-
-
-class StatusResult(object):
- """
- Encapsulates a reason for failure and an optional reason which defaults to
- ExitState.FAILED.
- """
-
- def __init__(self, reason, status):
- self._reason = reason
- if status not in ExitState.ALL_STATES:
- raise ValueError('Unknown task state: %r' % status)
- self._status = status
-
- @property
- def reason(self):
- return self._reason
-
- @property
- def status(self):
- return self._status
-
- def __repr__(self):
- return '%s(%r, status=%r)' % (
- self.__class__.__name__,
- self._reason,
- ExitState.ALL_STATES[self._status])
-
-
-class StatusChecker(Observable, Interface):
- """Interface to pluggable status checkers for the Aurora Executor."""
-
- @abstractproperty
- def status(self):
- """Return None under normal operations. Return StatusResult to indicate status proposal."""
-
- def start(self):
- """Invoked once the task has been started."""
- self.metrics.register(NamedGauge('enabled', 1))
-
- def stop(self):
- """Invoked once a non-None status has been reported."""
- pass
-
-
-class StatusCheckerProvider(Interface):
- @abstractmethod
- def from_assigned_task(self, assigned_task, sandbox):
- pass
-
-
-class Healthy(StatusChecker):
- @property
- def status(self):
- return None
-
-
-class ChainedStatusChecker(StatusChecker):
- def __init__(self, status_checkers):
- self._status_checkers = status_checkers
- self._status = None
- if not all(isinstance(h_i, StatusChecker) for h_i in status_checkers):
- raise TypeError('ChainedStatusChecker must take an iterable of StatusCheckers.')
- super(ChainedStatusChecker, self).__init__()
-
- @property
- def status(self):
- if self._status is None:
- for status_checker in self._status_checkers:
- log.debug('Checking status from %s' % status_checker.__class__.__name__)
- status_checker_status = status_checker.status
- if status_checker_status is not None:
- log.info('%s reported %s' % (status_checker.__class__.__name__, status_checker_status))
- if not isinstance(status_checker_status, StatusResult):
- raise TypeError('StatusChecker returned something other than a StatusResult: got %s' %
- type(status_checker_status))
- self._status = status_checker_status
- break
- return self._status
-
- def start(self):
- for status_checker in self._status_checkers:
- status_checker.start()
-
- def stop(self):
- for status_checker in self._status_checkers:
- status_checker.stop()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/task_info.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/task_info.py b/src/main/python/twitter/aurora/executor/common/task_info.py
deleted file mode 100644
index 1a7cf27..0000000
--- a/src/main/python/twitter/aurora/executor/common/task_info.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import json
-
-from twitter.aurora.config.port_resolver import PortResolver
-from twitter.aurora.config.schema.base import MesosJob, MesosTaskInstance
-from twitter.aurora.config.thrift import task_instance_from_job
-from twitter.common import log
-
-from gen.twitter.aurora.ttypes import AssignedTask
-
-from pystachio import Ref
-from thrift.Thrift import TException
-from thrift.TSerialization import deserialize as thrift_deserialize
-
-
-def assigned_task_from_mesos_task(task):
- """Deserialize AssignedTask from a launchTask task protocol buffer."""
- try:
- assigned_task = thrift_deserialize(AssignedTask(), task.data)
- except (EOFError, TException) as e:
- raise ValueError('Could not deserialize task! %s' % e)
- return assigned_task
-
-
-def mesos_job_from_assigned_task(assigned_task):
- """Deserialize a MesosJob pystachio struct from an AssignedTask."""
- thermos_task = assigned_task.task.executorConfig.data
- try:
- json_blob = json.loads(thermos_task)
- except (TypeError, ValueError):
- return None
- if 'instance' in json_blob:
- # This is a MesosTaskInstance so we cannot get a MesosJob from this assigned_task
- return None
- return MesosJob.json_loads(thermos_task)
-
-
-def mesos_task_instance_from_assigned_task(assigned_task):
- """Deserialize MesosTaskInstance from an AssignedTask thrift."""
- thermos_task = assigned_task.task.executorConfig.data
-
- if not thermos_task:
- raise ValueError('Task did not have a thermos config!')
-
- try:
- json_blob = json.loads(thermos_task)
- except (TypeError, ValueError) as e:
- raise ValueError('Could not deserialize thermos config: %s' % e)
-
- # As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
- # or we can be sending a MesosJob. So handle both possible cases. Once everyone
- # is using MesosJob, then we can begin to leverage additional information that
- # becomes available such as cluster.
- if 'instance' in json_blob:
- return MesosTaskInstance.json_loads(thermos_task)
-
- # This is a MesosJob
- mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task), assigned_task.instanceId)
- for ref in refs:
- # If the ref is {{thermos.task_id}} or a subscope of
- # {{thermos.ports}}, it currently gets bound by the Thermos Runner,
- # so we must leave them unbound.
- #
- # {{thermos.user}} is a legacy binding which we can safely ignore.
- #
- # TODO(wickman) These should be rewritten by the mesos client to use
- # %%style%% replacements in order to allow us to better type-check configs
- # client-side.
- if ref == Ref.from_address('thermos.task_id'):
- continue
- if Ref.subscope(Ref.from_address('thermos.ports'), ref):
- continue
- if ref == Ref.from_address('thermos.user'):
- continue
- raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, refs)))
- return mti
-
-
-def resolve_ports(mesos_task, portmap):
- """Given a MesosTaskInstance and the portmap of resolved ports from the scheduler,
- create a fully resolved map of port name => port number for the thermos
- runner and discovery manager."""
- task_portmap = mesos_task.announce().portmap().get() if mesos_task.has_announce() else {}
- task_portmap.update(portmap)
- task_portmap = PortResolver.resolve(task_portmap)
-
- for name, port in task_portmap.items():
- if not isinstance(port, int):
- log.warning('Task has unmapped port: %s => %s' % (name, port))
-
- return dict((name, port) for (name, port) in task_portmap.items() if isinstance(port, int))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/task_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/task_runner.py b/src/main/python/twitter/aurora/executor/common/task_runner.py
deleted file mode 100644
index c7a57cb..0000000
--- a/src/main/python/twitter/aurora/executor/common/task_runner.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from abc import abstractmethod
-
-from twitter.common.lang import Interface
-
-from .status_checker import StatusChecker
-
-
-class TaskError(Exception):
- pass
-
-
-class TaskRunner(StatusChecker):
- # For now, TaskRunner should just maintain the StatusChecker API.
- pass
-
-
-class TaskRunnerProvider(Interface):
- @abstractmethod
- def from_assigned_task(self, assigned_task, sandbox):
- pass
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/executor_base.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/executor_base.py b/src/main/python/twitter/aurora/executor/executor_base.py
deleted file mode 100644
index 4fd9aa4..0000000
--- a/src/main/python/twitter/aurora/executor/executor_base.py
+++ /dev/null
@@ -1,103 +0,0 @@
-from twitter.common import log
-
-from gen.twitter.aurora.constants import TERMINAL_STATES as AURORA_TERMINAL_STATES
-from gen.twitter.aurora.ttypes import ScheduleStatus
-from gen.twitter.thermos.ttypes import TaskState
-
-import mesos
-import mesos_pb2 as mesos_pb
-
-
-class ThermosExecutorBase(mesos.Executor):
- # Statuses are hard, let's go shopping.
- MESOS_STATES = {
- mesos_pb.TASK_STARTING: 'STARTING',
- mesos_pb.TASK_RUNNING: 'RUNNING',
- mesos_pb.TASK_FINISHED: 'FINISHED',
- mesos_pb.TASK_FAILED: 'FAILED',
- mesos_pb.TASK_KILLED: 'KILLED',
- mesos_pb.TASK_LOST: 'LOST',
- }
-
- THERMOS_TO_MESOS_STATES = {
- TaskState.ACTIVE: mesos_pb.TASK_RUNNING,
- TaskState.SUCCESS: mesos_pb.TASK_FINISHED,
- TaskState.FAILED: mesos_pb.TASK_FAILED,
- TaskState.KILLED: mesos_pb.TASK_KILLED,
- TaskState.LOST: mesos_pb.TASK_LOST,
- }
-
- THERMOS_TO_TWITTER_STATES = {
- TaskState.ACTIVE: ScheduleStatus.RUNNING,
- TaskState.CLEANING: ScheduleStatus.RUNNING,
- TaskState.FINALIZING: ScheduleStatus.RUNNING,
- TaskState.SUCCESS: ScheduleStatus.FINISHED,
- TaskState.FAILED: ScheduleStatus.FAILED,
- TaskState.KILLED: ScheduleStatus.KILLED,
- TaskState.LOST: ScheduleStatus.LOST,
- }
-
- @staticmethod
- def twitter_status_is_terminal(status):
- return status in AURORA_TERMINAL_STATES
-
- @staticmethod
- def mesos_status_is_terminal(status):
- return status in (
- mesos_pb.TASK_FAILED,
- mesos_pb.TASK_FINISHED,
- mesos_pb.TASK_KILLED,
- mesos_pb.TASK_LOST,
- )
-
- @staticmethod
- def thermos_status_is_terminal(status):
- return status in (
- TaskState.FAILED,
- TaskState.KILLED,
- TaskState.LOST,
- TaskState.SUCCESS,
- )
-
- def __init__(self):
- self._slave_id = None
-
- def log(self, msg):
- log.info('Executor [%s]: %s' % (self._slave_id, msg))
-
- def registered(self, driver, executor_info, framework_info, slave_info):
- self.log('registered() called with:')
- self.log(' ExecutorInfo: %s' % executor_info)
- self.log(' FrameworkInfo: %s' % framework_info)
- self.log(' SlaveInfo: %s' % slave_info)
- self._driver = driver
- self._executor_info = executor_info
- self._framework_info = framework_info
- self._slave_info = slave_info
-
- def reregistered(self, driver, slave_info):
- self.log('reregistered() called with:')
- self.log(' SlaveInfo: %s' % slave_info)
-
- def disconnected(self, driver):
- self.log('disconnected() called')
-
- def send_update(self, driver, task_id, state, message=None):
- update = mesos_pb.TaskStatus()
- if not isinstance(state, int):
- raise TypeError('Invalid state type %s, should be int.' % type(state))
- if state not in self.MESOS_STATES:
- raise ValueError('Invalid state: %s' % state)
- update.state = state
- update.task_id.value = task_id
- if message:
- update.message = str(message)
- self.log('Updating %s => %s' % (task_id, self.MESOS_STATES[state]))
- self.log(' Reason: %s' % message)
- driver.sendStatusUpdate(update)
-
- def frameworkMessage(self, driver, message):
- self.log('frameworkMessage() got message: %s, ignoring.' % message)
-
- def error(self, driver, message):
- self.log('Received error message: %s' % message)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/executor_detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/executor_detector.py b/src/main/python/twitter/aurora/executor/executor_detector.py
deleted file mode 100644
index c26c00e..0000000
--- a/src/main/python/twitter/aurora/executor/executor_detector.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import os
-from glob import glob
-
-from twitter.common.string import ScanfParser
-
-
-# TODO(wickman) MESOS-2805 This makes an assumption about the directory
-# layout Mesos provides. Ideally we never need to do this and we should
-# work with the Mesos core team to make it unnecessary.
-class ExecutorDetector(object):
- class Error(Exception): pass
- class CannotFindRoot(Error): pass
-
- LOG_PATH = 'executor_logs'
- RESOURCE_PATH = 'resource_usage.recordio'
- VARS_PATH = 'executor_vars.json'
- PATTERN = [
- '%(root)s',
- 'slaves',
- '%(slave_id)s',
- 'frameworks',
- '%(framework_id)s',
- 'executors',
- '%(executor_id)s',
- 'runs',
- '%(run)s']
- EXTRACTOR = ScanfParser(os.path.join(*PATTERN))
-
- @classmethod
- def find_root(cls, path):
- """Does this path appear to match the executor directory pattern?"""
-
- def root_from_path(path):
- path = os.path.normpath(path)
- path_vector = path.split(os.path.sep)
- pattern_vector = cls.PATTERN
- if len(path_vector) < len(pattern_vector):
- return None
- for pattern, path_component in zip(reversed(pattern_vector), reversed(path_vector)):
- if pattern.startswith('%'):
- continue
- if path_component != pattern:
- return None
- matched_path = os.path.join(*path_vector[-len(pattern_vector) + 1:])
- return os.path.normpath(path[:-len(matched_path)])
-
- while path != os.path.dirname(path):
- root = root_from_path(path)
- if root:
- return root
- path = os.path.dirname(path)
-
- @classmethod
- def match(cls, path):
- try:
- return cls.EXTRACTOR.parse(path)
- except ScanfParser.ParseError:
- return None
-
- @classmethod
- def path(cls, result):
- return os.path.join(*cls.PATTERN) % result.groups()
-
- @classmethod
- def find(cls, root, slave_id='*', framework_id='*', executor_id='*', run='*'):
- mixins = dict(
- root=root, slave_id=slave_id, framework_id=framework_id, executor_id=executor_id, run=run)
- return filter(None, map(cls.match, glob(os.path.join(*cls.PATTERN) % mixins)))
-
- def __init__(self, root=None):
- self.root = root or self.find_root(os.getcwd())
- if self.root is None:
- raise self.CannotFindRoot('Not a valid executor root!')
-
- def __iter__(self):
- for extraction in self.find(root=self.root):
- yield extraction
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/executor_vars.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/executor_vars.py b/src/main/python/twitter/aurora/executor/executor_vars.py
deleted file mode 100644
index 7f54d8f..0000000
--- a/src/main/python/twitter/aurora/executor/executor_vars.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import os
-import time
-
-from twitter.common.exceptions import ExceptionalThread
-from twitter.common.metrics import (
- LambdaGauge,
- MutatorGauge,
- NamedGauge,
- Observable)
-from twitter.common.python.dirwrapper import PythonDirectoryWrapper
-from twitter.common.python.pex import PexInfo
-from twitter.common.quantity import Amount, Time
-from twitter.common.string.scanf import ScanfParser
-
-import psutil
-
-
-class ExecutorVars(Observable, ExceptionalThread):
- """
- Executor exported /vars wrapper.
-
- Currently writes to disk and communicates through the Aurora Observer,
- pending MESOS-433.
- """
- MUTATOR_METRICS = ('rss', 'cpu', 'thermos_pss', 'thermos_cpu')
- RELEASE_TAG_FORMAT = ScanfParser('%(project)s_R%(release)d')
- DEPLOY_TAG_FORMAT = ScanfParser('%(project)s_%(environment)s_%(release)d_R%(deploy)d')
- PROJECT_NAMES = ('thermos', 'thermos_executor')
- COLLECTION_INTERVAL = Amount(1, Time.MINUTES)
-
- @classmethod
- def get_release_from_tag(cls, tag):
- def parse_from(parser):
- try:
- scanf = parser.parse(tag)
- if scanf and scanf.project in cls.PROJECT_NAMES:
- return scanf.release
- except ScanfParser.ParseError:
- pass
- release = parse_from(cls.RELEASE_TAG_FORMAT)
- if release is None:
- release = parse_from(cls.DEPLOY_TAG_FORMAT)
- if release is None:
- release = 'UNKNOWN'
- return release
-
- @classmethod
- def get_release_from_binary(cls, binary):
- try:
- pex_info = PexInfo.from_pex(PythonDirectoryWrapper.get(binary))
- return cls.get_release_from_tag(pex_info.build_properties.get('tag', ''))
- except PythonDirectoryWrapper.Error:
- return 'UNKNOWN'
-
- def __init__(self, clock=time):
- self._clock = clock
- self._self = psutil.Process(os.getpid())
- if hasattr(self._self, 'getcwd'):
- self._version = self.get_release_from_binary(
- os.path.join(self._self.getcwd(), self._self.cmdline[1]))
- else:
- self._version = 'UNKNOWN'
- self.metrics.register(NamedGauge('version', self._version))
- self._orphan = False
- self.metrics.register(LambdaGauge('orphan', lambda: int(self._orphan)))
- self._metrics = dict((metric, MutatorGauge(metric, 0)) for metric in self.MUTATOR_METRICS)
- for metric in self._metrics.values():
- self.metrics.register(metric)
- ExceptionalThread.__init__(self)
- self.daemon = True
-
- def write_metric(self, metric, value):
- self._metrics[metric].write(value)
-
- @classmethod
- def thermos_children(cls, parent):
- try:
- for child in parent.get_children():
- yield child # thermos_runner
- try:
- for grandchild in child.get_children():
- yield grandchild # thermos_coordinator
- except psutil.Error:
- continue
- except psutil.Error:
- return
-
- @classmethod
- def aggregate_memory(cls, process, attribute='pss'):
- try:
- return sum(getattr(mmap, attribute) for mmap in process.get_memory_maps())
- except (psutil.Error, AttributeError):
- # psutil on OS X does not support get_memory_maps
- return 0
-
- @classmethod
- def cpu_rss_pss(cls, process):
- return (process.get_cpu_percent(0),
- process.get_memory_info().rss,
- cls.aggregate_memory(process, attribute='pss'))
-
- def run(self):
- while True:
- self._clock.sleep(self.COLLECTION_INTERVAL.as_(Time.SECONDS))
- self.sample()
-
- def sample(self):
- try:
- executor_cpu, executor_rss, _ = self.cpu_rss_pss(self._self)
- self.write_metric('cpu', executor_cpu)
- self.write_metric('rss', executor_rss)
- self._orphan = self._self.ppid == 1
- except psutil.Error:
- return False
-
- try:
- child_stats = map(self.cpu_rss_pss, self.thermos_children(self._self))
- self.write_metric('thermos_cpu', sum(stat[0] for stat in child_stats))
- self.write_metric('thermos_pss', sum(stat[2] for stat in child_stats))
- except psutil.Error:
- pass
-
- return True
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/gc_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/gc_executor.py b/src/main/python/twitter/aurora/executor/gc_executor.py
deleted file mode 100644
index f17d0a2..0000000
--- a/src/main/python/twitter/aurora/executor/gc_executor.py
+++ /dev/null
@@ -1,499 +0,0 @@
-"""Thermos garbage-collection (GC) executor
-
-This module containts the Thermos GC executor, responsible for garbage collecting old tasks and
-reconciling task states with the Mesos scheduler. It is intended to be run periodically on Mesos
-slaves utilising the Thermos executor.
-
-"""
-
-import os
-import threading
-import time
-
-from twitter.common.collections import OrderedDict
-from twitter.common.exceptions import ExceptionalThread
-from twitter.common.metrics import Observable
-from twitter.common.metrics.gauge import AtomicGauge
-from twitter.common.quantity import Amount, Time
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-from twitter.thermos.common.path import TaskPath
-from twitter.thermos.core.inspector import CheckpointInspector
-from twitter.thermos.core.helper import TaskKiller
-from twitter.thermos.monitoring.detector import TaskDetector
-from twitter.thermos.monitoring.garbage import TaskGarbageCollector
-
-from gen.twitter.aurora.comm.ttypes import (
- AdjustRetainedTasks,
- DeletedTasks,
- SchedulerMessage)
-from gen.twitter.aurora.ttypes import ScheduleStatus
-
-from .common.sandbox import DirectorySandbox, SandboxInterface
-from .executor_base import ThermosExecutorBase
-from .executor_detector import ExecutorDetector
-
-import mesos_pb2 as mesos_pb
-import psutil
-from thrift.TSerialization import deserialize as thrift_deserialize
-from thrift.TSerialization import serialize as thrift_serialize
-
-
-class ThermosGCExecutor(ThermosExecutorBase, ExceptionalThread, Observable):
- """
- Thermos GC Executor, responsible for:
- - garbage collecting old tasks to make sure they don't clutter up the system
- - state reconciliation with the scheduler (in case it thinks we're running
- something we're not or vice versa.)
- """
- MAX_PID_TIME_DRIFT = Amount(10, Time.SECONDS)
- MAX_CHECKPOINT_TIME_DRIFT = Amount(1, Time.HOURS) # maximum runner disconnection time
-
- # how old a task must be before we're willing to kill it, assuming that there could be
- # slight races in the following scenario:
- # launch gc with retained_tasks={t1, t2, t3}
- # launch task t4
- MINIMUM_KILL_AGE = Amount(10, Time.MINUTES)
-
- # wait time between checking for new GC events from the slave and/or cleaning orphaned tasks
- POLL_WAIT = Amount(5, Time.MINUTES)
-
- # maximum amount of time the executor will wait with no tasks before it exits.
- MAXIMUM_EXECUTOR_WAIT = Amount(15, Time.MINUTES)
-
- # maximum lifetime of this executor. this is to prevent older GC executor binaries from
- # running forever
- MAXIMUM_EXECUTOR_LIFETIME = Amount(1, Time.DAYS)
-
- PERSISTENCE_WAIT = Amount(5, Time.SECONDS)
-
- def __init__(self,
- checkpoint_root,
- verbose=True,
- task_killer=TaskKiller,
- executor_detector=ExecutorDetector,
- task_garbage_collector=TaskGarbageCollector,
- clock=time):
- ThermosExecutorBase.__init__(self)
- ExceptionalThread.__init__(self)
- self.daemon = True
- self._stop_event = threading.Event()
- self._gc_task_queue = OrderedDict() # mapping of task_id => (TaskInfo, AdjustRetainedTasks), in
- # the order in which they were received via a launchTask.
- self._driver = None # cache the ExecutorDriver provided by the slave, so we can use it
- # out of band from slave-initiated callbacks. This should be supplied by
- # ThermosExecutorBase.registered() when the executor first registers with
- # the slave.
- self._slave_id = None # cache the slave ID provided by the slave
- self._task_id = None # the task_id currently being executed by the ThermosGCExecutor, if any
- self._start_time = None # the start time of a task currently being executed, if any
- self._detector = executor_detector()
- self._collector = task_garbage_collector(root=checkpoint_root)
- self._clock = clock
- self._task_killer = task_killer
- self._checkpoint_root = checkpoint_root
- self._dropped_tasks = AtomicGauge('dropped_tasks')
- self.metrics.register(self._dropped_tasks)
-
- def _runner_ckpt(self, task_id):
- """Return the runner checkpoint file for a given task_id."""
- return TaskPath(root=self._checkpoint_root, task_id=task_id).getpath('runner_checkpoint')
-
- def _terminate_task(self, task_id, kill=True):
- """Terminate a task using the associated task killer. Returns a boolean indicating success."""
- killer = self._task_killer(task_id, self._checkpoint_root)
- self.log('Terminating %s...' % task_id)
- runner_terminate = killer.kill if kill else killer.lose
- try:
- runner_terminate(force=True)
- return True
- except Exception as e:
- self.log('Could not terminate: %s' % e)
- return False
-
- def partition_tasks(self):
- """Return active/finished tasks as discovered from the checkpoint root."""
- detector = TaskDetector(root=self._checkpoint_root)
- active_tasks = set(t_id for _, t_id in detector.get_task_ids(state='active'))
- finished_tasks = set(t_id for _, t_id in detector.get_task_ids(state='finished'))
- return active_tasks, finished_tasks
-
- def get_states(self, task_id):
- """Returns the (timestamp, status) tuples of the task or [] if could not replay."""
- statuses = CheckpointDispatcher.iter_statuses(self._runner_ckpt(task_id))
- try:
- return [(state.timestamp_ms / 1000.0, state.state) for state in statuses]
- except CheckpointDispatcher.ErrorRecoveringState:
- return []
-
- def get_sandbox(self, task_id):
- """Returns the sandbox of the task, or None if it has not yet been initialized."""
- try:
- for update in CheckpointDispatcher.iter_updates(self._runner_ckpt(task_id)):
- if update.runner_header and update.runner_header.sandbox:
- return update.runner_header.sandbox
- except CheckpointDispatcher.ErrorRecoveringState:
- return None
-
- def maybe_terminate_unknown_task(self, task_id):
- """Terminate a task if we believe the scheduler doesn't know about it.
-
- It's possible for the scheduler to queue a GC and launch a task afterwards, in which
- case we may see actively running tasks that the scheduler did not report in the
- AdjustRetainedTasks message.
-
- Returns:
- boolean indicating whether the task was terminated
- """
- states = self.get_states(task_id)
- if states:
- task_start_time, _ = states[0]
- if self._start_time - task_start_time > self.MINIMUM_KILL_AGE.as_(Time.SECONDS):
- return self._terminate_task(task_id)
- return False
-
- def should_gc_task(self, task_id):
- """Check if a possibly-corrupt task should be locally GCed
-
- A task should be GCed if its checkpoint stream appears to be corrupted and the kill age
- threshold is exceeded.
-
- Returns:
- set, containing the task_id if it should be marked for local GC, or empty otherwise
- """
- runner_ckpt = self._runner_ckpt(task_id)
- if not os.path.exists(runner_ckpt):
- return set()
- latest_update = os.path.getmtime(runner_ckpt)
- if self._start_time - latest_update > self.MINIMUM_KILL_AGE.as_(Time.SECONDS):
- self.log('Got corrupt checkpoint file for %s - marking for local GC' % task_id)
- return set([task_id])
- else:
- self.log('Checkpoint file unreadable, but not yet beyond MINIMUM_KILL_AGE threshold')
- return set()
-
- def reconcile_states(self, driver, retained_tasks):
- """Reconcile states that the scheduler thinks tasks are in vs what they really are in.
-
- Local vs Scheduler => Action
- ===================================
- ACTIVE ACTIVE => no-op
- ACTIVE STARTING => no-op
- ACTIVE TERMINAL => maybe kill task*
- ACTIVE !EXISTS => maybe kill task*
- TERMINAL ACTIVE => send actual status**
- TERMINAL STARTING => send actual status**
- TERMINAL TERMINAL => no-op
- TERMINAL !EXISTS => gc locally
- !EXISTS ACTIVE => send LOST**
- !EXISTS STARTING => no-op
- !EXISTS TERMINAL => gc remotely
-
- * - Only kill if this does not appear to be a race condition.
- ** - These appear to have no effect
-
- Side effecting operations:
- ACTIVE | (TERMINAL / !EXISTS) => maybe kill
- TERMINAL | !EXISTS => delete
- !EXISTS | TERMINAL => delete
-
- Returns tuple of (local_gc, remote_gc, updates), where:
- local_gc - set of task_ids to be GCed locally
- remote_gc - set of task_ids to be deleted on the scheduler
- updates - dictionary of updates sent to the scheduler (task_id: ScheduleStatus)
- """
- def partition(rt):
- active, starting, finished = set(), set(), set()
- for task_id, schedule_status in rt.items():
- if self.twitter_status_is_terminal(schedule_status):
- finished.add(task_id)
- elif (schedule_status == ScheduleStatus.STARTING or
- schedule_status == ScheduleStatus.ASSIGNED):
- starting.add(task_id)
- else:
- active.add(task_id)
- return active, starting, finished
-
- local_active, local_finished = self.partition_tasks()
- sched_active, sched_starting, sched_finished = partition(retained_tasks)
- local_task_ids = local_active | local_finished
- sched_task_ids = sched_active | sched_starting | sched_finished
- all_task_ids = local_task_ids | sched_task_ids
-
- self.log('Told to retain the following task ids:')
- for task_id, schedule_status in retained_tasks.items():
- self.log(' => %s as %s' % (task_id, ScheduleStatus._VALUES_TO_NAMES[schedule_status]))
-
- self.log('Local active tasks:')
- for task_id in local_active:
- self.log(' => %s' % task_id)
-
- self.log('Local finished tasks:')
- for task_id in local_finished:
- self.log(' => %s' % task_id)
-
- local_gc, remote_gc = set(), set()
- updates = {}
-
- for task_id in all_task_ids:
- if task_id in local_active and task_id not in (sched_active | sched_starting):
- self.log('Inspecting task %s for termination.' % task_id)
- if not self.maybe_terminate_unknown_task(task_id):
- local_gc.update(self.should_gc_task(task_id))
- if task_id in local_finished and task_id not in sched_task_ids:
- self.log('Queueing task %s for local deletion.' % task_id)
- local_gc.add(task_id)
- if task_id in local_finished and task_id in (sched_active | sched_starting):
- self.log('Task %s finished but scheduler thinks active/starting.' % task_id)
- states = self.get_states(task_id)
- if states:
- _, last_state = states[-1]
- updates[task_id] = self.THERMOS_TO_TWITTER_STATES.get(last_state, ScheduleStatus.UNKNOWN)
- self.send_update(
- driver,
- task_id,
- self.THERMOS_TO_MESOS_STATES.get(last_state, mesos_pb.TASK_LOST),
- 'Task finish detected by GC executor.')
- else:
- local_gc.update(self.should_gc_task(task_id))
- if task_id in sched_finished and task_id not in local_task_ids:
- self.log('Queueing task %s for remote deletion.' % task_id)
- remote_gc.add(task_id)
- if task_id not in local_task_ids and task_id in sched_active:
- self.log('Know nothing about task %s, telling scheduler of LOSS.' % task_id)
- updates[task_id] = ScheduleStatus.LOST
- self.send_update(driver, task_id, mesos_pb.TASK_LOST, 'GC executor found no trace of task.')
- if task_id not in local_task_ids and task_id in sched_starting:
- self.log('Know nothing about task %s, but scheduler says STARTING - passing' % task_id)
-
- return local_gc, remote_gc, updates
-
- def clean_orphans(self, driver):
- """Inspect checkpoints for trees that have been kill -9'ed but not properly cleaned up."""
- self.log('Checking for orphaned tasks')
- active_tasks, _ = self.partition_tasks()
- updates = {}
-
- inspector = CheckpointInspector(self._checkpoint_root)
-
- def is_our_process(process, uid, timestamp):
- if process.uids.real != uid:
- return False
- estimated_start_time = self._clock.time() - process.create_time
- return abs(timestamp - estimated_start_time) < self.MAX_PID_TIME_DRIFT.as_(Time.SECONDS)
-
- for task_id in active_tasks:
- self.log('Inspecting running task: %s' % task_id)
- inspection = inspector.inspect(task_id)
- if not inspection:
- self.log(' - Error inspecting task runner')
- continue
- latest_runner = inspection.runner_processes[-1]
- # Assume that it has not yet started?
- if not latest_runner:
- self.log(' - Task has no registered runners.')
- continue
- runner_pid, runner_uid, timestamp_ms = latest_runner
- try:
- runner_process = psutil.Process(runner_pid)
- if is_our_process(runner_process, runner_uid, timestamp_ms / 1000.0):
- self.log(' - Runner appears healthy.')
- continue
- except psutil.NoSuchProcess:
- # Runner is dead
- pass
- except psutil.Error as err:
- self.log(' - Error sampling runner process [pid=%s]: %s' % (runner_pid, err))
- continue
- try:
- latest_update = os.path.getmtime(self._runner_ckpt(task_id))
- except (IOError, OSError) as err:
- self.log(' - Error accessing runner ckpt: %s' % err)
- continue
- if self._clock.time() - latest_update < self.MAX_CHECKPOINT_TIME_DRIFT.as_(Time.SECONDS):
- self.log(' - Runner is dead but under LOST threshold.')
- continue
- self.log(' - Runner is dead but beyond LOST threshold: %.1fs' % (
- self._clock.time() - latest_update))
- if self._terminate_task(task_id, kill=False):
- updates[task_id] = ScheduleStatus.LOST
- self.send_update(
- driver, task_id, mesos_pb.TASK_LOST, 'GC executor detected failed task runner.')
-
- return updates
-
- def _erase_sandbox(self, task_id):
- # TODO(wickman) Only mesos should be in the business of garbage collecting sandboxes.
- header_sandbox = self.get_sandbox(task_id)
- directory_sandbox = DirectorySandbox(header_sandbox) if header_sandbox else None
- if directory_sandbox and directory_sandbox.exists():
- self.log('Destroying DirectorySandbox for %s' % task_id)
- directory_sandbox.destroy()
- else:
- self.log('Found no sandboxes for %s' % task_id)
-
- def _gc(self, task_id):
- """Erase the sandbox, logs and metadata of the given task."""
- self.log('Erasing sandbox for %s' % task_id)
- self._erase_sandbox(task_id)
- self.log('Erasing logs for %s' % task_id)
- self._collector.erase_logs(task_id)
- self.log('Erasing metadata for %s' % task_id)
- self._collector.erase_metadata(task_id)
-
- def garbage_collect(self, force_delete=frozenset()):
- """Garbage collect tasks on the system no longer active or in the supplied force_delete.
-
- Return a set of task_ids representing the tasks that were garbage collected.
- """
- active_tasks, finished_tasks = self.partition_tasks()
- retained_executors = set(iter(self.linked_executors))
- self.log('Executor sandboxes retained by Mesos:')
- if retained_executors:
- for r_e in sorted(retained_executors):
- self.log(' %s' % r_e)
- else:
- self.log(' None')
- for task_id in (active_tasks - retained_executors):
- self.log('ERROR: Active task %s had its executor sandbox pulled.' % task_id)
- gc_tasks = (finished_tasks - retained_executors) | force_delete
- for gc_task in gc_tasks:
- self._gc(gc_task)
- return gc_tasks
-
- @property
- def linked_executors(self):
- """Generator yielding the executor sandboxes detected on the system."""
- thermos_executor_prefix = 'thermos-'
- for executor in self._detector:
- # It's possible for just the 'latest' symlink to be present but no run directories.
- # This indicates that the task has been fully garbage collected.
- if executor.executor_id.startswith(thermos_executor_prefix) and executor.run != 'latest':
- yield executor.executor_id[len(thermos_executor_prefix):]
-
- def _run_gc(self, task, retain_tasks, retain_start):
- """
- Reconcile the set of tasks to retain (provided by the scheduler) with the current state of
- executors on this system. Garbage collect tasks/executors as appropriate.
-
- Not re-entrant! Previous executions must complete (and clear self._task_id) before this can be
- invoked.
-
- Potentially blocking (e.g. on I/O) in self.garbage_collect()
-
- Args:
- task: TaskInfo provided by the slave
- retain_tasks: mapping of task_id => ScheduleStatus, describing what the scheduler thinks is
- running on this system
- retain_start: the time at which the retain_tasks message is effective -- this means that
- tasks started after the retain_tasks message is effective are skipped
- until future GC runs.
- """
- task_id = task.task_id.value
- if self._task_id is not None:
- raise RuntimeError('_run_gc() called [task_id=%s], but already running [task_id=%s]'
- % (task_id, self._task_id))
- self._task_id = task_id
- self.log('Launching garbage collection [task_id=%s]' % task_id)
- self._start_time = retain_start
- local_gc, remote_gc, _ = self.reconcile_states(self._driver, retain_tasks)
- deleted_tasks = set(retain_tasks).intersection(self.garbage_collect(local_gc)) | remote_gc
- if deleted_tasks:
- self._driver.sendFrameworkMessage(thrift_serialize(
- SchedulerMessage(deletedTasks=DeletedTasks(taskIds=deleted_tasks))))
- self.send_update(
- self._driver, task.task_id.value, mesos_pb.TASK_FINISHED, 'Garbage collection finished.')
- self.log('Garbage collection complete [task_id=%s]' % task_id)
- self._task_id = self._start_time = None
-
- def run(self):
- """Main GC executor event loop.
-
- Periodically perform state reconciliation with the set of tasks provided
- by the slave, and garbage collect orphaned tasks on the system.
- """
- run_start = self._clock.time()
- last_gc_run = self._clock.time()
-
- def should_terminate():
- now = self._clock.time()
- if now > run_start + self.MAXIMUM_EXECUTOR_LIFETIME.as_(Time.SECONDS):
- return True
- if now > last_gc_run + self.MAXIMUM_EXECUTOR_WAIT.as_(Time.SECONDS):
- return True
- return self._stop_event.is_set()
-
- while not should_terminate():
- try:
- _, (task, retain_tasks, retain_start) = self._gc_task_queue.popitem(0)
- last_gc_run = retain_start
- self._run_gc(task, retain_tasks, retain_start)
- except KeyError: # no enqueued GC tasks
- pass
- if self._driver is not None:
- self.clean_orphans(self._driver)
- self._stop_event.wait(self.POLL_WAIT.as_(Time.SECONDS))
-
- # shutdown
- if self._driver is not None:
- try:
- prev_task_id, _ = self._gc_task_queue.popitem(0)
- except KeyError: # no enqueued GC tasks
- pass
- else:
- self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
- 'Garbage collection skipped - GC executor shutting down')
- # TODO(jon) Remove this once external MESOS-243 is resolved.
- self.log('Sleeping briefly to mitigate https://issues.apache.org/jira/browse/MESOS-243')
- self._clock.sleep(self.PERSISTENCE_WAIT.as_(Time.SECONDS))
-
- self._driver.stop()
-
- """ Mesos Executor API methods follow """
-
- def launchTask(self, driver, task):
- """Queue a new garbage collection run, and drop any currently-enqueued runs."""
- if self._slave_id is None:
- self._slave_id = task.slave_id.value
- task_id = task.task_id.value
- self.log('launchTask() got task_id: %s' % task_id)
- if task_id == self._task_id:
- self.log('=> GC with task_id %s currently running - ignoring' % task_id)
- return
- elif task_id in self._gc_task_queue:
- self.log('=> Already have task_id %s queued - ignoring' % task_id)
- return
- try:
- art = thrift_deserialize(AdjustRetainedTasks(), task.data)
- except Exception as err:
- self.log('Error deserializing task: %s' % err)
- self.send_update(
- self._driver, task_id, mesos_pb.TASK_FAILED, 'Deserialization of GC task failed')
- return
- try:
- prev_task_id, _ = self._gc_task_queue.popitem(0)
- except KeyError: # no enqueued GC tasks - reset counter
- self._dropped_tasks.write(0)
- else:
- self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
- self._dropped_tasks.increment()
- self.log('=> Updating scheduler')
- self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
- 'Garbage collection skipped - GC executor received another task')
- self.log('=> Adding %s to GC queue' % task_id)
- self._gc_task_queue[task_id] = (task, art.retainedTasks, self._clock.time())
-
- def killTask(self, driver, task_id):
- """Remove the specified task from the queue, if it's not yet run. Otherwise, no-op."""
- self.log('killTask() got task_id: %s' % task_id)
- task = self._gc_task_queue.pop(task_id, None)
- if task is not None:
- self.log('=> Removed %s from queued GC tasks' % task_id)
- elif task_id == self._task_id:
- self.log('=> GC with task_id %s currently running - ignoring' % task_id)
- else:
- self.log('=> Unknown task_id %s - ignoring' % task_id)
-
- def shutdown(self, driver):
- """Trigger the Executor to shut down as soon as the current GC run is finished."""
- self.log('shutdown() called - setting stop event')
- self._stop_event.set()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/status_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/status_manager.py b/src/main/python/twitter/aurora/executor/status_manager.py
deleted file mode 100644
index c974597..0000000
--- a/src/main/python/twitter/aurora/executor/status_manager.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import time
-
-from twitter.common import log
-from twitter.common.exceptions import ExceptionalThread
-from twitter.common.quantity import Amount, Time
-
-from .common.status_checker import StatusChecker
-
-
-class StatusManager(ExceptionalThread):
- """
- An agent that periodically checks the health of a task via StatusCheckers that
- provide HTTP health checking, resource consumption, etc.
-
- If any of the status interfaces return a status, the Status Manager
- invokes the user-supplied callback with the status.
- """
- POLL_WAIT = Amount(500, Time.MILLISECONDS)
-
- def __init__(self, status_checker, callback, clock=time):
- if not isinstance(status_checker, StatusChecker):
- raise TypeError('status_checker must be a StatusChecker, got %s' % type(status_checker))
- if not callable(callback):
- raise TypeError('callback needs to be callable!')
- self._status_checker = status_checker
- self._callback = callback
- self._clock = clock
- super(StatusManager, self).__init__()
- self.daemon = True
-
- def run(self):
- while True:
- status_result = self._status_checker.status
- if status_result is not None:
- log.info('Status manager got %s' % status_result)
- self._callback(status_result)
- break
- else:
- self._clock.sleep(self.POLL_WAIT.as_(Time.SECONDS))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/thermos_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/thermos_executor.py b/src/main/python/twitter/aurora/executor/thermos_executor.py
deleted file mode 100644
index 52b514a..0000000
--- a/src/main/python/twitter/aurora/executor/thermos_executor.py
+++ /dev/null
@@ -1,289 +0,0 @@
-import os
-import threading
-import time
-import traceback
-
-from twitter.common import log
-from twitter.common.concurrent import deadline, defer, Timeout
-from twitter.common.metrics import Observable
-from twitter.common.quantity import Amount, Time
-
-from .common.kill_manager import KillManager
-from .common.sandbox import DirectorySandbox, SandboxProvider
-from .common.status_checker import ChainedStatusChecker, ExitState
-from .common.task_info import (
- assigned_task_from_mesos_task,
- mesos_task_instance_from_assigned_task,
- resolve_ports,
-)
-from .common.task_runner import (
- TaskError,
- TaskRunner,
- TaskRunnerProvider,
-)
-from .executor_base import ThermosExecutorBase
-from .status_manager import StatusManager
-
-import mesos_pb2 as mesos_pb
-
-
-class DefaultSandboxProvider(SandboxProvider):
- SANDBOX_NAME = 'sandbox'
-
- def from_assigned_task(self, assigned_task):
- mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
- return DirectorySandbox(
- os.path.realpath(self.SANDBOX_NAME),
- mesos_task.role().get())
-
-
-class ThermosExecutor(Observable, ThermosExecutorBase):
- PERSISTENCE_WAIT = Amount(5, Time.SECONDS)
- SANDBOX_INITIALIZATION_TIMEOUT = Amount(10, Time.MINUTES)
- START_TIMEOUT = Amount(2, Time.MINUTES)
- STOP_TIMEOUT = Amount(2, Time.MINUTES)
- STOP_WAIT = Amount(5, Time.SECONDS)
-
- def __init__(self,
- runner_provider,
- status_manager_class=StatusManager,
- sandbox_provider=DefaultSandboxProvider,
- status_providers=(),
- clock=time):
-
- ThermosExecutorBase.__init__(self)
- if not isinstance(runner_provider, TaskRunnerProvider):
- raise TypeError('runner_provider must be a TaskRunnerProvider, got %s' %
- type(runner_provider))
- self._runner = None
- self._runner_provider = runner_provider
- self._clock = clock
- self._task_id = None
- self._status_providers = status_providers
- self._status_manager = None
- self._status_manager_class = status_manager_class
- self._sandbox = None
- self._sandbox_provider = sandbox_provider()
- self._kill_manager = KillManager()
- # Events that are exposed for interested entities
- self.runner_aborted = threading.Event()
- self.runner_started = threading.Event()
- self.sandbox_initialized = threading.Event()
- self.sandbox_created = threading.Event()
- self.terminated = threading.Event()
- self.launched = threading.Event()
-
- @property
- def runner(self):
- return self._runner
-
- def _die(self, driver, status, msg):
- log.fatal(msg)
- self.send_update(driver, self._task_id, status, msg)
- defer(driver.stop, delay=self.STOP_WAIT)
-
- def _run(self, driver, assigned_task, mesos_task):
- """
- Commence running a Task.
- - Initialize the sandbox
- - Start the ThermosTaskRunner (fork the Thermos TaskRunner)
- - Set up necessary HealthCheckers
- - Set up DiscoveryManager, if applicable
- - Set up ResourceCheckpointer
- - Set up StatusManager, and attach HealthCheckers
- """
- self.send_update(driver, self._task_id, mesos_pb.TASK_STARTING, 'Initializing sandbox.')
-
- if not self._initialize_sandbox(driver, assigned_task):
- return
-
- # Fully resolve the portmap
- portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
-
- # start the process on a separate thread and give the message processing thread back
- # to the driver
- try:
- self._runner = self._runner_provider.from_assigned_task(assigned_task, self._sandbox)
- except TaskError as e:
- self._die(driver, mesos_pb.TASK_FAILED, str(e))
- return
-
- if not isinstance(self._runner, TaskRunner):
- self._die(driver, mesos_pb.TASK_FAILED, 'Unrecognized task!')
- return
-
- if not self._start_runner(driver, assigned_task, mesos_task, portmap):
- return
-
- self.send_update(driver, self._task_id, mesos_pb.TASK_RUNNING)
-
- self._start_status_manager(driver, assigned_task, mesos_task, portmap)
-
- def _initialize_sandbox(self, driver, assigned_task):
- self._sandbox = self._sandbox_provider.from_assigned_task(assigned_task)
- self.sandbox_initialized.set()
- try:
- deadline(self._sandbox.create, timeout=self.SANDBOX_INITIALIZATION_TIMEOUT,
- daemon=True, propagate=True)
- except Timeout:
- self._die(driver, mesos_pb.TASK_FAILED, 'Timed out waiting for sandbox to initialize!')
- return
- except self._sandbox.Error as e:
- self._die(driver, mesos_pb.TASK_FAILED, 'Failed to initialize sandbox: %s' % e)
- return
- self.sandbox_created.set()
- return True
-
- def _start_runner(self, driver, assigned_task, mesos_task, portmap):
- if self.runner_aborted.is_set():
- self._die(driver, mesos_pb.TASK_KILLED, 'Task killed during initialization.')
-
- try:
- deadline(self._runner.start, timeout=self.START_TIMEOUT, propagate=True)
- except TaskError as e:
- self._die(driver, mesos_pb.TASK_FAILED, 'Task initialization failed: %s' % e)
- return False
- except Timeout:
- self._die(driver, mesos_pb.TASK_LOST, 'Timed out waiting for task to start!')
- return False
-
- self.runner_started.set()
- log.debug('Task started.')
-
- return True
-
- def _start_status_manager(self, driver, assigned_task, mesos_task, portmap):
- status_checkers = [self._kill_manager]
- self.metrics.register_observable('kill_manager', self._kill_manager)
-
- for status_provider in self._status_providers:
- status_checker = status_provider.from_assigned_task(assigned_task, self._sandbox)
- if status_checker is None:
- continue
- status_checkers.append(status_checker)
- # self.metrics.register_observable()
-
- self._chained_checker = ChainedStatusChecker(status_checkers)
- self._chained_checker.start()
-
- # chain the runner to the other checkers, but do not chain .start()/.stop()
- complete_checker = ChainedStatusChecker([self._runner, self._chained_checker])
- self._status_manager = self._status_manager_class(
- complete_checker, self._shutdown, clock=self._clock)
- self._status_manager.start()
-
- def _signal_kill_manager(self, driver, task_id, reason):
- if self._task_id is None:
- log.error('Was asked to kill task but no task running!')
- return
- if task_id != self._task_id:
- log.error('Asked to kill a task other than what we are running!')
- return
- if not self.sandbox_created.is_set():
- log.error('Asked to kill task with incomplete sandbox - aborting runner start')
- self.runner_aborted.set()
- return
- self.log('Activating kill manager.')
- self._kill_manager.kill(reason)
-
- @classmethod
- def translate_exit_state_to_mesos(cls, exit_state):
- # Put into executor_base?
- if exit_state == ExitState.FAILED:
- return mesos_pb.TASK_FAILED
- elif exit_state == ExitState.KILLED:
- return mesos_pb.TASK_KILLED
- elif exit_state == ExitState.FINISHED:
- return mesos_pb.TASK_FINISHED
- elif exit_state == ExitState.LOST:
- return mesos_pb.TASK_LOST
- log.error('Unknown exit state, defaulting to TASK_FINISHED.')
- return mesos_pb.TASK_FINISHED
-
- def _shutdown(self, status_result):
- runner_status = self._runner.status
-
- try:
- deadline(self._runner.stop, timeout=self.STOP_TIMEOUT)
- except Timeout:
- log.error('Failed to stop runner within deadline.')
-
- try:
- deadline(self._chained_checker.stop, timeout=self.STOP_TIMEOUT)
- except Timeout:
- log.error('Failed to stop all checkers within deadline.')
-
- # If the runner was alive when _shutdown was called, defer to the status_result,
- # otherwise the runner's terminal state is the preferred state.
- exit_status = runner_status or status_result
-
- self.send_update(
- self._driver,
- self._task_id,
- self.translate_exit_state_to_mesos(exit_status.status),
- status_result.reason)
-
- self.terminated.set()
- defer(self._driver.stop, delay=self.PERSISTENCE_WAIT)
-
- """ Mesos Executor API methods follow """
-
- def launchTask(self, driver, task):
- """
- Invoked when a task has been launched on this executor (initiated via Scheduler::launchTasks).
- Note that this task can be realized with a thread, a process, or some simple computation,
- however, no other callbacks will be invoked on this executor until this callback has returned.
- """
- self.launched.set()
- self.log('launchTask got task: %s:%s' % (task.name, task.task_id.value))
-
- # TODO(wickman) Update the tests to call registered(), then remove this line and issue
- # an assert if self._driver is not populated.
- self._driver = driver
-
- if self._runner:
- log.error('Already running a task! %s' % self._task_id)
- self.send_update(driver, task.task_id.value, mesos_pb.TASK_LOST,
- "Task already running on this executor: %s" % self._task_id)
- return
-
- self._slave_id = task.slave_id.value
- self._task_id = task.task_id.value
-
- try:
- assigned_task = assigned_task_from_mesos_task(task)
- mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
- except Exception as e:
- log.fatal('Could not deserialize AssignedTask')
- log.fatal(traceback.format_exc())
- self.send_update(
- driver, self._task_id, mesos_pb.TASK_FAILED, "Could not deserialize task: %s" % e)
- defer(driver.stop, delay=self.STOP_WAIT)
- return
-
- defer(lambda: self._run(driver, assigned_task, mesos_task))
-
- def killTask(self, driver, task_id):
- """
- Invoked when a task running within this executor has been killed (via
- SchedulerDriver::killTask). Note that no status update will be sent on behalf of the executor,
- the executor is responsible for creating a new TaskStatus (i.e., with TASK_KILLED) and invoking
- ExecutorDriver::sendStatusUpdate.
- """
- self.log('killTask got task_id: %s' % task_id)
- self._signal_kill_manager(driver, task_id.value, "Instructed to kill task.")
- self.log('killTask returned.')
-
- def shutdown(self, driver):
- """
- Invoked when the executor should terminate all of its currently running tasks. Note that after
- Mesos has determined that an executor has terminated any tasks that the executor did not send
- terminal status updates for (e.g., TASK_KILLED, TASK_FINISHED, TASK_FAILED, etc) a TASK_LOST
- status update will be created.
-
- """
- self.log('shutdown called')
- if self._task_id:
- self.log('shutting down %s' % self._task_id)
- self._signal_kill_manager(driver, self._task_id, "Told to shut down executor.")
- self.log('shutdown returned')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/thermos_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/thermos_runner.py b/src/main/python/twitter/aurora/executor/thermos_runner.py
deleted file mode 100644
index 0c9af05..0000000
--- a/src/main/python/twitter/aurora/executor/thermos_runner.py
+++ /dev/null
@@ -1,138 +0,0 @@
-import functools
-import os
-import signal
-import sys
-import traceback
-
-from twitter.common import app, log
-from twitter.thermos.common.options import add_port_to
-from twitter.thermos.common.planner import TaskPlanner
-from twitter.thermos.config.loader import ThermosConfigLoader
-from twitter.thermos.core.runner import TaskRunner
-
-
-app.add_option(
- "--thermos_json",
- dest="thermos_json",
- default=None,
- help="read a thermos Task from a serialized json blob")
-
-
-app.add_option(
- "--sandbox",
- dest="sandbox",
- metavar="PATH",
- default=None,
- help="the sandbox in which this task should run")
-
-
-app.add_option(
- "--checkpoint_root",
- dest="checkpoint_root",
- metavar="PATH",
- default=None,
- help="the path where we will store checkpoints")
-
-
-app.add_option(
- "--task_id",
- dest="task_id",
- metavar="STRING",
- default=None,
- help="The id to which this task should be bound, created if it does not exist.")
-
-
-app.add_option(
- "--setuid",
- dest="setuid",
- metavar="USER",
- default=None,
- help="setuid tasks to this user, requires superuser privileges.")
-
-
-app.add_option(
- "--enable_chroot",
- dest="chroot",
- default=False,
- action='store_true',
- help="chroot tasks to the sandbox before executing them.")
-
-
-app.add_option(
- "--port",
- type='string',
- nargs=1,
- action='callback',
- callback=add_port_to('prebound_ports'),
- dest='prebound_ports',
- default={},
- metavar = "NAME:PORT",
- help="bind a numbered port PORT to name NAME")
-
-
-def get_task_from_options(opts):
- tasks = ThermosConfigLoader.load_json(opts.thermos_json)
- if len(tasks.tasks()) == 0:
- app.error("No tasks specified!")
- if len(tasks.tasks()) > 1:
- app.error("Multiple tasks in config but no task name specified!")
- task = tasks.tasks()[0]
- if not task.task.check().ok():
- app.error(task.task.check().message())
- return task
-
-
-def runner_teardown(runner, sig=signal.SIGUSR1, frame=None):
- """Destroy runner on SIGUSR1 (kill) or SIGUSR2 (lose)"""
- op = 'kill' if sig == signal.SIGUSR1 else 'lose'
- log.info('Thermos runner got signal %s, shutting down.' % sig)
- log.info('Interrupted frame:')
- if frame:
- for line in ''.join(traceback.format_stack(frame)).splitlines():
- log.info(line)
- runner.close_ckpt()
- log.info('Calling runner.%s()' % op)
- getattr(runner, op)()
- sys.exit(0)
-
-
-class CappedTaskPlanner(TaskPlanner):
- TOTAL_RUN_LIMIT = 100
-
-
-def proxy_main(args, opts):
- assert opts.thermos_json and os.path.exists(opts.thermos_json)
- assert opts.sandbox
- assert opts.checkpoint_root
-
- thermos_task = get_task_from_options(opts)
- prebound_ports = opts.prebound_ports
- missing_ports = set(thermos_task.ports()) - set(prebound_ports)
-
- if missing_ports:
- app.error('ERROR! Unbound ports: %s' % ' '.join(port for port in missing_ports))
-
- task_runner = TaskRunner(
- thermos_task.task,
- opts.checkpoint_root,
- opts.sandbox,
- task_id=opts.task_id,
- user=opts.setuid,
- portmap=prebound_ports,
- chroot=opts.chroot,
- planner_class=CappedTaskPlanner
- )
-
- for sig in (signal.SIGUSR1, signal.SIGUSR2):
- signal.signal(sig, functools.partial(runner_teardown, task_runner))
-
- try:
- task_runner.run()
- except TaskRunner.InternalError as err:
- app.error('Internal error: %s' % err)
- except TaskRunner.InvalidTask as err:
- app.error(str(err))
- except TaskRunner.StateError:
- app.error('Task appears to already be in a terminal state.')
- except KeyboardInterrupt:
- runner_teardown(task_runner)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/thermos_task_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/thermos_task_runner.py b/src/main/python/twitter/aurora/executor/thermos_task_runner.py
deleted file mode 100644
index dba99d2..0000000
--- a/src/main/python/twitter/aurora/executor/thermos_task_runner.py
+++ /dev/null
@@ -1,331 +0,0 @@
-import errno
-import getpass
-import os
-import signal
-import subprocess
-import threading
-import time
-
-from twitter.aurora.common.http_signaler import HttpSignaler
-from twitter.common import log
-from twitter.common.dirutil import chmod_plus_x, safe_mkdtemp
-from twitter.common.log.options import LogOptions
-from twitter.common.quantity import Amount, Time
-from twitter.thermos.common.path import TaskPath
-from twitter.thermos.config.loader import ThermosTaskWrapper
-from twitter.thermos.core import runner as core
-from twitter.thermos.monitoring.monitor import TaskMonitor
-
-from gen.twitter.thermos.ttypes import TaskState
-
-from .common.status_checker import ExitState, StatusResult
-from .common.task_info import (
- mesos_task_instance_from_assigned_task,
- resolve_ports,
-)
-from .common.task_runner import (
- TaskError,
- TaskRunner,
- TaskRunnerProvider,
-)
-
-
-class ThermosTaskRunner(TaskRunner):
- ESCALATION_WAIT = Amount(5, Time.SECONDS)
- EXIT_STATE_MAP = {
- TaskState.ACTIVE: StatusResult('Runner died while task was active.', ExitState.LOST),
- TaskState.FAILED: StatusResult('Task failed.', ExitState.FAILED),
- TaskState.KILLED: StatusResult('Task killed.', ExitState.KILLED),
- TaskState.LOST: StatusResult('Task lost.', ExitState.LOST),
- TaskState.SUCCESS: StatusResult('Task finished.', ExitState.FINISHED),
- }
- MAX_WAIT = Amount(1, Time.MINUTES)
- PEX_NAME = 'thermos_runner.pex'
- POLL_INTERVAL = Amount(500, Time.MILLISECONDS)
- THERMOS_PREEMPTION_WAIT = Amount(1, Time.MINUTES)
-
- def __init__(self,
- runner_pex,
- task_id,
- task,
- role,
- portmap,
- sandbox,
- checkpoint_root=None,
- artifact_dir=None,
- clock=time):
- """
- runner_pex location of the thermos_runner pex that this task runner should use
- task_id task_id assigned by scheduler
- task thermos pystachio Task object
- role role to run the task under
- portmap { name => port } dictionary
- sandbox the sandbox object
- checkpoint_root the checkpoint root for the thermos runner
- artifact_dir scratch space for the thermos runner (basically cwd of thermos.pex)
- clock clock
- """
- self._runner_pex = runner_pex
- self._task_id = task_id
- self._task = task
- self._popen = None
- self._monitor = None
- self._status = None
- self._ports = portmap
- self._root = sandbox.root
- self._checkpoint_root = checkpoint_root or TaskPath.DEFAULT_CHECKPOINT_ROOT
- self._enable_chroot = sandbox.chrooted
- self._role = role
- self._clock = clock
- self._artifact_dir = artifact_dir or safe_mkdtemp()
-
- # wait events
- self._dead = threading.Event()
- self._kill_signal = threading.Event()
- self.forking = threading.Event()
- self.forked = threading.Event()
-
- try:
- with open(os.path.join(self._artifact_dir, 'task.json'), 'w') as fp:
- self._task_filename = fp.name
- ThermosTaskWrapper(self._task).to_file(self._task_filename)
- except ThermosTaskWrapper.InvalidTask as e:
- raise TaskError('Failed to load task: %s' % e)
-
- def _terminate_http(self):
- if 'health' not in self._ports:
- return
-
- http_signaler = HttpSignaler(self._ports['health'])
-
- # pass 1
- http_signaler.quitquitquit()
- self._clock.sleep(self.ESCALATION_WAIT.as_(Time.SECONDS))
- if self.status is not None:
- return True
-
- # pass 2
- http_signaler.abortabortabort()
- self._clock.sleep(self.ESCALATION_WAIT.as_(Time.SECONDS))
- if self.status is not None:
- return True
-
- @property
- def artifact_dir(self):
- return self._artifact_dir
-
- def task_state(self):
- return self._monitor.task_state() if self._monitor else None
-
- @property
- def is_alive(self):
- """
- Is the process underlying the Thermos task runner alive?
- """
- if not self._popen:
- return False
- if self._dead.is_set():
- return False
-
- # N.B. You cannot mix this code and any code that relies upon os.wait
- # mechanisms with blanket child process collection. One example is the
- # Thermos task runner which calls os.wait4 -- without refactoring, you
- # should not mix a Thermos task runner in the same process as this
- # thread.
- try:
- pid, _ = os.waitpid(self._popen.pid, os.WNOHANG)
- if pid == 0:
- return True
- else:
- log.info('Detected runner termination: pid=%s' % pid)
- except OSError as e:
- log.error('is_alive got OSError: %s' % e)
- if e.errno != errno.ECHILD:
- raise
-
- self._dead.set()
- return False
-
- def compute_status(self):
- if self.is_alive:
- return None
- exit_state = self.EXIT_STATE_MAP.get(self.task_state())
- if exit_state is None:
- log.error('Received unexpected exit state from TaskMonitor.')
- return exit_state
-
- def terminate_runner(self, as_loss=False):
- """
- Terminate the underlying runner process, if it exists.
- """
- if self._kill_signal.is_set():
- log.warning('Duplicate kill/lose signal received, ignoring.')
- return
- self._kill_signal.set()
- if self.is_alive:
- sig = 'SIGUSR2' if as_loss else 'SIGUSR1'
- log.info('Runner is alive, sending %s' % sig)
- try:
- self._popen.send_signal(getattr(signal, sig))
- except OSError as e:
- log.error('Got OSError sending %s: %s' % (sig, e))
- else:
- log.info('Runner is dead, skipping kill.')
-
- def kill(self):
- self.terminate_runner()
-
- def lose(self):
- self.terminate_runner(as_loss=True)
-
- def quitquitquit(self):
- """Bind to the process tree of a Thermos task and kill it with impunity."""
- try:
- runner = core.TaskRunner.get(self._task_id, self._checkpoint_root)
- if runner:
- log.info('quitquitquit calling runner.kill')
- # Right now preemption wait is hardcoded, though it may become configurable in the future.
- runner.kill(force=True, preemption_wait=self.THERMOS_PREEMPTION_WAIT)
- else:
- log.error('Could not instantiate runner!')
- except core.TaskRunner.Error as e:
- log.error('Could not quitquitquit runner: %s' % e)
-
- def _cmdline(self):
- params = dict(log_dir=LogOptions.log_dir(),
- log_to_disk='DEBUG',
- checkpoint_root=self._checkpoint_root,
- sandbox=self._root,
- task_id=self._task_id,
- thermos_json=self._task_filename)
-
- if getpass.getuser() == 'root':
- params.update(setuid=self._role)
-
- cmdline_args = [self._runner_pex]
- cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items())
- if self._enable_chroot:
- cmdline_args.extend(['--enable_chroot'])
- for name, port in self._ports.items():
- cmdline_args.extend(['--port=%s:%s' % (name, port)])
- return cmdline_args
-
- # --- public interface
- def start(self, timeout=MAX_WAIT):
- """Fork the task runner and return once the underlying task is running, up to timeout."""
- self.forking.set()
-
- try:
- chmod_plus_x(self._runner_pex)
- except OSError as e:
- if e.errno != errno.EPERM:
- raise TaskError('Failed to chmod +x runner: %s' % e)
-
- self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root), self._task_id)
-
- cmdline_args = self._cmdline()
- log.info('Forking off runner with cmdline: %s' % ' '.join(cmdline_args))
-
- try:
- self._popen = subprocess.Popen(cmdline_args)
- except OSError as e:
- raise TaskError(e)
-
- self.forked.set()
-
- log.debug('Waiting for task to start.')
-
- def is_started():
- return self._monitor and (self._monitor.active or self._monitor.finished)
-
- waited = Amount(0, Time.SECONDS)
- while not is_started() and waited < timeout:
- log.debug(' - sleeping...')
- self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
- waited += self.POLL_INTERVAL
-
- if not is_started():
- log.error('Task did not start with in deadline, forcing loss.')
- self.lose()
- raise TaskError('Task did not start within deadline.')
-
- def stop(self, timeout=MAX_WAIT):
- """Stop the runner. If it's already completed, no-op. If it's still running, issue a kill."""
- log.info('ThermosTaskRunner is shutting down.')
-
- if not self.forking.is_set():
- raise TaskError('Failed to call TaskRunner.start.')
-
- log.info('Invoking runner HTTP teardown.')
- self._terminate_http()
-
- log.info('Invoking runner.kill')
- self.kill()
-
- waited = Amount(0, Time.SECONDS)
- while self.is_alive and waited < timeout:
- self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
- waited += self.POLL_INTERVAL
-
- if not self.is_alive and self.task_state() != TaskState.ACTIVE:
- return
-
- log.info('Thermos task did not shut down cleanly, rebinding to kill.')
- self.quitquitquit()
-
- while not self._monitor.finished and waited < timeout:
- self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
- waited += self.POLL_INTERVAL
-
- if not self._monitor.finished:
- raise TaskError('Task did not stop within deadline.')
-
- @property
- def status(self):
- """Return the StatusResult of this task runner. This returns None as
- long as no terminal state is reached."""
- if self._status is None:
- self._status = self.compute_status()
- return self._status
-
-
-class DefaultThermosTaskRunnerProvider(TaskRunnerProvider):
- def __init__(self,
- pex_location,
- checkpoint_root=None,
- artifact_dir=None,
- task_runner_class=ThermosTaskRunner,
- max_wait=Amount(1, Time.MINUTES),
- preemption_wait=Amount(1, Time.MINUTES),
- poll_interval=Amount(500, Time.MILLISECONDS),
- clock=time):
- self._artifact_dir = artifact_dir or safe_mkdtemp()
- self._checkpoint_root = checkpoint_root
- self._clock = clock
- self._max_wait = max_wait
- self._pex_location = pex_location
- self._poll_interval = poll_interval
- self._preemption_wait = preemption_wait
- self._task_runner_class = task_runner_class
-
- def from_assigned_task(self, assigned_task, sandbox):
- task_id = assigned_task.taskId
- role = assigned_task.task.owner.role
- mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
- mesos_ports = resolve_ports(mesos_task, assigned_task.assignedPorts)
-
- class ProvidedThermosTaskRunner(self._task_runner_class):
- MAX_WAIT = self._max_wait
- POLL_INTERVAL = self._poll_interval
- THERMOS_PREEMPTION_WAIT = self._preemption_wait
-
- return ProvidedThermosTaskRunner(
- self._pex_location,
- task_id,
- mesos_task.task(),
- role,
- mesos_ports,
- sandbox,
- checkpoint_root=self._checkpoint_root,
- artifact_dir=self._artifact_dir,
- clock=self._clock)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/tools/java/organize_imports.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/tools/java/organize_imports.py b/src/main/python/twitter/aurora/tools/java/organize_imports.py
deleted file mode 100644
index 48a0821..0000000
--- a/src/main/python/twitter/aurora/tools/java/organize_imports.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/bin/env python
-'''
-Organizes a java source file's import statements in a way that pleases Twitter's checkstyle
-configuration.
-This expects exactly one argument: the name of the file to modify with preferred import
-ordering.
-'''
-
-from __future__ import print_function
-
-import re
-import sys
-from collections import defaultdict
-
-
-IMPORT_RE = re.compile('import(?: static)? (.*);')
-def get_group(import_statement):
- matcher = IMPORT_RE.match(import_statement)
- assert matcher, 'Could not parse import statement: %s' % import_statement
- class_name = matcher.group(1)
- if class_name.startswith('com.twitter'):
- group = 'com.twitter'
- else:
- group = class_name.split('.')[0]
- return group
-
-
-def index_by_group(import_statements):
- groups = defaultdict(list)
- for line in import_statements:
- groups[get_group(line)].append(line)
- return groups
-
-
-IMPORT_CLASS_RE = re.compile(
- 'import(?: static)? (?P<outer>[^A-Z]*[A-Z]\w+)(?:\.(?P<inners>[\w][^;]*))?')
-def get_all_group_lines(import_groups):
- if not import_groups:
- return []
-
- def get_group_lines(group):
- def comparator(x, y):
- # These shenanigans are used to properly order imports for inner classes.
- # So we get ordering like:
- # import com.foo.Bar;
- # import com.foo.Bar.Baz;
- # (this is not lexicographical, so normal sort won't suffice)
- x_m = IMPORT_CLASS_RE.match(x)
- y_m = IMPORT_CLASS_RE.match(y)
- if x_m.group('outer') == y_m.group('outer'):
- return cmp(x_m.group('inners'), y_m.group('inners'))
- else:
- return cmp(x, y)
- lines = sorted(import_groups[group], comparator)
- lines.append('')
- return lines
-
- all_lines = []
- explicit_groups = ['java', 'javax', 'scala', 'com', 'net', 'org', 'com.twitter']
- for group in explicit_groups:
- if group in import_groups:
- all_lines += get_group_lines(group)
-
- # Gather remaining groups.
- remaining_groups = sorted(set(import_groups.keys()) - set(explicit_groups))
- for group in remaining_groups:
- all_lines += get_group_lines(group)
- return all_lines
-
-
-if len(sys.argv) != 2:
- print('usage: %s FILE' % sys.argv[0])
- sys.exit(1)
-
-BEFORE_IMPORTS = 'before_imports'
-IMPORTS = 'imports'
-STATIC_IMPORTS = 'static_imports'
-AFTER_IMPORTS = 'after_imports'
-
-print('Organizing imports in %s' % sys.argv[1])
-lines_before_imports = []
-import_lines = []
-static_import_lines = []
-lines_after_imports = []
-with open(sys.argv[1], 'r') as f:
- position = BEFORE_IMPORTS
- for line in f:
- line = line.rstrip()
- if position == BEFORE_IMPORTS:
- if line.startswith('import'):
- position = IMPORTS
- else:
- lines_before_imports.append(line)
- if position == IMPORTS:
- if line.startswith('import static'):
- position = STATIC_IMPORTS
- elif line.startswith('import'):
- import_lines.append(line)
- elif line.strip():
- position = AFTER_IMPORTS
- if position == STATIC_IMPORTS:
- if line.startswith('import static'):
- static_import_lines.append(line)
- elif line.strip():
- position = AFTER_IMPORTS
- if position == AFTER_IMPORTS:
- lines_after_imports.append(line)
-
-import_groups = index_by_group(import_lines)
-static_import_groups = index_by_group(static_import_lines)
-
-def ensure_line_padding(lines):
- if lines and lines[-1] != '':
- lines.append('')
- return lines
-
-file_lines = lines_before_imports
-if import_groups:
- ensure_line_padding(file_lines)
- file_lines += get_all_group_lines(import_groups)
-if static_import_groups:
- ensure_line_padding(file_lines)
- file_lines += get_all_group_lines(static_import_groups)
-if lines_after_imports:
- ensure_line_padding(file_lines)
- file_lines += lines_after_imports
-
-with open(sys.argv[1], 'w') as f:
- for line in file_lines:
- print(line, file=f)
[04/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/assets/observer.js
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/assets/observer.js b/src/main/python/twitter/thermos/observer/http/assets/observer.js
deleted file mode 100644
index 84aec5d..0000000
--- a/src/main/python/twitter/thermos/observer/http/assets/observer.js
+++ /dev/null
@@ -1,202 +0,0 @@
-var Task = new Class({
- data: {},
-
- initialize: function(taskId, taskType) {
- this.taskType = taskType
- this.taskId = taskId
- this.visible = false
- this.setElement()
- },
-
- // task[active.task_id.processes.waiting]
- taskStr: function(components) {
- return 'task[' + ([this.taskType, this.taskId].append(components).join()) + ']'
- },
-
- transitionElement: function(dom_id, newval) {
- if (!$(dom_id)) {
- // alert('could not find dom_id' + dom_id)
- return
- }
-
- curval = $(dom_id).innerHTML
-
- // handle specialcases
- newval = this.translateElement(dom_id, newval)
-
- if (newval != curval) {
- $(dom_id).innerHTML = newval
- var morphElement = new Fx.Morph(dom_id, {
- duration: 'long',
- transition: Fx.Transitions.Sine.easeOut
- });
- morphElement.start({ 'color': ["#00FF00", "#222222"]})
- }
- },
-
- applyUpdate: function(update) {
- this.data = update
- this.updateElement()
- },
-
- /*
- TODO(wickman) Instead do the 2-3 necessary transition functions, then make
- a schema that maps attributes to the appropriate transition function.
- */
- translateElement: function(dom_id, value) {
- if (dom_id === this.taskStr(['processes', 'running']) ||
- dom_id === this.taskStr(['processes', 'waiting']) ||
- dom_id === this.taskStr(['processes', 'success']) ||
- dom_id === this.taskStr(['processes', 'failed'])) {
- return value.length
- }
-
- if (dom_id == this.taskStr(['resource_consumption', 'cpu'])) {
- if (value) {
- return value.toFixed(2)
- }
- }
-
- return value
- },
-
- /*
- TODO(wickman) Ditto to above.
- */
- updateElement: function() {
- for (prefix_attr in this.data) {
- if (this.data.hasOwnProperty(prefix_attr)) {
- if (prefix_attr == "task_id") {
- this.transitionElement(this.taskStr([prefix_attr]), this.translateUid(this.data[prefix_attr]))
- } else if (instanceOf(this.data[prefix_attr], String) || instanceOf(this.data[prefix_attr], Number)) {
- this.transitionElement(this.taskStr([prefix_attr]), this.data[prefix_attr])
- } else if (instanceOf(this.data[prefix_attr], Object)) {
- for (suffix_attr in this.data[prefix_attr])
- if (this.data[prefix_attr].hasOwnProperty(suffix_attr))
- this.transitionElement(this.taskStr([prefix_attr, suffix_attr]),
- this.data[prefix_attr][suffix_attr])
- }
- }
- }
- },
-
- translateUid: function(taskId) {
- return "<a href='/task/" + taskId + "'>" + taskId + "</a>"
- },
-
- setElement: function() {
- this.element = new Element('tr', {
- 'id': 'task[' + this.taskType + '][' + this.taskId + ']'}).adopt(
- new Element('td', { 'id': this.taskStr(['task_id']), 'html': this.translateUid(this.taskId)}),
- new Element('td', { 'id': this.taskStr(['name']) }),
- new Element('td', { 'id': this.taskStr(['resource_consumption', 'cpu']) }),
- new Element('td', { 'id': this.taskStr(['resource_consumption', 'ram']) }),
- new Element('td', { 'id': this.taskStr(['resource_consumption', 'disk']) }),
- new Element('td', { 'id': this.taskStr(['processes', 'waiting']) }),
- new Element('td', { 'id': this.taskStr(['processes', 'running']) }),
- new Element('td', { 'id': this.taskStr(['processes', 'success']) }),
- new Element('td', { 'id': this.taskStr(['processes', 'failed']) }))
- }
-})
-
-var TableManager = new Class({
- activeTasks: {},
- visibleChildren: [],
-
- initialize: function(tableType) {
- this.tableType = tableType
- this.setElement()
- this.getChildren()
- this.startPolling()
- },
-
- setElement: function() {
- this.element = new Element('table', { 'id': 'table[' + this.tableType + ']', 'class': 'common-table zebra-striped', 'cellpadding': '0' })
- this.element.adopt(
- new Element('thead').
- adopt(new Element('tr', { 'id': 'task[' + this.tableType + '][superheader]', 'class': 'meta-headers' })
- .adopt(new Element('th', { 'html': "", 'colspan': 2 }),
- new Element('th', { 'html': "consumed", 'colspan': 3 }),
- new Element('th', { 'html': "processes", 'colspan': 4 })
- )
- )
- .adopt(new Element('tr', { 'id': 'task[' + this.tableType + '][header]' })
- .adopt(new Element('th', { 'id': 'task[' + this.tableType + '][header][task_id]', 'html': "task_id" }),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][name]', 'html': "name" }),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][resource_consumption][cpu]', 'html': "cpu" }),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][resource_consumption][ram]', 'html': "ram" }),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][resource_consumption][disk]','html': "disk" }),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][waiting]', 'html': "waiting" }),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][running]', 'html': "running"}),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][success]', 'html': "success" }),
- new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][failed]', 'html': "failed" })
- )
- )
- )
- this.tbody = new Element('tbody', { 'id': this.tableType + '_tbody' })
- this.element.adopt(this.tbody)
- },
-
- toElement: function() {
- return this.element
- },
-
- startPolling: function() {
- this.taskIdPoller = setInterval(this.tableType + 'TableManager.getChildren();', 2500)
- this.dataPoller = setInterval(this.tableType + 'TableManager.refreshVisibleChildren();', 5300)
- },
-
- getChildren: function() {
- new Request.JSON({
- 'url': '/j/task_ids/' + this.tableType + '/-20',
- 'method': 'get',
- 'onComplete': function(response) {
- if (response) {
- var newChildren = Array.from(response.task_ids)
- this.visibleChildren = newChildren
-
- // first set all children to invisible
- for (taskId in this.activeTasks) {
- this.activeTasks[taskId].visible = false
- }
-
- // set new children visible
- for (var k = 0; k < newChildren.length; k++) {
- if (!(newChildren[k] in this.activeTasks)) {
- this.activeTasks[newChildren[k]] = new Task(newChildren[k], this.tableType)
- }
- this.activeTasks[newChildren[k]].visible = true
- }
-
- // clear then adopt them
- while (this.tbody.childNodes.length > 0)
- this.tbody.removeChild(this.tbody.firstChild);
- for (var k = 0; k < newChildren.length; k++)
- this.tbody.adopt(this.activeTasks[newChildren[k]].element)
- } else {
- clearInterval(this.taskIdPoller)
- }
- }.bind(this)
- }).send()
- },
-
- refreshVisibleChildren: function() {
- // first get visible children
- new Request.JSON({
- 'url': '/j/task',
- 'method': 'get',
- 'data': { 'task_id': this.visibleChildren.join() },
- 'onComplete': function(response) {
- if (response) {
- if (!response) return;
- for (taskId in response) {
- if (response.hasOwnProperty(taskId))
- this.activeTasks[taskId].applyUpdate(response[taskId])
- }
- } else {
- clearInterval(this.dataPoller)
- }
- }.bind(this)
- }).send()
- },
-})
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/file_browser.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/file_browser.py b/src/main/python/twitter/thermos/observer/http/file_browser.py
deleted file mode 100644
index 45c5708..0000000
--- a/src/main/python/twitter/thermos/observer/http/file_browser.py
+++ /dev/null
@@ -1,124 +0,0 @@
-import os
-import pprint
-from xml.sax.saxutils import escape
-
-from twitter.common import log
-from twitter.common.http import HttpServer
-
-import bottle
-from mako.template import Template
-
-from .templating import HttpTemplate
-
-
-MB = 1024 * 1024
-DEFAULT_CHUNK_LENGTH = MB
-MAX_CHUNK_LENGTH = 16 * MB
-
-
-def _read_chunk(filename, offset=None, length=None):
- offset = offset or -1
- length = length or -1
-
- try:
- length = long(length)
- offset = long(offset)
- except ValueError:
- return {}
-
- if not os.path.isfile(filename):
- return {}
-
- try:
- fstat = os.stat(filename)
- except Exception as e:
- log.error('Could not read from %s: %s' % (filename, e))
- return {}
-
- if offset == -1:
- offset = fstat.st_size
-
- if length == -1:
- length = fstat.st_size - offset
-
- with open(filename, "r") as fp:
- fp.seek(offset)
- try:
- data = fp.read(length)
- except IOError as e:
- log.error('Failed to read %s: %s' % (filename, e), exc_info=True)
- return {}
-
- if data:
- return dict(offset=offset, length=len(data), data=escape(data.decode('utf8', 'replace')))
-
- return dict(offset=offset, length=0)
-
-
-class TaskObserverFileBrowser(object):
- """
- Mixin for Thermos observer File browser.
- """
-
- @HttpServer.route("/logs/:task_id/:process/:run/:logtype")
- @HttpServer.mako_view(HttpTemplate.load('logbrowse'))
- def handle_logs(self, task_id, process, run, logtype):
- types = self._observer.logs(task_id, process, int(run))
- if logtype not in types:
- bottle.abort(404, "No such log type: %s" % logtype)
- base, path = types[logtype]
- filename = os.path.join(base, path)
- return {
- 'task_id': task_id,
- 'filename': filename,
- 'process': process,
- 'run': run,
- 'logtype': logtype
- }
-
- @HttpServer.route("/logdata/:task_id/:process/:run/:logtype")
- def handle_logdata(self, task_id, process, run, logtype):
- offset = self.Request.GET.get('offset', -1)
- length = self.Request.GET.get('length', -1)
- types = self._observer.logs(task_id, process, int(run))
- if logtype not in types:
- return {}
- chroot, path = types[logtype]
- return _read_chunk(os.path.join(chroot, path), offset, length)
-
- @HttpServer.route("/file/:task_id/:path#.+#")
- @HttpServer.mako_view(HttpTemplate.load('filebrowse'))
- def handle_file(self, task_id, path):
- if path is None:
- bottle.abort(404, "No such file")
- return {
- 'task_id': task_id,
- 'filename': path,
- }
-
- @HttpServer.route("/filedata/:task_id/:path#.+#")
- def handle_filedata(self, task_id, path):
- if path is None:
- return {}
- offset = self.Request.GET.get('offset', -1)
- length = self.Request.GET.get('length', -1)
- chroot, path = self._observer.valid_file(task_id, path)
- if chroot is None or path is None:
- return {}
- return _read_chunk(os.path.join(chroot, path), offset, length)
-
- @HttpServer.route("/browse/:task_id")
- @HttpServer.route("/browse/:task_id/:path#.*#")
- @HttpServer.mako_view(HttpTemplate.load('filelist'))
- def handle_dir(self, task_id, path=None):
- if path == "":
- path = None
- chroot, path = self._observer.valid_path(task_id, path)
- return dict(task_id=task_id, chroot=chroot, path=path)
-
- @HttpServer.route("/download/:task_id/:path#.+#")
- def handle_download(self, task_id, path=None):
- chroot, path = self._observer.valid_path(task_id, path)
- if path is None:
- bottle.abort(404, "No such file")
- return bottle.static_file(path, root=chroot, download=True)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/http_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/http_observer.py b/src/main/python/twitter/thermos/observer/http/http_observer.py
deleted file mode 100644
index b73f17f..0000000
--- a/src/main/python/twitter/thermos/observer/http/http_observer.py
+++ /dev/null
@@ -1,133 +0,0 @@
-"""HTTP interface to the Thermos TaskObserver
-
-This modules provides an HTTP server which exposes information about Thermos tasks running on a
-system. To do this, it relies heavily on the Thermos TaskObserver.
-
-"""
-
-import os
-import socket
-
-from twitter.common import log
-from twitter.common.http import HttpServer
-
-from .file_browser import TaskObserverFileBrowser
-from .json import TaskObserverJSONBindings
-from .static_assets import StaticAssets
-from .templating import HttpTemplate
-
-
-class BottleObserver(HttpServer, StaticAssets, TaskObserverFileBrowser, TaskObserverJSONBindings):
- """
- A bottle wrapper around a Thermos TaskObserver.
- """
-
- def __init__(self, observer):
- self._observer = observer
- StaticAssets.__init__(self)
- TaskObserverFileBrowser.__init__(self)
- TaskObserverJSONBindings.__init__(self)
- HttpServer.__init__(self)
-
- @HttpServer.route("/")
- @HttpServer.view(HttpTemplate.load('index'))
- def handle_index(self):
- return dict(hostname=socket.gethostname())
-
- @HttpServer.route("/main")
- @HttpServer.route("/main/:type")
- @HttpServer.route("/main/:type/:offset")
- @HttpServer.route("/main/:type/:offset/:num")
- @HttpServer.mako_view(HttpTemplate.load('main'))
- def handle_main(self, type=None, offset=None, num=None):
- if type not in (None, 'all', 'finished', 'active'):
- HttpServer.abort(404, 'Invalid task type: %s' % type)
- if offset is not None:
- try:
- offset = int(offset)
- except ValueError:
- HttpServer.abort(404, 'Invalid offset: %s' % offset)
- if num is not None:
- try:
- num = int(num)
- except ValueError:
- HttpServer.abort(404, 'Invalid count: %s' % num)
- return self._observer.main(type, offset, num)
-
- @HttpServer.route("/task/:task_id")
- @HttpServer.mako_view(HttpTemplate.load('task'))
- def handle_task(self, task_id):
- task = self.get_task(task_id)
- processes = self._observer.processes([task_id])
- if not processes.get(task_id, None):
- HttpServer.abort(404, 'Unknown task_id: %s' % task_id)
- processes = processes[task_id]
- state = self._observer.state(task_id)
-
- return dict(
- task_id = task_id,
- task = task,
- statuses = self._observer.task_statuses(task_id),
- user = task['user'],
- ports = task['ports'],
- processes = processes,
- chroot = state.get('sandbox', ''),
- launch_time = state.get('launch_time', 0),
- hostname = state.get('hostname', 'localhost'),
- )
-
- def get_task(self, task_id):
- task = self._observer._task(task_id)
- if not task:
- HttpServer.abort(404, "Failed to find task %s. Try again shortly." % task_id)
- return task
-
- @HttpServer.route("/rawtask/:task_id")
- @HttpServer.mako_view(HttpTemplate.load('rawtask'))
- def handle_rawtask(self, task_id):
- task = self.get_task(task_id)
- state = self._observer.state(task_id)
- return dict(
- hostname = state.get('hostname', 'localhost'),
- task_id = task_id,
- task_struct = task['task_struct']
- )
-
- @HttpServer.route("/process/:task_id/:process_id")
- @HttpServer.mako_view(HttpTemplate.load('process'))
- def handle_process(self, task_id, process_id):
- all_processes = {}
- current_run = self._observer.process(task_id, process_id)
- if not current_run:
- HttpServer.abort(404, 'Invalid task/process combination: %s/%s' % (task_id, process_id))
- process = self._observer.process_from_name(task_id, process_id)
- if process is None:
- msg = 'Could not recover process: %s/%s' % (task_id, process_id)
- log.error(msg)
- HttpServer.abort(404, msg)
-
- current_run_number = current_run['process_run']
- all_processes[current_run_number] = current_run
- for run in range(current_run_number):
- all_processes[run] = self._observer.process(task_id, process_id, run)
- def convert_process_tuple(run_tuple):
- process_tuple = dict(state = run_tuple['state'])
- if 'start_time' in run_tuple:
- process_tuple.update(start_time = run_tuple['start_time'])
- if 'stop_time' in run_tuple:
- process_tuple.update(stop_time = run_tuple['stop_time'])
- return process_tuple
-
- template = {
- 'task_id': task_id,
- 'process': {
- 'name': process_id,
- 'status': all_processes[current_run_number]["state"],
- 'cmdline': process.cmdline().get()
- },
- }
- template['process'].update(**all_processes[current_run_number].get('used', {}))
- template['runs'] = dict((run, convert_process_tuple(run_tuple))
- for run, run_tuple in all_processes.items())
- log.info('Rendering template is: %s' % template)
- return template
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/json.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/json.py b/src/main/python/twitter/thermos/observer/http/json.py
deleted file mode 100644
index 9212f0e..0000000
--- a/src/main/python/twitter/thermos/observer/http/json.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import urllib
-
-from twitter.common.http import HttpServer
-
-
-class TaskObserverJSONBindings(object):
- """
- Mixin for Thermos observer JSON endpoints.
- """
-
- @HttpServer.route("/j/task_ids")
- @HttpServer.route("/j/task_ids/:which")
- @HttpServer.route("/j/task_ids/:which/:offset")
- @HttpServer.route("/j/task_ids/:which/:offset/:num")
- def handle_task_ids(self, which=None, offset=None, num=None):
- return self._observer.task_ids(
- which,
- int(offset) if offset is not None else 0,
- int(num) if num is not None else 20)
-
- @HttpServer.route("/j/task_id_count")
- def handle_task_id_count(self):
- return self._observer.task_id_count()
-
- @HttpServer.route("/j/task")
- def handle_tasks(self):
- """
- Additional parameters:
- task_id = comma separated list of task_ids.
- """
- task_ids = HttpServer.Request.GET.get('task_id', [])
- if task_ids:
- task_ids = urllib.unquote(task_ids).split(',')
- return self._observer.task(task_ids)
-
- @HttpServer.route("/j/task/:task_id")
- def handle_task(self, task_id):
- return self._observer.task([task_id])
-
- @HttpServer.route("/j/process/:task_id")
- @HttpServer.route("/j/process/:task_id/:process")
- @HttpServer.route("/j/process/:task_id/:process/:run")
- def handle_process(self, task_id, process=None, run=None):
- return self._observer.process(task_id, process, run)
-
- @HttpServer.route("/j/processes")
- def handle_processes(self):
- """
- Additional parameters:
- task_ids = comma separated list of task_ids.
- """
- task_ids = HttpServer.Request.GET.get('task_id', [])
- if task_ids:
- task_ids = urllib.unquote(task_ids).split(',')
- return self._observer.processes(task_ids)
-
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/static_assets.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/static_assets.py b/src/main/python/twitter/thermos/observer/http/static_assets.py
deleted file mode 100644
index cf3ff08..0000000
--- a/src/main/python/twitter/thermos/observer/http/static_assets.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import mimetypes
-import os
-
-from twitter.common import log
-from twitter.common.http.server import HttpServer
-
-from bottle import HTTPResponse
-import pkg_resources
-
-
-class StaticAssets(object):
- """
- Serve the /assets directory.
- """
- def __init__(self):
- self._assets = {}
- self._detect_assets()
-
- def _detect_assets(self):
- log.info('detecting assets...')
- assets = pkg_resources.resource_listdir(__name__, 'assets')
- cached_assets = {}
- for asset in assets:
- log.info(' detected asset: %s' % asset)
- cached_assets[asset] = pkg_resources.resource_string(
- __name__, os.path.join('assets', asset))
- self._assets = cached_assets
-
- @HttpServer.route("/favicon.ico")
- def handle_favicon(self):
- HttpServer.redirect("/assets/favicon.ico")
-
- @HttpServer.route("/assets/:filename")
- def handle_asset(self, filename):
- # TODO(wickman) Add static_content to bottle.
- if filename in self._assets:
- mimetype, encoding = mimetypes.guess_type(filename)
- headers = {}
- if mimetype: headers['Content-Type'] = mimetype
- if encoding: headers['Content-Encoding'] = encoding
- return HTTPResponse(self._assets[filename], header=headers)
- else:
- HttpServer.abort(404, 'Unknown asset: %s' % filename)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/filebrowse.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/filebrowse.tpl b/src/main/python/twitter/thermos/observer/http/templates/filebrowse.tpl
deleted file mode 100644
index 871e31e..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/filebrowse.tpl
+++ /dev/null
@@ -1,81 +0,0 @@
-<%def name="download_link()">
- <a href='/download/${task_id}/${filename}'><font size=1>download</font></a>
-</%def>
-
-<html>
-
-<head>
- <meta charset="utf-8">
- <title></title>
-
- <style type="text/css">
- .log {
- font-family: "Inconsolata", "Monaco", "Courier New", "Courier";
- line-height:14px;
- font-size: 12px;
- }
-
- .invert {
- color: #FFFFFF;
- text-decoration: none;
- background: #000000;
- }
- </style>
-</head>
-
-<link rel="stylesheet"
- type="text/css"
- href="/assets/bootstrap.css"/>
-<style type="text/css">
-div.tight
-{
- height:100%;
- overflow:scroll;
-}
-</style>
-
-<title>file browser ${task_id}</title>
-<body>
- <div> <strong> filename </strong> ${filename} </div>
- <div> <strong> dl </strong> ${download_link()} </div>
- <div style="position: absolute; left: 5px; top: 0px;">
- <p id="indicator" class="log invert"></p>
- </div>
-
- <div id="data" class="log" style="white-space:pre-wrap; background-color:#EEEEEE;"></div>
-</body>
-
-<script src="/assets/jquery.js"></script>
-<script src="/assets/jquery.pailer.js"></script>
-
-<script>
- function resize() {
- var margin_left = parseInt($('body').css('margin-left'));
- var margin_top = parseInt($('body').css('margin-top'));
- var margin_bottom = parseInt($('body').css('margin-bottom'));
- $('#data').width($(window).width() - margin_left);
- $('#data').height($(window).height() - margin_top - margin_bottom);
- }
-
- $(window).resize(resize);
-
- $(document).ready(function() {
- resize();
-
- $('#data').pailer({
- 'read': function(options) {
- var settings = $.extend({
- 'offset': -1,
- 'length': -1
- }, options);
-
- var url = "/filedata/${task_id}/${filename}"
- + '?offset=' + settings.offset
- + '&length=' + settings.length;
- return $.getJSON(url);
- },
- 'indicator': $('#indicator')
- });
- });
-</script>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/filelist.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/filelist.tpl b/src/main/python/twitter/thermos/observer/http/templates/filelist.tpl
deleted file mode 100644
index 12a257e..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/filelist.tpl
+++ /dev/null
@@ -1,141 +0,0 @@
-<%doc>
- Template arguments:
- task_id
- chroot
- path
- dirs
- files
-</%doc>
-
-
-<%!
- import os
-
- from datetime import datetime
- import grp
- import os
- import pwd
- import stat
- import sys
-
- NOW = datetime.now()
-
- def format_mode(sres):
- mode = sres.st_mode
-
- root = (mode & 0700) >> 6
- group = (mode & 0070) >> 3
- user = (mode & 07)
-
- def stat_type(md):
- if stat.S_ISDIR(md):
- return 'd'
- elif stat.S_ISSOCK(md):
- return 's'
- else:
- return '-'
-
- def triple(md):
- return '%c%c%c' % (
- 'r' if md & 0b100 else '-',
- 'w' if md & 0b010 else '-',
- 'x' if md & 0b001 else '-')
-
- return ''.join([stat_type(mode), triple(root), triple(group), triple(user)])
-
- def format_mtime(mtime):
- dt = datetime.fromtimestamp(mtime)
- return '%s %2d %5s' % (dt.strftime('%b'), dt.day,
- dt.year if dt.year != NOW.year else dt.strftime('%H:%M'))
-
- def format_prefix(filename, sres):
- try:
- pwent = pwd.getpwuid(sres.st_uid)
- user = pwent.pw_name
- except KeyError:
- user = sres.st_uid
-
- try:
- grent = grp.getgrgid(sres.st_gid)
- group = grent.gr_name
- except KeyError:
- group = sres.st_gid
-
- return '%s %3d %10s %10s %10d %s' % (
- format_mode(sres),
- sres.st_nlink,
- user,
- group,
- sres.st_size,
- format_mtime(sres.st_mtime),
- )
-%>
-
-<%def name="download_link(filename)"><a href='/download/${task_id}/${os.path.join(path, filename)}'><font size=1>dl</font></a></%def>
-<%def name="directory_link(dirname)"><a href='/browse/${task_id}/${os.path.join(path, dirname)}'>${dirname}</a></%def>
-<%def name="file_link(filename)"><a href='/file/${task_id}/${os.path.join(path, filename)}'>${filename}</a></%def>
-
-<html>
-
-<link rel="stylesheet"
- type="text/css"
- href="/assets/bootstrap.css"/>
-<style type="text/css">
-div.tight
-{
- height:85%;
- overflow:auto;
-}
-</style>
-
-<title>path browser for ${task_id}</title>
-
-
-% if chroot is not None:
-<body>
- <div class="container">
- <div class="span6">
- <strong> task id </strong> ${task_id}
- </div>
- <div class="span6">
- <strong> path </strong> ${path}
- </div>
- <div class="span12 tight">
- <pre>
-
-% if path != ".":
- <%
- listing = ['..'] + os.listdir(os.path.join(chroot, path))
- %>\
-% else:
- <%
- listing = os.listdir(os.path.join(chroot, path))
- %>\
-% endif
-
-<% listing.sort() %>
-
-% for fn in listing:
-<%
- try:
- sres = os.stat(os.path.join(chroot, path, fn))
- except OSError:
- continue
-%>\
- % if not stat.S_ISDIR(sres.st_mode):
-${format_prefix(fn, sres)} ${file_link(fn)} ${download_link(fn)}
- % else:
-${format_prefix(fn, sres)} ${directory_link(fn)}
- % endif
-% endfor
- </pre>
- </div>
- </div>
-</body>
-% else:
-<body>
- This task is running without a chroot.
-</body>
-% endif
-
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/home.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/home.tpl b/src/main/python/twitter/thermos/observer/http/templates/home.tpl
deleted file mode 100644
index 4e7426b..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/home.tpl
+++ /dev/null
@@ -1,58 +0,0 @@
-<html>
-<title>thermos(${hostname})</title>
-
-<link rel="stylesheet"
- type="text/css"
- href="/assets/bootstrap.css"/>
-
-<body>
-
-<%!
- import socket
- import time
-
- def pretty_time(seconds):
- return time.asctime(time.localtime(seconds))
-%>
-
-<div class="container">
- <h3> host ${socket.gethostname()} </h3>
-
- <div class="content" id="defaultLayout">
- <table class="zebra-striped">
- <thead>
- <tr>
- <th colspan=3> task </th>
- <th colspan=4> resources </th>
- <th colspan=3> links </th>
- </tr>
-
- <tr>
- <th> name </th> <th> role </th> <th> status </th>
- <th> procs </th> <th> cpu </th> <th> ram </th> <th> disk </th>
- <th> task </th> <th> chroot </th> <th> ports </th>
- </tr>
- </thead>
- <tbody>
-
- % for proc_name, proc in sorted(processes.items()):
- <tr>
- <td> ${proc["process_name"]} </td>
- <td> ${proc["process_run"]} </td>
- <td> ${proc["state"]} </td>
- <td> ${pretty_time(float(proc["start_time"])/1000.0) if "start_time" in proc else ""} </td>
- <td> ${pretty_time(float(proc["stop_time"])/1000.0) if "stop_time" in proc else ""} </td>
- <td> ${'%.3f' % proc["used"]["cpu"] if "used" in proc else ""} </td>
- <td> ${'%dMB' % (proc["used"]["ram"] / 1024 / 1024) if "used" in proc else ""} </td>
- <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stdout">stdout</a> </td>
- <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stderr">stderr</a> </td>
- </tr>
- % endfor
- </tbody>
- </table>
- </div>
-
-</div>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/index.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/index.tpl b/src/main/python/twitter/thermos/observer/http/templates/index.tpl
deleted file mode 100644
index d662c60..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/index.tpl
+++ /dev/null
@@ -1,51 +0,0 @@
-<html>
-<title>thermos({{hostname}})</title>
-
-<link rel="stylesheet"
- type="text/css"
- href="assets/bootstrap.css"/>
-
-<script src="assets/jquery.js"></script>
-
-<body>
-
-<div class="container" id="defaultLayout">
- <div id="activeTaskContainer" class='uber-container'>
- <div class="active-container" data-url="main/active">
- </div>
- </div>
- <br><br>
- <div id="finishedTaskContainer" class='uber-container'>
- <div class="finished-container" data-url="main/finished">
- </div>
- </div>
-</div>
-
-<script type="text/javascript">
-
-$(document).on('click', 'a.refresh-container', function(e) {
- e.preventDefault()
- topLevelDivContainer = $(this).closest('.uber-container')
- divDataUrl = $(this).attr('data-url')
- $.ajax({
- 'type': 'GET',
- 'dataType': 'html',
- 'url': divDataUrl,
- success: function(data, xhr, err) {
- $(topLevelDivContainer).html(data)
- }
- })
- })
-
-refreshDivs = function() {
- $('#activeTaskContainer').load($('.uber-container .active-container').attr('data-url'))
- $('#finishedTaskContainer').load($('.uber-container .finished-container').attr('data-url'))
-}
-
-$(document).bind('ready', refreshDivs)
-setInterval(refreshDivs, 10000)
-
-</script>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/logbrowse.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/logbrowse.tpl b/src/main/python/twitter/thermos/observer/http/templates/logbrowse.tpl
deleted file mode 100644
index 059979c..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/logbrowse.tpl
+++ /dev/null
@@ -1,80 +0,0 @@
-<%def name="download_link()">
- <a href='/download/${task_id}/${filename}'><font size=1>download</font></a>
-</%def>
-
-<html>
-
-<head>
- <meta charset="utf-8">
- <title></title>
-
- <style type="text/css">
- .log {
- font-family: "Inconsolata", "Monaco", "Courier New", "Courier";
- line-height:14px;
- font-size: 12px;
- }
-
- .invert {
- color: #FFFFFF;
- text-decoration: none;
- background: #000000;
- }
- </style>
-</head>
-
-<link rel="stylesheet"
- type="text/css"
- href="/assets/bootstrap.css"/>
-<style type="text/css">
-div.tight
-{
- height:100%;
- overflow:scroll;
-}
-</style>
-
-<title>log browser ${task_id}</title>
-<body>
- <div> <strong> log </strong> ${logtype} <strong> ${download_link()} </strong> </div>
- <div style="position: absolute; left: 5px; top: 0px;">
- <p id="indicator" class="log invert"></p>
- </div>
-
- <div id="data" class="log" style="white-space:pre-wrap; background-color:#EEEEEE;"></div>
-</body>
-
-<script src="/assets/jquery.js"></script>
-<script src="/assets/jquery.pailer.js"></script>
-
-<script>
- function resize() {
- var margin_left = parseInt($('body').css('margin-left'));
- var margin_top = parseInt($('body').css('margin-top'));
- var margin_bottom = parseInt($('body').css('margin-bottom'));
- $('#data').width($(window).width() - margin_left);
- $('#data').height($(window).height() - margin_top - margin_bottom);
- }
-
- $(window).resize(resize);
-
- $(document).ready(function() {
- resize();
-
- $('#data').pailer({
- 'read': function(options) {
- var settings = $.extend({
- 'offset': -1,
- 'length': -1
- }, options);
-
- var url = "/logdata/${task_id}/${process}/${run}/${logtype}"
- + '?offset=' + settings.offset
- + '&length=' + settings.length;
- return $.getJSON(url);
- },
- 'indicator': $('#indicator')
- });
- });
-</script>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/main.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/main.tpl b/src/main/python/twitter/thermos/observer/http/templates/main.tpl
deleted file mode 100644
index d3419ab..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/main.tpl
+++ /dev/null
@@ -1,101 +0,0 @@
-<%doc>
- Template arguments:
- type
- offset
- num
- tasks
- task_count
-</%doc>
-
-<%!
-import socket
-import time
-
-try:
- from twitter.common import app
- observer_port = app.get_options().twitter_common_http_root_server_port
-except (ImportError, AttributeError) as e:
- observer_port = 1338
-
-host = socket.gethostname()
-num_tasks = 20
-
-def pretty_time(seconds=time.time()):
- return time.strftime('%m/%d %H:%M:%S', time.gmtime(seconds))
-
-%>
-
-<div class="${type}-container"
- data-url="/main/${type}/${offset}/${num}">
- <div class="row-fluid">
- <div class="span2">
- % if offset > 0:
- <a class="refresh-container" href="#"
- data-url="/main/${type}/${offset-num_tasks}/${num}">
- ← newer
- </a>
- % endif
- </div>
- % if offset + num_tasks < task_count:
- <div class="span2">
- <a class="refresh-container" href="#"
- data-url="/main/${type}/${offset+num_tasks}/${num}">
- older →
- </a>
- </div>
- % endif
- </div>
- <div class="content" id="defaultLayout">
- <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
- <thead>
- <tr>
- <th colspan=4> ${type} tasks ${offset}...${min(task_count, offset+num_tasks) - 1} of ${task_count} </th>
- <th colspan=3> resources </th>
- <th colspan=3> links </th>
- </tr>
-
- <tr>
- <th> name </th> <th> role </th> <th> launched </th> <th> status </th>
- <th> cpu </th> <th> ram </th> <th> disk </th>
- <th> task </th> <th> chroot </th> <th> ports </th>
- </tr>
- </thead>
- <tbody>
-
- % for row in tasks:
- <tr>
- <td> ${row["name"]} </td>
- <td> ${row["role"]} </td>
- <td> ${pretty_time(row["launch_timestamp"])} </td>
- <td> ${row["state"]} @
- ${pretty_time(row["state_timestamp"]) if row["state_timestamp"] else ""}</td>
-
- <td> ${'%.3f' % row["cpu"] if row["cpu"] > 0 else ""} </td>
- <td> ${'%.1fMB' % (row["ram"] / 1024. / 1024.) if row["ram"] > 0 else ""} </td>
- <td> ${'%dGB' % (row["disk"] / 1024 / 1024 / 1024) if row["disk"] > 0 else ""} </td>
-
- <td> <a href="/task/${row['task_id']}">info</a> </td>
- <td> <a href="/browse/${row['task_id']}">browse</a> </td>
- <td>
- % if type == 'active':
- % if 'http' in row["ports"]:
- <a href="http://${host}:${row['ports']['http']}">http</a>
- % else:
- <span class="muted">http</span>
- % endif
- % if 'admin' in row["ports"]:
- <a href="http://${host}:${row['ports']['admin']}">admin</a>
- % else:
- <span class="muted">admin</span>
- % endif
- % if set(row["ports"]) - set(['http', 'admin']):
- <a href="/task/${row['task_id']}">...</a>
- % endif
- % endif
- </td>
- </tr>
- % endfor
- </tbody>
- </table>
- </div>
-</div>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/process.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/process.tpl b/src/main/python/twitter/thermos/observer/http/templates/process.tpl
deleted file mode 100644
index a7fb90d..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/process.tpl
+++ /dev/null
@@ -1,109 +0,0 @@
-<%doc>
-Template arguments:
- task_id
- process {
- cpu:/ram: (optional)
- cmdline:
- name:
- }
-
- runs = {
- number: {
- start_time: (optional)
- stop_time: (optional)
- state:
- }
- }
-
- --
- for each run:
- run | state | started | finished | stdout | stderr
-</%doc>
-
-<%!
-import socket
-import time
-from xml.sax.saxutils import escape
-
-def pretty_time(seconds=time.time()):
- return time.strftime('%m/%d %H:%M:%S', time.gmtime(seconds))
-%>
-
-<html>
-<title>thermos(${socket.gethostname()})</title>
-
-<link rel="stylesheet"
- type="text/css"
- href="/assets/bootstrap.css"/>
-<body>
-
-
-<div class="container">
- <div class="row">
- <div class="span6" id="leftBar">
- <dl>
- <dt> process </dt>
- <dd> <strong> parent task </strong> <a href="/task/${task_id}">${task_id}</a> </dd>
- <dd> <strong> process name </strong> ${process["name"]} </dd>
- <dd> <strong> status </strong> ${process["status"]} </dd>
- </dl>
- </div>
-
- <div class="span6" id="rightBar">
- <dl>
- <dt> resources </dt>
- <dd> <strong> cpu </strong> ${'%.3f' % process["cpu"] if "cpu" in process else "N/A"} </dd>
- <dd> <strong> ram </strong> ${'%.1fMB' % (process["ram"] / 1024. / 1024.) if "ram" in process else "N/A"} </dd>
- <dd> <strong> total user </strong> ${'%.1fs' % process["user"] if "user" in process else "N/A"} </dd>
- <dd> <strong> total sys </strong> ${'%.1fs' % process["system"] if "system" in process else "N/A"} </dd>
- <dd> <strong> threads </strong> ${process["threads"] if "threads" in process else "N/A"} </dd>
- </dl>
- </div>
- </div>
-
-
- <strong> cmdline </strong><br>
- <div class="container">
-<pre>
-${escape(process["cmdline"])}
-</pre>
- </div><br><br>
-
-
- <strong> runs </strong>
- <div class="container">
- <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
- <thead>
- <tr>
- <th colspan=2> </th>
- <th colspan=2> time </th>
- <th colspan=2> logs </th>
- </tr>
-
- <tr>
- <th> run </th>
- <th> status </th>
- <th> started </th> <th> finished </th>
- <th> stdout </th> <th> stderr </th>
- </tr>
- </thead>
- <tbody>
-
- % for run, process_dict in sorted(runs.items(), reverse=True):
- <tr>
- <td> ${run} </td>
- <td> ${process_dict["state"]} </td>
- <td> ${pretty_time(process_dict["start_time"]) if "start_time" in process_dict else ""} </td>
- <td> ${pretty_time(process_dict["stop_time"]) if "stop_time" in process_dict else ""} </td>
- <td> <a href="/logs/${task_id}/${process["name"]}/${run}/stdout">stdout</a> </td>
- <td> <a href="/logs/${task_id}/${process["name"]}/${run}/stderr">stderr</a> </td>
- </tr>
- % endfor
- </tbody>
- </table>
- </div>
-
-</div>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/rawtask.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/rawtask.tpl b/src/main/python/twitter/thermos/observer/http/templates/rawtask.tpl
deleted file mode 100644
index c4b7f52..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/rawtask.tpl
+++ /dev/null
@@ -1,28 +0,0 @@
-<%doc>
- Template arguments:
- hostname
- task_id
- task_struct
-</%doc>
-
-<html>
-<title>thermos(${hostname})</title>
-
-<link rel="stylesheet"
- type="text/css"
- href="/assets/bootstrap.css"/>
-<%!
- from json import dumps
- def print_task(task):
- return dumps(task.get(), indent=4)
-%>
-
-<body>
-<div class="container">
- <h3> task ${task_id} </h3>
- <div class="content" id="rawTask">
- <pre>${print_task(task_struct)}</pre>
- </div>
-</div>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templates/task.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templates/task.tpl b/src/main/python/twitter/thermos/observer/http/templates/task.tpl
deleted file mode 100644
index b71c80d..0000000
--- a/src/main/python/twitter/thermos/observer/http/templates/task.tpl
+++ /dev/null
@@ -1,140 +0,0 @@
-<%doc>
- Template arguments:
- task_id
- task
- statuses
- user
- ports
- processes
- chroot
- launch_time
- hostname
-</%doc>
-
-<html>
-<title>thermos(${hostname})</title>
-
-<link rel="stylesheet"
- type="text/css"
- href="/assets/bootstrap.css"/>
-
-<body>
-
-<%!
- import time
-
- def pretty_time(seconds=time.time()):
- return time.strftime('%m/%d %H:%M:%S', time.gmtime(seconds))
-
- def get(task, typ):
- return task['resource_consumption'][typ]
-
-%>
-
-<div class="container">
-
- <h3> task ${task_id} </h3>
-
- <div class="row">
- <div class="span6" id="leftBar">
- <dl>
- <dt> task </dt>
- <dd> <strong> status </strong> ${statuses[-1][0] if statuses else 'UNKNOWN'} </dd>
- <dd> <strong> user </strong> ${user} </dd>
- <dt> ports </dt>
- % for port_name, port_number in ports.items():
- <dd> <strong> ${port_name} </strong> <a href="http://${hostname}:${port_number}">${port_number}</a> </dd>
- %endfor
- </dl>
- </div>
-
- <div class="span6" id="rightBar">
- <dl>
- <dt> header </dt>
- <dd> <strong> chroot </strong> <a href="/browse/${task_id}">browse</a> </dd>
- <dd> <strong> hostname </strong> <a href="/">${hostname}</a> </dd>
- <dd> <strong> launch time </strong> ${pretty_time(launch_time)} </dd>
- <dd> <strong> task config </strong> <a href="/rawtask/${task_id}">view</a> </dd>
- </dl>
- </div>
- </div>
-
- <div class="row-fluid">
- <div class="span8" id="taskLayout">
- <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
- <thead>
- <tr>
- <th colspan=1> task status </th>
- <th colspan=1> time </th>
- </tr>
- </thead>
-
- <tbody>
- % for status, timestamp in sorted(statuses, key=lambda status: status[1]):
- <tr>
- <td> ${status} </td> <td> ${pretty_time(timestamp)} </td>
- </tr>
- % endfor
- </tbody>
- </table>
- </div>
-
- <div class="span4" id="taskResources">
- <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
- <thead>
- <tr>
- <th> cpu </th>
- <th> ram </th>
- <th> disk </th>
- </tr>
- </thead>
-
- <tbody>
- <tr>
- <td> ${'%.3f' % get(task, 'cpu')} </td>
- <td> ${'%.1fMB' % (get(task, 'ram') / 1024. / 1024.)} </td>
- <td> ${'%.1fGB' % (get(task, 'disk') / 1024. / 1024. / 1024.)} </td>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
-
- <div class="content" id="processesLayout">
- <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
- <thead>
- <tr>
- <th colspan=3> process </th>
- <th colspan=2> time </th>
- <th colspan=2> used </th>
- <th colspan=2> logs </th>
- </tr>
-
- <tr>
- <th> name </th> <th> run </th> <th> status </th> <th> started </th> <th> finished </th>
- <th> cpu </th> <th> ram </th>
- <th> stdout </th> <th> stderr </th>
- </tr>
- </thead>
-
- <tbody>
- % for proc_name, proc in sorted(processes.items(), key=lambda item: item[1].get('start_time')):
- <tr>
- <td> <a href="/process/${task_id}/${proc["process_name"]}">${proc["process_name"]}</a> </td>
- <td> ${proc["process_run"]} </td>
- <td> ${proc["state"]} </td>
- <td> ${pretty_time(proc["start_time"]) if "start_time" in proc else ""} </td>
- <td> ${pretty_time(proc["stop_time"]) if "stop_time" in proc else ""} </td>
- <td> ${'%.3f' % proc["used"]["cpu"] if "used" in proc else ""} </td>
- <td> ${'%dMB' % (proc["used"]["ram"] / 1024 / 1024) if "used" in proc else ""} </td>
- <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stdout">stdout</a> </td>
- <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stderr">stderr</a> </td>
- </tr>
- % endfor
- </tbody>
- </table>
- </div>
-</div>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/templating.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/templating.py b/src/main/python/twitter/thermos/observer/http/templating.py
deleted file mode 100644
index 783308b..0000000
--- a/src/main/python/twitter/thermos/observer/http/templating.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import os
-import pkg_resources
-
-
-class HttpTemplate(object):
- @staticmethod
- def load(name):
- return pkg_resources.resource_string(
- __name__, os.path.join('templates', '%s.tpl' % name))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/observed_task.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/observed_task.py b/src/main/python/twitter/thermos/observer/observed_task.py
deleted file mode 100644
index f995998..0000000
--- a/src/main/python/twitter/thermos/observer/observed_task.py
+++ /dev/null
@@ -1,136 +0,0 @@
-from abc import abstractproperty
-import os
-
-from twitter.common import log
-from twitter.common.lang import AbstractClass
-from twitter.thermos.config.loader import ThermosTaskWrapper
-from twitter.thermos.config.schema import ThermosContext
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-
-from pystachio import Environment
-
-
-class ObservedTask(AbstractClass):
- """ Represents a Task being observed """
-
- @classmethod
- def safe_mtime(cls, path):
- try:
- return os.path.getmtime(path)
- except OSError:
- return None
-
- def __init__(self, task_id, pathspec):
- self._task_id = task_id
- self._pathspec = pathspec
- self._mtime = self._get_mtime()
-
- @abstractproperty
- def type(self):
- """Indicates the type of task (active or finished)"""
-
- def _read_task(self, memoized={}):
- """Read the corresponding task from disk and return a ThermosTask. Memoizes already-read tasks.
- """
- if self._task_id not in memoized:
- path = self._pathspec.given(task_id=self._task_id, state=self.type).getpath('task_path')
- if os.path.exists(path):
- task = ThermosTaskWrapper.from_file(path)
- if task is None:
- log.error('Error reading ThermosTask from %s in observer.' % path)
- else:
- context = self.context(self._task_id)
- if not context:
- log.warning('Task not yet available: %s' % self._task_id)
- task = task.task() % Environment(thermos=context)
- memoized[self._task_id] = task
-
- return memoized.get(self._task_id, None)
-
- def _get_mtime(self):
- """Retrieve the mtime of the task's state directory"""
- get_path = lambda state: self._pathspec.given(
- task_id=self._task_id, state=state).getpath('task_path')
- mtime = self.safe_mtime(get_path('active'))
- if mtime is None:
- mtime = self.safe_mtime(get_path('finished'))
- if mtime is None:
- log.error("Couldn't get mtime for task %s!" % self._task_id)
- return mtime
-
- def context(self, task_id):
- state = self.state
- if state.header is None:
- return None
- return ThermosContext(
- ports=state.header.ports if state.header.ports else {},
- task_id=state.header.task_id,
- user=state.header.user,
- )
-
- @property
- def task(self):
- """Return a ThermosTask representing this task"""
- return self._read_task()
-
- @property
- def task_id(self):
- """Return the task's task_id"""
- return self._task_id
-
- @property
- def mtime(self):
- """Return mtime of task file"""
- return self._mtime
-
- @abstractproperty
- def state(self):
- """Return state of task (gen.twitter.thermos.ttypes.RunnerState)"""
-
-
-class ActiveObservedTask(ObservedTask):
- """An active Task known by the TaskObserver"""
-
- def __init__(self, task_id, pathspec, task_monitor, resource_monitor):
- super(ActiveObservedTask, self).__init__(task_id, pathspec)
- self._task_monitor = task_monitor
- self._resource_monitor = resource_monitor
-
- @property
- def type(self):
- return 'active'
-
- @property
- def state(self):
- """Return a RunnerState representing the current state of task, retrieved from TaskMonitor"""
- return self.task_monitor.get_state()
-
- @property
- def task_monitor(self):
- """Return a TaskMonitor monitoring this task"""
- return self._task_monitor
-
- @property
- def resource_monitor(self):
- """Return a ResourceMonitor implementation monitoring this task's resources"""
- return self._resource_monitor
-
-
-class FinishedObservedTask(ObservedTask):
- """A finished Task known by the TaskObserver"""
-
- def __init__(self, task_id, pathspec):
- super(FinishedObservedTask, self).__init__(task_id, pathspec)
- self._state = None
-
- @property
- def type(self):
- return 'finished'
-
- @property
- def state(self):
- """Return final state of Task (RunnerState, read from disk and cached for future access)"""
- if self._state is None:
- path = self._pathspec.given(task_id=self._task_id).getpath('runner_checkpoint')
- self._state = CheckpointDispatcher.from_file(path)
- return self._state
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/task_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/task_observer.py b/src/main/python/twitter/thermos/observer/task_observer.py
deleted file mode 100644
index b6ebcf7..0000000
--- a/src/main/python/twitter/thermos/observer/task_observer.py
+++ /dev/null
@@ -1,640 +0,0 @@
-"""Observe Thermos tasks on a system
-
-This module provides a number of classes for exposing information about running (active) and
-finished Thermos tasks on a system. The primary entry point is the TaskObserver, a thread which
-polls a designated Thermos checkpoint root and collates information about all tasks it discovers.
-
-"""
-from operator import attrgetter
-import os
-import threading
-import time
-
-from twitter.common import log
-from twitter.common.exceptions import ExceptionalThread
-from twitter.common.lang import Lockable
-from twitter.common.quantity import Amount, Time
-
-from twitter.thermos.common.path import TaskPath
-from twitter.thermos.monitoring.detector import TaskDetector
-from twitter.thermos.monitoring.monitor import TaskMonitor
-from twitter.thermos.monitoring.process import ProcessSample
-from twitter.thermos.monitoring.resource import ResourceMonitorBase, TaskResourceMonitor
-
-from gen.twitter.thermos.ttypes import ProcessState, TaskState
-
-from .observed_task import ActiveObservedTask, FinishedObservedTask
-
-
-class TaskObserver(ExceptionalThread, Lockable):
- """
- The TaskObserver monitors the thermos checkpoint root for active/finished
- tasks. It is used to be the oracle of the state of all thermos tasks on
- a machine.
-
- It currently returns JSON, but really should just return objects. We should
- then build an object->json translator.
- """
- class UnexpectedError(Exception): pass
- class UnexpectedState(Exception): pass
-
- POLLING_INTERVAL = Amount(1, Time.SECONDS)
-
- def __init__(self, root, resource_monitor_class=TaskResourceMonitor):
- self._pathspec = TaskPath(root=root)
- self._detector = TaskDetector(root)
- if not issubclass(resource_monitor_class, ResourceMonitorBase):
- raise ValueError("resource monitor class must implement ResourceMonitorBase!")
- self._resource_monitor = resource_monitor_class
- self._active_tasks = {} # task_id => ActiveObservedTask
- self._finished_tasks = {} # task_id => FinishedObservedTask
- self._stop_event = threading.Event()
- ExceptionalThread.__init__(self)
- Lockable.__init__(self)
- self.daemon = True
-
- @property
- def active_tasks(self):
- """Return a dictionary of active Tasks"""
- return self._active_tasks
-
- @property
- def finished_tasks(self):
- """Return a dictionary of finished Tasks"""
- return self._finished_tasks
-
- @property
- def all_tasks(self):
- """Return a dictionary of all Tasks known by the TaskObserver"""
- return dict(self.active_tasks.items() + self.finished_tasks.items())
-
- def stop(self):
- self._stop_event.set()
-
- def start(self):
- ExceptionalThread.start(self)
-
- @Lockable.sync
- def add_active_task(self, task_id):
- if task_id in self.finished_tasks:
- log.error('Found an active task (%s) in finished tasks?' % task_id)
- return
- task_monitor = TaskMonitor(self._pathspec, task_id)
- if not task_monitor.get_state().header:
- log.info('Unable to load task "%s"' % task_id)
- return
- sandbox = task_monitor.get_state().header.sandbox
- resource_monitor = self._resource_monitor(task_monitor, sandbox)
- resource_monitor.start()
- self._active_tasks[task_id] = ActiveObservedTask(
- task_id=task_id, pathspec=self._pathspec,
- task_monitor=task_monitor, resource_monitor=resource_monitor
- )
-
- @Lockable.sync
- def add_finished_task(self, task_id):
- self._finished_tasks[task_id] = FinishedObservedTask(
- task_id=task_id, pathspec=self._pathspec
- )
-
- @Lockable.sync
- def active_to_finished(self, task_id):
- self.remove_active_task(task_id)
- self.add_finished_task(task_id)
-
- @Lockable.sync
- def remove_active_task(self, task_id):
- task = self.active_tasks.pop(task_id)
- task.resource_monitor.kill()
-
- @Lockable.sync
- def remove_finished_task(self, task_id):
- self.finished_tasks.pop(task_id)
-
- def run(self):
- """
- The internal thread for the observer. This periodically polls the
- checkpoint root for new tasks, or transitions of tasks from active to
- finished state.
- """
- while not self._stop_event.is_set():
- time.sleep(self.POLLING_INTERVAL.as_(Time.SECONDS))
-
- active_tasks = [task_id for _, task_id in self._detector.get_task_ids(state='active')]
- finished_tasks = [task_id for _, task_id in self._detector.get_task_ids(state='finished')]
-
- with self.lock:
-
- # Ensure all tasks currently detected on the system are observed appropriately
- for active in active_tasks:
- if active not in self.active_tasks:
- log.debug('task_id %s (unknown) -> active' % active)
- self.add_active_task(active)
- for finished in finished_tasks:
- if finished in self.active_tasks:
- log.debug('task_id %s active -> finished' % finished)
- self.active_to_finished(finished)
- elif finished not in self.finished_tasks:
- log.debug('task_id %s (unknown) -> finished' % finished)
- self.add_finished_task(finished)
-
- # Remove ObservedTasks for tasks no longer detected on the system
- for unknown in set(self.active_tasks) - set(active_tasks + finished_tasks):
- log.debug('task_id %s active -> (unknown)' % unknown)
- self.remove_active_task(unknown)
- for unknown in set(self.finished_tasks) - set(active_tasks + finished_tasks):
- log.debug('task_id %s finished -> (unknown)' % unknown)
- self.remove_finished_task(unknown)
-
- @Lockable.sync
- def process_from_name(self, task_id, process_id):
- if task_id in self.all_tasks:
- task = self.all_tasks[task_id].task
- if task:
- for process in task.processes():
- if process.name().get() == process_id:
- return process
-
- @Lockable.sync
- def task_count(self):
- """
- Return the count of tasks that could be ready properly from disk.
- This may be <= self.task_id_count()
- """
- return dict(
- active=len(self.active_tasks),
- finished=len(self.finished_tasks),
- all=len(self.all_tasks),
- )
-
- @Lockable.sync
- def task_id_count(self):
- """
- Return the raw count of active and finished task_ids from the TaskDetector.
- """
- num_active = len(list(self._detector.get_task_ids(state='active')))
- num_finished = len(list(self._detector.get_task_ids(state='finished')))
- return dict(active=num_active, finished=num_finished, all=num_active + num_finished)
-
- def _get_tasks_of_type(self, type):
- """Convenience function to return all tasks of a given type"""
- tasks = {
- 'active': self.active_tasks,
- 'finished': self.finished_tasks,
- 'all': self.all_tasks,
- }.get(type, None)
-
- if tasks is None:
- log.error('Unknown task type %s' % type)
- return {}
-
- return tasks
-
- @Lockable.sync
- def state(self, task_id):
- """Return a dict containing mapped information about a task's state"""
- real_state = self.raw_state(task_id)
- if real_state is None or real_state.header is None:
- return {}
- else:
- return dict(
- task_id=real_state.header.task_id,
- launch_time=real_state.header.launch_time_ms / 1000.0,
- sandbox=real_state.header.sandbox,
- hostname=real_state.header.hostname,
- user=real_state.header.user
- )
-
- @Lockable.sync
- def raw_state(self, task_id):
- """
- Return the current runner state (thrift blob: gen.twitter.thermos.ttypes.RunnerState)
- of a given task id
- """
- if task_id not in self.all_tasks:
- return None
- return self.all_tasks[task_id].state
-
- @Lockable.sync
- def _task_processes(self, task_id):
- """
- Return the processes of a task given its task_id.
-
- Returns a map from state to processes in that state, where possible
- states are: waiting, running, success, failed.
- """
- if task_id not in self.all_tasks:
- return {}
- state = self.raw_state(task_id)
- if state is None or state.header is None:
- return {}
-
- waiting, running, success, failed, killed = [], [], [], [], []
- for process, runs in state.processes.items():
- # No runs ==> nothing started.
- if len(runs) == 0:
- waiting.append(process)
- else:
- if runs[-1].state in (None, ProcessState.WAITING, ProcessState.LOST):
- waiting.append(process)
- elif runs[-1].state in (ProcessState.FORKED, ProcessState.RUNNING):
- running.append(process)
- elif runs[-1].state == ProcessState.SUCCESS:
- success.append(process)
- elif runs[-1].state == ProcessState.FAILED:
- failed.append(process)
- elif runs[-1].state == ProcessState.KILLED:
- killed.append(process)
- else:
- # TODO(wickman) Consider log.error instead of raising.
- raise self.UnexpectedState(
- "Unexpected ProcessHistoryState: %s" % state.processes[process].state)
-
- return dict(waiting=waiting, running=running, success=success, failed=failed, killed=killed)
-
- @Lockable.sync
- def main(self, type=None, offset=None, num=None):
- """Return a set of information about tasks, optionally filtered
-
- Args:
- type = (all|active|finished|None) [default: all]
- offset = offset into the list of task_ids [default: 0]
- num = number of results to return [default: 20]
-
- Tasks are sorted by interest:
- - active tasks are sorted by start time
- - finished tasks are sorted by completion time
-
- Returns:
- {
- tasks: [task_id_1, ..., task_id_N],
- type: query type,
- offset: next offset,
- num: next num
- }
-
- """
- type = type or 'all'
- offset = offset or 0
- num = num or 20
-
- # Get a list of all ObservedTasks of requested type
- tasks = sorted((task for task in self._get_tasks_of_type(type).values()),
- key=attrgetter('mtime'), reverse=True)
-
- # Filter by requested offset + number of results
- end = num
- if offset < 0:
- offset = offset % len(tasks) if len(tasks) > abs(offset) else 0
- end += offset
- tasks = tasks[offset:end]
-
- def task_row(observed_task):
- """Generate an output row for a Task"""
- task = self._task(observed_task.task_id)
- # tasks include those which could not be found properly and are hence empty {}
- if task:
- return dict(
- task_id=observed_task.task_id,
- name=task['name'],
- role=task['user'],
- launch_timestamp=task['launch_timestamp'],
- state=task['state'],
- state_timestamp=task['state_timestamp'],
- ports=task['ports'],
- **task['resource_consumption'])
-
- return dict(
- tasks=filter(None, map(task_row, tasks)),
- type=type,
- offset=offset,
- num=num,
- task_count=self.task_count()[type],
- )
-
- def _sample(self, task_id):
- if task_id not in self.active_tasks:
- log.debug("Task %s not found in active tasks" % task_id)
- sample = ProcessSample.empty().to_dict()
- sample['disk'] = 0
- else:
- resource_sample = self.active_tasks[task_id].resource_monitor.sample()[1]
- sample = resource_sample.process_sample.to_dict()
- sample['disk'] = resource_sample.disk_usage
- log.debug("Got sample for task %s: %s" % (task_id, sample))
- return sample
-
- @Lockable.sync
- def task_statuses(self, task_id):
- """
- Return the sequence of task states.
-
- [(task_state [string], timestamp), ...]
- """
-
- # Unknown task_id.
- if task_id not in self.all_tasks:
- return []
-
- task = self.all_tasks[task_id]
- if task is None:
- return []
-
- state = self.raw_state(task_id)
- if state is None or state.header is None:
- return []
-
- # Get the timestamp of the transition into the current state.
- return [
- (TaskState._VALUES_TO_NAMES.get(st.state, 'UNKNOWN'), st.timestamp_ms / 1000)
- for st in state.statuses]
-
- @Lockable.sync
- def _task(self, task_id):
- """
- Return composite information about a particular task task_id, given the below
- schema.
-
- {
- task_id: string,
- name: string,
- user: string,
- launch_timestamp: seconds,
- state: string [ACTIVE, SUCCESS, FAILED]
- ports: { name1: 'url', name2: 'url2' }
- resource_consumption: { cpu:, ram:, disk: }
- processes: { -> names only
- waiting: [],
- running: [],
- success: [],
- failed: []
- }
- }
- """
- # Unknown task_id.
- if task_id not in self.all_tasks:
- return {}
-
- task = self.all_tasks[task_id].task
- if task is None:
- # TODO(wickman) Can this happen?
- log.error('Could not find task: %s' % task_id)
- return {}
-
- state = self.raw_state(task_id)
- if state is None or state.header is None:
- # TODO(wickman) Can this happen?
- return {}
-
- # Get the timestamp of the transition into the current state.
- current_state = state.statuses[-1].state
- last_state = state.statuses[0]
- state_timestamp = 0
- for status in state.statuses:
- if status.state == current_state and last_state != current_state:
- state_timestamp = status.timestamp_ms / 1000
- last_state = status.state
-
- return dict(
- task_id=task_id,
- name=task.name().get(),
- launch_timestamp=state.statuses[0].timestamp_ms / 1000,
- state=TaskState._VALUES_TO_NAMES[state.statuses[-1].state],
- state_timestamp=state_timestamp,
- user=state.header.user,
- resource_consumption=self._sample(task_id),
- ports=state.header.ports,
- processes=self._task_processes(task_id),
- task_struct=task,
- )
-
- @Lockable.sync
- def _get_process_resource_consumption(self, task_id, process_name):
- if task_id not in self.active_tasks:
- log.debug("Task %s not found in active tasks" % task_id)
- return ProcessSample.empty().to_dict()
- sample = self.active_tasks[task_id].resource_monitor.sample_by_process(process_name).to_dict()
- log.debug('Resource consumption (%s, %s) => %s' % (task_id, process_name, sample))
- return sample
-
- @Lockable.sync
- def _get_process_tuple(self, history, run):
- """
- Return the basic description of a process run if it exists, otherwise
- an empty dictionary.
-
- {
- process_name: string
- process_run: int
- state: string [WAITING, FORKED, RUNNING, SUCCESS, KILLED, FAILED, LOST]
- (optional) start_time: seconds from epoch
- (optional) stop_time: seconds from epoch
- }
- """
- if len(history) == 0:
- return {}
- if run >= len(history):
- return {}
- else:
- process_run = history[run]
- run = run % len(history)
- d = dict(
- process_name=process_run.process,
- process_run=run,
- state=ProcessState._VALUES_TO_NAMES[process_run.state],
- )
- if process_run.start_time:
- d.update(start_time=process_run.start_time)
- if process_run.stop_time:
- d.update(stop_time=process_run.stop_time)
- return d
-
- @Lockable.sync
- def process(self, task_id, process, run=None):
- """
- Returns a process run, where the schema is given below:
-
- {
- process_name: string
- process_run: int
- used: { cpu: float, ram: int bytes, disk: int bytes }
- start_time: (time since epoch in millis (utc))
- stop_time: (time since epoch in millis (utc))
- state: string [WAITING, FORKED, RUNNING, SUCCESS, KILLED, FAILED, LOST]
- }
-
- If run is None, return the latest run.
- """
- state = self.raw_state(task_id)
- if state is None or state.header is None:
- return {}
- if process not in state.processes:
- return {}
- history = state.processes[process]
- run = int(run) if run is not None else -1
- tup = self._get_process_tuple(history, run)
- if not tup:
- return {}
- if tup.get('state') == 'RUNNING':
- tup.update(used=self._get_process_resource_consumption(task_id, process))
- return tup
-
- @Lockable.sync
- def _processes(self, task_id):
- """
- Return
- {
- process1: { ... }
- process2: { ... }
- ...
- processN: { ... }
- }
-
- where processK is the latest run of processK and in the schema as
- defined by process().
- """
-
- if task_id not in self.all_tasks:
- return {}
- state = self.raw_state(task_id)
- if state is None or state.header is None:
- return {}
-
- processes = self._task_processes(task_id)
- d = dict()
- for process_type in processes:
- for process_name in processes[process_type]:
- d[process_name] = self.process(task_id, process_name)
- return d
-
- @Lockable.sync
- def processes(self, task_ids):
- """
- Given a list of task_ids, returns a map of task_id => processes, where processes
- is defined by the schema in _processes.
- """
- if not isinstance(task_ids, (list, tuple)):
- return {}
- return dict((task_id, self._processes(task_id)) for task_id in task_ids)
-
- @Lockable.sync
- def get_run_number(self, runner_state, process, run=None):
- if runner_state is not None and runner_state.processes is not None:
- run = run if run is not None else -1
- if run < len(runner_state.processes[process]):
- if len(runner_state.processes[process]) > 0:
- return run % len(runner_state.processes[process])
-
- @Lockable.sync
- def logs(self, task_id, process, run=None):
- """
- Given a task_id and a process and (optional) run number, return a dict:
- {
- stderr: [dir, filename]
- stdout: [dir, filename]
- }
-
- If the run number is unspecified, uses the latest run.
-
- TODO(wickman) Just return the filenames directly?
- """
- runner_state = self.raw_state(task_id)
- if runner_state is None or runner_state.header is None:
- return {}
- run = self.get_run_number(runner_state, process, run)
- if run is None:
- return {}
- log_path = self._pathspec.given(task_id=task_id, process=process, run=run,
- log_dir=runner_state.header.log_dir).getpath('process_logdir')
- return dict(
- stdout=[log_path, 'stdout'],
- stderr=[log_path, 'stderr']
- )
-
- @staticmethod
- def _sanitize_path(base_path, relpath):
- """
- Attempts to sanitize a path through path normalization, also making sure
- that the relative path is contained inside of base_path.
- """
- if relpath is None:
- relpath = "."
- normalized_base = os.path.realpath(base_path)
- normalized = os.path.realpath(os.path.join(base_path, relpath))
- if normalized.startswith(normalized_base):
- return (normalized_base, os.path.relpath(normalized, normalized_base))
- return (None, None)
-
- @Lockable.sync
- def valid_file(self, task_id, path):
- """
- Like valid_path, but also verify the given path is a file
- """
- chroot, path = self.valid_path(task_id, path)
- if chroot and path and os.path.isfile(os.path.join(chroot, path)):
- return chroot, path
- return None, None
-
- @Lockable.sync
- def valid_path(self, task_id, path):
- """
- Given a task_id and a path within that task_id's sandbox, verify:
- (1) it's actually in the sandbox and not outside
- (2) it's a valid, existing path
- Returns chroot and the pathname relative to that chroot.
- """
- runner_state = self.raw_state(task_id)
- if runner_state is None or runner_state.header is None:
- return None, None
- try:
- chroot = runner_state.header.sandbox
- except AttributeError:
- return None, None
- chroot, path = self._sanitize_path(chroot, path)
- if chroot and path:
- return chroot, path
- return None, None
-
- @Lockable.sync
- def files(self, task_id, path=None):
- """
- Returns dictionary
- {
- task_id: task_id
- chroot: absolute directory on machine
- path: sanitized relative path w.r.t. chroot
- dirs: list of directories
- files: list of files
- }
- """
- # TODO(jon): DEPRECATED: most of the necessary logic is handled directly in the templates.
- # Also, global s/chroot/sandbox/?
- empty = dict(task_id=task_id, chroot=None, path=None, dirs=None, files=None)
- path = path if path is not None else '.'
- runner_state = self.raw_state(task_id)
- if runner_state is None:
- return empty
- try:
- chroot = runner_state.header.sandbox
- except AttributeError:
- return empty
- if chroot is None: # chroot-less job
- return empty
- chroot, path = self._sanitize_path(chroot, path)
- if (chroot is None or path is None
- or not os.path.isdir(os.path.join(chroot, path))):
- return empty
- names = os.listdir(os.path.join(chroot, path))
- dirs, files = [], []
- for name in names:
- if os.path.isdir(os.path.join(chroot, path, name)):
- dirs.append(name)
- else:
- files.append(name)
- return dict(
- task_id=task_id,
- chroot=chroot,
- path=path,
- dirs=dirs,
- files=files
- )
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/testing/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/testing/BUILD b/src/main/python/twitter/thermos/testing/BUILD
deleted file mode 100644
index b5485d0..0000000
--- a/src/main/python/twitter/thermos/testing/BUILD
+++ /dev/null
@@ -1,14 +0,0 @@
-python_library(
- name = 'runner',
- sources = ['runner.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/contextutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/python/twitter/thermos/config'),
- pants('src/main/python/twitter/thermos/core'),
- pants('src/main/python/twitter/thermos:thrift'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/testing/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/testing/__init__.py b/src/main/python/twitter/thermos/testing/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/testing/runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/testing/runner.py b/src/main/python/twitter/thermos/testing/runner.py
deleted file mode 100644
index 930f41b..0000000
--- a/src/main/python/twitter/thermos/testing/runner.py
+++ /dev/null
@@ -1,191 +0,0 @@
-from __future__ import print_function
-
-import atexit
-import errno
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import time
-
-from twitter.common import log
-from twitter.common.contextutil import temporary_file, environment_as
-from twitter.thermos.common.path import TaskPath
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-from twitter.thermos.config.loader import ThermosTaskWrapper
-from thrift.TSerialization import deserialize as thrift_deserialize
-
-from gen.twitter.thermos.ttypes import (
- TaskState,
- RunnerCkpt,
- RunnerState,
-)
-
-
-class Runner(object):
- RUN_JOB_SCRIPT = """
-import os
-import random
-import sys
-from twitter.common import log
-from twitter.common.log.options import LogOptions
-from twitter.thermos.config.loader import ThermosConfigLoader
-from twitter.thermos.core.helper import TaskRunnerHelper
-from twitter.thermos.core.runner import TaskRunner, TaskRunnerUniversalHandler
-from thrift.TSerialization import serialize as thrift_serialize
-
-random.seed(%(random_seed)d)
-
-log.init('runner_base')
-LogOptions.set_disk_log_level('DEBUG')
-
-task = ThermosConfigLoader.load_json('%(filename)s')
-task = task.tasks()[0].task
-
-success_rate=%(success_rate)d
-
-class AngryHandler(TaskRunnerUniversalHandler):
- def checkpoint(self, record):
- if not self._runner._recovery:
- if random.randint(0, 100) <= success_rate:
- super(AngryHandler, self).checkpoint(record)
- else:
- sys.exit(1)
-
-sandbox = os.path.join('%(sandbox)s', '%(task_id)s')
-args = {}
-args['task_id'] = '%(task_id)s'
-if %(portmap)s:
- args['portmap'] = %(portmap)s
-args['universal_handler'] = AngryHandler
-
-runner = TaskRunner(task, '%(root)s', sandbox, **args)
-runner.run()
-
-with open('%(state_filename)s', 'w') as fp:
- fp.write(thrift_serialize(runner.state))
-"""
-
- def __init__(self, task, portmap={}, success_rate=100, random_seed=31337):
- """
- task = Thermos task
- portmap = port map
- success_rate = success rate of writing checkpoint to disk
- """
- self.task = task
-
- with temporary_file(cleanup=False) as fp:
- self.job_filename = fp.name
- fp.write(ThermosTaskWrapper(task).to_json())
-
- self.state_filename = tempfile.mktemp()
- self.tempdir = tempfile.mkdtemp()
- self.task_id = '%s-runner-base' % int(time.time()*1000000)
- self.sandbox = os.path.join(self.tempdir, 'sandbox')
- self.portmap = portmap
- self.cleaned = False
- self.pathspec = TaskPath(root = self.tempdir, task_id = self.task_id)
- self.script_filename = None
- self.success_rate = success_rate
- self.random_seed = random_seed
- self._run_count = 0
-
- @property
- def pid(self):
- return self.po.pid
-
- @property
- def root(self):
- return self.tempdir
-
- def run(self):
- self._run_count += 1
- atexit.register(self.cleanup)
-
- if self.script_filename:
- os.unlink(self.script_filename)
-
- with temporary_file(cleanup=False) as fp:
- self.script_filename = fp.name
- fp.write(self.RUN_JOB_SCRIPT % {
- 'filename': self.job_filename,
- 'sandbox': self.sandbox,
- 'root': self.tempdir,
- 'task_id': self.task_id,
- 'state_filename': self.state_filename,
- 'portmap': repr(self.portmap),
- 'success_rate': self.success_rate,
- 'random_seed': self.random_seed + self._run_count,
- })
-
- with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
- self.po = subprocess.Popen([sys.executable, self.script_filename],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- try:
- so, se = self.po.communicate()
- except OSError as e:
- if e.errno == errno.ECHILD:
- so = se = 'Killed'
- else:
- raise
-
- rc = self.po.returncode
- if rc != 0:
- if os.path.exists(self.job_filename):
- config = open(self.job_filename).read()
- else:
- config = 'Nonexistent!'
- if 'THERMOS_DEBUG' in os.environ:
- print("Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n" % (
- config, so, se))
-
- try:
- with open(self.state_filename, 'r') as fp:
- self.state = thrift_deserialize(RunnerState(), fp.read())
- except Exception as e:
- if 'THERMOS_DEBUG' in os.environ:
- print('Failed to load Runner state: %s' % e, file=sys.stderr)
- self.state = RunnerState()
-
- try:
- self.reconstructed_state = CheckpointDispatcher.from_file(
- self.pathspec.getpath('runner_checkpoint'))
- except:
- self.reconstructed_state = None
- self.initialized = True
- return rc
-
- def cleanup(self):
- if not self.cleaned:
- if hasattr(self, 'po'):
- try:
- self.po.kill()
- except:
- pass
- os.unlink(self.job_filename)
- os.unlink(self.script_filename)
- if 'THERMOS_DEBUG' not in os.environ:
- shutil.rmtree(self.tempdir, ignore_errors=True)
- else:
- print('Logs saved in %s' % self.tempdir)
- self.cleaned = True
-
-
-class RunnerTestBase(object):
- @classmethod
- def task(cls):
- raise NotImplementedError
-
- @classmethod
- def setup_class(cls):
- cls.runner = Runner(cls.task(), portmap=getattr(cls, 'portmap', {}))
- cls.runner.run()
- cls.state = cls.runner.state
-
- @classmethod
- def teardown_class(cls):
- cls.runner.cleanup()
-
- def test_runner_state_reconstruction(self):
- assert self.state == self.runner.reconstructed_state
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css
deleted file mode 100644
index ee6da23..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * File: jquery.dataTables.css
- * Version: 1.9.4
- * Author: Allan Jardine (www.sprymedia.co.uk)
- * Info: www.datatables.net
- *
- * Copyright 2008-2012 Allan Jardine, all rights reserved.
- *
- * This source file is free software, under either the GPL v2 license or a
- * BSD style license, available at:
- * http://datatables.net/license_gpl2
- * http://datatables.net/license_bsd
- *
- * This source file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
- */
-
-div.dataTables_length label {
- float: left;
- text-align: left;
-}
-
-div.dataTables_length select {
- width: 75px;
-}
-
-div.dataTables_filter label {
- float: right;
-}
-
-div.dataTables_info {
- padding-top: 8px;
-}
-
-div.dataTables_paginate {
- float: right;
- margin: 0;
-}
-
-table.table {
- clear: both;
- margin-bottom: 6px !important;
- max-width: none !important;
-}
-
-table.table thead .sorting,
-table.table thead .sorting_asc,
-table.table thead .sorting_desc,
-table.table thead .sorting_asc_disabled,
-table.table thead .sorting_desc_disabled {
- cursor: pointer;
- *cursor: hand;
-}
-
-table.table thead .sorting { background: url('/images/sort_both.png') no-repeat center right; }
-table.table thead .sorting_asc { background: url('/images/sort_asc.png') no-repeat center right; }
-table.table thead .sorting_desc { background: url('/images/sort_desc.png') no-repeat center right; }
-
-table.table thead .sorting_asc_disabled { background: url('/images/sort_asc_disabled.png') no-repeat center right; }
-table.table thead .sorting_desc_disabled { background: url('/images/sort_desc_disabled.png') no-repeat center right; }
-
-table.dataTable th:active {
- outline: none;
-}
-
-/* Scrolling */
-div.dataTables_scrollHead table {
- margin-bottom: 0 !important;
- border-bottom-left-radius: 0;
- border-bottom-right-radius: 0;
-}
-
-div.dataTables_scrollHead table thead tr:last-child th:first-child,
-div.dataTables_scrollHead table thead tr:last-child td:first-child {
- border-bottom-left-radius: 0 !important;
- border-bottom-right-radius: 0 !important;
-}
-
-div.dataTables_scrollBody table {
- border-top: none;
- margin-bottom: 0 !important;
-}
-
-div.dataTables_scrollBody tbody tr:first-child th,
-div.dataTables_scrollBody tbody tr:first-child td {
- border-top: none;
-}
-
-div.dataTables_scrollFoot table {
- border-top: none;
-}
-
-
-
-
-/*
- * TableTools styles
- */
-.table tbody tr.active td,
-.table tbody tr.active th {
- background-color: #08C;
- color: white;
-}
-
-.table tbody tr.active:hover td,
-.table tbody tr.active:hover th {
- background-color: #0075b0 !important;
-}
-
-.table-striped tbody tr.active:nth-child(odd) td,
-.table-striped tbody tr.active:nth-child(odd) th {
- background-color: #017ebc;
-}
-
-table.DTTT_selectable tbody tr {
- cursor: pointer;
- *cursor: hand;
-}
-
-div.DTTT .btn {
- color: #333 !important;
- font-size: 12px;
-}
-
-div.DTTT .btn:hover {
- text-decoration: none !important;
-}
-
-
-ul.DTTT_dropdown.dropdown-menu a {
- color: #333 !important; /* needed only when demo_page.css is included */
-}
-
-ul.DTTT_dropdown.dropdown-menu li:hover a {
- background-color: #0088cc;
- color: white !important;
-}
-
-/* TableTools information display */
-div.DTTT_print_info.modal {
- height: 150px;
- margin-top: -75px;
- text-align: center;
-}
-
-div.DTTT_print_info h6 {
- font-weight: normal;
- font-size: 28px;
- line-height: 28px;
- margin: 1em;
-}
-
-div.DTTT_print_info p {
- font-size: 14px;
- line-height: 20px;
-}
-
-
-
-/*
- * FixedColumns styles
- */
-div.DTFC_LeftHeadWrapper table,
-div.DTFC_LeftFootWrapper table,
-table.DTFC_Cloned tr.even {
- background-color: white;
-}
-
-div.DTFC_LeftHeadWrapper table {
- margin-bottom: 0 !important;
- border-top-right-radius: 0 !important;
- border-bottom-left-radius: 0 !important;
- border-bottom-right-radius: 0 !important;
-}
-
-div.DTFC_LeftHeadWrapper table thead tr:last-child th:first-child,
-div.DTFC_LeftHeadWrapper table thead tr:last-child td:first-child {
- border-bottom-left-radius: 0 !important;
- border-bottom-right-radius: 0 !important;
-}
-
-div.DTFC_LeftBodyWrapper table {
- border-top: none;
- margin-bottom: 0 !important;
-}
-
-div.DTFC_LeftBodyWrapper tbody tr:first-child th,
-div.DTFC_LeftBodyWrapper tbody tr:first-child td {
- border-top: none;
-}
-
-div.DTFC_LeftFootWrapper table {
- border-top: none;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_disabled.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_disabled.png
deleted file mode 100644
index 881de79..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_disabled.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled.png
deleted file mode 100644
index c608682..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png
deleted file mode 100644
index d300f10..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_disabled.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_disabled.png
deleted file mode 100644
index 6a6ded7..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_disabled.png and /dev/null differ
[16/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/assets/observer.js
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/assets/observer.js b/src/main/python/apache/thermos/observer/http/assets/observer.js
new file mode 100644
index 0000000..84aec5d
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/assets/observer.js
@@ -0,0 +1,202 @@
+var Task = new Class({
+ data: {},
+
+ initialize: function(taskId, taskType) {
+ this.taskType = taskType
+ this.taskId = taskId
+ this.visible = false
+ this.setElement()
+ },
+
+ // task[active.task_id.processes.waiting]
+ taskStr: function(components) {
+ return 'task[' + ([this.taskType, this.taskId].append(components).join()) + ']'
+ },
+
+ transitionElement: function(dom_id, newval) {
+ if (!$(dom_id)) {
+ // alert('could not find dom_id' + dom_id)
+ return
+ }
+
+ curval = $(dom_id).innerHTML
+
+ // handle specialcases
+ newval = this.translateElement(dom_id, newval)
+
+ if (newval != curval) {
+ $(dom_id).innerHTML = newval
+ var morphElement = new Fx.Morph(dom_id, {
+ duration: 'long',
+ transition: Fx.Transitions.Sine.easeOut
+ });
+ morphElement.start({ 'color': ["#00FF00", "#222222"]})
+ }
+ },
+
+ applyUpdate: function(update) {
+ this.data = update
+ this.updateElement()
+ },
+
+ /*
+ TODO(wickman) Instead do the 2-3 necessary transition functions, then make
+ a schema that maps attributes to the appropriate transition function.
+ */
+ translateElement: function(dom_id, value) {
+ if (dom_id === this.taskStr(['processes', 'running']) ||
+ dom_id === this.taskStr(['processes', 'waiting']) ||
+ dom_id === this.taskStr(['processes', 'success']) ||
+ dom_id === this.taskStr(['processes', 'failed'])) {
+ return value.length
+ }
+
+ if (dom_id == this.taskStr(['resource_consumption', 'cpu'])) {
+ if (value) {
+ return value.toFixed(2)
+ }
+ }
+
+ return value
+ },
+
+ /*
+ TODO(wickman) Ditto to above.
+ */
+ updateElement: function() {
+ for (prefix_attr in this.data) {
+ if (this.data.hasOwnProperty(prefix_attr)) {
+ if (prefix_attr == "task_id") {
+ this.transitionElement(this.taskStr([prefix_attr]), this.translateUid(this.data[prefix_attr]))
+ } else if (instanceOf(this.data[prefix_attr], String) || instanceOf(this.data[prefix_attr], Number)) {
+ this.transitionElement(this.taskStr([prefix_attr]), this.data[prefix_attr])
+ } else if (instanceOf(this.data[prefix_attr], Object)) {
+ for (suffix_attr in this.data[prefix_attr])
+ if (this.data[prefix_attr].hasOwnProperty(suffix_attr))
+ this.transitionElement(this.taskStr([prefix_attr, suffix_attr]),
+ this.data[prefix_attr][suffix_attr])
+ }
+ }
+ }
+ },
+
+ translateUid: function(taskId) {
+ return "<a href='/task/" + taskId + "'>" + taskId + "</a>"
+ },
+
+ setElement: function() {
+ this.element = new Element('tr', {
+ 'id': 'task[' + this.taskType + '][' + this.taskId + ']'}).adopt(
+ new Element('td', { 'id': this.taskStr(['task_id']), 'html': this.translateUid(this.taskId)}),
+ new Element('td', { 'id': this.taskStr(['name']) }),
+ new Element('td', { 'id': this.taskStr(['resource_consumption', 'cpu']) }),
+ new Element('td', { 'id': this.taskStr(['resource_consumption', 'ram']) }),
+ new Element('td', { 'id': this.taskStr(['resource_consumption', 'disk']) }),
+ new Element('td', { 'id': this.taskStr(['processes', 'waiting']) }),
+ new Element('td', { 'id': this.taskStr(['processes', 'running']) }),
+ new Element('td', { 'id': this.taskStr(['processes', 'success']) }),
+ new Element('td', { 'id': this.taskStr(['processes', 'failed']) }))
+ }
+})
+
+var TableManager = new Class({
+ activeTasks: {},
+ visibleChildren: [],
+
+ initialize: function(tableType) {
+ this.tableType = tableType
+ this.setElement()
+ this.getChildren()
+ this.startPolling()
+ },
+
+ setElement: function() {
+ this.element = new Element('table', { 'id': 'table[' + this.tableType + ']', 'class': 'common-table zebra-striped', 'cellpadding': '0' })
+ this.element.adopt(
+ new Element('thead').
+ adopt(new Element('tr', { 'id': 'task[' + this.tableType + '][superheader]', 'class': 'meta-headers' })
+ .adopt(new Element('th', { 'html': "", 'colspan': 2 }),
+ new Element('th', { 'html': "consumed", 'colspan': 3 }),
+ new Element('th', { 'html': "processes", 'colspan': 4 })
+ )
+ )
+ .adopt(new Element('tr', { 'id': 'task[' + this.tableType + '][header]' })
+ .adopt(new Element('th', { 'id': 'task[' + this.tableType + '][header][task_id]', 'html': "task_id" }),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][name]', 'html': "name" }),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][resource_consumption][cpu]', 'html': "cpu" }),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][resource_consumption][ram]', 'html': "ram" }),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][resource_consumption][disk]','html': "disk" }),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][waiting]', 'html': "waiting" }),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][running]', 'html': "running"}),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][success]', 'html': "success" }),
+ new Element('th', { 'id': 'task[' + this.tableType + '][header][processes][failed]', 'html': "failed" })
+ )
+ )
+ )
+ this.tbody = new Element('tbody', { 'id': this.tableType + '_tbody' })
+ this.element.adopt(this.tbody)
+ },
+
+ toElement: function() {
+ return this.element
+ },
+
+ startPolling: function() {
+ this.taskIdPoller = setInterval(this.tableType + 'TableManager.getChildren();', 2500)
+ this.dataPoller = setInterval(this.tableType + 'TableManager.refreshVisibleChildren();', 5300)
+ },
+
+ getChildren: function() {
+ new Request.JSON({
+ 'url': '/j/task_ids/' + this.tableType + '/-20',
+ 'method': 'get',
+ 'onComplete': function(response) {
+ if (response) {
+ var newChildren = Array.from(response.task_ids)
+ this.visibleChildren = newChildren
+
+ // first set all children to invisible
+ for (taskId in this.activeTasks) {
+ this.activeTasks[taskId].visible = false
+ }
+
+ // set new children visible
+ for (var k = 0; k < newChildren.length; k++) {
+ if (!(newChildren[k] in this.activeTasks)) {
+ this.activeTasks[newChildren[k]] = new Task(newChildren[k], this.tableType)
+ }
+ this.activeTasks[newChildren[k]].visible = true
+ }
+
+ // clear then adopt them
+ while (this.tbody.childNodes.length > 0)
+ this.tbody.removeChild(this.tbody.firstChild);
+ for (var k = 0; k < newChildren.length; k++)
+ this.tbody.adopt(this.activeTasks[newChildren[k]].element)
+ } else {
+ clearInterval(this.taskIdPoller)
+ }
+ }.bind(this)
+ }).send()
+ },
+
+ refreshVisibleChildren: function() {
+ // first get visible children
+ new Request.JSON({
+ 'url': '/j/task',
+ 'method': 'get',
+ 'data': { 'task_id': this.visibleChildren.join() },
+ 'onComplete': function(response) {
+ if (response) {
+ if (!response) return;
+ for (taskId in response) {
+ if (response.hasOwnProperty(taskId))
+ this.activeTasks[taskId].applyUpdate(response[taskId])
+ }
+ } else {
+ clearInterval(this.dataPoller)
+ }
+ }.bind(this)
+ }).send()
+ },
+})
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/file_browser.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/file_browser.py b/src/main/python/apache/thermos/observer/http/file_browser.py
new file mode 100644
index 0000000..45c5708
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/file_browser.py
@@ -0,0 +1,124 @@
+import os
+import pprint
+from xml.sax.saxutils import escape
+
+from twitter.common import log
+from twitter.common.http import HttpServer
+
+import bottle
+from mako.template import Template
+
+from .templating import HttpTemplate
+
+
+MB = 1024 * 1024
+DEFAULT_CHUNK_LENGTH = MB
+MAX_CHUNK_LENGTH = 16 * MB
+
+
+def _read_chunk(filename, offset=None, length=None):
+ offset = offset or -1
+ length = length or -1
+
+ try:
+ length = long(length)
+ offset = long(offset)
+ except ValueError:
+ return {}
+
+ if not os.path.isfile(filename):
+ return {}
+
+ try:
+ fstat = os.stat(filename)
+ except Exception as e:
+ log.error('Could not read from %s: %s' % (filename, e))
+ return {}
+
+ if offset == -1:
+ offset = fstat.st_size
+
+ if length == -1:
+ length = fstat.st_size - offset
+
+ with open(filename, "r") as fp:
+ fp.seek(offset)
+ try:
+ data = fp.read(length)
+ except IOError as e:
+ log.error('Failed to read %s: %s' % (filename, e), exc_info=True)
+ return {}
+
+ if data:
+ return dict(offset=offset, length=len(data), data=escape(data.decode('utf8', 'replace')))
+
+ return dict(offset=offset, length=0)
+
+
+class TaskObserverFileBrowser(object):
+ """
+ Mixin for Thermos observer File browser.
+ """
+
+ @HttpServer.route("/logs/:task_id/:process/:run/:logtype")
+ @HttpServer.mako_view(HttpTemplate.load('logbrowse'))
+ def handle_logs(self, task_id, process, run, logtype):
+ types = self._observer.logs(task_id, process, int(run))
+ if logtype not in types:
+ bottle.abort(404, "No such log type: %s" % logtype)
+ base, path = types[logtype]
+ filename = os.path.join(base, path)
+ return {
+ 'task_id': task_id,
+ 'filename': filename,
+ 'process': process,
+ 'run': run,
+ 'logtype': logtype
+ }
+
+ @HttpServer.route("/logdata/:task_id/:process/:run/:logtype")
+ def handle_logdata(self, task_id, process, run, logtype):
+ offset = self.Request.GET.get('offset', -1)
+ length = self.Request.GET.get('length', -1)
+ types = self._observer.logs(task_id, process, int(run))
+ if logtype not in types:
+ return {}
+ chroot, path = types[logtype]
+ return _read_chunk(os.path.join(chroot, path), offset, length)
+
+ @HttpServer.route("/file/:task_id/:path#.+#")
+ @HttpServer.mako_view(HttpTemplate.load('filebrowse'))
+ def handle_file(self, task_id, path):
+ if path is None:
+ bottle.abort(404, "No such file")
+ return {
+ 'task_id': task_id,
+ 'filename': path,
+ }
+
+ @HttpServer.route("/filedata/:task_id/:path#.+#")
+ def handle_filedata(self, task_id, path):
+ if path is None:
+ return {}
+ offset = self.Request.GET.get('offset', -1)
+ length = self.Request.GET.get('length', -1)
+ chroot, path = self._observer.valid_file(task_id, path)
+ if chroot is None or path is None:
+ return {}
+ return _read_chunk(os.path.join(chroot, path), offset, length)
+
+ @HttpServer.route("/browse/:task_id")
+ @HttpServer.route("/browse/:task_id/:path#.*#")
+ @HttpServer.mako_view(HttpTemplate.load('filelist'))
+ def handle_dir(self, task_id, path=None):
+ if path == "":
+ path = None
+ chroot, path = self._observer.valid_path(task_id, path)
+ return dict(task_id=task_id, chroot=chroot, path=path)
+
+ @HttpServer.route("/download/:task_id/:path#.+#")
+ def handle_download(self, task_id, path=None):
+ chroot, path = self._observer.valid_path(task_id, path)
+ if path is None:
+ bottle.abort(404, "No such file")
+ return bottle.static_file(path, root=chroot, download=True)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/http_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/http_observer.py b/src/main/python/apache/thermos/observer/http/http_observer.py
new file mode 100644
index 0000000..b73f17f
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/http_observer.py
@@ -0,0 +1,133 @@
+"""HTTP interface to the Thermos TaskObserver
+
+This modules provides an HTTP server which exposes information about Thermos tasks running on a
+system. To do this, it relies heavily on the Thermos TaskObserver.
+
+"""
+
+import os
+import socket
+
+from twitter.common import log
+from twitter.common.http import HttpServer
+
+from .file_browser import TaskObserverFileBrowser
+from .json import TaskObserverJSONBindings
+from .static_assets import StaticAssets
+from .templating import HttpTemplate
+
+
+class BottleObserver(HttpServer, StaticAssets, TaskObserverFileBrowser, TaskObserverJSONBindings):
+ """
+ A bottle wrapper around a Thermos TaskObserver.
+ """
+
+ def __init__(self, observer):
+ self._observer = observer
+ StaticAssets.__init__(self)
+ TaskObserverFileBrowser.__init__(self)
+ TaskObserverJSONBindings.__init__(self)
+ HttpServer.__init__(self)
+
+ @HttpServer.route("/")
+ @HttpServer.view(HttpTemplate.load('index'))
+ def handle_index(self):
+ return dict(hostname=socket.gethostname())
+
+ @HttpServer.route("/main")
+ @HttpServer.route("/main/:type")
+ @HttpServer.route("/main/:type/:offset")
+ @HttpServer.route("/main/:type/:offset/:num")
+ @HttpServer.mako_view(HttpTemplate.load('main'))
+ def handle_main(self, type=None, offset=None, num=None):
+ if type not in (None, 'all', 'finished', 'active'):
+ HttpServer.abort(404, 'Invalid task type: %s' % type)
+ if offset is not None:
+ try:
+ offset = int(offset)
+ except ValueError:
+ HttpServer.abort(404, 'Invalid offset: %s' % offset)
+ if num is not None:
+ try:
+ num = int(num)
+ except ValueError:
+ HttpServer.abort(404, 'Invalid count: %s' % num)
+ return self._observer.main(type, offset, num)
+
+ @HttpServer.route("/task/:task_id")
+ @HttpServer.mako_view(HttpTemplate.load('task'))
+ def handle_task(self, task_id):
+ task = self.get_task(task_id)
+ processes = self._observer.processes([task_id])
+ if not processes.get(task_id, None):
+ HttpServer.abort(404, 'Unknown task_id: %s' % task_id)
+ processes = processes[task_id]
+ state = self._observer.state(task_id)
+
+ return dict(
+ task_id = task_id,
+ task = task,
+ statuses = self._observer.task_statuses(task_id),
+ user = task['user'],
+ ports = task['ports'],
+ processes = processes,
+ chroot = state.get('sandbox', ''),
+ launch_time = state.get('launch_time', 0),
+ hostname = state.get('hostname', 'localhost'),
+ )
+
+ def get_task(self, task_id):
+ task = self._observer._task(task_id)
+ if not task:
+ HttpServer.abort(404, "Failed to find task %s. Try again shortly." % task_id)
+ return task
+
+ @HttpServer.route("/rawtask/:task_id")
+ @HttpServer.mako_view(HttpTemplate.load('rawtask'))
+ def handle_rawtask(self, task_id):
+ task = self.get_task(task_id)
+ state = self._observer.state(task_id)
+ return dict(
+ hostname = state.get('hostname', 'localhost'),
+ task_id = task_id,
+ task_struct = task['task_struct']
+ )
+
+ @HttpServer.route("/process/:task_id/:process_id")
+ @HttpServer.mako_view(HttpTemplate.load('process'))
+ def handle_process(self, task_id, process_id):
+ all_processes = {}
+ current_run = self._observer.process(task_id, process_id)
+ if not current_run:
+ HttpServer.abort(404, 'Invalid task/process combination: %s/%s' % (task_id, process_id))
+ process = self._observer.process_from_name(task_id, process_id)
+ if process is None:
+ msg = 'Could not recover process: %s/%s' % (task_id, process_id)
+ log.error(msg)
+ HttpServer.abort(404, msg)
+
+ current_run_number = current_run['process_run']
+ all_processes[current_run_number] = current_run
+ for run in range(current_run_number):
+ all_processes[run] = self._observer.process(task_id, process_id, run)
+ def convert_process_tuple(run_tuple):
+ process_tuple = dict(state = run_tuple['state'])
+ if 'start_time' in run_tuple:
+ process_tuple.update(start_time = run_tuple['start_time'])
+ if 'stop_time' in run_tuple:
+ process_tuple.update(stop_time = run_tuple['stop_time'])
+ return process_tuple
+
+ template = {
+ 'task_id': task_id,
+ 'process': {
+ 'name': process_id,
+ 'status': all_processes[current_run_number]["state"],
+ 'cmdline': process.cmdline().get()
+ },
+ }
+ template['process'].update(**all_processes[current_run_number].get('used', {}))
+ template['runs'] = dict((run, convert_process_tuple(run_tuple))
+ for run, run_tuple in all_processes.items())
+ log.info('Rendering template is: %s' % template)
+ return template
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/json.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/json.py b/src/main/python/apache/thermos/observer/http/json.py
new file mode 100644
index 0000000..9212f0e
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/json.py
@@ -0,0 +1,56 @@
+import urllib
+
+from twitter.common.http import HttpServer
+
+
+class TaskObserverJSONBindings(object):
+ """
+ Mixin for Thermos observer JSON endpoints.
+ """
+
+ @HttpServer.route("/j/task_ids")
+ @HttpServer.route("/j/task_ids/:which")
+ @HttpServer.route("/j/task_ids/:which/:offset")
+ @HttpServer.route("/j/task_ids/:which/:offset/:num")
+ def handle_task_ids(self, which=None, offset=None, num=None):
+ return self._observer.task_ids(
+ which,
+ int(offset) if offset is not None else 0,
+ int(num) if num is not None else 20)
+
+ @HttpServer.route("/j/task_id_count")
+ def handle_task_id_count(self):
+ return self._observer.task_id_count()
+
+ @HttpServer.route("/j/task")
+ def handle_tasks(self):
+ """
+ Additional parameters:
+ task_id = comma separated list of task_ids.
+ """
+ task_ids = HttpServer.Request.GET.get('task_id', [])
+ if task_ids:
+ task_ids = urllib.unquote(task_ids).split(',')
+ return self._observer.task(task_ids)
+
+ @HttpServer.route("/j/task/:task_id")
+ def handle_task(self, task_id):
+ return self._observer.task([task_id])
+
+ @HttpServer.route("/j/process/:task_id")
+ @HttpServer.route("/j/process/:task_id/:process")
+ @HttpServer.route("/j/process/:task_id/:process/:run")
+ def handle_process(self, task_id, process=None, run=None):
+ return self._observer.process(task_id, process, run)
+
+ @HttpServer.route("/j/processes")
+ def handle_processes(self):
+ """
+ Additional parameters:
+ task_ids = comma separated list of task_ids.
+ """
+ task_ids = HttpServer.Request.GET.get('task_id', [])
+ if task_ids:
+ task_ids = urllib.unquote(task_ids).split(',')
+ return self._observer.processes(task_ids)
+
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/static_assets.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/static_assets.py b/src/main/python/apache/thermos/observer/http/static_assets.py
new file mode 100644
index 0000000..cf3ff08
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/static_assets.py
@@ -0,0 +1,43 @@
+import mimetypes
+import os
+
+from twitter.common import log
+from twitter.common.http.server import HttpServer
+
+from bottle import HTTPResponse
+import pkg_resources
+
+
+class StaticAssets(object):
+ """
+ Serve the /assets directory.
+ """
+ def __init__(self):
+ self._assets = {}
+ self._detect_assets()
+
+ def _detect_assets(self):
+ log.info('detecting assets...')
+ assets = pkg_resources.resource_listdir(__name__, 'assets')
+ cached_assets = {}
+ for asset in assets:
+ log.info(' detected asset: %s' % asset)
+ cached_assets[asset] = pkg_resources.resource_string(
+ __name__, os.path.join('assets', asset))
+ self._assets = cached_assets
+
+ @HttpServer.route("/favicon.ico")
+ def handle_favicon(self):
+ HttpServer.redirect("/assets/favicon.ico")
+
+ @HttpServer.route("/assets/:filename")
+ def handle_asset(self, filename):
+ # TODO(wickman) Add static_content to bottle.
+ if filename in self._assets:
+ mimetype, encoding = mimetypes.guess_type(filename)
+ headers = {}
+ if mimetype: headers['Content-Type'] = mimetype
+ if encoding: headers['Content-Encoding'] = encoding
+ return HTTPResponse(self._assets[filename], header=headers)
+ else:
+ HttpServer.abort(404, 'Unknown asset: %s' % filename)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/filebrowse.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/filebrowse.tpl b/src/main/python/apache/thermos/observer/http/templates/filebrowse.tpl
new file mode 100644
index 0000000..871e31e
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/filebrowse.tpl
@@ -0,0 +1,81 @@
+<%def name="download_link()">
+ <a href='/download/${task_id}/${filename}'><font size=1>download</font></a>
+</%def>
+
+<html>
+
+<head>
+ <meta charset="utf-8">
+ <title></title>
+
+ <style type="text/css">
+ .log {
+ font-family: "Inconsolata", "Monaco", "Courier New", "Courier";
+ line-height:14px;
+ font-size: 12px;
+ }
+
+ .invert {
+ color: #FFFFFF;
+ text-decoration: none;
+ background: #000000;
+ }
+ </style>
+</head>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="/assets/bootstrap.css"/>
+<style type="text/css">
+div.tight
+{
+ height:100%;
+ overflow:scroll;
+}
+</style>
+
+<title>file browser ${task_id}</title>
+<body>
+ <div> <strong> filename </strong> ${filename} </div>
+ <div> <strong> dl </strong> ${download_link()} </div>
+ <div style="position: absolute; left: 5px; top: 0px;">
+ <p id="indicator" class="log invert"></p>
+ </div>
+
+ <div id="data" class="log" style="white-space:pre-wrap; background-color:#EEEEEE;"></div>
+</body>
+
+<script src="/assets/jquery.js"></script>
+<script src="/assets/jquery.pailer.js"></script>
+
+<script>
+ function resize() {
+ var margin_left = parseInt($('body').css('margin-left'));
+ var margin_top = parseInt($('body').css('margin-top'));
+ var margin_bottom = parseInt($('body').css('margin-bottom'));
+ $('#data').width($(window).width() - margin_left);
+ $('#data').height($(window).height() - margin_top - margin_bottom);
+ }
+
+ $(window).resize(resize);
+
+ $(document).ready(function() {
+ resize();
+
+ $('#data').pailer({
+ 'read': function(options) {
+ var settings = $.extend({
+ 'offset': -1,
+ 'length': -1
+ }, options);
+
+ var url = "/filedata/${task_id}/${filename}"
+ + '?offset=' + settings.offset
+ + '&length=' + settings.length;
+ return $.getJSON(url);
+ },
+ 'indicator': $('#indicator')
+ });
+ });
+</script>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/filelist.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/filelist.tpl b/src/main/python/apache/thermos/observer/http/templates/filelist.tpl
new file mode 100644
index 0000000..12a257e
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/filelist.tpl
@@ -0,0 +1,141 @@
+<%doc>
+ Template arguments:
+ task_id
+ chroot
+ path
+ dirs
+ files
+</%doc>
+
+
+<%!
+ import os
+
+ from datetime import datetime
+ import grp
+ import os
+ import pwd
+ import stat
+ import sys
+
+ NOW = datetime.now()
+
+ def format_mode(sres):
+ mode = sres.st_mode
+
+ root = (mode & 0700) >> 6
+ group = (mode & 0070) >> 3
+ user = (mode & 07)
+
+ def stat_type(md):
+ if stat.S_ISDIR(md):
+ return 'd'
+ elif stat.S_ISSOCK(md):
+ return 's'
+ else:
+ return '-'
+
+ def triple(md):
+ return '%c%c%c' % (
+ 'r' if md & 0b100 else '-',
+ 'w' if md & 0b010 else '-',
+ 'x' if md & 0b001 else '-')
+
+ return ''.join([stat_type(mode), triple(root), triple(group), triple(user)])
+
+ def format_mtime(mtime):
+ dt = datetime.fromtimestamp(mtime)
+ return '%s %2d %5s' % (dt.strftime('%b'), dt.day,
+ dt.year if dt.year != NOW.year else dt.strftime('%H:%M'))
+
+ def format_prefix(filename, sres):
+ try:
+ pwent = pwd.getpwuid(sres.st_uid)
+ user = pwent.pw_name
+ except KeyError:
+ user = sres.st_uid
+
+ try:
+ grent = grp.getgrgid(sres.st_gid)
+ group = grent.gr_name
+ except KeyError:
+ group = sres.st_gid
+
+ return '%s %3d %10s %10s %10d %s' % (
+ format_mode(sres),
+ sres.st_nlink,
+ user,
+ group,
+ sres.st_size,
+ format_mtime(sres.st_mtime),
+ )
+%>
+
+<%def name="download_link(filename)"><a href='/download/${task_id}/${os.path.join(path, filename)}'><font size=1>dl</font></a></%def>
+<%def name="directory_link(dirname)"><a href='/browse/${task_id}/${os.path.join(path, dirname)}'>${dirname}</a></%def>
+<%def name="file_link(filename)"><a href='/file/${task_id}/${os.path.join(path, filename)}'>${filename}</a></%def>
+
+<html>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="/assets/bootstrap.css"/>
+<style type="text/css">
+div.tight
+{
+ height:85%;
+ overflow:auto;
+}
+</style>
+
+<title>path browser for ${task_id}</title>
+
+
+% if chroot is not None:
+<body>
+ <div class="container">
+ <div class="span6">
+ <strong> task id </strong> ${task_id}
+ </div>
+ <div class="span6">
+ <strong> path </strong> ${path}
+ </div>
+ <div class="span12 tight">
+ <pre>
+
+% if path != ".":
+ <%
+ listing = ['..'] + os.listdir(os.path.join(chroot, path))
+ %>\
+% else:
+ <%
+ listing = os.listdir(os.path.join(chroot, path))
+ %>\
+% endif
+
+<% listing.sort() %>
+
+% for fn in listing:
+<%
+ try:
+ sres = os.stat(os.path.join(chroot, path, fn))
+ except OSError:
+ continue
+%>\
+ % if not stat.S_ISDIR(sres.st_mode):
+${format_prefix(fn, sres)} ${file_link(fn)} ${download_link(fn)}
+ % else:
+${format_prefix(fn, sres)} ${directory_link(fn)}
+ % endif
+% endfor
+ </pre>
+ </div>
+ </div>
+</body>
+% else:
+<body>
+ This task is running without a chroot.
+</body>
+% endif
+
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/home.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/home.tpl b/src/main/python/apache/thermos/observer/http/templates/home.tpl
new file mode 100644
index 0000000..4e7426b
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/home.tpl
@@ -0,0 +1,58 @@
+<html>
+<title>thermos(${hostname})</title>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="/assets/bootstrap.css"/>
+
+<body>
+
+<%!
+ import socket
+ import time
+
+ def pretty_time(seconds):
+ return time.asctime(time.localtime(seconds))
+%>
+
+<div class="container">
+ <h3> host ${socket.gethostname()} </h3>
+
+ <div class="content" id="defaultLayout">
+ <table class="zebra-striped">
+ <thead>
+ <tr>
+ <th colspan=3> task </th>
+ <th colspan=4> resources </th>
+ <th colspan=3> links </th>
+ </tr>
+
+ <tr>
+ <th> name </th> <th> role </th> <th> status </th>
+ <th> procs </th> <th> cpu </th> <th> ram </th> <th> disk </th>
+ <th> task </th> <th> chroot </th> <th> ports </th>
+ </tr>
+ </thead>
+ <tbody>
+
+ % for proc_name, proc in sorted(processes.items()):
+ <tr>
+ <td> ${proc["process_name"]} </td>
+ <td> ${proc["process_run"]} </td>
+ <td> ${proc["state"]} </td>
+ <td> ${pretty_time(float(proc["start_time"])/1000.0) if "start_time" in proc else ""} </td>
+ <td> ${pretty_time(float(proc["stop_time"])/1000.0) if "stop_time" in proc else ""} </td>
+ <td> ${'%.3f' % proc["used"]["cpu"] if "used" in proc else ""} </td>
+ <td> ${'%dMB' % (proc["used"]["ram"] / 1024 / 1024) if "used" in proc else ""} </td>
+ <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stdout">stdout</a> </td>
+ <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stderr">stderr</a> </td>
+ </tr>
+ % endfor
+ </tbody>
+ </table>
+ </div>
+
+</div>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/index.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/index.tpl b/src/main/python/apache/thermos/observer/http/templates/index.tpl
new file mode 100644
index 0000000..d662c60
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/index.tpl
@@ -0,0 +1,51 @@
+<html>
+<title>thermos({{hostname}})</title>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="assets/bootstrap.css"/>
+
+<script src="assets/jquery.js"></script>
+
+<body>
+
+<div class="container" id="defaultLayout">
+ <div id="activeTaskContainer" class='uber-container'>
+ <div class="active-container" data-url="main/active">
+ </div>
+ </div>
+ <br><br>
+ <div id="finishedTaskContainer" class='uber-container'>
+ <div class="finished-container" data-url="main/finished">
+ </div>
+ </div>
+</div>
+
+<script type="text/javascript">
+
+$(document).on('click', 'a.refresh-container', function(e) {
+ e.preventDefault()
+ topLevelDivContainer = $(this).closest('.uber-container')
+ divDataUrl = $(this).attr('data-url')
+ $.ajax({
+ 'type': 'GET',
+ 'dataType': 'html',
+ 'url': divDataUrl,
+ success: function(data, xhr, err) {
+ $(topLevelDivContainer).html(data)
+ }
+ })
+ })
+
+refreshDivs = function() {
+ $('#activeTaskContainer').load($('.uber-container .active-container').attr('data-url'))
+ $('#finishedTaskContainer').load($('.uber-container .finished-container').attr('data-url'))
+}
+
+$(document).bind('ready', refreshDivs)
+setInterval(refreshDivs, 10000)
+
+</script>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/logbrowse.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/logbrowse.tpl b/src/main/python/apache/thermos/observer/http/templates/logbrowse.tpl
new file mode 100644
index 0000000..059979c
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/logbrowse.tpl
@@ -0,0 +1,80 @@
+<%def name="download_link()">
+ <a href='/download/${task_id}/${filename}'><font size=1>download</font></a>
+</%def>
+
+<html>
+
+<head>
+ <meta charset="utf-8">
+ <title></title>
+
+ <style type="text/css">
+ .log {
+ font-family: "Inconsolata", "Monaco", "Courier New", "Courier";
+ line-height:14px;
+ font-size: 12px;
+ }
+
+ .invert {
+ color: #FFFFFF;
+ text-decoration: none;
+ background: #000000;
+ }
+ </style>
+</head>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="/assets/bootstrap.css"/>
+<style type="text/css">
+div.tight
+{
+ height:100%;
+ overflow:scroll;
+}
+</style>
+
+<title>log browser ${task_id}</title>
+<body>
+ <div> <strong> log </strong> ${logtype} <strong> ${download_link()} </strong> </div>
+ <div style="position: absolute; left: 5px; top: 0px;">
+ <p id="indicator" class="log invert"></p>
+ </div>
+
+ <div id="data" class="log" style="white-space:pre-wrap; background-color:#EEEEEE;"></div>
+</body>
+
+<script src="/assets/jquery.js"></script>
+<script src="/assets/jquery.pailer.js"></script>
+
+<script>
+ function resize() {
+ var margin_left = parseInt($('body').css('margin-left'));
+ var margin_top = parseInt($('body').css('margin-top'));
+ var margin_bottom = parseInt($('body').css('margin-bottom'));
+ $('#data').width($(window).width() - margin_left);
+ $('#data').height($(window).height() - margin_top - margin_bottom);
+ }
+
+ $(window).resize(resize);
+
+ $(document).ready(function() {
+ resize();
+
+ $('#data').pailer({
+ 'read': function(options) {
+ var settings = $.extend({
+ 'offset': -1,
+ 'length': -1
+ }, options);
+
+ var url = "/logdata/${task_id}/${process}/${run}/${logtype}"
+ + '?offset=' + settings.offset
+ + '&length=' + settings.length;
+ return $.getJSON(url);
+ },
+ 'indicator': $('#indicator')
+ });
+ });
+</script>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/main.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/main.tpl b/src/main/python/apache/thermos/observer/http/templates/main.tpl
new file mode 100644
index 0000000..d3419ab
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/main.tpl
@@ -0,0 +1,101 @@
+<%doc>
+ Template arguments:
+ type
+ offset
+ num
+ tasks
+ task_count
+</%doc>
+
+<%!
+import socket
+import time
+
+try:
+ from twitter.common import app
+ observer_port = app.get_options().twitter_common_http_root_server_port
+except (ImportError, AttributeError) as e:
+ observer_port = 1338
+
+host = socket.gethostname()
+num_tasks = 20
+
+def pretty_time(seconds=time.time()):
+ return time.strftime('%m/%d %H:%M:%S', time.gmtime(seconds))
+
+%>
+
+<div class="${type}-container"
+ data-url="/main/${type}/${offset}/${num}">
+ <div class="row-fluid">
+ <div class="span2">
+ % if offset > 0:
+ <a class="refresh-container" href="#"
+ data-url="/main/${type}/${offset-num_tasks}/${num}">
+ ← newer
+ </a>
+ % endif
+ </div>
+ % if offset + num_tasks < task_count:
+ <div class="span2">
+ <a class="refresh-container" href="#"
+ data-url="/main/${type}/${offset+num_tasks}/${num}">
+ older →
+ </a>
+ </div>
+ % endif
+ </div>
+ <div class="content" id="defaultLayout">
+ <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
+ <thead>
+ <tr>
+ <th colspan=4> ${type} tasks ${offset}...${min(task_count, offset+num_tasks) - 1} of ${task_count} </th>
+ <th colspan=3> resources </th>
+ <th colspan=3> links </th>
+ </tr>
+
+ <tr>
+ <th> name </th> <th> role </th> <th> launched </th> <th> status </th>
+ <th> cpu </th> <th> ram </th> <th> disk </th>
+ <th> task </th> <th> chroot </th> <th> ports </th>
+ </tr>
+ </thead>
+ <tbody>
+
+ % for row in tasks:
+ <tr>
+ <td> ${row["name"]} </td>
+ <td> ${row["role"]} </td>
+ <td> ${pretty_time(row["launch_timestamp"])} </td>
+ <td> ${row["state"]} @
+ ${pretty_time(row["state_timestamp"]) if row["state_timestamp"] else ""}</td>
+
+ <td> ${'%.3f' % row["cpu"] if row["cpu"] > 0 else ""} </td>
+ <td> ${'%.1fMB' % (row["ram"] / 1024. / 1024.) if row["ram"] > 0 else ""} </td>
+ <td> ${'%dGB' % (row["disk"] / 1024 / 1024 / 1024) if row["disk"] > 0 else ""} </td>
+
+ <td> <a href="/task/${row['task_id']}">info</a> </td>
+ <td> <a href="/browse/${row['task_id']}">browse</a> </td>
+ <td>
+ % if type == 'active':
+ % if 'http' in row["ports"]:
+ <a href="http://${host}:${row['ports']['http']}">http</a>
+ % else:
+ <span class="muted">http</span>
+ % endif
+ % if 'admin' in row["ports"]:
+ <a href="http://${host}:${row['ports']['admin']}">admin</a>
+ % else:
+ <span class="muted">admin</span>
+ % endif
+ % if set(row["ports"]) - set(['http', 'admin']):
+ <a href="/task/${row['task_id']}">...</a>
+ % endif
+ % endif
+ </td>
+ </tr>
+ % endfor
+ </tbody>
+ </table>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/process.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/process.tpl b/src/main/python/apache/thermos/observer/http/templates/process.tpl
new file mode 100644
index 0000000..a7fb90d
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/process.tpl
@@ -0,0 +1,109 @@
+<%doc>
+Template arguments:
+ task_id
+ process {
+ cpu:/ram: (optional)
+ cmdline:
+ name:
+ }
+
+ runs = {
+ number: {
+ start_time: (optional)
+ stop_time: (optional)
+ state:
+ }
+ }
+
+ --
+ for each run:
+ run | state | started | finished | stdout | stderr
+</%doc>
+
+<%!
+import socket
+import time
+from xml.sax.saxutils import escape
+
+def pretty_time(seconds=time.time()):
+ return time.strftime('%m/%d %H:%M:%S', time.gmtime(seconds))
+%>
+
+<html>
+<title>thermos(${socket.gethostname()})</title>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="/assets/bootstrap.css"/>
+<body>
+
+
+<div class="container">
+ <div class="row">
+ <div class="span6" id="leftBar">
+ <dl>
+ <dt> process </dt>
+ <dd> <strong> parent task </strong> <a href="/task/${task_id}">${task_id}</a> </dd>
+ <dd> <strong> process name </strong> ${process["name"]} </dd>
+ <dd> <strong> status </strong> ${process["status"]} </dd>
+ </dl>
+ </div>
+
+ <div class="span6" id="rightBar">
+ <dl>
+ <dt> resources </dt>
+ <dd> <strong> cpu </strong> ${'%.3f' % process["cpu"] if "cpu" in process else "N/A"} </dd>
+ <dd> <strong> ram </strong> ${'%.1fMB' % (process["ram"] / 1024. / 1024.) if "ram" in process else "N/A"} </dd>
+ <dd> <strong> total user </strong> ${'%.1fs' % process["user"] if "user" in process else "N/A"} </dd>
+ <dd> <strong> total sys </strong> ${'%.1fs' % process["system"] if "system" in process else "N/A"} </dd>
+ <dd> <strong> threads </strong> ${process["threads"] if "threads" in process else "N/A"} </dd>
+ </dl>
+ </div>
+ </div>
+
+
+ <strong> cmdline </strong><br>
+ <div class="container">
+<pre>
+${escape(process["cmdline"])}
+</pre>
+ </div><br><br>
+
+
+ <strong> runs </strong>
+ <div class="container">
+ <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
+ <thead>
+ <tr>
+ <th colspan=2> </th>
+ <th colspan=2> time </th>
+ <th colspan=2> logs </th>
+ </tr>
+
+ <tr>
+ <th> run </th>
+ <th> status </th>
+ <th> started </th> <th> finished </th>
+ <th> stdout </th> <th> stderr </th>
+ </tr>
+ </thead>
+ <tbody>
+
+ % for run, process_dict in sorted(runs.items(), reverse=True):
+ <tr>
+ <td> ${run} </td>
+ <td> ${process_dict["state"]} </td>
+ <td> ${pretty_time(process_dict["start_time"]) if "start_time" in process_dict else ""} </td>
+ <td> ${pretty_time(process_dict["stop_time"]) if "stop_time" in process_dict else ""} </td>
+ <td> <a href="/logs/${task_id}/${process["name"]}/${run}/stdout">stdout</a> </td>
+ <td> <a href="/logs/${task_id}/${process["name"]}/${run}/stderr">stderr</a> </td>
+ </tr>
+ % endfor
+ </tbody>
+ </table>
+ </div>
+
+</div>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/rawtask.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/rawtask.tpl b/src/main/python/apache/thermos/observer/http/templates/rawtask.tpl
new file mode 100644
index 0000000..c4b7f52
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/rawtask.tpl
@@ -0,0 +1,28 @@
+<%doc>
+ Template arguments:
+ hostname
+ task_id
+ task_struct
+</%doc>
+
+<html>
+<title>thermos(${hostname})</title>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="/assets/bootstrap.css"/>
+<%!
+ from json import dumps
+ def print_task(task):
+ return dumps(task.get(), indent=4)
+%>
+
+<body>
+<div class="container">
+ <h3> task ${task_id} </h3>
+ <div class="content" id="rawTask">
+ <pre>${print_task(task_struct)}</pre>
+ </div>
+</div>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templates/task.tpl
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templates/task.tpl b/src/main/python/apache/thermos/observer/http/templates/task.tpl
new file mode 100644
index 0000000..b71c80d
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templates/task.tpl
@@ -0,0 +1,140 @@
+<%doc>
+ Template arguments:
+ task_id
+ task
+ statuses
+ user
+ ports
+ processes
+ chroot
+ launch_time
+ hostname
+</%doc>
+
+<html>
+<title>thermos(${hostname})</title>
+
+<link rel="stylesheet"
+ type="text/css"
+ href="/assets/bootstrap.css"/>
+
+<body>
+
+<%!
+ import time
+
+ def pretty_time(seconds=time.time()):
+ return time.strftime('%m/%d %H:%M:%S', time.gmtime(seconds))
+
+ def get(task, typ):
+ return task['resource_consumption'][typ]
+
+%>
+
+<div class="container">
+
+ <h3> task ${task_id} </h3>
+
+ <div class="row">
+ <div class="span6" id="leftBar">
+ <dl>
+ <dt> task </dt>
+ <dd> <strong> status </strong> ${statuses[-1][0] if statuses else 'UNKNOWN'} </dd>
+ <dd> <strong> user </strong> ${user} </dd>
+ <dt> ports </dt>
+ % for port_name, port_number in ports.items():
+ <dd> <strong> ${port_name} </strong> <a href="http://${hostname}:${port_number}">${port_number}</a> </dd>
+ %endfor
+ </dl>
+ </div>
+
+ <div class="span6" id="rightBar">
+ <dl>
+ <dt> header </dt>
+ <dd> <strong> chroot </strong> <a href="/browse/${task_id}">browse</a> </dd>
+ <dd> <strong> hostname </strong> <a href="/">${hostname}</a> </dd>
+ <dd> <strong> launch time </strong> ${pretty_time(launch_time)} </dd>
+ <dd> <strong> task config </strong> <a href="/rawtask/${task_id}">view</a> </dd>
+ </dl>
+ </div>
+ </div>
+
+ <div class="row-fluid">
+ <div class="span8" id="taskLayout">
+ <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
+ <thead>
+ <tr>
+ <th colspan=1> task status </th>
+ <th colspan=1> time </th>
+ </tr>
+ </thead>
+
+ <tbody>
+ % for status, timestamp in sorted(statuses, key=lambda status: status[1]):
+ <tr>
+ <td> ${status} </td> <td> ${pretty_time(timestamp)} </td>
+ </tr>
+ % endfor
+ </tbody>
+ </table>
+ </div>
+
+ <div class="span4" id="taskResources">
+ <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
+ <thead>
+ <tr>
+ <th> cpu </th>
+ <th> ram </th>
+ <th> disk </th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td> ${'%.3f' % get(task, 'cpu')} </td>
+ <td> ${'%.1fMB' % (get(task, 'ram') / 1024. / 1024.)} </td>
+ <td> ${'%.1fGB' % (get(task, 'disk') / 1024. / 1024. / 1024.)} </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+
+ <div class="content" id="processesLayout">
+ <table class="table table-bordered table-condensed table-striped" style="empty-cells:show;">
+ <thead>
+ <tr>
+ <th colspan=3> process </th>
+ <th colspan=2> time </th>
+ <th colspan=2> used </th>
+ <th colspan=2> logs </th>
+ </tr>
+
+ <tr>
+ <th> name </th> <th> run </th> <th> status </th> <th> started </th> <th> finished </th>
+ <th> cpu </th> <th> ram </th>
+ <th> stdout </th> <th> stderr </th>
+ </tr>
+ </thead>
+
+ <tbody>
+ % for proc_name, proc in sorted(processes.items(), key=lambda item: item[1].get('start_time')):
+ <tr>
+ <td> <a href="/process/${task_id}/${proc["process_name"]}">${proc["process_name"]}</a> </td>
+ <td> ${proc["process_run"]} </td>
+ <td> ${proc["state"]} </td>
+ <td> ${pretty_time(proc["start_time"]) if "start_time" in proc else ""} </td>
+ <td> ${pretty_time(proc["stop_time"]) if "stop_time" in proc else ""} </td>
+ <td> ${'%.3f' % proc["used"]["cpu"] if "used" in proc else ""} </td>
+ <td> ${'%dMB' % (proc["used"]["ram"] / 1024 / 1024) if "used" in proc else ""} </td>
+ <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stdout">stdout</a> </td>
+ <td> <a href="/logs/${task_id}/${proc["process_name"]}/${proc["process_run"]}/stderr">stderr</a> </td>
+ </tr>
+ % endfor
+ </tbody>
+ </table>
+ </div>
+</div>
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/templating.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/templating.py b/src/main/python/apache/thermos/observer/http/templating.py
new file mode 100644
index 0000000..783308b
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/templating.py
@@ -0,0 +1,9 @@
+import os
+import pkg_resources
+
+
+class HttpTemplate(object):
+ @staticmethod
+ def load(name):
+ return pkg_resources.resource_string(
+ __name__, os.path.join('templates', '%s.tpl' % name))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/observed_task.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/observed_task.py b/src/main/python/apache/thermos/observer/observed_task.py
new file mode 100644
index 0000000..f995998
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/observed_task.py
@@ -0,0 +1,136 @@
+from abc import abstractproperty
+import os
+
+from twitter.common import log
+from twitter.common.lang import AbstractClass
+from twitter.thermos.config.loader import ThermosTaskWrapper
+from twitter.thermos.config.schema import ThermosContext
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+
+from pystachio import Environment
+
+
+class ObservedTask(AbstractClass):
+ """ Represents a Task being observed """
+
+ @classmethod
+ def safe_mtime(cls, path):
+ try:
+ return os.path.getmtime(path)
+ except OSError:
+ return None
+
+ def __init__(self, task_id, pathspec):
+ self._task_id = task_id
+ self._pathspec = pathspec
+ self._mtime = self._get_mtime()
+
+ @abstractproperty
+ def type(self):
+ """Indicates the type of task (active or finished)"""
+
+ def _read_task(self, memoized={}):
+ """Read the corresponding task from disk and return a ThermosTask. Memoizes already-read tasks.
+ """
+ if self._task_id not in memoized:
+ path = self._pathspec.given(task_id=self._task_id, state=self.type).getpath('task_path')
+ if os.path.exists(path):
+ task = ThermosTaskWrapper.from_file(path)
+ if task is None:
+ log.error('Error reading ThermosTask from %s in observer.' % path)
+ else:
+ context = self.context(self._task_id)
+ if not context:
+ log.warning('Task not yet available: %s' % self._task_id)
+ task = task.task() % Environment(thermos=context)
+ memoized[self._task_id] = task
+
+ return memoized.get(self._task_id, None)
+
+ def _get_mtime(self):
+ """Retrieve the mtime of the task's state directory"""
+ get_path = lambda state: self._pathspec.given(
+ task_id=self._task_id, state=state).getpath('task_path')
+ mtime = self.safe_mtime(get_path('active'))
+ if mtime is None:
+ mtime = self.safe_mtime(get_path('finished'))
+ if mtime is None:
+ log.error("Couldn't get mtime for task %s!" % self._task_id)
+ return mtime
+
+ def context(self, task_id):
+ state = self.state
+ if state.header is None:
+ return None
+ return ThermosContext(
+ ports=state.header.ports if state.header.ports else {},
+ task_id=state.header.task_id,
+ user=state.header.user,
+ )
+
+ @property
+ def task(self):
+ """Return a ThermosTask representing this task"""
+ return self._read_task()
+
+ @property
+ def task_id(self):
+ """Return the task's task_id"""
+ return self._task_id
+
+ @property
+ def mtime(self):
+ """Return mtime of task file"""
+ return self._mtime
+
+ @abstractproperty
+ def state(self):
+ """Return state of task (gen.twitter.thermos.ttypes.RunnerState)"""
+
+
+class ActiveObservedTask(ObservedTask):
+ """An active Task known by the TaskObserver"""
+
+ def __init__(self, task_id, pathspec, task_monitor, resource_monitor):
+ super(ActiveObservedTask, self).__init__(task_id, pathspec)
+ self._task_monitor = task_monitor
+ self._resource_monitor = resource_monitor
+
+ @property
+ def type(self):
+ return 'active'
+
+ @property
+ def state(self):
+ """Return a RunnerState representing the current state of task, retrieved from TaskMonitor"""
+ return self.task_monitor.get_state()
+
+ @property
+ def task_monitor(self):
+ """Return a TaskMonitor monitoring this task"""
+ return self._task_monitor
+
+ @property
+ def resource_monitor(self):
+ """Return a ResourceMonitor implementation monitoring this task's resources"""
+ return self._resource_monitor
+
+
+class FinishedObservedTask(ObservedTask):
+ """A finished Task known by the TaskObserver"""
+
+ def __init__(self, task_id, pathspec):
+ super(FinishedObservedTask, self).__init__(task_id, pathspec)
+ self._state = None
+
+ @property
+ def type(self):
+ return 'finished'
+
+ @property
+ def state(self):
+ """Return final state of Task (RunnerState, read from disk and cached for future access)"""
+ if self._state is None:
+ path = self._pathspec.given(task_id=self._task_id).getpath('runner_checkpoint')
+ self._state = CheckpointDispatcher.from_file(path)
+ return self._state
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/task_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/task_observer.py b/src/main/python/apache/thermos/observer/task_observer.py
new file mode 100644
index 0000000..b6ebcf7
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/task_observer.py
@@ -0,0 +1,640 @@
+"""Observe Thermos tasks on a system
+
+This module provides a number of classes for exposing information about running (active) and
+finished Thermos tasks on a system. The primary entry point is the TaskObserver, a thread which
+polls a designated Thermos checkpoint root and collates information about all tasks it discovers.
+
+"""
+from operator import attrgetter
+import os
+import threading
+import time
+
+from twitter.common import log
+from twitter.common.exceptions import ExceptionalThread
+from twitter.common.lang import Lockable
+from twitter.common.quantity import Amount, Time
+
+from twitter.thermos.common.path import TaskPath
+from twitter.thermos.monitoring.detector import TaskDetector
+from twitter.thermos.monitoring.monitor import TaskMonitor
+from twitter.thermos.monitoring.process import ProcessSample
+from twitter.thermos.monitoring.resource import ResourceMonitorBase, TaskResourceMonitor
+
+from gen.twitter.thermos.ttypes import ProcessState, TaskState
+
+from .observed_task import ActiveObservedTask, FinishedObservedTask
+
+
+class TaskObserver(ExceptionalThread, Lockable):
+ """
+ The TaskObserver monitors the thermos checkpoint root for active/finished
+ tasks. It is used to be the oracle of the state of all thermos tasks on
+ a machine.
+
+ It currently returns JSON, but really should just return objects. We should
+ then build an object->json translator.
+ """
+ class UnexpectedError(Exception): pass
+ class UnexpectedState(Exception): pass
+
+ POLLING_INTERVAL = Amount(1, Time.SECONDS)
+
+ def __init__(self, root, resource_monitor_class=TaskResourceMonitor):
+ self._pathspec = TaskPath(root=root)
+ self._detector = TaskDetector(root)
+ if not issubclass(resource_monitor_class, ResourceMonitorBase):
+ raise ValueError("resource monitor class must implement ResourceMonitorBase!")
+ self._resource_monitor = resource_monitor_class
+ self._active_tasks = {} # task_id => ActiveObservedTask
+ self._finished_tasks = {} # task_id => FinishedObservedTask
+ self._stop_event = threading.Event()
+ ExceptionalThread.__init__(self)
+ Lockable.__init__(self)
+ self.daemon = True
+
+ @property
+ def active_tasks(self):
+ """Return a dictionary of active Tasks"""
+ return self._active_tasks
+
+ @property
+ def finished_tasks(self):
+ """Return a dictionary of finished Tasks"""
+ return self._finished_tasks
+
+ @property
+ def all_tasks(self):
+ """Return a dictionary of all Tasks known by the TaskObserver"""
+ return dict(self.active_tasks.items() + self.finished_tasks.items())
+
+ def stop(self):
+ self._stop_event.set()
+
+ def start(self):
+ ExceptionalThread.start(self)
+
+ @Lockable.sync
+ def add_active_task(self, task_id):
+ if task_id in self.finished_tasks:
+ log.error('Found an active task (%s) in finished tasks?' % task_id)
+ return
+ task_monitor = TaskMonitor(self._pathspec, task_id)
+ if not task_monitor.get_state().header:
+ log.info('Unable to load task "%s"' % task_id)
+ return
+ sandbox = task_monitor.get_state().header.sandbox
+ resource_monitor = self._resource_monitor(task_monitor, sandbox)
+ resource_monitor.start()
+ self._active_tasks[task_id] = ActiveObservedTask(
+ task_id=task_id, pathspec=self._pathspec,
+ task_monitor=task_monitor, resource_monitor=resource_monitor
+ )
+
+ @Lockable.sync
+ def add_finished_task(self, task_id):
+ self._finished_tasks[task_id] = FinishedObservedTask(
+ task_id=task_id, pathspec=self._pathspec
+ )
+
+ @Lockable.sync
+ def active_to_finished(self, task_id):
+ self.remove_active_task(task_id)
+ self.add_finished_task(task_id)
+
+ @Lockable.sync
+ def remove_active_task(self, task_id):
+ task = self.active_tasks.pop(task_id)
+ task.resource_monitor.kill()
+
+ @Lockable.sync
+ def remove_finished_task(self, task_id):
+ self.finished_tasks.pop(task_id)
+
+ def run(self):
+ """
+ The internal thread for the observer. This periodically polls the
+ checkpoint root for new tasks, or transitions of tasks from active to
+ finished state.
+ """
+ while not self._stop_event.is_set():
+ time.sleep(self.POLLING_INTERVAL.as_(Time.SECONDS))
+
+ active_tasks = [task_id for _, task_id in self._detector.get_task_ids(state='active')]
+ finished_tasks = [task_id for _, task_id in self._detector.get_task_ids(state='finished')]
+
+ with self.lock:
+
+ # Ensure all tasks currently detected on the system are observed appropriately
+ for active in active_tasks:
+ if active not in self.active_tasks:
+ log.debug('task_id %s (unknown) -> active' % active)
+ self.add_active_task(active)
+ for finished in finished_tasks:
+ if finished in self.active_tasks:
+ log.debug('task_id %s active -> finished' % finished)
+ self.active_to_finished(finished)
+ elif finished not in self.finished_tasks:
+ log.debug('task_id %s (unknown) -> finished' % finished)
+ self.add_finished_task(finished)
+
+ # Remove ObservedTasks for tasks no longer detected on the system
+ for unknown in set(self.active_tasks) - set(active_tasks + finished_tasks):
+ log.debug('task_id %s active -> (unknown)' % unknown)
+ self.remove_active_task(unknown)
+ for unknown in set(self.finished_tasks) - set(active_tasks + finished_tasks):
+ log.debug('task_id %s finished -> (unknown)' % unknown)
+ self.remove_finished_task(unknown)
+
+ @Lockable.sync
+ def process_from_name(self, task_id, process_id):
+ if task_id in self.all_tasks:
+ task = self.all_tasks[task_id].task
+ if task:
+ for process in task.processes():
+ if process.name().get() == process_id:
+ return process
+
+ @Lockable.sync
+ def task_count(self):
+ """
+ Return the count of tasks that could be ready properly from disk.
+ This may be <= self.task_id_count()
+ """
+ return dict(
+ active=len(self.active_tasks),
+ finished=len(self.finished_tasks),
+ all=len(self.all_tasks),
+ )
+
+ @Lockable.sync
+ def task_id_count(self):
+ """
+ Return the raw count of active and finished task_ids from the TaskDetector.
+ """
+ num_active = len(list(self._detector.get_task_ids(state='active')))
+ num_finished = len(list(self._detector.get_task_ids(state='finished')))
+ return dict(active=num_active, finished=num_finished, all=num_active + num_finished)
+
+ def _get_tasks_of_type(self, type):
+ """Convenience function to return all tasks of a given type"""
+ tasks = {
+ 'active': self.active_tasks,
+ 'finished': self.finished_tasks,
+ 'all': self.all_tasks,
+ }.get(type, None)
+
+ if tasks is None:
+ log.error('Unknown task type %s' % type)
+ return {}
+
+ return tasks
+
+ @Lockable.sync
+ def state(self, task_id):
+ """Return a dict containing mapped information about a task's state"""
+ real_state = self.raw_state(task_id)
+ if real_state is None or real_state.header is None:
+ return {}
+ else:
+ return dict(
+ task_id=real_state.header.task_id,
+ launch_time=real_state.header.launch_time_ms / 1000.0,
+ sandbox=real_state.header.sandbox,
+ hostname=real_state.header.hostname,
+ user=real_state.header.user
+ )
+
+ @Lockable.sync
+ def raw_state(self, task_id):
+ """
+ Return the current runner state (thrift blob: gen.twitter.thermos.ttypes.RunnerState)
+ of a given task id
+ """
+ if task_id not in self.all_tasks:
+ return None
+ return self.all_tasks[task_id].state
+
+ @Lockable.sync
+ def _task_processes(self, task_id):
+ """
+ Return the processes of a task given its task_id.
+
+ Returns a map from state to processes in that state, where possible
+ states are: waiting, running, success, failed.
+ """
+ if task_id not in self.all_tasks:
+ return {}
+ state = self.raw_state(task_id)
+ if state is None or state.header is None:
+ return {}
+
+ waiting, running, success, failed, killed = [], [], [], [], []
+ for process, runs in state.processes.items():
+ # No runs ==> nothing started.
+ if len(runs) == 0:
+ waiting.append(process)
+ else:
+ if runs[-1].state in (None, ProcessState.WAITING, ProcessState.LOST):
+ waiting.append(process)
+ elif runs[-1].state in (ProcessState.FORKED, ProcessState.RUNNING):
+ running.append(process)
+ elif runs[-1].state == ProcessState.SUCCESS:
+ success.append(process)
+ elif runs[-1].state == ProcessState.FAILED:
+ failed.append(process)
+ elif runs[-1].state == ProcessState.KILLED:
+ killed.append(process)
+ else:
+ # TODO(wickman) Consider log.error instead of raising.
+ raise self.UnexpectedState(
+ "Unexpected ProcessHistoryState: %s" % state.processes[process].state)
+
+ return dict(waiting=waiting, running=running, success=success, failed=failed, killed=killed)
+
+ @Lockable.sync
+ def main(self, type=None, offset=None, num=None):
+ """Return a set of information about tasks, optionally filtered
+
+ Args:
+ type = (all|active|finished|None) [default: all]
+ offset = offset into the list of task_ids [default: 0]
+ num = number of results to return [default: 20]
+
+ Tasks are sorted by interest:
+ - active tasks are sorted by start time
+ - finished tasks are sorted by completion time
+
+ Returns:
+ {
+ tasks: [task_id_1, ..., task_id_N],
+ type: query type,
+ offset: next offset,
+ num: next num
+ }
+
+ """
+ type = type or 'all'
+ offset = offset or 0
+ num = num or 20
+
+ # Get a list of all ObservedTasks of requested type
+ tasks = sorted((task for task in self._get_tasks_of_type(type).values()),
+ key=attrgetter('mtime'), reverse=True)
+
+ # Filter by requested offset + number of results
+ end = num
+ if offset < 0:
+ offset = offset % len(tasks) if len(tasks) > abs(offset) else 0
+ end += offset
+ tasks = tasks[offset:end]
+
+ def task_row(observed_task):
+ """Generate an output row for a Task"""
+ task = self._task(observed_task.task_id)
+ # tasks include those which could not be found properly and are hence empty {}
+ if task:
+ return dict(
+ task_id=observed_task.task_id,
+ name=task['name'],
+ role=task['user'],
+ launch_timestamp=task['launch_timestamp'],
+ state=task['state'],
+ state_timestamp=task['state_timestamp'],
+ ports=task['ports'],
+ **task['resource_consumption'])
+
+ return dict(
+ tasks=filter(None, map(task_row, tasks)),
+ type=type,
+ offset=offset,
+ num=num,
+ task_count=self.task_count()[type],
+ )
+
+ def _sample(self, task_id):
+ if task_id not in self.active_tasks:
+ log.debug("Task %s not found in active tasks" % task_id)
+ sample = ProcessSample.empty().to_dict()
+ sample['disk'] = 0
+ else:
+ resource_sample = self.active_tasks[task_id].resource_monitor.sample()[1]
+ sample = resource_sample.process_sample.to_dict()
+ sample['disk'] = resource_sample.disk_usage
+ log.debug("Got sample for task %s: %s" % (task_id, sample))
+ return sample
+
+ @Lockable.sync
+ def task_statuses(self, task_id):
+ """
+ Return the sequence of task states.
+
+ [(task_state [string], timestamp), ...]
+ """
+
+ # Unknown task_id.
+ if task_id not in self.all_tasks:
+ return []
+
+ task = self.all_tasks[task_id]
+ if task is None:
+ return []
+
+ state = self.raw_state(task_id)
+ if state is None or state.header is None:
+ return []
+
+ # Get the timestamp of the transition into the current state.
+ return [
+ (TaskState._VALUES_TO_NAMES.get(st.state, 'UNKNOWN'), st.timestamp_ms / 1000)
+ for st in state.statuses]
+
+ @Lockable.sync
+ def _task(self, task_id):
+ """
+ Return composite information about a particular task task_id, given the below
+ schema.
+
+ {
+ task_id: string,
+ name: string,
+ user: string,
+ launch_timestamp: seconds,
+ state: string [ACTIVE, SUCCESS, FAILED]
+ ports: { name1: 'url', name2: 'url2' }
+ resource_consumption: { cpu:, ram:, disk: }
+ processes: { -> names only
+ waiting: [],
+ running: [],
+ success: [],
+ failed: []
+ }
+ }
+ """
+ # Unknown task_id.
+ if task_id not in self.all_tasks:
+ return {}
+
+ task = self.all_tasks[task_id].task
+ if task is None:
+ # TODO(wickman) Can this happen?
+ log.error('Could not find task: %s' % task_id)
+ return {}
+
+ state = self.raw_state(task_id)
+ if state is None or state.header is None:
+ # TODO(wickman) Can this happen?
+ return {}
+
+ # Get the timestamp of the transition into the current state.
+ current_state = state.statuses[-1].state
+ last_state = state.statuses[0]
+ state_timestamp = 0
+ for status in state.statuses:
+ if status.state == current_state and last_state != current_state:
+ state_timestamp = status.timestamp_ms / 1000
+ last_state = status.state
+
+ return dict(
+ task_id=task_id,
+ name=task.name().get(),
+ launch_timestamp=state.statuses[0].timestamp_ms / 1000,
+ state=TaskState._VALUES_TO_NAMES[state.statuses[-1].state],
+ state_timestamp=state_timestamp,
+ user=state.header.user,
+ resource_consumption=self._sample(task_id),
+ ports=state.header.ports,
+ processes=self._task_processes(task_id),
+ task_struct=task,
+ )
+
+ @Lockable.sync
+ def _get_process_resource_consumption(self, task_id, process_name):
+ if task_id not in self.active_tasks:
+ log.debug("Task %s not found in active tasks" % task_id)
+ return ProcessSample.empty().to_dict()
+ sample = self.active_tasks[task_id].resource_monitor.sample_by_process(process_name).to_dict()
+ log.debug('Resource consumption (%s, %s) => %s' % (task_id, process_name, sample))
+ return sample
+
+ @Lockable.sync
+ def _get_process_tuple(self, history, run):
+ """
+ Return the basic description of a process run if it exists, otherwise
+ an empty dictionary.
+
+ {
+ process_name: string
+ process_run: int
+ state: string [WAITING, FORKED, RUNNING, SUCCESS, KILLED, FAILED, LOST]
+ (optional) start_time: seconds from epoch
+ (optional) stop_time: seconds from epoch
+ }
+ """
+ if len(history) == 0:
+ return {}
+ if run >= len(history):
+ return {}
+ else:
+ process_run = history[run]
+ run = run % len(history)
+ d = dict(
+ process_name=process_run.process,
+ process_run=run,
+ state=ProcessState._VALUES_TO_NAMES[process_run.state],
+ )
+ if process_run.start_time:
+ d.update(start_time=process_run.start_time)
+ if process_run.stop_time:
+ d.update(stop_time=process_run.stop_time)
+ return d
+
+ @Lockable.sync
+ def process(self, task_id, process, run=None):
+ """
+ Returns a process run, where the schema is given below:
+
+ {
+ process_name: string
+ process_run: int
+ used: { cpu: float, ram: int bytes, disk: int bytes }
+ start_time: (time since epoch in millis (utc))
+ stop_time: (time since epoch in millis (utc))
+ state: string [WAITING, FORKED, RUNNING, SUCCESS, KILLED, FAILED, LOST]
+ }
+
+ If run is None, return the latest run.
+ """
+ state = self.raw_state(task_id)
+ if state is None or state.header is None:
+ return {}
+ if process not in state.processes:
+ return {}
+ history = state.processes[process]
+ run = int(run) if run is not None else -1
+ tup = self._get_process_tuple(history, run)
+ if not tup:
+ return {}
+ if tup.get('state') == 'RUNNING':
+ tup.update(used=self._get_process_resource_consumption(task_id, process))
+ return tup
+
+ @Lockable.sync
+ def _processes(self, task_id):
+ """
+ Return
+ {
+ process1: { ... }
+ process2: { ... }
+ ...
+ processN: { ... }
+ }
+
+ where processK is the latest run of processK and in the schema as
+ defined by process().
+ """
+
+ if task_id not in self.all_tasks:
+ return {}
+ state = self.raw_state(task_id)
+ if state is None or state.header is None:
+ return {}
+
+ processes = self._task_processes(task_id)
+ d = dict()
+ for process_type in processes:
+ for process_name in processes[process_type]:
+ d[process_name] = self.process(task_id, process_name)
+ return d
+
+ @Lockable.sync
+ def processes(self, task_ids):
+ """
+ Given a list of task_ids, returns a map of task_id => processes, where processes
+ is defined by the schema in _processes.
+ """
+ if not isinstance(task_ids, (list, tuple)):
+ return {}
+ return dict((task_id, self._processes(task_id)) for task_id in task_ids)
+
+ @Lockable.sync
+ def get_run_number(self, runner_state, process, run=None):
+ if runner_state is not None and runner_state.processes is not None:
+ run = run if run is not None else -1
+ if run < len(runner_state.processes[process]):
+ if len(runner_state.processes[process]) > 0:
+ return run % len(runner_state.processes[process])
+
+ @Lockable.sync
+ def logs(self, task_id, process, run=None):
+ """
+ Given a task_id and a process and (optional) run number, return a dict:
+ {
+ stderr: [dir, filename]
+ stdout: [dir, filename]
+ }
+
+ If the run number is unspecified, uses the latest run.
+
+ TODO(wickman) Just return the filenames directly?
+ """
+ runner_state = self.raw_state(task_id)
+ if runner_state is None or runner_state.header is None:
+ return {}
+ run = self.get_run_number(runner_state, process, run)
+ if run is None:
+ return {}
+ log_path = self._pathspec.given(task_id=task_id, process=process, run=run,
+ log_dir=runner_state.header.log_dir).getpath('process_logdir')
+ return dict(
+ stdout=[log_path, 'stdout'],
+ stderr=[log_path, 'stderr']
+ )
+
+ @staticmethod
+ def _sanitize_path(base_path, relpath):
+ """
+ Attempts to sanitize a path through path normalization, also making sure
+ that the relative path is contained inside of base_path.
+ """
+ if relpath is None:
+ relpath = "."
+ normalized_base = os.path.realpath(base_path)
+ normalized = os.path.realpath(os.path.join(base_path, relpath))
+ if normalized.startswith(normalized_base):
+ return (normalized_base, os.path.relpath(normalized, normalized_base))
+ return (None, None)
+
+ @Lockable.sync
+ def valid_file(self, task_id, path):
+ """
+ Like valid_path, but also verify the given path is a file
+ """
+ chroot, path = self.valid_path(task_id, path)
+ if chroot and path and os.path.isfile(os.path.join(chroot, path)):
+ return chroot, path
+ return None, None
+
+ @Lockable.sync
+ def valid_path(self, task_id, path):
+ """
+ Given a task_id and a path within that task_id's sandbox, verify:
+ (1) it's actually in the sandbox and not outside
+ (2) it's a valid, existing path
+ Returns chroot and the pathname relative to that chroot.
+ """
+ runner_state = self.raw_state(task_id)
+ if runner_state is None or runner_state.header is None:
+ return None, None
+ try:
+ chroot = runner_state.header.sandbox
+ except AttributeError:
+ return None, None
+ chroot, path = self._sanitize_path(chroot, path)
+ if chroot and path:
+ return chroot, path
+ return None, None
+
+ @Lockable.sync
+ def files(self, task_id, path=None):
+ """
+ Returns dictionary
+ {
+ task_id: task_id
+ chroot: absolute directory on machine
+ path: sanitized relative path w.r.t. chroot
+ dirs: list of directories
+ files: list of files
+ }
+ """
+ # TODO(jon): DEPRECATED: most of the necessary logic is handled directly in the templates.
+ # Also, global s/chroot/sandbox/?
+ empty = dict(task_id=task_id, chroot=None, path=None, dirs=None, files=None)
+ path = path if path is not None else '.'
+ runner_state = self.raw_state(task_id)
+ if runner_state is None:
+ return empty
+ try:
+ chroot = runner_state.header.sandbox
+ except AttributeError:
+ return empty
+ if chroot is None: # chroot-less job
+ return empty
+ chroot, path = self._sanitize_path(chroot, path)
+ if (chroot is None or path is None
+ or not os.path.isdir(os.path.join(chroot, path))):
+ return empty
+ names = os.listdir(os.path.join(chroot, path))
+ dirs, files = [], []
+ for name in names:
+ if os.path.isdir(os.path.join(chroot, path, name)):
+ dirs.append(name)
+ else:
+ files.append(name)
+ return dict(
+ task_id=task_id,
+ chroot=chroot,
+ path=path,
+ dirs=dirs,
+ files=files
+ )
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/testing/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/testing/BUILD b/src/main/python/apache/thermos/testing/BUILD
new file mode 100644
index 0000000..b5485d0
--- /dev/null
+++ b/src/main/python/apache/thermos/testing/BUILD
@@ -0,0 +1,14 @@
+python_library(
+ name = 'runner',
+ sources = ['runner.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/contextutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/python/twitter/thermos/config'),
+ pants('src/main/python/twitter/thermos/core'),
+ pants('src/main/python/twitter/thermos:thrift'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/testing/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/testing/__init__.py b/src/main/python/apache/thermos/testing/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/testing/runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/testing/runner.py b/src/main/python/apache/thermos/testing/runner.py
new file mode 100644
index 0000000..930f41b
--- /dev/null
+++ b/src/main/python/apache/thermos/testing/runner.py
@@ -0,0 +1,191 @@
+from __future__ import print_function
+
+import atexit
+import errno
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+from twitter.common import log
+from twitter.common.contextutil import temporary_file, environment_as
+from twitter.thermos.common.path import TaskPath
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+from twitter.thermos.config.loader import ThermosTaskWrapper
+from thrift.TSerialization import deserialize as thrift_deserialize
+
+from gen.twitter.thermos.ttypes import (
+ TaskState,
+ RunnerCkpt,
+ RunnerState,
+)
+
+
+class Runner(object):
+ RUN_JOB_SCRIPT = """
+import os
+import random
+import sys
+from twitter.common import log
+from twitter.common.log.options import LogOptions
+from twitter.thermos.config.loader import ThermosConfigLoader
+from twitter.thermos.core.helper import TaskRunnerHelper
+from twitter.thermos.core.runner import TaskRunner, TaskRunnerUniversalHandler
+from thrift.TSerialization import serialize as thrift_serialize
+
+random.seed(%(random_seed)d)
+
+log.init('runner_base')
+LogOptions.set_disk_log_level('DEBUG')
+
+task = ThermosConfigLoader.load_json('%(filename)s')
+task = task.tasks()[0].task
+
+success_rate=%(success_rate)d
+
+class AngryHandler(TaskRunnerUniversalHandler):
+ def checkpoint(self, record):
+ if not self._runner._recovery:
+ if random.randint(0, 100) <= success_rate:
+ super(AngryHandler, self).checkpoint(record)
+ else:
+ sys.exit(1)
+
+sandbox = os.path.join('%(sandbox)s', '%(task_id)s')
+args = {}
+args['task_id'] = '%(task_id)s'
+if %(portmap)s:
+ args['portmap'] = %(portmap)s
+args['universal_handler'] = AngryHandler
+
+runner = TaskRunner(task, '%(root)s', sandbox, **args)
+runner.run()
+
+with open('%(state_filename)s', 'w') as fp:
+ fp.write(thrift_serialize(runner.state))
+"""
+
+ def __init__(self, task, portmap={}, success_rate=100, random_seed=31337):
+ """
+ task = Thermos task
+ portmap = port map
+ success_rate = success rate of writing checkpoint to disk
+ """
+ self.task = task
+
+ with temporary_file(cleanup=False) as fp:
+ self.job_filename = fp.name
+ fp.write(ThermosTaskWrapper(task).to_json())
+
+ self.state_filename = tempfile.mktemp()
+ self.tempdir = tempfile.mkdtemp()
+ self.task_id = '%s-runner-base' % int(time.time()*1000000)
+ self.sandbox = os.path.join(self.tempdir, 'sandbox')
+ self.portmap = portmap
+ self.cleaned = False
+ self.pathspec = TaskPath(root = self.tempdir, task_id = self.task_id)
+ self.script_filename = None
+ self.success_rate = success_rate
+ self.random_seed = random_seed
+ self._run_count = 0
+
+ @property
+ def pid(self):
+ return self.po.pid
+
+ @property
+ def root(self):
+ return self.tempdir
+
+ def run(self):
+ self._run_count += 1
+ atexit.register(self.cleanup)
+
+ if self.script_filename:
+ os.unlink(self.script_filename)
+
+ with temporary_file(cleanup=False) as fp:
+ self.script_filename = fp.name
+ fp.write(self.RUN_JOB_SCRIPT % {
+ 'filename': self.job_filename,
+ 'sandbox': self.sandbox,
+ 'root': self.tempdir,
+ 'task_id': self.task_id,
+ 'state_filename': self.state_filename,
+ 'portmap': repr(self.portmap),
+ 'success_rate': self.success_rate,
+ 'random_seed': self.random_seed + self._run_count,
+ })
+
+ with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
+ self.po = subprocess.Popen([sys.executable, self.script_filename],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ try:
+ so, se = self.po.communicate()
+ except OSError as e:
+ if e.errno == errno.ECHILD:
+ so = se = 'Killed'
+ else:
+ raise
+
+ rc = self.po.returncode
+ if rc != 0:
+ if os.path.exists(self.job_filename):
+ config = open(self.job_filename).read()
+ else:
+ config = 'Nonexistent!'
+ if 'THERMOS_DEBUG' in os.environ:
+ print("Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n" % (
+ config, so, se))
+
+ try:
+ with open(self.state_filename, 'r') as fp:
+ self.state = thrift_deserialize(RunnerState(), fp.read())
+ except Exception as e:
+ if 'THERMOS_DEBUG' in os.environ:
+ print('Failed to load Runner state: %s' % e, file=sys.stderr)
+ self.state = RunnerState()
+
+ try:
+ self.reconstructed_state = CheckpointDispatcher.from_file(
+ self.pathspec.getpath('runner_checkpoint'))
+ except:
+ self.reconstructed_state = None
+ self.initialized = True
+ return rc
+
+ def cleanup(self):
+ if not self.cleaned:
+ if hasattr(self, 'po'):
+ try:
+ self.po.kill()
+ except:
+ pass
+ os.unlink(self.job_filename)
+ os.unlink(self.script_filename)
+ if 'THERMOS_DEBUG' not in os.environ:
+ shutil.rmtree(self.tempdir, ignore_errors=True)
+ else:
+ print('Logs saved in %s' % self.tempdir)
+ self.cleaned = True
+
+
+class RunnerTestBase(object):
+ @classmethod
+ def task(cls):
+ raise NotImplementedError
+
+ @classmethod
+ def setup_class(cls):
+ cls.runner = Runner(cls.task(), portmap=getattr(cls, 'portmap', {}))
+ cls.runner.run()
+ cls.state = cls.runner.state
+
+ @classmethod
+ def teardown_class(cls):
+ cls.runner.cleanup()
+
+ def test_runner_state_reconstruction(self):
+ assert self.state == self.runner.reconstructed_state
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/__init__.py b/src/main/python/twitter/__init__.py
deleted file mode 100644
index de40ea7..0000000
--- a/src/main/python/twitter/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__import__('pkg_resources').declare_namespace(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/BUILD b/src/main/python/twitter/aurora/BUILD
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/BUILD.thirdparty
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/BUILD.thirdparty b/src/main/python/twitter/aurora/BUILD.thirdparty
deleted file mode 100644
index 7a43aca..0000000
--- a/src/main/python/twitter/aurora/BUILD.thirdparty
+++ /dev/null
@@ -1,19 +0,0 @@
-def make_dep(name, version, dependency_name=None):
- """Build a target from a specified dependency tuple.
-
- name is the target name, specified in other BUILD files.
- version is a hardcoded version string
- dependency_name is used to identify the specific binary to resolve
- """
- dependency_name = dependency_name or name
- versioned_name = "%s==%s" % (dependency_name, version)
- python_requirement(requirement=versioned_name, name=name)
-
-make_dep('argparse', '1.2.1')
-make_dep('mesos-core', '0.15.0-rc4', 'mesos')
-make_dep('mock', '1.0.1')
-make_dep('mox', '0.5.3')
-make_dep('psutil', '1.1.2')
-make_dep('pystachio', '0.7.2')
-make_dep('pyyaml', '3.10', 'PyYAML')
-make_dep('thrift', '0.9.1')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/__init__.py b/src/main/python/twitter/aurora/__init__.py
deleted file mode 100644
index b0d6433..0000000
--- a/src/main/python/twitter/aurora/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/admin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/admin/BUILD b/src/main/python/twitter/aurora/admin/BUILD
deleted file mode 100644
index c8089b4..0000000
--- a/src/main/python/twitter/aurora/admin/BUILD
+++ /dev/null
@@ -1,11 +0,0 @@
-python_library(
- name = 'mesos_maintenance',
- sources = 'mesos_maintenance.py',
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/aurora/client:api'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/admin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/admin/__init__.py b/src/main/python/twitter/aurora/admin/__init__.py
deleted file mode 100644
index e69de29..0000000
[25/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/hooks/hooked_api.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/hooks/hooked_api.py b/src/main/python/apache/aurora/client/hooks/hooked_api.py
new file mode 100644
index 0000000..cc4d3db
--- /dev/null
+++ b/src/main/python/apache/aurora/client/hooks/hooked_api.py
@@ -0,0 +1,167 @@
+import functools
+import traceback
+
+from twitter.common import log
+from twitter.aurora.client.api import AuroraClientAPI
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+
+from gen.twitter.aurora.ttypes import ResponseCode
+
+
+def _partial(function, *args, **kw):
+ """Returns a partial function __name__ inherited from parent function."""
+ partial = functools.partial(function, *args, **kw)
+ return functools.update_wrapper(partial, function)
+
+
+class HookConfig(object):
+ def __init__(self, config, job_key):
+ self.config = config
+ self.job_key = job_key or (config.job_key() if config is not None else None)
+
+ def __iter__(self):
+ yield self.config
+ yield self.job_key
+
+
+class NonHookedAuroraClientAPI(AuroraClientAPI):
+ """
+ This wraps those AuroraClientAPI methods that don't have an AuroraConfig 'config' param
+ to take an optional 'config' param which:
+ * contains the configured hooks (config.hooks)
+ * is dropped before the call is proxied to AuroraClientAPI
+ * is thus available to API methods in subclasses
+ """
+
+ def cancel_update(self, job_key, config=None):
+ return super(NonHookedAuroraClientAPI, self).cancel_update(job_key)
+
+ def kill_job(self, job_key, instances=None, lock=None, config=None):
+ return super(NonHookedAuroraClientAPI, self).kill_job(job_key, instances=instances, lock=lock)
+
+ def restart(self, job_key, shards, updater_config, health_check_interval_seconds, config=None):
+ return super(NonHookedAuroraClientAPI, self).restart(job_key, shards, updater_config,
+ health_check_interval_seconds)
+
+ def start_cronjob(self, job_key, config=None):
+ return super(NonHookedAuroraClientAPI, self).start_cronjob(job_key)
+
+
+class HookedAuroraClientAPI(NonHookedAuroraClientAPI):
+ """
+ Adds a hooking aspect/behaviour to the lifecycle of Mesos Client API methods
+ by injecting hooks (instances of twitter.aurora.client.hooks.Hooks)
+
+ * Hooks are available in the 'config' (AuroraConfig) param that each API call receives
+ * Each Hook is run around each API call:
+ * 'pre' hook before the call
+ * 'post' hook if the call succeeds
+ * 'err' hook if the call fails
+ * If the hook itself fails, then it is treated as a WARN rather than an ERROR
+ """
+
+ class Error(Exception): pass
+ class PreHooksStoppedCall(Error): pass
+ class APIError(Error):
+ def __init__(self, response):
+ self.response = response
+
+ def __str__(self):
+ return '%s: %s: %s' % (self.__class__.__name__,
+ ResponseCode._VALUES_TO_NAMES.get(self.response.responseCode, 'UNKNOWN'),
+ self.response.message)
+
+ @classmethod
+ def _meta_hook(cls, hook, hook_method):
+ def callback():
+ if hook_method is None:
+ return True
+ log.debug('Running %s in %s' % (hook_method.__name__, hook.__class__.__name__))
+ hook_result = False
+ try:
+ hook_result = hook_method()
+ if not hook_result:
+ log.debug('%s in %s returned False' % (hook_method.__name__,
+ hook.__class__.__name__))
+ except Exception:
+ log.warn('Error in %s in %s' %
+ (hook_method.__name__, hook.__class__.__name__))
+ log.warn(traceback.format_exc())
+ return hook_result
+ return callback
+
+ @classmethod
+ def _generate_method(cls, hook, config, job_key, event, method, extra_argument=None):
+ method_name, args, kw = method.__name__, method.args, method.keywords
+ kw = kw or {}
+ hook_method = getattr(hook, '%s_%s' % (event, method_name), None)
+ if callable(hook_method):
+ if extra_argument is not None:
+ hook_method = _partial(hook_method, extra_argument)
+ return _partial(hook_method, *args, **kw)
+ else:
+ hook_method = getattr(hook, 'generic_hook', None)
+ if hook_method is None:
+ return None
+ hook_method = _partial(hook_method, HookConfig(config, job_key),
+ event, method_name, extra_argument)
+ return _partial(hook_method, args, kw)
+
+ @classmethod
+ def _yield_hooks(cls, event, config, job_key, api_call, extra_argument=None):
+ hooks = config.hooks if config and config.raw().enable_hooks().get() else ()
+ for hook in hooks:
+ yield cls._meta_hook(hook,
+ cls._generate_method(hook, config, job_key, event, api_call, extra_argument))
+
+ @classmethod
+ def _invoke_hooks(cls, event, config, job_key, api_call, extra_argument=None):
+ hooks_passed = [hook() for hook in cls._yield_hooks(event, config, job_key, api_call,
+ extra_argument)]
+ return all(hooks_passed)
+
+ def _hooked_call(self, config, job_key, api_call):
+ if not self._invoke_hooks('pre', config, job_key, api_call):
+ raise self.PreHooksStoppedCall('Pre hooks stopped call to %s' % api_call.__name__)
+
+ try:
+ resp = api_call()
+ except Exception as e:
+ self._invoke_hooks('err', config, job_key, api_call, e)
+ raise # propagate since the API method call failed for unknown reasons
+
+ if resp.responseCode != ResponseCode.OK:
+ self._invoke_hooks('err', config, job_key, api_call, self.APIError(resp))
+ else:
+ self._invoke_hooks('post', config, job_key, api_call, resp)
+
+ return resp
+
+ def create_job(self, config, lock=None):
+ return self._hooked_call(config, None,
+ _partial(super(HookedAuroraClientAPI, self).create_job, config, lock))
+
+ def cancel_update(self, job_key, config=None):
+ return self._hooked_call(config, job_key,
+ _partial(super(HookedAuroraClientAPI, self).cancel_update,
+ job_key, config=config))
+
+ def kill_job(self, job_key, instances=None, lock=None, config=None):
+ return self._hooked_call(config, job_key,
+ _partial(super(HookedAuroraClientAPI, self).kill_job,
+ job_key, instances=instances, lock=lock, config=config))
+
+ def restart(self, job_key, shards, updater_config, health_check_interval_seconds, config=None):
+ return self._hooked_call(config, job_key,
+ _partial(super(HookedAuroraClientAPI, self).restart,
+ job_key, shards, updater_config, health_check_interval_seconds, config=config))
+
+ def start_cronjob(self, job_key, config=None):
+ return self._hooked_call(config, job_key,
+ _partial(super(HookedAuroraClientAPI, self).start_cronjob,
+ job_key, config=config))
+
+ def update_job(self, config, health_check_interval_seconds=3, instances=None):
+ return self._hooked_call(config, None,
+ _partial(super(HookedAuroraClientAPI, self).update_job,
+ config, health_check_interval_seconds=health_check_interval_seconds, instances=instances))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/options.py b/src/main/python/apache/aurora/client/options.py
new file mode 100644
index 0000000..7ced961
--- /dev/null
+++ b/src/main/python/apache/aurora/client/options.py
@@ -0,0 +1,199 @@
+import optparse
+
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+from twitter.thermos.common.options import add_binding_to
+
+
+__all__ = (
+ 'CLUSTER_CONFIG_OPTION',
+ 'CLUSTER_INVOKE_OPTION',
+ 'ENVIRONMENT_BIND_OPTION',
+ 'ENV_CONFIG_OPTION',
+ 'EXECUTOR_SANDBOX_OPTION',
+ 'FROM_JOBKEY_OPTION',
+ 'HEALTH_CHECK_INTERVAL_SECONDS_OPTION',
+ 'JSON_OPTION',
+ 'OPEN_BROWSER_OPTION',
+ 'SHARDS_OPTION',
+ 'SSH_USER_OPTION',
+ 'WAIT_UNTIL_OPTION',
+)
+
+
+def add_verbosity_options():
+ from twitter.common import app
+ from twitter.common.log.options import LogOptions
+
+ def set_quiet(option, _1, _2, parser):
+ setattr(parser.values, option.dest, 'quiet')
+ LogOptions.set_stderr_log_level('NONE')
+
+ def set_verbose(option, _1, _2, parser):
+ setattr(parser.values, option.dest, 'verbose')
+ LogOptions.set_stderr_log_level('DEBUG')
+
+ app.add_option('-v',
+ dest='verbosity',
+ default='normal',
+ action='callback',
+ callback=set_verbose,
+ help='Verbose logging. (default: %default)')
+
+ app.add_option('-q',
+ dest='verbosity',
+ default='normal',
+ action='callback',
+ callback=set_quiet,
+ help='Quiet logging. (default: %default)')
+
+
+def parse_shards_into(option, opt, value, parser):
+ """Parse lists of shard or shard ranges into a set().
+
+ Examples:
+ 0-2
+ 0,1-3,5
+ 1,3,5
+ """
+ def shard_range_parser(shards):
+ result = set()
+ for part in shards.split(','):
+ x = part.split('-')
+ result.update(range(int(x[0]), int(x[-1]) + 1))
+ return sorted(result)
+
+ try:
+ setattr(parser.values, option.dest, shard_range_parser(value))
+ except ValueError as e:
+ raise optparse.OptionValueError('Failed to parse: %s' % e)
+
+
+def parse_aurora_job_key_into(option, opt, value, parser):
+ try:
+ setattr(parser.values, option.dest, AuroraJobKey.from_path(value))
+ except AuroraJobKey.Error as e:
+ raise optparse.OptionValueError('Failed to parse: %s' % e)
+
+
+def make_env_option(explanation):
+ return optparse.Option(
+ '--env',
+ dest='env',
+ default=None,
+ help=explanation)
+
+
+OPEN_BROWSER_OPTION = optparse.Option(
+ '-o',
+ '--open_browser',
+ dest='open_browser',
+ action='store_true',
+ default=False,
+ help='Open a browser window to the job page after a job mutation.')
+
+
+SHARDS_OPTION = optparse.Option(
+ '--shards',
+ type='string',
+ dest='shards',
+ default=None,
+ action='callback',
+ callback=parse_shards_into,
+ help='A list of shard ids to act on. Can either be a comma-separated list (e.g. 0,1,2) '
+ 'or a range (e.g. 0-2) or any combination of the two (e.g. 0-2,5,7-9). If not set, '
+ 'all shards will be acted on.')
+
+
+FROM_JOBKEY_OPTION = optparse.Option('--from', dest='rename_from', type='string', default=None,
+ metavar='CLUSTER/ROLE/ENV/JOB', action='callback', callback=parse_aurora_job_key_into,
+ help='Job key to diff against.')
+
+
+JSON_OPTION = optparse.Option(
+ '-j',
+ '--json',
+ dest='json',
+ default=False,
+ action='store_true',
+ help='If specified, configuration is read in JSON format.')
+
+
+CLUSTER_CONFIG_OPTION = optparse.Option(
+ '--cluster',
+ dest='cluster',
+ default=None,
+ type='string',
+ help='Cluster to match when selecting a job from a configuration. Optional if only one job '
+ 'matching the given job name exists in the config.')
+
+
+CLUSTER_INVOKE_OPTION = optparse.Option(
+ '--cluster',
+ dest='cluster',
+ default=None,
+ type='string',
+ help='Cluster to invoke this command against. Deprecated in favor of the CLUSTER/ROLE/ENV/NAME '
+ 'syntax.')
+
+
+ENV_CONFIG_OPTION = make_env_option(
+ 'Environment to match when selecting a job from a configuration.')
+
+
+# This is for binding arbitrary points in the Thermos namespace to specific strings, e.g.
+# if a Thermos configuration has {{jvm.version}}, it can be bound explicitly from the
+# command-line with, for example, -E jvm.version=7
+ENVIRONMENT_BIND_OPTION = optparse.Option(
+ '-E',
+ type='string',
+ nargs=1,
+ action='callback',
+ default=[],
+ metavar='NAME=VALUE',
+ callback=add_binding_to('bindings'),
+ dest='bindings',
+ help='Bind a thermos mustache variable name to a value. '
+ 'Multiple flags may be used to specify multiple values.')
+
+
+EXECUTOR_SANDBOX_OPTION = optparse.Option(
+ '-e',
+ '--executor_sandbox',
+ action='store_true',
+ default=False,
+ dest='executor_sandbox',
+ help='Run the command in the executor sandbox instead of the task sandbox.')
+
+
+SSH_USER_OPTION = optparse.Option(
+ '--user',
+ dest='ssh_user',
+ default=None,
+ help="ssh as this user instead of the role.")
+
+
+CREATE_STATES = (
+ 'PENDING',
+ 'RUNNING',
+ 'FINISHED'
+)
+
+
+WAIT_UNTIL_OPTION = optparse.Option(
+ '--wait_until',
+ default='PENDING',
+ type='choice',
+ choices=('PENDING', 'RUNNING', 'FINISHED'),
+ metavar='STATE',
+ dest='wait_until',
+ help='Block the client until all the tasks have transitioned into the '
+ 'requested state. Options: %s. Default: %%default' % (', '.join(CREATE_STATES)))
+
+
+HEALTH_CHECK_INTERVAL_SECONDS_OPTION = optparse.Option(
+ '--updater_health_check_interval_seconds',
+ dest='health_check_interval_seconds',
+ type=int,
+ default=3,
+ help='Time interval between subsequent shard status checks.'
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/BUILD b/src/main/python/apache/aurora/common/BUILD
new file mode 100644
index 0000000..4e839f7
--- /dev/null
+++ b/src/main/python/apache/aurora/common/BUILD
@@ -0,0 +1,63 @@
+import os
+
+python_library(
+ name = 'aurora_job_key',
+ sources = ['aurora_job_key.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'cluster',
+ sources = ['cluster.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ ]
+)
+
+python_library(
+ name = 'clusters',
+ sources = ['clusters.py'],
+ dependencies = [
+ pants(':cluster'),
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ pants('aurora/twitterdeps/src/python/twitter/common/collections'),
+ ]
+)
+
+python_library(
+ name = 'cluster_option',
+ sources = ['cluster_option.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ ]
+)
+
+python_library(
+ name = 'http_signaler',
+ sources = ['http_signaler.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ ]
+)
+
+python_library(
+ name = 'common',
+ dependencies = [
+ pants(':aurora_job_key'),
+ pants(':cluster'),
+ pants(':cluster_option'),
+ pants(':clusters'),
+ pants(':http_signaler'),
+ pants('src/main/python/twitter/aurora/common/auth'),
+ ],
+ provides = setup_py(
+ name = 'twitter.aurora.common',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ description = 'Aurora common libraries.',
+ license = 'Apache License, Version 2.0',
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/__init__.py b/src/main/python/apache/aurora/common/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/aurora_job_key.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/aurora_job_key.py b/src/main/python/apache/aurora/common/aurora_job_key.py
new file mode 100644
index 0000000..81a8687
--- /dev/null
+++ b/src/main/python/apache/aurora/common/aurora_job_key.py
@@ -0,0 +1,97 @@
+import re
+
+from twitter.common.lang import Compatibility, total_ordering
+
+from gen.twitter.aurora.constants import GOOD_IDENTIFIER_PATTERN_PYTHON
+from gen.twitter.aurora.ttypes import Identity, JobKey, TaskQuery
+
+# TODO(ksweeney): This can just probably just extend namedtuple.
+@total_ordering
+class AuroraJobKey(object):
+ """A canonical representation of a key that can identify a job in any of the clusters the client
+ is aware of."""
+ class Error(Exception): pass
+ class TypeError(TypeError, Error): pass
+ class InvalidIdentifier(ValueError, Error): pass
+ class ParseError(ValueError, Error): pass
+
+ VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
+
+ def __init__(self, cluster, role, env, name):
+ if not isinstance(cluster, Compatibility.string):
+ raise self.TypeError("cluster should be a string, got %s" % (cluster.__class__.__name__))
+ self._cluster = cluster
+ self._role = self._assert_valid_identifier("role", role)
+ self._env = self._assert_valid_identifier("env", env)
+ self._name = self._assert_valid_identifier("name", name)
+
+ @classmethod
+ def from_path(cls, path):
+ try:
+ cluster, role, env, name = path.split('/', 4)
+ except ValueError:
+ raise cls.ParseError(
+ "Invalid path '%s'. path should be a string in the form CLUSTER/ROLE/ENV/NAME" % path)
+ return cls(cluster, role, env, name)
+
+ @classmethod
+ def from_thrift(cls, cluster, job_key):
+ if not isinstance(job_key, JobKey):
+ raise cls.TypeError("job_key must be a Thrift JobKey struct")
+ return cls(cluster, job_key.role, job_key.environment, job_key.name)
+
+ @classmethod
+ def _assert_valid_identifier(cls, field, identifier):
+ if not isinstance(identifier, Compatibility.string):
+ raise cls.TypeError("%s must be a string" % field)
+ if not cls.VALID_IDENTIFIER.match(identifier):
+ raise cls.InvalidIdentifier("Invalid %s '%s'" % (field, identifier))
+ return identifier
+
+ @property
+ def cluster(self):
+ return self._cluster
+
+ @property
+ def role(self):
+ return self._role
+
+ @property
+ def env(self):
+ return self._env
+
+ @property
+ def name(self):
+ return self._name
+
+ def to_path(self):
+ return "%s/%s/%s/%s" % (self.cluster, self.role, self.env, self.name)
+
+ def to_thrift(self):
+ return JobKey(role=self.role, environment=self.env, name=self.name)
+
+ def to_thrift_query(self):
+ return TaskQuery(owner=Identity(role=self.role), environment=self.env, jobName=self.name)
+
+ def __iter__(self):
+ """Support 'cluster, role, env, name = job_key' assignment."""
+ return iter((self._cluster, self._role, self._env, self._name))
+
+ def __repr__(self):
+ return "%s(%r, %r, %r, %r)" % (self.__class__, self._cluster, self._role, self._env, self._name)
+
+ def __str__(self):
+ return self.to_path()
+
+ def __hash__(self):
+ return hash(AuroraJobKey) + hash(self.to_path())
+
+ def __eq__(self, other):
+ if not isinstance(other, AuroraJobKey):
+ return NotImplemented
+ return self.to_path() == other.to_path()
+
+ def __lt__(self, other):
+ if not isinstance(other, AuroraJobKey):
+ return NotImplemented
+ return self.to_path() < other.to_path()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/auth/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/BUILD b/src/main/python/apache/aurora/common/auth/BUILD
new file mode 100644
index 0000000..fd5b024
--- /dev/null
+++ b/src/main/python/apache/aurora/common/auth/BUILD
@@ -0,0 +1,9 @@
+python_library(
+ name = 'auth',
+ sources = globs('*.py'),
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/auth/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/__init__.py b/src/main/python/apache/aurora/common/auth/__init__.py
new file mode 100644
index 0000000..5418228
--- /dev/null
+++ b/src/main/python/apache/aurora/common/auth/__init__.py
@@ -0,0 +1,2 @@
+from .auth_module_manager import make_session_key, register_auth_module, SessionKeyError
+from .auth_module import AuthModule, InsecureAuthModule
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/auth/auth_module.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/auth_module.py b/src/main/python/apache/aurora/common/auth/auth_module.py
new file mode 100644
index 0000000..7b146a9
--- /dev/null
+++ b/src/main/python/apache/aurora/common/auth/auth_module.py
@@ -0,0 +1,30 @@
+from abc import abstractmethod, abstractproperty
+
+import getpass
+import time
+
+from twitter.common.lang import Interface
+
+from gen.twitter.aurora.ttypes import SessionKey
+
+
+class AuthModule(Interface):
+ @abstractproperty
+ def mechanism(self):
+ """Return the mechanism provided by this AuthModule."""
+
+ @abstractmethod
+ def payload(self):
+ """Return the payload generated by the AuthModule."""
+
+ def __call__(self):
+ return SessionKey(mechanism=self.mechanism, data=self.payload())
+
+
+class InsecureAuthModule(AuthModule):
+ @property
+ def mechanism(self):
+ return 'UNAUTHENTICATED'
+
+ def payload(self):
+ return 'UNAUTHENTICATED'
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/auth/auth_module_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/auth/auth_module_manager.py b/src/main/python/apache/aurora/common/auth/auth_module_manager.py
new file mode 100644
index 0000000..03d51b3
--- /dev/null
+++ b/src/main/python/apache/aurora/common/auth/auth_module_manager.py
@@ -0,0 +1,47 @@
+from twitter.common import log
+
+from gen.twitter.aurora.ttypes import SessionKey
+
+from .auth_module import AuthModule, InsecureAuthModule
+
+
+_INSECURE_AUTH_MODULE = InsecureAuthModule()
+_AUTH_MODULES = {
+ _INSECURE_AUTH_MODULE.mechanism: _INSECURE_AUTH_MODULE
+}
+
+
+class SessionKeyError(Exception): pass
+
+
+def register_auth_module(auth_module):
+ """
+ Add an auth module into the registry used by make_session_key. An auth module is discovered
+ via its auth mechanism.
+
+ args:
+ auth_module: A 0-arg callable that should return a SessionKey or raises a SessionKeyError
+ and extend AuthModule.
+ """
+ if not isinstance(auth_module, AuthModule):
+ raise TypeError('Given auth module must be a AuthModule subclass, got %s' % type(auth_module))
+ if not callable(auth_module):
+ raise TypeError('auth_module should be callable.')
+ _AUTH_MODULES[auth_module.mechanism] = auth_module
+
+
+def make_session_key(auth_mechanism='UNAUTHENTICATED'):
+ """
+ Attempts to create a session key by calling the auth module registered to the auth mechanism.
+ If an auth module does not exist for an auth mechanism, an InsecureAuthModule will be used.
+ """
+ if not _AUTH_MODULES:
+ raise SessionKeyError('No auth modules have been registered. Please call register_auth_module.')
+
+ auth_module = _AUTH_MODULES.get(auth_mechanism) or _INSECURE_AUTH_MODULE
+ log.debug('Using auth module: %r' % auth_module)
+ session_key = auth_module()
+ if not isinstance(session_key, SessionKey):
+ raise SessionKeyError('Expected %r but got %r from auth module %r' % (
+ SessionKey, session_key.__class__, auth_module))
+ return session_key
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/cluster.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/cluster.py b/src/main/python/apache/aurora/common/cluster.py
new file mode 100644
index 0000000..21d8ac3
--- /dev/null
+++ b/src/main/python/apache/aurora/common/cluster.py
@@ -0,0 +1,81 @@
+from pystachio import Empty, Struct
+from pystachio.composite import Structural
+
+__all__ = ('Cluster',)
+
+
+# TODO(wickman) It seems like some of this Trait/Mixin stuff should be a
+# first-class construct in Pystachio. It could be a solution for extensible
+# Job/Task definitions.
+class Cluster(dict):
+ """Cluster encapsulates a set of K/V attributes describing cluster configurations.
+
+ Given a cluster, attributes may be accessed directly on them, e.g.
+ cluster.name
+ cluster.scheduler_zk_path
+
+ In order to enforce particular "traits" of Cluster, use Cluster.Trait to construct
+ enforceable schemas, e.g.
+
+ class ResolverTrait(Cluster.Trait):
+ scheduler_zk_ensemble = Required(String)
+ scheduler_zk_path = Default(String, '/twitter/service/mesos/prod/scheduler')
+
+ cluster = Cluster(name = 'west', scheduler_zk_ensemble = 'zookeeper.west.twttr.net')
+
+ # Ensures that scheduler_zk_ensemble is defined in the cluster or it will raise a TypeError
+ cluster.with_trait(ResolverTrait).scheduler_zk_ensemble
+
+ # Will use the default if none is provided on Cluster.
+ cluster.with_trait(ResolverTrait).scheduler_zk_path
+ """
+ Trait = Struct
+
+ def __init__(self, **kwargs):
+ self._traits = ()
+ super(Cluster, self).__init__(**kwargs)
+
+ def get_trait(self, trait):
+ """Given a Cluster.Trait, extract that trait."""
+ if not issubclass(trait, Structural):
+ raise TypeError('provided trait must be a Cluster.Trait subclass, got %s' % type(trait))
+ # TODO(wickman) Expose this in pystachio as a non-private or add a load method with strict=
+ return trait(trait._filter_against_schema(self))
+
+ def check_trait(self, trait):
+ """Given a Cluster.Trait, typecheck that trait."""
+ trait_check = self.get_trait(trait).check()
+ if not trait_check.ok():
+ raise TypeError(trait_check.message())
+
+ def with_traits(self, *traits):
+ """Return a cluster annotated with a set of traits."""
+ new_cluster = self.__class__(**self)
+ for trait in traits:
+ new_cluster.check_trait(trait)
+ new_cluster._traits = traits
+ return new_cluster
+
+ def with_trait(self, trait):
+ """Return a cluster annotated with a single trait (helper for self.with_traits)."""
+ return self.with_traits(trait)
+
+ def __setitem__(self, key, value):
+ raise TypeError('Clusters are immutable.')
+
+ def __getattr__(self, attribute):
+ for trait in self._traits:
+ expressed_trait = self.get_trait(trait)
+ if hasattr(expressed_trait, attribute):
+ value = getattr(expressed_trait, attribute)()
+ return None if value is Empty else value.get()
+ try:
+ return self[attribute]
+ except KeyError:
+ return self.__getattribute__(attribute)
+
+ def __copy__(self):
+ return self
+
+ def __deepcopy__(self, memo):
+ return self
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/cluster_option.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/cluster_option.py b/src/main/python/apache/aurora/common/cluster_option.py
new file mode 100644
index 0000000..3e07476
--- /dev/null
+++ b/src/main/python/apache/aurora/common/cluster_option.py
@@ -0,0 +1,82 @@
+from copy import copy
+from optparse import (
+ NO_DEFAULT,
+ OptionValueError,
+ Option)
+
+
+def _check_mesos_cluster(option, opt, value):
+ cluster_name = value
+ if option.clusters and cluster_name in option.clusters:
+ return option.clusters[cluster_name]
+ elif option.cluster_provider:
+ return option.cluster_provider(cluster_name)
+
+ cluster_list = ""
+ if option.clusters:
+ cluster_list = 'Valid options for clusters are %s' % ' '.join(option.clusters)
+
+ raise OptionValueError(
+ '%s is not a valid cluster for the %s option. %s' % (value, opt, cluster_list))
+
+
+class ClusterOption(Option):
+ """A command-line Option that requires a valid cluster name and returns a Cluster object.
+
+ Use in an @app.command_option decorator to avoid boilerplate. For example:
+
+ CLUSTER_PATH = os.path.expanduser('~/.clusters')
+ CLUSTERS = Clusters.from_json(CLUSTER_PATH)
+
+ @app.command
+ @app.command_option(ClusterOption('--cluster', default='smf1-test', clusters=CLUSTERS))
+ def get_health(args, options):
+ if options.cluster.zk_server:
+ do_something(options.cluster)
+
+ @app.command
+ @app.command_option(ClusterOption('-s',
+ '--source_cluster',
+ default='smf1-test',
+ clusters=CLUSTERS,
+ help='Source cluster to pull metadata from.'))
+ @app.command_option(ClusterOption('-d',
+ '--dest_cluster',
+ clusters=CLUSTERS,
+ default='smf1-test'))
+ def copy_metadata(args, options):
+ if not options.source_cluster:
+ print('required option source_cluster missing!')
+ metadata_copy(options.source_cluster, options.dest_cluster)
+ """
+
+ # Needed since we're creating a new type for validation - see optparse docs.
+ TYPES = copy(Option.TYPES) + ('mesos_cluster',)
+ TYPE_CHECKER = copy(Option.TYPE_CHECKER)
+ TYPE_CHECKER['mesos_cluster'] = _check_mesos_cluster
+
+ def __init__(self, *opt_str, **attrs):
+ """
+ *opt_str: Same meaning as in twitter.common.options.Option, at least one is required.
+ **attrs: See twitter.common.options.Option, with the following caveats:
+
+ Exactly one of the following must be provided:
+
+ clusters: A static Clusters object from which to pick clusters.
+ cluster_provider: A function that takes a cluster name and returns a Cluster object.
+ """
+ self.clusters = attrs.pop('clusters', None)
+ self.cluster_provider = attrs.pop('cluster_provider', None)
+ if not (self.clusters is not None) ^ (self.cluster_provider is not None):
+ raise ValueError('Must specify exactly one of clusters and cluster_provider.')
+
+ default_attrs = dict(
+ default=None,
+ action='store',
+ type='mesos_cluster',
+ help='Mesos cluster to use (Default: %%default)'
+ )
+
+ combined_attrs = default_attrs
+ combined_attrs.update(attrs) # Defensive copy
+ Option.__init__(self, *opt_str, **combined_attrs) # old-style superclass
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/clusters.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/clusters.py b/src/main/python/apache/aurora/common/clusters.py
new file mode 100644
index 0000000..2777569
--- /dev/null
+++ b/src/main/python/apache/aurora/common/clusters.py
@@ -0,0 +1,135 @@
+from __future__ import print_function
+
+from collections import Mapping, namedtuple
+from contextlib import contextmanager
+import itertools
+import json
+import os
+import sys
+
+from twitter.common.collections import maybe_list
+
+from .cluster import Cluster
+
+from pystachio import Required, String
+
+try:
+ import yaml
+ HAS_YAML = True
+except ImportError:
+ HAS_YAML = False
+
+
+__all__ = (
+ 'CLUSTERS',
+ 'Clusters',
+)
+
+
+class NameTrait(Cluster.Trait):
+ name = Required(String)
+
+
+Parser = namedtuple('Parser', 'loader exception')
+
+
+class Clusters(Mapping):
+ class Error(Exception): pass
+ class ClusterExists(Error): pass
+ class ClusterNotFound(KeyError, Error): pass
+ class UnknownFormatError(Error): pass
+ class ParseError(Error): pass
+
+ LOADERS = {'.json': Parser(json.load, ValueError)}
+ if HAS_YAML:
+ LOADERS['.yml'] = Parser(yaml.load, yaml.parser.ParserError)
+
+ @classmethod
+ def from_file(cls, filename):
+ return cls(list(cls.iter_clusters(filename)))
+
+ @classmethod
+ def iter_clusters(cls, filename):
+ _, ext = os.path.splitext(filename)
+ if ext not in cls.LOADERS:
+ raise cls.UnknownFormatError('Unknown clusters file extension: %r' % ext)
+ with open(filename) as fp:
+ loader, exc_type = cls.LOADERS[ext]
+ try:
+ document = loader(fp)
+ except exc_type as e:
+ raise cls.ParseError('Unable to parse %s: %s' % (filename, e))
+ if isinstance(document, list):
+ iterator = document
+ elif isinstance(document, dict):
+ iterator = document.values()
+ else:
+ raise cls.ParseError('Unknown layout in %s' % filename)
+ for document in iterator:
+ if not isinstance(document, dict):
+ raise cls.ParseError('Clusters must be maps of key/value pairs, got %s' % type(document))
+ # documents not adhering to NameTrait are ignored.
+ if 'name' not in document:
+ continue
+ yield Cluster(**document)
+
+ def __init__(self, cluster_list):
+ self.replace(cluster_list)
+
+ def replace(self, cluster_list):
+ self._clusters = {}
+ self.update(cluster_list)
+
+ def update(self, cluster_list):
+ cluster_list = maybe_list(cluster_list, expected_type=Cluster, raise_type=TypeError)
+ for cluster in cluster_list:
+ self.add(cluster)
+
+ def add(self, cluster):
+ """Add a cluster to this Clusters map."""
+ cluster = Cluster(**cluster)
+ cluster.check_trait(NameTrait)
+ self._clusters[cluster.name] = cluster
+
+ @contextmanager
+ def patch(self, cluster_list):
+ """Patch this Clusters instance with a new list of clusters in a
+ contextmanager. Intended for testing purposes."""
+ old_clusters = self._clusters.copy()
+ self.replace(cluster_list)
+ yield self
+ self._clusters = old_clusters
+
+ def __iter__(self):
+ return iter(self._clusters)
+
+ def __len__(self):
+ return len(self._clusters)
+
+ def __getitem__(self, name):
+ try:
+ return self._clusters[name]
+ except KeyError:
+ raise self.ClusterNotFound('Unknown cluster %s, valid clusters: %s' % (
+ name, ', '.join(self._clusters.keys())))
+
+
+
+DEFAULT_SEARCH_PATHS = (
+ os.environ.get('AURORA_CONFIG_ROOT') or '/etc/aurora',
+ os.path.expanduser('~/.aurora')
+)
+
+
+CLUSTERS = Clusters(())
+
+
+def load():
+ """(re-)load all clusters from the search path."""
+ for search_path, ext in itertools.product(DEFAULT_SEARCH_PATHS, Clusters.LOADERS):
+ filename = os.path.join(search_path, 'clusters' + ext)
+ if os.path.exists(filename):
+ CLUSTERS.update(Clusters.from_file(filename).values())
+
+
+load()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/common/http_signaler.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/common/http_signaler.py b/src/main/python/apache/aurora/common/http_signaler.py
new file mode 100644
index 0000000..778ddc4
--- /dev/null
+++ b/src/main/python/apache/aurora/common/http_signaler.py
@@ -0,0 +1,82 @@
+import contextlib
+from socket import timeout as SocketTimeout
+import sys
+
+from twitter.common import log
+from twitter.common.lang import Compatibility
+
+if Compatibility.PY3:
+ from http.client import HTTPException
+ import urllib.request as urllib_request
+ from urllib.error import URLError, HTTPError
+else:
+ from httplib import HTTPException
+ import urllib2 as urllib_request
+ from urllib2 import URLError, HTTPError
+
+
+class HttpSignaler(object):
+ """Simple HTTP endpoint wrapper to check health or trigger quitquitquit/abortabortabort"""
+ TIMEOUT_SECS = 1.0
+ FAILURE_REASON_LENGTH = 10
+
+ class Error(Exception): pass
+ class QueryError(Error): pass
+
+ def __init__(self, port, host='localhost', timeout_secs=TIMEOUT_SECS):
+ self._host = host
+ self._url_base = 'http://%s:%d/' % (host, port)
+ self._timeout_secs = timeout_secs
+
+ def url(self, endpoint):
+ return self._url_base + endpoint
+
+ @property
+ def opener(self):
+ return urllib_request.urlopen
+
+ def query(self, endpoint, data=None):
+ """Request an HTTP endpoint with a GET request (or POST if data is not None)"""
+ url = self.url(endpoint)
+ log.debug("%s: %s %s" % (self.__class__.__name__, 'GET' if data is None else 'POST', url))
+
+ def raise_error(reason):
+ raise self.QueryError('Failed to signal %s: %s' % (self.url(endpoint), reason))
+
+ try:
+ with contextlib.closing(
+ self.opener(url, data, timeout=self._timeout_secs)) as fp:
+ return fp.read()
+ except (HTTPException, SocketTimeout) as e:
+ # the type of an HTTPException is typically more useful than its contents (since for example
+ # BadStatusLines are often empty). likewise with socket.timeout.
+ raise_error('Error within %s' % e.__class__.__name__)
+ except (URLError, HTTPError) as e:
+ raise_error(e)
+ except Exception as e:
+ raise_error('Unexpected error: %s' % e)
+
+ def __call__(self, endpoint, use_post_method=False, expected_response=None):
+ """Returns a (boolean, string|None) tuple of (call success, failure reason)"""
+ try:
+ response = self.query(endpoint, '' if use_post_method else None).strip().lower()
+ if expected_response is not None and response != expected_response:
+ def shorten(string):
+ return (string if len(string) < self.FAILURE_REASON_LENGTH
+ else "%s..." % string[:self.FAILURE_REASON_LENGTH - 3])
+ reason = 'Response differs from expected response (expected "%s", got "%s")'
+ log.warning(reason % (expected_response, response))
+ return (False, reason % (shorten(str(expected_response)), shorten(str(response))))
+ else:
+ return (True, None)
+ except self.QueryError as e:
+ return (False, str(e))
+
+ def health(self):
+ return self('health', use_post_method=False, expected_response='ok')
+
+ def quitquitquit(self):
+ return self('quitquitquit', use_post_method=True)
+
+ def abortabortabort(self):
+ return self('abortabortabort', use_post_method=True)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/BUILD b/src/main/python/apache/aurora/config/BUILD
new file mode 100644
index 0000000..91cb3e4
--- /dev/null
+++ b/src/main/python/apache/aurora/config/BUILD
@@ -0,0 +1,43 @@
+import os
+
+# Alias for src/main/python/twitter/aurora/config/schema
+python_library(
+ name = 'schema',
+ dependencies = [
+ pants('src/main/python/twitter/aurora/config/schema'),
+ ]
+)
+
+python_library(
+ name = 'config',
+ sources = (
+ '__init__.py',
+ 'loader.py',
+ 'port_resolver.py',
+ 'thrift.py',
+ ),
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('src/main/python/twitter/aurora/common'),
+ pants('src/main/python/twitter/aurora/config/schema'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ],
+
+)
+
+python_library(
+ name = 'config-packaged',
+ dependencies = [
+ pants(':config'),
+
+ # covering dependencies
+ pants('src/main/python/twitter/thermos/config'),
+ ],
+ provides = setup_py(
+ name = 'twitter.aurora.config',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ description = 'Aurora/Thermos Pystachio schemas for describing job configurations.',
+ license = 'Apache License, Version 2.0',
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/__init__.py b/src/main/python/apache/aurora/config/__init__.py
new file mode 100644
index 0000000..a3ba981
--- /dev/null
+++ b/src/main/python/apache/aurora/config/__init__.py
@@ -0,0 +1,271 @@
+from __future__ import absolute_import
+
+from collections import defaultdict
+
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+from twitter.aurora.config.schema.base import MesosContext
+from twitter.thermos.config.loader import PortExtractor, ThermosTaskWrapper
+from twitter.thermos.config.schema import ThermosContext
+
+from .loader import AuroraConfigLoader
+from .port_resolver import PortResolver
+from .thrift import convert as convert_thrift, InvalidConfig as InvalidThriftConfig
+
+from pystachio import Empty, Environment, Ref
+
+__all__ = ('AuroraConfig', 'PortResolver')
+
+
+class AuroraConfig(object):
+ class Error(Exception): pass
+
+ class InvalidConfig(Error):
+ def __str__(self):
+ return 'The configuration was invalid: %s' % self.args[0]
+
+ @classmethod
+ def plugins(cls):
+ """A stack of callables to apply to the config on load."""
+ return []
+
+ @classmethod
+ def pick(cls, env, name, bindings, select_cluster=None, select_role=None, select_env=None):
+ # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
+
+ job_list = env.get('jobs', [])
+ if not job_list:
+ raise ValueError('No job defined in this config!')
+
+ def maybe_bind(j):
+ return j.bind(*bindings) if bindings else j
+
+ if name is None:
+ if len(job_list) > 1:
+ raise ValueError('Configuration has multiple jobs but no job name specified!')
+ return maybe_bind(job_list[0])
+
+ # TODO(wfarner): Rework this and calling code to make name optional as well.
+ def match_name(job):
+ return str(job.name()) == name
+ def match_cluster(job):
+ return select_cluster is None or str(job.cluster()) == select_cluster
+ def match_env(job):
+ return select_env is None or str(job.environment()) == select_env
+ def match_role(job):
+ return select_role is None or str(job.role()) == select_role
+
+ bound_jobs = map(maybe_bind, job_list)
+ matches = [j for j in bound_jobs if
+ all([match_cluster(j), match_role(j), match_env(j), match_name(j)])]
+
+ if len(matches) == 0:
+ msg = "Could not find job %s/%s/%s/%s\n" % (
+ select_cluster or '*', select_role or '*', select_env or '*', name)
+ for j in bound_jobs:
+ if j.environment() is Empty:
+ msg += "Job %s/%s/%s/%s in configuration file doesn't specify an environment\n" % (
+ j.cluster(), j.role(), '{MISSING}', j.name()
+ )
+ msg += cls._candidate_jobs_str(bound_jobs)
+ raise ValueError(msg)
+
+ elif len(matches) > 1:
+ msg = 'Multiple jobs match, please disambiguate by specifying a job key.\n'
+ msg += cls._candidate_jobs_str(bound_jobs)
+ raise ValueError(msg)
+ else:
+ return matches[0]
+
+ @staticmethod
+ def _candidate_jobs_str(job_list):
+ assert(job_list)
+ job_list = [" %s/%s/%s/%s" % (
+ j.cluster(), j.role(),
+ j.environment() if j.environment() is not Empty else "{MISSING}",
+ j.name())
+ for j in job_list]
+ return 'Candidates are:\n' + '\n'.join(job_list)
+
+ @classmethod
+ def apply_plugins(cls, config, env=None):
+ for plugin in cls.plugins():
+ if not callable(plugin):
+ raise cls.Error('Invalid configuration plugin %r, should be callable!' % plugin)
+ plugin(config, env)
+ return config
+
+ @classmethod
+ def load(
+ cls, filename, name=None, bindings=None,
+ select_cluster=None, select_role=None, select_env=None):
+ # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
+ env = AuroraConfigLoader.load(filename)
+ return cls.apply_plugins(
+ cls(cls.pick(env, name, bindings, select_cluster, select_role, select_env)), env)
+
+ @classmethod
+ def load_json(
+ cls, filename, name=None, bindings=None,
+ select_cluster=None, select_role=None, select_env=None):
+ # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
+ job = AuroraConfigLoader.load_json(filename)
+ return cls.apply_plugins(cls(job.bind(*bindings) if bindings else job))
+
+ @classmethod
+ def loads_json(cls, string, name=None, bindings=None, select_cluster=None, select_env=None):
+ # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
+ job = AuroraConfigLoader.loads_json(string)
+ return cls.apply_plugins(cls(job.bind(*bindings) if bindings else job))
+
+ @classmethod
+ def validate_job(cls, job):
+ """
+ Validate and sanitize the input job
+
+ Currently, the validation stage simply ensures that the job has all required fields.
+ self.InvalidConfig is raised if any required fields are not present.
+ """
+ def has(pystachio_type, thing):
+ return getattr(pystachio_type, 'has_%s' % thing)()
+ for required in ("cluster", "task", "role"):
+ if not has(job, required):
+ raise cls.InvalidConfig(
+ '%s required for job "%s"' % (required.capitalize(), job.name()))
+ if not has(job.task(), 'processes'):
+ raise cls.InvalidConfig('Processes required for task on job "%s"' % job.name())
+
+ @classmethod
+ def standard_bindings(cls, job):
+ # Rewrite now-deprecated bindings into their proper form.
+ return job.bind({
+ Ref.from_address('mesos.role'): '{{role}}',
+ Ref.from_address('mesos.cluster'): '{{cluster}}',
+ Ref.from_address('thermos.user'): '{{role}}',
+ })
+
+ def __init__(self, job):
+ self.validate_job(job)
+ self._job = self.standard_bindings(job)
+ self._packages = []
+ self.binding_dicts = defaultdict(dict)
+ self.hooks = []
+
+ def context(self, instance=None):
+ context = dict(instance=instance)
+ # Filter unspecified values
+ return Environment(mesos=MesosContext(dict((k, v) for k, v in context.items() if v)))
+
+ def job(self):
+ interpolated_job = self._job % self.context()
+
+ # TODO(wickman) Once thermos is onto thrift instead of pystachio, use
+ # %%replacements%% instead.
+ #
+ # Typecheck against the Job, with the following free variables unwrapped at the Task level:
+ # - a dummy {{mesos.instance}}
+ # - dummy values for the {{thermos.ports}} context, to allow for their use in task_links
+ env = dict(mesos=Environment(instance=0))
+ if interpolated_job.task_links() is not Empty:
+ try:
+ dummy_ports = dict(
+ (port, 31337) for port in PortExtractor.extract(interpolated_job.task_links()))
+ except PortExtractor.InvalidPorts as err:
+ raise self.InvalidConfig('Invalid port references in task_links! %s' % err)
+ env.update(thermos=ThermosContext(ports=dummy_ports))
+ typecheck = interpolated_job.bind(Environment(env)).check()
+ if not typecheck.ok():
+ raise self.InvalidConfig(typecheck.message())
+ interpolated_job = interpolated_job(task_links=self.task_links())
+ try:
+ return convert_thrift(interpolated_job, self._packages, self.ports())
+ except InvalidThriftConfig as e:
+ raise self.InvalidConfig(str(e))
+
+ def bind(self, binding):
+ self._job = self._job.bind(binding)
+
+ def raw(self):
+ return self._job
+
+ # This stinks to high heaven
+ def update_job(self, new_job):
+ self._job = new_job
+
+ def instances(self):
+ return self._job.instances().get()
+
+ def task(self, instance):
+ return (self._job % self.context(instance)).task()
+
+ def name(self):
+ return self._job.name().get()
+
+ def role(self):
+ return self._job.role().get()
+
+ def cluster(self):
+ return self._job.cluster().get()
+
+ def environment(self):
+ return self._job.environment().get()
+
+ def job_key(self):
+ return AuroraJobKey(self.cluster(), self.role(), self.environment(), self.name())
+
+ def ports(self):
+ """Return the list of ports that need to be allocated by the scheduler."""
+
+ # Strictly speaking this is wrong -- it is possible to do things like
+ # {{thermos.ports[instance_{{mesos.instance}}]}}
+ # which can only be extracted post-unwrapping. This means that validating
+ # the state of the announce configuration could be problematic if people
+ # try to do complicated things.
+ referenced_ports = ThermosTaskWrapper(self._job.task(), strict=False).ports()
+ resolved_portmap = PortResolver.resolve(self._job.announce().portmap().get()
+ if self._job.has_announce() else {})
+
+ # values of the portmap that are not integers => unallocated
+ unallocated = set(port for port in resolved_portmap.values() if not isinstance(port, int))
+
+ # find referenced {{thermos.portmap[ports]}} that are not resolved by the portmap
+ unresolved_references = set(
+ port for port in (resolved_portmap.get(port_ref, port_ref) for port_ref in referenced_ports)
+ if not isinstance(port, int))
+
+ return unallocated | unresolved_references
+
+ def has_health_port(self):
+ return "health" in ThermosTaskWrapper(self._job.task(), strict=False).ports()
+
+ def task_links(self):
+ # {{mesos.instance}} --> %shard_id%
+ # {{thermos.ports[foo]}} --> %port:foo%
+ task_links = self._job.task_links()
+ if task_links is Empty:
+ return task_links
+ _, uninterp = task_links.interpolate()
+ substitutions = {
+ Ref.from_address('mesos.instance'): '%shard_id%'
+ }
+ port_scope = Ref.from_address('thermos.ports')
+ for ref in uninterp:
+ subscope = port_scope.scoped_to(ref)
+ if subscope:
+ substitutions[ref] = '%%port:%s%%' % subscope.action().value
+ return task_links.bind(substitutions)
+
+ def update_config(self):
+ return self._job.update_config()
+
+ def add_package(self, package):
+ self._packages.append(package)
+
+ # TODO(wickman) Kill package() once MESOS-3191 is in.
+ def package(self):
+ pass
+
+ def is_dedicated(self):
+ return self._job.has_constraints() and 'dedicated' in self._job.constraints()
+
+ def __repr__(self):
+ return '%s(%r)' % (self.__class__.__name__, self._job)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/loader.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/loader.py b/src/main/python/apache/aurora/config/loader.py
new file mode 100644
index 0000000..e3e5559
--- /dev/null
+++ b/src/main/python/apache/aurora/config/loader.py
@@ -0,0 +1,63 @@
+import json
+import pkgutil
+import textwrap
+
+from twitter.aurora.config.schema import base as base_schema
+
+from pystachio.config import Config as PystachioConfig
+
+
+class AuroraConfigLoader(PystachioConfig):
+ SCHEMA_MODULES = []
+
+ @classmethod
+ def assembled_schema(cls, schema_modules):
+ default_schema = [super(AuroraConfigLoader, cls).DEFAULT_SCHEMA]
+ default_schema.extend('from %s import *' % module.__name__ for module in schema_modules)
+ return '\n'.join(default_schema)
+
+ @classmethod
+ def register_schema(cls, schema_module):
+ """Register the schema defined in schema_module, equivalent to doing
+
+ from schema_module.__name__ import *
+
+ before all pystachio configurations are evaluated.
+ """
+ cls.SCHEMA_MODULES.append(schema_module)
+ cls.DEFAULT_SCHEMA = cls.assembled_schema(cls.SCHEMA_MODULES)
+
+ @classmethod
+ def register_schemas_from(cls, package):
+ """Register schemas from all modules in a particular package."""
+ for _, submodule, is_package in pkgutil.iter_modules(package.__path__):
+ if is_package:
+ continue
+ cls.register_schema(
+ __import__('%s.%s' % (package.__name__, submodule), fromlist=[package.__name__]))
+
+ @classmethod
+ def flush_schemas(cls):
+ """Flush all schemas from AuroraConfigLoader. Intended for test use only."""
+ cls.SCHEMA_MODULES = []
+ cls.register_schema(base_schema)
+
+ @classmethod
+ def load(cls, loadable):
+ return cls.load_raw(loadable).environment
+
+ @classmethod
+ def load_raw(cls, loadable):
+ return cls(loadable)
+
+ @classmethod
+ def load_json(cls, filename):
+ with open(filename) as fp:
+ return base_schema.Job.json_load(fp)
+
+ @classmethod
+ def loads_json(cls, string):
+ return base_schema.Job(json.loads(string))
+
+
+AuroraConfigLoader.flush_schemas()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/port_resolver.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/port_resolver.py b/src/main/python/apache/aurora/config/port_resolver.py
new file mode 100644
index 0000000..486095f
--- /dev/null
+++ b/src/main/python/apache/aurora/config/port_resolver.py
@@ -0,0 +1,45 @@
+from twitter.common.lang import Compatibility
+
+
+class PortResolver(object):
+ class CycleException(Exception): pass
+
+ @classmethod
+ def resolve(cls, portmap):
+ """
+ Given an announce-style portmap, return a fully dereferenced portmap.
+
+ For example, given the portmap:
+ {
+ 'http': 80,
+ 'aurora: 'http',
+ 'https': 'aurora',
+ 'thrift': 'service'
+ }
+
+ Returns {'http': 80, 'aurora': 80, 'https': 80, 'thrift': 'service'}
+ """
+ for (name, port) in portmap.items():
+ if not isinstance(name, Compatibility.string):
+ raise ValueError('All portmap keys must be strings!')
+ if not isinstance(port, (int, Compatibility.string)):
+ raise ValueError('All portmap values must be strings or integers!')
+
+ portmap = portmap.copy()
+ for port in list(portmap):
+ try:
+ portmap[port] = int(portmap[port])
+ except ValueError:
+ continue
+
+ def resolve_one(static_port):
+ visited = set()
+ root = portmap[static_port]
+ while root in portmap:
+ visited.add(root)
+ if portmap[root] in visited:
+ raise cls.CycleException('Found cycle in portmap!')
+ root = portmap[root]
+ return root
+
+ return dict((name, resolve_one(name)) for name in portmap)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/recipes.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/recipes.py b/src/main/python/apache/aurora/config/recipes.py
new file mode 100644
index 0000000..b2dc23c
--- /dev/null
+++ b/src/main/python/apache/aurora/config/recipes.py
@@ -0,0 +1,44 @@
+import os
+
+from .loader import AuroraConfigLoader
+
+import pkg_resources
+
+
+class Recipes(object):
+ """
+ Encapsulate a registry of Recipes (i.e. tasks to mutate the behavior of other tasks.)
+ """
+ REGISTRY = {}
+ RECIPE_EXTENSION = '.aurora_recipe'
+
+ class Error(Exception): pass
+ class UnknownRecipe(Error): pass
+
+ @classmethod
+ def get(cls, name):
+ if name not in cls.REGISTRY:
+ raise cls.UnknownRecipe('Could not find recipe %s!' % name)
+ return cls.REGISTRY[name]
+
+ @classmethod
+ def include_one(cls, filename):
+ recipe_env = AuroraConfigLoader.load(filename)
+ cls.REGISTRY.update(recipe_env.get('recipes', {}))
+
+ @classmethod
+ def include_module(cls, module):
+ for filename in pkg_resources.resource_listdir(module, ''):
+ if filename.endswith(cls.RECIPE_EXTENSION):
+ cls.include_one(os.path.join(module.replace('.', os.sep), filename))
+
+ @classmethod
+ def include(cls, path):
+ if os.path.isfile(path):
+ cls.include_one(path)
+ elif os.path.isdir(path):
+ for filename in os.listdir(path):
+ if filename.endswith(cls.RECIPE_EXTENSION):
+ cls.include_one(os.path.join(path, filename))
+ else:
+ raise ValueError('Could not find %s' % path)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/repl.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/repl.py b/src/main/python/apache/aurora/config/repl.py
new file mode 100644
index 0000000..d26f94d
--- /dev/null
+++ b/src/main/python/apache/aurora/config/repl.py
@@ -0,0 +1,8 @@
+from twitter.aurora.config.loader import AuroraConfigLoader
+from twitter.common.lang import Compatibility
+
+
+
+import code
+code.interact('Mesos Config REPL',
+ local=Compatibility.exec_function(AuroraConfigLoader.DEFAULT_SCHEMA, globals()))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/schema/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/BUILD b/src/main/python/apache/aurora/config/schema/BUILD
new file mode 100644
index 0000000..fd08ec7
--- /dev/null
+++ b/src/main/python/apache/aurora/config/schema/BUILD
@@ -0,0 +1,9 @@
+python_library(
+ name = 'schema',
+ sources = ['base.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ pants('src/main/python/twitter/thermos/config:schema'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/schema/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/__init__.py b/src/main/python/apache/aurora/config/schema/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/schema/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py
new file mode 100644
index 0000000..b3d437f
--- /dev/null
+++ b/src/main/python/apache/aurora/config/schema/base.py
@@ -0,0 +1,122 @@
+from twitter.thermos.config.schema import *
+
+from gen.twitter.aurora.constants import DEFAULT_ENVIRONMENT
+
+
+# TODO(wickman) Bind {{mesos.instance}} to %shard_id%
+class MesosContext(Struct):
+ # The instance id (i.e. replica id, shard id) in the context of a task
+ instance = Required(Integer)
+
+
+# AppApp layout setup
+class AppPackage(Struct):
+ name = Required(String)
+ version = Default(String, 'latest')
+
+class AppLayout(Struct):
+ packages = Default(List(AppPackage), [])
+
+
+# The object bound into the {{packer}} namespace.
+# Referenced by
+# {{packer[role][name][version]}}
+#
+# Where version =
+# number (integer)
+# 'live' (live package)
+# 'latest' (highest version number)
+#
+# For example if you'd like to create a copy process for a particular
+# package,
+# copy_latest = Process(
+# name = 'copy-{{package_name}}',
+# cmdline = '{{packer[{{role}}][{{package_name}}][latest].copy_command}}')
+# processes = [
+# copy_latest.bind(package_name = 'labrat'),
+# copy_latest.bind(package_name = 'packer')
+# ]
+class PackerObject(Struct):
+ package = String
+ package_uri = String
+ copy_command = String
+
+
+class UpdateConfig(Struct):
+ batch_size = Default(Integer, 1)
+ restart_threshold = Default(Integer, 60)
+ watch_secs = Default(Integer, 30)
+ max_per_shard_failures = Default(Integer, 0)
+ max_total_failures = Default(Integer, 0)
+
+
+class HealthCheckConfig(Struct):
+ initial_interval_secs = Default(Float, 60.0)
+ interval_secs = Default(Float, 30.0)
+ timeout_secs = Default(Float, 1.0)
+ max_consecutive_failures = Default(Integer, 0)
+
+
+class Announcer(Struct):
+ primary_port = Default(String, 'http')
+
+ # Portmap can either alias two ports together, e.g.
+ # aurora <= http
+ # Or it can be used to alias static ports to endpoints, e.g.
+ # http <= 80
+ # https <= 443
+ # aurora <= https
+ portmap = Default(Map(String, String), {
+ 'aurora': '{{primary_port}}'
+ })
+
+
+# The executorConfig populated inside of TaskConfig.
+class MesosTaskInstance(Struct):
+ task = Required(Task)
+ layout = AppLayout
+ instance = Required(Integer)
+ role = Required(String)
+ announce = Announcer
+ environment = Default(String, DEFAULT_ENVIRONMENT)
+ health_check_interval_secs = Default(Integer, 30) # DEPRECATED (MESOS-2649)
+ health_check_config = Default(HealthCheckConfig, HealthCheckConfig())
+
+
+class MesosJob(Struct):
+ name = Default(String, '{{task.name}}')
+ role = Required(String)
+ contact = String
+ cluster = Required(String)
+ environment = Required(String)
+ instances = Default(Integer, 1)
+ task = Required(Task)
+ recipes = List(String)
+ announce = Announcer
+
+ cron_schedule = String
+ cron_policy = String # these two are aliases of each other. default is KILL_EXISTING
+ cron_collision_policy = String # if unspecified.
+ # cron_policy is DEPRECATED (MESOS-2491) in favor of
+ # cron_collision_policy.
+
+ update_config = Default(UpdateConfig, UpdateConfig())
+
+ constraints = Map(String, String)
+ daemon = Boolean # daemon and service are aliased together.
+ service = Boolean # daemon is DEPRECATED (MESOS-2492) in favor of
+ # service. by default, service is False.
+ max_task_failures = Default(Integer, 1)
+ production = Default(Boolean, False)
+ priority = Default(Integer, 0)
+ health_check_interval_secs = Integer # DEPRECATED in favor of health_check_config (MESOS-2649).
+ health_check_config = HealthCheckConfig
+ task_links = Map(String, String)
+
+ layout = AppLayout # DEPRECATED in favor of directory sandboxes
+
+ enable_hooks = Default(Boolean, False) # enable client API hooks; from env python-list 'hooks'
+
+
+Job = MesosJob
+Service = Job(service = True)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/config/thrift.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py
new file mode 100644
index 0000000..8f6a5ce
--- /dev/null
+++ b/src/main/python/apache/aurora/config/thrift.py
@@ -0,0 +1,259 @@
+import getpass
+import re
+
+from twitter.aurora.config.schema.base import (
+ HealthCheckConfig,
+ MesosContext,
+ MesosTaskInstance,
+)
+from twitter.common.lang import Compatibility
+from twitter.thermos.config.loader import ThermosTaskValidator
+
+from gen.twitter.aurora.constants import GOOD_IDENTIFIER_PATTERN_PYTHON, AURORA_EXECUTOR_NAME
+from gen.twitter.aurora.ttypes import (
+ Constraint,
+ CronCollisionPolicy,
+ ExecutorConfig,
+ Identity,
+ JobConfiguration,
+ JobKey,
+ LimitConstraint,
+ Package,
+ TaskConfig,
+ TaskConstraint,
+ ValueConstraint,
+)
+
+from pystachio import Empty, Ref
+
+__all__ = (
+ 'InvalidConfig',
+ 'convert'
+)
+
+
+class InvalidConfig(ValueError):
+ pass
+
+
+def constraints_to_thrift(constraints):
+ """Convert a python dictionary to a set of Constraint thrift objects."""
+ result = set()
+ for attribute, constraint_value in constraints.items():
+ assert isinstance(attribute, Compatibility.string) and (
+ isinstance(constraint_value, Compatibility.string)), (
+ "Both attribute name and value in constraints must be string")
+ constraint = Constraint()
+ constraint.name = attribute
+ task_constraint = TaskConstraint()
+ if constraint_value.startswith('limit:'):
+ task_constraint.limit = LimitConstraint()
+ try:
+ task_constraint.limit.limit = int(constraint_value.replace('limit:', '', 1))
+ except ValueError:
+ print('%s is not a valid limit value, must be integer' % constraint_value)
+ raise
+ else:
+ # Strip off the leading negation if present.
+ negated = constraint_value.startswith('!')
+ if negated:
+ constraint_value = constraint_value[1:]
+ task_constraint.value = ValueConstraint(negated, set(constraint_value.split(',')))
+ constraint.constraint = task_constraint
+ result.add(constraint)
+ return result
+
+
+def task_instance_from_job(job, instance):
+ instance_context = MesosContext(instance=instance)
+ # TODO(Sathya): Remove health_check_interval_secs references after deprecation cycle is complete.
+ health_check_config = HealthCheckConfig()
+ if job.has_health_check_interval_secs():
+ health_check_config = HealthCheckConfig(interval_secs=job.health_check_interval_secs().get())
+ elif job.has_health_check_config():
+ health_check_config = job.health_check_config()
+ ti = MesosTaskInstance(task=job.task(),
+ layout=job.layout(),
+ role=job.role(),
+ health_check_interval_secs=health_check_config.interval_secs().get(),
+ health_check_config=health_check_config,
+ instance=instance)
+ if job.has_announce():
+ ti = ti(announce=job.announce())
+ if job.has_environment():
+ ti = ti(environment=job.environment())
+ return ti.bind(mesos=instance_context).interpolate()
+
+
+def translate_cron_policy(policy):
+ cron_policy = CronCollisionPolicy._NAMES_TO_VALUES.get(policy.get())
+ if cron_policy is None:
+ raise InvalidConfig('Invalid cron policy: %s' % policy.get())
+ return cron_policy
+
+
+def fully_interpolated(pystachio_object, coerce_fn=lambda i: i):
+ # Extract a fully-interpolated unwrapped object from pystachio_object or raise InvalidConfig.
+ #
+ # TODO(ksweeney): Remove this once Pystachio 1.0 changes the behavior of interpolate() to return
+ # unwrapped objects and fail when there are any unbound refs.
+ if not pystachio_object.check().ok():
+ raise InvalidConfig(pystachio_object.check().message())
+
+ # If an object type-checks it's okay to use the raw value from the wrapped object returned by
+ # interpolate. Without the previous check value.get() could return a string with mustaches
+ # instead of an object of the expected type.
+ value, _ = pystachio_object.interpolate()
+ return coerce_fn(value.get())
+
+
+def select_cron_policy(cron_policy, cron_collision_policy):
+ if cron_policy is Empty and cron_collision_policy is Empty:
+ return CronCollisionPolicy.KILL_EXISTING
+ elif cron_policy is not Empty and cron_collision_policy is Empty:
+ return translate_cron_policy(cron_policy)
+ elif cron_policy is Empty and cron_collision_policy is not Empty:
+ return translate_cron_policy(cron_collision_policy)
+ else:
+ raise InvalidConfig('Specified both cron_policy and cron_collision_policy!')
+
+
+def select_service_bit(job):
+ if not job.has_daemon() and not job.has_service():
+ return False
+ elif job.has_daemon() and not job.has_service():
+ return fully_interpolated(job.daemon(), bool)
+ elif not job.has_daemon() and job.has_service():
+ return fully_interpolated(job.service(), bool)
+ else:
+ raise InvalidConfig('Specified both daemon and service bits!')
+
+
+# TODO(wickman) Due to MESOS-2718 we should revert to using the MesosTaskInstance.
+#
+# Using the MesosJob instead of the MesosTaskInstance was to allow for
+# planned future use of fields such as 'cluster' and to allow for conversion
+# from Job=>Task to be done entirely on the executor, but instead this had
+# made it impossible to run idempotent updates.
+#
+# In the meantime, we are erasing fields of the Job that are controversial.
+# This achieves roughly the same effect as using the MesosTaskInstance.
+# The future work is tracked at MESOS-2727.
+ALIASED_FIELDS = (
+ 'cron_policy',
+ 'cron_collision_policy',
+ 'update_config',
+ 'daemon',
+ 'service',
+ 'instances'
+)
+
+
+def filter_aliased_fields(job):
+ return job(**dict((key, Empty) for key in ALIASED_FIELDS))
+
+
+def assert_valid_field(field, identifier):
+ VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
+ if not isinstance(identifier, Compatibility.string):
+ raise InvalidConfig("%s must be a string" % field)
+ if not VALID_IDENTIFIER.match(identifier):
+ raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
+ return identifier
+
+
+MESOS_INSTANCE_REF = Ref.from_address('mesos.instance')
+THERMOS_PORT_SCOPE_REF = Ref.from_address('thermos.ports')
+THERMOS_TASK_ID_REF = Ref.from_address('thermos.task_id')
+
+
+# TODO(wickman) Make this a method directly on an AuroraConfig so that we don't
+# need the packages/ports shenanigans.
+def convert(job, packages=frozenset(), ports=frozenset()):
+ """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
+
+ owner = Identity(role=fully_interpolated(job.role()), user=getpass.getuser())
+ key = JobKey(
+ role=assert_valid_field('role', fully_interpolated(job.role())),
+ environment=assert_valid_field('environment', fully_interpolated(job.environment())),
+ name=assert_valid_field('name', fully_interpolated(job.name())))
+
+ task_raw = job.task()
+
+ MB = 1024 * 1024
+ task = TaskConfig()
+
+ def not_empty_or(item, default):
+ return default if item is Empty else fully_interpolated(item)
+
+ # job components
+ task.jobName = fully_interpolated(job.name())
+ task.environment = fully_interpolated(job.environment())
+ task.production = fully_interpolated(job.production(), bool)
+ task.isService = select_service_bit(job)
+ task.maxTaskFailures = fully_interpolated(job.max_task_failures())
+ task.priority = fully_interpolated(job.priority())
+ task.contactEmail = not_empty_or(job.contact(), None)
+
+ # Add package tuples to a task, to display in the scheduler UI.
+ task.packages = frozenset(
+ Package(role=str(role), name=str(package_name), version=int(version))
+ for role, package_name, version in packages)
+
+ # task components
+ if not task_raw.has_resources():
+ raise InvalidConfig('Task must specify resources!')
+
+ if (fully_interpolated(task_raw.resources().ram()) == 0
+ or fully_interpolated(task_raw.resources().disk()) == 0):
+ raise InvalidConfig('Must specify ram and disk resources, got ram:%r disk:%r' % (
+ fully_interpolated(task_raw.resources().ram()),
+ fully_interpolated(task_raw.resources().disk())))
+
+ task.numCpus = fully_interpolated(task_raw.resources().cpu())
+ task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB
+ task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB
+ if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0:
+ raise InvalidConfig('Task has invalid resources. cpu/ramMb/diskMb must all be positive: '
+ 'cpu:%r ramMb:%r diskMb:%r' % (task.numCpus, task.ramMb, task.diskMb))
+
+ task.owner = owner
+ task.requestedPorts = ports
+ task.taskLinks = not_empty_or(job.task_links(), {})
+ task.constraints = constraints_to_thrift(not_empty_or(job.constraints(), {}))
+
+ underlying, refs = job.interpolate()
+
+ # need to fake an instance id for the sake of schema checking
+ underlying_checked = underlying.bind(mesos = {'instance': 31337})
+ try:
+ ThermosTaskValidator.assert_valid_task(underlying_checked.task())
+ except ThermosTaskValidator.InvalidTaskError as e:
+ raise InvalidConfig('Task is invalid: %s' % e)
+ if not underlying_checked.check().ok():
+ raise InvalidConfig('Job not fully specified: %s' % underlying.check().message())
+
+ unbound = []
+ for ref in refs:
+ if ref == THERMOS_TASK_ID_REF or ref == MESOS_INSTANCE_REF or (
+ Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)):
+ continue
+ unbound.append(ref)
+
+ if unbound:
+ raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound)))
+
+ cron_schedule = not_empty_or(job.cron_schedule(), '')
+ cron_policy = select_cron_policy(job.cron_policy(), job.cron_collision_policy())
+
+ task.executorConfig = ExecutorConfig(
+ name=AURORA_EXECUTOR_NAME,
+ data=filter_aliased_fields(underlying).json_dumps())
+
+ return JobConfiguration(
+ key=key,
+ owner=owner,
+ cronSchedule=cron_schedule,
+ cronCollisionPolicy=cron_policy,
+ taskConfig=task,
+ instanceCount=fully_interpolated(job.instances()))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/BUILD b/src/main/python/apache/aurora/executor/BUILD
new file mode 100644
index 0000000..01701e9
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/BUILD
@@ -0,0 +1,139 @@
+import os
+
+python_library(
+ name = 'thermos_task_runner',
+ sources = ['thermos_task_runner.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/thermos/common'),
+ pants('src/main/python/twitter/thermos/config:schema'),
+ pants('src/main/python/twitter/thermos/core'),
+ pants('src/main/python/twitter/thermos/monitoring:monitor'),
+ pants('src/main/python/twitter/aurora/common:http_signaler'),
+ pants('src/main/python/twitter/aurora/executor/common:status_checker'),
+ pants('src/main/python/twitter/aurora/executor/common:task_info'),
+ pants('src/main/python/twitter/aurora/executor/common:task_runner'),
+ ]
+)
+
+python_library(
+ name = 'executor_detector',
+ sources = ['executor_detector.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/string'),
+ ]
+)
+
+python_library(
+ name = 'executor_vars',
+ sources = ['executor_vars.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:psutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
+ pants('aurora/twitterdeps/src/python/twitter/common/python'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('aurora/twitterdeps/src/python/twitter/common/string'),
+ ]
+)
+
+python_library(
+ name = 'status_manager',
+ sources = ['status_manager.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/aurora/executor/common:status_checker'),
+ ]
+)
+
+python_library(
+ name = 'thermos_executor_base',
+ sources = ['executor_base.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ pants('src/main/python/twitter/aurora:mesos-core'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'thermos_executor',
+ sources = ['thermos_executor.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ pants(':status_manager'),
+ pants(':thermos_executor_base'),
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/concurrent'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/aurora/executor/common:kill_manager'),
+ pants('src/main/python/twitter/aurora/executor/common:sandbox'),
+ pants('src/main/python/twitter/aurora/executor/common:task_info'),
+ pants('src/main/python/twitter/aurora/executor/common:task_runner'),
+ pants('src/main/python/twitter/aurora:mesos-core'),
+ ]
+)
+
+python_library(
+ name = 'thermos_runner',
+ sources = ['thermos_runner.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/thermos/common:planner'),
+ pants('src/main/python/twitter/thermos/config:schema'),
+ pants('src/main/python/twitter/thermos/core'),
+ ],
+)
+
+python_library(
+ name = 'gc_executor',
+ sources = ['gc_executor.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:psutil'),
+ pants(':executor_detector'),
+ pants(':thermos_executor_base'),
+ pants('aurora/twitterdeps/src/python/twitter/common/collections'),
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/python/twitter/thermos/core:helper'),
+ pants('src/main/python/twitter/thermos/core:inspector'),
+ pants('src/main/python/twitter/thermos/monitoring:detector'),
+ pants('src/main/python/twitter/thermos/monitoring:garbage'),
+ pants('src/main/python/twitter/aurora/config:schema'),
+ pants('src/main/python/twitter/aurora/executor/common:sandbox'),
+ pants('src/main/python/twitter/aurora:mesos-core'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'executor-packaged',
+ dependencies = [
+ # Covering dependencies
+ pants('src/main/python/twitter/aurora/common'),
+ pants('src/main/python/twitter/aurora/config'),
+ pants('src/main/python/twitter/thermos/common'),
+ pants('src/main/python/twitter/thermos/config'),
+ pants('src/main/python/twitter/thermos/core'),
+ pants('src/main/python/twitter/thermos/monitoring'),
+ ],
+ provides = setup_py(
+ name = 'twitter.aurora.executor',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ ).with_binaries(
+ gc_executor = pants('src/main/python/twitter/aurora/executor/bin:gc_executor'),
+ thermos_executor = pants('src/main/python/twitter/aurora/executor/bin:thermos_executor'),
+ thermos_runner = pants('src/main/python/twitter/aurora/executor/bin:thermos_runner'),
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/__init__.py b/src/main/python/apache/aurora/executor/__init__.py
new file mode 100644
index 0000000..b0d6433
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/BUILD b/src/main/python/apache/aurora/executor/bin/BUILD
new file mode 100644
index 0000000..5e9cab1
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/bin/BUILD
@@ -0,0 +1,47 @@
+python_binary(
+ name = 'thermos_executor',
+ source = 'thermos_executor_main.py',
+ entry_point = 'twitter.aurora.executor.bin.thermos_executor_main:proxy_main',
+ ignore_errors = True,
+ always_write_cache = True,
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
+ pants('src/main/python/twitter/aurora/executor/common:executor_timeout'),
+ pants('src/main/python/twitter/aurora/executor/common:health_checker'),
+ pants('src/main/python/twitter/aurora/executor/common:sandbox'),
+ pants('src/main/python/twitter/aurora/executor:executor_detector'),
+ pants('src/main/python/twitter/aurora/executor:executor_vars'),
+ pants('src/main/python/twitter/aurora/executor:thermos_executor'),
+ pants('src/main/python/twitter/aurora/executor:thermos_task_runner'),
+ ]
+)
+
+python_binary(
+ name = 'gc_executor',
+ source = 'gc_executor_main.py',
+ entry_point = 'twitter.aurora.executor.bin.gc_executor_main:proxy_main',
+ ignore_errors = True,
+ always_write_cache = True,
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/python/twitter/aurora/executor:executor_detector'),
+ pants('src/main/python/twitter/aurora/executor:executor_vars'),
+ pants('src/main/python/twitter/aurora/executor:gc_executor'),
+ ]
+)
+
+python_binary(
+ name = 'thermos_runner',
+ source = 'thermos_runner_main.py',
+ entry_point = 'twitter.aurora.executor.bin.thermos_runner_main:proxy_main',
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/executor:thermos_runner'),
+ ],
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/__init__.py b/src/main/python/apache/aurora/executor/bin/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/bin/gc_executor_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/gc_executor_main.py b/src/main/python/apache/aurora/executor/bin/gc_executor_main.py
new file mode 100644
index 0000000..fad20e6
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/bin/gc_executor_main.py
@@ -0,0 +1,45 @@
+"""Command-line entry point to the Thermos GC executor
+
+This module wraps the Thermos GC executor into an executable suitable for launching by a Mesos
+slave.
+
+"""
+
+from twitter.aurora.executor.executor_detector import ExecutorDetector
+from twitter.aurora.executor.gc_executor import ThermosGCExecutor
+from twitter.common import app, log
+from twitter.common.log.options import LogOptions
+from twitter.common.metrics.sampler import DiskMetricWriter
+from twitter.thermos.common.path import TaskPath
+
+import mesos
+
+
+app.configure(debug=True)
+
+
+# locate logs locally in executor sandbox
+LogOptions.set_simple(True)
+LogOptions.set_disk_log_level('DEBUG')
+LogOptions.set_log_dir(ExecutorDetector.LOG_PATH)
+
+
+def proxy_main():
+ def main():
+ # Create executor stub
+ thermos_gc_executor = ThermosGCExecutor(checkpoint_root=TaskPath.DEFAULT_CHECKPOINT_ROOT)
+ thermos_gc_executor.start()
+
+ # Start metrics collection
+ metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH)
+ metric_writer.start()
+
+ # Create driver stub
+ driver = mesos.MesosExecutorDriver(thermos_gc_executor)
+
+ # Start GC executor
+ driver.run()
+
+ log.info('MesosExecutorDriver.run() has finished.')
+
+ app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
new file mode 100644
index 0000000..31e718e
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/bin/thermos_executor_main.py
@@ -0,0 +1,67 @@
+"""Command-line entry point to the Thermos Executor
+
+This module wraps the Thermos Executor into an executable suitable for launching by a Mesos
+slave.
+
+"""
+
+import os
+
+from twitter.common import app, log
+from twitter.common.log.options import LogOptions
+
+from twitter.aurora.executor.common.executor_timeout import ExecutorTimeout
+from twitter.aurora.executor.common.health_checker import HealthCheckerProvider
+from twitter.aurora.executor.thermos_executor import ThermosExecutor
+from twitter.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider
+
+import mesos
+
+
+app.configure(debug=True)
+LogOptions.set_simple(True)
+LogOptions.set_disk_log_level('DEBUG')
+LogOptions.set_log_dir('.')
+
+
+# TODO(wickman) Consider just having the OSS version require pip installed
+# thermos_runner binaries on every machine and instead of embedding the pex
+# as a resource, shell out to one on the PATH.
+def dump_runner_pex():
+ import pkg_resources
+ import twitter.aurora.executor.resources
+ pex_name = 'thermos_runner.pex'
+ runner_pex = os.path.join(os.path.realpath('.'), pex_name)
+ with open(runner_pex, 'w') as fp:
+ # TODO(wickman) Use shutil.copyfileobj to reduce memory footprint here.
+ fp.write(pkg_resources.resource_stream(
+ twitter.aurora.executor.resources.__name__, pex_name).read())
+ return runner_pex
+
+
+def proxy_main():
+ def main():
+ runner_provider = DefaultThermosTaskRunnerProvider(
+ dump_runner_pex(),
+ artifact_dir=os.path.realpath('.'),
+ )
+
+ # Create executor stub
+ thermos_executor = ThermosExecutor(
+ runner_provider=runner_provider,
+ status_providers=(HealthCheckerProvider(),),
+ )
+
+ # Create driver stub
+ driver = mesos.MesosExecutorDriver(thermos_executor)
+
+ # This is an ephemeral executor -- shutdown if we receive no tasks within a certain
+ # time period
+ ExecutorTimeout(thermos_executor.launched, driver).start()
+
+ # Start executor
+ driver.run()
+
+ log.info('MesosExecutorDriver.run() has finished.')
+
+ app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py b/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
new file mode 100644
index 0000000..602111e
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/bin/thermos_runner_main.py
@@ -0,0 +1,12 @@
+from twitter.common import app
+from twitter.common.log.options import LogOptions
+from twitter.aurora.executor.thermos_runner import proxy_main as runner_proxy_main
+
+
+LogOptions.set_simple(True)
+
+
+def proxy_main():
+ main = runner_proxy_main
+
+ app.main()
[23/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py b/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
new file mode 100644
index 0000000..4376e85
--- /dev/null
+++ b/src/main/python/apache/aurora/tools/java/thrift_wrapper_codegen.py
@@ -0,0 +1,475 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import re
+import sys
+
+
+class Type(object):
+ '''A data type.'''
+
+ def __init__(self, name, package=None, immutable=False):
+ self.name = name
+ self.package = package
+ self.immutable = immutable
+
+ def absolute_name(self):
+ return '%s.%s' % (self.package, self.name) if self.package else self.name
+
+ def __str__(self):
+ return '%s (%smutable)' % (self.absolute_name(), 'im' if self.immutable else '')
+
+
+class PrimitiveType(Type):
+ '''A primitive type, with its associated typeboxed name.'''
+
+ def __init__(self, name, boxed_name):
+ Type.__init__(self, name, package=None, immutable=True)
+ self.boxed_name = boxed_name
+
+
+class ParameterizedType(Type):
+ '''A parameterized type, usually a collection.'''
+
+ def __init__(self, name, params):
+ Type.__init__(self, name, None)
+ self.params = params
+
+ def param_names(self):
+ def name(t):
+ if isinstance(t, StructType):
+ return t.codegen_name
+ elif isinstance(t, PrimitiveType):
+ return t.boxed_name
+ else:
+ return t.name
+ return ', '.join([name(p) for p in self.params])
+
+
+class StructType(Type):
+ '''A thrift-defined type, which composes other types as fields.'''
+
+ def __init__(self, name, package, kind, fields):
+ Type.__init__(self, name, package, kind == 'enum')
+ self.codegen_name = 'I%s' % name
+ self.kind = kind
+ self.fields = fields
+
+ def __str__(self):
+ return '%s %s { %s }' % (self.kind, self.name, ', '.join(map(str, self.fields)))
+
+
+class Field(object):
+ '''A field within a thrift structure.'''
+
+ def __init__(self, ttype, name):
+ self.ttype = ttype
+ self.name = name
+
+ def accessor_method(self):
+ return '%s%s' % (
+ 'is' if self.ttype.name == 'boolean' else 'get',
+ self.name[:1].capitalize() + self.name[1:])
+
+ def isset_method(self):
+ return 'isSet%s' % (self.name[0].upper() + self.name[1:])
+
+ def __str__(self):
+ return '%s: %s' % (self.name, self.ttype)
+
+
+FIELD_TEMPLATE = ''' public %(type)s %(fn_name)s() {
+ return %(field)s;
+ }'''
+
+
+# Template string for a method to access an immutable field.
+IMMUTABLE_FIELD_TEMPLATE = ''' public %(type)s %(fn_name)s() {
+ return wrapped.%(fn_name)s();
+ }'''
+
+
+STRUCT_DECLARATION = '''private final %(type)s %(field)s;'''
+STRUCT_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
+ ? null
+ : %(type)s.buildNoCopy(wrapped.%(fn_name)s());'''
+
+
+IMMUTABLE_COLLECTION_DECLARATION = '''private final Immutable%(collection)s<%(params)s> %(field)s;'''
+IMMUTABLE_COLLECTION_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
+ ? Immutable%(collection)s.<%(params)s>of()
+ : Immutable%(collection)s.copyOf(wrapped.%(fn_name)s());'''
+
+
+# Tempalte string for assignment for a collection field containing a struct.
+STRUCT_COLLECTION_FIELD_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
+ ? Immutable%(collection)s.<%(params)s>of()
+ : FluentIterable.from(wrapped.%(fn_name)s())
+ .transform(%(params)s.FROM_BUILDER)
+ .to%(collection)s();'''
+
+PACKAGE_NAME = 'com.twitter.aurora.scheduler.storage.entities'
+
+CLASS_TEMPLATE = '''/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package %(package)s;
+
+%(imports)s
+
+/**
+ * An immutable wrapper class.
+ * <p>
+ * This code is auto-generated, and should not be directly modified.
+ */
+public final class %(name)s {
+ private final %(wrapped)s wrapped;
+%(fields)s
+ private %(name)s(%(wrapped)s wrapped) {
+ this.wrapped = Preconditions.checkNotNull(wrapped);%(assignments)s
+ }
+
+ static %(name)s buildNoCopy(%(wrapped)s wrapped) {
+ return new %(name)s(wrapped);
+ }
+
+ public static %(name)s build(%(wrapped)s wrapped) {
+ return buildNoCopy(wrapped.deepCopy());
+ }
+
+ public static final Function<%(name)s, %(wrapped)s> TO_BUILDER =
+ new Function<%(name)s, %(wrapped)s>() {
+ @Override
+ public %(wrapped)s apply(%(name)s input) {
+ return input.newBuilder();
+ }
+ };
+
+ public static final Function<%(wrapped)s, %(name)s> FROM_BUILDER =
+ new Function<%(wrapped)s, %(name)s>() {
+ @Override
+ public %(name)s apply(%(wrapped)s input) {
+ return new %(name)s(input);
+ }
+ };
+
+ public static ImmutableList<%(wrapped)s> toBuildersList(Iterable<%(name)s> w) {
+ return FluentIterable.from(w).transform(TO_BUILDER).toList();
+ }
+
+ public static ImmutableList<%(name)s> listFromBuilders(Iterable<%(wrapped)s> b) {
+ return FluentIterable.from(b).transform(FROM_BUILDER).toList();
+ }
+
+ public static ImmutableSet<%(wrapped)s> toBuildersSet(Iterable<%(name)s> w) {
+ return FluentIterable.from(w).transform(TO_BUILDER).toSet();
+ }
+
+ public static ImmutableSet<%(name)s> setFromBuilders(Iterable<%(wrapped)s> b) {
+ return FluentIterable.from(b).transform(FROM_BUILDER).toSet();
+ }
+
+ public %(wrapped)s newBuilder() {
+ return wrapped.deepCopy();
+ }
+
+%(accessors)s
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof %(name)s)) {
+ return false;
+ }
+ %(name)s other = (%(name)s) o;
+ return wrapped.equals(other.wrapped);
+ }
+
+ @Override
+ public int hashCode() {
+ return wrapped.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return wrapped.toString();
+ }
+}'''
+
+
+class GeneratedCode(object):
+ def __init__(self, class_name, wrapped_type):
+ self._class_name = class_name
+ self._wrapped_type = wrapped_type
+ self._imports = set()
+ self._accessors = []
+ self._fields = []
+ self._assignments = []
+
+ def add(self, s, end='\n'):
+ print('This no longer does anything.')
+
+ def add_import(self, import_class):
+ self._imports.add(import_class)
+
+ def add_assignment(self, field, assignment):
+ self._fields.append(field)
+ self._assignments.append(assignment)
+
+ def add_accessor(self, accessor_method):
+ self._accessors.append(accessor_method)
+
+ def dump(self, f):
+ remaining_imports = list(self._imports)
+ import_groups = []
+ def remove_by_prefix(prefix):
+ group = [i for i in remaining_imports if i.startswith(prefix)]
+ remaining_imports[:] = [i for i in remaining_imports if not i.startswith(prefix)]
+ return group
+
+ def add_import_group(group):
+ if group:
+ import_groups.append('\n'.join(['import %s;' % i for i in sorted(group)]))
+
+ twitter_imports = remove_by_prefix('com.twitter')
+ add_import_group(remove_by_prefix('java'))
+ add_import_group(remove_by_prefix('com'))
+ add_import_group(remove_by_prefix('net'))
+ add_import_group(remove_by_prefix('org'))
+ add_import_group(twitter_imports)
+
+ print(CLASS_TEMPLATE % {
+ 'package': PACKAGE_NAME,
+ 'name': self._class_name,
+ 'wrapped': self._wrapped_type,
+ 'imports': '\n\n'.join(import_groups),
+ 'accessors': '\n\n'.join(self._accessors),
+ 'fields': (' ' + '\n '.join(self._fields) + '\n') if self._fields else '',
+ 'assignments': ('\n ' + '\n '.join(self._assignments)) if self._assignments else '',
+ }, file=f)
+
+
+# A namespace declaration, e.g.:
+# namespace java com.twitter.aurora.gen
+NAMESPACE_RE = 'namespace\s+(?P<lang>\w+)\s+(?P<namespace>[^\s]+)'
+
+# A possibly-parameterized type name, e.g.:
+# int
+# TaskConfig
+# Set<String>
+# Map<String, TaskConfig>
+TYPE_PATTERN = '(?P<type>\w+)(?:<(?P<params>[^>]+)>)?'
+
+
+# Matches a complete struct definnition, capturing the type and body.
+STRUCT_RE = '(?P<kind>enum|struct|union)\s+(?P<name>\w+)\s+{(?P<body>[^}]+)}'
+
+
+# A field definition within a struct, e.g.:
+# 1: string name
+# 15: Map<String, TaskConfig> configs # Configs mapped by name.
+FIELD_RE = '\s*\d+:\s+(?:(?:required|optional)\s+)?(%s)\s+(?P<name>\w+).*' % TYPE_PATTERN
+
+
+def parse_structs(thrift_defs):
+ '''Read all thrift structures found in a file.
+
+ This returns a list of Type objects representing the structs found
+ and the fields they contain.
+ '''
+ # Capture all namespace definitions.
+ namespaces = dict(re.findall(NAMESPACE_RE, thrift_defs))
+
+ def parse_field(field):
+ type_name = field.group('type')
+ type_params = field.group('params')
+ if type_params:
+ params = [Type(p) for p in type_params.replace(' ', '').split(',')]
+ ttype = ParameterizedType(type_name, params)
+ else:
+ ttype = Type(type_name)
+ return Field(ttype, field.group('name'))
+
+ def parse_fields(field_str):
+ return map(parse_field, re.finditer(FIELD_RE, field_str))
+
+ return [StructType(s.group('name'),
+ namespaces['java'],
+ s.group('kind'),
+ parse_fields(s.group('body')))
+ for s in re.finditer(STRUCT_RE, thrift_defs, flags=re.MULTILINE)]
+
+
+def generate_java(struct):
+ code = GeneratedCode(struct.codegen_name, struct.name)
+ code.add_import('com.google.common.base.Preconditions')
+ code.add_import('com.google.common.base.Function')
+ code.add_import('com.google.common.collect.ImmutableList')
+ code.add_import('com.google.common.collect.ImmutableSet')
+ code.add_import('com.google.common.collect.FluentIterable')
+ code.add_import(struct.absolute_name())
+
+ if struct.kind == 'union':
+ code.add_accessor(IMMUTABLE_FIELD_TEMPLATE
+ % {'type': '%s._Fields' % struct.name, 'fn_name': 'getSetField'})
+
+ # Accessor for each field.
+ for field in struct.fields:
+ if not (isinstance(field.ttype, StructType) and (field.ttype.kind == 'enum' or struct.kind == 'union')):
+ code.add_accessor(IMMUTABLE_FIELD_TEMPLATE
+ % {'type': 'boolean',
+ 'fn_name': field.isset_method()})
+ if field.ttype.immutable:
+ code.add_accessor(IMMUTABLE_FIELD_TEMPLATE % {'type': field.ttype.name,
+ 'fn_name': field.accessor_method()})
+ elif not struct.kind == 'union':
+ if isinstance(field.ttype, StructType):
+ return_type = field.ttype.codegen_name
+ elif isinstance(field.ttype, ParameterizedType):
+ return_type = '%s<%s>' % (field.ttype.name, field.ttype.param_names())
+ else:
+ return_type = field.ttype.name
+ code.add_accessor(FIELD_TEMPLATE % {'type': return_type,
+ 'fn_name': field.accessor_method(),
+ 'field': field.name})
+
+ if isinstance(field.ttype, StructType):
+ if field.ttype.kind == 'enum':
+ code.add_import(field.ttype.absolute_name())
+
+ if field.ttype.immutable:
+ # Direct accessor was already added.
+ pass
+ elif struct.kind == 'union':
+ copy_field = '%s.build(wrapped.%s())' % (field.ttype.codegen_name,
+ field.accessor_method())
+ code.add_accessor(FIELD_TEMPLATE % {'type': field.ttype.codegen_name,
+ 'fn_name': field.accessor_method(),
+ 'field': copy_field})
+ else:
+ args = {
+ 'field': field.name,
+ 'fn_name': field.accessor_method(),
+ 'isset': field.isset_method(),
+ 'type': field.ttype.codegen_name,
+ }
+ code.add_assignment(STRUCT_DECLARATION % args, STRUCT_ASSIGNMENT % args)
+ elif isinstance(field.ttype, ParameterizedType):
+ # Add necessary imports, supporting only List, Map, Set.
+ assert field.ttype.name in ['List', 'Map', 'Set'], 'Unrecognized type %s' % field.ttype.name
+ code.add_import('com.google.common.collect.Immutable%s' % field.ttype.name)
+ code.add_import('java.util.%s' % field.ttype.name)
+
+ params = field.ttype.params
+ if all([p.immutable for p in params]):
+ # All parameter types are immutable.
+ assignment = IMMUTABLE_COLLECTION_ASSIGNMENT
+ elif len(params) == 1:
+ # Only one non-immutable parameter.
+ # Assumes the parameter type is a struct and our code generator
+ # will make a compatible wrapper class and constructor.
+ assignment = STRUCT_COLLECTION_FIELD_ASSIGNMENT
+ else:
+ assert False, 'Unable to codegen accessor field for %s' % field.name
+ args = {'collection': field.ttype.name,
+ 'field': field.name,
+ 'fn_name': field.accessor_method(),
+ 'isset': field.isset_method(),
+ 'params': field.ttype.param_names()}
+ code.add_assignment(IMMUTABLE_COLLECTION_DECLARATION % args, assignment % args)
+ elif not field.ttype.immutable:
+ assert False, 'Making type %s immutable is not supported.' % field.ttype.name
+ return code
+
+
+THRIFT_ALIASES = {
+ 'bool': 'boolean',
+ 'i32': 'int',
+ 'i64': 'long',
+ 'double': 'double',
+ 'string': 'String',
+ 'list': 'List',
+ 'set': 'Set',
+ 'map': 'Map',
+ 'binary': 'byte[]',
+}
+
+
+def main(args):
+ if len(args) != 4:
+ print('usage: %s thrift_file struct_name output_directory' % sys.argv[0])
+ sys.exit(1)
+
+ thrift_file, struct_name, output_directory = sys.argv[1:]
+ print('Searching for %s in %s' % (sys.argv[2], sys.argv[1]))
+ with open(sys.argv[1]) as f:
+ # Load all structs found in the thrift file.
+ structs = parse_structs(f.read())
+
+ # The symbol table stores information about types we recognize.
+ # As new symbols are parsed, they are accumulated here.
+ # This is also seeded with JDK types.
+ # Note: byte[] is not immutable, but we'd rather accept it than copy.
+ primitives = dict((t, PrimitiveType(t, b)) for (t, b) in [('boolean', 'Boolean'),
+ ('int', 'Integer'),
+ ('long', 'Long'),
+ ('double', 'Double')])
+ lang_symbols = dict((t, Type(t, 'java.lang', immutable=True)) for t in ['String', 'byte[]'])
+ util_symbols = dict((t, Type(t, 'java.util')) for t in ['List', 'Map', 'Set'])
+ symbol_table = dict(primitives.items() + lang_symbols.items() + util_symbols.items())
+
+ def load_dependencies(struct):
+ # Fill in type information for fields by searching for dependencies.
+ for field in struct.fields:
+ if isinstance(field.ttype, ParameterizedType):
+ field.ttype.name = find_symbol(field.ttype.name).name
+ field.ttype.params = [find_symbol(p.name) for p in field.ttype.params]
+ else:
+ field.ttype = find_symbol(field.ttype.name)
+
+ def find_symbol(name):
+ name = THRIFT_ALIASES.get(name, name)
+ if name in symbol_table:
+ return symbol_table[name]
+
+ symbol = next((s for s in structs if s.name == name), None)
+ assert symbol, 'Failed to find required struct %s' % name
+ load_dependencies(symbol)
+ symbol_table[name] = symbol
+ return symbol
+
+ find_symbol(sys.argv[2])
+ print('Symbol table:')
+ for _, symbol in symbol_table.items():
+ print(' %s' % symbol)
+
+ for _, symbol in symbol_table.items():
+ if isinstance(symbol, StructType):
+ if symbol.kind == 'enum':
+ print('Skipping code generation for %s, since it is immutable' % symbol.name)
+ else:
+ package_dir = os.path.join(sys.argv[3], PACKAGE_NAME.replace('.', os.path.sep))
+ if not os.path.isdir(package_dir):
+ os.makedirs(package_dir)
+ gen_file = os.path.join(package_dir, '%s.java' % symbol.codegen_name)
+ print('Generating %s' % gen_file)
+ with open(gen_file, 'w') as f:
+ code = generate_java(symbol)
+ code.dump(f)
+
+
+if __name__ == '__main__':
+ main(sys.argv)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/BUILD b/src/main/python/apache/thermos/BUILD
new file mode 100644
index 0000000..937bcf3
--- /dev/null
+++ b/src/main/python/apache/thermos/BUILD
@@ -0,0 +1,18 @@
+import os
+
+page(name = 'readme', source = 'README.md')
+
+python_library(
+ name = 'thermos',
+ dependencies = [
+ pants('src/main/python/twitter/thermos/core'),
+ pants('src/main/python/twitter/thermos/monitoring'),
+ ],
+ provides = setup_py(
+ name = 'twitter.thermos',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ descripton = 'The Twitter Thermos runtime.',
+ ).with_binaries(
+ thermos = pants('src/main/python/twitter/thermos/bin:thermos'),
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/BUILD.thirdparty
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/BUILD.thirdparty b/src/main/python/apache/thermos/BUILD.thirdparty
new file mode 100644
index 0000000..5a54f4a
--- /dev/null
+++ b/src/main/python/apache/thermos/BUILD.thirdparty
@@ -0,0 +1,34 @@
+python_requirement(
+ name = 'psutil',
+ requirement = 'psutil==1.1.2',
+)
+
+python_requirement(
+ name = 'pystachio',
+ requirement = 'pystachio==0.7.2',
+)
+
+python_requirement(
+ name = 'mako',
+ requirement = 'mako==0.4.0',
+)
+
+python_requirement(
+ name = 'cherrypy',
+ requirement = 'cherrypy==3.2.2',
+)
+
+python_requirement(
+ name = 'bottle',
+ requirement = 'bottle==0.11.6',
+)
+
+python_requirement(
+ name = 'thrift',
+ requirement = 'thrift==0.9.1',
+)
+
+python_requirement(
+ name = 'mock',
+ requirement = 'mock==1.0.1',
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/README.md
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/README.md b/src/main/python/apache/thermos/README.md
new file mode 100644
index 0000000..e456e93
--- /dev/null
+++ b/src/main/python/apache/thermos/README.md
@@ -0,0 +1,461 @@
+** This document is deprecated and no longer updated. **
+
+# Thermos manual #
+
+[TOC]
+
+
+## tl;dr ##
+
+** You noticed the deprecation notice, right? **
+
+### What is Thermos? ###
+
+Thermos a simple process management framework used for orchestrating
+dependent processes within a single chroot. At Twitter, it is used as a
+process manager for Mesos tasks. In practice, there is a one-to-one
+correspondence between a Mesos task and Thermos task. This document
+describes how to use Thermos in a local development environment and does not
+describe how to run Thermos tasks on Mesos, though once you have a valid
+Thermos configuration doing so is a small step.
+
+### Simplest invocation ###
+
+Thermos lives in `science` at Twitter and publically on GitHub in `twitter-commons` (TBD).
+
+#### build ####
+
+Build the Thermos CLI and the Thermos observer.
+
+```shell
+$ ./pants src/python/twitter/thermos
+$ ./pants src/python/twitter/thermos/bin:thermos_observe
+```
+
+You can copy `dist/thermos.pex` to `thermos` somewhere in your `$PATH` or use a la carte.
+
+#### simplerun ####
+
+You can run Thermos tasks without first writing a configuration file using `simplerun`:
+
+```shell
+$ thermos simplerun 'echo hello world'
+Running command: 'echo hello world'
+ INFO] Forking Process(simple)
+ INFO] Process(simple) finished successfully [rc=0]
+ INFO] Killing task id: simple-20120529-162532.018646
+ INFO] => Current user: wickman
+ INFO] Kill complete.
+ INFO] Task succeeded.
+```
+
+#### introspection ####
+
+```shell
+$ thermos status --verbosity=3 simple-20120529-162532.018646
+Found task simple-20120529-162532.018646
+ simple-20120529-162532.018646 [owner: wickman] state: SUCCESS start: Tue May 29 16:25:32 2012
+ user: wickman ports: None
+ sandbox: None
+ process table:
+ - simple runs: 1 last: pid=57471, rc=0, finish:Tue May 29 16:25:32 2012, state:SUCCESS
+
+$ thermos tail simple-20120529-162532.018646
+Tail of terminal log /Users/wickman/.thermos/logs/simple-20120529-162532.018646/simple/0/stdout
+hello world
+```
+
+#### thermos observer ####
+
+```shell
+$ dist/thermos_observe.pex --root=$HOME/.thermos
+```
+
+This will fire up a webserver on `localhost:1338` that you can use as a web interface to interact with
+locally running Thermos tasks.
+
+
+## Building blocks ##
+
+Thermos is made of Processes and Tasks.
+
+### Thermos processes ###
+
+A Thermos process is simply a command-line that is invoked in a subshell. A single
+run of a process may succeed (have a zero exit status) or fail (have a
+non-zero exit status.) A process is considered permanently failed after a
+maximum number of individual run failures, by default one.
+
+### Thermos tasks ###
+
+A Thermos task is a collection of processes and constraints that dictate how
+and when to run them. By default there are no constraints between processes
+bound to a task and they run in parallel. The simplest (and currently only)
+Task level constraint is the `order` dependency: process B should not be run
+until process A has completed successfully. For example, process A could be
+`git clone` and process B could be `rake test`. It doesn't make sense to
+run process B until process A is fully completed and successful.
+
+
+### Thermos configuration ###
+
+The easiest way to invoke Thermos is using `thermos simplerun`. Under the
+covers, this synthesizes a Thermos task with a single Thermos process named
+`simple`. To do the same thing via Thermos configuration, add
+the following to `simple.thermos`:
+
+```python
+process = Process(name = 'simple', cmdline = 'echo hello world')
+task = Task(name = 'simple', processes = [process])
+export(task)
+```
+
+Then invoke by `thermos run simple.thermos`.
+
+Configuring Thermos is done through
+[pystachio](http://github.com/wickman/pystachio) templates. These templates
+behave like structurally typed collections of key-value pairs. You can
+either construct configuration through plain old Python objects (as above)
+or as Python dictionaries or JSON that is coerced into Python dictionaries.
+
+For example, the above configuration is equivalent to:
+
+```python
+export({
+ 'name': 'simple',
+ 'processes': [{
+ 'name': 'simple',
+ 'cmdline': 'echo hello world'
+ }]
+})
+```
+
+The full Thermos pystachio schema can be found at
+`src/python/twitter/thermos/config/schema.py` and is mostly described below.
+
+
+## Configuration reference ##
+
+### Process objects ###
+
+Processes fundamentally consist of a `name`, `cmdline`. You will rarely
+need to specify anything more.
+
+<table>
+ <tr> <td colspan=2> <b>Process schema</b> </td> </tr>
+ <tr> <td> <em>name</em> (required) </td> <td> Process name (String) </td> </tr>
+ <tr> <td> <em>cmdline</em> (required) </td> <td> Command line (String) </td> </tr>
+ <tr> <td> <em>max_failures</em> </td> <td> Max failures (Integer, default 1) </td> </tr>
+ <tr> <td> <em>daemon</em> </td> <td> Daemon process? (Boolean, default False) </td> </tr>
+ <tr> <td> <em>ephemeral</em> </td> <td> Ephemeral process? (Boolean, default False) </td> </tr>
+ <tr> <td> <em>min_duration</em> </td> <td> Min duration between runs in seconds (Integer, default 15) </td> </tr>
+ <tr> <td> <em>final</em> </td> <td> This is a finalizing process that should run last (Boolean, default False) </td> </tr>
+</table>
+
+
+#### name ####
+
+The name is any string that is a valid UNIX filename (specifically no
+slashes, NULLs or leading periods.) Each process name within a task must be
+unique.
+
+#### cmdline ####
+
+The command line is invoked in a bash subshell, so can be full-blown bash
+scripts, though no command-line arguments are supplied.
+
+#### max_failures ####
+
+The maximum number of failures (non-zero exit statuses) this process can
+sustain before being marked permanently failed and not retried. If a
+process becomes permanently failed, Thermos looks to the task failure limit
+(usually 1) to determine whether or not the Thermos task should be failed.
+
+Setting max_failures to 0 means that this process will be retried
+indefinitely until a successful (zero) exit status is achieved. It will be retried
+at most once every `min_duration` seconds in order to prevent DoSing the
+coordinating thermos scheduler.
+
+#### daemon ####
+
+By default Thermos processes are non-daemon. If `daemon` is set to True, a
+successful (zero) exit status will not prevent future process runs.
+Instead, the process will be reinvoked after min_duration seconds. However,
+the maximum failure limit will still apply. A combination of `daemon=True`
+and `max_failures=0` will cause a process to be retried indefinitely
+regardless of exit status. This should generally be avoided for very
+short-lived processes because of the accumulation of checkpointed state for
+each process run.
+
+#### ephemeral ####
+
+By default Thermos processes are non-ephemeral. If `ephemeral` is set to
+True, the status of the process is not used to determine whether the task in
+which it is bound has completed. Take for example a Thermos task with a
+non-ephemeral webserver process and an ephemeral logsaver process that
+periodically checkpoints its log files to a centralized data store. The
+task is considered finished once the webserver process has completed,
+regardless of the current status of the logsaver.
+
+#### min_duration ####
+
+Processes may succeed or fail multiple times throughout the duration of a
+single task. Each of these is called a "process run." The `min_duration` is the minimum
+number of seconds the scheduler waits between running the same process.
+
+#### final ####
+
+Processes can be grouped into two classes: ordinary processes and finalizing
+processes. By default, Thermos processes are ordinary. They run as long as
+the Thermos Task is considered healthy (i.e., no failure limits have been
+reached.) But once all regular Thermos processes have either finished or the
+Task has reached a certain failure threshold, it moves into a "finalization"
+stage and then runs all finalizing processes. These are typically processes
+necessary for cleaning up the task, such as log checkpointers, or perhaps
+e-mail notifications that the task has completed.
+
+Finalizing processes may not depend upon ordinary processes or vice-versa, however
+finalizing processes may depend upon other finalizing processes and will otherwise run as
+a typical process schedule.
+
+
+### Task objects ###
+
+Tasks fundamentally consist of a `name` and a list of processes `processes`.
+Processes can be further constrained with `constraints`
+
+<table>
+ <tr> <td colspan=2> <b>Task schema</b> </td> </tr>
+ <tr> <td> <em>name</em> (required) </td> <td> Task name (String) </td> </tr>
+ <tr> <td> <em>processes</em> (required) </td> <td> List of processes (List of Process objects) </td> </tr>
+ <tr> <td> <em>constraints</em> </td> <td> Constraints (List of Constraint objects, default []) </td> </tr>
+ <tr> <td> <em>resources</em> </td> <td> Resource footprint (Resource, optional) </td> </tr>
+ <tr> <td> <em>max_failures</em> </td> <td> Max failures (Integer, default 1) </td> </tr>
+ <tr> <td> <em>max_concurrency</em> </td> <td> Max concurrency (Integer, default 0 = unlimited concurrency) </td> </tr>
+ <tr> <td> <em>finalization_wait</em> </td> <td> Amount of time allocated to run finalizing processes (Integer in seconds, default 30) </td> </tr>
+</table>
+
+<table>
+ <tr> <td colspan=2> <b>Constraint schema</b> </td> </tr>
+ <tr> <td> <em>order</em> </td> <td> List of process names that should run in order (List of Strings)</td> </tr>
+</table>
+
+<table>
+ <tr> <td colspan=2> <b>Resource schema</b> </td> </tr>
+ <tr> <td> <em>cpu</em> (required) </td> <td> Number of cores (Float) </td> </tr>
+ <tr> <td> <em>ram</em> (required) </td> <td> Bytes of RAM (Integer) </td> </tr>
+ <tr> <td> <em>disk</em> (required) </td> <td> Bytes of disk (Integer) </td> </tr>
+</table>
+
+#### name ####
+
+The name is used to label the task and is used for reporting in the observer UI and for
+management in the thermos CLI.
+
+#### processes ####
+
+Processes is an unordered list of `Process` objects. In order to place temporal constraints upon
+them, you must use `constraints`.
+
+#### constraints ####
+
+A list of `Constraint` objects. Currently only one type of constraint is supported, the `order` constraint.
+`order` is a list of process names that should run in order. For example,
+
+```python
+process = Process(cmdline = "echo hello world")
+task = Task(name = "echoes", processes = [process(name = "first"), process(name = "second")],
+ constraints = [Constraint(order = ["first", "second"]))
+```
+
+Constraints can be supplied ad-hoc and in duplicate and not all processes need be constrained, however
+tasks with cycles will be rejected by the Thermos scheduler.
+
+#### resources ####
+
+`resources` is a `Resource` object described by `cpu`, `ram`, and `disk`. It is currently unused by
+Thermos but reserved for future use in constraining the resource consumption of a task.
+
+#### max_failures ####
+
+`max_failures` is the number of failed processes in order for this task to be marked as failed. A single
+process run does not consistute a failure. For example:
+
+```python
+template = Process(max_failures=10)
+task = Task(name = "fail", processes = [template(name = "failing", cmdline = "exit 1"),
+ template(name = "succeeding", cmdline = "exit 0")],
+ max_failures=2)
+```
+
+The `failing` process would fail 10 times before being marked as permanently
+failed, and the `succeeding` process would succeed on the first run. The
+task would succeed despite only allowing for two failed processes. To be
+more specific, there would be 10 failed process runs yet 1 failed process.
+
+#### max_concurrency ####
+
+For tasks with a number of expensive but otherwise independent processes, it
+may be desirable to limit the amount of concurrency provided by the Thermos
+scheduler rather than artificially constraining them through `order`
+constraints. For example, a test framework may generate a task with 100
+test run processes, but would like to run it on a machine with only 4 cores.
+You can limit the amount of parallelism to 4 by setting `max_concurrency=4`
+in your task configuration.
+
+For example, the following Thermos task spawns 180 processes ("mappers") to compute
+individual elements of a 180 degree sine table, all dependent upon one final process ("reducer")
+to tabulate the results:
+
+```python
+def make_mapper(id):
+ return Process(
+ name = "mapper%03d" % id,
+ cmdline = "echo 'scale=50;s(%d*4*a(1)/180)' | bc -l > temp.sine_table.%03d" % (id, id))
+
+def make_reducer():
+ return Process(name = "reducer",
+ cmdline = "cat temp.* | nl > sine_table.txt && rm -f temp.*")
+
+processes = map(make_mapper, range(180))
+
+task = Task(
+ name = "mapreduce",
+ processes = processes + [make_reducer()],
+ constraints = [Constraint(order = [mapper.name(), 'reducer']) for mapper in processes],
+ max_concurrency = 8)
+
+export(task)
+```
+
+#### finalization_wait ####
+
+Tasks have three active stages: ACTIVE, CLEANING and FINALIZING. The ACTIVE stage is when
+ordinary processes run. This stage will last as long as processes are running and the
+task is healthy. The moment either all processes have finished successfully or the task
+has reached a maximum process failure limit, it will go into CLEANING stage and send SIGTERMs
+to all currently running processes and their process trees. Once all processes have
+terminated, the task goes into FINALIZING stage and invokes the schedule of all processes
+with the "final" bit set.
+
+This whole process from the end of ACTIVE stage to the end of FINALIZING must take place within
+"finalization_wait" seconds. If it does not complete within that time, all remaining
+processes will be sent SIGKILLs (or if they depend upon processes that have not yet completed,
+will never be invoked.)
+
+Client applications with higher priority may be able to force a shorter
+finalization wait (e.g. through parameters to `thermos kill`), so this is
+mostly a best-effort signal.
+
+
+## REPL ##
+
+You can interactively experiment with the Thermos configuration REPL via the
+`src/python/twitter/thermos/config:repl` target:
+
+```python
+$ ./pants py src/python/twitter/thermos/config:repl
+Build operating on target: PythonBinary(src/python/twitter/thermos/config/BUILD:repl)
+Thermos Config REPL
+>>> boilerplate = Process(cmdline = "echo hello world")
+>>> boilerplate
+Process(cmdline=echo hello world, daemon=0, max_failures=1, ephemeral=0, min_duration=5)
+>>> boilerplate.check()
+TypeCheck(FAILED): Process[name] is required.
+>>> boilerplate(name = "hello world").check()
+TypeCheck(OK)
+```
+
+## Thermos templating ##
+
+The Thermos DSL is implemented in [pystachio](http://github.com/wickman/pystachio) which means that
+a simple Mustache-like templating layer is available for use when configuring tasks.
+
+### Ordinary templates ###
+
+By using Mustache style templates in your job, you can do allow some amount of runtime configuration
+of your tasks:
+
+```
+>>> process = Process(name = "hello", cmdline = "echo hello {{first}}")
+>>> process
+Process(cmdline=echo hello {{first}}, daemon=0, name=hello, max_failures=1, ephemeral=0, min_duration=5)
+>>> process.check()
+TypeCheck(FAILED): Process[cmdline] failed: Uninterpolated variables: {{first}}
+```
+
+This process leaves `{{first}}` as a free variable. It can be filled elsewhere in the configuration, e.g. via
+`%` or `bind`:
+```
+>>> process % {'first': 'brian'}
+Process(cmdline=echo hello brian, daemon=0, name=hello, max_failures=1, ephemeral=0, min_duration=5)
+>>> process.bind(first = 'brian')
+Process(cmdline=echo hello brian, daemon=0, name=hello, max_failures=1, ephemeral=0, min_duration=5)
+```
+
+If this is left unbound, the thermos CLI will complain:
+
+```
+$ thermos run thermos/examples/tutorial/unbound.thermos
+twitter.thermos.config.loader.InvalidTask: Task[processes] failed: Element in ProcessList failed check: Process[cmdline] failed: Uninterpolated variables: {{first}}
+```
+
+But free variables can be populated at runtime using the `-E` parameter:
+
+```
+$ thermos run -E first=brian thermos/examples/tutorial/unbound.thermos
+Writing log files to disk in /var/tmp
+ INFO] Forking Process(hello)
+ INFO] Process(hello) finished successfully [rc=0]
+ INFO] Killing task id: unbound-20120530-124903.934384
+ INFO] => Current user: wickman
+ INFO] Kill complete.
+ INFO] Task succeeded.
+```
+
+### Special templates ###
+
+Each Thermos task when run has a special `ThermosContext` template bound to the `thermos` variable.
+Currently this provides three things: `thermos.task_id`, `thermos.user` and `thermos.ports`. The
+`task_id` is the id generated (or supplied at runtime, in the case of the thermos CLI) for the task
+and the `thermos.user` is the real user the task runs as. `thermos.ports` is a mapping of named ports
+supplied by the user and exposed through the user interface. For example, to run (and background)
+the observer on port 1338:
+
+```
+$ thermos simplerun --daemon -P http:1338 'dist/thermos_observe.pex --http_port={{thermos.ports[http]}} --root=$HOME/.thermos'
+```
+
+If you go to http://localhost:1338, this bound port `http` will be exposed
+via the UI on both the main and task pages.
+
+To kill the background daemon:
+
+```
+$ thermos kill simple.*
+```
+
+### Includes ###
+
+It is possible to include other Thermos configurations via the `include` parameter. For example,
+`thermos/examples/tutorial/lib/math.thermos`:
+
+```python
+bc = Process(cmdline = "echo 'scale={{precision}};{{command}}' | bc -l")
+pi = bc(name = "pi").bind(command = "4*a(1)")
+e = bc(name = "e").bind(command = "e(1)")
+```
+
+and `thermos/examples/tutorial/pi.thermos`:
+
+```python
+include('lib/math.thermos')
+
+export(Task(name = "compute_pi", processes = [pi]))
+```
+
+can then be executed with the free `precision` variable:
+
+```shell
+$ thermos run -E precision=500 thermos/examples/tutorial/pi.thermos
+```
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/__init__.py b/src/main/python/apache/thermos/__init__.py
new file mode 100644
index 0000000..de40ea7
--- /dev/null
+++ b/src/main/python/apache/thermos/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/bin/BUILD b/src/main/python/apache/thermos/bin/BUILD
new file mode 100644
index 0000000..5aa41a2
--- /dev/null
+++ b/src/main/python/apache/thermos/bin/BUILD
@@ -0,0 +1,24 @@
+python_binary(
+ name = 'thermos_ckpt',
+ source = 'thermos_ckpt.py',
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/python/twitter/thermos/common'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift')
+ ]
+)
+
+python_binary(
+ name = 'thermos',
+ source = 'thermos.py',
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/thermos/common'),
+ pants('src/main/python/twitter/thermos/config:schema'),
+ pants('src/main/python/twitter/thermos/core'),
+ pants('src/main/python/twitter/thermos/monitoring'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ],
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/bin/__init__.py b/src/main/python/apache/thermos/bin/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/bin/thermos.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/bin/thermos.py b/src/main/python/apache/thermos/bin/thermos.py
new file mode 100644
index 0000000..5e695b4
--- /dev/null
+++ b/src/main/python/apache/thermos/bin/thermos.py
@@ -0,0 +1,653 @@
+# TODO(wickman) This needs some usage/help refactoring.
+
+from __future__ import print_function
+
+from collections import namedtuple
+import getpass
+import os
+import pprint
+import pwd
+import re
+import sys
+import time
+
+from twitter.common import app, log
+from twitter.common.log.options import LogOptions
+from twitter.common.dirutil import du, tail_f
+from twitter.common.dirutil.tail import tail as tail_closed
+from twitter.common.quantity import Amount, Time, Data
+from twitter.common.quantity.parse_simple import parse_time, parse_data
+from twitter.common.recordio import RecordIO, ThriftRecordReader
+from twitter.thermos.common.path import TkPath
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+from twitter.thermos.common.options import add_port_to, add_binding_to
+from twitter.thermos.config.loader import ThermosConfigLoader, ThermosTaskWrapper
+from twitter.thermos.config.schema import (
+ Process,
+ Resources,
+ Task)
+from twitter.thermos.core.helper import TaskRunnerHelper
+from twitter.thermos.core.runner import TaskRunner
+from twitter.thermos.monitoring.detector import TaskDetector
+from twitter.thermos.monitoring.garbage import TaskGarbageCollector, DefaultCollector
+from twitter.thermos.monitoring.monitor import TaskMonitor
+
+from gen.twitter.thermos.ttypes import (
+ ProcessState,
+ RunnerCkpt,
+ RunnerState,
+ TaskState)
+
+from pystachio import Ref
+from pystachio.naming import frozendict
+
+app.add_option("--root", dest="root", metavar="PATH",
+ default=TaskPath.DEFAULT_CHECKPOINT_ROOT,
+ help="the thermos config root")
+
+
+def set_keep(option, opt_str, value, parser):
+ setattr(parser.values, option.dest, opt_str.startswith('--keep'))
+
+
+def get_task_from_options(args, opts, **kw):
+ loader = ThermosConfigLoader.load_json if opts.json else ThermosConfigLoader.load
+
+ if len(args) != 1:
+ app.error('Should specify precisely one config, instead got: %s' % args)
+
+ tasks = loader(args[0], bindings=opts.bindings, **kw)
+
+ task_list = list(tasks.tasks())
+ if len(task_list) == 0:
+ app.error("No tasks specified!")
+
+ if opts.task is None and len(task_list) > 1:
+ app.error("Multiple tasks in config but no task name specified!")
+
+ task = None
+ if opts.task is not None:
+ for t in task_list:
+ if t.task().name().get() == opts.task:
+ task = t
+ break
+ if task is None:
+ app.error("Could not find task %s!" % opts.task)
+ else:
+ task = task_list[0]
+
+ if kw.get('strict', False):
+ if not task.task.check().ok():
+ app.error(task.task.check().message())
+
+ return task
+
+
+def daemonize():
+ def daemon_fork():
+ try:
+ if os.fork() > 0:
+ os._exit(0)
+ except OSError as e:
+ sys.stderr.write('[pid:%s] Failed to fork: %s\n' % (os.getpid(), e))
+ sys.exit(1)
+ daemon_fork()
+ os.setsid()
+ daemon_fork()
+ sys.stdin, sys.stdout, sys.stderr = (open('/dev/null', 'r'),
+ open('/dev/null', 'a+'),
+ open('/dev/null', 'a+', 0))
+
+
+def tasks_from_re(expressions, root, state=None):
+ task_ids = [t_id for _, t_id in TaskDetector(root=root).get_task_ids(state=state)]
+ matched_tasks = set()
+ for task_expr in map(re.compile, expressions):
+ for task_id in task_ids:
+ if task_expr.match(task_id):
+ matched_tasks.add(task_id)
+ return matched_tasks
+
+
+def _really_run(task, root, sandbox, task_id=None, user=None, prebound_ports=None, chroot=None,
+ daemon=False):
+ prebound_ports = prebound_ports or {}
+ missing_ports = set(task.ports()) - set(prebound_ports.keys())
+ if missing_ports:
+ app.error('ERROR! Unbound ports: %s' % ' '.join(port for port in missing_ports))
+ task_runner = TaskRunner(task.task, root, sandbox, task_id=task_id,
+ user=user, portmap=prebound_ports, chroot=chroot)
+ if daemon:
+ print('Daemonizing and starting runner.')
+ try:
+ log.teardown_stderr_logging()
+ daemonize()
+ except Exception as e:
+ print("Failed to daemonize: %s" % e)
+ sys.exit(1)
+ try:
+ task_runner.run()
+ except KeyboardInterrupt:
+ print('Got keyboard interrupt, killing job!')
+ task_runner.close_ckpt()
+ task_runner.kill()
+
+
+@app.command
+@app.command_option("--user", metavar="USER", default=getpass.getuser(), dest='user',
+ help="run as this user. if not $USER, must have setuid privilege.")
+@app.command_option("--enable_chroot", dest="chroot", default=False, action='store_true',
+ help="chroot tasks to the sandbox before executing them, requires "
+ "root privileges.")
+@app.command_option("--task", metavar="TASKNAME", default=None, dest='task',
+ help="The thermos task within the config that should be run. Only required if "
+ "there are multiple tasks exported from the thermos configuration.")
+@app.command_option("--task_id", metavar="STRING", default=None, dest='task_id',
+ help="The id to which this task should be bound, synthesized from the task "
+ "name if none provided.")
+@app.command_option("--json", default=False, action='store_true', dest='json',
+ help="Read the source file in json format.")
+@app.command_option("--sandbox", metavar="PATH", default="/var/lib/thermos/sandbox", dest='sandbox',
+ help="The sandbox in which to run the task.")
+@app.command_option("-P", "--port", type="string", nargs=1, action="callback",
+ callback=add_port_to('prebound_ports'), dest="prebound_ports", default=[],
+ metavar="NAME:PORT", help="bind named PORT to NAME.")
+@app.command_option("-E", "--environment", type="string", nargs=1, action="callback",
+ callback=add_binding_to('bindings'), default=[], dest="bindings",
+ metavar="NAME=VALUE",
+ help="bind the configuration environment variable NAME to VALUE.")
+@app.command_option("--daemon", default=False, action='store_true', dest='daemon',
+ help="fork and daemonize the thermos runner.")
+def run(args, options):
+
+ """Run a thermos task.
+
+ Usage: thermos run [options] config
+ Options:
+ --user=USER run as this user. if not $USER, must have setuid privilege.
+ --enable_chroot chroot into the sandbox for this task, requires superuser
+ privilege
+ --task=TASKNAME the thermos task within the config that should be run. only
+ required if there are multiple tasks exported from the thermos
+ configuration.
+ --task_id=STRING the id to which this task should be bound, synthesized from the
+ task name if none provided.
+ --json specify that the config is in json format instead of pystachio
+ --sandbox=PATH the sandbox in which to run the task
+ [default: /var/lib/thermos/sandbox]
+ -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
+ multiple times to bind multiple names.)
+ -E/--environment=NAME=VALUE bind the configuration environment variable NAME to
+ VALUE.
+ --daemon Fork and daemonize the task.
+ """
+ thermos_task = get_task_from_options(args, options)
+ _really_run(thermos_task,
+ options.root,
+ options.sandbox,
+ task_id=options.task_id,
+ user=options.user,
+ prebound_ports=options.prebound_ports,
+ chroot=options.chroot,
+ daemon=options.daemon)
+
+
+def inspect_unwrap(obj):
+ if isinstance(obj, frozendict):
+ return dict((key, inspect_unwrap(val)) for (key, val) in obj.items())
+ if isinstance(obj, (list, tuple, set)):
+ return tuple(inspect_unwrap(val) for val in obj)
+ return obj
+
+
+@app.command
+@app.command_option("--task", metavar="TASKNAME", default=None, dest='task',
+ help="The thermos task within the config that should be inspected. Only "
+ "required if there are multiple tasks exported from the thermos "
+ "configuration.")
+@app.command_option("--json", default=False, action='store_true', dest='json',
+ help="Read the source file in json format instead of pystachio.")
+@app.command_option("-P", "--port", type="string", nargs=1, action="callback",
+ callback=add_port_to('prebound_ports'), dest="prebound_ports", default=[],
+ metavar="NAME:PORT", help="bind named PORT to NAME.")
+@app.command_option("-E", "--environment", type="string", nargs=1, action="callback",
+ callback=add_binding_to('bindings'), default=[], dest="bindings",
+ metavar="NAME=VALUE",
+ help="bind the configuration environment variable NAME to VALUE.")
+def inspect(args, options):
+ """Inspect a thermos config and display the evaluated task
+
+ Usage: thermos inspect [options] config
+ Options:
+ --task=TASKNAME the thermos task within the config that should be inspected. Only
+ required if there are multiple tasks exported from the thermos
+ configuration.
+ --json specify that the config is in json format instead of pystachio
+ -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
+ multiple times to bind multiple names.)
+ -E/--environment=NAME=VALUE bind the configuration environment variable NAME to
+ VALUE.
+ """
+ thermos_task = get_task_from_options(args, options)
+ ti, _ = thermos_task.task().interpolate()
+ pprint.pprint(inspect_unwrap(ti.get()), indent=4)
+
+
+@app.command
+@app.command_option("--user", metavar="USER", default=getpass.getuser(), dest='user',
+ help="run as this user. if not $USER, must have setuid privilege.")
+@app.command_option("--name", metavar="STRING", default='simple', dest='name',
+ help="The name to give this task.")
+@app.command_option("--task_id", metavar="STRING", default=None, dest='task_id',
+ help="The id to which this task should be bound, synthesized from the task "
+ "name if none provided.")
+@app.command_option("-P", "--port", type="string", nargs=1, action="callback",
+ callback=add_port_to('prebound_ports'), dest="prebound_ports", default=[],
+ metavar="NAME:PORT", help="bind named PORT to NAME.")
+@app.command_option("-E", "--environment", type="string", nargs=1, action="callback",
+ callback=add_binding_to('bindings'), default=[], dest="bindings",
+ metavar="NAME=VALUE",
+ help="bind the configuration environment variable NAME to VALUE.")
+@app.command_option("--daemon", default=False, action='store_true', dest='daemon',
+ help="fork and daemonize the thermos runner.")
+def simplerun(args, options):
+ """Run a simple command line as a thermos task.
+
+ Usage: thermos simplerun [options] [--] commandline
+ Options:
+ --user=USER run as this user. if not $USER, must have setuid privilege.
+ --name=STRING the name to give this task. ('simple' by default)
+ --task_id=STRING the id to which this task should be bound, synthesized from the
+ task name if none provided.
+ -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
+ multiple times to bind multiple names.)
+ -E/--environment=NAME=VALUE bind the configuration environment variable NAME to
+ VALUE.
+ --daemon Fork and daemonize the task.
+ """
+ try:
+ cutoff = args.index('--')
+ cmdline = ' '.join(args[cutoff+1:])
+ except ValueError:
+ cmdline = ' '.join(args)
+
+ print("Running command: '%s'" % cmdline)
+
+ thermos_task = ThermosTaskWrapper(Task(
+ name = options.name,
+ resources = Resources(cpu = 1.0, ram = 256 * 1024 * 1024, disk = 0),
+ processes = [Process(name = options.name, cmdline = cmdline)]))
+
+ _really_run(thermos_task,
+ options.root,
+ None,
+ task_id=options.task_id,
+ user=options.user,
+ prebound_ports=options.prebound_ports,
+ chroot=False,
+ daemon=options.daemon)
+
+
+@app.command
+@app.command_option("--simple", default=False, dest='simple', action='store_true',
+ help="Only print the checkpoint records, do not replay them.")
+def read(args, options):
+ """Replay a thermos checkpoint.
+
+ Usage: thermos read [options] checkpoint_filename
+ Options:
+ --simple Do not replay the full task state machine. Only print out the contents of
+ each checkpoint log message.
+ """
+ if len(args) != 1:
+ app.error('Expected one checkpoint file, got %s' % len(args))
+ if not os.path.exists(args[0]):
+ app.error('Could not find %s' % args[0])
+
+ dispatcher = CheckpointDispatcher()
+ state = RunnerState(processes={})
+ with open(args[0], 'r') as fp:
+ try:
+ for record in ThriftRecordReader(fp, RunnerCkpt):
+ if not options.simple:
+ dispatcher.dispatch(state, record)
+ else:
+ print('CKPT: %s' % record)
+ except RecordIO.Error as err:
+ print("Failed to recover from %s: %s" % (fp.name, err))
+ return
+
+ if not options.simple:
+ if state is None or state.header is None:
+ print('Checkpoint stream CORRUPT or outdated format')
+ return
+ print('Recovered Task Header:')
+ print(' id: %s' % state.header.task_id)
+ print(' user: %s' % state.header.user)
+ print(' host: %s' % state.header.hostname)
+ print(' sandbox: %s' % state.header.sandbox)
+ if state.header.ports:
+ print(' ports: %s' % ' '.join(
+ '%s->%s' % (name, port) for (name, port) in state.header.ports.items()))
+ print('Recovered Task States:')
+ for task_status in state.statuses:
+ print(' %s [pid: %d] => %s' % (
+ time.asctime(time.localtime(task_status.timestamp_ms/1000.0)),
+ task_status.runner_pid,
+ TaskState._VALUES_TO_NAMES[task_status.state]))
+ print('Recovered Processes:')
+ for process, process_history in state.processes.items():
+ print(' %s runs: %s' % (process, len(process_history)))
+ for k in reversed(range(len(process_history))):
+ run = process_history[k]
+ print(' %2d: pid=%d, rc=%s, finish:%s, state:%s' % (
+ k,
+ run.pid,
+ run.return_code if run.return_code is not None else '',
+ time.asctime(time.localtime(run.stop_time)) if run.stop_time else 'None',
+ ProcessState._VALUES_TO_NAMES.get(run.state, 'Unknown')))
+
+
+@app.command
+def kill(args, options):
+ """Kill task(s)
+
+ Usage: thermos kill task_id1 [task_id2 ...]
+
+ Regular expressions may be used to match multiple tasks.
+ """
+ if not args:
+ print('Must specify tasks!', file=sys.stderr)
+ return
+
+ matched_tasks = tasks_from_re(args, options.root, state='active')
+
+ if not matched_tasks:
+ print('No active tasks matched.')
+ return
+
+ for task_id in matched_tasks:
+ print('Killing %s...' % task_id, end='')
+ TaskRunnerHelper.kill(task_id, options.root, force=True)
+ print('done.')
+
+
+@app.command
+@app.command_option("--max_age", metavar="AGE", default=None, dest='max_age',
+ help="Max age in human readable form, e.g. 2d5h or 7200s")
+@app.command_option("--max_tasks", metavar="NUM", default=None, dest='max_tasks',
+ help="Max number of tasks to keep.")
+@app.command_option("--max_space", metavar="SPACE", default=None, dest='max_space',
+ help="Max space to allow for tasks, e.g. 20G.")
+@app.command_option("--keep-data", "--delete-data",
+ metavar="PATH", default=True,
+ action='callback', callback=set_keep, dest='keep_data',
+ help="Keep data.")
+@app.command_option("--keep-logs", "--delete-logs",
+ metavar="PATH", default=True,
+ action='callback', callback=set_keep, dest='keep_logs',
+ help="Keep logs.")
+@app.command_option("--keep-metadata", "--delete-metadata",
+ metavar="PATH", default=True,
+ action='callback', callback=set_keep, dest='keep_metadata',
+ help="Keep metadata.")
+@app.command_option("--force", default=False, action='store_true', dest='force',
+ help="Perform garbage collection without confirmation")
+@app.command_option("--dryrun", default=False, action='store_true', dest='dryrun',
+ help="Don't actually run garbage collection.")
+def gc(args, options):
+ """Garbage collect task(s) and task metadata.
+
+ Usage: thermos gc [options] [task_id1 task_id2 ...]
+
+ If tasks specified, restrict garbage collection to only those tasks,
+ otherwise all tasks are considered. The optional constraints are still
+ honored.
+
+ Options:
+ --max_age=AGE Max age in quasi-human readable form, e.g. --max_age=2d5h,
+ format *d*h*m*s [default: skip]
+ --max_tasks=NUM Max number of tasks to keep [default: skip]
+ --max_space=SPACE Max space to allow for tasks [default: skip]
+ --[keep/delete-]metadata Garbage collect metadata [default: keep]
+ --[keep/delete-]logs Garbage collect logs [default: keep]
+ --[keep/delete-]data Garbage collect data [default: keep]
+ WARNING: Do NOT do this if your sandbox is $HOME.
+ --force Perform garbage collection without confirmation [default: false]
+ --dryrun Don't actually run garbage collection [default: false]
+ """
+ print('Analyzing root at %s' % options.root)
+ gc_options = {}
+ if options.max_age is not None:
+ gc_options['max_age'] = parse_time(options.max_age)
+ if options.max_space is not None:
+ gc_options['max_space'] = parse_data(options.max_space)
+ if options.max_tasks is not None:
+ gc_options['max_tasks'] = int(options.max_tasks)
+ gc_options.update(include_data = not options.keep_data,
+ include_metadata = not options.keep_metadata,
+ include_logs = not options.keep_logs,
+ verbose = True, logger = print)
+ tgc = TaskGarbageCollector(root=options.root)
+
+ if args:
+ gc_tasks = tasks_from_re(args, options.root, state='finished')
+ else:
+ print('No task ids specified, using default collector.')
+ gc_tasks = [task.task_id for task in DefaultCollector(tgc, **gc_options).run()]
+
+ if not gc_tasks:
+ print('No tasks to garbage collect. Exiting')
+ return
+
+ def maybe(function, *args):
+ if options.dryrun:
+ print(' would run %s%r' % (function.__name__, args))
+ else:
+ function(*args)
+
+ value = 'y'
+ if not options.force:
+ value = raw_input("Continue [y/N]? ") or 'N'
+ if value.lower() == 'y':
+ print('Running gc...')
+ tgc = TaskGarbageCollector(root=options.root)
+ for task in gc_tasks:
+ print(' Task %s ' % task, end='')
+ print('data (%s) ' % ('keeping' if options.keep_data else 'deleting'), end='')
+ print('logs (%s) ' % ('keeping' if options.keep_logs else 'deleting'), end='')
+ print('metadata (%s) ' % ('keeping' if options.keep_metadata else 'deleting'))
+ if not options.keep_data:
+ maybe(tgc.erase_data, task)
+ if not options.keep_logs:
+ maybe(tgc.erase_logs, task)
+ if not options.keep_metadata:
+ maybe(tgc.erase_metadata, task)
+ print('done.')
+ else:
+ print('Cancelling gc.')
+
+
+@app.command
+@app.command_option("--verbosity", default=0, dest='verbose', type='int',
+ help="Display more verbosity")
+@app.command_option("--only", default=None, dest='only', type='choice',
+ choices=('active', 'finished'), help="Display only tasks of this type.")
+def status(args, options):
+ """Get the status of task(s).
+
+ Usage: thermos status [options] [task_name(s) or task_regexp(s)]
+
+ Options:
+ --verbosity=LEVEL Verbosity level for logging. [default: 0]
+ --only=TYPE Only print tasks of TYPE (options: active finished)
+ """
+ detector = TaskDetector(root=options.root)
+
+ def format_task(task_id):
+ checkpoint_filename = detector.get_checkpoint(task_id)
+ checkpoint_stat = os.stat(checkpoint_filename)
+ try:
+ checkpoint_owner = pwd.getpwuid(checkpoint_stat.st_uid).pw_name
+ except:
+ checkpoint_owner = 'uid:%s' % checkpoint_stat.st_uid
+ print(' %-20s [owner: %8s]' % (task_id, checkpoint_owner), end='')
+ if options.verbose == 0:
+ print()
+ if options.verbose > 0:
+ state = CheckpointDispatcher.from_file(checkpoint_filename)
+ if state is None or state.header is None:
+ print(' - checkpoint stream CORRUPT or outdated format')
+ return
+ print(' state: %8s' % TaskState._VALUES_TO_NAMES.get(state.statuses[-1].state, 'Unknown'),
+ end='')
+ print(' start: %25s' % time.asctime(time.localtime(state.header.launch_time_ms/1000.0)))
+ if options.verbose > 1:
+ print(' user: %s' % state.header.user, end='')
+ if state.header.ports:
+ print(' ports: %s' % ' '.join('%s -> %s' % (key, val)
+ for key, val in state.header.ports.items()))
+ else:
+ print(' ports: None')
+ print(' sandbox: %s' % state.header.sandbox)
+ if options.verbose > 2:
+ print(' process table:')
+ for process, process_history in state.processes.items():
+ print(' - %s runs: %s' % (process, len(process_history)), end='')
+ last_run = process_history[-1]
+ print(' last: pid=%s, rc=%s, finish:%s, state:%s' % (
+ last_run.pid or 'None',
+ last_run.return_code if last_run.return_code is not None else '',
+ time.asctime(time.localtime(last_run.stop_time)) if last_run.stop_time else 'None',
+ ProcessState._VALUES_TO_NAMES.get(last_run.state, 'Unknown')))
+ print()
+
+ matchers = map(re.compile, args or ['.*'])
+ active = [t_id for _, t_id in detector.get_task_ids(state='active')
+ if any(pattern.match(t_id) for pattern in matchers)]
+ finished = [t_id for _, t_id in detector.get_task_ids(state='finished')
+ if any(pattern.match(t_id) for pattern in matchers)]
+
+ found = False
+ if options.only is None or options.only == 'active':
+ if active:
+ print('Active tasks:')
+ found = True
+ for task_id in active:
+ format_task(task_id)
+ print()
+
+ if options.only is None or options.only == 'finished':
+ if finished:
+ print('Finished tasks:')
+ found = True
+ for task_id in finished:
+ format_task(task_id)
+ print()
+
+ if not found:
+ print('No tasks found in root [%s]' % options.root)
+ sys.exit(1)
+
+
+
+@app.command
+@app.command_option("--stderr", default=False, dest='use_stderr', action='store_true',
+ help="Tail stderr instead of stdout")
+def tail(args, options):
+ """Tail the logs of a task process.
+
+ Usage: thermos tail task_name [process_name]
+ """
+ if len(args) == 0:
+ app.error('Expected a task to tail, got nothing!')
+ if len(args) not in (1, 2):
+ app.error('Expected at most two arguments (task and optional process), got %d' % len(args))
+
+ task_id = args[0]
+ detector = TaskDetector(root=options.root)
+ checkpoint = CheckpointDispatcher.from_file(detector.get_checkpoint(task_id))
+ log_dir = checkpoint.header.log_dir
+ process_runs = [(process, run) for (process, run) in detector.get_process_runs(task_id, log_dir)]
+ if len(args) == 2:
+ process_runs = [(process, run) for (process, run) in process_runs if process == args[1]]
+
+ if len(process_runs) == 0:
+ print('ERROR: No processes found.', file=sys.stderr)
+ sys.exit(1)
+
+ processes = set([process for process, _ in process_runs])
+ if len(processes) != 1:
+ print('ERROR: More than one process matches query.', file=sys.stderr)
+ sys.exit(1)
+
+ process = processes.pop()
+ run = max([run for _, run in process_runs])
+
+ logdir = TaskPath(root=options.root, task_id=args[0], process=process,
+ run=run, log_dir=log_dir).getpath('process_logdir')
+ logfile = os.path.join(logdir, 'stderr' if options.use_stderr else 'stdout')
+
+ monitor = TaskMonitor(TaskPath(root = options.root), args[0])
+ def log_is_active():
+ active_processes = monitor.get_active_processes()
+ for process_status, process_run in active_processes:
+ if process_status.process == process and process_run == run:
+ return True
+ return False
+
+ if not log_is_active():
+ print('Tail of terminal log %s' % logfile)
+ for line in tail_closed(logfile):
+ print(line.rstrip())
+ return
+
+ now = time.time()
+ next_check = now + 5.0
+ print('Tail of active log %s' % logfile)
+ for line in tail_f(logfile, include_last=True, forever=False):
+ print(line.rstrip())
+ if time.time() > next_check:
+ if not log_is_active():
+ break
+ else:
+ next_check = time.time() + 5.0
+
+
+@app.command
+def help(args, options):
+ """Get help about a specific command.
+ """
+ if len(args) == 0:
+ app.help()
+ for (command, doc) in app.get_commands_and_docstrings():
+ if args[0] == command:
+ print('command %s:' % command)
+ print(doc)
+ app.quit(0)
+ print('unknown command: %s' % args[0], file=sys.stderr)
+
+
+
+
+def generate_usage():
+ usage = """
+thermos
+
+commands:
+"""
+
+ for (command, doc) in app.get_commands_and_docstrings():
+ usage += ' ' + '%-10s' % command + '\t' + doc.split('\n')[0].strip() + '\n'
+ app.set_usage(usage)
+
+
+LogOptions.set_disk_log_level('NONE')
+LogOptions.set_stdout_log_level('INFO')
+generate_usage()
+
+
+proxy_main = app.main
+
+
+proxy_main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/bin/thermos_ckpt.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/bin/thermos_ckpt.py b/src/main/python/apache/thermos/bin/thermos_ckpt.py
new file mode 100755
index 0000000..36fd419
--- /dev/null
+++ b/src/main/python/apache/thermos/bin/thermos_ckpt.py
@@ -0,0 +1,55 @@
+import os
+import sys
+import pprint
+import time
+
+from gen.twitter.thermos.ttypes import RunnerState, RunnerCkpt, TaskState
+
+from twitter.common import app
+from twitter.common.recordio import RecordIO, ThriftRecordReader
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+
+app.add_option("--checkpoint", dest = "ckpt", metavar = "CKPT",
+ help = "read checkpoint from CKPT")
+app.add_option("--assemble", dest = "assemble", metavar = "CKPT", default=True,
+ help = "read checkpoint from CKPT")
+
+def main(args):
+ values = app.get_options()
+
+ if len(args) > 0:
+ print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % (" ".join(args))
+ app.help()
+ sys.exit(1)
+
+ if not values.ckpt:
+ print >> sys.stderr, "ERROR: must supply --checkpoint"
+ app.help()
+ sys.exit(1)
+
+ fp = file(values.ckpt, "r")
+ rr = ThriftRecordReader(fp, RunnerCkpt)
+ wrs = RunnerState(processes = {})
+ dispatcher = CheckpointDispatcher()
+ try:
+ for wts in rr:
+ print 'Recovering: ', wts
+ if values.assemble is True:
+ dispatcher.dispatch(wrs, wts)
+ except RecordIO.Error as err:
+ print 'Error recovering checkpoint stream: %s' % err
+ return
+ print '\n\n\n'
+ if values.assemble:
+ print 'Recovered Task Header'
+ pprint.pprint(wrs.header, indent=4)
+
+ print '\nRecovered Task States'
+ for task_status in wrs.statuses:
+ print ' %s [pid: %d] => %s' % (time.asctime(time.localtime(task_status.timestamp_ms/1000.0)),
+ task_status.runner_pid, TaskState._VALUES_TO_NAMES[task_status.state])
+
+ print '\nRecovered Processes'
+ pprint.pprint(wrs.processes, indent=4)
+
+app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/common/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/BUILD b/src/main/python/apache/thermos/common/BUILD
new file mode 100644
index 0000000..d97c771
--- /dev/null
+++ b/src/main/python/apache/thermos/common/BUILD
@@ -0,0 +1,45 @@
+import os
+
+python_library(
+ name = 'ckpt',
+ sources = ['ckpt.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift')
+ ]
+)
+
+python_library(
+ name = 'path',
+ sources = ['path.py'],
+)
+
+python_library(
+ name = 'planner',
+ sources = ['planner.py'],
+)
+
+python_library(
+ name = 'options',
+ sources = ['options.py'],
+ dependencies = [
+ pants('src/main/python/twitter/thermos:pystachio'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'common',
+ dependencies = [
+ pants(':ckpt'),
+ pants(':options'),
+ pants(':path'),
+ pants(':planner'),
+ ],
+ provides = setup_py(
+ name = 'twitter.thermos.common',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ description = 'Thermos common libraries.',
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/common/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/__init__.py b/src/main/python/apache/thermos/common/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/common/ckpt.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/ckpt.py b/src/main/python/apache/thermos/common/ckpt.py
new file mode 100644
index 0000000..9c5a949
--- /dev/null
+++ b/src/main/python/apache/thermos/common/ckpt.py
@@ -0,0 +1,375 @@
+"""Read checkpoint streams for the Thermos runner, and dispatch events on state transitions
+
+This module contains the CheckpointDispatcher, which reconstructs checkpoint streams containing the
+state of the Thermos runner and its constituent processes.
+
+It also defines several Handler interfaces to define behaviour on transitions in the Process and
+Task state machines.
+
+"""
+
+import os
+
+from twitter.common import log
+from twitter.common.recordio import RecordIO, ThriftRecordReader
+from gen.twitter.thermos.ttypes import (
+ ProcessState,
+ ProcessStatus,
+ RunnerCkpt,
+ RunnerState,
+ TaskState,
+)
+
+
+class UniversalStateHandler(object):
+ """
+ Generic interface for a handler to be called on any process/state transition, and at task
+ initialization
+ """
+ def on_process_transition(self, state, process_update):
+ pass
+
+ def on_task_transition(self, state, task_update):
+ pass
+
+ def on_initialization(self, header):
+ pass
+
+
+class ProcessStateHandler(object):
+ """
+ Interface for handlers for the Process state machine, called on process transitions
+
+ () - starting state, [] - terminal state
+
+ [FAILED]
+ ^
+ |
+ (WAITING) ----> FORKED ----> RUNNING -----> [KILLED]
+ | | |
+ v | `---> [SUCCESS]
+ [LOST] <------'
+ """
+ def on_waiting(self, process_update):
+ pass
+
+ def on_forked(self, process_update):
+ pass
+
+ def on_running(self, process_update):
+ pass
+
+ def on_success(self, process_update):
+ pass
+
+ def on_failed(self, process_update):
+ pass
+
+ def on_lost(self, process_update):
+ pass
+
+ def on_killed(self, process_update):
+ pass
+
+
+class TaskStateHandler(object):
+ """
+ Interface for handlers for the Task state machine, called on task transitions
+
+ () - starting state, [] - terminal state
+
+ .--------------------------------------------+----.
+ | | |
+ | .----------> [SUCCESS] | |
+ | | | |
+ | | .--------> [FAILED] | |
+ | | | | |
+ (ACTIVE) FINALIZING ---> [KILLED] <---' |
+ | ^ | .------^ |
+ | | | | |
+ `---> CLEANING ---' `----)--> [LOST] <---------'
+ | | | ^
+ | `----------------' |
+ `-------------------------'
+
+ ACTIVE -> KILLED/LOST only happens under garbage collection situations.
+ Ordinary task preemption/kill still goes through CLEANING/FINALIZING before
+ reaching a terminal state.
+ """
+
+ def on_active(self, task_update):
+ pass
+
+ def on_cleaning(self, task_update):
+ pass
+
+ def on_finalizing(self, task_update):
+ pass
+
+ def on_success(self, task_update):
+ pass
+
+ def on_failed(self, task_update):
+ pass
+
+ def on_killed(self, task_update):
+ pass
+
+ def on_lost(self, task_update):
+ pass
+
+
+def assert_nonempty(state, fields):
+ for field in fields:
+ assert getattr(state, field, None) is not None, "Missing field %s from %s!" % (field, state)
+
+
+def copy_fields(state, state_update, fields):
+ assert_nonempty(state_update, fields)
+ for field in fields:
+ setattr(state, field, getattr(state_update, field))
+
+
+class CheckpointDispatcher(object):
+ """
+ The reconstruction/dispatching mechanism for logic triggered on task/process state transitions.
+
+ Most applications should build an event-loop around the CheckpointDispatcher.
+ """
+
+ class Error(Exception): pass
+ class ErrorRecoveringState(Error): pass
+ class InvalidSequenceNumber(Error): pass
+ class InvalidHandler(Error): pass
+
+ @classmethod
+ def iter_updates(cls, filename):
+ try:
+ with open(filename) as fp:
+ rr = ThriftRecordReader(fp, RunnerCkpt)
+ for update in rr:
+ yield update
+ except (IOError, OSError, RecordIO.Error) as err:
+ raise cls.ErrorRecoveringState(err)
+
+ @classmethod
+ def iter_statuses(cls, filename):
+ for update in cls.iter_updates(filename):
+ if update.task_status:
+ yield update.task_status
+
+ @classmethod
+ def from_file(cls, filename, truncate=False):
+ """Reconstruct a RunnerState from a checkpoint stream contained in a file
+
+ Returns a hydrated RunnerState, or None on any failures.
+ """
+ state = RunnerState(processes={})
+ builder = cls()
+ try:
+ for update in cls.iter_updates(filename):
+ builder.dispatch(state, update, truncate=truncate)
+ return state
+ except cls.Error as e:
+ log.error('Failed to recover from %s: %s' % (filename, e))
+
+ def __init__(self):
+ self._task_handlers = []
+ self._process_handlers = []
+ self._universal_handlers = []
+
+ def register_handler(self, handler):
+ HANDLER_MAP = {
+ TaskStateHandler: self._task_handlers,
+ ProcessStateHandler: self._process_handlers,
+ UniversalStateHandler: self._universal_handlers
+ }
+
+ for handler_type, handler_list in HANDLER_MAP.items():
+ if isinstance(handler, handler_type):
+ handler_list.append(handler)
+ break
+ else:
+ raise self.InvalidHandler("Unknown handler type %s" % type(handler))
+
+ def _run_process_dispatch(self, state, process_update):
+ for handler in self._universal_handlers:
+ handler.on_process_transition(state, process_update)
+ for handler in self._process_handlers:
+ handler_function = 'on_' + ProcessState._VALUES_TO_NAMES[state].lower()
+ getattr(handler, handler_function)(process_update)
+
+ def _run_task_dispatch(self, state, task_update):
+ for handler in self._universal_handlers:
+ handler.on_task_transition(state, task_update)
+ for handler in self._task_handlers:
+ handler_function = 'on_' + TaskState._VALUES_TO_NAMES[state].lower()
+ getattr(handler, handler_function)(task_update)
+
+ def _run_header_dispatch(self, header):
+ for handler in self._universal_handlers:
+ handler.on_initialization(header)
+
+ @staticmethod
+ def is_terminal(process_state_update):
+ TERMINAL_STATES = [
+ ProcessState.SUCCESS,
+ ProcessState.FAILED,
+ ProcessState.KILLED,
+ ProcessState.LOST]
+ return process_state_update.state in TERMINAL_STATES
+
+ @classmethod
+ def _update_process_state(cls, process_state, process_state_update):
+ """
+ Apply process_state_update against process_state.
+ Raises ErrorRecoveringState on failure.
+ """
+ def assert_process_state_in(*expected_states):
+ if process_state.state not in expected_states:
+ raise cls.ErrorRecoveringState(
+ 'Detected invalid state transition %s => %s' % (
+ ProcessState._VALUES_TO_NAMES.get(process_state.state),
+ ProcessState._VALUES_TO_NAMES.get(process_state_update.state)))
+
+ # CREATION => WAITING
+ if process_state_update.state == ProcessState.WAITING:
+ assert_process_state_in(None)
+ required_fields = ['seq', 'state', 'process']
+ copy_fields(process_state, process_state_update, required_fields)
+
+ # WAITING => FORKED
+ elif process_state_update.state == ProcessState.FORKED:
+ assert_process_state_in(ProcessState.WAITING)
+ required_fields = ['seq', 'state', 'fork_time', 'coordinator_pid']
+ copy_fields(process_state, process_state_update, required_fields)
+
+ # FORKED => RUNNING
+ elif process_state_update.state == ProcessState.RUNNING:
+ assert_process_state_in(ProcessState.FORKED)
+ required_fields = ['seq', 'state', 'start_time', 'pid']
+ copy_fields(process_state, process_state_update, required_fields)
+
+ # RUNNING => SUCCESS
+ elif process_state_update.state == ProcessState.SUCCESS:
+ assert_process_state_in(ProcessState.RUNNING)
+ required_fields = ['seq', 'state', 'stop_time', 'return_code']
+ copy_fields(process_state, process_state_update, required_fields)
+
+ # RUNNING => FAILED
+ elif process_state_update.state == ProcessState.FAILED:
+ assert_process_state_in(ProcessState.RUNNING)
+ required_fields = ['seq', 'state', 'stop_time', 'return_code']
+ copy_fields(process_state, process_state_update, required_fields)
+
+ # {FORKED, RUNNING} => KILLED
+ elif process_state_update.state == ProcessState.KILLED:
+ assert_process_state_in(ProcessState.FORKED, ProcessState.RUNNING)
+ required_fields = ['seq', 'state', 'stop_time', 'return_code']
+ copy_fields(process_state, process_state_update, required_fields)
+
+ # {FORKED, RUNNING} => LOST
+ elif process_state_update.state == ProcessState.LOST:
+ assert_process_state_in(ProcessState.FORKED, ProcessState.RUNNING)
+ required_fields = ['seq', 'state']
+ copy_fields(process_state, process_state_update, required_fields)
+
+ else:
+ raise cls.ErrorRecoveringState(
+ "Unknown state = %s" % process_state_update.state)
+
+ def would_update(self, state, runner_ckpt):
+ """
+ Provided a ProcessStatus, would this perform a transition and update state?
+ """
+ process_update = runner_ckpt.process_status
+ if process_update is None:
+ return False
+ process = process_update.process
+ if process not in state.processes: # never seen before
+ return True
+ else:
+ # if this sequence number is ahead of the current high water mark, it would
+ # produce a transition
+ return state.processes[process][-1].seq < process_update.seq
+
+ def dispatch(self, state, runner_ckpt, recovery=False, truncate=False):
+ """
+ Given a RunnerState and a RunnerCkpt to apply to it, determine the appropriate action and
+ dispatch to the appropriate handlers.
+
+ state = RunnerState to be updated
+ runner_ckpt = RunnerCkpt update to apply
+ recovery = if true, enable recovery mode (accept out-of-order sequence updates)
+ truncate = if true, store only the latest task/process states, instead of
+ history for all runs.
+
+ Raises ErrorRecoveringState on failure.
+ """
+ # case 1: runner_header
+ # -> Initialization of the task stream.
+ if runner_ckpt.runner_header is not None:
+ if state.header is not None:
+ raise self.ErrorRecoveringState(
+ "Attempting to rebind task with different parameters!")
+ else:
+ log.debug('Initializing TaskRunner header to %s' % runner_ckpt.runner_header)
+ state.header = runner_ckpt.runner_header
+ self._run_header_dispatch(runner_ckpt.runner_header)
+ return
+
+ # case 2: task_status
+ # -> State transition on the task (ACTIVE, FAILED, SUCCESS, LOST)
+ if runner_ckpt.task_status is not None:
+ if state.statuses is None:
+ state.statuses = []
+ old_state = None
+ else:
+ old_state = state.statuses[-1].state
+ if not truncate:
+ state.statuses.append(runner_ckpt.task_status)
+ else:
+ state.statuses = [runner_ckpt.task_status]
+ new_state = runner_ckpt.task_status.state
+ log.debug('Flipping task state from %s to %s' % (
+ TaskState._VALUES_TO_NAMES.get(old_state, '(undefined)'),
+ TaskState._VALUES_TO_NAMES.get(new_state, '(undefined)')))
+ self._run_task_dispatch(new_state, runner_ckpt.task_status)
+ return
+
+ # case 3: process_status
+ # -> State transition on a process itself
+ # (WAITING, FORKED, RUNNING, SUCCESS, KILLED, FAILED, LOST)
+ if runner_ckpt.process_status is not None:
+ process_update = runner_ckpt.process_status
+ name = process_update.process
+ current_run = state.processes[name][-1] if name in state.processes else None
+ if current_run and process_update.seq != current_run.seq + 1:
+ if recovery:
+ log.debug('Skipping replayed out-of-order update: %s' % process_update)
+ return
+ else:
+ raise self.InvalidSequenceNumber(
+ "Out of order sequence number! %s => %s" % (current_run, process_update))
+
+ # One special case for WAITING: Initialize a new target ProcessState.
+ if process_update.state == ProcessState.WAITING:
+ assert current_run is None or self.is_terminal(current_run)
+ if name not in state.processes:
+ state.processes[name] = [ProcessStatus(seq=-1)]
+ else:
+ if not truncate:
+ state.processes[name].append(ProcessStatus(seq=current_run.seq))
+ else:
+ state.processes[name] = [ProcessStatus(seq=current_run.seq)]
+
+ # Run the process state machine.
+ log.debug('Running state machine for process=%s/seq=%s' % (name, process_update.seq))
+ if not state.processes or name not in state.processes:
+ raise self.ErrorRecoveringState("Encountered potentially out of order "
+ "process update. Are you sure this is a full checkpoint stream?")
+ self._update_process_state(state.processes[name][-1], process_update)
+ self._run_process_dispatch(process_update.state, process_update)
+ return
+
+ raise self.ErrorRecoveringState("Empty RunnerCkpt encountered!")
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/common/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/common/options.py b/src/main/python/apache/thermos/common/options.py
new file mode 100644
index 0000000..9948d03
--- /dev/null
+++ b/src/main/python/apache/thermos/common/options.py
@@ -0,0 +1,34 @@
+from pystachio import Ref
+
+class ParseError(Exception):
+ pass
+
+def add_port_to(option_name):
+ def add_port_callback(option, opt, value, parser):
+ if not getattr(parser.values, option_name, None):
+ setattr(parser.values, option_name, {})
+ try:
+ name, port = value.split(':')
+ except (ValueError, TypeError):
+ raise ParseError('Invalid value for %s: %s should be of form NAME:PORT' % (
+ opt, value))
+ try:
+ port = int(port)
+ except ValueError:
+ raise ParseError('Port does not appear to be an integer: %s' % port)
+ getattr(parser.values, option_name)[name] = port
+ return add_port_callback
+
+def add_binding_to(option_name):
+ def add_binding_callback(option, opt, value, parser):
+ if not getattr(parser.values, option_name, None):
+ setattr(parser.values, option_name, [])
+ if len(value.split('=')) != 2:
+ raise ParseError('Binding must be of the form NAME=VALUE')
+ name, value = value.split('=')
+ try:
+ ref = Ref.from_address(name)
+ except Ref.InvalidRefError as e:
+ raise ParseError('Could not parse ref %s: %s' % (name, e))
+ getattr(parser.values, option_name).append({ref: value})
+ return add_binding_callback
[14/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/cli/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/cli/BUILD b/src/main/python/twitter/aurora/client/cli/BUILD
deleted file mode 100644
index 0a44a21..0000000
--- a/src/main/python/twitter/aurora/client/cli/BUILD
+++ /dev/null
@@ -1,26 +0,0 @@
-
-python_binary(
- name='client',
- entry_point = 'twitter.aurora.cli:main',
- dependencies = [ pants(':cli') ],
- )
-
-python_library(
- name='cli',
- sources = [ '__init__.py', 'context.py', 'jobs.py', 'options.py' ],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/python'),
- pants('src/main/python/twitter/aurora/client/api:command_runner'),
- pants('src/main/python/twitter/aurora/client/api:disambiguator'),
- pants('src/main/python/twitter/aurora/client/api:job_monitor'),
- pants('src/main/python/twitter/aurora/client/api:updater'),
- pants('src/main/python/twitter/aurora/client/hooks'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/python/twitter/aurora/client:config'),
- pants('src/main/python/twitter/aurora/client:factory'),
- pants('src/main/python/twitter/aurora/client:options'),
- pants('src/main/python/twitter/aurora/common'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- pants('src/main/python/twitter/aurora:argparse')
- ]
- )
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/cli/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/cli/__init__.py b/src/main/python/twitter/aurora/client/cli/__init__.py
deleted file mode 100644
index 2c08cf9..0000000
--- a/src/main/python/twitter/aurora/client/cli/__init__.py
+++ /dev/null
@@ -1,213 +0,0 @@
-'''Command-line tooling infrastructure for aurora client v2.
-
-This provides a framework for a noun/verb command-line application. The application is structured
-around a collection of basic objects (nouns) that can be manipulated by the command line, where
-each type of object provides a collection of operations (verbs). Every command invocation
-consists of the name of the noun, followed by one of the verbs for that noun, followed by other
-arguments needed by the verb.
-
-For example:
-- To create a job, the noun is "job", the verb is "create":
- $ aurora job create us-west/www/prod/server server.aurora
-
-- To find out the resource quota for a specific user, the noun is "user" and the verb is
- "get_quota":
- $ aurora user get_quota mchucarroll
-'''
-
-from __future__ import print_function
-
-from abc import abstractmethod
-import argparse
-import sys
-
-
-# Constants for standard return codes.
-EXIT_OK = 0
-EXIT_INVALID_CONFIGURATION = 3
-EXIT_COMMAND_FAILURE = 4
-EXIT_INVALID_COMMAND = 5
-EXIT_INVALID_PARAMETER = 6
-EXIT_NETWORK_ERROR = 7
-EXIT_PERMISSION_VIOLATION = 8
-EXIT_TIMEOUT = 9
-EXIT_UNKNOWN_ERROR = 20
-
-
-class Context(object):
- class Error(Exception): pass
-
- class ArgumentException(Error): pass
-
- class CommandError(Error):
- def __init__(self, code, msg):
- super(Context.CommandError, self).__init__(msg)
- self.msg = msg
- self.code = code
-
- def set_options(self, options):
- """Add the options object to a context.
- This is separated from the constructor to make patching tests easier.
- """
- self.options = options
-
-
-class CommandOption(object):
- """A lightweight encapsulation of an argparse option specification, which can be used to
- define options that can be reused by multiple commands."""
-
- def __init__(self, *args, **kwargs):
- self.args = args
- self.kwargs = kwargs
-
- def add_to_parser(self, parser):
- parser.add_argument(*self.args, **self.kwargs)
-
-
-class AuroraCommand(object):
- def setup_options_parser(self, argparser):
- """Set up command line options parsing for this command.
- This is a thin veneer over the standard python argparse system.
- :param argparser: the argument parser where this command can add its arguments.
- """
- pass
-
- def add_option(self, argparser, option):
- """Add a predefined argument encapsulated an a CommandOption to an argument parser."""
- if not isinstance(option, CommandOption):
- raise TypeError('Command option object must be an instance of CommandOption')
- option.add_to_parser(argparser)
-
- @property
- def help(self):
- """The help message for a command that will be used in the argparse help message"""
-
- @property
- def name(self):
- """The command name"""
-
-
-class CommandLine(object):
- """The top-level object implementing a command-line application."""
-
- def __init__(self):
- self.nouns = {}
- self.parser = None
-
- def register_noun(self, noun):
- """Add a noun to the application"""
- if not isinstance(noun, Noun):
- raise TypeError('register_noun requires a Noun argument')
- self.nouns[noun.name] = noun
-
- def setup_options_parser(self):
- """ Build the options parsing for the application."""
- self.parser = argparse.ArgumentParser()
- subparser = self.parser.add_subparsers(dest='noun')
- for (name, noun) in self.nouns.items():
- noun_parser = subparser.add_parser(name, help=noun.help)
- noun.internal_setup_options_parser(noun_parser)
-
- def register_nouns(self):
- """This method should overridden by applications to register the collection of nouns
- that they can manipulate.
- """
- pass
-
- def execute(self, args):
- """Execute a command.
- :param args: the command-line arguments for the command. This only includes arguments
- that should be parsed by the application; it does not include sys.argv[0].
- """
- self.register_nouns()
- self.setup_options_parser()
- options = self.parser.parse_args(args)
- if options.noun not in self.nouns:
- raise ValueError('Unknown command: %s' % options.noun)
- noun = self.nouns[options.noun]
- context = noun.create_context()
- context.set_options(options)
- try:
- return noun.execute(context)
- except Context.CommandError as c:
- print('Error executing command: %s' % c.msg, file=sys.stderr)
- return c.code
-
-
-class Noun(AuroraCommand):
- """A type of object manipulated by a command line application"""
- class InvalidVerbException(Exception): pass
-
- def __init__(self):
- super(Noun, self).__init__()
- self.verbs = {}
-
- def register_verb(self, verb):
- """Add an operation supported for this noun."""
- if not isinstance(verb, Verb):
- raise TypeError('register_verb requires a Verb argument')
- self.verbs[verb.name] = verb
- verb._register(self)
-
- def internal_setup_options_parser(self, argparser):
- """Internal driver for the options processing framework."""
- self.setup_options_parser(argparser)
- subparser = argparser.add_subparsers(dest='verb')
- for (name, verb) in self.verbs.items():
- vparser = subparser.add_parser(name, help=verb.help)
- verb.setup_options_parser(vparser)
-
- @classmethod
- def create_context(cls):
- """Commands access state through a context object. The noun specifies what kind
- of context should be created for this noun's required state.
- """
- pass
-
- @abstractmethod
- def setup_options_parser(self, argparser):
- pass
-
- def execute(self, context):
- if context.options.verb not in self.verbs:
- raise self.InvalidVerbException('Noun %s does not have a verb %s' %
- (self.name, context.options.verb))
- self.verbs[context.options.verb].execute(context)
-
-
-class Verb(AuroraCommand):
- """An operation for a noun. Most application logic will live in verbs."""
-
- def _register(self, noun):
- """Create a link from a verb to its noun."""
- self.noun = noun
-
- @abstractmethod
- def setup_options_parser(self, argparser):
- pass
-
- def execute(self, context):
- pass
-
-
-class AuroraCommandLine(CommandLine):
- """ An example implementation of a command line application using this framework.
- This should probably eventually get moved in to its own source file.
- """
-
- @classmethod
- def get_description(cls):
- return 'Aurora client command line'
-
- def register_nouns(self):
- from .jobs import Job
- self.register_noun(Job())
-
-
-def main():
- cmd = AuroraCommandLine()
- cmd.execute(sys.argv[1:])
-
-
-if __name__ == '__main__':
- main(sys.argv)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/cli/context.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/cli/context.py b/src/main/python/twitter/aurora/client/cli/context.py
deleted file mode 100644
index 2ae92ec..0000000
--- a/src/main/python/twitter/aurora/client/cli/context.py
+++ /dev/null
@@ -1,49 +0,0 @@
-
-from twitter.aurora.client.base import synthesize_url
-from twitter.aurora.client.cli import Context, EXIT_NETWORK_ERROR
-from twitter.aurora.client.config import get_config
-from twitter.aurora.client.factory import make_client
-from twitter.common import log
-
-from gen.twitter.aurora.ttypes import ResponseCode
-
-
-class AuroraCommandContext(Context):
- """A context object used by Aurora commands to manage command processing state
- and common operations.
- """
-
- def get_api(self, cluster):
- """Creates an API object for a specified cluster"""
- return make_client(cluster)
-
- def get_job_config(self, job_key, config_file):
- """Loads a job configuration from a config file"""
- jobname = job_key.name
- return get_config(
- jobname,
- config_file,
- self.options.json,
- self.options.bindings,
- select_cluster=job_key.cluster,
- select_role=job_key.role,
- select_env=job_key.env)
-
- def open_page(self, url):
- import webbrowser
- webbrowser.open_new_tab(url)
-
- def open_job_page(self, api, config):
- self.open_page(synthesize_url(api.scheduler.scheduler().url, config.role(),
- config.environment(), config.name()))
-
- def handle_open(self, api):
- if self.options.open_browser:
- self.open_page(synthesize_url(api.scheduler.scheduler().url,
- self.options.jobspec.role, self.options.jobspec.env, self.options.jobspec.name))
-
- def check_and_log_response(self, resp):
- log.info('Response from scheduler: %s (message: %s)'
- % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
- if resp.responseCode != ResponseCode.OK:
- raise self.CommandError(EXIT_NETWORK_ERROR, resp.message)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/cli/jobs.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/cli/jobs.py b/src/main/python/twitter/aurora/client/cli/jobs.py
deleted file mode 100644
index e66f181..0000000
--- a/src/main/python/twitter/aurora/client/cli/jobs.py
+++ /dev/null
@@ -1,115 +0,0 @@
-from twitter.aurora.client.api.job_monitor import JobMonitor
-from twitter.aurora.client.cli import (
- EXIT_INVALID_CONFIGURATION,
- Noun,
- Verb
-)
-from twitter.aurora.client.cli.context import AuroraCommandContext
-from twitter.aurora.client.cli.options import (
- BIND_OPTION,
- BROWSER_OPTION,
- CONFIG_OPTION,
- JOBSPEC_OPTION,
- JSON_OPTION
-)
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-
-from pystachio.config import Config
-
-
-def parse_instances(instances):
- """Parse lists of instances or instance ranges into a set().
-
- Examples:
- 0-2
- 0,1-3,5
- 1,3,5
- """
- if instances is None or instances == '':
- return None
- result = set()
- for part in instances.split(','):
- x = part.split('-')
- result.update(range(int(x[0]), int(x[-1]) + 1))
- return sorted(result)
-
-
-class CreateJobCommand(Verb):
- @property
- def name(self):
- return 'create'
-
- @property
- def help(self):
- return 'Create a job using aurora'
-
- CREATE_STATES = ('PENDING', 'RUNNING', 'FINISHED')
-
- def setup_options_parser(self, parser):
- self.add_option(parser, BIND_OPTION)
- self.add_option(parser, BROWSER_OPTION)
- self.add_option(parser, JSON_OPTION)
- parser.add_argument('--wait_until', choices=self.CREATE_STATES,
- default='PENDING',
- help=('Block the client until all the tasks have transitioned into the requested state. '
- 'Default: PENDING'))
- self.add_option(parser, JOBSPEC_OPTION)
- self.add_option(parser, CONFIG_OPTION)
-
- def execute(self, context):
- try:
- config = context.get_job_config(context.options.jobspec, context.options.config_file)
- except Config.InvalidConfigError as e:
- raise context.CommandError(EXIT_INVALID_CONFIGURATION,
- 'Error loading job configuration: %s' % e)
- api = context.get_api(config.cluster())
- monitor = JobMonitor(api, config.role(), config.environment(), config.name())
- resp = api.create_job(config)
- context.check_and_log_response(resp)
- if context.options.open_browser:
- context.open_job_page(api, config)
- if context.options.wait_until == 'RUNNING':
- monitor.wait_until(monitor.running_or_finished)
- elif context.options.wait_until == 'FINISHED':
- monitor.wait_until(monitor.terminal)
-
-
-class KillJobCommand(Verb):
- @property
- def name(self):
- return 'kill'
-
- def setup_options_parser(self, parser):
- self.add_option(parser, BROWSER_OPTION)
- parser.add_argument('--instances', type=parse_instances, dest='instances', default=None,
- help='A list of instance ids to act on. Can either be a comma-separated list (e.g. 0,1,2) '
- 'or a range (e.g. 0-2) or any combination of the two (e.g. 0-2,5,7-9). If not set, '
- 'all instances will be acted on.')
- parser.add_argument('--config', type=str, default=None, dest='config',
- help='Config file for the job, possibly containing hooks')
- self.add_option(parser, JOBSPEC_OPTION)
-
- def execute(self, context):
- api = context.get_api(context.options.jobspec.cluster)
- resp = api.kill_job(context.options.jobspec, context.options.instances)
- context.check_and_log_response(resp)
- context.handle_open(api)
-
-
-class Job(Noun):
- @property
- def name(self):
- return 'job'
-
- @property
- def help(self):
- return "Work with an aurora job"
-
- @classmethod
- def create_context(cls):
- return AuroraCommandContext()
-
- def __init__(self):
- super(Job, self).__init__()
- self.register_verb(CreateJobCommand())
- self.register_verb(KillJobCommand())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/cli/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/cli/options.py b/src/main/python/twitter/aurora/client/cli/options.py
deleted file mode 100644
index 64a58f8..0000000
--- a/src/main/python/twitter/aurora/client/cli/options.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from twitter.aurora.client.cli import CommandOption
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-
-
-BIND_OPTION = CommandOption('--bind', type=str, default=[], dest='bindings',
- action='append',
- help='Bind a thermos mustache variable name to a value. '
- 'Multiple flags may be used to specify multiple values.')
-
-
-BROWSER_OPTION = CommandOption('--open-browser', default=False, dest='open_browser',
- action='store_true',
- help='open browser to view job page after job is created')
-
-
-CONFIG_OPTION = CommandOption('config_file', type='str', dest='config_file',
- help='pathname of the aurora configuration file contain the job specification')
-
-
-JOBSPEC_OPTION = CommandOption('jobspec', type=AuroraJobKey.from_path,
- help='Fully specified job key, in CLUSTER/ROLE/ENV/NAME format')
-
-
-JSON_OPTION = CommandOption('--json', default=False, dest='json', action='store_true',
- help='Read job configuration in json format')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/commands/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/commands/BUILD b/src/main/python/twitter/aurora/client/commands/BUILD
deleted file mode 100644
index 3bcb6fd..0000000
--- a/src/main/python/twitter/aurora/client/commands/BUILD
+++ /dev/null
@@ -1,81 +0,0 @@
-python_library(
- name = 'all',
- dependencies = [
- pants(':core'),
- pants(':help'),
- pants(':run'),
- pants(':ssh'),
- ]
-)
-
-python_library(
- name = 'admin',
- sources = ['admin.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/aurora/admin:mesos_maintenance'),
- pants('src/main/python/twitter/aurora/client/api'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/python/twitter/aurora/common:clusters'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'core',
- sources = ['core.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/python'),
- pants('src/main/python/twitter/aurora/client/api:command_runner'),
- pants('src/main/python/twitter/aurora/client/api:disambiguator'),
- pants('src/main/python/twitter/aurora/client/api:job_monitor'),
- pants('src/main/python/twitter/aurora/client/api:updater'),
- pants('src/main/python/twitter/aurora/client/hooks'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/python/twitter/aurora/client:config'),
- pants('src/main/python/twitter/aurora/client:factory'),
- pants('src/main/python/twitter/aurora/client:options'),
- pants('src/main/python/twitter/aurora/common'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'help',
- sources = ['help.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('src/main/python/twitter/aurora/client:base'),
- ]
-)
-
-python_library(
- name = 'run',
- sources = ['run.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('src/main/python/twitter/aurora/client/api:command_runner'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/python/twitter/aurora/client:options'),
- pants('src/main/python/twitter/aurora/common:aurora_job_key'),
- pants('src/main/python/twitter/aurora/common:clusters'),
- ]
-)
-
-python_library(
- name = 'ssh',
- sources = ['ssh.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('src/main/python/twitter/aurora/client/api:command_runner'),
- pants('src/main/python/twitter/aurora/client:base'),
- pants('src/main/python/twitter/aurora/client:factory'),
- pants('src/main/python/twitter/aurora/client:options'),
- pants('src/main/python/twitter/aurora/common:aurora_job_key'),
- pants('src/main/python/twitter/aurora/common:clusters'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/commands/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/commands/__init__.py b/src/main/python/twitter/aurora/client/commands/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/commands/admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/commands/admin.py b/src/main/python/twitter/aurora/client/commands/admin.py
deleted file mode 100644
index c1c9c42..0000000
--- a/src/main/python/twitter/aurora/client/commands/admin.py
+++ /dev/null
@@ -1,406 +0,0 @@
-from __future__ import print_function
-
-"""Command-line client for managing admin-only interactions with the aurora scheduler.
-"""
-
-import os
-import optparse
-import subprocess
-
-from twitter.aurora.admin.mesos_maintenance import MesosMaintenance
-from twitter.aurora.client.api import AuroraClientAPI
-from twitter.aurora.client.base import check_and_log_response, die, requires
-from twitter.aurora.common.clusters import CLUSTERS
-from twitter.common import app, log
-from twitter.common.quantity import Amount, Data
-from twitter.common.quantity.parse_simple import parse_data
-
-from gen.twitter.aurora.constants import ACTIVE_STATES, TERMINAL_STATES
-from gen.twitter.aurora.ttypes import (
- ResponseCode,
- ScheduleStatus,
- TaskQuery,
-)
-
-
-GROUPING_OPTION = optparse.Option(
- '--grouping',
- type='choice',
- choices=MesosMaintenance.GROUPING_FUNCTIONS.keys(),
- metavar='GROUPING',
- default=MesosMaintenance.DEFAULT_GROUPING,
- dest='grouping',
- help='Grouping function to use to group hosts. Options: %s. Default: %%default' % (
- ', '.join(MesosMaintenance.GROUPING_FUNCTIONS.keys())))
-
-
-def parse_hosts(options):
- if not (options.filename or options.hosts):
- die('Please specify either --filename or --hosts')
- if options.filename:
- with open(options.filename, 'r') as hosts:
- hosts = [hostname.strip() for hostname in hosts]
- elif options.hosts:
- hosts = [hostname.strip() for hostname in options.hosts.split(",")]
- if not hosts:
- die('No valid hosts found.')
- return hosts
-
-
-@app.command
-@app.command_option('--force', dest='force', default=False, action='store_true',
- help='Force expensive queries to run.')
-@app.command_option('--shards', dest='shards', default=None,
- help='Only match given shards of a job.')
-@app.command_option('--states', dest='states', default='RUNNING',
- help='Only match tasks with given state(s).')
-@app.command_option('-l', '--listformat', dest='listformat',
- default="%role%/%jobName%/%instanceId% %status%",
- help='Format string of job/task items to print out.')
-# TODO(ksweeney): Allow query by environment here.
-def query(args, options):
- """usage: query [--shards=N[,N,...]]
- [--states=State[,State,...]]
- cluster [role [job]]
-
- Query Mesos about jobs and tasks.
- """
- def _convert_fmt_string(fmtstr):
- import re
- def convert(match):
- return "%%(%s)s" % match.group(1)
- return re.sub(r'%(\w+)%', convert, fmtstr)
-
- def flatten_task(t, d={}):
- for key in t.__dict__.keys():
- val = getattr(t, key)
- try:
- val.__dict__.keys()
- except AttributeError:
- d[key] = val
- else:
- flatten_task(val, d)
-
- return d
-
- def map_values(d):
- default_value = lambda v: v
- mapping = {
- 'status': lambda v: ScheduleStatus._VALUES_TO_NAMES[v],
- }
- return dict(
- (k, mapping.get(k, default_value)(v)) for (k, v) in d.items()
- )
-
- for state in options.states.split(','):
- if state not in ScheduleStatus._NAMES_TO_VALUES:
- msg = "Unknown state '%s' specified. Valid states are:\n" % state
- msg += ','.join(ScheduleStatus._NAMES_TO_VALUES.keys())
- die(msg)
-
- # Role, Job, Instances, States, and the listformat
- if len(args) == 0:
- die('Must specify at least cluster.')
-
- cluster = args[0]
- role = args[1] if len(args) > 1 else None
- job = args[2] if len(args) > 2 else None
- instances = set(map(int, options.shards.split(','))) if options.shards else set()
-
- if options.states:
- states = set(map(ScheduleStatus._NAMES_TO_VALUES.get, options.states.split(',')))
- else:
- states = ACTIVE_STATES | TERMINAL_STATES
- listformat = _convert_fmt_string(options.listformat)
-
- # Figure out "expensive" queries here and bone if they do not have --force
- # - Does not specify role
- if role is None and not options.force:
- die('--force is required for expensive queries (no role specified)')
-
- # - Does not specify job
- if job is None and not options.force:
- die('--force is required for expensive queries (no job specified)')
-
- # - Specifies status outside of ACTIVE_STATES
- if not (states <= ACTIVE_STATES) and not options.force:
- die('--force is required for expensive queries (states outside ACTIVE states')
-
- api = AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
- query_info = api.query(api.build_query(role, job, instances=instances, statuses=states))
- tasks = query_info.result.scheduleStatusResult.tasks
- if query_info.responseCode != ResponseCode.OK:
- die('Failed to query scheduler: %s' % query_info.message)
- if tasks is None:
- return
-
- try:
- for task in tasks:
- d = flatten_task(task)
- print(listformat % map_values(d))
- except KeyError:
- msg = "Unknown key in format string. Valid keys are:\n"
- msg += ','.join(d.keys())
- die(msg)
-
-
-@app.command
-@requires.exactly('cluster', 'role', 'cpu', 'ramMb', 'diskMb')
-def set_quota(cluster, role, cpu_str, ram_mb_str, disk_mb_str):
- """usage: set_quota cluster role cpu ramMb diskMb
-
- Alters the amount of production quota allocated to a user.
- """
- try:
- cpu = float(cpu_str)
- ram_mb = int(ram_mb_str)
- disk_mb = int(disk_mb_str)
- except ValueError:
- log.error('Invalid value')
-
- options = app.get_options()
- resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).set_quota(role, cpu, ram_mb, disk_mb)
- check_and_log_response(resp)
-
-
-@app.command
-@app.command_option('--filename', dest='filename', default=None,
- help='Name of the file with hostnames')
-@app.command_option('--hosts', dest='hosts', default=None,
- help='Comma separated list of hosts')
-@requires.exactly('cluster')
-def start_maintenance_hosts(cluster):
- """usage: start_maintenance_hosts cluster [--filename=filename]
- [--hosts=hosts]
- """
- options = app.get_options()
- MesosMaintenance(CLUSTERS[cluster], options.verbosity).start_maintenance(parse_hosts(options))
-
-
-@app.command
-@app.command_option('--filename', dest='filename', default=None,
- help='Name of the file with hostnames')
-@app.command_option('--hosts', dest='hosts', default=None,
- help='Comma separated list of hosts')
-@requires.exactly('cluster')
-def end_maintenance_hosts(cluster):
- """usage: end_maintenance_hosts cluster [--filename=filename]
- [--hosts=hosts]
- """
- options = app.get_options()
- MesosMaintenance(CLUSTERS[cluster], options.verbosity).end_maintenance(parse_hosts(options))
-
-
-@app.command
-@app.command_option('--filename', dest='filename', default=None,
- help='Name of the file with hostnames')
-@app.command_option('--hosts', dest='hosts', default=None,
- help='Comma separated list of hosts')
-@app.command_option('--batch_size', dest='batch_size', default=1,
- help='Number of groups to operate on at a time.')
-@app.command_option('--post_drain_script', dest='post_drain_script', default=None,
- help='Path to a script to run for each host.')
-@app.command_option(GROUPING_OPTION)
-@requires.exactly('cluster')
-def perform_maintenance_hosts(cluster):
- """usage: perform_maintenance cluster [--filename=filename]
- [--hosts=hosts]
- [--batch_size=num]
- [--post_drain_script=path]
- [--grouping=function]
-
- Asks the scheduler to remove any running tasks from the machine and remove it
- from service temporarily, perform some action on them, then return the machines
- to service.
- """
- options = app.get_options()
- drainable_hosts = parse_hosts(options)
-
- if options.post_drain_script:
- if not os.path.exists(options.post_drain_script):
- die("No such file: %s" % options.post_drain_script)
- cmd = os.path.abspath(options.post_drain_script)
- drained_callback = lambda host: subprocess.Popen([cmd, host])
- else:
- drained_callback = None
-
- MesosMaintenance(CLUSTERS[cluster], options.verbosity).perform_maintenance(
- drainable_hosts,
- batch_size=int(options.batch_size),
- callback=drained_callback,
- grouping_function=options.grouping)
-
-
-@app.command
-@app.command_option('--filename', dest='filename', default=None,
- help='Name of the file with hostnames')
-@app.command_option('--hosts', dest='hosts', default=None,
- help='Comma separated list of hosts')
-@requires.exactly('cluster')
-def host_maintenance_status(cluster):
- """usage: host_maintenance_status cluster [--filename=filename]
- [--hosts=hosts]
-
- Check on the schedulers maintenance status for a list of hosts in the cluster.
- """
- options = app.get_options()
- checkable_hosts = parse_hosts(options)
- statuses = MesosMaintenance(CLUSTERS[cluster], options.verbosity).check_status(checkable_hosts)
- for pair in statuses:
- log.info("%s is in state: %s" % pair)
-
-
-@app.command
-@requires.exactly('cluster', 'role', 'cpu', 'ram', 'disk')
-def increase_quota(cluster, role, cpu_str, ram_str, disk_str):
- """usage: increase_quota cluster role cpu ram[unit] disk[unit]
-
- Increases the amount of production quota allocated to a user.
- """
- cpu = float(cpu_str)
- ram = parse_data(ram_str)
- disk = parse_data(disk_str)
-
- options = app.get_options()
- client = AuroraClientAPI(CLUSTERS[cluster], options.verbosity == 'verbose')
- resp = client.get_quota(role)
- quota = resp.result.getQuotaResult.quota
- log.info('Current quota for %s:\n\tCPU\t%s\n\tRAM\t%s MB\n\tDisk\t%s MB' %
- (role, quota.numCpus, quota.ramMb, quota.diskMb))
-
- new_cpu = cpu + quota.numCpus
- new_ram = ram + Amount(quota.ramMb, Data.MB)
- new_disk = disk + Amount(quota.diskMb, Data.MB)
-
- log.info('Attempting to update quota for %s to\n\tCPU\t%s\n\tRAM\t%s MB\n\tDisk\t%s MB' %
- (role, new_cpu, new_ram.as_(Data.MB), new_disk.as_(Data.MB)))
-
- resp = client.set_quota(role, new_cpu, new_ram.as_(Data.MB), new_disk.as_(Data.MB))
- check_and_log_response(resp)
-
-
-@app.command
-@requires.exactly('cluster')
-def scheduler_backup_now(cluster):
- """usage: scheduler_backup_now cluster
-
- Immediately initiates a full storage backup.
- """
- options = app.get_options()
- check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity).perform_backup())
-
-
-@app.command
-@requires.exactly('cluster')
-def scheduler_list_backups(cluster):
- """usage: scheduler_list_backups cluster
-
- Lists backups available for recovery.
- """
- options = app.get_options()
- resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).list_backups()
- check_and_log_response(resp)
- backups = resp.result.listBackupsResult.backups
- print('%s available backups:' % len(backups))
- for backup in backups:
- print(backup)
-
-
-@app.command
-@requires.exactly('cluster', 'backup_id')
-def scheduler_stage_recovery(cluster, backup_id):
- """usage: scheduler_stage_recovery cluster backup_id
-
- Stages a backup for recovery.
- """
- options = app.get_options()
- check_and_log_response(
- AuroraClientAPI(CLUSTERS[cluster], options.verbosity).stage_recovery(backup_id))
-
-
-@app.command
-@requires.exactly('cluster')
-def scheduler_print_recovery_tasks(cluster):
- """usage: scheduler_print_recovery_tasks cluster
-
- Prints all active tasks in a staged recovery.
- """
- options = app.get_options()
- resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).query_recovery(
- TaskQuery(statuses=ACTIVE_STATES))
- check_and_log_response(resp)
- log.info('Role\tJob\tShard\tStatus\tTask ID')
- for task in resp.tasks:
- assigned = task.assignedTask
- conf = assigned.task
- log.info('\t'.join((conf.owner.role,
- conf.jobName,
- str(assigned.instanceId),
- ScheduleStatus._VALUES_TO_NAMES[task.status],
- assigned.taskId)))
-
-
-@app.command
-@requires.exactly('cluster', 'task_ids')
-def scheduler_delete_recovery_tasks(cluster, task_ids):
- """usage: scheduler_delete_recovery_tasks cluster task_ids
-
- Deletes a comma-separated list of task IDs from a staged recovery.
- """
- ids = set(task_ids.split(','))
- options = app.get_options()
- check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
- .delete_recovery_tasks(TaskQuery(taskIds=ids)))
-
-
-@app.command
-@requires.exactly('cluster')
-def scheduler_commit_recovery(cluster):
- """usage: scheduler_commit_recovery cluster
-
- Commits a staged recovery.
- """
- options = app.get_options()
- check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
- .commit_recovery())
-
-
-@app.command
-@requires.exactly('cluster')
-def scheduler_unload_recovery(cluster):
- """usage: scheduler_unload_recovery cluster
-
- Unloads a staged recovery.
- """
- options = app.get_options()
- check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
- .unload_recovery())
-
-
-@app.command
-@requires.exactly('cluster')
-def scheduler_list_job_updates(cluster):
- """usage: scheduler_list_job_updates cluster
-
- Lists in-flight job updates.
- """
- options = app.get_options()
- resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).get_job_updates()
- check_and_log_response(resp)
- print('Role\tEnv\tJob')
- for update in resp.jobUpdates:
- print('%s\t%s\t%s' % (
- update.jobKey.role if update.jobKey else update.roleDeprecated,
- update.jobKey.environment if update.jobKey else None,
- update.jobKey.name if update.jobKey else update.jobDeprecated))
-
-
-@app.command
-@requires.exactly('cluster')
-def scheduler_snapshot(cluster):
- """usage: scheduler_snapshot cluster
-
- Request that the scheduler perform a storage snapshot and block until complete.
- """
- options = app.get_options()
- check_and_log_response(AuroraClientAPI(CLUSTERS['cluster'], options.verbosity).snapshot())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/commands/core.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/commands/core.py b/src/main/python/twitter/aurora/client/commands/core.py
deleted file mode 100644
index 712c7ea..0000000
--- a/src/main/python/twitter/aurora/client/commands/core.py
+++ /dev/null
@@ -1,600 +0,0 @@
-"""Command-line client for managing jobs with the Aurora scheduler.
-"""
-
-from __future__ import print_function
-
-import collections
-from datetime import datetime
-import json
-import os
-import pprint
-import subprocess
-import sys
-import time
-from tempfile import NamedTemporaryFile
-
-from twitter.common import app, log
-from twitter.common.python.pex import PexInfo
-from twitter.common.python.dirwrapper import PythonDirectoryWrapper
-
-from twitter.aurora.client.base import (
- check_and_log_response,
- deprecation_warning,
- die,
- handle_open,
- requires,
- synthesize_url)
-from twitter.aurora.client.api.disambiguator import LiveJobDisambiguator
-from twitter.aurora.client.api.job_monitor import JobMonitor
-from twitter.aurora.client.api.updater_util import UpdaterConfig
-from twitter.aurora.client.config import get_config
-from twitter.aurora.client.factory import make_client, make_client_factory
-from twitter.aurora.client.options import (
- CLUSTER_CONFIG_OPTION,
- CLUSTER_INVOKE_OPTION,
- ENV_CONFIG_OPTION,
- ENVIRONMENT_BIND_OPTION,
- FROM_JOBKEY_OPTION,
- HEALTH_CHECK_INTERVAL_SECONDS_OPTION,
- JSON_OPTION,
- OPEN_BROWSER_OPTION,
- SHARDS_OPTION,
- WAIT_UNTIL_OPTION)
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-
-from gen.twitter.aurora.constants import ACTIVE_STATES, CURRENT_API_VERSION, AURORA_EXECUTOR_NAME
-from gen.twitter.aurora.ttypes import ExecutorConfig, ResponseCode, ScheduleStatus
-
-
-def get_job_config(job_spec, config_file, options):
- try:
- job_key = AuroraJobKey.from_path(job_spec)
- select_cluster = job_key.cluster
- select_env = job_key.env
- select_role = job_key.role
- jobname = job_key.name
- except AuroraJobKey.Error:
- deprecation_warning('Please refer to your job in CLUSTER/ROLE/ENV/NAME format.')
- select_cluster = options.cluster if options.cluster else None
- select_env = options.env
- select_role = None
- jobname = job_spec
- try:
- json_option = options.json
- except AttributeError:
- json_option = False
- try:
- bindings = options.bindings
- except AttributeError:
- bindings = ()
- return get_config(
- jobname,
- config_file,
- json_option,
- bindings,
- select_cluster=select_cluster,
- select_role=select_role,
- select_env=select_env)
-
-@app.command
-def version(args):
- """usage: version
-
- Prints information about the version of the aurora client being run.
- """
- try:
- pexpath = sys.argv[0]
- pex_info = PexInfo.from_pex(PythonDirectoryWrapper.get(pexpath))
- print("Aurora client build info:")
- print("\tsha: %s" % pex_info.build_properties['sha'])
- print("\tdate: %s" % pex_info.build_properties['date'])
- except (IOError, PythonDirectoryWrapper.Error):
- print("Aurora client build info not available")
- print("Aurora API version: %s" % CURRENT_API_VERSION)
-
-
-@app.command
-@app.command_option(ENVIRONMENT_BIND_OPTION)
-@app.command_option(OPEN_BROWSER_OPTION)
-@app.command_option(CLUSTER_CONFIG_OPTION)
-@app.command_option(ENV_CONFIG_OPTION)
-@app.command_option(JSON_OPTION)
-@app.command_option(WAIT_UNTIL_OPTION)
-@requires.exactly('cluster/role/env/job', 'config')
-def create(job_spec, config_file):
- """usage: create cluster/role/env/job config
-
- Creates a job based on a configuration file.
- """
- options = app.get_options()
- try:
- config = get_job_config(job_spec, config_file, options)
- except ValueError as v:
- print("Error: %s" % v)
- sys.exit(1)
- api = make_client(config.cluster())
- monitor = JobMonitor(api, config.role(), config.environment(), config.name())
- resp = api.create_job(config)
- check_and_log_response(resp)
- handle_open(api.scheduler.scheduler().url, config.role(), config.environment(), config.name())
- if options.wait_until == 'RUNNING':
- monitor.wait_until(monitor.running_or_finished)
- elif options.wait_until == 'FINISHED':
- monitor.wait_until(monitor.terminal)
-
-
-@app.command
-@app.command_option(ENVIRONMENT_BIND_OPTION)
-@app.command_option(CLUSTER_CONFIG_OPTION)
-@app.command_option(ENV_CONFIG_OPTION)
-@app.command_option(JSON_OPTION)
-@app.command_option(FROM_JOBKEY_OPTION)
-@requires.exactly('cluster/role/env/job', 'config')
-def diff(job_spec, config_file):
- """usage: diff cluster/role/env/job config
-
- Compares a job configuration against a running job.
- By default the diff will be displayed using 'diff', though you may choose an alternate
- diff program by specifying the DIFF_VIEWER environment variable."""
- options = app.get_options()
- config = get_job_config(job_spec, config_file, options)
- if options.rename_from:
- cluster, role, env, name = options.rename_from
- else:
- cluster = config.cluster()
- role = config.role()
- env = config.environment()
- name = config.name()
- api = make_client(cluster)
- resp = api.query(api.build_query(role, name, statuses=ACTIVE_STATES, env=env))
- if resp.responseCode != ResponseCode.OK:
- die('Request failed, server responded with "%s"' % resp.message)
- remote_tasks = [t.assignedTask.task for t in resp.result.scheduleStatusResult.tasks]
- resp = api.populate_job_config(config)
- if resp.responseCode != ResponseCode.OK:
- die('Request failed, server responded with "%s"' % resp.message)
- local_tasks = resp.result.populateJobResult.populated
-
- pp = pprint.PrettyPrinter(indent=2)
- def pretty_print_task(task):
- # The raw configuration is not interesting - we only care about what gets parsed.
- task.configuration = None
- task.executorConfig = ExecutorConfig(
- name=AURORA_EXECUTOR_NAME,
- data=json.loads(task.executorConfig.data))
- return pp.pformat(vars(task))
-
- def pretty_print_tasks(tasks):
- return ',\n'.join([pretty_print_task(t) for t in tasks])
-
- def dump_tasks(tasks, out_file):
- out_file.write(pretty_print_tasks(tasks))
- out_file.write('\n')
- out_file.flush()
-
- diff_program = os.environ.get('DIFF_VIEWER', 'diff')
- with NamedTemporaryFile() as local:
- dump_tasks(local_tasks, local)
- with NamedTemporaryFile() as remote:
- dump_tasks(remote_tasks, remote)
- result = subprocess.call([diff_program, remote.name, local.name])
- # Unlike most commands, diff doesn't return zero on success; it returns
- # 1 when a successful diff is non-empty.
- if result != 0 and result != 1:
- return result
- else:
- return 0
-
-
-@app.command(name='open')
-def do_open(args, _):
- """usage: open cluster[/role[/env/job]]
-
- Opens the scheduler page for a cluster, role or job in the default web browser.
- """
- cluster_name = role = env = job = None
- args = args[0].split("/")
- if len(args) > 0:
- cluster_name = args[0]
- if len(args) > 1:
- role = args[1]
- if len(args) > 2:
- env = args[2]
- if len(args) > 3:
- job = args[3]
- else:
- # TODO(ksweeney): Remove this after MESOS-2945 is completed.
- die('env scheduler pages are not yet implemented, please specify job')
-
- if not cluster_name:
- die('cluster is required')
-
- api = make_client(cluster_name)
-
- import webbrowser
- webbrowser.open_new_tab(synthesize_url(api.scheduler.scheduler().url, role, env, job))
-
-
-@app.command
-@app.command_option('--local', dest='local', default=False, action='store_true',
- help='Inspect the configuration as would be created by the "spawn" command.')
-@app.command_option('--raw', dest='raw', default=False, action='store_true',
- help='Show the raw configuration.')
-@app.command_option(ENVIRONMENT_BIND_OPTION)
-@app.command_option(CLUSTER_CONFIG_OPTION)
-@app.command_option(ENV_CONFIG_OPTION)
-@app.command_option(JSON_OPTION)
-@requires.exactly('cluster/role/env/job', 'config')
-def inspect(job_spec, config_file):
- """usage: inspect cluster/role/env/job config
-
- Verifies that a job can be parsed from a configuration file, and displays
- the parsed configuration.
- """
- options = app.get_options()
- config = get_job_config(job_spec, config_file, options)
- if options.raw:
- print('Parsed job config: %s' % config.job())
- return
-
- job_thrift = config.job()
- job = config.raw()
- job_thrift = config.job()
- print('Job level information')
- print(' name: %s' % job.name())
- print(' role: %s' % job.role())
- print(' contact: %s' % job.contact())
- print(' cluster: %s' % job.cluster())
- print(' instances: %s' % job.instances())
- if job.has_cron_schedule():
- print(' cron:')
- print(' schedule: %s' % job.cron_schedule())
- print(' policy: %s' % job.cron_collision_policy())
- if job.has_constraints():
- print(' constraints:')
- for constraint, value in job.constraints().get().items():
- print(' %s: %s' % (constraint, value))
- print(' service: %s' % job_thrift.taskConfig.isService)
- print(' production: %s' % bool(job.production().get()))
- print()
-
- task = job.task()
- print('Task level information')
- print(' name: %s' % task.name())
- if len(task.constraints().get()) > 0:
- print(' constraints:')
- for constraint in task.constraints():
- print(' %s' % (' < '.join(st.get() for st in constraint.order())))
- print()
-
- processes = task.processes()
- for process in processes:
- print('Process %s:' % process.name())
- if process.daemon().get():
- print(' daemon')
- if process.ephemeral().get():
- print(' ephemeral')
- if process.final().get():
- print(' final')
- print(' cmdline:')
- for line in process.cmdline().get().splitlines():
- print(' ' + line)
- print()
-
-
-@app.command
-@app.command_option(CLUSTER_INVOKE_OPTION)
-@app.command_option(OPEN_BROWSER_OPTION)
-def start_cron(args, options):
- """usage: start_cron cluster/role/env/job
-
- Invokes a cron job immediately, out of its normal cron cycle.
- This does not affect the cron cycle in any way.
- """
-
- api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
- args, options, make_client_factory())
- config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
- resp = api.start_cronjob(job_key, config=config)
- check_and_log_response(resp)
- handle_open(api.scheduler.scheduler().url, job_key.role, job_key.env, job_key.name)
-
-
-@app.command
-@app.command_option(
- '--pretty',
- dest='pretty',
- default=False,
- action='store_true',
- help='Show job information in prettyprinted format')
-@app.command_option(
- '--show-cron',
- '-c',
- dest='show_cron_schedule',
- default=False,
- action='store_true',
- help='List jobs registered with the Aurora scheduler')
-@requires.exactly('cluster/role')
-def list_jobs(cluster_and_role):
- """usage: list_jobs [--show-cron] cluster/role/env/job
-
- Shows all jobs that match the job-spec known by the scheduler.
- If --show-cron is specified, then also shows the registered cron schedule.
- """
- def show_job_simple(job):
- if options.show_cron_schedule:
- print(('{0}/{1.key.role}/{1.key.environment}/{1.key.name}' +
- '\t\'{1.cronSchedule}\'\t{1.cronCollisionPolicy}').format(cluster, job))
- else:
- print('{0}/{1.key.role}/{1.key.environment}/{1.key.name}'.format(cluster, job))
-
- def show_job_pretty(job):
- print("Job %s/%s/%s/%s:" %
- (cluster, job.key.role, job.key.environment, job.key.name))
- print('\tcron schedule: %s' % job.cronSchedule)
- print('\tcron policy: %s' % job.cronCollisionPolicy)
-
- options = app.get_options()
- if options.show_cron_schedule and options.pretty:
- print_fn = show_job_pretty
- else:
- print_fn = show_job_simple
- # Take the cluster_and_role parameter, and split it into its two components.
- if cluster_and_role.count('/') != 1:
- die('list_jobs parameter must be in cluster/role format')
- (cluster,role) = cluster_and_role.split('/')
- api = make_client(cluster)
- resp = api.get_jobs(role)
- check_and_log_response(resp)
- for job in resp.result.getJobsResult.configs:
- print_fn(job)
-
-
-@app.command
-@app.command_option(CLUSTER_INVOKE_OPTION)
-@app.command_option(OPEN_BROWSER_OPTION)
-@app.command_option(SHARDS_OPTION)
-def kill(args, options):
- """usage: kill cluster/role/env/job
-
- Kills a running job, blocking until all tasks have terminated.
-
- Default behaviour is to kill all shards in the job, but the kill
- can be limited to specific shards with the --shards option
- """
- api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
- args, options, make_client_factory())
- options = app.get_options()
- config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
- resp = api.kill_job(job_key, options.shards, config=config)
- check_and_log_response(resp)
- handle_open(api.scheduler.scheduler().url, job_key.role, job_key.env, job_key.name)
-
-
-@app.command
-@app.command_option(CLUSTER_INVOKE_OPTION)
-def status(args, options):
- """usage: status cluster/role/env/job
-
- Fetches and prints information about the active tasks in a job.
- """
- def is_active(task):
- return task.status in ACTIVE_STATES
-
- def print_task(scheduled_task):
- assigned_task = scheduled_task.assignedTask
- taskInfo = assigned_task.task
- taskString = ''
- if taskInfo:
- taskString += '''cpus: %s, ram: %s MB, disk: %s MB''' % (taskInfo.numCpus,
- taskInfo.ramMb,
- taskInfo.diskMb)
- if assigned_task.assignedPorts:
- taskString += '\n\tports: %s' % assigned_task.assignedPorts
- taskString += '\n\tfailure count: %s (max %s)' % (scheduled_task.failureCount,
- taskInfo.maxTaskFailures)
- taskString += '\n\tevents:'
- for event in scheduled_task.taskEvents:
- taskString += '\n\t\t %s %s: %s' % (datetime.fromtimestamp(event.timestamp / 1000),
- ScheduleStatus._VALUES_TO_NAMES[event.status],
- event.message)
- taskString += '\n\tpackages:'
- for pkg in assigned_task.task.packages:
- taskString += ('\n\t\trole: %s, package: %s, version: %s' % (pkg.role, pkg.name, pkg.version))
-
- return taskString
-
- def print_tasks(tasks):
- for task in tasks:
- taskString = print_task(task)
-
- log.info('role: %s, env: %s, name: %s, shard: %s, status: %s on %s\n%s' %
- (task.assignedTask.task.owner.role,
- task.assignedTask.task.environment,
- task.assignedTask.task.jobName,
- task.assignedTask.instanceId,
- ScheduleStatus._VALUES_TO_NAMES[task.status],
- task.assignedTask.slaveHost,
- taskString))
- for pkg in task.assignedTask.task.packages:
- log.info('\tpackage %s/%s/%s' % (pkg.role, pkg.name, pkg.version))
-
- api, job_key, _ = LiveJobDisambiguator.disambiguate_args_or_die(
- args, options, make_client_factory())
- resp = api.check_status(job_key)
- check_and_log_response(resp)
-
- tasks = resp.result.scheduleStatusResult.tasks
- if tasks:
- active_tasks = filter(is_active, tasks)
- log.info('Active Tasks (%s)' % len(active_tasks))
- print_tasks(active_tasks)
- inactive_tasks = filter(lambda x: not is_active(x), tasks)
- log.info('Inactive Tasks (%s)' % len(inactive_tasks))
- print_tasks(inactive_tasks)
- else:
- log.info('No tasks found.')
-
-
-@app.command
-@app.command_option(SHARDS_OPTION)
-@app.command_option(ENVIRONMENT_BIND_OPTION)
-@app.command_option(CLUSTER_CONFIG_OPTION)
-@app.command_option(ENV_CONFIG_OPTION)
-@app.command_option(JSON_OPTION)
-@app.command_option(HEALTH_CHECK_INTERVAL_SECONDS_OPTION)
-@app.command_option(
- '--force',
- dest='force',
- default=True, # TODO(maximk): Temporary bandaid for MESOS-4310 until a better fix is available.
- action='store_true',
- help='Turn off warning message that the update looks large enough to be disruptive.')
-@requires.exactly('cluster/role/env/job', 'config')
-def update(job_spec, config_file):
- """usage: update cluster/role/env/job config
-
- Performs a rolling upgrade on a running job, using the update configuration
- within the config file as a control for update velocity and failure tolerance.
-
- Updates are fully controlled client-side, so aborting an update halts the
- update and leaves the job in a 'locked' state on the scheduler.
- Subsequent update attempts will fail until the update is 'unlocked' using the
- 'cancel_update' command.
-
- The updater only takes action on shards in a job that have changed, meaning
- that changing a single shard will only induce a restart on the changed shard.
-
- You may want to consider using the 'diff' subcommand before updating,
- to preview what changes will take effect.
- """
- def warn_if_dangerous_change(api, job_spec, config):
- # Get the current job status, so that we can check if there's anything
- # dangerous about this update.
- job_key = AuroraJobKey(config.cluster(), config.role(), config.environment(), config.name())
- resp = api.query(api.build_query(config.role(), config.name(),
- statuses=ACTIVE_STATES, env=config.environment()))
- if resp.responseCode != ResponseCode.OK:
- die('Could not get job status from server for comparison: %s' % resp.message)
- remote_tasks = [t.assignedTask.task for t in resp.result.scheduleStatusResult.tasks]
- resp = api.populate_job_config(config)
- if resp.responseCode != ResponseCode.OK:
- die('Server could not populate job config for comparison: %s' % resp.message)
- local_task_count = len(resp.result.populateJobResult.populated)
- remote_task_count = len(remote_tasks)
- if (local_task_count >= 4 * remote_task_count or local_task_count <= 4 * remote_task_count
- or local_task_count == 0):
- print('Warning: this update is a large change. Press ^c within 5 seconds to abort')
- time.sleep(5)
-
- options = app.get_options()
- config = get_job_config(job_spec, config_file, options)
- api = make_client(config.cluster())
- if not options.force:
- warn_if_dangerous_change(api, job_spec, config)
- resp = api.update_job(config, options.health_check_interval_seconds, options.shards)
- check_and_log_response(resp)
-
-
-@app.command
-@app.command_option(CLUSTER_INVOKE_OPTION)
-@app.command_option(HEALTH_CHECK_INTERVAL_SECONDS_OPTION)
-@app.command_option(OPEN_BROWSER_OPTION)
-@app.command_option(SHARDS_OPTION)
-@app.command_option(
- '--batch_size',
- dest='batch_size',
- type=int,
- default=1,
- help='Number of shards to be restarted in one iteration.')
-@app.command_option(
- '--max_per_shard_failures',
- dest='max_per_shard_failures',
- type=int,
- default=0,
- help='Maximum number of restarts per shard during restart. Increments total failure count when '
- 'this limit is exceeded.')
-@app.command_option(
- '--max_total_failures',
- dest='max_total_failures',
- type=int,
- default=0,
- help='Maximum number of shard failures to be tolerated in total during restart.')
-@app.command_option(
- '--restart_threshold',
- dest='restart_threshold',
- type=int,
- default=60,
- help='Maximum number of seconds before a shard must move into the RUNNING state before '
- 'considered a failure.')
-@app.command_option(
- '--watch_secs',
- dest='watch_secs',
- type=int,
- default=30,
- help='Minimum number of seconds a shard must remain in RUNNING state before considered a '
- 'success.')
-def restart(args, options):
- """usage: restart cluster/role/env/job
- [--shards=SHARDS]
- [--batch_size=INT]
- [--updater_health_check_interval_seconds=SECONDS]
- [--max_per_shard_failures=INT]
- [--max_total_failures=INT]
- [--restart_threshold=INT]
- [--watch_secs=SECONDS]
-
- Performs a rolling restart of shards within a job.
-
- Restarts are fully controlled client-side, so aborting halts the restart.
- """
- api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
- args, options, make_client_factory())
- config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
- updater_config = UpdaterConfig(
- options.batch_size,
- options.restart_threshold,
- options.watch_secs,
- options.max_per_shard_failures,
- options.max_total_failures)
- resp = api.restart(job_key, options.shards, updater_config,
- options.health_check_interval_seconds, config=config)
- check_and_log_response(resp)
- handle_open(api.scheduler.scheduler().url, job_key.role, job_key.env, job_key.name)
-
-
-@app.command
-@app.command_option(CLUSTER_INVOKE_OPTION)
-def cancel_update(args, options):
- """usage: cancel_update cluster/role/env/job
-
- Unlocks a job for updates.
- A job may be locked if a client's update session terminated abnormally,
- or if another user is actively updating the job. This command should only
- be used when the user is confident that they are not conflicting with another user.
- """
- api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
- args, options, make_client_factory())
- config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
- resp = api.cancel_update(job_key, config=config)
- check_and_log_response(resp)
-
-
-@app.command
-@app.command_option(CLUSTER_INVOKE_OPTION)
-@requires.exactly('role')
-def get_quota(role):
- """usage: get_quota --cluster=CLUSTER role
-
- Prints the production quota that has been allocated to a user.
- """
- options = app.get_options()
- resp = make_client(options.cluster).get_quota(role)
- quota = resp.result.getQuotaResult.quota
-
- quota_fields = [
- ('CPU', quota.numCpus),
- ('RAM', '%f GB' % (float(quota.ramMb) / 1024)),
- ('Disk', '%f GB' % (float(quota.diskMb) / 1024))
- ]
- log.info('Quota for %s:\n\t%s' %
- (role, '\n\t'.join(['%s\t%s' % (k, v) for (k, v) in quota_fields])))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/commands/help.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/commands/help.py b/src/main/python/twitter/aurora/client/commands/help.py
deleted file mode 100644
index a74c607..0000000
--- a/src/main/python/twitter/aurora/client/commands/help.py
+++ /dev/null
@@ -1,53 +0,0 @@
-from __future__ import print_function
-
-import collections
-import sys
-
-from twitter.aurora.client.base import die
-from twitter.common import app
-
-
-def make_commands_str(commands):
- commands.sort()
- if len(commands) == 1:
- return str(commands[0])
- elif len(commands) == 2:
- return '%s (or %s)' % (str(commands[0]), str(commands[1]))
- else:
- return '%s (or any of: %s)' % (str(commands[0]), ' '.join(map(str, commands[1:])))
-
-
-def generate_full_usage():
- docs_to_commands = collections.defaultdict(list)
- for (command, doc) in app.get_commands_and_docstrings():
- if doc is not None:
- docs_to_commands[doc].append(command)
- def make_docstring(item):
- (doc_text, commands) = item
- def format_line(line):
- return ' %s\n' % line.lstrip()
- stripped = ''.join(map(format_line, doc_text.splitlines()))
- return '%s\n%s' % (make_commands_str(commands), stripped)
- usage = sorted(map(make_docstring, docs_to_commands.items()))
- return 'Available commands:\n\n' + '\n'.join(usage)
-
-
-@app.command
-def help(args):
- """usage: help [subcommand]
-
- Prints help for using the aurora client, or one of its specific subcommands.
- """
- if not args:
- print(generate_full_usage())
- sys.exit(0)
-
- if len(args) > 1:
- die('Please specify at most one subcommand.')
-
- subcmd = args[0]
- if subcmd in app.get_commands():
- app.command_parser(subcmd).print_help()
- else:
- print('Subcommand %s not found.' % subcmd)
- sys.exit(1)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/commands/run.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/commands/run.py b/src/main/python/twitter/aurora/client/commands/run.py
deleted file mode 100644
index 494ce47..0000000
--- a/src/main/python/twitter/aurora/client/commands/run.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from twitter.common import app
-from twitter.aurora.client.base import die
-from twitter.aurora.client.options import (
- EXECUTOR_SANDBOX_OPTION,
- SSH_USER_OPTION,
-)
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-from twitter.aurora.common.clusters import CLUSTERS
-from twitter.aurora.client.api.command_runner import DistributedCommandRunner
-
-
-@app.command
-@app.command_option('-t', '--threads', type=int, default=1, dest='num_threads',
- help='The number of threads to use.')
-@app.command_option(SSH_USER_OPTION)
-@app.command_option(EXECUTOR_SANDBOX_OPTION)
-def run(args, options):
- """usage: run cluster/role/env/job cmd
-
- Runs a shell command on all machines currently hosting shards of a single job.
-
- This feature supports the same command line wildcards that are used to
- populate a job's commands.
-
- This means anything in the {{mesos.*}} and {{thermos.*}} namespaces.
- """
- # TODO(William Farner): Add support for invoking on individual shards.
- # TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010).
- if not args:
- die('job path is required')
- job_path = args.pop(0)
- try:
- cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
- except AuroraJobKey.Error as e:
- die('Invalid job path "%s": %s' % (job_path, e))
-
- command = ' '.join(args)
- cluster = CLUSTERS[cluster_name]
- dcr = DistributedCommandRunner(cluster, role, env, [name], options.ssh_user)
- dcr.run(command, parallelism=options.num_threads, executor_sandbox=options.executor_sandbox)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/commands/ssh.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/commands/ssh.py b/src/main/python/twitter/aurora/client/commands/ssh.py
deleted file mode 100644
index 109229f..0000000
--- a/src/main/python/twitter/aurora/client/commands/ssh.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import subprocess
-
-from twitter.common import app
-from twitter.aurora.client.base import check_and_log_response, die
-from twitter.aurora.client.factory import make_client
-from twitter.aurora.client.options import (
- EXECUTOR_SANDBOX_OPTION,
- SSH_USER_OPTION,
-)
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-from twitter.aurora.client.api.command_runner import DistributedCommandRunner
-
-
-@app.command
-@app.command_option(EXECUTOR_SANDBOX_OPTION)
-@app.command_option(SSH_USER_OPTION)
-@app.command_option('-L', dest='tunnels', action='append', metavar='PORT:NAME',
- default=[],
- help="Add tunnel from local port PORT to remote named port NAME.")
-def ssh(args, options):
- """usage: ssh cluster/role/env/job shard [args...]
-
- Initiate an SSH session on the machine that a shard is running on.
- """
- if not args:
- die('Job path is required')
- job_path = args.pop(0)
- try:
- cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
- except AuroraJobKey.Error as e:
- die('Invalid job path "%s": %s' % (job_path, e))
- if not args:
- die('Shard is required')
- try:
- shard = int(args.pop(0))
- except ValueError:
- die('Shard must be an integer')
- api = make_client(cluster_name)
- resp = api.query(api.build_query(role, name, set([int(shard)]), env=env))
- check_and_log_response(resp)
-
- first_task = resp.result.scheduleStatusResult.tasks[0]
- remote_cmd = 'bash' if not args else ' '.join(args)
- command = DistributedCommandRunner.substitute(remote_cmd, first_task,
- api.cluster, executor_sandbox=options.executor_sandbox)
-
- ssh_command = ['ssh', '-t']
-
- role = first_task.assignedTask.task.owner.role
- slave_host = first_task.assignedTask.slaveHost
-
- for tunnel in options.tunnels:
- try:
- port, name = tunnel.split(':')
- port = int(port)
- except ValueError:
- die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
- if name not in first_task.assignedTask.assignedPorts:
- die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
- ssh_command += [
- '-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
-
- ssh_command += ['%s@%s' % (options.ssh_user or role, slave_host), command]
- return subprocess.call(ssh_command)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/config.py b/src/main/python/twitter/aurora/client/config.py
deleted file mode 100644
index 32df4eb..0000000
--- a/src/main/python/twitter/aurora/client/config.py
+++ /dev/null
@@ -1,208 +0,0 @@
-'''Library of utilities called by the mesos client binary
-'''
-
-from __future__ import print_function
-
-import functools
-import math
-import posixpath
-import re
-import sys
-
-from twitter.common import app, log
-
-from twitter.aurora.client import binding_helper
-from twitter.aurora.client.base import deprecation_warning, die
-from twitter.aurora.config import AuroraConfig
-from twitter.thermos.config.schema_helpers import Tasks
-
-from gen.twitter.aurora.constants import DEFAULT_ENVIRONMENT
-
-from pystachio import Empty, Ref
-
-
-APPAPP_DEPRECATION_WARNING = """
-The use of app-app is deprecated. Please reach out to mesos-team@twitter.com for advice on
-migrating your application away from app-app layouts to an alternative packaging solution.
-"""
-
-
-def _warn_on_appapp_layouts(config):
- if config.raw().has_layout():
- deprecation_warning(APPAPP_DEPRECATION_WARNING)
-
-
-CRON_DEPRECATION_WARNING = """
-The "cron_policy" parameter to Jobs has been renamed to "cron_collision_policy".
-Please update your Jobs accordingly.
-"""
-
-
-def _warn_on_deprecated_cron_policy(config):
- if config.raw().cron_policy() is not Empty:
- deprecation_warning(CRON_DEPRECATION_WARNING)
-
-
-DAEMON_DEPRECATION_WARNING = """
-The "daemon" parameter to Jobs is deprecated in favor of the "service" parameter.
-Please update your Job to set "service = True" instead of "daemon = True", or use
-the top-level Service() instead of Job().
-"""
-
-
-def _warn_on_deprecated_daemon_job(config):
- if config.raw().daemon() is not Empty:
- deprecation_warning(DAEMON_DEPRECATION_WARNING)
-
-
-HEALTH_CHECK_INTERVAL_SECS_DEPRECATION_WARNING = """
-The "health_check_interval_secs" parameter to Jobs is deprecated in favor of the
-"health_check_config" parameter. Please update your Job to set the parameter by creating a new
-HealthCheckConfig.
-
-See the HealthCheckConfig section of the Configuration Reference page for more information:
-http://go/auroraconfig/#Aurora%2BThermosConfigurationReference-HealthCheckConfig
-"""
-
-
-def _warn_on_deprecated_health_check_interval_secs(config):
- if config.raw().health_check_interval_secs() is not Empty:
- deprecation_warning(HEALTH_CHECK_INTERVAL_SECS_DEPRECATION_WARNING)
-
-
-ANNOUNCE_WARNING = """
-Announcer specified primary port as '%(primary_port)s' but no processes have bound that port.
-If you would like to utilize this port, you should listen on {{thermos.ports[%(primary_port)s]}}
-from some Process bound to your task.
-"""
-
-
-def _validate_announce_configuration(config):
- if not config.raw().has_announce():
- return
-
- primary_port = config.raw().announce().primary_port().get()
- if primary_port not in config.ports():
- print(ANNOUNCE_WARNING % {'primary_port': primary_port}, file=sys.stderr)
-
- if config.raw().has_announce() and not config.raw().has_constraints() or (
- 'dedicated' not in config.raw().constraints()):
- for port in config.raw().announce().portmap().get().values():
- try:
- port = int(port)
- except ValueError:
- continue
- raise ValueError('Job must be dedicated in order to specify static ports!')
-
-
-STAGING_RE = re.compile(r'^staging\d*$')
-
-
-def _validate_environment_name(config):
- env_name = str(config.raw().environment())
- if STAGING_RE.match(env_name):
- return
- if env_name not in ('prod', 'devel', 'test'):
- raise ValueError('Environment name should be one of "prod", "devel", "test" or '
- 'staging<number>! Got %s' % env_name)
-
-
-UPDATE_CONFIG_MAX_FAILURES_ERROR = '''
-max_total_failures in update_config must be lesser than the job size.
-Based on your job size (%s) you should use max_total_failures <= %s.
-
-See http://go/auroraconfig for details.
-'''
-
-
-UPDATE_CONFIG_DEDICATED_THRESHOLD_ERROR = '''
-Since this is a dedicated job, you must set your max_total_failures in
-your update configuration to no less than 2%% of your job size.
-Based on your job size (%s) you should use max_total_failures >= %s.
-
-See http://go/auroraconfig for details.
-'''
-
-
-def _validate_update_config(config):
- job_size = config.instances()
- max_failures = config.update_config().max_total_failures().get()
-
- if max_failures >= job_size:
- die(UPDATE_CONFIG_MAX_FAILURES_ERROR % (job_size, job_size - 1))
-
- if config.is_dedicated():
- min_failure_threshold = int(math.floor(job_size * 0.02))
- if max_failures < min_failure_threshold:
- die(UPDATE_CONFIG_DEDICATED_THRESHOLD_ERROR % (job_size, min_failure_threshold))
-
-
-HEALTH_CHECK_INTERVAL_SECS_ERROR = '''
-health_check_interval_secs paramater to Job has been deprecated. Please specify health_check_config
-only.
-
-See http://go/auroraconfig/#Aurora%2BThermosConfigurationReference-HealthCheckConfig
-'''
-
-
-def _validate_health_check_config(config):
- # TODO(Sathya): Remove this check after health_check_interval_secs deprecation cycle is complete.
- if config.raw().has_health_check_interval_secs() and config.raw().has_health_check_config():
- die(HEALTH_CHECK_INTERVAL_SECS_ERROR)
-
-
-DEFAULT_ENVIRONMENT_WARNING = '''
-Job did not specify environment, auto-populating to "%s".
-'''
-
-
-def _inject_default_environment(config):
- if not config.raw().has_environment():
- print(DEFAULT_ENVIRONMENT_WARNING % DEFAULT_ENVIRONMENT, file=sys.stderr)
- config.update_job(config.raw()(environment=DEFAULT_ENVIRONMENT))
-
-
-def validate_config(config, env=None):
- _validate_update_config(config)
- _validate_health_check_config(config)
- _validate_announce_configuration(config)
- _validate_environment_name(config)
-
-
-def populate_namespaces(config, env=None):
- _inject_default_environment(config)
- _warn_on_deprecated_cron_policy(config)
- _warn_on_deprecated_daemon_job(config)
- _warn_on_deprecated_health_check_interval_secs(config)
- _warn_on_appapp_layouts(config)
- return config
-
-
-def inject_hooks(config, env=None):
- config.hooks = (env or {}).get('hooks', [])
-
-
-class AnnotatedAuroraConfig(AuroraConfig):
- @classmethod
- def plugins(cls):
- return (inject_hooks,
- functools.partial(binding_helper.apply_all),
- functools.partial(populate_namespaces),
- validate_config)
-
-
-def get_config(jobname,
- config_file,
- json=False,
- bindings=(),
- select_cluster=None,
- select_role=None,
- select_env=None):
- """Creates and returns a config object contained in the provided file."""
- loader = AnnotatedAuroraConfig.load_json if json else AnnotatedAuroraConfig.load
- return loader(config_file,
- jobname,
- bindings,
- select_cluster=select_cluster,
- select_role=select_role,
- select_env=select_env)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/factory.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/factory.py b/src/main/python/twitter/aurora/client/factory.py
deleted file mode 100644
index 7a44e3b..0000000
--- a/src/main/python/twitter/aurora/client/factory.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import functools
-
-from twitter.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
-from twitter.aurora.common.cluster import Cluster
-from twitter.aurora.common.clusters import CLUSTERS
-from twitter.common import app
-
-from .base import die
-
-
-# TODO(wickman) Kill make_client and make_client_factory as part of MESOS-3801.
-# These are currently necessary indirections for the LiveJobDisambiguator among
-# other things but can go away once those are scrubbed.
-
-def make_client_factory():
- verbose = getattr(app.get_options(), 'verbosity', 'normal') == 'verbose'
- class TwitterAuroraClientAPI(HookedAuroraClientAPI):
- def __init__(self, cluster, *args, **kw):
- if cluster not in CLUSTERS:
- die('Unknown cluster: %s' % cluster)
- super(TwitterAuroraClientAPI, self).__init__(CLUSTERS[cluster], *args, **kw)
- return functools.partial(TwitterAuroraClientAPI, verbose=verbose)
-
-
-def make_client(cluster):
- factory = make_client_factory()
- return factory(cluster.name if isinstance(cluster, Cluster) else cluster)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/hooks/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/hooks/BUILD b/src/main/python/twitter/aurora/client/hooks/BUILD
deleted file mode 100644
index c3d0a1a..0000000
--- a/src/main/python/twitter/aurora/client/hooks/BUILD
+++ /dev/null
@@ -1,10 +0,0 @@
-python_library(
- name = 'hooks',
- sources = ['__init__.py', 'hooked_api.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/client:api'),
- pants('src/main/python/twitter/aurora/common:aurora_job_key'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/hooks/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/hooks/__init__.py b/src/main/python/twitter/aurora/client/hooks/__init__.py
deleted file mode 100644
index 3091f67..0000000
--- a/src/main/python/twitter/aurora/client/hooks/__init__.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""
-A hooks implementation for the Aurora client.
-
-The Hook protocol is the following:
- Any object may be passed in as a hook.
-
- If the object has pre_<api method name> defined that is callable, it will be called with:
- method(*args, **kw)
-
- where *args and **kw are the arguments and keyword arguments passed into
- the original APi call. This is done prior to the invocation of the API
- call. If this method returns Falsy, the API call will be aborted.
-
- If the object has an err_<api method name> defined that is callable, it will be called with:
- method(exc, *args, **kw)
-
- If the object has a post_<api method name> defined that is callable, it will be called with:
- method(result, *args, **kw)
-
- These methods are called after the respective API call has been made. The
- return codes of err and post methods are ignored.
-
-If the object does not have any of these attributes, it will instead delegate to the
-'generic_hook' method, if available. The method signature for generic_hook is:
-
- generic_hook(hook_config, event, method_name, result_or_err, args, kw)
-
-Where hook_config is a namedtuple of 'config' and 'job_key', event is one of
-'pre', 'err', 'post', method_name is the API method name, and args, kw are
-the arguments / keyword arguments. result_or_err is a tri_state:
- - None for pre hooks
- - result for post hooks
- - exc for err hooks
-
-Examples:
-
- class Logger(object):
- '''Just logs every at all point for all API calls'''
- def generic_hook(self, hook_config, event, method_name, result_or_err, *args, **kw)
- log.info('%s: %s_%s of %s' % (self.__class__.__name__, event, method_name, job_key))
-
- class KillConfirmer(object):
- def confirm(self, msg):
- return True if raw_input(msg).lower() == 'yes' else False
-
- def pre_kill(self, job_key, shards=None):
- shards = ('shards %s' % shards) if shards is not None else 'all shards'
- return self.confirm('Are you sure you want to kill %s? (yes/no): ' % (job_key, shards))
-"""
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/hooks/hooked_api.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/hooks/hooked_api.py b/src/main/python/twitter/aurora/client/hooks/hooked_api.py
deleted file mode 100644
index cc4d3db..0000000
--- a/src/main/python/twitter/aurora/client/hooks/hooked_api.py
+++ /dev/null
@@ -1,167 +0,0 @@
-import functools
-import traceback
-
-from twitter.common import log
-from twitter.aurora.client.api import AuroraClientAPI
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-
-from gen.twitter.aurora.ttypes import ResponseCode
-
-
-def _partial(function, *args, **kw):
- """Returns a partial function __name__ inherited from parent function."""
- partial = functools.partial(function, *args, **kw)
- return functools.update_wrapper(partial, function)
-
-
-class HookConfig(object):
- def __init__(self, config, job_key):
- self.config = config
- self.job_key = job_key or (config.job_key() if config is not None else None)
-
- def __iter__(self):
- yield self.config
- yield self.job_key
-
-
-class NonHookedAuroraClientAPI(AuroraClientAPI):
- """
- This wraps those AuroraClientAPI methods that don't have an AuroraConfig 'config' param
- to take an optional 'config' param which:
- * contains the configured hooks (config.hooks)
- * is dropped before the call is proxied to AuroraClientAPI
- * is thus available to API methods in subclasses
- """
-
- def cancel_update(self, job_key, config=None):
- return super(NonHookedAuroraClientAPI, self).cancel_update(job_key)
-
- def kill_job(self, job_key, instances=None, lock=None, config=None):
- return super(NonHookedAuroraClientAPI, self).kill_job(job_key, instances=instances, lock=lock)
-
- def restart(self, job_key, shards, updater_config, health_check_interval_seconds, config=None):
- return super(NonHookedAuroraClientAPI, self).restart(job_key, shards, updater_config,
- health_check_interval_seconds)
-
- def start_cronjob(self, job_key, config=None):
- return super(NonHookedAuroraClientAPI, self).start_cronjob(job_key)
-
-
-class HookedAuroraClientAPI(NonHookedAuroraClientAPI):
- """
- Adds a hooking aspect/behaviour to the lifecycle of Mesos Client API methods
- by injecting hooks (instances of twitter.aurora.client.hooks.Hooks)
-
- * Hooks are available in the 'config' (AuroraConfig) param that each API call receives
- * Each Hook is run around each API call:
- * 'pre' hook before the call
- * 'post' hook if the call succeeds
- * 'err' hook if the call fails
- * If the hook itself fails, then it is treated as a WARN rather than an ERROR
- """
-
- class Error(Exception): pass
- class PreHooksStoppedCall(Error): pass
- class APIError(Error):
- def __init__(self, response):
- self.response = response
-
- def __str__(self):
- return '%s: %s: %s' % (self.__class__.__name__,
- ResponseCode._VALUES_TO_NAMES.get(self.response.responseCode, 'UNKNOWN'),
- self.response.message)
-
- @classmethod
- def _meta_hook(cls, hook, hook_method):
- def callback():
- if hook_method is None:
- return True
- log.debug('Running %s in %s' % (hook_method.__name__, hook.__class__.__name__))
- hook_result = False
- try:
- hook_result = hook_method()
- if not hook_result:
- log.debug('%s in %s returned False' % (hook_method.__name__,
- hook.__class__.__name__))
- except Exception:
- log.warn('Error in %s in %s' %
- (hook_method.__name__, hook.__class__.__name__))
- log.warn(traceback.format_exc())
- return hook_result
- return callback
-
- @classmethod
- def _generate_method(cls, hook, config, job_key, event, method, extra_argument=None):
- method_name, args, kw = method.__name__, method.args, method.keywords
- kw = kw or {}
- hook_method = getattr(hook, '%s_%s' % (event, method_name), None)
- if callable(hook_method):
- if extra_argument is not None:
- hook_method = _partial(hook_method, extra_argument)
- return _partial(hook_method, *args, **kw)
- else:
- hook_method = getattr(hook, 'generic_hook', None)
- if hook_method is None:
- return None
- hook_method = _partial(hook_method, HookConfig(config, job_key),
- event, method_name, extra_argument)
- return _partial(hook_method, args, kw)
-
- @classmethod
- def _yield_hooks(cls, event, config, job_key, api_call, extra_argument=None):
- hooks = config.hooks if config and config.raw().enable_hooks().get() else ()
- for hook in hooks:
- yield cls._meta_hook(hook,
- cls._generate_method(hook, config, job_key, event, api_call, extra_argument))
-
- @classmethod
- def _invoke_hooks(cls, event, config, job_key, api_call, extra_argument=None):
- hooks_passed = [hook() for hook in cls._yield_hooks(event, config, job_key, api_call,
- extra_argument)]
- return all(hooks_passed)
-
- def _hooked_call(self, config, job_key, api_call):
- if not self._invoke_hooks('pre', config, job_key, api_call):
- raise self.PreHooksStoppedCall('Pre hooks stopped call to %s' % api_call.__name__)
-
- try:
- resp = api_call()
- except Exception as e:
- self._invoke_hooks('err', config, job_key, api_call, e)
- raise # propagate since the API method call failed for unknown reasons
-
- if resp.responseCode != ResponseCode.OK:
- self._invoke_hooks('err', config, job_key, api_call, self.APIError(resp))
- else:
- self._invoke_hooks('post', config, job_key, api_call, resp)
-
- return resp
-
- def create_job(self, config, lock=None):
- return self._hooked_call(config, None,
- _partial(super(HookedAuroraClientAPI, self).create_job, config, lock))
-
- def cancel_update(self, job_key, config=None):
- return self._hooked_call(config, job_key,
- _partial(super(HookedAuroraClientAPI, self).cancel_update,
- job_key, config=config))
-
- def kill_job(self, job_key, instances=None, lock=None, config=None):
- return self._hooked_call(config, job_key,
- _partial(super(HookedAuroraClientAPI, self).kill_job,
- job_key, instances=instances, lock=lock, config=config))
-
- def restart(self, job_key, shards, updater_config, health_check_interval_seconds, config=None):
- return self._hooked_call(config, job_key,
- _partial(super(HookedAuroraClientAPI, self).restart,
- job_key, shards, updater_config, health_check_interval_seconds, config=config))
-
- def start_cronjob(self, job_key, config=None):
- return self._hooked_call(config, job_key,
- _partial(super(HookedAuroraClientAPI, self).start_cronjob,
- job_key, config=config))
-
- def update_job(self, config, health_check_interval_seconds=3, instances=None):
- return self._hooked_call(config, None,
- _partial(super(HookedAuroraClientAPI, self).update_job,
- config, health_check_interval_seconds=health_check_interval_seconds, instances=instances))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/client/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/client/options.py b/src/main/python/twitter/aurora/client/options.py
deleted file mode 100644
index 7ced961..0000000
--- a/src/main/python/twitter/aurora/client/options.py
+++ /dev/null
@@ -1,199 +0,0 @@
-import optparse
-
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-from twitter.thermos.common.options import add_binding_to
-
-
-__all__ = (
- 'CLUSTER_CONFIG_OPTION',
- 'CLUSTER_INVOKE_OPTION',
- 'ENVIRONMENT_BIND_OPTION',
- 'ENV_CONFIG_OPTION',
- 'EXECUTOR_SANDBOX_OPTION',
- 'FROM_JOBKEY_OPTION',
- 'HEALTH_CHECK_INTERVAL_SECONDS_OPTION',
- 'JSON_OPTION',
- 'OPEN_BROWSER_OPTION',
- 'SHARDS_OPTION',
- 'SSH_USER_OPTION',
- 'WAIT_UNTIL_OPTION',
-)
-
-
-def add_verbosity_options():
- from twitter.common import app
- from twitter.common.log.options import LogOptions
-
- def set_quiet(option, _1, _2, parser):
- setattr(parser.values, option.dest, 'quiet')
- LogOptions.set_stderr_log_level('NONE')
-
- def set_verbose(option, _1, _2, parser):
- setattr(parser.values, option.dest, 'verbose')
- LogOptions.set_stderr_log_level('DEBUG')
-
- app.add_option('-v',
- dest='verbosity',
- default='normal',
- action='callback',
- callback=set_verbose,
- help='Verbose logging. (default: %default)')
-
- app.add_option('-q',
- dest='verbosity',
- default='normal',
- action='callback',
- callback=set_quiet,
- help='Quiet logging. (default: %default)')
-
-
-def parse_shards_into(option, opt, value, parser):
- """Parse lists of shard or shard ranges into a set().
-
- Examples:
- 0-2
- 0,1-3,5
- 1,3,5
- """
- def shard_range_parser(shards):
- result = set()
- for part in shards.split(','):
- x = part.split('-')
- result.update(range(int(x[0]), int(x[-1]) + 1))
- return sorted(result)
-
- try:
- setattr(parser.values, option.dest, shard_range_parser(value))
- except ValueError as e:
- raise optparse.OptionValueError('Failed to parse: %s' % e)
-
-
-def parse_aurora_job_key_into(option, opt, value, parser):
- try:
- setattr(parser.values, option.dest, AuroraJobKey.from_path(value))
- except AuroraJobKey.Error as e:
- raise optparse.OptionValueError('Failed to parse: %s' % e)
-
-
-def make_env_option(explanation):
- return optparse.Option(
- '--env',
- dest='env',
- default=None,
- help=explanation)
-
-
-OPEN_BROWSER_OPTION = optparse.Option(
- '-o',
- '--open_browser',
- dest='open_browser',
- action='store_true',
- default=False,
- help='Open a browser window to the job page after a job mutation.')
-
-
-SHARDS_OPTION = optparse.Option(
- '--shards',
- type='string',
- dest='shards',
- default=None,
- action='callback',
- callback=parse_shards_into,
- help='A list of shard ids to act on. Can either be a comma-separated list (e.g. 0,1,2) '
- 'or a range (e.g. 0-2) or any combination of the two (e.g. 0-2,5,7-9). If not set, '
- 'all shards will be acted on.')
-
-
-FROM_JOBKEY_OPTION = optparse.Option('--from', dest='rename_from', type='string', default=None,
- metavar='CLUSTER/ROLE/ENV/JOB', action='callback', callback=parse_aurora_job_key_into,
- help='Job key to diff against.')
-
-
-JSON_OPTION = optparse.Option(
- '-j',
- '--json',
- dest='json',
- default=False,
- action='store_true',
- help='If specified, configuration is read in JSON format.')
-
-
-CLUSTER_CONFIG_OPTION = optparse.Option(
- '--cluster',
- dest='cluster',
- default=None,
- type='string',
- help='Cluster to match when selecting a job from a configuration. Optional if only one job '
- 'matching the given job name exists in the config.')
-
-
-CLUSTER_INVOKE_OPTION = optparse.Option(
- '--cluster',
- dest='cluster',
- default=None,
- type='string',
- help='Cluster to invoke this command against. Deprecated in favor of the CLUSTER/ROLE/ENV/NAME '
- 'syntax.')
-
-
-ENV_CONFIG_OPTION = make_env_option(
- 'Environment to match when selecting a job from a configuration.')
-
-
-# This is for binding arbitrary points in the Thermos namespace to specific strings, e.g.
-# if a Thermos configuration has {{jvm.version}}, it can be bound explicitly from the
-# command-line with, for example, -E jvm.version=7
-ENVIRONMENT_BIND_OPTION = optparse.Option(
- '-E',
- type='string',
- nargs=1,
- action='callback',
- default=[],
- metavar='NAME=VALUE',
- callback=add_binding_to('bindings'),
- dest='bindings',
- help='Bind a thermos mustache variable name to a value. '
- 'Multiple flags may be used to specify multiple values.')
-
-
-EXECUTOR_SANDBOX_OPTION = optparse.Option(
- '-e',
- '--executor_sandbox',
- action='store_true',
- default=False,
- dest='executor_sandbox',
- help='Run the command in the executor sandbox instead of the task sandbox.')
-
-
-SSH_USER_OPTION = optparse.Option(
- '--user',
- dest='ssh_user',
- default=None,
- help="ssh as this user instead of the role.")
-
-
-CREATE_STATES = (
- 'PENDING',
- 'RUNNING',
- 'FINISHED'
-)
-
-
-WAIT_UNTIL_OPTION = optparse.Option(
- '--wait_until',
- default='PENDING',
- type='choice',
- choices=('PENDING', 'RUNNING', 'FINISHED'),
- metavar='STATE',
- dest='wait_until',
- help='Block the client until all the tasks have transitioned into the '
- 'requested state. Options: %s. Default: %%default' % (', '.join(CREATE_STATES)))
-
-
-HEALTH_CHECK_INTERVAL_SECONDS_OPTION = optparse.Option(
- '--updater_health_check_interval_seconds',
- dest='health_check_interval_seconds',
- type=int,
- default=3,
- help='Time interval between subsequent shard status checks.'
-)
[46/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/HttpStatsFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/HttpStatsFilter.java b/src/main/java/com/twitter/aurora/scheduler/http/HttpStatsFilter.java
deleted file mode 100644
index a846383..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/HttpStatsFilter.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.io.IOException;
-
-import javax.servlet.FilterChain;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.servlet.http.HttpServletResponseWrapper;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-import com.twitter.common.net.http.filters.AbstractHttpFilter;
-import com.twitter.common.stats.SlidingStats;
-
-/**
- * An HTTP filter that exports counts and timing for requests based on response code.
- */
-public class HttpStatsFilter extends AbstractHttpFilter {
-
- private final LoadingCache<Integer, SlidingStats> counters = CacheBuilder.newBuilder()
- .build(new CacheLoader<Integer, SlidingStats>() {
- @Override public SlidingStats load(Integer status) {
- return new SlidingStats("http_" + status + "_responses", "nanos");
- }
- });
-
- private static class ResponseWithStatus extends HttpServletResponseWrapper {
- // 200 response code is the default if none is explicitly set.
- private int wrappedStatus = 200;
-
- ResponseWithStatus(HttpServletResponse resp) {
- super(resp);
- }
-
- @Override public void setStatus(int sc) {
- super.setStatus(sc);
- wrappedStatus = sc;
- }
-
- @Override public void setStatus(int sc, String sm) {
- super.setStatus(sc, sm);
- wrappedStatus = sc;
- }
- }
-
- @Override
- public void doFilter(HttpServletRequest request, HttpServletResponse response, FilterChain chain)
- throws IOException, ServletException {
-
- long start = System.nanoTime();
- ResponseWithStatus wrapper = new ResponseWithStatus(response);
- chain.doFilter(request, wrapper);
- counters.getUnchecked(wrapper.wrappedStatus).accumulate(System.nanoTime() - start);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/JerseyTemplateServlet.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/JerseyTemplateServlet.java b/src/main/java/com/twitter/aurora/scheduler/http/JerseyTemplateServlet.java
deleted file mode 100644
index 5c13de5..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/JerseyTemplateServlet.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.io.StringWriter;
-
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response;
-
-import org.antlr.stringtemplate.StringTemplate;
-
-import com.twitter.common.base.Closure;
-import com.twitter.common.util.templating.StringTemplateHelper;
-import com.twitter.common.util.templating.StringTemplateHelper.TemplateException;
-
-/**
- * Base class for common functions needed in a jersey stringtemplate servlet.
- */
-abstract class JerseyTemplateServlet {
-
- private final StringTemplateHelper templateHelper;
-
- JerseyTemplateServlet(String templatePath) {
- templateHelper = new StringTemplateHelper(getClass(), templatePath, true);
- }
-
- protected final Response fillTemplate(Closure<StringTemplate> populator) {
- StringWriter output = new StringWriter();
- try {
- templateHelper.writeTemplate(output, populator);
- } catch (TemplateException e) {
- throw new WebApplicationException(e);
- }
- return Response.ok(output.toString()).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirect.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirect.java b/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirect.java
deleted file mode 100644
index 20e4446..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirect.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.net.InetSocketAddress;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-import javax.servlet.http.HttpServletRequest;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.net.HostAndPort;
-import com.google.common.util.concurrent.Atomics;
-
-import com.twitter.common.application.modules.LocalServiceRegistry;
-import com.twitter.common.net.pool.DynamicHostSet;
-import com.twitter.common.net.pool.DynamicHostSet.HostChangeMonitor;
-import com.twitter.common.net.pool.DynamicHostSet.MonitorException;
-import com.twitter.thrift.Endpoint;
-import com.twitter.thrift.ServiceInstance;
-
-/**
- * Redirect logic for finding the leading scheduler in the event that this process is not the
- * leader.
- */
-public class LeaderRedirect {
-
- // TODO(wfarner): Should we tie this directly to the producer of the node (HttpModule? It seems
- // like the right thing to do, but would introduce an otherwise unnecessary dependency.
- @VisibleForTesting
- static final String HTTP_PORT_NAME = "http";
-
- private static final Logger LOG = Logger.getLogger(LeaderRedirect.class.getName());
-
- private final LocalServiceRegistry serviceRegistry;
- private final DynamicHostSet<ServiceInstance> schedulers;
-
- private final AtomicReference<ServiceInstance> leader = Atomics.newReference();
-
- @Inject
- LeaderRedirect(LocalServiceRegistry serviceRegistry, DynamicHostSet<ServiceInstance> schedulers) {
- this.serviceRegistry = Preconditions.checkNotNull(serviceRegistry);
- this.schedulers = Preconditions.checkNotNull(schedulers);
- }
-
- /**
- * Initiates the monitor that will watch the scheduler host set.
- *
- * @throws MonitorException If monitoring failed to initialize.
- */
- public void monitor() throws MonitorException {
- schedulers.watch(new SchedulerMonitor());
- }
-
- private Optional<HostAndPort> getLeaderHttp() {
- ServiceInstance leadingScheduler = leader.get();
- if (leadingScheduler == null) {
- return Optional.absent();
- }
-
- if (leadingScheduler.isSetAdditionalEndpoints()) {
- Endpoint leaderHttp = leadingScheduler.getAdditionalEndpoints().get(HTTP_PORT_NAME);
- if (leaderHttp != null && leaderHttp.isSetHost() && leaderHttp.isSetPort()) {
- return Optional.of(HostAndPort.fromParts(leaderHttp.getHost(), leaderHttp.getPort()));
- }
- }
-
- LOG.warning("Leader service instance seems to be incomplete: " + leadingScheduler);
- return Optional.absent();
- }
-
- private Optional<HostAndPort> getLocalHttp() {
- InetSocketAddress localHttp = serviceRegistry.getAuxiliarySockets().get(HTTP_PORT_NAME);
- return (localHttp == null) ? Optional.<HostAndPort>absent()
- : Optional.of(HostAndPort.fromParts(localHttp.getHostName(), localHttp.getPort()));
- }
-
- /**
- * Gets the optional HTTP endpoint that should be redirected to in the event that this
- * scheduler is not the leader.
- *
- * @return Optional redirect target.
- */
- @VisibleForTesting
- Optional<HostAndPort> getRedirect() {
- Optional<HostAndPort> leaderHttp = getLeaderHttp();
- Optional<HostAndPort> localHttp = getLocalHttp();
-
- if (leaderHttp.isPresent()) {
- if (leaderHttp.equals(localHttp)) {
- return Optional.absent();
- } else {
- return leaderHttp;
- }
- } else {
- LOG.info("No leader found, not redirecting.");
- return Optional.absent();
- }
- }
-
- /**
- * Gets the optional redirect URI target in the event that this process is not the leading
- * scheduler.
- *
- * @param req HTTP request.
- * @return An optional redirect destination to route the request to the leading scheduler.
- */
- public Optional<String> getRedirectTarget(HttpServletRequest req) {
- Optional<HostAndPort> redirectTarget = getRedirect();
- if (redirectTarget.isPresent()) {
- HostAndPort target = redirectTarget.get();
- StringBuilder redirect = new StringBuilder()
- .append(req.getScheme())
- .append("://")
- .append(target.getHostText())
- .append(":")
- .append(target.getPort())
- .append(req.getRequestURI());
-
- String queryString = req.getQueryString();
- if (queryString != null) {
- redirect.append("?").append(queryString);
- }
-
- return Optional.of(redirect.toString());
- } else {
- return Optional.absent();
- }
- }
-
- /**
- * Monitor to track scheduler leader changes.
- */
- private class SchedulerMonitor implements HostChangeMonitor<ServiceInstance> {
- @Override public void onChange(ImmutableSet<ServiceInstance> hostSet) {
- switch (hostSet.size()) {
- case 0:
- LOG.warning("No schedulers in host set, will not redirect despite not being leader.");
- leader.set(null);
- break;
-
- case 1:
- LOG.info("Found leader scheduler at " + hostSet);
- leader.set(Iterables.getOnlyElement(hostSet));
- break;
-
- default:
- LOG.severe("Multiple schedulers detected, will not redirect: " + hostSet);
- leader.set(null);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirectFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirectFilter.java b/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirectFilter.java
deleted file mode 100644
index 86f0567..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/LeaderRedirectFilter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.io.IOException;
-
-import javax.inject.Inject;
-import javax.servlet.FilterChain;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-
-import com.twitter.common.net.http.filters.AbstractHttpFilter;
-
-/**
- * An HTTP filter that will redirect the request to the leading scheduler.
- */
-public class LeaderRedirectFilter extends AbstractHttpFilter {
-
- private final LeaderRedirect redirector;
-
- @Inject
- LeaderRedirectFilter(LeaderRedirect redirector) {
- this.redirector = Preconditions.checkNotNull(redirector);
- }
-
- @Override
- public void doFilter(HttpServletRequest request, HttpServletResponse response, FilterChain chain)
- throws IOException, ServletException {
-
- Optional<String> leaderRedirect = redirector.getRedirectTarget(request);
- if (leaderRedirect.isPresent()) {
- response.sendRedirect(leaderRedirect.get());
- } else {
- chain.doFilter(request, response);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java b/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java
deleted file mode 100644
index fb71539..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
-import static com.twitter.aurora.gen.MaintenanceMode.SCHEDULED;
-
-/**
- * Servlet that exposes the maintenance state of hosts.
- */
-@Path("/maintenance")
-public class Maintenance {
- private final Storage storage;
-
- @Inject
- Maintenance(Storage storage) {
- this.storage = Preconditions.checkNotNull(storage);
- }
-
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- public Response getHosts() {
- return storage.weaklyConsistentRead(new Work.Quiet<Response>() {
- @Override public Response apply(StoreProvider storeProvider) {
- Multimap<MaintenanceMode, String> hostsByMode =
- Multimaps.transformValues(
- Multimaps.index(storeProvider.getAttributeStore().getHostAttributes(), GET_MODE),
- HOST_NAME);
-
- Map<MaintenanceMode, Object> hosts = Maps.newHashMap();
- hosts.put(DRAINED, ImmutableSet.copyOf(hostsByMode.get(DRAINED)));
- hosts.put(SCHEDULED, ImmutableSet.copyOf(hostsByMode.get(SCHEDULED)));
- hosts.put(DRAINING, getTasksByHosts(storeProvider, hostsByMode.get(DRAINING)).asMap());
- return Response.ok(hosts).build();
- }
- });
- }
-
- private Multimap<String, String> getTasksByHosts(StoreProvider provider, Iterable<String> hosts) {
- ImmutableSet.Builder<IScheduledTask> drainingTasks = ImmutableSet.builder();
- for (String host : hosts) {
- drainingTasks.addAll(provider.getTaskStore().fetchTasks(Query.slaveScoped(host).active()));
- }
- return Multimaps.transformValues(
- Multimaps.index(drainingTasks.build(), TASK_TO_HOST),
- Tasks.SCHEDULED_TO_ID);
- }
-
- private static final Function<IScheduledTask, String> TASK_TO_HOST =
- new Function<IScheduledTask, String>() {
- @Override public String apply(IScheduledTask task) {
- return task.getAssignedTask().getSlaveHost();
- }
- };
-
- private static final Function<HostAttributes, String> HOST_NAME =
- new Function<HostAttributes, String>() {
- @Override public String apply(HostAttributes attributes) {
- return attributes.getHost();
- }
- };
-
- private static final Function<HostAttributes, MaintenanceMode> GET_MODE =
- new Function<HostAttributes, MaintenanceMode>() {
- @Override public MaintenanceMode apply(HostAttributes attrs) {
- return attrs.getMode();
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/Mname.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Mname.java b/src/main/java/com/twitter/aurora/scheduler/http/Mname.java
deleted file mode 100644
index c97560f..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/Mname.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import javax.inject.Inject;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-import javax.ws.rs.core.UriBuilder;
-import javax.ws.rs.core.UriInfo;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-
-import static javax.ws.rs.core.Response.Status.NOT_FOUND;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.RUNNING;
-
-/**
- * Simple redirector from the canonical name of a task to its configured HTTP port.
- *
- * <p>Forwards for GET, PUT, POST and DELETE requests using HTTP 307 allowing compliant clients to
- * seamlessly perform re-directed mutations.
- */
-@Path("/mname")
-public class Mname {
-
- private static final Set<String> HTTP_PORT_NAMES = ImmutableSet.of(
- "health", "http", "HTTP", "web");
-
- private final Storage storage;
-
- @Inject
- public Mname(Storage storage) {
- this.storage = checkNotNull(storage);
- }
-
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response getUsage() {
- return Response
- .status(Status.BAD_REQUEST)
- .entity("<html>Usage: /mname/{role}/{env}/{job}/{instance}</html>")
- .build();
- }
-
- @GET
- @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
- @Produces(MediaType.TEXT_HTML)
- public Response getWithForwardRequest(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @PathParam("forward") String forward,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
- }
-
- @PUT
- @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
- @Produces(MediaType.TEXT_HTML)
- public Response putWithForwardRequest(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @PathParam("forward") String forward,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
- }
-
- @POST
- @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
- @Produces(MediaType.TEXT_HTML)
- public Response postWithForwardRequest(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @PathParam("forward") String forward,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
- }
-
- @DELETE
- @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
- @Produces(MediaType.TEXT_HTML)
- public Response deleteWithForwardRequest(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @PathParam("forward") String forward,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
- }
-
- @GET
- @Path("/{role}/{env}/{job}/{instance}")
- @Produces(MediaType.TEXT_HTML)
- public Response get(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
- }
-
- @PUT
- @Path("/{role}/{env}/{job}/{instance}")
- @Produces(MediaType.TEXT_HTML)
- public Response put(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
- }
-
- @POST
- @Path("/{role}/{env}/{job}/{instance}")
- @Produces(MediaType.TEXT_HTML)
- public Response post(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
- }
-
- @DELETE
- @Path("/{role}/{env}/{job}/{instance}")
- @Produces(MediaType.TEXT_HTML)
- public Response delete(
- @PathParam("role") String role,
- @PathParam("env") String env,
- @PathParam("job") String job,
- @PathParam("instance") int instanceId,
- @Context UriInfo uriInfo) {
-
- return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
- }
-
- private Response get(
- String role,
- String env,
- String job,
- int instanceId,
- UriInfo uriInfo,
- Optional<String> forwardRequest) {
-
- IScheduledTask task = Iterables.getOnlyElement(
- Storage.Util.consistentFetchTasks(storage,
- Query.instanceScoped(JobKeys.from(role, env, job), instanceId).active()),
- null);
- if (task == null) {
- return respond(NOT_FOUND, "No such live instance found.");
- }
-
- if (task.getStatus() != RUNNING) {
- return respond(NOT_FOUND, "The selected instance is currently in state " + task.getStatus());
- }
-
- IAssignedTask assignedTask = task.getAssignedTask();
- Optional<Integer> port = getRedirectPort(assignedTask);
- if (!port.isPresent()) {
- return respond(NOT_FOUND, "The task does not have a registered http port.");
- }
-
- UriBuilder redirect = UriBuilder
- .fromPath(forwardRequest.or("/"))
- .scheme("http")
- .host(assignedTask.getSlaveHost())
- .port(port.get());
- for (Entry<String, List<String>> entry : uriInfo.getQueryParameters().entrySet()) {
- for (String value : entry.getValue()) {
- redirect.queryParam(entry.getKey(), value);
- }
- }
-
- return Response.temporaryRedirect(redirect.build()).build();
- }
-
- @VisibleForTesting
- static Optional<Integer> getRedirectPort(IAssignedTask task) {
- Map<String, Integer> ports = task.isSetAssignedPorts()
- ? task.getAssignedPorts() : ImmutableMap.<String, Integer>of();
- for (String httpPortName : HTTP_PORT_NAMES) {
- Integer port = ports.get(httpPortName);
- if (port != null) {
- return Optional.of(port);
- }
- }
- return Optional.absent();
- }
-
- private Response respond(Status status, String message) {
- return Response.status(status).entity(message).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/Offers.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Offers.java b/src/main/java/com/twitter/aurora/scheduler/http/Offers.java
deleted file mode 100644
index e90f1cc..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/Offers.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.mesos.Protos.Attribute;
-import org.apache.mesos.Protos.ExecutorID;
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.Resource;
-import org.apache.mesos.Protos.Value.Range;
-
-import com.twitter.aurora.scheduler.async.OfferQueue;
-
-/**
- * Servlet that exposes resource offers that the scheduler is currently retaining.
- */
-@Path("/offers")
-public class Offers {
-
- private final OfferQueue offerQueue;
-
- @Inject
- Offers(OfferQueue offerQueue) {
- this.offerQueue = Preconditions.checkNotNull(offerQueue);
- }
-
- /**
- * Dumps the offers queued in the scheduler.
- *
- * @return HTTP response.
- */
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- public Response getOffers() {
- return Response.ok(
- FluentIterable.from(offerQueue.getOffers()).transform(TO_BEAN).toList()).build();
- }
-
- private static final Function<ExecutorID, String> EXECUTOR_ID_TOSTRING =
- new Function<ExecutorID, String>() {
- @Override public String apply(ExecutorID id) {
- return id.getValue();
- }
- };
-
- private static final Function<Range, Object> RANGE_TO_BEAN = new Function<Range, Object>() {
- @Override public Object apply(Range range) {
- return range.getBegin() + "-" + range.getEnd();
- }
- };
-
- private static final Function<Attribute, Object> ATTRIBUTE_TO_BEAN =
- new Function<Attribute, Object>() {
- @Override public Object apply(Attribute attr) {
- ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
- builder.put("name", attr.getName());
- if (attr.hasScalar()) {
- builder.put("scalar", attr.getScalar().getValue());
- }
- if (attr.hasRanges()) {
- builder.put("ranges", immutable(attr.getRanges().getRangeList(), RANGE_TO_BEAN));
- }
- if (attr.hasSet()) {
- builder.put("set", attr.getSet().getItemList());
- }
- if (attr.hasText()) {
- builder.put("text", attr.getText().getValue());
- }
- return builder.build();
- }
- };
-
- private static final Function<Resource, Object> RESOURCE_TO_BEAN =
- new Function<Resource, Object>() {
- @Override public Object apply(Resource resource) {
- ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
- builder.put("name", resource.getName());
- if (resource.hasScalar()) {
- builder.put("scalar", resource.getScalar().getValue());
- }
- if (resource.hasRanges()) {
- builder.put("ranges", immutable(resource.getRanges().getRangeList(), RANGE_TO_BEAN));
- }
- if (resource.hasSet()) {
- builder.put("set", resource.getSet().getItemList());
- }
- return builder.build();
- }
- };
-
- private static <A, B> Iterable<B> immutable(Iterable<A> iterable, Function<A, B> transform) {
- return FluentIterable.from(iterable).transform(transform).toList();
- }
-
- private static final Function<Offer, Map<String, ?>> TO_BEAN =
- new Function<Offer, Map<String, ?>>() {
- @Override public Map<String, ?> apply(Offer offer) {
- return ImmutableMap.<String, Object>builder()
- .put("id", offer.getId().getValue())
- .put("framework_id", offer.getFrameworkId().getValue())
- .put("slave_id", offer.getSlaveId().getValue())
- .put("hostname", offer.getHostname())
- .put("resources", immutable(offer.getResourcesList(), RESOURCE_TO_BEAN))
- .put("attributes", immutable(offer.getAttributesList(), ATTRIBUTE_TO_BEAN))
- .put("executor_ids", immutable(offer.getExecutorIdsList(), EXECUTOR_ID_TOSTRING))
- .build();
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/PendingTasks.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/PendingTasks.java b/src/main/java/com/twitter/aurora/scheduler/http/PendingTasks.java
deleted file mode 100644
index 359fb97..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/PendingTasks.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.base.Preconditions;
-
-import com.twitter.aurora.scheduler.async.TaskGroups;
-
-/**
- * Servlet that exposes detailed information about tasks that are pending.
- */
-@Path("/pendingtasks")
-public class PendingTasks {
-
- private final TaskGroups taskGroups;
-
- @Inject
- PendingTasks(TaskGroups taskGroups) {
- this.taskGroups = Preconditions.checkNotNull(taskGroups);
- }
-
- /**
- * Returns information about pending tasks.
- *
- * @return HTTP response.
- */
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- public Response getOffers() {
- return Response.ok(taskGroups.getGroups()).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/Quotas.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Quotas.java b/src/main/java/com/twitter/aurora/scheduler/http/Quotas.java
deleted file mode 100644
index 1183275..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/Quotas.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-
-import org.codehaus.jackson.annotate.JsonProperty;
-
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-
-/**
- * Servlet that exposes allocated resource quotas.
- */
-@Path("/quotas")
-public class Quotas {
-
- private final Storage storage;
-
- @Inject
- Quotas(Storage storage) {
- this.storage = Preconditions.checkNotNull(storage);
- }
-
- /**
- * Dumps allocated resource quotas.
- *
- * @return HTTP response.
- */
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- public Response getOffers(@QueryParam("role") final String role) {
- return storage.weaklyConsistentRead(new Work.Quiet<Response>() {
- @Override public Response apply(StoreProvider storeProvider) {
- Map<String, IQuota> quotas;
- if (role == null) {
- quotas = storeProvider.getQuotaStore().fetchQuotas();
- } else {
- Optional<IQuota> quota = storeProvider.getQuotaStore().fetchQuota(role);
- if (quota.isPresent()) {
- quotas = ImmutableMap.of(role, quota.get());
- } else {
- quotas = ImmutableMap.of();
- }
- }
-
- return Response.ok(Maps.transformValues(quotas, TO_BEAN)).build();
- }
- });
- }
-
- private static final Function<IQuota, QuotaBean> TO_BEAN = new Function<IQuota, QuotaBean>() {
- @Override public QuotaBean apply(IQuota quota) {
- return new QuotaBean(quota.getNumCpus(), quota.getRamMb(), quota.getDiskMb());
- }
- };
-
- private static final class QuotaBean {
- private final double cpu;
- private final long ramMb;
- private final long diskMb;
-
- private QuotaBean(double cpu, long ramMb, long diskMb) {
- this.cpu = cpu;
- this.ramMb = ramMb;
- this.diskMb = diskMb;
- }
-
- @JsonProperty("cpu_cores")
- public double getCpu() {
- return cpu;
- }
-
- @JsonProperty("ram_mb")
- public long getRamMb() {
- return ramMb;
- }
-
- @JsonProperty("disk_mb")
- public long getDiskMb() {
- return diskMb;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzHome.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzHome.java b/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzHome.java
deleted file mode 100644
index 07a3259..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzHome.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Set;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.base.Function;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-
-import org.antlr.stringtemplate.StringTemplate;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.state.CronJobManager;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.base.Closure;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * HTTP interface to serve as a HUD for the aurora scheduler.
- */
-@Path("/scheduler")
-public class SchedulerzHome extends JerseyTemplateServlet {
-
- private static final Function<String, Role> CREATE_ROLE = new Function<String, Role>() {
- @Override public Role apply(String ownerRole) {
- Role role = new Role();
- role.role = ownerRole;
- return role;
- }
- };
-
- private final Storage storage;
- private final CronJobManager cronScheduler;
- private final String clusterName;
-
- /**
- * Creates a new scheduler home servlet.
- *
- * @param storage Backing store to fetch tasks from.
- * @param cronScheduler Cron scheduler.
- * @param clusterName Name of the serving cluster.
- */
- @Inject
- public SchedulerzHome(
- Storage storage,
- CronJobManager cronScheduler,
- @ClusterName String clusterName) {
-
- super("schedulerzhome");
- this.storage = checkNotNull(storage);
- this.cronScheduler = checkNotNull(cronScheduler);
- this.clusterName = checkNotBlank(clusterName);
- }
-
- /**
- * Fetches the scheduler landing page.
- *
- * @return HTTP response.
- */
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response get() {
- return fillTemplate(new Closure<StringTemplate>() {
- @Override public void execute(StringTemplate template) {
- template.setAttribute("cluster_name", clusterName);
-
- LoadingCache<String, Role> owners =
- CacheBuilder.newBuilder().build(CacheLoader.from(CREATE_ROLE));
-
- // TODO(William Farner): Render this page without an expensive query.
- Set<IScheduledTask> tasks =
- Storage.Util.weaklyConsistentFetchTasks(storage, Query.unscoped());
- for (ITaskConfig task : Iterables.transform(tasks, Tasks.SCHEDULED_TO_INFO)) {
- owners.getUnchecked(task.getOwner().getRole()).accumulate(task);
- }
-
- // Add cron job counts for each role.
- for (IJobConfiguration job : cronScheduler.getJobs()) {
- owners.getUnchecked(job.getOwner().getRole()).accumulate(job);
- }
-
- template.setAttribute(
- "owners",
- DisplayUtils.ROLE_ORDERING.sortedCopy(owners.asMap().values()));
- }
- });
- }
-
- /**
- * Template object to represent a role.
- */
- static class Role {
- private String role;
- private Set<String> jobs = Sets.newHashSet();
- private Set<String> cronJobs = Sets.newHashSet();
-
- private void accumulate(ITaskConfig task) {
- jobs.add(task.getJobName());
- }
-
- private void accumulate(IJobConfiguration job) {
- cronJobs.add(job.getKey().getName());
- }
-
- public String getRole() {
- return role;
- }
-
- public int getJobCount() {
- return jobs.size();
- }
-
- public int getCronJobCount() {
- return cronJobs.size();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java b/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java
deleted file mode 100644
index 12b0bec..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzJob.java
+++ /dev/null
@@ -1,432 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.HashBiMap;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-import com.google.common.collect.Ordering;
-
-import org.antlr.stringtemplate.StringTemplate;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.apiConstants;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
-import com.twitter.aurora.scheduler.metadata.NearestFit;
-import com.twitter.aurora.scheduler.state.CronJobManager;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IConstraint;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConstraint;
-import com.twitter.aurora.scheduler.storage.entities.ITaskEvent;
-import com.twitter.common.base.Closure;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.ASSIGNED;
-import static com.twitter.aurora.gen.ScheduleStatus.FAILED;
-import static com.twitter.aurora.gen.ScheduleStatus.FINISHED;
-import static com.twitter.aurora.gen.ScheduleStatus.KILLED;
-import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
-import static com.twitter.aurora.gen.ScheduleStatus.LOST;
-import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
-import static com.twitter.aurora.gen.ScheduleStatus.RUNNING;
-import static com.twitter.aurora.gen.ScheduleStatus.STARTING;
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * HTTP interface to view information about a job in the aurora scheduler.
- */
-@Path("/scheduler/{role}/{environment}/{job}")
-public class SchedulerzJob extends JerseyTemplateServlet {
- private static final String STATUS_FILTER_PARAM = "status";
- private static final String ADMIN_VIEW_PARAM = "admin";
-
- // Pagination controls.
- private static final String OFFSET_PARAM = "o";
- private static final int PAGE_SIZE = 50;
-
- private static final Ordering<IScheduledTask> INSTANCE_ID_COMPARATOR =
- Ordering.natural().onResultOf(Tasks.SCHEDULED_TO_INSTANCE_ID);
-
- private static final Map<ScheduleStatus, Set<ScheduleStatus>> FILTER_MAP =
- ImmutableMap.<ScheduleStatus, Set<ScheduleStatus>>builder()
- .put(PENDING, EnumSet.of(PENDING))
- .put(RUNNING, EnumSet.of(ASSIGNED, STARTING, RUNNING, KILLING))
- .put(FINISHED, EnumSet.of(KILLED, FINISHED))
- .put(FAILED, EnumSet.of(LOST, FAILED))
- .build();
-
- private static final Comparator<IScheduledTask> REVERSE_CHRON_COMPARATOR =
- new Comparator<IScheduledTask>() {
- @Override public int compare(IScheduledTask taskA, IScheduledTask taskB) {
- // Sort in reverse chronological order.
- Iterable<ITaskEvent> taskAEvents = taskA.getTaskEvents();
- Iterable<ITaskEvent> taskBEvents = taskB.getTaskEvents();
-
- boolean taskAHasEvents = taskAEvents != null && !Iterables.isEmpty(taskAEvents);
- boolean taskBHasEvents = taskBEvents != null && !Iterables.isEmpty(taskBEvents);
- if (taskAHasEvents && taskBHasEvents) {
- return Long.signum(Iterables.getLast(taskBEvents).getTimestamp()
- - Iterables.getLast(taskAEvents).getTimestamp());
- } else {
- return 0;
- }
- }
- };
-
- private static final Function<Veto, String> GET_REASON = new Function<Veto, String>() {
- @Override public String apply(Veto veto) {
- return veto.getReason();
- }
- };
-
- // Double percents to escape formatting sequence.
- private static final String PORT_FORMAT = "%%port:%s%%";
- // TODO(William Farner): Search for usage of this, figure out a deprecation strategy to switch
- // to %instance_id%.
- private static final String INSTANCE_ID_REGEXP = "%shard_id%";
- private static final String TASK_ID_REGEXP = "%task_id%";
- private static final String HOST_REGEXP = "%host%";
-
- private static String expandText(String value, IAssignedTask task) {
- String expanded = value
- .replaceAll(INSTANCE_ID_REGEXP, String.valueOf(task.getInstanceId()))
- .replaceAll(TASK_ID_REGEXP, task.getTaskId());
-
- if (task.isSetSlaveHost()) {
- expanded = expanded.replaceAll(HOST_REGEXP, task.getSlaveHost());
- }
-
- // Expand ports.
- if (task.isSetAssignedPorts()) {
- for (Map.Entry<String, Integer> portEntry : task.getAssignedPorts().entrySet()) {
- expanded = expanded.replaceAll(
- String.format(PORT_FORMAT, portEntry.getKey()),
- String.valueOf(portEntry.getValue()));
- }
- }
-
- return expanded;
- }
-
- private final Function<IScheduledTask, Map<String, Object>> taskToStringMap =
- new Function<IScheduledTask, Map<String, Object>>() {
- @Override public Map<String, Object> apply(IScheduledTask scheduledTask) {
- final IAssignedTask task = scheduledTask.getAssignedTask();
- ImmutableMap.Builder<String, Object> builder = ImmutableMap.<String, Object>builder()
- .put("taskId", task.getTaskId())
- .put("instanceId", task.getInstanceId())
- .put("slaveHost", task.isSetSlaveHost() ? task.getSlaveHost() : "")
- .put("status", scheduledTask.getStatus())
- .put("statusTimestamp", Iterables.getLast(scheduledTask.getTaskEvents()).getTimestamp())
- .put("taskEvents", scheduledTask.getTaskEvents());
-
- if (scheduledTask.getStatus() == ScheduleStatus.PENDING) {
- String pendingReason;
- Set<Veto> vetoes = nearestFit.getNearestFit(task.getTaskId());
- if (vetoes.isEmpty()) {
- pendingReason = "No matching hosts.";
- } else {
- pendingReason = Joiner.on(",").join(Iterables.transform(vetoes, GET_REASON));
- }
- builder.put("pendingReason", pendingReason);
- }
-
- Function<String, String> expander = new Function<String, String>() {
- @Override public String apply(String input) {
- return expandText(input, task);
- }
- };
-
- Map<String, String> links = ImmutableMap.of();
- if (apiConstants.LIVE_STATES.contains(scheduledTask.getStatus())) {
- links =
- ImmutableMap.copyOf(Maps.transformValues(task.getTask().getTaskLinks(), expander));
- }
- builder.put("links", links);
- builder.put("executorPort", 1338);
- if (task.isSetSlaveHost()) {
- builder.put("executorUri",
- "http://" + task.getSlaveHost() + ":1338/task/" + task.getTaskId());
- }
- return builder.build();
- }
- };
-
- private final Storage storage;
- private final String clusterName;
- private final NearestFit nearestFit;
- private final CronJobManager cronJobManager;
-
- /**
- * Creates a new job servlet.
- *
- * @param storage Backing store to fetch tasks from.
- * @param clusterName Name of the serving cluster.
- */
- @Inject
- public SchedulerzJob(
- Storage storage,
- CronJobManager cronJobManager,
- @ClusterName String clusterName,
- NearestFit nearestFit) {
-
- super("schedulerzjob");
- this.storage = checkNotNull(storage);
- this.clusterName = checkNotBlank(clusterName);
- this.nearestFit = checkNotNull(nearestFit);
- this.cronJobManager = checkNotNull(cronJobManager);
- }
-
- private static <T> Iterable<T> offsetAndLimit(Iterable<T> iterable, int offset) {
- return ImmutableList.copyOf(Iterables.limit(Iterables.skip(iterable, offset), PAGE_SIZE));
- }
-
- private static String scaleMb(long mb) {
- return (mb >= 1024) ? ((mb / 1024) + " GiB") : (mb + " MiB");
- }
-
- private static final Function<IConstraint, String> DISPLAY_CONSTRAINT =
- new Function<IConstraint, String>() {
- @Override public String apply(IConstraint constraint) {
- StringBuilder sb = new StringBuilder().append(constraint.getName()).append(": ");
- ITaskConstraint taskConstraint = constraint.getConstraint();
- switch (taskConstraint.getSetField()) {
- case VALUE:
- if (taskConstraint.getValue().isNegated()) {
- sb.append("not ");
- }
- sb.append(Joiner.on(", ").join(taskConstraint.getValue().getValues()));
- break;
-
- case LIMIT:
- sb.append("limit ").append(taskConstraint.getLimit().getLimit());
- break;
-
- default:
- sb.append("Unhandled constraint type " + taskConstraint.getSetField());
- }
-
- return sb.toString();
- }
- };
-
- private static final Function<ITaskConfig, SchedulingDetails> CONFIG_TO_DETAILS =
- new Function<ITaskConfig, SchedulingDetails>() {
- @Override public SchedulingDetails apply(ITaskConfig task) {
- String resources = Joiner.on(", ").join(
- "cpu: " + task.getNumCpus(),
- "ram: " + scaleMb(task.getRamMb()),
- "disk: " + scaleMb(task.getDiskMb()));
- ImmutableMap.Builder<String, Object> details = ImmutableMap.<String, Object>builder()
- .put("resources", resources);
- if (!task.getConstraints().isEmpty()) {
- Iterable<String> displayConstraints = FluentIterable.from(task.getConstraints())
- .transform(DISPLAY_CONSTRAINT)
- .toSortedList(Ordering.<String>natural());
- details.put("constraints", Joiner.on(", ").join(displayConstraints));
- }
- if (task.isIsService()) {
- details.put("service", "true");
- }
- if (task.isProduction()) {
- details.put("production", "true");
- }
- if (!task.getRequestedPorts().isEmpty()) {
- details.put("ports",
- Joiner.on(", ").join(ImmutableSortedSet.copyOf(task.getRequestedPorts())));
- }
- if (!task.getPackages().isEmpty()) {
- List<String> packages = Ordering.natural().sortedCopy(
- Iterables.transform(task.getPackages(), TransformationUtils.PACKAGE_TOSTRING));
- details.put(
- "packages",
- Joiner.on(',').join(packages));
- }
- details.put("contact", task.isSetContactEmail() ? task.getContactEmail() : "none");
- return new SchedulingDetails(details.build());
- }
- };
-
- static class SchedulingDetails {
- private final Map<String, Object> details;
-
- SchedulingDetails(ImmutableMap<String, Object> details) {
- this.details = details;
- }
-
- public Map<String, Object> getDetails() {
- return details;
- }
-
- @Override
- public int hashCode() {
- return details.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof SchedulingDetails)) {
- return false;
- }
-
- SchedulingDetails other = (SchedulingDetails) o;
- return other.details.equals(details);
- }
- }
-
-
-
- private static Map<String, SchedulingDetails> buildSchedulingTable(
- Iterable<IAssignedTask> tasks) {
-
- Map<Integer, ITaskConfig> byInstance = Maps.transformValues(
- Maps.uniqueIndex(tasks, Tasks.ASSIGNED_TO_INSTANCE_ID),
- Tasks.ASSIGNED_TO_INFO);
- Map<Integer, SchedulingDetails> detailsByInstance =
- Maps.transformValues(byInstance, CONFIG_TO_DETAILS);
- Multimap<SchedulingDetails, Integer> instancesByDetails = Multimaps.invertFrom(
- Multimaps.forMap(detailsByInstance), HashMultimap.<SchedulingDetails, Integer>create());
- Map<SchedulingDetails, String> instanceStringsByDetails =
- Maps.transformValues(instancesByDetails.asMap(), TransformationUtils.INSTANCES_TOSTRING);
- return HashBiMap.create(instanceStringsByDetails).inverse();
- }
-
- /**
- * Fetches the landing page for a job within a role.
- *
- * @return HTTP response.
- */
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response get(
- @PathParam("role") final String role,
- @PathParam("environment") final String environment,
- @PathParam("job") final String job,
- @QueryParam(OFFSET_PARAM) final int offset,
- @QueryParam(STATUS_FILTER_PARAM) final String filterArg,
- @QueryParam(ADMIN_VIEW_PARAM) final String adminView) {
-
- return fillTemplate(new Closure<StringTemplate>() {
- @Override public void execute(StringTemplate template) {
- template.setAttribute("cluster_name", clusterName);
- template.setAttribute(ADMIN_VIEW_PARAM, adminView != null);
- IJobKey jobKey = JobKeys.from(role, environment, job);
-
- boolean isCron = cronJobManager.hasJob(jobKey);
- template.setAttribute("is_cron", isCron);
-
- ScheduleStatus statusFilter = null;
- if (filterArg != null) {
- template.setAttribute(STATUS_FILTER_PARAM, filterArg);
-
- try {
- statusFilter = ScheduleStatus.valueOf(filterArg.toUpperCase());
- } catch (IllegalArgumentException e) {
- template.setAttribute("exception", "Invalid status type: " + filterArg);
- return;
- }
- }
-
- template.setAttribute("role", role);
- template.setAttribute("environment", environment);
- template.setAttribute("job", job);
- template.setAttribute("statsUrl", DisplayUtils.getJobDashboardUrl(jobKey));
- boolean hasMore = false;
-
- Query.Builder builder = Query.jobScoped(JobKeys.from(role, environment, job));
-
- Optional<Query.Builder> activeQuery = Optional.absent();
- Optional<Query.Builder> completedQuery = Optional.absent();
- if (statusFilter != null) {
- Collection<ScheduleStatus> queryStatuses = FILTER_MAP.get(statusFilter);
- if (Tasks.isActive(statusFilter)) {
- activeQuery = Optional.of(builder.byStatus(queryStatuses));
- } else {
- completedQuery = Optional.of(builder.byStatus(queryStatuses));
- }
- } else {
- activeQuery = Optional.of(builder.active());
- completedQuery = Optional.of(builder.terminal());
- }
-
- if (activeQuery.isPresent()) {
- Set<IScheduledTask> activeTasks =
- Storage.Util.weaklyConsistentFetchTasks(storage, activeQuery.get());
- List<IScheduledTask> liveTasks = INSTANCE_ID_COMPARATOR.sortedCopy(activeTasks);
- template.setAttribute("activeTasks",
- ImmutableList.copyOf(
- Iterables.transform(offsetAndLimit(liveTasks, offset), taskToStringMap)));
- hasMore = hasMore || (liveTasks.size() > (offset + PAGE_SIZE));
- template.setAttribute("schedulingDetails",
- buildSchedulingTable(Iterables.transform(liveTasks, Tasks.SCHEDULED_TO_ASSIGNED)));
- }
- if (completedQuery.isPresent()) {
- List<IScheduledTask> completedTasks = Lists.newArrayList(
- Storage.Util.weaklyConsistentFetchTasks(storage, completedQuery.get()));
- Collections.sort(completedTasks, REVERSE_CHRON_COMPARATOR);
- template.setAttribute("completedTasks",
- ImmutableList.copyOf(
- Iterables.transform(offsetAndLimit(completedTasks, offset), taskToStringMap)));
- hasMore = hasMore || (completedTasks.size() > (offset + PAGE_SIZE));
- }
-
- template.setAttribute("offset", offset);
- if (offset > 0) {
- template.setAttribute("prevOffset", Math.max(0, offset - PAGE_SIZE));
- }
- if (hasMore) {
- template.setAttribute("nextOffset", offset + PAGE_SIZE);
- }
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java b/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java
deleted file mode 100644
index 756c672..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/SchedulerzRole.java
+++ /dev/null
@@ -1,393 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.Sets;
-
-import org.antlr.stringtemplate.StringTemplate;
-
-import com.twitter.aurora.gen.CronCollisionPolicy;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.gen.apiConstants;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.cron.CronPredictor;
-import com.twitter.aurora.scheduler.quota.QuotaManager;
-import com.twitter.aurora.scheduler.quota.Quotas;
-import com.twitter.aurora.scheduler.state.CronJobManager;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.base.Closure;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.scheduler.base.Tasks.GET_STATUS;
-import static com.twitter.aurora.scheduler.base.Tasks.LATEST_ACTIVITY;
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * HTTP interface to provide information about jobs for a specific role.
- */
-@Path("/scheduler/{role}")
-public class SchedulerzRole extends JerseyTemplateServlet {
-
- private static final List<ScheduleStatus> STATUSES = ImmutableList.<ScheduleStatus>builder()
- .addAll(apiConstants.TERMINAL_STATES)
- .addAll(apiConstants.ACTIVE_STATES)
- .build();
-
- // The freshest task is the latest active task
- // or the latest inactive task if no active task exists.
- private static final Ordering<IScheduledTask> FRESH_TASK_ORDER =
- Ordering.explicit(STATUSES).onResultOf(GET_STATUS).compound(LATEST_ACTIVITY);
-
- @VisibleForTesting
- static IScheduledTask getFreshestTask(Iterable<IScheduledTask> tasks) {
- return FRESH_TASK_ORDER.max(tasks);
- }
-
- private final Storage storage;
- private final CronJobManager cronJobManager;
- private final CronPredictor cronPredictor;
- private final String clusterName;
- private final QuotaManager quotaManager;
-
- @Inject
- SchedulerzRole(
- Storage storage,
- CronJobManager cronJobManager,
- CronPredictor cronPredictor,
- @ClusterName String clusterName,
- QuotaManager quotaManager) {
-
- super("schedulerzrole");
- this.storage = checkNotNull(storage);
- this.cronJobManager = checkNotNull(cronJobManager);
- this.cronPredictor = checkNotNull(cronPredictor);
- this.clusterName = checkNotBlank(clusterName);
- this.quotaManager = checkNotNull(quotaManager);
- }
-
- /**
- * Fetches the landing page for a role.
- *
- * @return HTTP response.
- */
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response get(@PathParam("role") final String role) {
- return processRequest(Optional.of(role), Optional.<String>absent());
- }
-
- private Response processRequest(final Optional<String> role, final Optional<String> environment) {
- return fillTemplate(new Closure<StringTemplate>() {
- @Override public void execute(StringTemplate template) {
-
- if (!role.isPresent()) {
- template.setAttribute("exception", "Please specify a user.");
- return;
- }
-
- Map<IJobKey, Map<?, ?>> cronJobs = fetchCronJobsBy(role.get(), environment);
- List<Job> jobs = fetchJobsBy(role.get(), environment, cronJobs);
- if (jobs.isEmpty() && cronJobs.isEmpty()) {
- String msg = "No jobs found for role " + role.get()
- + (environment.isPresent() ? (" and environment " + environment.get()) : "");
- throw new WebApplicationException(Response.status(Status.NOT_FOUND).entity(msg).build());
- }
-
- template.setAttribute("cluster_name", clusterName);
- template.setAttribute("role", role.get());
- template.setAttribute("environment", environment.orNull());
- template.setAttribute("jobs", jobs);
- template.setAttribute("cronJobs", cronJobs.values());
-
- // TODO(Suman Karumuri): In future compute consumption for role and environment.
- template.setAttribute("prodResourcesUsed", quotaManager.getConsumption(role.get()));
- template.setAttribute("nonProdResourcesUsed", getNonProdConsumption(role.get()));
- template.setAttribute("resourceQuota", getQuota(role.get()));
- }
- });
- }
-
- private IQuota getQuota(final String role) {
- return Storage.Util.consistentFetchQuota(storage, role).or(Quotas.noQuota());
- }
-
- private IQuota getNonProdConsumption(String role) {
- FluentIterable<ITaskConfig> tasks = FluentIterable
- .from(Storage.Util.weaklyConsistentFetchTasks(storage, Query.roleScoped(role).active()))
- .transform(Tasks.SCHEDULED_TO_INFO)
- .filter(Predicates.not(Tasks.IS_PRODUCTION));
-
- return Quotas.fromTasks(tasks);
- }
-
- /**
- * Display jobs for a role and environment.
- */
- @Path("/{environment}")
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response get(
- @PathParam("role") final String role,
- @PathParam("environment") final String environment) {
-
- Optional<String> env = Optional.of(environment);
- if (env.isPresent() && env.get().isEmpty()) {
- env = Optional.absent();
- }
-
- return processRequest(Optional.of(role), env);
- }
-
- private Map<IJobKey, Map<?, ?>> fetchCronJobsBy(
- final String role,
- final Optional<String> environment) {
-
- Predicate<IJobConfiguration> byRoleEnv = new Predicate<IJobConfiguration>() {
- @Override public boolean apply(IJobConfiguration job) {
- boolean roleMatch = job.getOwner().getRole().equals(role);
- boolean envMatch = !environment.isPresent()
- || job.getKey().getEnvironment().equals(environment.get());
- return roleMatch && envMatch;
- }
- };
-
- Iterable<IJobConfiguration> jobs = FluentIterable
- .from(cronJobManager.getJobs())
- .filter(byRoleEnv);
-
- return Maps.transformValues(Maps.uniqueIndex(jobs, JobKeys.FROM_CONFIG),
- new Function<IJobConfiguration, Map<?, ?>>() {
- @Override public Map<?, ?> apply(IJobConfiguration job) {
- return ImmutableMap.<Object, Object>builder()
- .put("jobKey", job.getKey())
- .put("name", job.getKey().getName())
- .put("environment", job.getKey().getEnvironment())
- .put("pendingTaskCount", job.getInstanceCount())
- .put("cronSchedule", job.getCronSchedule())
- .put("nextRun", cronPredictor.predictNextRun(job.getCronSchedule()).getTime())
- .put("cronCollisionPolicy", cronCollisionPolicy(job))
- .put("packages", getPackages(job))
- .build();
- }
- });
- }
-
- private static CronCollisionPolicy cronCollisionPolicy(IJobConfiguration jobConfiguration) {
- return CronJobManager.orDefault(jobConfiguration.getCronCollisionPolicy());
- }
-
- private static String getPackages(IJobConfiguration job) {
- Set<String> packages = Sets.newHashSet();
-
- // Insert all packages for all tasks in the set to eliminate duplicates
- ITaskConfig task = job.getTaskConfig();
- if (!task.getPackages().isEmpty()) {
- packages.addAll(Lists.newArrayList(
- Iterables.transform(task.getPackages(), TransformationUtils.PACKAGE_TOSTRING)));
- }
- return Joiner.on(',').join(packages);
- }
-
- private List<Job> fetchJobsBy(
- final String role,
- final Optional<String> environment,
- final Map<IJobKey, Map<?, ?>> cronJobs) {
-
- final Function<Map.Entry<IJobKey, Collection<IScheduledTask>>, Job> toJob =
- new Function<Map.Entry<IJobKey, Collection<IScheduledTask>>, Job>() {
- @Override public Job apply(Map.Entry<IJobKey, Collection<IScheduledTask>> tasksByJobKey) {
- IJobKey jobKey = tasksByJobKey.getKey();
- Collection<IScheduledTask> tasks = tasksByJobKey.getValue();
-
- Job job = new Job();
- job.environment = jobKey.getEnvironment();
- job.name = jobKey.getName();
-
- // Pick the freshest task's config and associate it with the job.
- ITaskConfig freshestConfig = getFreshestTask(tasks).getAssignedTask().getTask();
- job.production = freshestConfig.isProduction();
-
- // TODO(Suman Karumuri): Add a source/job type to TaskConfig and replace logic below
- if (freshestConfig.isIsService()) {
- job.type = JobType.SERVICE;
- } else if (cronJobs.containsKey(jobKey)) {
- job.type = JobType.CRON;
- } else {
- job.type = JobType.ADHOC;
- }
-
- for (IScheduledTask task : tasks) {
- switch (task.getStatus()) {
- case INIT:
- case PENDING:
- job.pendingTaskCount++;
- break;
-
- case ASSIGNED:
- case STARTING:
- case RESTARTING:
- case RUNNING:
- case KILLING:
- case PREEMPTING:
- job.activeTaskCount++;
- break;
-
- case KILLED:
- case FINISHED:
- job.finishedTaskCount++;
- break;
-
- case LOST:
- case FAILED:
- case UNKNOWN:
- job.failedTaskCount++;
- Date now = new Date();
- long elapsedMillis = now.getTime()
- - Iterables.getLast(task.getTaskEvents()).getTimestamp();
-
- if (Amount.of(elapsedMillis, Time.MILLISECONDS).as(Time.HOURS) < 6) {
- job.recentlyFailedTaskCount++;
- }
- break;
-
- default:
- throw new IllegalArgumentException("Unsupported status: " + task.getStatus());
- }
- }
-
- return job;
- }
- };
-
- Query.Builder query = environment.isPresent()
- ? Query.envScoped(role, environment.get())
- : Query.roleScoped(role);
-
- Multimap<IJobKey, IScheduledTask> tasks =
- Tasks.byJobKey(Storage.Util.weaklyConsistentFetchTasks(storage, query));
-
- Iterable<Job> jobs = FluentIterable
- .from(tasks.asMap().entrySet())
- .transform(toJob);
-
- return DisplayUtils.JOB_ORDERING.sortedCopy(jobs);
- }
-
- /**
- * Template object to represent a job.
- */
- static class Job {
- private String name;
- private String environment;
- private int pendingTaskCount = 0;
- private int activeTaskCount = 0;
- private int finishedTaskCount = 0;
- private int failedTaskCount = 0;
- private int recentlyFailedTaskCount = 0;
- private boolean production = false;
- private JobType type;
-
- public String getName() {
- return name;
- }
-
- public String getEnvironment() {
- return environment;
- }
-
- public int getPendingTaskCount() {
- return pendingTaskCount;
- }
-
- public int getActiveTaskCount() {
- return activeTaskCount;
- }
-
- public int getFinishedTaskCount() {
- return finishedTaskCount;
- }
-
- public int getFailedTaskCount() {
- return failedTaskCount;
- }
-
- public int getRecentlyFailedTaskCount() {
- return recentlyFailedTaskCount;
- }
-
- public boolean getProduction() {
- return production;
- }
-
- public String getType() {
- return type.toString();
- }
- }
-
- static enum JobType {
- ADHOC("adhoc"), CRON("cron"), SERVICE("service");
-
- private String jobType;
-
- private JobType(String jobType) {
- this.jobType = jobType;
- }
-
- public String toString() {
- return jobType;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/ServletModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/ServletModule.java b/src/main/java/com/twitter/aurora/scheduler/http/ServletModule.java
deleted file mode 100644
index 825e2e1..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/ServletModule.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.io.Resources;
-import com.google.common.net.MediaType;
-import com.google.inject.AbstractModule;
-import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
-import com.google.inject.servlet.GuiceFilter;
-import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
-import com.sun.jersey.guice.JerseyServletModule;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-
-import com.twitter.aurora.scheduler.quota.QuotaManager;
-import com.twitter.aurora.scheduler.state.CronJobManager;
-import com.twitter.aurora.scheduler.state.SchedulerCore;
-import com.twitter.common.application.http.Registration;
-import com.twitter.common.application.modules.LifecycleModule;
-import com.twitter.common.application.modules.LocalServiceRegistry;
-import com.twitter.common.base.ExceptionalCommand;
-import com.twitter.common.net.pool.DynamicHostSet;
-import com.twitter.common.net.pool.DynamicHostSet.MonitorException;
-import com.twitter.thrift.ServiceInstance;
-
-import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS;
-import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_RESPONSE_FILTERS;
-import static com.sun.jersey.api.json.JSONConfiguration.FEATURE_POJO_MAPPING;
-
-/**
- * Binding module for scheduler HTTP servlets.
- */
-public class ServletModule extends AbstractModule {
-
- private static final Map<String, String> CONTAINER_PARAMS = ImmutableMap.of(
- FEATURE_POJO_MAPPING, Boolean.TRUE.toString(),
- PROPERTY_CONTAINER_REQUEST_FILTERS, GZIPContentEncodingFilter.class.getName(),
- PROPERTY_CONTAINER_RESPONSE_FILTERS, GZIPContentEncodingFilter.class.getName());
-
- @Override
- protected void configure() {
- requireBinding(SchedulerCore.class);
- requireBinding(CronJobManager.class);
- requireBinding(Key.get(String.class, ClusterName.class));
- requireBinding(QuotaManager.class);
-
- // Bindings required for the leader redirector.
- requireBinding(LocalServiceRegistry.class);
- requireBinding(Key.get(new TypeLiteral<DynamicHostSet<ServiceInstance>>() { }));
- Registration.registerServletFilter(binder(), GuiceFilter.class, "/*");
- install(new JerseyServletModule() {
- private void registerJerseyEndpoint(String indexPath, Class<?>... servlets) {
- filter(indexPath + "*").through(LeaderRedirectFilter.class);
- filter(indexPath + "*").through(GuiceContainer.class, CONTAINER_PARAMS);
- Registration.registerEndpoint(binder(), indexPath);
- for (Class<?> servlet : servlets) {
- bind(servlet);
- }
- }
-
- @Override protected void configureServlets() {
- bind(HttpStatsFilter.class).in(Singleton.class);
- filter("/scheduler*").through(HttpStatsFilter.class);
- bind(LeaderRedirectFilter.class).in(Singleton.class);
- registerJerseyEndpoint("/cron", Cron.class);
- registerJerseyEndpoint("/maintenance", Maintenance.class);
- registerJerseyEndpoint("/mname", Mname.class);
- registerJerseyEndpoint("/offers", Offers.class);
- registerJerseyEndpoint("/pendingtasks", PendingTasks.class);
- registerJerseyEndpoint("/quotas", Quotas.class);
- registerJerseyEndpoint(
- "/scheduler",
- SchedulerzHome.class,
- SchedulerzRole.class,
- SchedulerzJob.class);
- registerJerseyEndpoint("/slaves", Slaves.class);
- registerJerseyEndpoint("/structdump", StructDump.class);
- registerJerseyEndpoint("/utilization", Utilization.class);
- }
- });
-
- // Static assets.
- registerJQueryAssets();
- registerBootstrapAssets();
-
- registerAsset("assets/util.js", "/js/util.js");
- registerAsset("assets/dictionary.js", "/js/dictionary.js");
- registerAsset("assets/images/viz.png", "/images/viz.png");
- registerAsset("assets/images/aurora.png", "/images/aurora.png");
-
- // Register datatables
- registerAsset("assets/datatables/css/jquery.dataTables.css", "/css/jquery.dataTables.css");
- registerAsset("assets/datatables/images/back_disabled.png", "/images/back_disabled.png");
- registerAsset(
- "assets/datatables/images/back_enabled_hover.png",
- "/images/back_enabled_hover.png");
- registerAsset("assets/datatables/images/back_enabled.png", "/images/back_enabled.png");
- registerAsset(
- "assets/datatables/images/forward_disabled.png",
- "/images/forward_disabled.png");
- registerAsset(
- "assets/datatables/images/forward_enabled_hover.png",
- "/images/forward_enabled_hover.png");
- registerAsset(
- "assets/datatables/images/forward_enabled.png",
- "/images/forward_enabled.png");
- registerAsset(
- "assets/datatables/images/sort_asc_disabled.png",
- "/images/sort_asc_disabled.png");
- registerAsset("assets/datatables/images/sort_asc.png", "/images/sort_asc.png");
- registerAsset("assets/datatables/images/sort_both.png", "/images/sort_both.png");
- registerAsset(
- "assets/datatables/images/sort_desc_disabled.png",
- "/images/sort_desc_disabled.png");
- registerAsset("assets/datatables/images/sort_desc.png", "/images/sort_desc.png");
- registerAsset(
- "assets/datatables/js/jquery.dataTables.min.js",
- "/js/jquery.dataTables.min.js");
- registerAsset(
- "assets/datatables/js/dataTables.bootstrap.js",
- "/js/dataTables.bootstrap.js");
- registerAsset(
- "assets/datatables/js/dataTables.localstorage.js",
- "/js/dataTables.localstorage.js");
- registerAsset(
- "assets/datatables/js/dataTables.htmlNumberType.js",
- "/js/dataTables.htmlNumberType.js");
-
- bind(LeaderRedirect.class).in(Singleton.class);
- LifecycleModule.bindStartupAction(binder(), RedirectMonitor.class);
- }
-
- private void registerJQueryAssets() {
- registerAsset("bower_components/jquery/jquery.js", "/js/jquery.min.js", false);
- }
-
- private void registerBootstrapAssets() {
- final String BOOTSTRAP_PATH = "bower_components/bootstrap.css/";
-
- registerAsset(BOOTSTRAP_PATH + "js/bootstrap.min.js", "/js/bootstrap.min.js", false);
- registerAsset(BOOTSTRAP_PATH + "css/bootstrap.min.css", "/css/bootstrap.min.css", false);
- registerAsset(BOOTSTRAP_PATH + "css/bootstrap-responsive.min.css",
- "/css/bootstrap-responsive.min.css",
- false);
- registerAsset(BOOTSTRAP_PATH + "img/glyphicons-halflings-white.png",
- "/img/glyphicons-halflings-white.png",
- false);
- registerAsset(BOOTSTRAP_PATH + "img/glyphicons-halflings.png",
- "/img/glyphicons-halflings.png",
- false);
- }
-
- private void registerAsset(String resourceLocation, String registerLocation) {
- registerAsset(resourceLocation, registerLocation, true);
- }
-
- private void registerAsset(String resourceLocation, String registerLocation, boolean isRelative) {
- String mediaType = getMediaType(registerLocation).toString();
-
- if (isRelative) {
- Registration.registerHttpAsset(
- binder(),
- registerLocation,
- ServletModule.class,
- resourceLocation,
- mediaType,
- true);
- } else {
- Registration.registerHttpAsset(
- binder(),
- registerLocation,
- Resources.getResource(resourceLocation),
- mediaType,
- true);
- }
- }
-
- private MediaType getMediaType(String filePath) {
- if (filePath.endsWith(".png")) {
- return MediaType.PNG;
- } else if (filePath.endsWith(".js")) {
- return MediaType.JAVASCRIPT_UTF_8;
- } else if (filePath.endsWith(".html")) {
- return MediaType.HTML_UTF_8;
- } else if (filePath.endsWith(".css")) {
- return MediaType.CSS_UTF_8;
- } else {
- throw new IllegalArgumentException("Could not determine media type for " + filePath);
- }
- }
-
- static class RedirectMonitor implements ExceptionalCommand<MonitorException> {
-
- private final LeaderRedirect redirector;
-
- @Inject
- RedirectMonitor(LeaderRedirect redirector) {
- this.redirector = Preconditions.checkNotNull(redirector);
- }
-
- @Override public void execute() throws MonitorException {
- redirector.monitor();
- }
- }
-}
[05/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/assets/mootools-core.js
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/assets/mootools-core.js b/src/main/python/twitter/thermos/observer/http/assets/mootools-core.js
deleted file mode 100644
index 959afa4..0000000
--- a/src/main/python/twitter/thermos/observer/http/assets/mootools-core.js
+++ /dev/null
@@ -1,5789 +0,0 @@
-/*
----
-
-name: Core
-
-description: The heart of MooTools.
-
-license: MIT-style license.
-
-copyright: Copyright (c) 2006-2010 [Valerio Proietti](http://mad4milk.net/).
-
-authors: The MooTools production team (http://mootools.net/developers/)
-
-inspiration:
- - Class implementation inspired by [Base.js](http://dean.edwards.name/weblog/2006/03/base/) Copyright (c) 2006 Dean Edwards, [GNU Lesser General Public License](http://opensource.org/licenses/lgpl-license.php)
- - Some functionality inspired by [Prototype.js](http://prototypejs.org) Copyright (c) 2005-2007 Sam Stephenson, [MIT License](http://opensource.org/licenses/mit-license.php)
-
-provides: [Core, MooTools, Type, typeOf, instanceOf, Native]
-
-...
-*/
-
-(function(){
-
-this.MooTools = {
- version: '1.3dev',
- build: '29de8c7a2a562582a4ec2e42feb8cd667217b3fa'
-};
-
-// typeOf, instanceOf
-
-var typeOf = this.typeOf = function(item){
- if (item == null) return 'null';
- if (item.$family) return item.$family();
-
- if (item.nodeName){
- if (item.nodeType == 1) return 'element';
- if (item.nodeType == 3) return (/\S/).test(item.nodeValue) ? 'textnode' : 'whitespace';
- } else if (typeof item.length == 'number'){
- if (item.callee) return 'arguments';
- if ('item' in item) return 'collection';
- }
-
- return typeof item;
-};
-
-var instanceOf = this.instanceOf = function(item, object){
- if (item == null) return false;
- var constructor = item.$constructor || item.constructor;
- while (constructor){
- if (constructor === object) return true;
- constructor = constructor.parent;
- }
- return item instanceof object;
-};
-
-// Function overloading
-
-var Function = this.Function;
-
-var enumerables = true;
-for (var i in {toString: 1}) enumerables = null;
-if (enumerables) enumerables = ['hasOwnProperty', 'valueOf', 'isPrototypeOf', 'propertyIsEnumerable', 'toLocaleString', 'toString', 'constructor'];
-
-Function.prototype.overloadSetter = function(usePlural){
- var self = this;
- return function(a, b){
- if (a == null) return this;
- if (usePlural || typeof a != 'string'){
- for (var k in a) self.call(this, k, a[k]);
- if (enumerables) for (var i = enumerables.length; i--;){
- k = enumerables[i];
- if (a.hasOwnProperty(k)) self.call(this, k, a[k]);
- }
- } else {
- self.call(this, a, b);
- }
- return this;
- };
-};
-
-Function.prototype.overloadGetter = function(usePlural){
- var self = this;
- return function(a){
- var args, result;
- if (usePlural || typeof a != 'string') args = a;
- else if (arguments.length > 1) args = arguments;
- if (args){
- result = {};
- for (var i = 0; i < args.length; i++) result[args[i]] = self.call(this, args[i]);
- } else {
- result = self.call(this, a);
- }
- return result;
- };
-};
-
-Function.prototype.extend = function(key, value){
- this[key] = value;
-}.overloadSetter();
-
-Function.prototype.implement = function(key, value){
- this.prototype[key] = value;
-}.overloadSetter();
-
-// From
-
-var slice = Array.prototype.slice;
-
-Function.from = function(item){
- return (typeOf(item) == 'function') ? item : function(){
- return item;
- };
-};
-
-Array.from = function(item){
- if (item == null) return [];
- return (Type.isEnumerable(item) && typeof item != 'string') ? (typeOf(item) == 'array') ? item : slice.call(item) : [item];
-};
-
-Number.from = function(item){
- var number = parseFloat(item);
- return isFinite(number) ? number : null;
-};
-
-String.from = function(item){
- return item + '';
-};
-
-// hide, protect
-
-Function.implement({
-
- hide: function(){
- this.$hidden = true;
- return this;
- },
-
- protect: function(){
- this.$protected = true;
- return this;
- }
-
-});
-
-// Type
-
-var Type = this.Type = function(name, object){
- if (name){
- var lower = name.toLowerCase();
- var typeCheck = function(item){
- return (typeOf(item) == lower);
- };
-
- Type['is' + name] = typeCheck;
- if (object != null){
- object.prototype.$family = (function(){
- return lower;
- }).hide();
- //<1.2compat>
- object.type = typeCheck;
- //</1.2compat>
- }
- }
-
- if (object == null) return null;
-
- object.extend(this);
- object.$constructor = Type;
- object.prototype.$constructor = object;
-
- return object;
-};
-
-var toString = Object.prototype.toString;
-
-Type.isEnumerable = function(item){
- return (item != null && typeof item.length == 'number' && toString.call(item) != '[object Function]' );
-};
-
-var hooks = {};
-
-var hooksOf = function(object){
- var type = typeOf(object.prototype);
- return hooks[type] || (hooks[type] = []);
-};
-
-var implement = function(name, method){
- if (method && method.$hidden) return this;
-
- var hooks = hooksOf(this);
-
- for (var i = 0; i < hooks.length; i++){
- var hook = hooks[i];
- if (typeOf(hook) == 'type') implement.call(hook, name, method);
- else hook.call(this, name, method);
- }
-
- var previous = this.prototype[name];
- if (previous == null || !previous.$protected) this.prototype[name] = method;
-
- if (this[name] == null && typeOf(method) == 'function') extend.call(this, name, function(item){
- return method.apply(item, slice.call(arguments, 1));
- });
-
- return this;
-};
-
-var extend = function(name, method){
- if (method && method.$hidden) return this;
- var previous = this[name];
- if (previous == null || !previous.$protected) this[name] = method;
- return this;
-};
-
-Type.implement({
-
- implement: implement.overloadSetter(),
-
- extend: extend.overloadSetter(),
-
- alias: function(name, existing){
- implement.call(this, name, this.prototype[existing]);
- }.overloadSetter(),
-
- mirror: function(hook){
- hooksOf(this).push(hook);
- return this;
- }
-
-});
-
-new Type('Type', Type);
-
-// Default Types
-
-var force = function(name, object, methods){
- var isType = (object != Object),
- prototype = object.prototype;
-
- if (isType) object = new Type(name, object);
-
- for (var i = 0, l = methods.length; i < l; i++){
- var key = methods[i],
- generic = object[key],
- proto = prototype[key];
-
- if (generic) generic.protect();
-
- if (isType && proto){
- delete prototype[key];
- prototype[key] = proto.protect();
- }
- }
-
- if (isType) object.implement(prototype);
-
- return force;
-};
-
-force('String', String, [
- 'charAt', 'charCodeAt', 'concat', 'indexOf', 'lastIndexOf', 'match', 'quote', 'replace', 'search',
- 'slice', 'split', 'substr', 'substring', 'toLowerCase', 'toUpperCase'
-])('Array', Array, [
- 'pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift', 'concat', 'join', 'slice',
- 'indexOf', 'lastIndexOf', 'filter', 'forEach', 'every', 'map', 'some', 'reduce', 'reduceRight'
-])('Number', Number, [
- 'toExponential', 'toFixed', 'toLocaleString', 'toPrecision'
-])('Function', Function, [
- 'apply', 'call', 'bind'
-])('RegExp', RegExp, [
- 'exec', 'test'
-])('Object', Object, [
- 'create', 'defineProperty', 'defineProperties', 'keys',
- 'getPrototypeOf', 'getOwnPropertyDescriptor', 'getOwnPropertyNames',
- 'preventExtensions', 'isExtensible', 'seal', 'isSealed', 'freeze', 'isFrozen'
-])('Date', Date, ['now']);
-
-Object.extend = extend.overloadSetter();
-
-Date.extend('now', function(){
- return +(new Date);
-});
-
-new Type('Boolean', Boolean);
-
-// fixes NaN returning as Number
-
-Number.prototype.$family = function(){
- return isFinite(this) ? 'number' : 'null';
-}.hide();
-
-// Number.random
-
-Number.extend('random', function(min, max){
- return Math.floor(Math.random() * (max - min + 1) + min);
-});
-
-// forEach, each
-
-Object.extend('forEach', function(object, fn, bind){
- for (var key in object){
- if (object.hasOwnProperty(key)) fn.call(bind, object[key], key, object);
- }
-});
-
-Object.each = Object.forEach;
-
-Array.implement({
-
- forEach: function(fn, bind){
- for (var i = 0, l = this.length; i < l; i++){
- if (i in this) fn.call(bind, this[i], i, this);
- }
- },
-
- each: function(fn, bind){
- Array.forEach(this, fn, bind);
- return this;
- }
-
-});
-
-// Array & Object cloning, Object merging and appending
-
-var cloneOf = function(item){
- switch (typeOf(item)){
- case 'array': return item.clone();
- case 'object': return Object.clone(item);
- default: return item;
- }
-};
-
-Array.implement('clone', function(){
- var i = this.length, clone = new Array(i);
- while (i--) clone[i] = cloneOf(this[i]);
- return clone;
-});
-
-var mergeOne = function(source, key, current){
- switch (typeOf(current)){
- case 'object':
- if (typeOf(source[key]) == 'object') Object.merge(source[key], current);
- else source[key] = Object.clone(current);
- break;
- case 'array': source[key] = current.clone(); break;
- default: source[key] = current;
- }
- return source;
-};
-
-Object.extend({
-
- merge: function(source, k, v){
- if (typeOf(k) == 'string') return mergeOne(source, k, v);
- for (var i = 1, l = arguments.length; i < l; i++){
- var object = arguments[i];
- for (var key in object) mergeOne(source, key, object[key]);
- }
- return source;
- },
-
- clone: function(object){
- var clone = {};
- for (var key in object) clone[key] = cloneOf(object[key]);
- return clone;
- },
-
- append: function(original){
- for (var i = 1, l = arguments.length; i < l; i++){
- var extended = arguments[i] || {};
- for (var key in extended) original[key] = extended[key];
- }
- return original;
- }
-
-});
-
-// Object-less types
-
-['Object', 'WhiteSpace', 'TextNode', 'Collection', 'Arguments'].each(function(name){
- new Type(name);
-});
-
-// Unique ID
-
-var UID = Date.now();
-
-String.extend('generateUID', function(){
- return (UID++).toString(36);
-});
-
-//<1.2compat>
-
-var Hash = this.Hash = new Type('Hash', function(object){
- if (typeOf(object) == 'hash') object = Object.clone(object.getClean());
- for (var key in object) this[key] = object[key];
- return this;
-});
-
-Hash.implement({
-
- forEach: function(fn, bind){
- Object.forEach(this, fn, bind);
- },
-
- getClean: function(){
- var clean = {};
- for (var key in this){
- if (this.hasOwnProperty(key)) clean[key] = this[key];
- }
- return clean;
- },
-
- getLength: function(){
- var length = 0;
- for (var key in this){
- if (this.hasOwnProperty(key)) length++;
- }
- return length;
- }
-
-});
-
-Hash.alias('each', 'forEach');
-
-Object.type = Type.isObject;
-
-var Native = this.Native = function(properties){
- return new Type(properties.name, properties.initialize);
-};
-
-Native.type = Type.type;
-
-Native.implement = function(objects, methods){
- for (var i = 0; i < objects.length; i++) objects[i].implement(methods);
- return Native;
-};
-
-var arrayType = Array.type;
-Array.type = function(item){
- return instanceOf(item, Array) || arrayType(item);
-};
-
-this.$A = function(item){
- return Array.from(item).slice();
-};
-
-this.$arguments = function(i){
- return function(){
- return arguments[i];
- };
-};
-
-this.$chk = function(obj){
- return !!(obj || obj === 0);
-};
-
-this.$clear = function(timer){
- clearTimeout(timer);
- clearInterval(timer);
- return null;
-};
-
-this.$defined = function(obj){
- return (obj != null);
-};
-
-this.$each = function(iterable, fn, bind){
- var type = typeOf(iterable);
- ((type == 'arguments' || type == 'collection' || type == 'array' || type == 'elements') ? Array : Object).each(iterable, fn, bind);
-};
-
-this.$empty = function(){};
-
-this.$extend = function(original, extended){
- return Object.append(original, extended);
-};
-
-this.$H = function(object){
- return new Hash(object);
-};
-
-this.$merge = function(){
- var args = Array.slice(arguments);
- args.unshift({});
- return Object.merge.apply(null, args);
-};
-
-this.$lambda = Function.from;
-this.$mixin = Object.merge;
-this.$random = Number.random;
-this.$splat = Array.from;
-this.$time = Date.now;
-
-this.$type = function(object){
- var type = typeOf(object);
- if (type == 'elements') return 'array';
- return (type == 'null') ? false : type;
-};
-
-this.$unlink = function(object){
- switch (typeOf(object)){
- case 'object': return Object.clone(object);
- case 'array': return Array.clone(object);
- case 'hash': return new Hash(object);
- default: return object;
- }
-};
-
-//</1.2compat>
-
-})();
-
-
-/*
----
-
-name: Array
-
-description: Contains Array Prototypes like each, contains, and erase.
-
-license: MIT-style license.
-
-requires: Type
-
-provides: Array
-
-...
-*/
-
-Array.implement({
-
- invoke: function(methodName){
- var args = Array.slice(arguments, 1);
- return this.map(function(item){
- return item[methodName].apply(item, args);
- });
- },
-
- every: function(fn, bind){
- for (var i = 0, l = this.length; i < l; i++){
- if ((i in this) && !fn.call(bind, this[i], i, this)) return false;
- }
- return true;
- },
-
- filter: function(fn, bind){
- var results = [];
- for (var i = 0, l = this.length; i < l; i++){
- if ((i in this) && fn.call(bind, this[i], i, this)) results.push(this[i]);
- }
- return results;
- },
-
- clean: function(){
- return this.filter(function(item){
- return item != null;
- });
- },
-
- indexOf: function(item, from){
- var len = this.length;
- for (var i = (from < 0) ? Math.max(0, len + from) : from || 0; i < len; i++){
- if (this[i] === item) return i;
- }
- return -1;
- },
-
- map: function(fn, bind){
- var results = [];
- for (var i = 0, l = this.length; i < l; i++){
- if (i in this) results[i] = fn.call(bind, this[i], i, this);
- }
- return results;
- },
-
- some: function(fn, bind){
- for (var i = 0, l = this.length; i < l; i++){
- if ((i in this) && fn.call(bind, this[i], i, this)) return true;
- }
- return false;
- },
-
- associate: function(keys){
- var obj = {}, length = Math.min(this.length, keys.length);
- for (var i = 0; i < length; i++) obj[keys[i]] = this[i];
- return obj;
- },
-
- link: function(object){
- var result = {};
- for (var i = 0, l = this.length; i < l; i++){
- for (var key in object){
- if (object[key](this[i])){
- result[key] = this[i];
- delete object[key];
- break;
- }
- }
- }
- return result;
- },
-
- contains: function(item, from){
- return this.indexOf(item, from) != -1;
- },
-
- append: function(array){
- this.push.apply(this, array);
- return this;
- },
-
- getLast: function(){
- return (this.length) ? this[this.length - 1] : null;
- },
-
- getRandom: function(){
- return (this.length) ? this[Number.random(0, this.length - 1)] : null;
- },
-
- include: function(item){
- if (!this.contains(item)) this.push(item);
- return this;
- },
-
- combine: function(array){
- for (var i = 0, l = array.length; i < l; i++) this.include(array[i]);
- return this;
- },
-
- erase: function(item){
- for (var i = this.length; i--;){
- if (this[i] === item) this.splice(i, 1);
- }
- return this;
- },
-
- empty: function(){
- this.length = 0;
- return this;
- },
-
- flatten: function(){
- var array = [];
- for (var i = 0, l = this.length; i < l; i++){
- var type = typeOf(this[i]);
- if (type == 'null') continue;
- array = array.concat((type == 'array' || type == 'collection' || type == 'arguments' || instanceOf(this[i], Array)) ? Array.flatten(this[i]) : this[i]);
- }
- return array;
- },
-
- pick: function(){
- for (var i = 0, l = this.length; i < l; i++){
- if (this[i] != null) return this[i];
- }
- return null;
- },
-
- hexToRgb: function(array){
- if (this.length != 3) return null;
- var rgb = this.map(function(value){
- if (value.length == 1) value += value;
- return value.toInt(16);
- });
- return (array) ? rgb : 'rgb(' + rgb + ')';
- },
-
- rgbToHex: function(array){
- if (this.length < 3) return null;
- if (this.length == 4 && this[3] == 0 && !array) return 'transparent';
- var hex = [];
- for (var i = 0; i < 3; i++){
- var bit = (this[i] - 0).toString(16);
- hex.push((bit.length == 1) ? '0' + bit : bit);
- }
- return (array) ? hex : '#' + hex.join('');
- }
-
-});
-
-//<1.2compat>
-
-Array.alias('extend', 'append');
-
-var $pick = function(){
- return Array.from(arguments).pick();
-};
-
-//</1.2compat>
-
-
-/*
----
-
-name: String
-
-description: Contains String Prototypes like camelCase, capitalize, test, and toInt.
-
-license: MIT-style license.
-
-requires: Type
-
-provides: String
-
-...
-*/
-
-String.implement({
-
- test: function(regex, params){
- return ((typeOf(regex) == 'regexp') ? regex : new RegExp('' + regex, params)).test(this);
- },
-
- contains: function(string, separator){
- return (separator) ? (separator + this + separator).indexOf(separator + string + separator) > -1 : this.indexOf(string) > -1;
- },
-
- trim: function(){
- return this.replace(/^\s+|\s+$/g, '');
- },
-
- clean: function(){
- return this.replace(/\s+/g, ' ').trim();
- },
-
- camelCase: function(){
- return this.replace(/-\D/g, function(match){
- return match.charAt(1).toUpperCase();
- });
- },
-
- hyphenate: function(){
- return this.replace(/[A-Z]/g, function(match){
- return ('-' + match.charAt(0).toLowerCase());
- });
- },
-
- capitalize: function(){
- return this.replace(/\b[a-z]/g, function(match){
- return match.toUpperCase();
- });
- },
-
- escapeRegExp: function(){
- return this.replace(/([-.*+?^${}()|[\]\/\\])/g, '\\$1');
- },
-
- toInt: function(base){
- return parseInt(this, base || 10);
- },
-
- toFloat: function(){
- return parseFloat(this);
- },
-
- hexToRgb: function(array){
- var hex = this.match(/^#?(\w{1,2})(\w{1,2})(\w{1,2})$/);
- return (hex) ? hex.slice(1).hexToRgb(array) : null;
- },
-
- rgbToHex: function(array){
- var rgb = this.match(/\d{1,3}/g);
- return (rgb) ? rgb.rgbToHex(array) : null;
- },
-
- substitute: function(object, regexp){
- return this.replace(regexp || (/\\?\{([^{}]+)\}/g), function(match, name){
- if (match.charAt(0) == '\\') return match.slice(1);
- return (object[name] != null) ? object[name] : '';
- });
- }
-
-});
-
-
-/*
----
-
-name: Number
-
-description: Contains Number Prototypes like limit, round, times, and ceil.
-
-license: MIT-style license.
-
-requires: Type
-
-provides: Number
-
-...
-*/
-
-Number.implement({
-
- limit: function(min, max){
- return Math.min(max, Math.max(min, this));
- },
-
- round: function(precision){
- precision = Math.pow(10, precision || 0).toFixed(precision < 0 ? -precision : 0);
- return Math.round(this * precision) / precision;
- },
-
- times: function(fn, bind){
- for (var i = 0; i < this; i++) fn.call(bind, i, this);
- },
-
- toFloat: function(){
- return parseFloat(this);
- },
-
- toInt: function(base){
- return parseInt(this, base || 10);
- }
-
-});
-
-Number.alias('each', 'times');
-
-(function(math){
- var methods = {};
- math.each(function(name){
- if (!Number[name]) methods[name] = function(){
- return Math[name].apply(null, [this].concat(Array.from(arguments)));
- };
- });
- Number.implement(methods);
-})(['abs', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'exp', 'floor', 'log', 'max', 'min', 'pow', 'sin', 'sqrt', 'tan']);
-
-
-/*
----
-
-name: Function
-
-description: Contains Function Prototypes like create, bind, pass, and delay.
-
-license: MIT-style license.
-
-requires: Type
-
-provides: Function
-
-...
-*/
-
-Function.extend({
-
- attempt: function(){
- for (var i = 0, l = arguments.length; i < l; i++){
- try {
- return arguments[i]();
- } catch (e){}
- }
- return null;
- }
-
-});
-
-Function.implement({
-
- attempt: function(args, bind){
- try {
- return this.apply(bind, Array.from(args));
- } catch (e){}
-
- return null;
- },
-
- bind: function(bind){
- var self = this,
- args = (arguments.length > 1) ? Array.slice(arguments, 1) : null;
-
- return function(){
- if (!args && !arguments.length) return self.call(bind);
- if (args && arguments.length) return self.apply(bind, args.concat(Array.from(arguments)));
- return self.apply(bind, args || arguments);
- };
- },
-
- pass: function(args, bind){
- var self = this;
- if (args != null) args = Array.from(args);
- return function(){
- return self.apply(bind, args || arguments);
- };
- },
-
- delay: function(delay, bind, args){
- return setTimeout(this.pass(args, bind), delay);
- },
-
- periodical: function(periodical, bind, args){
- return setInterval(this.pass(args, bind), periodical);
- }
-
-});
-
-//<1.2compat>
-
-delete Function.prototype.bind;
-
-Function.implement({
-
- create: function(options){
- var self = this;
- options = options || {};
- return function(event){
- var args = options.arguments;
- args = (args != null) ? Array.from(args) : Array.slice(arguments, (options.event) ? 1 : 0);
- if (options.event) args = [event || window.event].extend(args);
- var returns = function(){
- return self.apply(options.bind || null, args);
- };
- if (options.delay) return setTimeout(returns, options.delay);
- if (options.periodical) return setInterval(returns, options.periodical);
- if (options.attempt) return Function.attempt(returns);
- return returns();
- };
- },
-
- bind: function(bind, args){
- var self = this;
- if (args != null) args = Array.from(args);
- return function(){
- return self.apply(bind, args || arguments);
- };
- },
-
- bindWithEvent: function(bind, args){
- var self = this;
- if (args != null) args = Array.from(args);
- return function(event){
- return self.apply(bind, (args == null) ? arguments : [event].concat(args));
- };
- },
-
- run: function(args, bind){
- return this.apply(bind, Array.from(args));
- }
-
-});
-
-var $try = Function.attempt;
-
-//</1.2compat>
-
-
-/*
----
-
-name: Object
-
-description: Object generic methods
-
-license: MIT-style license.
-
-requires: Type
-
-provides: [Object, Hash]
-
-...
-*/
-
-
-Object.extend({
-
- subset: function(object, keys){
- var results = {};
- for (var i = 0, l = keys.length; i < l; i++){
- var k = keys[i];
- results[k] = object[k];
- }
- return results;
- },
-
- map: function(object, fn, bind){
- var results = {};
- for (var key in object){
- if (object.hasOwnProperty(key)) results[key] = fn.call(bind, object[key], key, object);
- }
- return results;
- },
-
- filter: function(object, fn, bind){
- var results = {};
- Object.each(object, function(value, key){
- if (fn.call(bind, value, key, object)) results[key] = value;
- });
- return results;
- },
-
- every: function(object, fn, bind){
- for (var key in object){
- if (object.hasOwnProperty(key) && !fn.call(bind, object[key], key)) return false;
- }
- return true;
- },
-
- some: function(object, fn, bind){
- for (var key in object){
- if (object.hasOwnProperty(key) && fn.call(bind, object[key], key)) return true;
- }
- return false;
- },
-
- keys: function(object){
- var keys = [];
- for (var key in object){
- if (object.hasOwnProperty(key)) keys.push(key);
- }
- return keys;
- },
-
- values: function(object){
- var values = [];
- for (var key in object){
- if (object.hasOwnProperty(key)) values.push(object[key]);
- }
- return values;
- },
-
- getLength: function(object){
- return Object.keys(object).length;
- },
-
- keyOf: function(object, value){
- for (var key in object){
- if (object.hasOwnProperty(key) && object[key] === value) return key;
- }
- return null;
- },
-
- contains: function(object, value){
- return Object.keyOf(object, value) != null;
- },
-
- toQueryString: function(object, base){
- var queryString = [];
-
- Object.each(object, function(value, key){
- if (base) key = base + '[' + key + ']';
- var result;
- switch (typeOf(value)){
- case 'object': result = Object.toQueryString(value, key); break;
- case 'array':
- var qs = {};
- value.each(function(val, i){
- qs[i] = val;
- });
- result = Object.toQueryString(qs, key);
- break;
- default: result = key + '=' + encodeURIComponent(value);
- }
- if (value != null) queryString.push(result);
- });
-
- return queryString.join('&');
- }
-
-});
-
-
-//<1.2compat>
-
-Hash.implement({
-
- has: Object.prototype.hasOwnProperty,
-
- keyOf: function(value){
- return Object.keyOf(this, value);
- },
-
- hasValue: function(value){
- return Object.contains(this, value);
- },
-
- extend: function(properties){
- Hash.each(properties || {}, function(value, key){
- Hash.set(this, key, value);
- }, this);
- return this;
- },
-
- combine: function(properties){
- Hash.each(properties || {}, function(value, key){
- Hash.include(this, key, value);
- }, this);
- return this;
- },
-
- erase: function(key){
- if (this.hasOwnProperty(key)) delete this[key];
- return this;
- },
-
- get: function(key){
- return (this.hasOwnProperty(key)) ? this[key] : null;
- },
-
- set: function(key, value){
- if (!this[key] || this.hasOwnProperty(key)) this[key] = value;
- return this;
- },
-
- empty: function(){
- Hash.each(this, function(value, key){
- delete this[key];
- }, this);
- return this;
- },
-
- include: function(key, value){
- if (this[key] == null) this[key] = value;
- return this;
- },
-
- map: function(fn, bind){
- return new Hash(Object.map(this, fn, bind));
- },
-
- filter: function(fn, bind){
- return new Hash(Object.filter(this, fn, bind));
- },
-
- every: function(fn, bind){
- return Object.every(this, fn, bind);
- },
-
- some: function(fn, bind){
- return Object.some(this, fn, bind);
- },
-
- getKeys: function(){
- return Object.keys(this);
- },
-
- getValues: function(){
- return Object.values(this);
- },
-
- toQueryString: function(base){
- return Object.toQueryString(this, base);
- }
-
-});
-
-Hash.extend = Object.append;
-
-Hash.alias({indexOf: 'keyOf', contains: 'hasValue'});
-
-//</1.2compat>
-
-
-/*
----
-
-name: Browser
-
-description: The Browser Object. Contains Browser initialization, Window and Document, and the Browser Hash.
-
-license: MIT-style license.
-
-requires: [Array, Function, Number, String]
-
-provides: [Browser, Window, Document]
-
-...
-*/
-
-(function(){
-
-var document = this.document;
-var window = document.window = this;
-
-var UID = 1;
-
-this.$uid = (window.ActiveXObject) ? function(item){
- return (item.uid || (item.uid = [UID++]))[0];
-} : function(item){
- return item.uid || (item.uid = UID++);
-};
-
-$uid(window);
-$uid(document);
-
-var ua = navigator.userAgent.toLowerCase(),
- platform = navigator.platform.toLowerCase(),
- UA = ua.match(/(opera|ie|firefox|chrome|version)[\s\/:]([\w\d\.]+)?.*?(safari|version[\s\/:]([\w\d\.]+)|$)/) || [null, 'unknown', 0];
-
-var Browser = this.Browser = {
-
- extend: Function.prototype.extend,
-
- name: (UA[1] == 'version') ? UA[3] : UA[1],
-
- version: parseFloat((UA[1] == 'opera' && UA[4]) ? UA[4] : UA[2]),
-
- Platform: {
- name: ua.match(/ip(?:ad|od|hone)/) ? 'ios' : (ua.match(/(?:webos|android)/) || platform.match(/mac|win|linux/) || ['other'])[0]
- },
-
- Features: {
- xpath: !!(document.evaluate),
- air: !!(window.runtime),
- query: !!(document.querySelector),
- json: !!(window.JSON)
- },
-
- Plugins: {}
-
-};
-
-Browser[Browser.name] = true;
-Browser[Browser.name + parseInt(Browser.version, 10)] = true;
-Browser.Platform[Browser.Platform.name] = true;
-
-// Request
-
-Browser.Request = (function(){
-
- var XMLHTTP = function(){
- return new XMLHttpRequest();
- };
-
- var MSXML2 = function(){
- return new ActiveXObject('MSXML2.XMLHTTP');
- };
-
- var MSXML = function(){
- return new ActiveXObject('Microsoft.XMLHTTP');
- };
-
- return Function.attempt(function(){
- XMLHTTP();
- return XMLHTTP;
- }, function(){
- MSXML2();
- return MSXML2;
- }, function(){
- MSXML();
- return MSXML;
- });
-
-})();
-
-Browser.Features.xhr = !!(Browser.Request);
-
-// Flash detection
-
-var version = (Function.attempt(function(){
- return navigator.plugins['Shockwave Flash'].description;
-}, function(){
- return new ActiveXObject('ShockwaveFlash.ShockwaveFlash').GetVariable('$version');
-}) || '0 r0').match(/\d+/g);
-
-Browser.Plugins.Flash = {
- version: Number(version[0] || '0.' + version[1]) || 0,
- build: Number(version[2]) || 0
-};
-
-// String scripts
-
-Browser.exec = function(text){
- if (!text) return text;
- if (window.execScript){
- window.execScript(text);
- } else {
- var script = document.createElement('script');
- script.setAttribute('type', 'text/javascript');
- script.text = text;
- document.head.appendChild(script);
- document.head.removeChild(script);
- }
- return text;
-};
-
-String.implement('stripScripts', function(exec){
- var scripts = '';
- var text = this.replace(/<script[^>]*>([\s\S]*?)<\/script>/gi, function(all, code){
- scripts += code + '\n';
- return '';
- });
- if (exec === true) Browser.exec(scripts);
- else if (typeOf(exec) == 'function') exec(scripts, text);
- return text;
-});
-
-// Window, Document
-
-Browser.extend({
- Document: this.Document,
- Window: this.Window,
- Element: this.Element,
- Event: this.Event
-});
-
-this.Window = this.$constructor = new Type('Window', function(){});
-
-this.$family = Function.from('window').hide();
-
-Window.mirror(function(name, method){
- window[name] = method;
-});
-
-this.Document = document.$constructor = new Type('Document', function(){});
-
-document.$family = Function.from('document').hide();
-
-Document.mirror(function(name, method){
- document[name] = method;
-});
-
-document.html = document.documentElement;
-document.head = document.getElementsByTagName('head')[0];
-
-if (document.execCommand) try {
- document.execCommand("BackgroundImageCache", false, true);
-} catch (e){}
-
-if (this.attachEvent && !this.addEventListener){
- var unloadEvent = function(){
- this.detachEvent('onunload', unloadEvent);
- document.head = document.html = document.window = null;
- };
- this.attachEvent('onunload', unloadEvent);
-}
-
-// IE fails on collections and <select>.options (refers to <select>)
-var arrayFrom = Array.from;
-try {
- arrayFrom(document.html.childNodes);
-} catch(e){
- Array.from = function(item){
- if (typeof item != 'string' && Type.isEnumerable(item) && typeOf(item) != 'array'){
- var i = item.length, array = new Array(i);
- while (i--) array[i] = item[i];
- return array;
- }
- return arrayFrom(item);
- };
-
- var prototype = Array.prototype,
- slice = prototype.slice;
- ['pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift', 'concat', 'join', 'slice'].each(function(name){
- var method = prototype[name];
- Array[name] = function(item){
- return method.apply(Array.from(item), slice.call(arguments, 1));
- };
- });
-}
-
-//<1.2compat>
-
-if (Browser.Platform.ios) Browser.Platform.ipod = true;
-
-Browser.Engine = {};
-
-var setEngine = function(name, version){
- Browser.Engine.name = name;
- Browser.Engine[name + version] = true;
- Browser.Engine.version = version;
-};
-
-if (Browser.ie){
- Browser.Engine.trident = true;
-
- switch (Browser.version){
- case 6: setEngine('trident', 4); break;
- case 7: setEngine('trident', 5); break;
- case 8: setEngine('trident', 6);
- }
-}
-
-if (Browser.firefox){
- Browser.Engine.gecko = true;
-
- if (Browser.version >= 3) setEngine('gecko', 19);
- else setEngine('gecko', 18);
-}
-
-if (Browser.safari || Browser.chrome){
- Browser.Engine.webkit = true;
-
- switch (Browser.version){
- case 2: setEngine('webkit', 419); break;
- case 3: setEngine('webkit', 420); break;
- case 4: setEngine('webkit', 525);
- }
-}
-
-if (Browser.opera){
- Browser.Engine.presto = true;
-
- if (Browser.version >= 9.6) setEngine('presto', 960);
- else if (Browser.version >= 9.5) setEngine('presto', 950);
- else setEngine('presto', 925);
-}
-
-if (Browser.name == 'unknown'){
- switch ((ua.match(/(?:webkit|khtml|gecko)/) || [])[0]){
- case 'webkit':
- case 'khtml':
- Browser.Engine.webkit = true;
- break;
- case 'gecko':
- Browser.Engine.gecko = true;
- }
-}
-
-this.$exec = Browser.exec;
-
-//</1.2compat>
-
-})();
-
-
-/*
----
-
-name: Event
-
-description: Contains the Event Class, to make the event object cross-browser.
-
-license: MIT-style license.
-
-requires: [Window, Document, Array, Function, String, Object]
-
-provides: Event
-
-...
-*/
-
-var Event = new Type('Event', function(event, win){
- if (!win) win = window;
- var doc = win.document;
- event = event || win.event;
- if (event.$extended) return event;
- this.$extended = true;
- var type = event.type,
- target = event.target || event.srcElement,
- page = {},
- client = {};
- while (target && target.nodeType == 3) target = target.parentNode;
-
- if (type.indexOf('key') != -1){
- var code = event.which || event.keyCode;
- var key = Object.keyOf(Event.Keys, code);
- if (type == 'keydown'){
- var fKey = code - 111;
- if (fKey > 0 && fKey < 13) key = 'f' + fKey;
- }
- if (!key) key = String.fromCharCode(code).toLowerCase();
- } else if (type.test(/click|mouse|menu/i)){
- doc = (!doc.compatMode || doc.compatMode == 'CSS1Compat') ? doc.html : doc.body;
- page = {
- x: (event.pageX != null) ? event.pageX : event.clientX + doc.scrollLeft,
- y: (event.pageY != null) ? event.pageY : event.clientY + doc.scrollTop
- };
- client = {
- x: (event.pageX != null) ? event.pageX - win.pageXOffset : event.clientX,
- y: (event.pageY != null) ? event.pageY - win.pageYOffset : event.clientY
- };
- if (type.test(/DOMMouseScroll|mousewheel/)){
- var wheel = (event.wheelDelta) ? event.wheelDelta / 120 : -(event.detail || 0) / 3;
- }
- var rightClick = (event.which == 3) || (event.button == 2),
- related = null;
- if (type.test(/over|out/)){
- related = event.relatedTarget || event[(type == 'mouseover' ? 'from' : 'to') + 'Element'];
- var testRelated = function(){
- while (related && related.nodeType == 3) related = related.parentNode;
- return true;
- };
- var hasRelated = (Browser.firefox2) ? testRelated.attempt() : testRelated();
- related = (hasRelated) ? related : null;
- }
- } else if (type.test(/gesture|touch/i)){
- this.rotation = event.rotation;
- this.scale = event.scale;
- this.targetTouches = event.targetTouches;
- this.changedTouches = event.changedTouches;
- var touches = this.touches = event.touches;
- if (touches && touches[0]){
- var touch = touches[0];
- page = {x: touch.pageX, y: touch.pageY};
- client = {x: touch.clientX, y: touch.clientY};
- }
- }
-
- return Object.append(this, {
- event: event,
- type: type,
-
- page: page,
- client: client,
- rightClick: rightClick,
-
- wheel: wheel,
-
- relatedTarget: document.id(related),
- target: document.id(target),
-
- code: code,
- key: key,
-
- shift: event.shiftKey,
- control: event.ctrlKey,
- alt: event.altKey,
- meta: event.metaKey
- });
-});
-
-Event.Keys = {
- 'enter': 13,
- 'up': 38,
- 'down': 40,
- 'left': 37,
- 'right': 39,
- 'esc': 27,
- 'space': 32,
- 'backspace': 8,
- 'tab': 9,
- 'delete': 46
-};
-
-//<1.2compat>
-
-Event.Keys = new Hash(Event.Keys);
-
-//</1.2compat>
-
-Event.implement({
-
- stop: function(){
- return this.stopPropagation().preventDefault();
- },
-
- stopPropagation: function(){
- if (this.event.stopPropagation) this.event.stopPropagation();
- else this.event.cancelBubble = true;
- return this;
- },
-
- preventDefault: function(){
- if (this.event.preventDefault) this.event.preventDefault();
- else this.event.returnValue = false;
- return this;
- }
-
-});
-
-
-/*
----
-
-name: Class
-
-description: Contains the Class Function for easily creating, extending, and implementing reusable Classes.
-
-license: MIT-style license.
-
-requires: [Array, String, Function, Number]
-
-provides: Class
-
-...
-*/
-
-(function(){
-
-var Class = this.Class = new Type('Class', function(params){
- if (instanceOf(params, Function)) params = {initialize: params};
-
- var newClass = function(){
- reset(this);
- if (newClass.$prototyping) return this;
- this.$caller = null;
- var value = (this.initialize) ? this.initialize.apply(this, arguments) : this;
- this.$caller = this.caller = null;
- return value;
- }.extend(this).implement(params);
-
- newClass.$constructor = Class;
- newClass.prototype.$constructor = newClass;
- newClass.prototype.parent = parent;
-
- return newClass;
-});
-
-var parent = function(){
- if (!this.$caller) throw new Error('The method "parent" cannot be called.');
- var name = this.$caller.$name,
- parent = this.$caller.$owner.parent,
- previous = (parent) ? parent.prototype[name] : null;
- if (!previous) throw new Error('The method "' + name + '" has no parent.');
- return previous.apply(this, arguments);
-};
-
-var reset = function(object){
- for (var key in object){
- var value = object[key];
- switch (typeOf(value)){
- case 'object':
- var F = function(){};
- F.prototype = value;
- object[key] = reset(new F);
- break;
- case 'array': object[key] = value.clone(); break;
- }
- }
- return object;
-};
-
-var wrap = function(self, key, method){
- if (method.$origin) method = method.$origin;
- var wrapper = function(){
- if (method.$protected && this.$caller == null) throw new Error('The method "' + key + '" cannot be called.');
- var caller = this.caller, current = this.$caller;
- this.caller = current; this.$caller = wrapper;
- var result = method.apply(this, arguments);
- this.$caller = current; this.caller = caller;
- return result;
- }.extend({$owner: self, $origin: method, $name: key});
- return wrapper;
-};
-
-var implement = function(key, value, retain){
- if (Class.Mutators.hasOwnProperty(key)){
- value = Class.Mutators[key].call(this, value);
- if (value == null) return this;
- }
-
- if (typeOf(value) == 'function'){
- if (value.$hidden) return this;
- this.prototype[key] = (retain) ? value : wrap(this, key, value);
- } else {
- Object.merge(this.prototype, key, value);
- }
-
- return this;
-};
-
-var getInstance = function(klass){
- klass.$prototyping = true;
- var proto = new klass;
- delete klass.$prototyping;
- return proto;
-};
-
-Class.implement('implement', implement.overloadSetter());
-
-Class.Mutators = {
-
- Extends: function(parent){
- this.parent = parent;
- this.prototype = getInstance(parent);
- },
-
- Implements: function(items){
- Array.from(items).each(function(item){
- var instance = new item;
- for (var key in instance) implement.call(this, key, instance[key], true);
- }, this);
- }
-};
-
-})();
-
-
-/*
----
-
-name: Class.Extras
-
-description: Contains Utility Classes that can be implemented into your own Classes to ease the execution of many common tasks.
-
-license: MIT-style license.
-
-requires: Class
-
-provides: [Class.Extras, Chain, Events, Options]
-
-...
-*/
-
-(function(){
-
-this.Chain = new Class({
-
- $chain: [],
-
- chain: function(){
- this.$chain.append(Array.flatten(arguments));
- return this;
- },
-
- callChain: function(){
- return (this.$chain.length) ? this.$chain.shift().apply(this, arguments) : false;
- },
-
- clearChain: function(){
- this.$chain.empty();
- return this;
- }
-
-});
-
-var removeOn = function(string){
- return string.replace(/^on([A-Z])/, function(full, first){
- return first.toLowerCase();
- });
-};
-
-this.Events = new Class({
-
- $events: {},
-
- addEvent: function(type, fn, internal){
- type = removeOn(type);
-
- /*<1.2compat>*/
- if (fn == $empty) return this;
- /*</1.2compat>*/
-
- this.$events[type] = (this.$events[type] || []).include(fn);
- if (internal) fn.internal = true;
- return this;
- },
-
- addEvents: function(events){
- for (var type in events) this.addEvent(type, events[type]);
- return this;
- },
-
- fireEvent: function(type, args, delay){
- type = removeOn(type);
- var events = this.$events[type];
- if (!events) return this;
- args = Array.from(args);
- events.each(function(fn){
- if (delay) fn.delay(delay, this, args);
- else fn.apply(this, args);
- }, this);
- return this;
- },
-
- removeEvent: function(type, fn){
- type = removeOn(type);
- var events = this.$events[type];
- if (events && !fn.internal){
- var index = events.indexOf(fn);
- if (index != -1) delete events[index];
- }
- return this;
- },
-
- removeEvents: function(events){
- var type;
- if (typeOf(events) == 'object'){
- for (type in events) this.removeEvent(type, events[type]);
- return this;
- }
- if (events) events = removeOn(events);
- for (type in this.$events){
- if (events && events != type) continue;
- var fns = this.$events[type];
- for (var i = fns.length; i--;) this.removeEvent(type, fns[i]);
- }
- return this;
- }
-
-});
-
-this.Options = new Class({
-
- setOptions: function(){
- var options = this.options = Object.merge.apply(null, [{}, this.options].append(arguments));
- if (!this.addEvent) return this;
- for (var option in options){
- if (typeOf(options[option]) != 'function' || !(/^on[A-Z]/).test(option)) continue;
- this.addEvent(option, options[option]);
- delete options[option];
- }
- return this;
- }
-
-});
-
-})();
-
-
-/*
----
-name: Slick.Parser
-description: Standalone CSS3 Selector parser
-provides: Slick.Parser
-...
-*/
-
-(function(){
-
-var parsed,
- separatorIndex,
- combinatorIndex,
- reversed,
- cache = {},
- reverseCache = {},
- reUnescape = /\\/g;
-
-var parse = function(expression, isReversed){
- if (expression == null) return null;
- if (expression.Slick === true) return expression;
- expression = ('' + expression).replace(/^\s+|\s+$/g, '');
- reversed = !!isReversed;
- var currentCache = (reversed) ? reverseCache : cache;
- if (currentCache[expression]) return currentCache[expression];
- parsed = {Slick: true, expressions: [], raw: expression, reverse: function(){
- return parse(this.raw, true);
- }};
- separatorIndex = -1;
- while (expression != (expression = expression.replace(regexp, parser)));
- parsed.length = parsed.expressions.length;
- return currentCache[expression] = (reversed) ? reverse(parsed) : parsed;
-};
-
-var reverseCombinator = function(combinator){
- if (combinator === '!') return ' ';
- else if (combinator === ' ') return '!';
- else if ((/^!/).test(combinator)) return combinator.replace(/^!/, '');
- else return '!' + combinator;
-};
-
-var reverse = function(expression){
- var expressions = expression.expressions;
- for (var i = 0; i < expressions.length; i++){
- var exp = expressions[i];
- var last = {parts: [], tag: '*', combinator: reverseCombinator(exp[0].combinator)};
-
- for (var j = 0; j < exp.length; j++){
- var cexp = exp[j];
- if (!cexp.reverseCombinator) cexp.reverseCombinator = ' ';
- cexp.combinator = cexp.reverseCombinator;
- delete cexp.reverseCombinator;
- }
-
- exp.reverse().push(last);
- }
- return expression;
-};
-
-var escapeRegExp = function(string){// Credit: XRegExp 0.6.1 (c) 2007-2008 Steven Levithan <http://stevenlevithan.com/regex/xregexp/> MIT License
- return string.replace(/[-[\]{}()*+?.\\^$|,#\s]/g, "\\$&");
-};
-
-var regexp = new RegExp(
-/*
-#!/usr/bin/env ruby
-puts "\t\t" + DATA.read.gsub(/\(\?x\)|\s+#.*$|\s+|\\$|\\n/,'')
-__END__
- "(?x)^(?:\
- \\s* ( , ) \\s* # Separator \n\
- | \\s* ( <combinator>+ ) \\s* # Combinator \n\
- | ( \\s+ ) # CombinatorChildren \n\
- | ( <unicode>+ | \\* ) # Tag \n\
- | \\# ( <unicode>+ ) # ID \n\
- | \\. ( <unicode>+ ) # ClassName \n\
- | # Attribute \n\
- \\[ \
- \\s* (<unicode1>+) (?: \
- \\s* ([*^$!~|]?=) (?: \
- \\s* (?:\
- ([\"']?)(.*?)\\9 \
- )\
- ) \
- )? \\s* \
- \\](?!\\]) \n\
- | :+ ( <unicode>+ )(?:\
- \\( (?:\
- (?:([\"'])([^\\12]*)\\12)|((?:\\([^)]+\\)|[^()]*)+)\
- ) \\)\
- )?\
- )"
-*/
- "^(?:\\s*(,)\\s*|\\s*(<combinator>+)\\s*|(\\s+)|(<unicode>+|\\*)|\\#(<unicode>+)|\\.(<unicode>+)|\\[\\s*(<unicode1>+)(?:\\s*([*^$!~|]?=)(?:\\s*(?:([\"']?)(.*?)\\9)))?\\s*\\](?!\\])|:+(<unicode>+)(?:\\((?:(?:([\"'])([^\\12]*)\\12)|((?:\\([^)]+\\)|[^()]*)+))\\))?)"
- .replace(/<combinator>/, '[' + escapeRegExp(">+~`!@$%^&={}\\;</") + ']')
- .replace(/<unicode>/g, '(?:[\\w\\u00a1-\\uFFFF-]|\\\\[^\\s0-9a-f])')
- .replace(/<unicode1>/g, '(?:[:\\w\\u00a1-\\uFFFF-]|\\\\[^\\s0-9a-f])')
-);
-
-function parser(
- rawMatch,
-
- separator,
- combinator,
- combinatorChildren,
-
- tagName,
- id,
- className,
-
- attributeKey,
- attributeOperator,
- attributeQuote,
- attributeValue,
-
- pseudoClass,
- pseudoQuote,
- pseudoClassQuotedValue,
- pseudoClassValue
-){
- if (separator || separatorIndex === -1){
- parsed.expressions[++separatorIndex] = [];
- combinatorIndex = -1;
- if (separator) return '';
- }
-
- if (combinator || combinatorChildren || combinatorIndex === -1){
- combinator = combinator || ' ';
- var currentSeparator = parsed.expressions[separatorIndex];
- if (reversed && currentSeparator[combinatorIndex])
- currentSeparator[combinatorIndex].reverseCombinator = reverseCombinator(combinator);
- currentSeparator[++combinatorIndex] = {combinator: combinator, tag: '*'};
- }
-
- var currentParsed = parsed.expressions[separatorIndex][combinatorIndex];
-
- if (tagName){
- currentParsed.tag = tagName.replace(reUnescape, '');
-
- } else if (id){
- currentParsed.id = id.replace(reUnescape, '');
-
- } else if (className){
- className = className.replace(reUnescape, '');
-
- if (!currentParsed.classList) currentParsed.classList = [];
- if (!currentParsed.classes) currentParsed.classes = [];
- currentParsed.classList.push(className);
- currentParsed.classes.push({
- value: className,
- regexp: new RegExp('(^|\\s)' + escapeRegExp(className) + '(\\s|$)')
- });
-
- } else if (pseudoClass){
- pseudoClassValue = pseudoClassValue || pseudoClassQuotedValue;
- pseudoClassValue = pseudoClassValue ? pseudoClassValue.replace(reUnescape, '') : null;
-
- if (!currentParsed.pseudos) currentParsed.pseudos = [];
- currentParsed.pseudos.push({
- key: pseudoClass.replace(reUnescape, ''),
- value: pseudoClassValue
- });
-
- } else if (attributeKey){
- attributeKey = attributeKey.replace(reUnescape, '');
- attributeValue = (attributeValue || '').replace(reUnescape, '');
-
- var test, regexp;
-
- switch (attributeOperator){
- case '^=' : regexp = new RegExp( '^'+ escapeRegExp(attributeValue) ); break;
- case '$=' : regexp = new RegExp( escapeRegExp(attributeValue) +'$' ); break;
- case '~=' : regexp = new RegExp( '(^|\\s)'+ escapeRegExp(attributeValue) +'(\\s|$)' ); break;
- case '|=' : regexp = new RegExp( '^'+ escapeRegExp(attributeValue) +'(-|$)' ); break;
- case '=' : test = function(value){
- return attributeValue == value;
- }; break;
- case '*=' : test = function(value){
- return value && value.indexOf(attributeValue) > -1;
- }; break;
- case '!=' : test = function(value){
- return attributeValue != value;
- }; break;
- default : test = function(value){
- return !!value;
- };
- }
-
- if (attributeValue == '' && (/^[*$^]=$/).test(attributeOperator)) test = function(){
- return false;
- };
-
- if (!test) test = function(value){
- return value && regexp.test(value);
- };
-
- if (!currentParsed.attributes) currentParsed.attributes = [];
- currentParsed.attributes.push({
- key: attributeKey,
- operator: attributeOperator,
- value: attributeValue,
- test: test
- });
-
- }
-
- return '';
-};
-
-// Slick NS
-
-var Slick = (this.Slick || {});
-
-Slick.parse = function(expression){
- return parse(expression);
-};
-
-Slick.escapeRegExp = escapeRegExp;
-
-if (!this.Slick) this.Slick = Slick;
-
-}).apply(/*<CommonJS>*/(typeof exports != 'undefined') ? exports : /*</CommonJS>*/this);
-
-
-/*
----
-name: Slick.Finder
-description: The new, superfast css selector engine.
-provides: Slick.Finder
-requires: Slick.Parser
-...
-*/
-
-(function(){
-
-var local = {};
-
-// Feature / Bug detection
-
-local.isNativeCode = function(fn){
- return (/\{\s*\[native code\]\s*\}/).test('' + fn);
-};
-
-local.isXML = function(document){
- return (!!document.xmlVersion) || (!!document.xml) || (Object.prototype.toString.call(document) === '[object XMLDocument]') ||
- (document.nodeType === 9 && document.documentElement.nodeName !== 'HTML');
-};
-
-local.setDocument = function(document){
-
- // convert elements / window arguments to document. if document cannot be extrapolated, the function returns.
-
- if (document.nodeType === 9); // document
- else if (document.ownerDocument) document = document.ownerDocument; // node
- else if (document.navigator) document = document.document; // window
- else return;
-
- // check if it's the old document
-
- if (this.document === document) return;
- this.document = document;
- var root = this.root = document.documentElement;
-
- this.isXMLDocument = this.isXML(document);
-
- this.brokenStarGEBTN
- = this.starSelectsClosedQSA
- = this.idGetsName
- = this.brokenMixedCaseQSA
- = this.brokenGEBCN
- = this.brokenCheckedQSA
- = this.brokenEmptyAttributeQSA
- = this.isHTMLDocument
- = false;
-
- var starSelectsClosed, starSelectsComments,
- brokenSecondClassNameGEBCN, cachedGetElementsByClassName;
-
- var selected, id;
- var testNode = document.createElement('div');
- root.appendChild(testNode);
-
- // on non-HTML documents innerHTML and getElementsById doesnt work properly
- try {
- id = 'slick_getbyid_test';
- testNode.innerHTML = '<a id="'+id+'"></a>';
- this.isHTMLDocument = !!document.getElementById(id);
- } catch(e){};
-
- if (this.isHTMLDocument){
-
- testNode.style.display = 'none';
-
- // IE returns comment nodes for getElementsByTagName('*') for some documents
- testNode.appendChild(document.createComment(''));
- starSelectsComments = (testNode.getElementsByTagName('*').length > 0);
-
- // IE returns closed nodes (EG:"</foo>") for getElementsByTagName('*') for some documents
- try {
- testNode.innerHTML = 'foo</foo>';
- selected = testNode.getElementsByTagName('*');
- starSelectsClosed = (selected && selected.length && selected[0].nodeName.charAt(0) == '/');
- } catch(e){};
-
- this.brokenStarGEBTN = starSelectsComments || starSelectsClosed;
-
- // IE 8 returns closed nodes (EG:"</foo>") for querySelectorAll('*') for some documents
- if (testNode.querySelectorAll) try {
- testNode.innerHTML = 'foo</foo>';
- selected = testNode.querySelectorAll('*');
- this.starSelectsClosedQSA = (selected && selected.length && selected[0].nodeName.charAt(0) == '/');
- } catch(e){};
-
- // IE returns elements with the name instead of just id for getElementsById for some documents
- try {
- id = 'slick_id_gets_name';
- testNode.innerHTML = '<a name="'+id+'"></a><b id="'+id+'"></b>';
- this.idGetsName = document.getElementById(id) === testNode.firstChild;
- } catch(e){};
-
- // Safari 3.2 querySelectorAll doesnt work with mixedcase on quirksmode
- try {
- testNode.innerHTML = '<a class="MiXedCaSe"></a>';
- this.brokenMixedCaseQSA = !testNode.querySelectorAll('.MiXedCaSe').length;
- } catch(e){};
-
- try {
- testNode.innerHTML = '<a class="f"></a><a class="b"></a>';
- testNode.getElementsByClassName('b').length;
- testNode.firstChild.className = 'b';
- cachedGetElementsByClassName = (testNode.getElementsByClassName('b').length != 2);
- } catch(e){};
-
- // Opera 9.6 getElementsByClassName doesnt detects the class if its not the first one
- try {
- testNode.innerHTML = '<a class="a"></a><a class="f b a"></a>';
- brokenSecondClassNameGEBCN = (testNode.getElementsByClassName('a').length != 2);
- } catch(e){};
-
- this.brokenGEBCN = cachedGetElementsByClassName || brokenSecondClassNameGEBCN;
-
- // Webkit dont return selected options on querySelectorAll
- try {
- testNode.innerHTML = '<select><option selected="selected">a</option></select>';
- this.brokenCheckedQSA = (testNode.querySelectorAll(':checked').length == 0);
- } catch(e){};
-
- // IE returns incorrect results for attr[*^$]="" selectors on querySelectorAll
- try {
- testNode.innerHTML = '<a class=""></a>';
- this.brokenEmptyAttributeQSA = (testNode.querySelectorAll('[class*=""]').length != 0);
- } catch(e){};
-
- }
-
- root.removeChild(testNode);
- testNode = null;
-
- // hasAttribute
-
- this.hasAttribute = (root && this.isNativeCode(root.hasAttribute)) ? function(node, attribute) {
- return node.hasAttribute(attribute);
- } : function(node, attribute) {
- node = node.getAttributeNode(attribute);
- return !!(node && (node.specified || node.nodeValue));
- };
-
- // contains
- // FIXME: Add specs: local.contains should be different for xml and html documents?
- this.contains = (root && this.isNativeCode(root.contains)) ? function(context, node){
- return context.contains(node);
- } : (root && root.compareDocumentPosition) ? function(context, node){
- return context === node || !!(context.compareDocumentPosition(node) & 16);
- } : function(context, node){
- if (node) do {
- if (node === context) return true;
- } while ((node = node.parentNode));
- return false;
- };
-
- // document order sorting
- // credits to Sizzle (http://sizzlejs.com/)
-
- this.documentSorter = (root.compareDocumentPosition) ? function(a, b){
- if (!a.compareDocumentPosition || !b.compareDocumentPosition) return 0;
- return a.compareDocumentPosition(b) & 4 ? -1 : a === b ? 0 : 1;
- } : ('sourceIndex' in root) ? function(a, b){
- if (!a.sourceIndex || !b.sourceIndex) return 0;
- return a.sourceIndex - b.sourceIndex;
- } : (document.createRange) ? function(a, b){
- if (!a.ownerDocument || !b.ownerDocument) return 0;
- var aRange = a.ownerDocument.createRange(), bRange = b.ownerDocument.createRange();
- aRange.setStart(a, 0);
- aRange.setEnd(a, 0);
- bRange.setStart(b, 0);
- bRange.setEnd(b, 0);
- return aRange.compareBoundaryPoints(Range.START_TO_END, bRange);
- } : null ;
-
- this.getUID = (this.isHTMLDocument) ? this.getUIDHTML : this.getUIDXML;
-
-};
-
-// Main Method
-
-local.search = function(context, expression, append, first){
-
- var found = this.found = (first) ? null : (append || []);
-
- // context checks
-
- if (!context) return found; // No context
- if (context.navigator) context = context.document; // Convert the node from a window to a document
- else if (!context.nodeType) return found; // Reject misc junk input
-
- // setup
-
- var parsed, i;
-
- var uniques = this.uniques = {};
-
- if (this.document !== (context.ownerDocument || context)) this.setDocument(context);
-
- // should sort if there are nodes in append and if you pass multiple expressions.
- // should remove duplicates if append already has items
- var shouldUniques = !!(append && append.length);
-
- // avoid duplicating items already in the append array
- if (shouldUniques) for (i = found.length; i--;) this.uniques[this.getUID(found[i])] = true;
-
- // expression checks
-
- if (typeof expression == 'string'){ // expression is a string
-
- // Overrides
-
- for (i = this.overrides.length; i--;){
- var override = this.overrides[i];
- if (override.regexp.test(expression)){
- var result = override.method.call(context, expression, found, first);
- if (result === false) continue;
- if (result === true) return found;
- return result;
- }
- }
-
- parsed = this.Slick.parse(expression);
- if (!parsed.length) return found;
- } else if (expression == null){ // there is no expression
- return found;
- } else if (expression.Slick){ // expression is a parsed Slick object
- parsed = expression;
- } else if (this.contains(context.documentElement || context, expression)){ // expression is a node
- (found) ? found.push(expression) : found = expression;
- return found;
- } else { // other junk
- return found;
- }
-
- // cache elements for the nth selectors
-
- /*<pseudo-selectors>*//*<nth-pseudo-selectors>*/
-
- this.posNTH = {};
- this.posNTHLast = {};
- this.posNTHType = {};
- this.posNTHTypeLast = {};
-
- /*</nth-pseudo-selectors>*//*</pseudo-selectors>*/
-
- // if append is null and there is only a single selector with one expression use pushArray, else use pushUID
- this.push = (!shouldUniques && (first || (parsed.length == 1 && parsed.expressions[0].length == 1))) ? this.pushArray : this.pushUID;
-
- if (found == null) found = [];
-
- // default engine
-
- var j, m, n;
- var combinator, tag, id, classList, classes, attributes, pseudos;
- var currentItems, currentExpression, currentBit, lastBit, expressions = parsed.expressions;
-
- search: for (i = 0; (currentExpression = expressions[i]); i++) for (j = 0; (currentBit = currentExpression[j]); j++){
-
- combinator = 'combinator:' + currentBit.combinator;
- if (!this[combinator]) continue search;
-
- tag = (this.isXMLDocument) ? currentBit.tag : currentBit.tag.toUpperCase();
- id = currentBit.id;
- classList = currentBit.classList;
- classes = currentBit.classes;
- attributes = currentBit.attributes;
- pseudos = currentBit.pseudos;
- lastBit = (j === (currentExpression.length - 1));
-
- this.bitUniques = {};
-
- if (lastBit){
- this.uniques = uniques;
- this.found = found;
- } else {
- this.uniques = {};
- this.found = [];
- }
-
- if (j === 0){
- this[combinator](context, tag, id, classes, attributes, pseudos, classList);
- if (first && lastBit && found.length) break search;
- } else {
- if (first && lastBit) for (m = 0, n = currentItems.length; m < n; m++){
- this[combinator](currentItems[m], tag, id, classes, attributes, pseudos, classList);
- if (found.length) break search;
- } else for (m = 0, n = currentItems.length; m < n; m++) this[combinator](currentItems[m], tag, id, classes, attributes, pseudos, classList);
- }
-
- currentItems = this.found;
- }
-
- if (shouldUniques || (parsed.expressions.length > 1)) this.sort(found);
-
- return (first) ? (found[0] || null) : found;
-};
-
-// Utils
-
-local.uidx = 1;
-local.uidk = 'slick:uniqueid';
-
-local.getUIDXML = function(node){
- var uid = node.getAttribute(this.uidk);
- if (!uid){
- uid = this.uidx++;
- node.setAttribute(this.uidk, uid);
- }
- return uid;
-};
-
-local.getUIDHTML = function(node){
- return node.uniqueNumber || (node.uniqueNumber = this.uidx++);
-};
-
-// sort based on the setDocument documentSorter method.
-
-local.sort = function(results){
- if (!this.documentSorter) return results;
- results.sort(this.documentSorter);
- return results;
-};
-
-/*<pseudo-selectors>*//*<nth-pseudo-selectors>*/
-
-local.cacheNTH = {};
-
-local.matchNTH = /^([+-]?\d*)?([a-z]+)?([+-]\d+)?$/;
-
-local.parseNTHArgument = function(argument){
- var parsed = argument.match(this.matchNTH);
- if (!parsed) return false;
- var special = parsed[2] || false;
- var a = parsed[1] || 1;
- if (a == '-') a = -1;
- var b = +parsed[3] || 0;
- parsed =
- (special == 'n') ? {a: a, b: b} :
- (special == 'odd') ? {a: 2, b: 1} :
- (special == 'even') ? {a: 2, b: 0} : {a: 0, b: a};
-
- return (this.cacheNTH[argument] = parsed);
-};
-
-local.createNTHPseudo = function(child, sibling, positions, ofType){
- return function(node, argument){
- var uid = this.getUID(node);
- if (!this[positions][uid]){
- var parent = node.parentNode;
- if (!parent) return false;
- var el = parent[child], count = 1;
- if (ofType){
- var nodeName = node.nodeName;
- do {
- if (el.nodeName !== nodeName) continue;
- this[positions][this.getUID(el)] = count++;
- } while ((el = el[sibling]));
- } else {
- do {
- if (el.nodeType !== 1) continue;
- this[positions][this.getUID(el)] = count++;
- } while ((el = el[sibling]));
- }
- }
- argument = argument || 'n';
- var parsed = this.cacheNTH[argument] || this.parseNTHArgument(argument);
- if (!parsed) return false;
- var a = parsed.a, b = parsed.b, pos = this[positions][uid];
- if (a == 0) return b == pos;
- if (a > 0){
- if (pos < b) return false;
- } else {
- if (b < pos) return false;
- }
- return ((pos - b) % a) == 0;
- };
-};
-
-/*</nth-pseudo-selectors>*//*</pseudo-selectors>*/
-
-local.pushArray = function(node, tag, id, classes, attributes, pseudos){
- if (this.matchSelector(node, tag, id, classes, attributes, pseudos)) this.found.push(node);
-};
-
-local.pushUID = function(node, tag, id, classes, attributes, pseudos){
- var uid = this.getUID(node);
- if (!this.uniques[uid] && this.matchSelector(node, tag, id, classes, attributes, pseudos)){
- this.uniques[uid] = true;
- this.found.push(node);
- }
-};
-
-local.matchNode = function(node, selector){
- var parsed = this.Slick.parse(selector);
- if (!parsed) return true;
-
- // simple (single) selectors
- if(parsed.length == 1 && parsed.expressions[0].length == 1){
- var exp = parsed.expressions[0][0];
- return this.matchSelector(node, (this.isXMLDocument) ? exp.tag : exp.tag.toUpperCase(), exp.id, exp.classes, exp.attributes, exp.pseudos);
- }
-
- var nodes = this.search(this.document, parsed);
- for (var i = 0, item; item = nodes[i++];){
- if (item === node) return true;
- }
- return false;
-};
-
-local.matchPseudo = function(node, name, argument){
- var pseudoName = 'pseudo:' + name;
- if (this[pseudoName]) return this[pseudoName](node, argument);
- var attribute = this.getAttribute(node, name);
- return (argument) ? argument == attribute : !!attribute;
-};
-
-local.matchSelector = function(node, tag, id, classes, attributes, pseudos){
- if (tag){
- if (tag == '*'){
- if (node.nodeName < '@') return false; // Fix for comment nodes and closed nodes
- } else {
- if (node.nodeName != tag) return false;
- }
- }
-
- if (id && node.getAttribute('id') != id) return false;
-
- var i, part, cls;
- if (classes) for (i = classes.length; i--;){
- cls = ('className' in node) ? node.className : node.getAttribute('class');
- if (!(cls && classes[i].regexp.test(cls))) return false;
- }
- if (attributes) for (i = attributes.length; i--;){
- part = attributes[i];
- if (part.operator ? !part.test(this.getAttribute(node, part.key)) : !this.hasAttribute(node, part.key)) return false;
- }
- if (pseudos) for (i = pseudos.length; i--;){
- part = pseudos[i];
- if (!this.matchPseudo(node, part.key, part.value)) return false;
- }
- return true;
-};
-
-var combinators = {
-
- ' ': function(node, tag, id, classes, attributes, pseudos, classList){ // all child nodes, any level
-
- var i, item, children;
-
- if (this.isHTMLDocument){
- getById: if (id){
- item = this.document.getElementById(id);
- if ((!item && node.all) || (this.idGetsName && item && item.getAttributeNode('id').nodeValue != id)){
- // all[id] returns all the elements with that name or id inside node
- // if theres just one it will return the element, else it will be a collection
- children = node.all[id];
- if (!children) return;
- if (!children[0]) children = [children];
- for (i = 0; item = children[i++];) if (item.getAttributeNode('id').nodeValue == id){
- this.push(item, tag, null, classes, attributes, pseudos);
- break;
- }
- return;
- }
- if (!item){
- // if the context is in the dom we return, else we will try GEBTN, breaking the getById label
- if (this.contains(this.document.documentElement, node)) return;
- else break getById;
- } else if (this.document !== node && !this.contains(node, item)) return;
- this.push(item, tag, null, classes, attributes, pseudos);
- return;
- }
- getByClass: if (classes && node.getElementsByClassName && !this.brokenGEBCN){
- children = node.getElementsByClassName(classList.join(' '));
- if (!(children && children.length)) break getByClass;
- for (i = 0; item = children[i++];) this.push(item, tag, id, null, attributes, pseudos);
- return;
- }
- }
- getByTag: {
- children = node.getElementsByTagName(tag);
- if (!(children && children.length)) break getByTag;
- if (!this.brokenStarGEBTN) tag = null;
- for (i = 0; item = children[i++];) this.push(item, tag, id, classes, attributes, pseudos);
- }
- },
-
- '>': function(node, tag, id, classes, attributes, pseudos){ // direct children
- if ((node = node.firstChild)) do {
- if (node.nodeType === 1) this.push(node, tag, id, classes, attributes, pseudos);
- } while ((node = node.nextSibling));
- },
-
- '+': function(node, tag, id, classes, attributes, pseudos){ // next sibling
- while ((node = node.nextSibling)) if (node.nodeType === 1){
- this.push(node, tag, id, classes, attributes, pseudos);
- break;
- }
- },
-
- '^': function(node, tag, id, classes, attributes, pseudos){ // first child
- node = node.firstChild;
- if (node){
- if (node.nodeType === 1) this.push(node, tag, id, classes, attributes, pseudos);
- else this['combinator:+'](node, tag, id, classes, attributes, pseudos);
- }
- },
-
- '~': function(node, tag, id, classes, attributes, pseudos){ // next siblings
- while ((node = node.nextSibling)){
- if (node.nodeType !== 1) continue;
- var uid = this.getUID(node);
- if (this.bitUniques[uid]) break;
- this.bitUniques[uid] = true;
- this.push(node, tag, id, classes, attributes, pseudos);
- }
- },
-
- '++': function(node, tag, id, classes, attributes, pseudos){ // next sibling and previous sibling
- this['combinator:+'](node, tag, id, classes, attributes, pseudos);
- this['combinator:!+'](node, tag, id, classes, attributes, pseudos);
- },
-
- '~~': function(node, tag, id, classes, attributes, pseudos){ // next siblings and previous siblings
- this['combinator:~'](node, tag, id, classes, attributes, pseudos);
- this['combinator:!~'](node, tag, id, classes, attributes, pseudos);
- },
-
- '!': function(node, tag, id, classes, attributes, pseudos){ // all parent nodes up to document
- while ((node = node.parentNode)) if (node !== this.document) this.push(node, tag, id, classes, attributes, pseudos);
- },
-
- '!>': function(node, tag, id, classes, attributes, pseudos){ // direct parent (one level)
- node = node.parentNode;
- if (node !== this.document) this.push(node, tag, id, classes, attributes, pseudos);
- },
-
- '!+': function(node, tag, id, classes, attributes, pseudos){ // previous sibling
- while ((node = node.previousSibling)) if (node.nodeType === 1){
- this.push(node, tag, id, classes, attributes, pseudos);
- break;
- }
- },
-
- '!^': function(node, tag, id, classes, attributes, pseudos){ // last child
- node = node.lastChild;
- if (node){
- if (node.nodeType === 1) this.push(node, tag, id, classes, attributes, pseudos);
- else this['combinator:!+'](node, tag, id, classes, attributes, pseudos);
- }
- },
-
- '!~': function(node, tag, id, classes, attributes, pseudos){ // previous siblings
- while ((node = node.previousSibling)){
- if (node.nodeType !== 1) continue;
- var uid = this.getUID(node);
- if (this.bitUniques[uid]) break;
- this.bitUniques[uid] = true;
- this.push(node, tag, id, classes, attributes, pseudos);
- }
- }
-
-};
-
-for (var c in combinators) local['combinator:' + c] = combinators[c];
-
-var pseudos = {
-
- /*<pseudo-selectors>*/
-
- 'empty': function(node){
- var child = node.firstChild;
- return !(child && child.nodeType == 1) && !(node.innerText || node.textContent || '').length;
- },
-
- 'not': function(node, expression){
- return !this.matchNode(node, expression);
- },
-
- 'contains': function(node, text){
- return (node.innerText || node.textContent || '').indexOf(text) > -1;
- },
-
- 'first-child': function(node){
- while ((node = node.previousSibling)) if (node.nodeType === 1) return false;
- return true;
- },
-
- 'last-child': function(node){
- while ((node = node.nextSibling)) if (node.nodeType === 1) return false;
- return true;
- },
-
- 'only-child': function(node){
- var prev = node;
- while ((prev = prev.previousSibling)) if (prev.nodeType === 1) return false;
- var next = node;
- while ((next = next.nextSibling)) if (next.nodeType === 1) return false;
- return true;
- },
-
- /*<nth-pseudo-selectors>*/
-
- 'nth-child': local.createNTHPseudo('firstChild', 'nextSibling', 'posNTH'),
-
- 'nth-last-child': local.createNTHPseudo('lastChild', 'previousSibling', 'posNTHLast'),
-
- 'nth-of-type': local.createNTHPseudo('firstChild', 'nextSibling', 'posNTHType', true),
-
- 'nth-last-of-type': local.createNTHPseudo('lastChild', 'previousSibling', 'posNTHTypeLast', true),
-
- 'index': function(node, index){
- return this['pseudo:nth-child'](node, '' + index + 1);
- },
-
- 'even': function(node, argument){
- return this['pseudo:nth-child'](node, '2n');
- },
-
- 'odd': function(node, argument){
- return this['pseudo:nth-child'](node, '2n+1');
- },
-
- /*</nth-pseudo-selectors>*/
-
- /*<of-type-pseudo-selectors>*/
-
- 'first-of-type': function(node){
- var nodeName = node.nodeName;
- while ((node = node.previousSibling)) if (node.nodeName === nodeName) return false;
- return true;
- },
-
- 'last-of-type': function(node){
- var nodeName = node.nodeName;
- while ((node = node.nextSibling)) if (node.nodeName === nodeName) return false;
- return true;
- },
-
- 'only-of-type': function(node){
- var prev = node, nodeName = node.nodeName;
- while ((prev = prev.previousSibling)) if (prev.nodeName === nodeName) return false;
- var next = node;
- while ((next = next.nextSibling)) if (next.nodeName === nodeName) return false;
- return true;
- },
-
- /*</of-type-pseudo-selectors>*/
-
- // custom pseudos
-
- 'enabled': function(node){
- return (node.disabled === false);
- },
-
- 'disabled': function(node){
- return (node.disabled === true);
- },
-
- 'checked': function(node){
- return node.checked || node.selected;
- },
-
- 'focus': function(node){
- return this.isHTMLDocument && this.document.activeElement === node && (node.href || node.type || this.hasAttribute(node, 'tabindex'));
- },
-
- 'root': function(node){
- return (node === this.root);
- },
-
- 'selected': function(node){
- return node.selected;
- }
-
- /*</pseudo-selectors>*/
-};
-
-for (var p in pseudos) local['pseudo:' + p] = pseudos[p];
-
-// attributes methods
-
-local.attributeGetters = {
-
- 'class': function(){
- return ('className' in this) ? this.className : this.getAttribute('class');
- },
-
- 'for': function(){
- return ('htmlFor' in this) ? this.htmlFor : this.getAttribute('for');
- },
-
- 'href': function(){
- return ('href' in this) ? this.getAttribute('href', 2) : this.getAttribute('href');
- },
-
- 'style': function(){
- return (this.style) ? this.style.cssText : this.getAttribute('style');
- }
-
-};
-
-local.getAttribute = function(node, name){
- // FIXME: check if getAttribute() will get input elements on a form on this browser
- // getAttribute is faster than getAttributeNode().nodeValue
- var method = this.attributeGetters[name];
- if (method) return method.call(node);
- var attributeNode = node.getAttributeNode(name);
- return attributeNode ? attributeNode.nodeValue : null;
-};
-
-// overrides
-
-local.overrides = [];
-
-local.override = function(regexp, method){
- this.overrides.push({regexp: regexp, method: method});
-};
-
-/*<overrides>*/
-
-/*<query-selector-override>*/
-
-var reEmptyAttribute = /\[.*[*$^]=(?:["']{2})?\]/;
-
-local.override(/./, function(expression, found, first){ //querySelectorAll override
-
- if (!this.querySelectorAll || this.nodeType != 9 || !local.isHTMLDocument || local.brokenMixedCaseQSA ||
- (local.brokenCheckedQSA && expression.indexOf(':checked') > -1) ||
- (local.brokenEmptyAttributeQSA && reEmptyAttribute.test(expression)) || Slick.disableQSA) return false;
-
- var nodes, node;
- try {
- if (first) return this.querySelector(expression) || null;
- else nodes = this.querySelectorAll(expression);
- } catch(error){
- return false;
- }
-
- var i, hasOthers = !!(found.length);
-
- if (local.starSelectsClosedQSA) for (i = 0; node = nodes[i++];){
- if (node.nodeName > '@' && (!hasOthers || !local.uniques[local.getUIDHTML(node)])) found.push(node);
- } else for (i = 0; node = nodes[i++];){
- if (!hasOthers || !local.uniques[local.getUIDHTML(node)]) found.push(node);
- }
-
- if (hasOthers) local.sort(found);
-
- return true;
-
-});
-
-/*</query-selector-override>*/
-
-/*<tag-override>*/
-
-local.override(/^[\w-]+$|^\*$/, function(expression, found, first){ // tag override
- var tag = expression;
- if (tag == '*' && local.brokenStarGEBTN) return false;
-
- var nodes = this.getElementsByTagName(tag);
-
- if (first) return nodes[0] || null;
- var i, node, hasOthers = !!(found.length);
-
- for (i = 0; node = nodes[i++];){
- if (!hasOthers || !local.uniques[local.getUID(node)]) found.push(node);
- }
-
- if (hasOthers) local.sort(found);
-
- return true;
-});
-
-/*</tag-override>*/
-
-/*<class-override>*/
-
-local.override(/^\.[\w-]+$/, function(expression, found, first){ // class override
- if (!local.isHTMLDocument || (!this.getElementsByClassName && this.querySelectorAll)) return false;
-
- var nodes, node, i, hasOthers = !!(found && found.length), className = expression.substring(1);
- if (this.getElementsByClassName && !local.brokenGEBCN){
- nodes = this.getElementsByClassName(className);
- if (first) return nodes[0] || null;
- for (i = 0; node = nodes[i++];){
- if (!hasOthers || !local.uniques[local.getUIDHTML(node)]) found.push(node);
- }
- } else {
- var matchClass = new RegExp('(^|\\s)'+ Slick.escapeRegExp(className) +'(\\s|$)');
- nodes = this.getElementsByTagName('*');
- for (i = 0; node = nodes[i++];){
- className = node.className;
- if (!className || !matchClass.test(className)) continue;
- if (first) return node;
- if (!hasOthers || !local.uniques[local.getUIDHTML(node)]) found.push(node);
- }
- }
- if (hasOthers) local.sort(found);
- return (first) ? null : true;
-});
-
-/*</class-override>*/
-
-/*<id-override>*/
-
-local.override(/^#[\w-]+$/, function(expression, found, first){ // ID override
- if (!local.isHTMLDocument || this.nodeType != 9) return false;
-
- var id = expression.substring(1), el = this.getElementById(id);
- if (!el) return found;
- if (local.idGetsName && el.getAttributeNode('id').nodeValue != id) return false;
- if (first) return el || null;
- var hasOthers = !!(found.length);
- if (!hasOthers || !local.uniques[local.getUIDHTML(el)]) found.push(el);
- if (hasOthers) local.sort(found);
- return true;
-});
-
-/*</id-override>*/
-
-/*</overrides>*/
-
-if (typeof document != 'undefined') local.setDocument(document);
-
-// Slick
-
-var Slick = local.Slick = (this.Slick || {});
-
-Slick.version = '0.9dev';
-
-// Slick finder
-
-Slick.search = function(context, expression, append){
- return local.search(context, expression, append);
-};
-
-Slick.find = function(context, expression){
- return local.search(context, expression, null, true);
-};
-
-// Slick containment checker
-
-Slick.contains = function(container, node){
- local.setDocument(container);
- return local.contains(container, node);
-};
-
-// Slick attribute getter
-
-Slick.getAttribute = function(node, name){
- return local.getAttribute(node, name);
-};
-
-// Slick matcher
-
-Slick.match = function(node, selector){
- if (!(node && selector)) return false;
- if (!selector || selector === node) return true;
- if (typeof selector != 'string') return false;
- local.setDocument(node);
- return local.matchNode(node, selector);
-};
-
-// Slick attribute accessor
-
-Slick.defineAttributeGetter = function(name, fn){
- local.attributeGetters[name] = fn;
- return this;
-};
-
-Slick.lookupAttributeGetter = function(name){
- return local.attributeGetters[name];
-};
-
-// Slick pseudo accessor
-
-Slick.definePseudo = function(name, fn){
- local['pseudo:' + name] = function(node, argument){
- return fn.call(node, argument);
- };
- return this;
-};
-
-Slick.lookupPseudo = function(name){
- var pseudo = local['pseudo:' + name];
- if (pseudo) return function(argument){
- return pseudo.call(this, argument);
- };
- return null;
-};
-
-// Slick overrides accessor
-
-Slick.override = function(regexp, fn){
- local.override(regexp, fn);
- return this;
-};
-
-Slick.isXML = local.isXML;
-
-Slick.uidOf = function(node){
- return local.getUIDHTML(node);
-};
-
-if (!this.Slick) this.Slick = Slick;
-
-}).apply(/*<CommonJS>*/(typeof exports != 'undefined') ? exports : /*</CommonJS>*/this);
-
-
-/*
----
-
-name: Element
-
-description: One of the most important items in MooTools. Contains the dollar function, the dollars function, and an handful of cross-browser, time-saver methods to let you easily work with HTML Elements.
-
-license: MIT-style license.
-
-requires: [Window, Document, Array, String, Function, Number, Slick.Parser, Slick.Finder]
-
-provides: [Element, Elements, $, $$, Iframe, Selectors]
-
-...
-*/
-
-var Element = function(tag, props){
- var konstructor = Element.Constructors[tag];
- if (konstructor) return konstructor(props);
- if (typeof tag != 'string') return document.id(tag).set(props);
-
- if (!props) props = {};
-
- if (!tag.test(/^[\w-]+$/)){
- var parsed = Slick.parse(tag).expressions[0][0];
- tag = (parsed.tag == '*') ? 'div' : parsed.tag;
- if (parsed.id && props.id == null) props.id = parsed.id;
-
- var attributes = parsed.attributes;
- if (attributes) for (var i = 0, l = attributes.length; i < l; i++){
- var attr = attributes[i];
- if (attr.value != null && attr.operator == '=' && props[attr.key] == null)
- props[attr.key] = attr.value;
- }
-
- if (parsed.classList && props['class'] == null) props['class'] = parsed.classList.join(' ');
- }
-
- return document.newElement(tag, props);
-};
-
-if (Browser.Element) Element.prototype = Browser.Element.prototype;
-
-new Type('Element', Element).mirror(function(name){
- if (Array.prototype[name]) return;
-
- var obj = {};
- obj[name] = function(){
- var results = [], args = arguments, elements = true;
- for (var i = 0, l = this.length; i < l; i++){
- var element = this[i], result = results[i] = element[name].apply(element, args);
- elements = (elements && typeOf(result) == 'element');
- }
- return (elements) ? new Elements(results) : results;
- };
-
- Elements.implement(obj);
-});
-
-if (!Browser.Element){
- Element.parent = Object;
-
- Element.Prototype = {'$family': Function.from('element').hide()};
-
- Element.mirror(function(name, method){
- Element.Prototype[name] = method;
- });
-}
-
-Element.Constructors = {};
-
-//<1.2compat>
-
-Element.Constructors = new Hash;
-
-//</1.2compat>
-
-var IFrame = new Type('IFrame', function(){
- var params = Array.link(arguments, {
- properties: Type.isObject,
- iframe: function(obj){
- return (obj != null);
- }
- });
-
- var props = params.properties || {}, iframe;
- if (params.iframe) iframe = document.id(params.iframe);
- var onload = props.onload || function(){};
- delete props.onload;
- props.id = props.name = [props.id, props.name, iframe ? (iframe.id || iframe.name) : 'IFrame_' + String.generateUID()].pick();
- iframe = new Element(iframe || 'iframe', props);
-
- var onLoad = function(){
- onload.call(iframe.contentWindow);
- };
-
- if (window.frames[props.id]) onLoad();
- else iframe.addListener('load', onLoad);
- return iframe;
-});
-
-var Elements = this.Elements = function(nodes){
- if (nodes && nodes.length){
- var uniques = {}, node;
- for (var i = 0; node = nodes[i++];){
- var uid = Slick.uidOf(node);
- if (!uniques[uid]){
- uniques[uid] = true;
- this.push(node);
- }
- }
- }
-};
-
-Elements.prototype = {length: 0};
-Elements.parent = Array;
-
-new Type('Elements', Elements).implement({
-
- filter: function(filter, bind){
- if (!filter) return this;
- return new Elements(Array.filter(this, (typeOf(filter) == 'string') ? function(item){
- return item.match(filter);
- } : filter, bind));
- }.protect(),
-
- push: function(){
- var length = this.length;
- for (var i = 0, l = arguments.length; i < l; i++){
- var item = document.id(arguments[i]);
- if (item) this[length++] = item;
- }
- return (this.length = length);
- }.protect(),
-
- concat: function(){
- var newElements = new Elements(this);
- for (var i = 0, l = arguments.length; i < l; i++){
- var item = arguments[i];
- if (Type.isEnumerable(item)) newElements.append(item);
- else newElements.push(item);
- }
- return newElements;
- }.protect(),
-
- append: function(collection){
- for (var i = 0, l = collection.length; i < l; i++) this.push(collection[i]);
- return this;
- }.protect(),
-
- empty: function(){
- while (this.length) delete this[--this.length];
- return this;
- }.protect()
-
-});
-
-(function(){
-
-// FF, IE
-var splice = Array.prototype.splice, object = {'0': 0, '1': 1, length: 2};
-
-splice.call(object, 1, 1);
-if (object[1] == 1) Elements.implement('splice', function(){
- var length = this.length;
- splice.apply(this, arguments);
- while (length >= this.length) delete this[length--];
- return this;
-}.protect());
-
-Elements.implement(Array.prototype);
-
-Array.mirror(Elements);
-
-/*<ltIE8>*/
-var createElementAcceptsHTML;
-try {
- var x = document.createElement('<input name=x>');
- createElementAcceptsHTML = (x.name == 'x');
-} catch(e){}
-
-var escapeQuotes = function(html){
- return ('' + html).replace(/&/g, '&').replace(/"/g, '"');
-};
-/*</ltIE8>*/
-
-Document.implement({
-
- newElement: function(tag, props){
- if (props && props.checked != null) props.defaultChecked = props.checked;
- /*<ltIE8>*/// Fix for readonly name and type properties in IE < 8
- if (createElementAcceptsHTML && props){
- tag = '<' + tag;
- if (props.name) tag += ' name="' + escapeQuotes(props.name) + '"';
- if (props.type) tag += ' type="' + escapeQuotes(props.type) + '"';
- tag += '>';
- delete props.name;
- delete props.type;
- }
- /*</ltIE8>*/
- return this.id(this.createElement(tag)).set(props);
- }
-
-});
-
-})();
-
-Document.implement({
-
- newTextNode: function(text){
- return this.createTextNode(text);
- },
-
- getDocument: function(){
- return this;
- },
-
- getWindow: function(){
- return this.window;
- },
-
- id: (function(){
-
- var types = {
-
- string: function(id, nocash, doc){
- id = Slick.find(doc, '#' + id.replace(/(\W)/g, '\\$1'));
- return (id) ? types.element(id, nocash) : null;
- },
-
- element: function(el, nocash){
- $uid(el);
- if (!nocash && !el.$family && !(/^object|embed$/i).test(el.tagName)){
- Object.append(el, Element.Prototype);
- }
- return el;
- },
-
- object: function(obj, nocash, doc){
- if (obj.toElement) return types.element(obj.toElement(doc), nocash);
- return null;
- }
-
- };
-
- types.textnode = types.whitespace = types.window = types.document = function(zero){
- return zero;
- };
-
- return function(el, nocash, doc){
- if (el && el.$family && el.uid) return el;
- var type = typeOf(el);
- return (types[type]) ? types[type](el, nocash, doc || document) : null;
- };
-
- })()
-
-});
-
-if (window.$ == null) Window.implement('$', function(el, nc){
- return document.id(el, nc, this.document);
-});
-
-Window.implement({
-
- getDocument: function(){
- return this.document;
- },
-
- getWindow: function(){
- return this;
- }
-
-});
-
-[Document, Element].invoke('implement', {
-
- getElements: function(expression){
- return Slick.search(this, expression, new Elements);
- },
-
- getElement: function(expression){
- return document.id(Slick.find(this, expression));
- }
-
-});
-
-//<1.2compat>
-
-(function(search, find, match){
-
- this.Selectors = {};
- var pseudos = this.Selectors.Pseudo = new Hash();
-
- var addSlickPseudos = function(){
- for (var name in pseudos) if (pseudos.hasOwnProperty(name)){
- Slick.definePseudo(name, pseudos[name]);
- delete pseudos[name];
- }
- };
-
- Slick.search = function(context, expression, append){
- addSlickPseudos();
- return search.call(this, context, expression, append);
- };
-
- Slick.find = function(context, expression){
- addSlickPseudos();
- return find.call(this, context, expression);
- };
-
- Slick.match = function(node, selector){
- addSlickPseudos();
- return match.call(this, node, selector);
- };
-
-})(Slick.search, Slick.find, Slick.match);
-
-if (window.$$ == null) Window.implement('$$', function(selector){
- var elements = new Elements;
- if (arguments.length == 1 && typeof selector == 'string') return Slick.search(this.document, selector, elements);
- var args = Array.flatten(arguments);
- for (var i = 0, l = args.length; i < l; i++){
- var item = args[i];
- switch (typeOf(item)){
- case 'element': elements.push(item); break;
- case 'string': Slick.search(this.document, item, elements);
- }
- }
- return elements;
-});
-
-//</1.2compat>
-
-if (window.$$ == null) Window.implement('$$', function(selector){
- if (arguments.length == 1){
- if (typeof selector == 'string') return Slick.search(this.document, selector, new Elements);
- else if (Type.isEnumerable(selector)) return new Elements(selector);
- }
- return new Elements(arguments);
-});
-
-(function(){
-
-var collected = {}, storage = {};
-var props = {input: 'checked', option: 'selected', textarea: 'value'};
-
-var get = function(uid){
- return (storage[uid] || (storage[uid] = {}));
-};
-
-var clean = function(item){
- if (item.removeEvents) item.removeEvents();
- if (item.clearAttributes) item.clearAttributes();
- var uid = item.uid;
- if (uid != null){
- delete collected[uid];
- delete storage[uid];
- }
- return item;
-};
-
-var camels = ['defaultValue', 'accessKey', 'cellPadding', 'cellSpacing', 'colSpan', 'frameBorder', 'maxLength', 'readOnly',
- 'rowSpan', 'tabIndex', 'useMap'
-];
-var bools = ['compact', 'nowrap', 'ismap', 'declare', 'noshade', 'checked', 'disabled', 'readOnly', 'multiple', 'selected',
- 'noresize', 'defer'
-];
- var attributes = {
- 'html': 'innerHTML',
- 'class': 'className',
- 'for': 'htmlFor',
- 'text': (function(){
- var temp = document.createElement('div');
- return (temp.innerText == null) ? 'textContent' : 'innerText';
- })()
-};
-var readOnly = ['type'];
-var expandos = ['value', 'defaultValue'];
-var uriAttrs = /^(?:href|src|usemap)$/i;
-
-bools = bools.associate(bools);
-camels = camels.associate(camels.map(String.toLowerCase));
-readOnly = readOnly.associate(readOnly);
-
-Object.append(attributes, expandos.associate(expandos));
-
-var inserters = {
-
- before: function(context, element){
- var parent = element.parentNode;
- if (parent) parent.insertBefore(context, element);
- },
-
- after: function(context, element){
- var parent = element.parentNode;
- if (parent) parent.insertBefore(context, element.nextSibling);
- },
-
- bottom: function(context, element){
- element.appendChild(context);
- },
-
- top: function(context, element){
- element.insertBefore(context, element.firstChild);
- }
-
-};
-
-inserters.inside = inserters.bottom;
-
-//<1.2compat>
-
-Object.each(inserters, function(inserter, where){
-
- where = where.capitalize();
-
- var methods = {};
-
- methods['inject' + where] = function(el){
- inserter(this, document.id(el, true));
- return this;
- };
-
- methods['grab' + where] = function(el){
- inserter(document.id(el, true), this);
- return this;
- };
-
- Element.implement(methods);
-
-});
-
-//</1.2compat>
-
-var injectCombinator = function(expression, combinator){
- if (!expression) return combinator;
-
- expression = Slick.parse(expression);
-
- var expressions = expression.expressions;
- for (var i = expressions.length; i--;)
- expressions[i][0].combinator = combinator;
-
- return expression;
-};
-
-Element.implement({
-
- set: function(prop, value){
- var property = Element.Properties[prop];
- (property && property.set) ? property.set.call(this, value) : this.setProperty(prop, value);
- }.overloadSetter(),
-
- get: function(prop){
- var property = Element.Properties[prop];
- return (property && property.get) ? property.get.apply(this) : this.getProperty(prop);
- }.overloadGetter(),
-
- erase: function(prop){
- var property = Element.Properties[prop];
- (property && property.erase) ? property.erase.apply(this) : this.removeProperty(prop);
- return this;
- },
-
- setProperty: function(attribute, value){
- attribute = camels[attribute] || attribute;
- if (value == null) return this.removeProperty(attribute);
- var key = attributes[attribute];
- (key) ? this[key] = value :
- (bools[attribute]) ? this[attribute] = !!value : this.setAttribute(attribute, '' + value);
- return this;
- },
-
- setProperties: function(attributes){
- for (var attribute in attributes) this.setProperty(attribute, attributes[attribute]);
- return this;
- },
-
- getProperty: function(attribute){
- attribute = camels[attribute] || attribute;
- var key = attributes[attribute] || readOnly[attribute];
- return (key) ? this[key] :
- (bools[attribute]) ? !!this[attribute] :
- (uriAttrs.test(attribute) ? this.getAttribute(attribute, 2) :
- (key = this.getAttributeNode(attribute)) ? key.nodeValue : null) || null;
- },
-
- getProperties: function(){
- var args = Array.from(arguments);
- return args.map(this.getProperty, this).associate(args);
- },
-
- removeProperty: function(attribute){
- attribute = camels[attribute] || attribute;
- var key = attributes[attribute];
- (key) ? this[key] = '' :
- (bools[attribute]) ? this[attribute] = false : this.removeAttribute(attribute);
- return this;
- },
-
- removeProperties: function(){
- Array.each(arguments, this.removeProperty, this);
- return this;
- },
-
- hasClass: function(className){
- return this.className.clean().contains(className, ' ');
- },
-
- addClass: function(className){
- if (!this.hasClass(className)) this
<TRUNCATED>
[08/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/assets/bootstrap.css
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/assets/bootstrap.css b/src/main/python/twitter/thermos/observer/http/assets/bootstrap.css
deleted file mode 100644
index 1c75d0c..0000000
--- a/src/main/python/twitter/thermos/observer/http/assets/bootstrap.css
+++ /dev/null
@@ -1,9 +0,0 @@
-/*!
- * Bootstrap v2.0.3
- *
- * Copyright 2012 Twitter, Inc
- * Licensed under the Apache License v2.0
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Designed and built with all the love in the world @twitter by @mdo and @fat.
- */article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-app
earance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:28px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box}body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;line-height:18px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover{color:#005580;text-decoration:underline}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;content:""}.row:after{clear:both}[class*="span"]{float:left;margin-left:20px}.container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.span
12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offset1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:28px;margin-left:2.127659574%;*margin-left:2.0744680846382977%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:99.9999
9998999999%;*width:99.94680850063828%}.row-fluid .span11{width:91.489361693%;*width:91.4361702036383%}.row-fluid .span10{width:82.97872339599999%;*width:82.92553190663828%}.row-fluid .span9{width:74.468085099%;*width:74.4148936096383%}.row-fluid .span8{width:65.95744680199999%;*width:65.90425531263828%}.row-fluid .span7{width:57.446808505%;*width:57.3936170156383%}.row-fluid .span6{width:48.93617020799999%;*width:48.88297871863829%}.row-fluid .span5{width:40.425531911%;*width:40.3723404216383%}.row-fluid .span4{width:31.914893614%;*width:31.8617021246383%}.row-fluid .span3{width:23.404255317%;*width:23.3510638276383%}.row-fluid .span2{width:14.89361702%;*width:14.8404255306383%}.row-fluid .span1{width:6.382978723%;*width:6.329787233638298%}.container{margin-right:auto;margin-left:auto;*zoom:1}.container:before,.container:after{display:table;content:""}.container:after{clear:both}.container-fluid{padding-right:20px;padding-left:20px;*zoom:1}.container-fluid:before,.container-fluid:af
ter{display:table;content:""}.container-fluid:after{clear:both}p{margin:0 0 9px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;line-height:18px}p small{font-size:11px;color:#999}.lead{margin-bottom:18px;font-size:20px;font-weight:200;line-height:27px}h1,h2,h3,h4,h5,h6{margin:0;font-family:inherit;font-weight:bold;color:inherit;text-rendering:optimizelegibility}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small{font-weight:normal;color:#999}h1{font-size:30px;line-height:36px}h1 small{font-size:18px}h2{font-size:24px;line-height:36px}h2 small{font-size:18px}h3{font-size:18px;line-height:27px}h3 small{font-size:14px}h4,h5,h6{line-height:18px}h4{font-size:14px}h4 small{font-size:12px}h5{font-size:12px}h6{font-size:11px;color:#999;text-transform:uppercase}.page-header{padding-bottom:17px;margin:18px 0;border-bottom:1px solid #eee}.page-header h1{line-height:1}ul,ol{padding:0;margin:0 0 9px 25px}ul ul,ul ol,ol ol,ol ul{margin-bottom:0}ul{list-style:disc}ol{list-
style:decimal}li{line-height:18px}ul.unstyled,ol.unstyled{margin-left:0;list-style:none}dl{margin-bottom:18px}dt,dd{line-height:18px}dt{font-weight:bold;line-height:17px}dd{margin-left:9px}.dl-horizontal dt{float:left;width:120px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:130px}hr{margin:18px 0;border:0;border-top:1px solid #eee;border-bottom:1px solid #fff}strong{font-weight:bold}em{font-style:italic}.muted{color:#999}abbr[title]{cursor:help;border-bottom:1px dotted #ddd}abbr.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:0 0 0 15px;margin:0 0 18px;border-left:5px solid #eee}blockquote p{margin-bottom:0;font-size:16px;font-weight:300;line-height:22.5px}blockquote small{display:block;line-height:18px;color:#999}blockquote small:before{content:'\2014 \00A0'}blockquote.pull-right{float:right;padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0}blockquote.pull-right p,b
lockquote.pull-right small{text-align:right}q:before,q:after,blockquote:before,blockquote:after{content:""}address{display:block;margin-bottom:18px;font-style:normal;line-height:18px}small{font-size:100%}cite{font-style:normal}code,pre{padding:0 3px 2px;font-family:Menlo,Monaco,Consolas,"Courier New",monospace;font-size:12px;color:#333;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}code{padding:2px 4px;color:#d14;background-color:#f7f7f9;border:1px solid #e1e1e8}pre{display:block;padding:8.5px;margin:0 0 9px;font-size:12.025px;line-height:18px;word-break:break-all;word-wrap:break-word;white-space:pre;white-space:pre-wrap;background-color:#f5f5f5;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}pre.prettyprint{margin-bottom:18px}pre code{padding:0;color:inherit;background-color:transparent;border:0}.pre-scrollable{max-height:340px;overflow-y:scroll}form{margin:0 0 18px}fieldset{padding:0;mar
gin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:27px;font-size:19.5px;line-height:36px;color:#333;border:0;border-bottom:1px solid #eee}legend small{font-size:13.5px;color:#999}label,input,button,select,textarea{font-size:13px;font-weight:normal;line-height:18px}input,button,select,textarea{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}label{display:block;margin-bottom:5px;color:#333}input,textarea,select,.uneditable-input{display:inline-block;width:210px;height:18px;padding:4px;margin-bottom:9px;font-size:13px;line-height:18px;color:#555;background-color:#fff;border:1px solid #ccc;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.uneditable-textarea{width:auto;height:auto}label input,label textarea,label select{display:block}input[type="image"],input[type="checkbox"],input[type="radio"]{width:auto;height:auto;padding:0;margin:3px 0;*margin-top:0;line-height:normal;cursor:pointer;background-color:transparent;border:0 \9;-webkit-bord
er-radius:0;-moz-border-radius:0;border-radius:0}input[type="image"]{border:0}input[type="file"]{width:auto;padding:initial;line-height:initial;background-color:#fff;background-color:initial;border:initial;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}input[type="button"],input[type="reset"],input[type="submit"]{width:auto;height:auto}select,input[type="file"]{height:28px;*margin-top:4px;line-height:28px}input[type="file"]{line-height:18px \9}select{width:220px;background-color:#fff}select[multiple],select[size]{height:auto}input[type="image"]{-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}textarea{height:auto}input[type="hidden"]{display:none}.radio,.checkbox{min-height:18px;padding-left:18px}.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-18px}.controls>.radio:first-child,.controls>.checkbox:first-child{padding-top:5px}.radio.inline,.checkbox.inline{display:inline-block;padding-top:5px;margin-bottom:0;vertical-align:
middle}.radio.inline+.radio.inline,.checkbox.inline+.checkbox.inline{margin-left:10px}input,textarea{-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border linear .2s,box-shadow linear .2s;-moz-transition:border linear .2s,box-shadow linear .2s;-ms-transition:border linear .2s,box-shadow linear .2s;-o-transition:border linear .2s,box-shadow linear .2s;transition:border linear .2s,box-shadow linear .2s}input:focus,textarea:focus{border-color:rgba(82,168,236,0.8);outline:0;outline:thin dotted \9;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6)}input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus,select:focus{outline:thin dotted #333;outline:5px auto -webkit-f
ocus-ring-color;outline-offset:-2px;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.input-mini{width:60px}.input-small{width:90px}.input-medium{width:150px}.input-large{width:210px}.input-xlarge{width:270px}.input-xxlarge{width:530px}input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"]{float:none;margin-left:0}input,textarea,.uneditable-input{margin-left:0}input.span12,textarea.span12,.uneditable-input.span12{width:930px}input.span11,textarea.span11,.uneditable-input.span11{width:850px}input.span10,textarea.span10,.uneditable-input.span10{width:770px}input.span9,textarea.span9,.uneditable-input.span9{width:690px}input.span8,textarea.span8,.uneditable-input.span8{width:610px}input.span7,textarea.span7,.uneditable-input.span7{width:530px}input.span6,textarea.span6,.uneditable-input.s
pan6{width:450px}input.span5,textarea.span5,.uneditable-input.span5{width:370px}input.span4,textarea.span4,.uneditable-input.span4{width:290px}input.span3,textarea.span3,.uneditable-input.span3{width:210px}input.span2,textarea.span2,.uneditable-input.span2{width:130px}input.span1,textarea.span1,.uneditable-input.span1{width:50px}input[disabled],select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#eee;border-color:#ddd}input[type="radio"][disabled],input[type="checkbox"][disabled],input[type="radio"][readonly],input[type="checkbox"][readonly]{background-color:transparent}.control-group.warning>label,.control-group.warning .help-block,.control-group.warning .help-inline{color:#c09853}.control-group.warning input,.control-group.warning select,.control-group.warning textarea{color:#c09853;border-color:#c09853}.control-group.warning input:focus,.control-group.warning select:focus,.control-group.warning textarea:focus
{border-color:#a47e3c;-webkit-box-shadow:0 0 6px #dbc59e;-moz-box-shadow:0 0 6px #dbc59e;box-shadow:0 0 6px #dbc59e}.control-group.warning .input-prepend .add-on,.control-group.warning .input-append .add-on{color:#c09853;background-color:#fcf8e3;border-color:#c09853}.control-group.error>label,.control-group.error .help-block,.control-group.error .help-inline{color:#b94a48}.control-group.error input,.control-group.error select,.control-group.error textarea{color:#b94a48;border-color:#b94a48}.control-group.error input:focus,.control-group.error select:focus,.control-group.error textarea:focus{border-color:#953b39;-webkit-box-shadow:0 0 6px #d59392;-moz-box-shadow:0 0 6px #d59392;box-shadow:0 0 6px #d59392}.control-group.error .input-prepend .add-on,.control-group.error .input-append .add-on{color:#b94a48;background-color:#f2dede;border-color:#b94a48}.control-group.success>label,.control-group.success .help-block,.control-group.success .help-inline{color:#468847}.control-group.success
input,.control-group.success select,.control-group.success textarea{color:#468847;border-color:#468847}.control-group.success input:focus,.control-group.success select:focus,.control-group.success textarea:focus{border-color:#356635;-webkit-box-shadow:0 0 6px #7aba7b;-moz-box-shadow:0 0 6px #7aba7b;box-shadow:0 0 6px #7aba7b}.control-group.success .input-prepend .add-on,.control-group.success .input-append .add-on{color:#468847;background-color:#dff0d8;border-color:#468847}input:focus:required:invalid,textarea:focus:required:invalid,select:focus:required:invalid{color:#b94a48;border-color:#ee5f5b}input:focus:required:invalid:focus,textarea:focus:required:invalid:focus,select:focus:required:invalid:focus{border-color:#e9322d;-webkit-box-shadow:0 0 6px #f8b9b7;-moz-box-shadow:0 0 6px #f8b9b7;box-shadow:0 0 6px #f8b9b7}.form-actions{padding:17px 20px 18px;margin-top:18px;margin-bottom:18px;background-color:#f5f5f5;border-top:1px solid #ddd;*zoom:1}.form-actions:before,.form-actions:aft
er{display:table;content:""}.form-actions:after{clear:both}.uneditable-input{overflow:hidden;white-space:nowrap;cursor:not-allowed;background-color:#fff;border-color:#eee;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);box-shadow:inset 0 1px 2px rgba(0,0,0,0.025)}:-moz-placeholder{color:#999}::-webkit-input-placeholder{color:#999}.help-block,.help-inline{color:#555}.help-block{display:block;margin-bottom:9px}.help-inline{display:inline-block;*display:inline;padding-left:5px;vertical-align:middle;*zoom:1}.input-prepend,.input-append{margin-bottom:5px}.input-prepend input,.input-append input,.input-prepend select,.input-append select,.input-prepend .uneditable-input,.input-append .uneditable-input{position:relative;margin-bottom:0;*margin-left:0;vertical-align:middle;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend input:focus,.input-append input:focus,.input-prepend select:
focus,.input-append select:focus,.input-prepend .uneditable-input:focus,.input-append .uneditable-input:focus{z-index:2}.input-prepend .uneditable-input,.input-append .uneditable-input{border-left-color:#ccc}.input-prepend .add-on,.input-append .add-on{display:inline-block;width:auto;height:18px;min-width:16px;padding:4px 5px;font-weight:normal;line-height:18px;text-align:center;text-shadow:0 1px 0 #fff;vertical-align:middle;background-color:#eee;border:1px solid #ccc}.input-prepend .add-on,.input-append .add-on,.input-prepend .btn,.input-append .btn{margin-left:-1px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend .active,.input-append .active{background-color:#a9dba9;border-color:#46a546}.input-prepend .add-on,.input-prepend .btn{margin-right:-1px}.input-prepend .add-on:first-child,.input-prepend .btn:first-child{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append input,.input-append select,.input-appe
nd .uneditable-input{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append .uneditable-input{border-right-color:#ccc;border-left-color:#eee}.input-append .add-on:last-child,.input-append .btn:last-child{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend.input-append input,.input-prepend.input-append select,.input-prepend.input-append .uneditable-input{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend.input-append .add-on:first-child,.input-prepend.input-append .btn:first-child{margin-right:-1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-prepend.input-append .add-on:last-child,.input-prepend.input-append .btn:last-child{margin-left:-1px;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.search-query{padding-right:14px;padding-right:4px \9;padding-left:14px;pad
ding-left:4px \9;margin-bottom:0;-webkit-border-radius:14px;-moz-border-radius:14px;border-radius:14px}.form-search input,.form-inline input,.form-horizontal input,.form-search textarea,.form-inline textarea,.form-horizontal textarea,.form-search select,.form-inline select,.form-horizontal select,.form-search .help-inline,.form-inline .help-inline,.form-horizontal .help-inline,.form-search .uneditable-input,.form-inline .uneditable-input,.form-horizontal .uneditable-input,.form-search .input-prepend,.form-inline .input-prepend,.form-horizontal .input-prepend,.form-search .input-append,.form-inline .input-append,.form-horizontal .input-append{display:inline-block;*display:inline;margin-bottom:0;*zoom:1}.form-search .hide,.form-inline .hide,.form-horizontal .hide{display:none}.form-search label,.form-inline label{display:inline-block}.form-search .input-append,.form-inline .input-append,.form-search .input-prepend,.form-inline .input-prepend{margin-bottom:0}.form-search .radio,.form-s
earch .checkbox,.form-inline .radio,.form-inline .checkbox{padding-left:0;margin-bottom:0;vertical-align:middle}.form-search .radio input[type="radio"],.form-search .checkbox input[type="checkbox"],.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:left;margin-right:3px;margin-left:0}.control-group{margin-bottom:9px}legend+.control-group{margin-top:18px;-webkit-margin-top-collapse:separate}.form-horizontal .control-group{margin-bottom:18px;*zoom:1}.form-horizontal .control-group:before,.form-horizontal .control-group:after{display:table;content:""}.form-horizontal .control-group:after{clear:both}.form-horizontal .control-label{float:left;width:140px;padding-top:5px;text-align:right}.form-horizontal .controls{*display:inline-block;*padding-left:20px;margin-left:160px;*margin-left:0}.form-horizontal .controls:first-child{*padding-left:160px}.form-horizontal .help-block{margin-top:9px;margin-bottom:0}.form-horizontal .form-actions{padding-left:1
60px}table{max-width:100%;background-color:transparent;border-collapse:collapse;border-spacing:0}.table{width:100%;margin-bottom:18px}.table th,.table td{padding:8px;line-height:18px;text-align:left;vertical-align:top;border-top:1px solid #ddd}.table th{font-weight:bold}.table thead th{vertical-align:bottom}.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0}.table tbody+tbody{border-top:2px solid #ddd}.table-condensed th,.table-condensed td{padding:4px 5px}.table-bordered{border:1px solid #ddd;border-collapse:separate;*border-collapse:collapsed;border-left:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-bordered th,.table-bordered td{border-left:1px solid #ddd}.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child t
h,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child th:first-child,.table-bordered tbody:first-child tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered thead:first-child tr:first-child th:last-child,.table-bordered tbody:first-child tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topright:4px}.table-bordered thead:last-child tr:last-child th:first-child,.table-bordered tbody:last-child tr:last-child td:first-child{-webkit-border-radius:0 0 0 4px;-moz-border-radius:0 0 0 4px;bor
der-radius:0 0 0 4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px}.table-bordered thead:last-child tr:last-child th:last-child,.table-bordered tbody:last-child tr:last-child td:last-child{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#f9f9f9}.table tbody tr:hover td,.table tbody tr:hover th{background-color:#f5f5f5}table .span1{float:none;width:44px;margin-left:0}table .span2{float:none;width:124px;margin-left:0}table .span3{float:none;width:204px;margin-left:0}table .span4{float:none;width:284px;margin-left:0}table .span5{float:none;width:364px;margin-left:0}table .span6{float:none;width:444px;margin-left:0}table .span7{float:none;width:524px;margin-left:0}table .span8{float:none;width:604px;margin-left:0}table .span9{float:none;width:684px;margin-left:0}table .span10{
float:none;width:764px;margin-left:0}table .span11{float:none;width:844px;margin-left:0}table .span12{float:none;width:924px;margin-left:0}table .span13{float:none;width:1004px;margin-left:0}table .span14{float:none;width:1084px;margin-left:0}table .span15{float:none;width:1164px;margin-left:0}table .span16{float:none;width:1244px;margin-left:0}table .span17{float:none;width:1324px;margin-left:0}table .span18{float:none;width:1404px;margin-left:0}table .span19{float:none;width:1484px;margin-left:0}table .span20{float:none;width:1564px;margin-left:0}table .span21{float:none;width:1644px;margin-left:0}table .span22{float:none;width:1724px;margin-left:0}table .span23{float:none;width:1804px;margin-left:0}table .span24{float:none;width:1884px;margin-left:0}[class^="icon-"],[class*=" icon-"]{display:inline-block;width:14px;height:14px;*margin-right:.3em;line-height:14px;vertical-align:text-top;background-image:url("../img/glyphicons-halflings.png");background-position:14px 14px;backgroun
d-repeat:no-repeat}[class^="icon-"]:last-child,[class*=" icon-"]:last-child{*margin-left:0}.icon-white{background-image:url("../img/glyphicons-halflings-white.png")}.icon-glass{background-position:0 0}.icon-music{background-position:-24px 0}.icon-search{background-position:-48px 0}.icon-envelope{background-position:-72px 0}.icon-heart{background-position:-96px 0}.icon-star{background-position:-120px 0}.icon-star-empty{background-position:-144px 0}.icon-user{background-position:-168px 0}.icon-film{background-position:-192px 0}.icon-th-large{background-position:-216px 0}.icon-th{background-position:-240px 0}.icon-th-list{background-position:-264px 0}.icon-ok{background-position:-288px 0}.icon-remove{background-position:-312px 0}.icon-zoom-in{background-position:-336px 0}.icon-zoom-out{background-position:-360px 0}.icon-off{background-position:-384px 0}.icon-signal{background-position:-408px 0}.icon-cog{background-position:-432px 0}.icon-trash{background-position:-456px 0}.icon-home{ba
ckground-position:0 -24px}.icon-file{background-position:-24px -24px}.icon-time{background-position:-48px -24px}.icon-road{background-position:-72px -24px}.icon-download-alt{background-position:-96px -24px}.icon-download{background-position:-120px -24px}.icon-upload{background-position:-144px -24px}.icon-inbox{background-position:-168px -24px}.icon-play-circle{background-position:-192px -24px}.icon-repeat{background-position:-216px -24px}.icon-refresh{background-position:-240px -24px}.icon-list-alt{background-position:-264px -24px}.icon-lock{background-position:-287px -24px}.icon-flag{background-position:-312px -24px}.icon-headphones{background-position:-336px -24px}.icon-volume-off{background-position:-360px -24px}.icon-volume-down{background-position:-384px -24px}.icon-volume-up{background-position:-408px -24px}.icon-qrcode{background-position:-432px -24px}.icon-barcode{background-position:-456px -24px}.icon-tag{background-position:0 -48px}.icon-tags{background-position:-25px -48p
x}.icon-book{background-position:-48px -48px}.icon-bookmark{background-position:-72px -48px}.icon-print{background-position:-96px -48px}.icon-camera{background-position:-120px -48px}.icon-font{background-position:-144px -48px}.icon-bold{background-position:-167px -48px}.icon-italic{background-position:-192px -48px}.icon-text-height{background-position:-216px -48px}.icon-text-width{background-position:-240px -48px}.icon-align-left{background-position:-264px -48px}.icon-align-center{background-position:-288px -48px}.icon-align-right{background-position:-312px -48px}.icon-align-justify{background-position:-336px -48px}.icon-list{background-position:-360px -48px}.icon-indent-left{background-position:-384px -48px}.icon-indent-right{background-position:-408px -48px}.icon-facetime-video{background-position:-432px -48px}.icon-picture{background-position:-456px -48px}.icon-pencil{background-position:0 -72px}.icon-map-marker{background-position:-24px -72px}.icon-adjust{background-position:-48
px -72px}.icon-tint{background-position:-72px -72px}.icon-edit{background-position:-96px -72px}.icon-share{background-position:-120px -72px}.icon-check{background-position:-144px -72px}.icon-move{background-position:-168px -72px}.icon-step-backward{background-position:-192px -72px}.icon-fast-backward{background-position:-216px -72px}.icon-backward{background-position:-240px -72px}.icon-play{background-position:-264px -72px}.icon-pause{background-position:-288px -72px}.icon-stop{background-position:-312px -72px}.icon-forward{background-position:-336px -72px}.icon-fast-forward{background-position:-360px -72px}.icon-step-forward{background-position:-384px -72px}.icon-eject{background-position:-408px -72px}.icon-chevron-left{background-position:-432px -72px}.icon-chevron-right{background-position:-456px -72px}.icon-plus-sign{background-position:0 -96px}.icon-minus-sign{background-position:-24px -96px}.icon-remove-sign{background-position:-48px -96px}.icon-ok-sign{background-position:-72
px -96px}.icon-question-sign{background-position:-96px -96px}.icon-info-sign{background-position:-120px -96px}.icon-screenshot{background-position:-144px -96px}.icon-remove-circle{background-position:-168px -96px}.icon-ok-circle{background-position:-192px -96px}.icon-ban-circle{background-position:-216px -96px}.icon-arrow-left{background-position:-240px -96px}.icon-arrow-right{background-position:-264px -96px}.icon-arrow-up{background-position:-289px -96px}.icon-arrow-down{background-position:-312px -96px}.icon-share-alt{background-position:-336px -96px}.icon-resize-full{background-position:-360px -96px}.icon-resize-small{background-position:-384px -96px}.icon-plus{background-position:-408px -96px}.icon-minus{background-position:-433px -96px}.icon-asterisk{background-position:-456px -96px}.icon-exclamation-sign{background-position:0 -120px}.icon-gift{background-position:-24px -120px}.icon-leaf{background-position:-48px -120px}.icon-fire{background-position:-72px -120px}.icon-eye-ope
n{background-position:-96px -120px}.icon-eye-close{background-position:-120px -120px}.icon-warning-sign{background-position:-144px -120px}.icon-plane{background-position:-168px -120px}.icon-calendar{background-position:-192px -120px}.icon-random{background-position:-216px -120px}.icon-comment{background-position:-240px -120px}.icon-magnet{background-position:-264px -120px}.icon-chevron-up{background-position:-288px -120px}.icon-chevron-down{background-position:-313px -119px}.icon-retweet{background-position:-336px -120px}.icon-shopping-cart{background-position:-360px -120px}.icon-folder-close{background-position:-384px -120px}.icon-folder-open{background-position:-408px -120px}.icon-resize-vertical{background-position:-432px -119px}.icon-resize-horizontal{background-position:-456px -118px}.icon-hdd{background-position:0 -144px}.icon-bullhorn{background-position:-24px -144px}.icon-bell{background-position:-48px -144px}.icon-certificate{background-position:-72px -144px}.icon-thumbs-up
{background-position:-96px -144px}.icon-thumbs-down{background-position:-120px -144px}.icon-hand-right{background-position:-144px -144px}.icon-hand-left{background-position:-168px -144px}.icon-hand-up{background-position:-192px -144px}.icon-hand-down{background-position:-216px -144px}.icon-circle-arrow-right{background-position:-240px -144px}.icon-circle-arrow-left{background-position:-264px -144px}.icon-circle-arrow-up{background-position:-288px -144px}.icon-circle-arrow-down{background-position:-312px -144px}.icon-globe{background-position:-336px -144px}.icon-wrench{background-position:-360px -144px}.icon-tasks{background-position:-384px -144px}.icon-filter{background-position:-408px -144px}.icon-briefcase{background-position:-432px -144px}.icon-fullscreen{background-position:-456px -144px}.dropup,.dropdown{position:relative}.dropdown-toggle{*margin-bottom:-3px}.dropdown-toggle:active,.open .dropdown-toggle{outline:0}.caret{display:inline-block;width:0;height:0;vertical-align:top;
border-top:4px solid #000;border-right:4px solid transparent;border-left:4px solid transparent;content:"";opacity:.3;filter:alpha(opacity=30)}.dropdown .caret{margin-top:8px;margin-left:2px}.dropdown:hover .caret,.open .caret{opacity:1;filter:alpha(opacity=100)}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:4px 0;margin:1px 0 0;list-style:none;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{*width:100%;height:1px;margin:8px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bo
ttom:1px solid #fff}.dropdown-menu a{display:block;padding:3px 15px;clear:both;font-weight:normal;line-height:18px;color:#333;white-space:nowrap}.dropdown-menu li>a:hover,.dropdown-menu .active>a,.dropdown-menu .active>a:hover{color:#fff;text-decoration:none;background-color:#08c}.open{*z-index:1000}.open .dropdown-menu{display:block}.pull-right .dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid #000;content:"\2191"}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px}.typeahead{margin-top:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #eee;border:1px solid rgba(0,0,0,0.05);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);b
ox-shadow:inset 0 1px 1px rgba(0,0,0,0.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,0.15)}.well-large{padding:24px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.well-small{padding:9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.fade{opacity:0;filter:alpha(opacity=0);-webkit-transition:opacity .15s linear;-moz-transition:opacity .15s linear;-ms-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1;filter:alpha(opacity=100)}.collapse{position:relative;height:0;overflow:hidden;-webkit-transition:height .35s ease;-moz-transition:height .35s ease;-ms-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.collapse.in{height:auto}.close{float:right;font-size:20px;font-weight:bold;line-height:18px;color:#000;text-shadow:0 1px 0 #fff;opacity:.2;filter:alpha(opacity=20)}.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:
.4;filter:alpha(opacity=40)}button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none}.btn{display:inline-block;*display:inline;padding:4px 10px 4px;margin-bottom:0;*margin-left:.3em;font-size:13px;line-height:18px;*line-height:20px;color:#333;text-align:center;text-shadow:0 1px 1px rgba(255,255,255,0.75);vertical-align:middle;cursor:pointer;background-color:#f5f5f5;*background-color:#e6e6e6;background-image:-ms-linear-gradient(top,#fff,#e6e6e6);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));background-image:-webkit-linear-gradient(top,#fff,#e6e6e6);background-image:-o-linear-gradient(top,#fff,#e6e6e6);background-image:linear-gradient(top,#fff,#e6e6e6);background-image:-moz-linear-gradient(top,#fff,#e6e6e6);background-repeat:repeat-x;border:1px solid #ccc;*border:0;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);border-color:#e6e6e6 #e6e6e6 #bfbfbf;border-bottom-color:#b3b3b3;-webkit-border-radius:4px;-mo
z-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffff',endColorstr='#e6e6e6',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);*zoom:1;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn:hover,.btn:active,.btn.active,.btn.disabled,.btn[disabled]{background-color:#e6e6e6;*background-color:#d9d9d9}.btn:active,.btn.active{background-color:#ccc \9}.btn:first-child{*margin-left:0}.btn:hover{color:#333;text-decoration:none;background-color:#e6e6e6;*background-color:#d9d9d9;background-position:0 -15px;-webkit-transition:background-position .1s linear;-moz-transition:background-position .1s linear;-ms-transition:background-position .1s linear;-o-transition:background-position .1s linear;transition:background-position .1
s linear}.btn:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.active,.btn:active{background-color:#e6e6e6;background-color:#d9d9d9 \9;background-image:none;outline:0;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn.disabled,.btn[disabled]{cursor:default;background-color:#e6e6e6;background-image:none;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-large{padding:9px 14px;font-size:15px;line-height:normal;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.btn-large [class^="icon-"]{margin-top:1px}.btn-small{padding:5px 9px;font-size:11px;line-height:16px}.btn-small [class^="icon-"]{margin-top:-1px}.btn-mini{padding:2px 6px;font-size:11px;line-height:14px}.btn-primary,.btn-primary:hover,.
btn-warning,.btn-warning:hover,.btn-danger,.btn-danger:hover,.btn-success,.btn-success:hover,.btn-info,.btn-info:hover,.btn-inverse,.btn-inverse:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.btn-primary.active,.btn-warning.active,.btn-danger.active,.btn-success.active,.btn-info.active,.btn-inverse.active{color:rgba(255,255,255,0.75)}.btn{border-color:#ccc;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25)}.btn-primary{background-color:#0074cc;*background-color:#05c;background-image:-ms-linear-gradient(top,#08c,#05c);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#05c));background-image:-webkit-linear-gradient(top,#08c,#05c);background-image:-o-linear-gradient(top,#08c,#05c);background-image:-moz-linear-gradient(top,#08c,#05c);background-image:linear-gradient(top,#08c,#05c);background-repeat:repeat-x;border-color:#05c #05c #003580;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(sta
rtColorstr='#0088cc',endColorstr='#0055cc',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-primary:hover,.btn-primary:active,.btn-primary.active,.btn-primary.disabled,.btn-primary[disabled]{background-color:#05c;*background-color:#004ab3}.btn-primary:active,.btn-primary.active{background-color:#004099 \9}.btn-warning{background-color:#faa732;*background-color:#f89406;background-image:-ms-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;border-color:#f89406 #f89406 #ad6704;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450',endColorstr='#f89406
',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-warning:hover,.btn-warning:active,.btn-warning.active,.btn-warning.disabled,.btn-warning[disabled]{background-color:#f89406;*background-color:#df8505}.btn-warning:active,.btn-warning.active{background-color:#c67605 \9}.btn-danger{background-color:#da4f49;*background-color:#bd362f;background-image:-ms-linear-gradient(top,#ee5f5b,#bd362f);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#bd362f));background-image:-webkit-linear-gradient(top,#ee5f5b,#bd362f);background-image:-o-linear-gradient(top,#ee5f5b,#bd362f);background-image:-moz-linear-gradient(top,#ee5f5b,#bd362f);background-image:linear-gradient(top,#ee5f5b,#bd362f);background-repeat:repeat-x;border-color:#bd362f #bd362f #802420;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ee5f5b',endColorstr='#bd362f',GradientType=0);filter:progid:dximage
transform.microsoft.gradient(enabled=false)}.btn-danger:hover,.btn-danger:active,.btn-danger.active,.btn-danger.disabled,.btn-danger[disabled]{background-color:#bd362f;*background-color:#a9302a}.btn-danger:active,.btn-danger.active{background-color:#942a25 \9}.btn-success{background-color:#5bb75b;*background-color:#51a351;background-image:-ms-linear-gradient(top,#62c462,#51a351);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#51a351));background-image:-webkit-linear-gradient(top,#62c462,#51a351);background-image:-o-linear-gradient(top,#62c462,#51a351);background-image:-moz-linear-gradient(top,#62c462,#51a351);background-image:linear-gradient(top,#62c462,#51a351);background-repeat:repeat-x;border-color:#51a351 #51a351 #387038;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#62c462',endColorstr='#51a351',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.
btn-success:hover,.btn-success:active,.btn-success.active,.btn-success.disabled,.btn-success[disabled]{background-color:#51a351;*background-color:#499249}.btn-success:active,.btn-success.active{background-color:#408140 \9}.btn-info{background-color:#49afcd;*background-color:#2f96b4;background-image:-ms-linear-gradient(top,#5bc0de,#2f96b4);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#2f96b4));background-image:-webkit-linear-gradient(top,#5bc0de,#2f96b4);background-image:-o-linear-gradient(top,#5bc0de,#2f96b4);background-image:-moz-linear-gradient(top,#5bc0de,#2f96b4);background-image:linear-gradient(top,#5bc0de,#2f96b4);background-repeat:repeat-x;border-color:#2f96b4 #2f96b4 #1f6377;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de',endColorstr='#2f96b4',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-info:hover,.btn-info:active,.btn-info
.active,.btn-info.disabled,.btn-info[disabled]{background-color:#2f96b4;*background-color:#2a85a0}.btn-info:active,.btn-info.active{background-color:#24748c \9}.btn-inverse{background-color:#414141;*background-color:#222;background-image:-ms-linear-gradient(top,#555,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#555),to(#222));background-image:-webkit-linear-gradient(top,#555,#222);background-image:-o-linear-gradient(top,#555,#222);background-image:-moz-linear-gradient(top,#555,#222);background-image:linear-gradient(top,#555,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#555555',endColorstr='#222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-inverse:hover,.btn-inverse:active,.btn-inverse.active,.btn-inverse.disabled,.btn-inverse[disabled]{background-color:#222;*background-color:#1515
15}.btn-inverse:active,.btn-inverse.active{background-color:#080808 \9}button.btn,input[type="submit"].btn{*padding-top:2px;*padding-bottom:2px}button.btn::-moz-focus-inner,input[type="submit"].btn::-moz-focus-inner{padding:0;border:0}button.btn.btn-large,input[type="submit"].btn.btn-large{*padding-top:7px;*padding-bottom:7px}button.btn.btn-small,input[type="submit"].btn.btn-small{*padding-top:3px;*padding-bottom:3px}button.btn.btn-mini,input[type="submit"].btn.btn-mini{*padding-top:1px;*padding-bottom:1px}.btn-group{position:relative;*margin-left:.3em;*zoom:1}.btn-group:before,.btn-group:after{display:table;content:""}.btn-group:after{clear:both}.btn-group:first-child{*margin-left:0}.btn-group+.btn-group{margin-left:5px}.btn-toolbar{margin-top:9px;margin-bottom:9px}.btn-toolbar .btn-group{display:inline-block;*display:inline;*zoom:1}.btn-group>.btn{position:relative;float:left;margin-left:-1px;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group>.btn:first-child{
margin-left:0;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-bottomleft:4px;-moz-border-radius-topleft:4px}.btn-group>.btn:last-child,.btn-group>.dropdown-toggle{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-bottomright:4px}.btn-group>.btn.large:first-child{margin-left:0;-webkit-border-bottom-left-radius:6px;border-bottom-left-radius:6px;-webkit-border-top-left-radius:6px;border-top-left-radius:6px;-moz-border-radius-bottomleft:6px;-moz-border-radius-topleft:6px}.btn-group>.btn.large:last-child,.btn-group>.large.dropdown-toggle{-webkit-border-top-right-radius:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;border-bottom-right-radius:6px;-moz-border-radius-topright:6px;-moz-border-radius-bottomright:6px}.btn-group>.btn:hover
,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:2}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.dropdown-toggle{*padding-top:4px;padding-right:8px;*padding-bottom:4px;padding-left:8px;-webkit-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn-group>.btn-mini.dropdown-toggle{padding-right:5px;padding-left:5px}.btn-group>.btn-small.dropdown-toggle{*padding-top:4px;*padding-bottom:4px}.btn-group>.btn-large.dropdown-toggle{padding-right:12px;padding-left:12px}.btn-group.open .dropdown-toggle{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgb
a(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn-group.open .btn.dropdown-toggle{background-color:#e6e6e6}.btn-group.open .btn-primary.dropdown-toggle{background-color:#05c}.btn-group.open .btn-warning.dropdown-toggle{background-color:#f89406}.btn-group.open .btn-danger.dropdown-toggle{background-color:#bd362f}.btn-group.open .btn-success.dropdown-toggle{background-color:#51a351}.btn-group.open .btn-info.dropdown-toggle{background-color:#2f96b4}.btn-group.open .btn-inverse.dropdown-toggle{background-color:#222}.btn .caret{margin-top:7px;margin-left:0}.btn:hover .caret,.open.btn-group .caret{opacity:1;filter:alpha(opacity=100)}.btn-mini .caret{margin-top:5px}.btn-small .caret{margin-top:6px}.btn-large .caret{margin-top:6px;border-top-width:5px;border-right-width:5px;border-left-width:5px}.dropup .btn-large .caret{border-top:0;border-bottom:5px solid #000}.btn-primary .caret,.btn-warning .caret,.btn-danger .caret,.btn
-info .caret,.btn-success .caret,.btn-inverse .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:.75;filter:alpha(opacity=75)}.alert{padding:8px 35px 8px 14px;margin-bottom:18px;color:#c09853;text-shadow:0 1px 0 rgba(255,255,255,0.5);background-color:#fcf8e3;border:1px solid #fbeed5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.alert-heading{color:inherit}.alert .close{position:relative;top:-2px;right:-21px;line-height:18px}.alert-success{color:#468847;background-color:#dff0d8;border-color:#d6e9c6}.alert-danger,.alert-error{color:#b94a48;background-color:#f2dede;border-color:#eed3d7}.alert-info{color:#3a87ad;background-color:#d9edf7;border-color:#bce8f1}.alert-block{padding-top:14px;padding-bottom:14px}.alert-block>p,.alert-block>ul{margin-bottom:0}.alert-block p+p{margin-top:5px}.nav{margin-bottom:18px;margin-left:0;list-style:none}.nav>li>a{display:block}.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>.pull-right{float:right}.nav .n
av-header{display:block;padding:3px 15px;font-size:11px;font-weight:bold;line-height:18px;color:#999;text-shadow:0 1px 0 rgba(255,255,255,0.5);text-transform:uppercase}.nav li+.nav-header{margin-top:9px}.nav-list{padding-right:15px;padding-left:15px;margin-bottom:0}.nav-list>li>a,.nav-list .nav-header{margin-right:-15px;margin-left:-15px;text-shadow:0 1px 0 rgba(255,255,255,0.5)}.nav-list>li>a{padding:3px 15px}.nav-list>.active>a,.nav-list>.active>a:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.2);background-color:#08c}.nav-list [class^="icon-"]{margin-right:2px}.nav-list .divider{*width:100%;height:1px;margin:8px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.nav-tabs,.nav-pills{*zoom:1}.nav-tabs:before,.nav-pills:before,.nav-tabs:after,.nav-pills:after{display:table;content:""}.nav-tabs:after,.nav-pills:after{clear:both}.nav-tabs>li,.nav-pills>li{float:left}.nav-tabs>li>a,.nav-pills>li>a{padding-right:12px;padding-left:12px;margi
n-right:2px;line-height:14px}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{margin-bottom:-1px}.nav-tabs>li>a{padding-top:8px;padding-bottom:8px;line-height:18px;border:1px solid transparent;-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>.active>a,.nav-tabs>.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-pills>li>a{padding-top:8px;padding-bottom:8px;margin-top:2px;margin-bottom:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.nav-pills>.active>a,.nav-pills>.active>a:hover{color:#fff;background-color:#08c}.nav-stacked>li{float:none}.nav-stacked>li>a{margin-right:0}.nav-tabs.nav-stacked{border-bottom:0}.nav-tabs.nav-stacked>li>a{border:1px solid #ddd;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.nav-tabs.nav-stacked>li:first-child>a{-webkit-border-radius:4px 4px 0 0;
-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs.nav-stacked>li:last-child>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.nav-tabs.nav-stacked>li>a:hover{z-index:2;border-color:#ddd}.nav-pills.nav-stacked>li>a{margin-bottom:3px}.nav-pills.nav-stacked>li:last-child>a{margin-bottom:1px}.nav-tabs .dropdown-menu{-webkit-border-radius:0 0 5px 5px;-moz-border-radius:0 0 5px 5px;border-radius:0 0 5px 5px}.nav-pills .dropdown-menu{-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-tabs .dropdown-toggle .caret,.nav-pills .dropdown-toggle .caret{margin-top:6px;border-top-color:#08c;border-bottom-color:#08c}.nav-tabs .dropdown-toggle:hover .caret,.nav-pills .dropdown-toggle:hover .caret{border-top-color:#005580;border-bottom-color:#005580}.nav-tabs .active .dropdown-toggle .caret,.nav-pills .active .dropdown-toggle .caret{border-top-color:#333;border-bottom-color:#333}.nav>.dropdown.active>a:hover{color:#000;
cursor:pointer}.nav-tabs .open .dropdown-toggle,.nav-pills .open .dropdown-toggle,.nav>li.dropdown.open.active>a:hover{color:#fff;background-color:#999;border-color:#999}.nav li.dropdown.open .caret,.nav li.dropdown.open.active .caret,.nav li.dropdown.open a:hover .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:1;filter:alpha(opacity=100)}.tabs-stacked .open>a:hover{border-color:#999}.tabbable{*zoom:1}.tabbable:before,.tabbable:after{display:table;content:""}.tabbable:after{clear:both}.tab-content{overflow:auto}.tabs-below>.nav-tabs,.tabs-right>.nav-tabs,.tabs-left>.nav-tabs{border-bottom:0}.tab-content>.tab-pane,.pill-content>.pill-pane{display:none}.tab-content>.active,.pill-content>.active{display:block}.tabs-below>.nav-tabs{border-top:1px solid #ddd}.tabs-below>.nav-tabs>li{margin-top:-1px;margin-bottom:0}.tabs-below>.nav-tabs>li>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.tabs-below>.nav-tabs>li>a:hover{border-top-
color:#ddd;border-bottom-color:transparent}.tabs-below>.nav-tabs>.active>a,.tabs-below>.nav-tabs>.active>a:hover{border-color:transparent #ddd #ddd #ddd}.tabs-left>.nav-tabs>li,.tabs-right>.nav-tabs>li{float:none}.tabs-left>.nav-tabs>li>a,.tabs-right>.nav-tabs>li>a{min-width:74px;margin-right:0;margin-bottom:3px}.tabs-left>.nav-tabs{float:left;margin-right:19px;border-right:1px solid #ddd}.tabs-left>.nav-tabs>li>a{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px}.tabs-left>.nav-tabs>li>a:hover{border-color:#eee #ddd #eee #eee}.tabs-left>.nav-tabs .active>a,.tabs-left>.nav-tabs .active>a:hover{border-color:#ddd transparent #ddd #ddd;*border-right-color:#fff}.tabs-right>.nav-tabs{float:right;margin-left:19px;border-left:1px solid #ddd}.tabs-right>.nav-tabs>li>a{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0}.tabs-right>.nav-tabs>li>a:hover{border-color:#eee #eee #ee
e #ddd}.tabs-right>.nav-tabs .active>a,.tabs-right>.nav-tabs .active>a:hover{border-color:#ddd #ddd #ddd transparent;*border-left-color:#fff}.navbar{*position:relative;*z-index:2;margin-bottom:18px;overflow:visible}.navbar-inner{min-height:40px;padding-right:20px;padding-left:20px;background-color:#2c2c2c;background-image:-moz-linear-gradient(top,#333,#222);background-image:-ms-linear-gradient(top,#333,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#333),to(#222));background-image:-webkit-linear-gradient(top,#333,#222);background-image:-o-linear-gradient(top,#333,#222);background-image:linear-gradient(top,#333,#222);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#333333',endColorstr='#222222',GradientType=0);-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1);
box-shadow:0 1px 3px rgba(0,0,0,0.25),inset 0 -1px 0 rgba(0,0,0,0.1)}.navbar .container{width:auto}.nav-collapse.collapse{height:auto}.navbar{color:#999}.navbar .brand:hover{text-decoration:none}.navbar .brand{display:block;float:left;padding:8px 20px 12px;margin-left:-20px;font-size:20px;font-weight:200;line-height:1;color:#999}.navbar .navbar-text{margin-bottom:0;line-height:40px}.navbar .navbar-link{color:#999}.navbar .navbar-link:hover{color:#fff}.navbar .btn,.navbar .btn-group{margin-top:5px}.navbar .btn-group .btn{margin:0}.navbar-form{margin-bottom:0;*zoom:1}.navbar-form:before,.navbar-form:after{display:table;content:""}.navbar-form:after{clear:both}.navbar-form input,.navbar-form select,.navbar-form .radio,.navbar-form .checkbox{margin-top:5px}.navbar-form input,.navbar-form select{display:inline-block;margin-bottom:0}.navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px}.navbar-form .input-append,.navbar-form .
input-prepend{margin-top:6px;white-space:nowrap}.navbar-form .input-append input,.navbar-form .input-prepend input{margin-top:0}.navbar-search{position:relative;float:left;margin-top:6px;margin-bottom:0}.navbar-search .search-query{padding:4px 9px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;font-weight:normal;line-height:1;color:#fff;background-color:#626262;border:1px solid #151515;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-webkit-transition:none;-moz-transition:none;-ms-transition:none;-o-transition:none;transition:none}.navbar-search .search-query:-moz-placeholder{color:#ccc}.navbar-search .search-query::-webkit-input-placeholder{color:#ccc}.navbar-search .search-query:focus,.navbar-search .search-query.focused{padding:5px 10px;color:#333;text-shadow:0 1px 0 #fff;
background-color:#fff;border:0;outline:0;-webkit-box-shadow:0 0 3px rgba(0,0,0,0.15);-moz-box-shadow:0 0 3px rgba(0,0,0,0.15);box-shadow:0 0 3px rgba(0,0,0,0.15)}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030;margin-bottom:0}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding-right:0;padding-left:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.navbar-fixed-top{top:0}.navbar-fixed-bottom{bottom:0}.navbar .nav{position:relative;left:0;display:block;float:left;margin:0 10px 0 0}.navbar .nav.pull-right{float:right}.navbar .nav>li{display:block;float:left}.navbar .nav>li>a{float:none;padding:9px 10px 11px;line-height:19px;color:#999;text-decoration:none;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.navbar .btn{display:inline-block;padding:4px 10px 4px;margin:5px 5px 6px;line-height:18px}.navbar .btn-group{padding:5px 5px 6px;margin:0}.navbar .nav>li>
a:hover{color:#fff;text-decoration:none;background-color:transparent}.navbar .nav .active>a,.navbar .nav .active>a:hover{color:#fff;text-decoration:none;background-color:#222}.navbar .divider-vertical{width:1px;height:40px;margin:0 9px;overflow:hidden;background-color:#222;border-right:1px solid #333}.navbar .nav.pull-right{margin-right:0;margin-left:10px}.navbar .btn-navbar{display:none;float:right;padding:7px 10px;margin-right:5px;margin-left:5px;background-color:#2c2c2c;*background-color:#222;background-image:-ms-linear-gradient(top,#333,#222);background-image:-webkit-gradient(linear,0 0,0 100%,from(#333),to(#222));background-image:-webkit-linear-gradient(top,#333,#222);background-image:-o-linear-gradient(top,#333,#222);background-image:linear-gradient(top,#333,#222);background-image:-moz-linear-gradient(top,#333,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradi
ent(startColorstr='#333333',endColorstr='#222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075)}.navbar .btn-navbar:hover,.navbar .btn-navbar:active,.navbar .btn-navbar.active,.navbar .btn-navbar.disabled,.navbar .btn-navbar[disabled]{background-color:#222;*background-color:#151515}.navbar .btn-navbar:active,.navbar .btn-navbar.active{background-color:#080808 \9}.navbar .btn-navbar .icon-bar{display:block;width:18px;height:2px;background-color:#f5f5f5;-webkit-border-radius:1px;-moz-border-radius:1px;border-radius:1px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,0.25);-moz-box-shadow:0 1px 0 rgba(0,0,0,0.25);box-shadow:0 1px 0 rgba(0,0,0,0.25)}.btn-navbar .icon-bar+.icon-bar{margin-top:3px}.navbar .dropdown-men
u:before{position:absolute;top:-7px;left:9px;display:inline-block;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-left:7px solid transparent;border-bottom-color:rgba(0,0,0,0.2);content:''}.navbar .dropdown-menu:after{position:absolute;top:-6px;left:10px;display:inline-block;border-right:6px solid transparent;border-bottom:6px solid #fff;border-left:6px solid transparent;content:''}.navbar-fixed-bottom .dropdown-menu:before{top:auto;bottom:-7px;border-top:7px solid #ccc;border-bottom:0;border-top-color:rgba(0,0,0,0.2)}.navbar-fixed-bottom .dropdown-menu:after{top:auto;bottom:-6px;border-top:6px solid #fff;border-bottom:0}.navbar .nav li.dropdown .dropdown-toggle .caret,.navbar .nav li.dropdown.open .caret{border-top-color:#fff;border-bottom-color:#fff}.navbar .nav li.dropdown.active .caret{opacity:1;filter:alpha(opacity=100)}.navbar .nav li.dropdown.open>.dropdown-toggle,.navbar .nav li.dropdown.active>.dropdown-toggle,.navbar .nav li.dropdown.open.active>.drop
down-toggle{background-color:transparent}.navbar .nav li.dropdown.active>.dropdown-toggle:hover{color:#fff}.navbar .pull-right .dropdown-menu,.navbar .dropdown-menu.pull-right{right:0;left:auto}.navbar .pull-right .dropdown-menu:before,.navbar .dropdown-menu.pull-right:before{right:12px;left:auto}.navbar .pull-right .dropdown-menu:after,.navbar .dropdown-menu.pull-right:after{right:13px;left:auto}.breadcrumb{padding:7px 14px;margin:0 0 18px;list-style:none;background-color:#fbfbfb;background-image:-moz-linear-gradient(top,#fff,#f5f5f5);background-image:-ms-linear-gradient(top,#fff,#f5f5f5);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#f5f5f5));background-image:-webkit-linear-gradient(top,#fff,#f5f5f5);background-image:-o-linear-gradient(top,#fff,#f5f5f5);background-image:linear-gradient(top,#fff,#f5f5f5);background-repeat:repeat-x;border:1px solid #ddd;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;filter:progid:dximagetransform.microsoft.gra
dient(startColorstr='#ffffff',endColorstr='#f5f5f5',GradientType=0);-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.breadcrumb li{display:inline-block;*display:inline;text-shadow:0 1px 0 #fff;*zoom:1}.breadcrumb .divider{padding:0 5px;color:#999}.breadcrumb .active a{color:#333}.pagination{height:36px;margin:18px 0}.pagination ul{display:inline-block;*display:inline;margin-bottom:0;margin-left:0;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;*zoom:1;-webkit-box-shadow:0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:0 1px 2px rgba(0,0,0,0.05);box-shadow:0 1px 2px rgba(0,0,0,0.05)}.pagination li{display:inline}.pagination a{float:left;padding:0 14px;line-height:34px;text-decoration:none;border:1px solid #ddd;border-left-width:0}.pagination a:hover,.pagination .active a{background-color:#f5f5f5}.pagination .active a{color:#999;cursor:default}.pagination .disabled span,.pagination .disabled a,.pagination .disabled a:
hover{color:#999;cursor:default;background-color:transparent}.pagination li:first-child a{border-left-width:1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.pagination li:last-child a{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.pagination-centered{text-align:center}.pagination-right{text-align:right}.pager{margin-bottom:18px;margin-left:0;text-align:center;list-style:none;*zoom:1}.pager:before,.pager:after{display:table;content:""}.pager:after{clear:both}.pager li{display:inline}.pager a{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.pager a:hover{text-decoration:none;background-color:#f5f5f5}.pager .next a{float:right}.pager .previous a{float:left}.pager .disabled a,.pager .disabled a:hover{color:#999;cursor:default;background-color:#fff}.modal-open .dropdown-menu{z-index:2050}.modal-
open .dropdown.open{*z-index:2050}.modal-open .popover{z-index:2060}.modal-open .tooltip{z-index:2070}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop,.modal-backdrop.fade.in{opacity:.8;filter:alpha(opacity=80)}.modal{position:fixed;top:50%;left:50%;z-index:1050;width:560px;margin:-250px 0 0 -280px;overflow:auto;background-color:#fff;border:1px solid #999;border:1px solid rgba(0,0,0,0.3);*border:1px solid #999;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3);-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.modal.fade{top:-25%;-webkit-transition:opacity .3s linear,top .3s ease-out;-moz-transition:opacity .3s linear,top .3s ease-out;-ms-transition:opacity .3s linear,top .3s ease-out;-o-transition:opacity
.3s linear,top .3s ease-out;transition:opacity .3s linear,top .3s ease-out}.modal.fade.in{top:50%}.modal-header{padding:9px 15px;border-bottom:1px solid #eee}.modal-header .close{margin-top:2px}.modal-body{max-height:400px;padding:15px;overflow-y:auto}.modal-form{margin-bottom:0}.modal-footer{padding:14px 15px 15px;margin-bottom:0;text-align:right;background-color:#f5f5f5;border-top:1px solid #ddd;-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;*zoom:1;-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.modal-footer:before,.modal-footer:after{display:table;content:""}.modal-footer:after{clear:both}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.tooltip{position:absolute;z-index:1020;display:block;padding:5px;font-size:11px;opacity:0;filter:alpha(opacity=0);visibility:visible}.tooltip.in{opacity:.8;filter:alpha(opacity=80)}.toolti
p.top{margin-top:-2px}.tooltip.right{margin-left:2px}.tooltip.bottom{margin-top:2px}.tooltip.left{margin-left:-2px}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-top:5px solid #000;border-right:5px solid transparent;border-left:5px solid transparent}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-top:5px solid transparent;border-bottom:5px solid transparent;border-left:5px solid #000}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-right:5px solid transparent;border-bottom:5px solid #000;border-left:5px solid transparent}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-top:5px solid transparent;border-right:5px solid #000;border-bottom:5px solid transparent}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;text-decoration:none;background-color:#000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0}.popover{pos
ition:absolute;top:0;left:0;z-index:1010;display:none;padding:5px}.popover.top{margin-top:-5px}.popover.right{margin-left:5px}.popover.bottom{margin-top:5px}.popover.left{margin-left:-5px}.popover.top .arrow{bottom:0;left:50%;margin-left:-5px;border-top:5px solid #000;border-right:5px solid transparent;border-left:5px solid transparent}.popover.right .arrow{top:50%;left:0;margin-top:-5px;border-top:5px solid transparent;border-right:5px solid #000;border-bottom:5px solid transparent}.popover.bottom .arrow{top:0;left:50%;margin-left:-5px;border-right:5px solid transparent;border-bottom:5px solid #000;border-left:5px solid transparent}.popover.left .arrow{top:50%;right:0;margin-top:-5px;border-top:5px solid transparent;border-bottom:5px solid transparent;border-left:5px solid #000}.popover .arrow{position:absolute;width:0;height:0}.popover-inner{width:280px;padding:3px;overflow:hidden;background:#000;background:rgba(0,0,0,0.8);-webkit-border-radius:6px;-moz-border-radius:6px;border-ra
dius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3)}.popover-title{padding:9px 15px;line-height:1;background-color:#f5f5f5;border-bottom:1px solid #eee;-webkit-border-radius:3px 3px 0 0;-moz-border-radius:3px 3px 0 0;border-radius:3px 3px 0 0}.popover-content{padding:14px;background-color:#fff;-webkit-border-radius:0 0 3px 3px;-moz-border-radius:0 0 3px 3px;border-radius:0 0 3px 3px;-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.popover-content p,.popover-content ul,.popover-content ol{margin-bottom:0}.thumbnails{margin-left:-20px;list-style:none;*zoom:1}.thumbnails:before,.thumbnails:after{display:table;content:""}.thumbnails:after{clear:both}.row-fluid .thumbnails{margin-left:0}.thumbnails>li{float:left;margin-bottom:18px;margin-left:20px}.thumbnail{display:block;padding:4px;line-height:1;border:1px solid #ddd;-webkit-border-radius:4px;-moz-border-rad
ius:4px;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:0 1px 1px rgba(0,0,0,0.075);box-shadow:0 1px 1px rgba(0,0,0,0.075)}a.thumbnail:hover{border-color:#08c;-webkit-box-shadow:0 1px 4px rgba(0,105,214,0.25);-moz-box-shadow:0 1px 4px rgba(0,105,214,0.25);box-shadow:0 1px 4px rgba(0,105,214,0.25)}.thumbnail>img{display:block;max-width:100%;margin-right:auto;margin-left:auto}.thumbnail .caption{padding:9px}.label,.badge{font-size:10.998px;font-weight:bold;line-height:14px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);white-space:nowrap;vertical-align:baseline;background-color:#999}.label{padding:1px 4px 2px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.badge{padding:1px 9px 2px;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px}a.label:hover,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.label-important,.badge-important{background-color:#b94a48}.label-important[href],.badge-important[href]{backgr
ound-color:#953b39}.label-warning,.badge-warning{background-color:#f89406}.label-warning[href],.badge-warning[href]{background-color:#c67605}.label-success,.badge-success{background-color:#468847}.label-success[href],.badge-success[href]{background-color:#356635}.label-info,.badge-info{background-color:#3a87ad}.label-info[href],.badge-info[href]{background-color:#2d6987}.label-inverse,.badge-inverse{background-color:#333}.label-inverse[href],.badge-inverse[href]{background-color:#1a1a1a}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-moz-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-ms-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:0 0}to{background-position:40px 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:18px;mar
gin-bottom:18px;overflow:hidden;background-color:#f7f7f7;background-image:-moz-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-ms-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#f5f5f5),to(#f9f9f9));background-image:-webkit-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-o-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:linear-gradient(top,#f5f5f5,#f9f9f9);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#f5f5f5',endColorstr='#f9f9f9',GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress .bar{width:0;height:18px;font-size:12px;color:#fff;text-align:center;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e90d2;background-image:-moz-linear-gradient(top,#149bdf,#0480be);background-image:-webk
it-gradient(linear,0 0,0 100%,from(#149bdf),to(#0480be));background-image:-webkit-linear-gradient(top,#149bdf,#0480be);background-image:-o-linear-gradient(top,#149bdf,#0480be);background-image:linear-gradient(top,#149bdf,#0480be);background-image:-ms-linear-gradient(top,#149bdf,#0480be);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#149bdf',endColorstr='#0480be',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box;-webkit-transition:width .6s ease;-moz-transition:width .6s ease;-ms-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress-striped .bar{background-color:#149bdf;background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15)
50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50
%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;-moz-background-size:40px 40px;-o-background-size:40px 40px;background-size:40px 40px}.progress.active .bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-moz-animation:progress-bar-stripes 2s linear infinite;-ms-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-danger .bar{background-color:#dd514c;background-image:-moz-linear-gradient(top,#ee5f5b,#c43c35);background-image:-ms-linear-gradient(top,#ee5f5b,#c43c35);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#c43c35));background-image:-webkit-linear-gradient(top,#ee5f5b,#c43c35);background-image:-o-linear-gradient(top,#ee5f5b,#c43c35);background-image:linear-gradient(top,#ee5f5b,#c43c35);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startCo
lorstr='#ee5f5b',endColorstr='#c43c35',GradientType=0)}.progress-danger.progress-striped .bar{background-color:#ee5f5b;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-4
5deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-success .bar{background-color:#5eb95e;background-image:-moz-linear-gradient(top,#62c462,#57a957);background-image:-ms-linear-gradient(top,#62c462,#57a957);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#57a957));background-image:-webkit-linear-gradient(top,#62c462,#57a957);background-image:-o-linear-gradient(top,#62c462,#57a957);background-image:linear-gradient(top,#62c462,#57a957);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#62c462',endColorstr='#57a957',GradientType=0)}.progress-success.progress-striped .bar{background-color:#62c462;background-image:-webkit-gradient(linear,0 100%
,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:li
near-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-info .bar{background-color:#4bb1cf;background-image:-moz-linear-gradient(top,#5bc0de,#339bb9);background-image:-ms-linear-gradient(top,#5bc0de,#339bb9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#339bb9));background-image:-webkit-linear-gradient(top,#5bc0de,#339bb9);background-image:-o-linear-gradient(top,#5bc0de,#339bb9);background-image:linear-gradient(top,#5bc0de,#339bb9);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de',endColorstr='#339bb9',GradientType=0)}.progress-info.progress-striped .bar{background-color:#5bc0de;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255
,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress
-warning .bar{background-color:#faa732;background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:-ms-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450',endColorstr='#f89406',GradientType=0)}.progress-warning.progress-striped .bar{background-color:#fbb450;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,r
gba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-ms-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(-45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.accordion{margin-bottom:18px}.accordion-group{margin-bottom:2px;border:1px solid #e5e5e5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.accordion-heading
{border-bottom:0}.accordion-heading .accordion-toggle{display:block;padding:8px 15px}.accordion-toggle{cursor:pointer}.accordion-inner{padding:9px 15px;border-top:1px solid #e5e5e5}.carousel{position:relative;margin-bottom:18px;line-height:1}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel .item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-moz-transition:.6s ease-in-out left;-ms-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel .item>img{display:block;line-height:1}.carousel .active,.carousel .next,.carousel .prev{display:block}.carousel .active{left:0}.carousel .next,.carousel .prev{position:absolute;top:0;width:100%}.carousel .next{left:100%}.carousel .prev{left:-100%}.carousel .next.left,.carousel .prev.right{left:0}.carousel .active.left{left:-100%}.carousel .active.right{left:100%}.carousel-control{position:absolute;top:40%;left:15px;width:40px;height:40px;margin-top:-20p
x;font-size:60px;font-weight:100;line-height:30px;color:#fff;text-align:center;background:#222;border:3px solid #fff;-webkit-border-radius:23px;-moz-border-radius:23px;border-radius:23px;opacity:.5;filter:alpha(opacity=50)}.carousel-control.right{right:15px;left:auto}.carousel-control:hover{color:#fff;text-decoration:none;opacity:.9;filter:alpha(opacity=90)}.carousel-caption{position:absolute;right:0;bottom:0;left:0;padding:10px 15px 5px;background:#333;background:rgba(0,0,0,0.75)}.carousel-caption h4,.carousel-caption p{color:#fff}.hero-unit{padding:60px;margin-bottom:30px;background-color:#eee;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.hero-unit h1{margin-bottom:0;font-size:60px;line-height:1;letter-spacing:-1px;color:inherit}.hero-unit p{font-size:18px;font-weight:200;line-height:27px;color:inherit}.pull-right{float:right}.pull-left{float:left}.hide{display:none}.show{display:block}.invisible{visibility:hidden}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/assets/favicon.ico
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/assets/favicon.ico b/src/main/python/twitter/thermos/observer/http/assets/favicon.ico
deleted file mode 100644
index 048d2e4..0000000
Binary files a/src/main/python/twitter/thermos/observer/http/assets/favicon.ico and /dev/null differ
[50/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/codec/ThriftBinaryCodec.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/codec/ThriftBinaryCodec.java b/src/main/java/com/twitter/aurora/codec/ThriftBinaryCodec.java
deleted file mode 100644
index 2443078..0000000
--- a/src/main/java/com/twitter/aurora/codec/ThriftBinaryCodec.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.codec;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.thrift.TBase;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TProtocolFactory;
-
-/**
- * Codec that works for thrift objects.
- */
-public final class ThriftBinaryCodec {
-
- /**
- * Protocol factory used for all thrift encoding and decoding.
- */
- public static final TProtocolFactory PROTOCOL_FACTORY = new TBinaryProtocol.Factory();
-
- private ThriftBinaryCodec() {
- // Utility class.
- }
-
- /**
- * Identical to {@link #decodeNonNull(Class, byte[])}, but allows for a null buffer.
- *
- * @param clazz Class to instantiate and deserialize to.
- * @param buffer Buffer to decode.
- * @param <T> Target type.
- * @return A populated message, or {@code null} if the buffer was {@code null}.
- * @throws CodingException If the message could not be decoded.
- */
- @Nullable
- public static <T extends TBase<T, ?>> T decode(Class<T> clazz, @Nullable byte[] buffer)
- throws CodingException {
-
- if (buffer == null) {
- return null;
- }
- return decodeNonNull(clazz, buffer);
- }
-
- /**
- * Decodes a binary-encoded byte array into a target type.
- *
- * @param clazz Class to instantiate and deserialize to.
- * @param buffer Buffer to decode.
- * @param <T> Target type.
- * @return A populated message.
- * @throws CodingException If the message could not be decoded.
- */
- public static <T extends TBase<T, ?>> T decodeNonNull(Class<T> clazz, byte[] buffer)
- throws CodingException {
-
- Preconditions.checkNotNull(clazz);
- Preconditions.checkNotNull(buffer);
-
- try {
- T t = clazz.newInstance();
- new TDeserializer(PROTOCOL_FACTORY).deserialize(t, buffer);
- return t;
- } catch (IllegalAccessException e) {
- throw new CodingException("Failed to access constructor for target type.", e);
- } catch (InstantiationException e) {
- throw new CodingException("Failed to instantiate target type.", e);
- } catch (TException e) {
- throw new CodingException("Failed to deserialize thrift object.", e);
- }
- }
-
- /**
- * Identical to {@link #encodeNonNull(TBase)}, but allows for a null input.
- *
- * @param tBase Object to encode.
- * @return Encoded object, or {@code null} if the argument was {@code null}.
- * @throws CodingException If the object could not be encoded.
- */
- @Nullable
- public static byte[] encode(@Nullable TBase<?, ?> tBase) throws CodingException {
- if (tBase == null) {
- return null;
- }
- return encodeNonNull(tBase);
- }
-
- /**
- * Encodes a thrift object into a binary array.
- *
- * @param tBase Object to encode.
- * @return Encoded object.
- * @throws CodingException If the object could not be encoded.
- */
- public static byte[] encodeNonNull(TBase<?, ?> tBase) throws CodingException {
- Preconditions.checkNotNull(tBase);
-
- try {
- return new TSerializer(PROTOCOL_FACTORY).serialize(tBase);
- } catch (TException e) {
- throw new CodingException("Failed to serialize: " + tBase, e);
- }
- }
-
- /**
- * Thrown when serialization or deserialization failed.
- */
- public static class CodingException extends Exception {
- public CodingException(String message) {
- super(message);
- }
- public CodingException(String msg, Throwable cause) {
- super(msg, cause);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/Driver.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/Driver.java b/src/main/java/com/twitter/aurora/scheduler/Driver.java
deleted file mode 100644
index aa77887..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/Driver.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
-
-import org.apache.mesos.Protos;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.Status;
-import org.apache.mesos.Protos.TaskInfo;
-import org.apache.mesos.SchedulerDriver;
-
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.StateMachine;
-
-import static org.apache.mesos.Protos.Status.DRIVER_RUNNING;
-
-/**
- * Wraps the mesos Scheduler driver to ensure its used in a valid lifecycle; namely:
- * <pre>
- * (run -> kill*)? -> stop*
- * </pre>
- *
- * Also ensures the driver is only asked for when needed.
- */
-public interface Driver {
-
- /**
- * Launches a task.
- *
- * @param offerId ID of the resource offer to accept with the task.
- * @param task Task to launch.
- */
- void launchTask(OfferID offerId, TaskInfo task);
-
- /**
- * Declines a resource offer.
- *
- * @param offerId ID of the offer to decline.
- */
- void declineOffer(OfferID offerId);
-
- /**
- * Sends a kill task request for the given {@code taskId} to the mesos master.
- *
- * @param taskId The id of the task to kill.
- */
- void killTask(String taskId);
-
- /**
- * Stops the underlying driver if it is running, otherwise does nothing.
- */
- void stop();
-
- /**
- * Starts the underlying driver. Can only be called once.
- *
- * @return The status of the underlying driver run request.
- */
- Protos.Status start();
-
- /**
- * Blocks until the underlying driver is stopped or aborted.
- *
- * @return The status of the underlying driver upon exit.
- */
- Protos.Status join();
-
- /**
- * Mesos driver implementation.
- */
- static class DriverImpl implements Driver {
- private static final Logger LOG = Logger.getLogger(Driver.class.getName());
-
- /**
- * Driver states.
- */
- enum State {
- INIT,
- RUNNING,
- STOPPED
- }
-
- private final StateMachine<State> stateMachine;
- private final Supplier<Optional<SchedulerDriver>> driverSupplier;
- private final AtomicLong killFailures = Stats.exportLong("scheduler_driver_kill_failures");
-
- /**
- * Creates a driver manager that will only ask for the underlying mesos driver when actually
- * needed.
- *
- * @param driverSupplier A factory for the underlying driver.
- */
- @Inject
- DriverImpl(Supplier<Optional<SchedulerDriver>> driverSupplier) {
- this.driverSupplier = driverSupplier;
- this.stateMachine =
- StateMachine.<State>builder("scheduler_driver")
- .initialState(State.INIT)
- .addState(State.INIT, State.RUNNING, State.STOPPED)
- .addState(State.RUNNING, State.STOPPED)
- .logTransitions()
- .throwOnBadTransition(true)
- .build();
- }
-
- private synchronized SchedulerDriver get(State expected) {
- // TODO(William Farner): Formalize the failure case here by throwing a checked exception.
- stateMachine.checkState(expected);
- // This will and should fail if the driver is not present.
- return driverSupplier.get().get();
- }
-
- @Override
- public void launchTask(OfferID offerId, TaskInfo task) {
- get(State.RUNNING).launchTasks(offerId, ImmutableList.of(task));
- }
-
- @Override
- public void declineOffer(OfferID offerId) {
- get(State.RUNNING).declineOffer(offerId);
- }
-
- @Override
- public Protos.Status start() {
- SchedulerDriver driver = get(State.INIT);
- stateMachine.transition(State.RUNNING);
- return driver.start();
- }
-
- @Override
- public Status join() {
- return get(State.RUNNING).join();
- }
-
- @Override
- public synchronized void stop() {
- if (stateMachine.getState() == State.RUNNING) {
- SchedulerDriver driver = get(State.RUNNING);
- driver.stop(true /* failover */);
- stateMachine.transition(State.STOPPED);
- }
- }
-
- @Override
- public void killTask(String taskId) {
- SchedulerDriver driver = get(State.RUNNING);
- Protos.Status status = driver.killTask(Protos.TaskID.newBuilder().setValue(taskId).build());
-
- if (status != DRIVER_RUNNING) {
- LOG.severe(String.format("Attempt to kill task %s failed with code %s",
- taskId, status));
- killFailures.incrementAndGet();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/DriverFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/DriverFactory.java b/src/main/java/com/twitter/aurora/scheduler/DriverFactory.java
deleted file mode 100644
index e39bb09..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/DriverFactory.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Properties;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.inject.Provider;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import com.google.protobuf.ByteString;
-
-import org.apache.mesos.MesosSchedulerDriver;
-import org.apache.mesos.Protos.Credential;
-import org.apache.mesos.Protos.FrameworkID;
-import org.apache.mesos.Protos.FrameworkInfo;
-import org.apache.mesos.Scheduler;
-import org.apache.mesos.SchedulerDriver;
-
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.args.constraints.NotNull;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-
-/**
- * Factory to create scheduler driver instances.
- */
-public interface DriverFactory extends Function<String, SchedulerDriver> {
-
- static class DriverFactoryImpl implements DriverFactory {
- private static final Logger LOG = Logger.getLogger(DriverFactory.class.getName());
-
- @NotNull
- @CmdLine(name = "mesos_master_address",
- help = "Address for the mesos master, can be a socket address or zookeeper path.")
- private static final Arg<String> MESOS_MASTER_ADDRESS = Arg.create();
-
- @VisibleForTesting
- static final String PRINCIPAL_KEY = "aurora_authentication_principal";
- @VisibleForTesting
- static final String SECRET_KEY = "aurora_authentication_secret";
- @CmdLine(name = "framework_authentication_file",
- help = "Properties file which contains framework credentials to authenticate with Mesos"
- + "master. Must contain the properties '" + PRINCIPAL_KEY + "' and "
- + "'" + SECRET_KEY + "'.")
- private static final Arg<File> FRAMEWORK_AUTHENTICATION_FILE = Arg.create();
-
- @CmdLine(name = "framework_failover_timeout",
- help = "Time after which a framework is considered deleted. SHOULD BE VERY HIGH.")
- private static final Arg<Amount<Long, Time>> FRAMEWORK_FAILOVER_TIMEOUT =
- Arg.create(Amount.of(21L, Time.DAYS));
-
- /**
- * Require Mesos slaves to have checkpointing enabled. Slaves with checkpointing enabled will
- * attempt to write checkpoints when required by a task's framework. These checkpoints allow
- * executors to be reattached rather than killed when a slave is restarted.
- *
- * This flag is dangerous! When enabled tasks will not launch on slaves without checkpointing
- * enabled.
- *
- * Behavior is as follows:
- * (Scheduler -require_slave_checkpoint=true, Slave --checkpoint=true):
- * Tasks will launch. Checkpoints will be written.
- * (Scheduler -require_slave_checkpoint=true, Slave --checkpoint=false):
- * Tasks WILL NOT launch.
- * (Scheduler -require_slave_checkpoint=false, Slave --checkpoint=true):
- * Tasks will launch. Checkpoints will not be written.
- * (Scheduler -require_slave_checkpoint=false, Slave --checkpoint=false):
- * Tasks will launch. Checkpoints will not be written.
- *
- * TODO(ksweeney): Remove warning table after https://issues.apache.org/jira/browse/MESOS-444
- * is resolved.
- */
- @CmdLine(name = "require_slave_checkpoint",
- help = "DANGEROUS! Require Mesos slaves to have checkpointing enabled. When enabled a "
- + "slave restart should not kill executors, but the scheduler will not be able to "
- + "launch tasks on slaves without --checkpoint=true in their command lines. See "
- + "DriverFactory.java for more information.")
- private static final Arg<Boolean> REQUIRE_SLAVE_CHECKPOINT = Arg.create(false);
-
- private static final String EXECUTOR_USER = "root";
-
- private static final String TWITTER_FRAMEWORK_NAME = "TwitterScheduler";
-
- private final Provider<Scheduler> scheduler;
-
- @Inject
- DriverFactoryImpl(Provider<Scheduler> scheduler) {
- this.scheduler = Preconditions.checkNotNull(scheduler);
- }
-
- @VisibleForTesting
- static Properties parseCredentials(InputStream credentialsStream) {
- Properties properties = new Properties();
- try {
- properties.load(credentialsStream);
- } catch (IOException e) {
- LOG.severe("Unable to load authentication file");
- throw Throwables.propagate(e);
- }
- Preconditions.checkState(properties.containsKey(PRINCIPAL_KEY),
- "The framework authentication file is missing the key: %s", PRINCIPAL_KEY);
- Preconditions.checkState(properties.containsKey(SECRET_KEY),
- "The framework authentication file is missing the key: %s", SECRET_KEY);
- return properties;
- }
-
- @Override
- public SchedulerDriver apply(@Nullable String frameworkId) {
- LOG.info("Connecting to mesos master: " + MESOS_MASTER_ADDRESS.get());
-
- FrameworkInfo.Builder frameworkInfo = FrameworkInfo.newBuilder()
- .setUser(EXECUTOR_USER)
- .setName(TWITTER_FRAMEWORK_NAME)
- .setCheckpoint(REQUIRE_SLAVE_CHECKPOINT.get())
- .setFailoverTimeout(FRAMEWORK_FAILOVER_TIMEOUT.get().as(Time.SECONDS));
-
- if (frameworkId != null) {
- LOG.info("Found persisted framework ID: " + frameworkId);
- frameworkInfo.setId(FrameworkID.newBuilder().setValue(frameworkId));
- } else {
- LOG.warning("Did not find a persisted framework ID, connecting as a new framework.");
- }
-
- if (FRAMEWORK_AUTHENTICATION_FILE.hasAppliedValue()) {
- Properties properties;
- try {
- properties = parseCredentials(new FileInputStream(FRAMEWORK_AUTHENTICATION_FILE.get()));
- } catch (FileNotFoundException e) {
- LOG.severe("Authentication File not Found");
- throw Throwables.propagate(e);
- }
-
- LOG.info(String.format("Connecting to master using authentication (principal: %s).",
- properties.get(PRINCIPAL_KEY)));
-
- Credential credential = Credential.newBuilder()
- .setPrincipal(properties.getProperty(PRINCIPAL_KEY))
- .setSecret(ByteString.copyFromUtf8(properties.getProperty(SECRET_KEY)))
- .build();
-
- return new MesosSchedulerDriver(
- scheduler.get(),
- frameworkInfo.build(),
- MESOS_MASTER_ADDRESS.get(),
- credential);
- } else {
- LOG.warning("Connecting to master without authentication!");
- return new MesosSchedulerDriver(
- scheduler.get(),
- frameworkInfo.build(),
- MESOS_MASTER_ADDRESS.get());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/MesosSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/MesosSchedulerImpl.java b/src/main/java/com/twitter/aurora/scheduler/MesosSchedulerImpl.java
deleted file mode 100644
index fb41405..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/MesosSchedulerImpl.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import org.apache.mesos.Protos.ExecutorID;
-import org.apache.mesos.Protos.FrameworkID;
-import org.apache.mesos.Protos.MasterInfo;
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.SlaveID;
-import org.apache.mesos.Protos.TaskInfo;
-import org.apache.mesos.Protos.TaskStatus;
-import org.apache.mesos.Scheduler;
-import org.apache.mesos.SchedulerDriver;
-
-import com.twitter.aurora.GuiceUtils.AllowUnchecked;
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.gen.comm.SchedulerMessage;
-import com.twitter.aurora.scheduler.base.Conversions;
-import com.twitter.aurora.scheduler.base.SchedulerException;
-import com.twitter.aurora.scheduler.configuration.Resources;
-import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.Event;
-import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
-import com.twitter.aurora.scheduler.state.SchedulerCore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.common.application.Lifecycle;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.stats.Stats;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Location for communication with mesos.
- */
-class MesosSchedulerImpl implements Scheduler {
- private static final Logger LOG = Logger.getLogger(MesosSchedulerImpl.class.getName());
-
- private final AtomicLong resourceOffers = Stats.exportLong("scheduler_resource_offers");
- private final AtomicLong failedOffers = Stats.exportLong("scheduler_failed_offers");
- private final AtomicLong failedStatusUpdates = Stats.exportLong("scheduler_status_updates");
- private final AtomicLong frameworkDisconnects =
- Stats.exportLong("scheduler_framework_disconnects");
- private final AtomicLong frameworkReregisters =
- Stats.exportLong("scheduler_framework_reregisters");
- private final AtomicLong lostExecutors = Stats.exportLong("scheduler_lost_executors");
-
- private final List<TaskLauncher> taskLaunchers;
-
- private final Storage storage;
- private final SchedulerCore schedulerCore;
- private final Lifecycle lifecycle;
- private volatile boolean registered = false;
-
- /**
- * Creates a new scheduler.
- *
- * @param schedulerCore Core scheduler.
- * @param lifecycle Application lifecycle manager.
- * @param taskLaunchers Task launchers.
- */
- @Inject
- public MesosSchedulerImpl(
- Storage storage,
- SchedulerCore schedulerCore,
- final Lifecycle lifecycle,
- List<TaskLauncher> taskLaunchers) {
-
- this.storage = checkNotNull(storage);
- this.schedulerCore = checkNotNull(schedulerCore);
- this.lifecycle = checkNotNull(lifecycle);
- this.taskLaunchers = checkNotNull(taskLaunchers);
- }
-
- @Override
- public void slaveLost(SchedulerDriver schedulerDriver, SlaveID slaveId) {
- LOG.info("Received notification of lost slave: " + slaveId);
- }
-
- @SendNotification(after = Event.DriverRegistered)
- @Override
- public void registered(
- SchedulerDriver driver,
- final FrameworkID frameworkId,
- MasterInfo masterInfo) {
-
- LOG.info("Registered with ID " + frameworkId + ", master: " + masterInfo);
-
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- storeProvider.getSchedulerStore().saveFrameworkId(frameworkId.getValue());
- }
- });
- registered = true;
- }
-
- @SendNotification(after = Event.DriverDisconnected)
- @Override
- public void disconnected(SchedulerDriver schedulerDriver) {
- LOG.warning("Framework disconnected.");
- frameworkDisconnects.incrementAndGet();
- }
-
- @Override
- public void reregistered(SchedulerDriver schedulerDriver, MasterInfo masterInfo) {
- LOG.info("Framework re-registered with master " + masterInfo);
- frameworkReregisters.incrementAndGet();
- }
-
- private static boolean fitsInOffer(TaskInfo task, Offer offer) {
- return Resources.from(offer).greaterThanOrEqual(Resources.from(task.getResourcesList()));
- }
-
- @Timed("scheduler_resource_offers")
- @Override
- public void resourceOffers(SchedulerDriver driver, List<Offer> offers) {
- Preconditions.checkState(registered, "Must be registered before receiving offers.");
-
- for (final Offer offer : offers) {
- log(Level.FINE, "Received offer: %s", offer);
- resourceOffers.incrementAndGet();
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- storeProvider.getAttributeStore().saveHostAttributes(Conversions.getAttributes(offer));
- }
- });
-
- // Ordering of task launchers is important here, since offers are consumed greedily.
- // TODO(William Farner): Refactor this area of code now that the primary task launcher
- // is asynchronous.
- for (TaskLauncher launcher : taskLaunchers) {
- Optional<TaskInfo> task = Optional.absent();
- try {
- task = launcher.createTask(offer);
- } catch (SchedulerException e) {
- LOG.log(Level.WARNING, "Failed to schedule offers.", e);
- failedOffers.incrementAndGet();
- }
-
- if (task.isPresent()) {
- if (fitsInOffer(task.get(), offer)) {
- driver.launchTasks(offer.getId(), ImmutableList.of(task.get()));
- break;
- } else {
- LOG.warning("Insufficient resources to launch task " + task);
- }
- }
- }
- }
- }
-
- @Override
- public void offerRescinded(SchedulerDriver schedulerDriver, OfferID offerId) {
- LOG.info("Offer rescinded: " + offerId);
- for (TaskLauncher launcher : taskLaunchers) {
- launcher.cancelOffer(offerId);
- }
- }
-
- @AllowUnchecked
- @Timed("scheduler_status_update")
- @Override
- public void statusUpdate(SchedulerDriver driver, TaskStatus status) {
- String info = status.hasData() ? status.getData().toStringUtf8() : null;
- String infoMsg = info != null ? " with info " + info : "";
- String coreMsg = status.hasMessage() ? " with core message " + status.getMessage() : "";
- LOG.info("Received status update for task " + status.getTaskId().getValue()
- + " in state " + status.getState() + infoMsg + coreMsg);
-
- try {
- for (TaskLauncher launcher : taskLaunchers) {
- if (launcher.statusUpdate(status)) {
- return;
- }
- }
- } catch (SchedulerException e) {
- LOG.log(Level.SEVERE, "Status update failed due to scheduler exception: " + e, e);
- // We re-throw the exception here in an effort to NACK the status update and trigger an
- // abort of the driver. Previously we directly issued driver.abort(), but the re-entrancy
- // guarantees of that are uncertain (and we believe it was not working). However, this
- // was difficult to discern since logging is unreliable during JVM shutdown and we would not
- // see the above log message.
- throw e;
- }
-
- LOG.warning("Unhandled status update " + status);
- failedStatusUpdates.incrementAndGet();
- }
-
- @Override
- public void error(SchedulerDriver driver, String message) {
- LOG.severe("Received error message: " + message);
- lifecycle.shutdown();
- }
-
- @Override
- public void executorLost(SchedulerDriver schedulerDriver, ExecutorID executorID, SlaveID slaveID,
- int status) {
-
- LOG.info("Lost executor " + executorID);
- lostExecutors.incrementAndGet();
- }
-
- @Timed("scheduler_framework_message")
- @Override
- public void frameworkMessage(SchedulerDriver driver, ExecutorID executor, SlaveID slave,
- byte[] data) {
-
- if (data == null) {
- LOG.info("Received empty framework message.");
- return;
- }
-
- try {
- SchedulerMessage schedulerMsg = ThriftBinaryCodec.decode(SchedulerMessage.class, data);
- if (schedulerMsg == null || !schedulerMsg.isSet()) {
- LOG.warning("Received empty scheduler message.");
- return;
- }
-
- switch (schedulerMsg.getSetField()) {
- case DELETED_TASKS:
- // TODO(William Farner): Refactor this to use a thinner interface here. As it stands
- // it is odd that we route the registered() call to schedulerCore via the
- // registeredListener and call the schedulerCore directly here.
- schedulerCore.tasksDeleted(schedulerMsg.getDeletedTasks().getTaskIds());
- break;
-
- default:
- LOG.warning("Received unhandled scheduler message type: " + schedulerMsg.getSetField());
- break;
- }
- } catch (ThriftBinaryCodec.CodingException e) {
- LOG.log(Level.SEVERE, "Failed to decode framework message.", e);
- }
- }
-
- private static void log(Level level, String message, Object... args) {
- if (LOG.isLoggable(level)) {
- LOG.log(level, String.format(message, args));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/MesosTaskFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/MesosTaskFactory.java b/src/main/java/com/twitter/aurora/scheduler/MesosTaskFactory.java
deleted file mode 100644
index 5f73f71..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/MesosTaskFactory.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.protobuf.ByteString;
-
-import org.apache.mesos.Protos.ExecutorID;
-import org.apache.mesos.Protos.ExecutorInfo;
-import org.apache.mesos.Protos.Resource;
-import org.apache.mesos.Protos.SlaveID;
-import org.apache.mesos.Protos.TaskID;
-import org.apache.mesos.Protos.TaskInfo;
-
-import com.twitter.aurora.Protobufs;
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.scheduler.base.CommandUtil;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.SchedulerException;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.configuration.Resources;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.quantity.Data;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * A factory to create mesos task objects.
- */
-public interface MesosTaskFactory {
-
- /**
- * Creates a mesos task object.
- *
- * @param task Assigned task to translate into a task object.
- * @param slaveId Id of the slave the task is being assigned to.
- * @return A new task.
- * @throws SchedulerException If the task could not be encoded.
- */
- TaskInfo createFrom(IAssignedTask task, SlaveID slaveId) throws SchedulerException;
-
- static class ExecutorConfig {
- private final String executorPath;
-
- public ExecutorConfig(String executorPath) {
- this.executorPath = checkNotBlank(executorPath);
- }
-
- String getExecutorPath() {
- return executorPath;
- }
- }
-
- static class MesosTaskFactoryImpl implements MesosTaskFactory {
- private static final Logger LOG = Logger.getLogger(MesosTaskFactoryImpl.class.getName());
- private static final String EXECUTOR_PREFIX = "thermos-";
-
- /**
- * Name to associate with task executors.
- */
- @VisibleForTesting
- static final String EXECUTOR_NAME = "aurora.task";
-
- private final String executorPath;
-
- @Inject
- MesosTaskFactoryImpl(ExecutorConfig executorConfig) {
- this.executorPath = executorConfig.getExecutorPath();
- }
-
- @VisibleForTesting
- static ExecutorID getExecutorId(String taskId) {
- return ExecutorID.newBuilder().setValue(EXECUTOR_PREFIX + taskId).build();
- }
-
- public static String getJobSourceName(IJobKey jobkey) {
- return String.format("%s.%s.%s", jobkey.getRole(), jobkey.getEnvironment(), jobkey.getName());
- }
-
- public static String getJobSourceName(ITaskConfig task) {
- return getJobSourceName(JobKeys.from(task));
- }
-
- public static String getInstanceSourceName(ITaskConfig task, int instanceId) {
- return String.format("%s.%s", getJobSourceName(task), instanceId);
- }
-
- @Override
- public TaskInfo createFrom(IAssignedTask task, SlaveID slaveId) throws SchedulerException {
- checkNotNull(task);
- byte[] taskInBytes;
- try {
- taskInBytes = ThriftBinaryCodec.encode(task.newBuilder());
- } catch (ThriftBinaryCodec.CodingException e) {
- LOG.log(Level.SEVERE, "Unable to serialize task.", e);
- throw new SchedulerException("Internal error.", e);
- }
-
- ITaskConfig config = task.getTask();
- List<Resource> resources;
- if (task.isSetAssignedPorts()) {
- resources = Resources.from(config)
- .toResourceList(ImmutableSet.copyOf(task.getAssignedPorts().values()));
- } else {
- resources = ImmutableList.of();
- }
-
- if (LOG.isLoggable(Level.FINE)) {
- LOG.fine("Setting task resources to "
- + Iterables.transform(resources, Protobufs.SHORT_TOSTRING));
- }
- TaskInfo.Builder taskBuilder =
- TaskInfo.newBuilder()
- .setName(JobKeys.toPath(Tasks.ASSIGNED_TO_JOB_KEY.apply(task)))
- .setTaskId(TaskID.newBuilder().setValue(task.getTaskId()))
- .setSlaveId(slaveId)
- .addAllResources(resources)
- .setData(ByteString.copyFrom(taskInBytes));
-
- ExecutorInfo executor = ExecutorInfo.newBuilder()
- .setCommand(CommandUtil.create(executorPath))
- .setExecutorId(getExecutorId(task.getTaskId()))
- .setName(EXECUTOR_NAME)
- .setSource(getInstanceSourceName(config, task.getInstanceId()))
- .addResources(Resources.makeMesosResource(Resources.CPUS, ResourceSlot.EXECUTOR_CPUS))
- .addResources(
- Resources.makeMesosResource(Resources.RAM_MB, ResourceSlot.EXECUTOR_RAM.as(Data.MB)))
- .build();
- return taskBuilder
- .setExecutor(executor)
- .build();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/ResourceSlot.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/ResourceSlot.java b/src/main/java/com/twitter/aurora/scheduler/ResourceSlot.java
deleted file mode 100644
index a9c14e6..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/ResourceSlot.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.Arrays;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-
-import com.twitter.aurora.scheduler.configuration.Resources;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Data;
-
-import static org.apache.mesos.Protos.Offer;
-
-/**
- * Resource containing class that is aware of executor overhead.
- */
-public final class ResourceSlot {
-
- private final Resources resources;
-
- /**
- * CPU allocated for each executor.
- */
- @VisibleForTesting
- static final double EXECUTOR_CPUS = 0.25;
-
- /**
- * RAM required for the executor. Executors in the wild have been observed using 48-54MB RSS,
- * setting to 128MB to be extra vigilant initially.
- */
- @VisibleForTesting
- static final Amount<Long, Data> EXECUTOR_RAM = Amount.of(128L, Data.MB);
-
- private ResourceSlot(Resources r) {
- this.resources = r;
- }
-
- public static ResourceSlot from(ITaskConfig task) {
- double totalCPU = task.getNumCpus() + EXECUTOR_CPUS;
- Amount<Long, Data> totalRAM = Amount.of(task.getRamMb() + EXECUTOR_RAM.as(Data.MB), Data.MB);
- Amount<Long, Data> disk = Amount.of(task.getDiskMb(), Data.MB);
- return new ResourceSlot(
- new Resources(totalCPU, totalRAM, disk, task.getRequestedPorts().size()));
- }
-
- public static ResourceSlot from(Offer offer) {
- return new ResourceSlot(Resources.from(offer));
- }
-
- public double getNumCpus() {
- return resources.getNumCpus();
- }
-
- public Amount<Long, Data> getRam() {
- return resources.getRam();
- }
-
- public Amount<Long, Data> getDisk() {
- return resources.getDisk();
- }
-
- public int getNumPorts() {
- return resources.getNumPorts();
- }
-
- @VisibleForTesting
- public static ResourceSlot from(double cpu,
- Amount<Long, Data> ram,
- Amount<Long, Data> disk,
- int ports) {
- double totalCPU = cpu + EXECUTOR_CPUS;
- Amount<Long, Data> totalRAM = Amount.of(ram.as(Data.MB) + EXECUTOR_RAM.as(Data.MB), Data.MB);
-
- return new ResourceSlot(new Resources(totalCPU, totalRAM, disk, ports));
- }
-
- public static ResourceSlot sum(ResourceSlot... rs) {
- return sum(Arrays.asList(rs));
- }
-
- public static ResourceSlot sum(Iterable<ResourceSlot> rs) {
- Resources r = Resources.sum(Iterables.transform(rs, new Function<ResourceSlot, Resources>() {
- @Override public Resources apply(ResourceSlot input) {
- return input.resources;
- }
- }));
-
- return new ResourceSlot(r);
- }
-
- public static final Ordering<ResourceSlot> ORDER = new Ordering<ResourceSlot>() {
- @Override public int compare(ResourceSlot left, ResourceSlot right) {
- return Resources.RESOURCE_ORDER.compare(left.resources, right.resources);
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/SchedulerLifecycle.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/SchedulerLifecycle.java b/src/main/java/com/twitter/aurora/scheduler/SchedulerLifecycle.java
deleted file mode 100644
index f90869d..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/SchedulerLifecycle.java
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.base.Supplier;
-import com.google.common.eventbus.Subscribe;
-import com.google.common.util.concurrent.Atomics;
-
-import org.apache.mesos.Protos;
-import org.apache.mesos.SchedulerDriver;
-
-import com.twitter.aurora.scheduler.events.PubsubEvent.DriverRegistered;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.StorageBackfill;
-import com.twitter.common.application.Lifecycle;
-import com.twitter.common.base.Closure;
-import com.twitter.common.base.Closures;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.StatImpl;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-import com.twitter.common.util.StateMachine;
-import com.twitter.common.util.StateMachine.Transition;
-import com.twitter.common.zookeeper.Group.JoinException;
-import com.twitter.common.zookeeper.ServerSet;
-import com.twitter.common.zookeeper.SingletonService.LeaderControl;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.common.zookeeper.SingletonService.LeadershipListener;
-
-/**
- * The central driver of the scheduler runtime lifecycle. Handles the transitions from startup and
- * initialization through acting as a standby scheduler / log replica and finally to becoming the
- * scheduler leader.
- * <p>
- * The (enforced) call order to be used with this class is:
- * <ol>
- * <li>{@link #prepare()}, to initialize the storage system.</li>
- * <li>{@link LeadershipListener#onLeading(LeaderControl) onLeading()} on the
- * {@link LeadershipListener LeadershipListener}
- * returned from {@link #prepare()}, signaling that this process has exclusive control of the
- * cluster.</li>
- * <li>{@link #registered(DriverRegistered) registered()},
- * indicating that registration with the mesos master has succeeded.
- * At this point, the scheduler's presence will be announced via
- * {@link LeaderControl#advertise() advertise()}.</li>
- * </ol>
- * If this call order is broken, calls will fail by throwing
- * {@link java.lang.IllegalStateException}.
- * <p>
- * At any point in the lifecycle, the scheduler will respond to
- * {@link LeadershipListener#onDefeated(com.twitter.common.zookeeper.ServerSet.EndpointStatus)
- * onDefeated()} by initiating a clean shutdown using {@link Lifecycle#shutdown() shutdown()}.
- * A clean shutdown will also be initiated if control actions fail during normal state transitions.
- */
-public class SchedulerLifecycle implements EventSubscriber {
-
- private static final Logger LOG = Logger.getLogger(SchedulerLifecycle.class.getName());
-
- private enum State {
- IDLE,
- PREPARING_STORAGE,
- STORAGE_PREPARED,
- LEADER_AWAITING_REGISTRATION,
- REGISTERED_LEADER,
- RUNNING,
- DEAD
- }
-
- private static final Predicate<Transition<State>> IS_DEAD = new Predicate<Transition<State>>() {
- @Override public boolean apply(Transition<State> state) {
- return state.getTo() == State.DEAD;
- }
- };
-
- private static final Predicate<Transition<State>> NOT_DEAD = Predicates.not(IS_DEAD);
-
- private final LeadershipListener leadershipListener;
- private final AtomicBoolean registrationAcked = new AtomicBoolean(false);
- private final AtomicReference<LeaderControl> leaderControl = Atomics.newReference();
- private final StateMachine<State> stateMachine;
-
- @Inject
- SchedulerLifecycle(
- final DriverFactory driverFactory,
- final NonVolatileStorage storage,
- final Lifecycle lifecycle,
- final Driver driver,
- final DriverReference driverRef,
- final LeadingOptions leadingOptions,
- final ScheduledExecutorService executorService,
- final Clock clock) {
-
- this(
- driverFactory,
- storage,
- lifecycle,
- driver,
- driverRef,
- new DelayedActions() {
- @Override public void blockingDriverJoin(Runnable runnable) {
- executorService.execute(runnable);
- }
-
- @Override public void onAutoFailover(Runnable runnable) {
- executorService.schedule(
- runnable,
- leadingOptions.leadingTimeLimit.getValue(),
- leadingOptions.leadingTimeLimit.getUnit().getTimeUnit());
- }
-
- @Override public void onRegistrationTimeout(Runnable runnable) {
- LOG.info(
- "Giving up on registration in " + leadingOptions.registrationDelayLimit);
- executorService.schedule(
- runnable,
- leadingOptions.registrationDelayLimit.getValue(),
- leadingOptions.registrationDelayLimit.getUnit().getTimeUnit());
- }
- },
- clock);
- }
-
- @VisibleForTesting
- SchedulerLifecycle(
- final DriverFactory driverFactory,
- final NonVolatileStorage storage,
- final Lifecycle lifecycle,
- // TODO(wfarner): The presence of Driver and DriverReference is quite confusing. Figure out
- // a clean way to collapse the duties of DriverReference into DriverImpl.
- final Driver driver,
- final DriverReference driverRef,
- final DelayedActions delayedActions,
- final Clock clock) {
-
- Stats.export(new StatImpl<Integer>("framework_registered") {
- @Override public Integer read() {
- return registrationAcked.get() ? 1 : 0;
- }
- });
- for (final State state : State.values()) {
- Stats.export(new StatImpl<Integer>("scheduler_lifecycle_" + state) {
- @Override public Integer read() {
- return (state == stateMachine.getState()) ? 1 : 0;
- }
- });
- }
-
- final Closure<Transition<State>> prepareStorage = new Closure<Transition<State>>() {
- @Override public void execute(Transition<State> transition) {
- try {
- storage.prepare();
- stateMachine.transition(State.STORAGE_PREPARED);
- } catch (RuntimeException e) {
- stateMachine.transition(State.DEAD);
- throw e;
- }
- }
- };
-
- final Closure<Transition<State>> handleLeading = new Closure<Transition<State>>() {
- @Override public void execute(Transition<State> transition) {
- LOG.info("Elected as leading scheduler!");
- storage.start(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- StorageBackfill.backfill(storeProvider, clock);
- }
- });
-
- @Nullable final String frameworkId = storage.consistentRead(
- new Work.Quiet<String>() {
- @Override public String apply(StoreProvider storeProvider) {
- return storeProvider.getSchedulerStore().fetchFrameworkId();
- }
- });
- driverRef.set(driverFactory.apply(frameworkId));
-
- delayedActions.onRegistrationTimeout(
- new Runnable() {
- @Override public void run() {
- if (!registrationAcked.get()) {
- LOG.severe(
- "Framework has not been registered within the tolerated delay.");
- stateMachine.transition(State.DEAD);
- }
- }
- });
-
- delayedActions.onAutoFailover(
- new Runnable() {
- @Override public void run() {
- LOG.info("Triggering automatic failover.");
- stateMachine.transition(State.DEAD);
- }
- });
-
- Protos.Status status = driver.start();
- LOG.info("Driver started with code " + status);
- delayedActions.blockingDriverJoin(new Runnable() {
- @Override public void run() {
- // Blocks until driver exits.
- driver.join();
- stateMachine.transition(State.DEAD);
- }
- });
- }
- };
-
- final Closure<Transition<State>> handleRegistered = new Closure<Transition<State>>() {
- @Override public void execute(Transition<State> transition) {
- registrationAcked.set(true);
- try {
- leaderControl.get().advertise();
- } catch (JoinException e) {
- LOG.log(Level.SEVERE, "Failed to advertise leader, shutting down.", e);
- stateMachine.transition(State.DEAD);
- } catch (InterruptedException e) {
- LOG.log(Level.SEVERE, "Interrupted while advertising leader, shutting down.", e);
- stateMachine.transition(State.DEAD);
- Thread.currentThread().interrupt();
- }
- }
- };
-
- final Closure<Transition<State>> shutDown = new Closure<Transition<State>>() {
- private final AtomicBoolean invoked = new AtomicBoolean(false);
- @Override public void execute(Transition<State> transition) {
- if (!invoked.compareAndSet(false, true)) {
- LOG.info("Shutdown already invoked, ignoring extra call.");
- return;
- }
-
- // TODO(wfarner): Consider using something like guava's Closer to abstractly tear down
- // resources here.
- try {
- LeaderControl control = leaderControl.get();
- if (control != null) {
- try {
- control.leave();
- } catch (JoinException e) {
- LOG.log(Level.WARNING, "Failed to leave leadership: " + e, e);
- } catch (ServerSet.UpdateException e) {
- LOG.log(Level.WARNING, "Failed to leave server set: " + e, e);
- }
- }
-
- // TODO(wfarner): Re-evaluate tear-down ordering here. Should the top-level shutdown
- // be invoked first, or the underlying critical components?
- driver.stop();
- storage.stop();
- } finally {
- lifecycle.shutdown();
- }
- }
- };
-
- stateMachine = StateMachine.<State>builder("SchedulerLifecycle")
- .initialState(State.IDLE)
- .logTransitions()
- .addState(
- Closures.filter(NOT_DEAD, prepareStorage),
- State.IDLE,
- State.PREPARING_STORAGE, State.DEAD)
- .addState(
- State.PREPARING_STORAGE,
- State.STORAGE_PREPARED, State.DEAD)
- .addState(
- Closures.filter(NOT_DEAD, handleLeading),
- State.STORAGE_PREPARED,
- State.LEADER_AWAITING_REGISTRATION, State.DEAD)
- .addState(
- Closures.filter(NOT_DEAD, handleRegistered),
- State.LEADER_AWAITING_REGISTRATION,
- State.REGISTERED_LEADER, State.DEAD)
- .addState(
- State.REGISTERED_LEADER,
- State.RUNNING, State.DEAD)
- .addState(
- State.RUNNING,
- State.DEAD)
- .addState(
- State.DEAD,
- // Allow cycles in DEAD to prevent throwing and avoid the need for call-site checking.
- State.DEAD
- )
- .onAnyTransition(
- Closures.filter(IS_DEAD, shutDown))
- .build();
-
- this.leadershipListener = new SchedulerCandidateImpl(stateMachine, leaderControl);
- }
-
- /**
- * Prepares a scheduler to offer itself as a leader candidate. After this call the scheduler will
- * host a live log replica and start syncing data from the leader via the log until it gets called
- * upon to lead.
- *
- * @return A listener that can be offered for leadership of a distributed election.
- */
- public LeadershipListener prepare() {
- stateMachine.transition(State.PREPARING_STORAGE);
- return leadershipListener;
- }
-
- @Subscribe
- public void registered(DriverRegistered event) {
- stateMachine.transition(State.REGISTERED_LEADER);
- }
-
- /**
- * Maintains a reference to the driver.
- */
- static class DriverReference implements Supplier<Optional<SchedulerDriver>> {
- private final AtomicReference<SchedulerDriver> driver = Atomics.newReference();
-
- @Override public Optional<SchedulerDriver> get() {
- return Optional.fromNullable(driver.get());
- }
-
- private void set(SchedulerDriver ref) {
- driver.set(ref);
- }
- }
-
- private static class SchedulerCandidateImpl implements LeadershipListener {
- private final StateMachine<State> stateMachine;
- private final AtomicReference<LeaderControl> leaderControl;
-
- SchedulerCandidateImpl(
- StateMachine<State> stateMachine,
- AtomicReference<LeaderControl> leaderControl) {
-
- this.stateMachine = stateMachine;
- this.leaderControl = leaderControl;
- }
-
- @Override public void onLeading(LeaderControl control) {
- leaderControl.set(control);
- stateMachine.transition(State.LEADER_AWAITING_REGISTRATION);
- }
-
- @Override public void onDefeated(@Nullable ServerSet.EndpointStatus status) {
- LOG.severe("Lost leadership, committing suicide.");
- stateMachine.transition(State.DEAD);
- }
- }
-
- public static class LeadingOptions {
- private final Amount<Long, Time> registrationDelayLimit;
- private final Amount<Long, Time> leadingTimeLimit;
-
- /**
- * Creates a new collection of options for tuning leadership behavior.
- *
- * @param registrationDelayLimit Maximum amount of time to wait for framework registration to
- * complete.
- * @param leadingTimeLimit Maximum amount of time to serve as leader before abdicating.
- */
- public LeadingOptions(
- Amount<Long, Time> registrationDelayLimit,
- Amount<Long, Time> leadingTimeLimit) {
-
- Preconditions.checkArgument(
- registrationDelayLimit.getValue() >= 0,
- "Registration delay limit must be positive.");
- Preconditions.checkArgument(
- leadingTimeLimit.getValue() >= 0,
- "Leading time limit must be positive.");
-
- this.registrationDelayLimit = checkNotNull(registrationDelayLimit);
- this.leadingTimeLimit = checkNotNull(leadingTimeLimit);
- }
- }
-
- @VisibleForTesting
- interface DelayedActions {
- void blockingDriverJoin(Runnable runnable);
-
- void onAutoFailover(Runnable runnable);
-
- void onRegistrationTimeout(Runnable runnable);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/SchedulerModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/SchedulerModule.java b/src/main/java/com/twitter/aurora/scheduler/SchedulerModule.java
deleted file mode 100644
index be4c2b1..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/SchedulerModule.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-
-import javax.inject.Singleton;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.inject.AbstractModule;
-import com.google.inject.PrivateModule;
-import com.google.inject.Provides;
-import com.google.inject.TypeLiteral;
-
-import org.apache.mesos.Scheduler;
-import org.apache.mesos.SchedulerDriver;
-
-import com.twitter.aurora.scheduler.Driver.DriverImpl;
-import com.twitter.aurora.scheduler.SchedulerLifecycle.DriverReference;
-import com.twitter.aurora.scheduler.SchedulerLifecycle.LeadingOptions;
-import com.twitter.aurora.scheduler.TaskIdGenerator.TaskIdGeneratorImpl;
-import com.twitter.aurora.scheduler.events.PubsubEventModule;
-import com.twitter.aurora.scheduler.periodic.GcExecutorLauncher;
-import com.twitter.aurora.scheduler.periodic.GcExecutorLauncher.GcExecutorSettings;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-
-/**
- * Binding module for top-level scheduling logic.
- */
-public class SchedulerModule extends AbstractModule {
-
- @CmdLine(name = "executor_gc_interval",
- help = "Max interval on which to run the GC executor on a host to clean up dead tasks.")
- private static final Arg<Amount<Long, Time>> EXECUTOR_GC_INTERVAL =
- Arg.create(Amount.of(1L, Time.HOURS));
-
- @CmdLine(name = "gc_executor_path", help = "Path to the gc executor launch script.")
- private static final Arg<String> GC_EXECUTOR_PATH = Arg.create(null);
-
- @CmdLine(name = "max_registration_delay",
- help = "Max allowable delay to allow the driver to register before aborting")
- private static final Arg<Amount<Long, Time>> MAX_REGISTRATION_DELAY =
- Arg.create(Amount.of(1L, Time.MINUTES));
-
- @CmdLine(name = "max_leading_duration",
- help = "After leading for this duration, the scheduler should commit suicide.")
- private static final Arg<Amount<Long, Time>> MAX_LEADING_DURATION =
- Arg.create(Amount.of(1L, Time.DAYS));
-
- @Override
- protected void configure() {
- bind(Driver.class).to(DriverImpl.class);
- bind(DriverImpl.class).in(Singleton.class);
- bind(new TypeLiteral<Supplier<Optional<SchedulerDriver>>>() { }).to(DriverReference.class);
- bind(DriverReference.class).in(Singleton.class);
-
- bind(Scheduler.class).to(MesosSchedulerImpl.class);
- bind(MesosSchedulerImpl.class).in(Singleton.class);
-
- bind(TaskIdGenerator.class).to(TaskIdGeneratorImpl.class);
-
- bind(GcExecutorSettings.class).toInstance(new GcExecutorSettings(
- EXECUTOR_GC_INTERVAL.get(),
- Optional.fromNullable(GC_EXECUTOR_PATH.get())));
-
- bind(GcExecutorLauncher.class).in(Singleton.class);
- bind(UserTaskLauncher.class).in(Singleton.class);
-
- install(new PrivateModule() {
- @Override protected void configure() {
- bind(LeadingOptions.class).toInstance(
- new LeadingOptions(MAX_REGISTRATION_DELAY.get(), MAX_LEADING_DURATION.get()));
- final ScheduledExecutorService executor = Executors.newScheduledThreadPool(
- 1,
- new ThreadFactoryBuilder().setNameFormat("Lifecycle-%d").setDaemon(true).build());
- bind(ScheduledExecutorService.class).toInstance(executor);
- bind(SchedulerLifecycle.class).in(Singleton.class);
- expose(SchedulerLifecycle.class);
- }
- });
-
- PubsubEventModule.bindSubscriber(binder(), SchedulerLifecycle.class);
- PubsubEventModule.bindSubscriber(binder(), TaskVars.class);
- }
-
- @Provides
- @Singleton
- List<TaskLauncher> provideTaskLaunchers(
- GcExecutorLauncher gcLauncher,
- UserTaskLauncher userTaskLauncher) {
-
- return ImmutableList.of(gcLauncher, userTaskLauncher);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/TaskIdGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/TaskIdGenerator.java b/src/main/java/com/twitter/aurora/scheduler/TaskIdGenerator.java
deleted file mode 100644
index 240649e..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/TaskIdGenerator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.UUID;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Preconditions;
-
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.util.Clock;
-
-/**
- * A function that generates universally-unique (not guaranteed, but highly confident) task IDs.
- */
-public interface TaskIdGenerator {
-
- /**
- * Generates a universally-unique ID for the task. This is not necessarily a repeatable
- * operation, two subsequent invocations with the same object need not return the same value.
- *
- * @param task Configuration of the task to create an ID for.
- * @param instanceId Instance ID for the task.
- * @return A universally-unique ID for the task.
- */
- String generate(ITaskConfig task, int instanceId);
-
- class TaskIdGeneratorImpl implements TaskIdGenerator {
- private final Clock clock;
-
- @Inject
- TaskIdGeneratorImpl(Clock clock) {
- this.clock = Preconditions.checkNotNull(clock);
- }
-
- @Override
- public String generate(ITaskConfig task, int instanceId) {
- String sep = "-";
- return new StringBuilder()
- .append(clock.nowMillis()) // Allows chronological sorting.
- .append(sep)
- .append(task.getOwner().getRole()) // Identification and collision prevention.
- .append(sep)
- .append(task.getEnvironment())
- .append(sep)
- .append(task.getJobName())
- .append(sep)
- .append(instanceId) // Collision prevention within job.
- .append(sep)
- .append(UUID.randomUUID()) // Just-in-case collision prevention.
- .toString().replaceAll("[^\\w-]", sep); // Constrain character set.
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/TaskLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/TaskLauncher.java b/src/main/java/com/twitter/aurora/scheduler/TaskLauncher.java
deleted file mode 100644
index aade6da..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/TaskLauncher.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import com.google.common.base.Optional;
-
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.TaskInfo;
-import org.apache.mesos.Protos.TaskStatus;
-
-/**
- * A receiver of resource offers and task status updates.
- */
-public interface TaskLauncher {
-
- /**
- * Grants a resource offer to the task launcher, which will be passed to any subsequent task
- * launchers if this one does not accept.
- * <p>
- * A task launcher may choose to retain an offer for later use. Any retained offers must be
- * cleaned up with {@link #cancelOffer(OfferID)}.
- *
- * @param offer The resource offer.
- * @return A task, absent if the launcher chooses not to accept the offer.
- */
- Optional<TaskInfo> createTask(Offer offer);
-
- /**
- * Informs the launcher that a status update has been received for a task. If the task is not
- * associated with the launcher, it should return {@code false} so that another launcher may
- * receive it.
- *
- * @param status The status update.
- * @return {@code true} if the status is relevant to the launcher and should not be delivered to
- * other launchers, {@code false} otherwise.
- */
- boolean statusUpdate(TaskStatus status);
-
- /**
- * Informs the launcher that a previously-advertised offer is canceled and may not be used.
- *
- * @param offer The canceled offer.
- */
- void cancelOffer(OfferID offer);
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/TaskVars.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/TaskVars.java b/src/main/java/com/twitter/aurora/scheduler/TaskVars.java
deleted file mode 100644
index 5574631..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/TaskVars.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterables;
-import com.google.common.eventbus.Subscribe;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
-import com.twitter.aurora.scheduler.storage.AttributeStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.stats.StatsProvider;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A container that tracks and exports stat counters for tasks.
- */
-class TaskVars implements EventSubscriber {
- private static final Logger LOG = Logger.getLogger(TaskVars.class.getName());
-
- // Used to ignore pubsub events sent before storage has completely started. This avoids a
- // miscount where a StorageStarted consumer is invoked before storageStarted is invoked here,
- // and pubsub events are fired for tasks that we have not yet counted. For example, if
- // tasksDeleted is invoked, we would end up with a negative count.
- private volatile boolean storageStarted = false;
-
- private final LoadingCache<String, AtomicLong> countersByStatus;
- private final LoadingCache<String, AtomicLong> countersByRack;
-
- private final Storage storage;
-
- @Inject
- TaskVars(Storage storage, final StatsProvider statProvider) {
- this.storage = checkNotNull(storage);
- checkNotNull(statProvider);
- countersByStatus = CacheBuilder.newBuilder().build(new CacheLoader<String, AtomicLong>() {
- @Override public AtomicLong load(String statName) {
- return statProvider.makeCounter(statName);
- }
- });
- countersByRack = CacheBuilder.newBuilder().build(new CacheLoader<String, AtomicLong>() {
- @Override public AtomicLong load(String rack) {
- return statProvider.makeCounter(rackStatName(rack));
- }
- });
- }
-
- @VisibleForTesting
- static String getVarName(ScheduleStatus status) {
- return "task_store_" + status;
- }
-
- @VisibleForTesting
- static String rackStatName(String rack) {
- return "tasks_lost_rack_" + rack;
- }
-
- private static final Predicate<Attribute> IS_RACK = new Predicate<Attribute>() {
- @Override public boolean apply(Attribute attr) {
- return "rack".equals(attr.getName());
- }
- };
-
- private static final Function<Attribute, String> ATTR_VALUE = new Function<Attribute, String>() {
- @Override public String apply(Attribute attr) {
- return Iterables.getOnlyElement(attr.getValues());
- }
- };
-
- private AtomicLong getCounter(ScheduleStatus status) {
- return countersByStatus.getUnchecked(getVarName(status));
- }
-
- private void incrementCount(ScheduleStatus status) {
- getCounter(status).incrementAndGet();
- }
-
- private void decrementCount(ScheduleStatus status) {
- getCounter(status).decrementAndGet();
- }
-
- @Subscribe
- public void taskChangedState(TaskStateChange stateChange) {
- if (!storageStarted) {
- return;
- }
-
- IScheduledTask task = stateChange.getTask();
- if (stateChange.getOldState() != ScheduleStatus.INIT) {
- decrementCount(stateChange.getOldState());
- }
- incrementCount(task.getStatus());
-
- if (stateChange.getNewState() == ScheduleStatus.LOST) {
- final String host = stateChange.getTask().getAssignedTask().getSlaveHost();
- Optional<String> rack = storage.consistentRead(new Work.Quiet<Optional<String>>() {
- @Override public Optional<String> apply(StoreProvider storeProvider) {
- Optional<Attribute> rack = FluentIterable
- .from(AttributeStore.Util.attributesOrNone(storeProvider, host))
- .firstMatch(IS_RACK);
- return rack.transform(ATTR_VALUE);
- }
- });
-
- if (rack.isPresent()) {
- countersByRack.getUnchecked(rack.get()).incrementAndGet();
- } else {
- LOG.warning("Failed to find rack attribute associated with host " + host);
- }
- }
- }
-
- @Subscribe
- public void storageStarted(StorageStarted event) {
- for (IScheduledTask task : Storage.Util.consistentFetchTasks(storage, Query.unscoped())) {
- incrementCount(task.getStatus());
- }
-
- // Dummy read the counter for each status counter. This is important to guarantee a stat with
- // value zero is present for each state, even if all states are not represented in the task
- // store.
- for (ScheduleStatus status : ScheduleStatus.values()) {
- getCounter(status);
- }
- storageStarted = true;
- }
-
- @Subscribe
- public void tasksDeleted(final TasksDeleted event) {
- if (!storageStarted) {
- return;
- }
-
- for (IScheduledTask task : event.getTasks()) {
- decrementCount(task.getStatus());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/UserTaskLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/UserTaskLauncher.java b/src/main/java/com/twitter/aurora/scheduler/UserTaskLauncher.java
deleted file mode 100644
index 0fb3bbb..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/UserTaskLauncher.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler;
-
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.TaskInfo;
-import org.apache.mesos.Protos.TaskStatus;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.async.OfferQueue;
-import com.twitter.aurora.scheduler.base.Conversions;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.SchedulerException;
-import com.twitter.aurora.scheduler.state.StateManager;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A task launcher that matches resource offers against user tasks.
- */
-class UserTaskLauncher implements TaskLauncher {
-
- private static final Logger LOG = Logger.getLogger(UserTaskLauncher.class.getName());
-
- @VisibleForTesting
- static final String MEMORY_LIMIT_EXCEEDED = "MEMORY STATISTICS";
-
- @VisibleForTesting
- static final String MEMORY_LIMIT_DISPLAY = "Task used more memory than requested.";
-
- private final OfferQueue offerQueue;
- private final StateManager stateManager;
-
- @Inject
- UserTaskLauncher(OfferQueue offerQueue, StateManager stateManager) {
- this.offerQueue = checkNotNull(offerQueue);
- this.stateManager = checkNotNull(stateManager);
- }
-
- @Override
- public Optional<TaskInfo> createTask(Offer offer) {
- checkNotNull(offer);
-
- offerQueue.addOffer(offer);
- return Optional.absent();
- }
-
- @Override
- public synchronized boolean statusUpdate(TaskStatus status) {
- @Nullable String message = null;
- if (status.hasMessage()) {
- message = status.getMessage();
- }
-
- try {
- ScheduleStatus translatedState = Conversions.convertProtoState(status.getState());
- // TODO(William Farner): Remove this hack once Mesos API change is done.
- // Tracked by: https://issues.apache.org/jira/browse/MESOS-343
- if ((translatedState == ScheduleStatus.FAILED)
- && (message != null)
- && (message.contains(MEMORY_LIMIT_EXCEEDED))) {
- message = MEMORY_LIMIT_DISPLAY;
- }
-
- stateManager.changeState(
- Query.taskScoped(status.getTaskId().getValue()),
- translatedState,
- Optional.fromNullable(message));
- } catch (SchedulerException e) {
- LOG.log(Level.WARNING, "Failed to update status for: " + status, e);
- throw e;
- }
- return true;
- }
-
- @Override
- public void cancelOffer(OfferID offer) {
- offerQueue.cancelOffer(offer);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/app/AppModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/app/AppModule.java b/src/main/java/com/twitter/aurora/scheduler/app/AppModule.java
deleted file mode 100644
index 24702b0..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/app/AppModule.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.app;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
-import com.google.inject.AbstractModule;
-import com.google.inject.Key;
-import com.google.inject.Provides;
-
-import org.apache.mesos.Scheduler;
-import org.apache.zookeeper.data.ACL;
-
-import com.twitter.aurora.GuiceUtils;
-import com.twitter.aurora.scheduler.SchedulerModule;
-import com.twitter.aurora.scheduler.async.AsyncModule;
-import com.twitter.aurora.scheduler.events.PubsubEventModule;
-import com.twitter.aurora.scheduler.filter.SchedulingFilterImpl;
-import com.twitter.aurora.scheduler.http.ClusterName;
-import com.twitter.aurora.scheduler.http.ServletModule;
-import com.twitter.aurora.scheduler.metadata.MetadataModule;
-import com.twitter.aurora.scheduler.quota.QuotaModule;
-import com.twitter.aurora.scheduler.state.StateModule;
-import com.twitter.aurora.scheduler.stats.AsyncStatsModule;
-import com.twitter.common.application.ShutdownRegistry;
-import com.twitter.common.application.modules.LifecycleModule;
-import com.twitter.common.base.Command;
-import com.twitter.common.inject.TimedInterceptor;
-import com.twitter.common.net.pool.DynamicHostSet;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.stats.StatsProvider;
-import com.twitter.common.util.Clock;
-import com.twitter.common.zookeeper.ServerSet;
-import com.twitter.common.zookeeper.ServerSetImpl;
-import com.twitter.common.zookeeper.SingletonService;
-import com.twitter.common.zookeeper.ZooKeeperClient;
-import com.twitter.common.zookeeper.ZooKeeperClient.Credentials;
-import com.twitter.common.zookeeper.ZooKeeperUtils;
-import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule.ClientConfig;
-import com.twitter.thrift.ServiceInstance;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * Binding module for the aurora scheduler application.
- */
-class AppModule extends AbstractModule {
- private static final Logger LOG = Logger.getLogger(AppModule.class.getName());
-
- private final String clusterName;
- private final String serverSetPath;
- private final ClientConfig zkClientConfig;
-
- AppModule(String clusterName, String serverSetPath, ClientConfig zkClientConfig) {
- this.clusterName = checkNotBlank(clusterName);
- this.serverSetPath = checkNotBlank(serverSetPath);
- this.zkClientConfig = checkNotNull(zkClientConfig);
- }
-
- @Override
- protected void configure() {
- // Enable intercepted method timings and context classloader repair.
- TimedInterceptor.bind(binder());
- GuiceUtils.bindJNIContextClassLoader(binder(), Scheduler.class);
- GuiceUtils.bindExceptionTrap(binder(), Scheduler.class);
-
- bind(Clock.class).toInstance(Clock.SYSTEM_CLOCK);
-
- bind(Key.get(String.class, ClusterName.class)).toInstance(clusterName);
-
- // Filter layering: notifier filter -> base impl
- PubsubEventModule.bind(binder(), SchedulingFilterImpl.class);
- bind(SchedulingFilterImpl.class).in(Singleton.class);
-
- LifecycleModule.bindStartupAction(binder(), RegisterShutdownStackPrinter.class);
-
- install(new AsyncModule());
- install(new AsyncStatsModule());
- install(new MetadataModule());
- install(new QuotaModule());
- install(new ServletModule());
- install(new SchedulerModule());
- install(new StateModule());
-
- bind(StatsProvider.class).toInstance(Stats.STATS_PROVIDER);
- }
-
- /**
- * Command to register a thread stack printer that identifies initiator of a shutdown.
- */
- private static class RegisterShutdownStackPrinter implements Command {
- private static final Function<StackTraceElement, String> STACK_ELEM_TOSTRING =
- new Function<StackTraceElement, String>() {
- @Override public String apply(StackTraceElement element) {
- return element.getClassName() + "." + element.getMethodName()
- + String.format("(%s:%s)", element.getFileName(), element.getLineNumber());
- }
- };
-
- private final ShutdownRegistry shutdownRegistry;
-
- @Inject
- RegisterShutdownStackPrinter(ShutdownRegistry shutdownRegistry) {
- this.shutdownRegistry = shutdownRegistry;
- }
-
- @Override
- public void execute() {
- shutdownRegistry.addAction(new Command() {
- @Override public void execute() {
- Thread thread = Thread.currentThread();
- String message = new StringBuilder()
- .append("Thread: ").append(thread.getName())
- .append(" (id ").append(thread.getId()).append(")")
- .append("\n")
- .append(Joiner.on("\n ").join(
- Iterables.transform(Arrays.asList(thread.getStackTrace()), STACK_ELEM_TOSTRING)))
- .toString();
-
- LOG.info("Shutdown initiated by: " + message);
- }
- });
- }
- }
-
- @Provides
- @Singleton
- List<ACL> provideAcls() {
- if (zkClientConfig.credentials == Credentials.NONE) {
- LOG.warning("Running without ZooKeeper digest credentials. ZooKeeper ACLs are disabled.");
- return ZooKeeperUtils.OPEN_ACL_UNSAFE;
- } else {
- return ZooKeeperUtils.EVERYONE_READ_CREATOR_ALL;
- }
- }
-
- @Provides
- @Singleton
- ServerSet provideServerSet(ZooKeeperClient client, List<ACL> zooKeeperAcls) {
- return new ServerSetImpl(client, zooKeeperAcls, serverSetPath);
- }
-
- @Provides
- @Singleton
- DynamicHostSet<ServiceInstance> provideSchedulerHostSet(ServerSet serverSet) {
- // Used for a type re-binding of the serverset.
- return serverSet;
- }
-
- @Provides
- @Singleton
- SingletonService provideSingletonService(
- ZooKeeperClient client,
- ServerSet serverSet,
- List<ACL> zookeeperAcls) {
-
- return new SingletonService(
- serverSet,
- SingletonService.createSingletonCandidate(client, serverSetPath, zookeeperAcls));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/app/Log4jConfigurator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/app/Log4jConfigurator.java b/src/main/java/com/twitter/aurora/scheduler/app/Log4jConfigurator.java
deleted file mode 100644
index 0ea2204..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/app/Log4jConfigurator.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.app;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.log4j.BasicConfigurator;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import com.twitter.common.logging.RootLogConfig;
-import com.twitter.common.logging.RootLogConfig.Configuration;
-import com.twitter.common.logging.log4j.GlogLayout;
-
-/**
- * Configures log4j logging.
- */
-final class Log4jConfigurator {
- private static final java.util.logging.Logger LOG =
- java.util.logging.Logger.getLogger(Log4jConfigurator.class.getName());
-
- /**
- * Configures log4j to log to stderr with a glog format.
- *
- * @param glogConfig The glog configuration in effect.
- */
- static void configureConsole(Configuration glogConfig) {
- Preconditions.checkNotNull(glogConfig);
-
- BasicConfigurator.configure(
- new ConsoleAppender(new GlogLayout(), ConsoleAppender.SYSTEM_ERR));
- Logger.getRootLogger().setLevel(getLevel(glogConfig));
- }
-
- private static Level getLevel(RootLogConfig.Configuration logConfig) {
- switch (logConfig.getVlog()) {
- case FINEST: // fall through
- case FINER: // fall through
- case FINE: // fall through
- case CONFIG:
- return Level.TRACE;
- case INFO:
- return Level.INFO;
- case WARNING:
- return Level.WARN;
- case SEVERE:
- return Level.ERROR;
- default:
- LOG.warning("Mapping unexpected vlog value of " + logConfig.getVlog() + " to log4j TRACE");
- return Level.TRACE;
- }
- }
-
- private Log4jConfigurator() {
- // Utility class.
- }
-}
[02/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/util.js
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/util.js b/src/main/resources/com/twitter/aurora/scheduler/http/assets/util.js
deleted file mode 100644
index 5dd0d17..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/assets/util.js
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Given an epoch milliseconds timestamp, generate a string representation that includes
- * GMT and local time.
- */
-function printDate(timestamp) {
- function pad(number) {
- number = '' + number;
- if (number.length < 2) {
- return '0' + number;
- }
- return number;
- }
- var d = new Date(timestamp);
- return pad(d.getUTCMonth() + 1) + "/"
- + pad(d.getUTCDate()) + " "
- + pad(d.getUTCHours()) + ":"
- + pad(d.getUTCMinutes()) + ":"
- + pad(d.getUTCSeconds())
- + ' UTC ('
- + pad(d.getMonth() + 1) + "/"
- + pad(d.getDate()) + " "
- + pad(d.getHours()) + ":"
- + pad(d.getMinutes()) + ":"
- + pad(d.getSeconds())
- + ' local)';
-}
-
-/**
- * Given the current time and an epoch milliseconds timestamp, generate a human-readable
- * representation of how long has elapsed from timestamp to now.
- */
-function printElapsed(newerTimestamp, olderTimestamp) {
- var elapsed = newerTimestamp - olderTimestamp;
- if (elapsed <= 0) {
- return "epsilon";
- }
- var SEC = 1000;
- var MIN = 60 * SEC;
- var HOUR = 60 * MIN;
- var DAY = 24 * HOUR;
- if (elapsed < 2 * SEC) {
- return Math.floor(elapsed) + " msec";
- }
- if (elapsed < 2 * MIN) {
- return Math.floor(elapsed / SEC) + " secs";
- }
- if (elapsed < 2 * HOUR) {
- return Math.floor(elapsed / MIN) + " mins";
- }
- if (elapsed < 2 * DAY) {
- return Math.floor(elapsed / HOUR) + " hours";
- }
- return Math.floor(elapsed / DAY) + " days";
-}
-
-function scaleMb(sizeInMb) {
- var size = sizeInMb;
- var SCALE = ["MiB", "GiB", "TiB", "PiB", "EiB"];
- var unit = 0;
- while(size >= 1024 && unit < SCALE.length) {
- size = size / 1024;
- unit++;
- }
- return size.toFixed(2).toString() + " " + SCALE[unit];
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzhome.st
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzhome.st b/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzhome.st
deleted file mode 100644
index 5e540f3..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzhome.st
+++ /dev/null
@@ -1,53 +0,0 @@
-<!DOCTYPE html>
-<html>
- <head>
- <meta charset="UTF-8">
- <meta name="google" value="notranslate">
- <link rel="icon" href="/images/aurora.png" type="image/png" />
- <title>Aurora $cluster_name$ scheduler</title>
- <link href="/css/bootstrap.min.css" rel="stylesheet" />
- <link href="/css/jquery.dataTables.css" rel="stylesheet" />
- <script type="text/javascript" src="/js/jquery.min.js"></script>
- <script type="text/javascript" src="/js/bootstrap.min.js"></script>
- <script type="text/javascript" src="/js/jquery.dataTables.min.js"></script>
- <script type="text/javascript" src="/js/dataTables.bootstrap.js"></script>
- <script type="text/javascript" src="/js/dataTables.localstorage.js"></script>
- </head>
- <body>
- <div class="container-fluid">
- <div class="row-fluid">
- <center>
- <h2>Scheduled Jobs</h2>
- </center>
- </div>
-
- <table id="jobs-table" class="table table-bordered table-striped table-condensed table-hover">
- <thead>
- <th>Role
- <th>Jobs
- <th>Cron Jobs
- </thead>
- <tbody>
- $owners:{ owner |
- <tr>
- <td><a href="/scheduler/$owner.role$">$owner.role$</a>
- <td>$owner.jobCount$
- <td>$owner.cronJobCount$
- </tr>
- }$
- </tbody>
- </table>
- $if(exception)$
- <font color='red'>Exception: $exception$</font>
- $endif$
- </div>
-
- <script type="text/javascript">
- jQuery(function(){
- jQuery('#jobs-table').dataTable({
- 'bStateSave': true
- });
- });
- </script>
- </body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzjob.st
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzjob.st b/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzjob.st
deleted file mode 100644
index 28b5667..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzjob.st
+++ /dev/null
@@ -1,315 +0,0 @@
-<!DOCTYPE html>
-<html>
- <head>
- <meta charset="UTF-8">
- <meta name="google" value="notranslate">
- <link rel="icon" href="/images/aurora.png" type="image/png" />
- <title>Aurora $cluster_name$ scheduler - Tasks for $role$/$environment$/$job$</title>
- <link href="/css/bootstrap.min.css" rel="stylesheet" />
- <script type="text/javascript" src="/js/jquery.min.js"></script>
- <script type="text/javascript" src="/js/bootstrap.min.js"></script>
- <script type="text/javascript" src="/js/util.js"></script>
- <script type="text/javascript" src="/js/dictionary.js"></script>
- <style>
- .expand-btn:hover{
- text-decoration: none;
- }
- .task-status {
- cursor: help;
- border-bottom: 1px #aaaaaa dashed;
- }
- </style>
- </head>
- <body>
- <div class="container-fluid">
- <div class="row-fluid">
- <div style="float:right; position:absolute; right:20px">
- <a href="$statsUrl$"><img src="/images/viz.png" alt="Container stats on viz" /></a>
- </div>
- <center>
- <h2>Job: $job$</h2>
- <h5 style='margin-top: -12px;'>Role: $role$ - Environment: $environment$</h5>
- </center>
- </div>
-
- <div class="row-fluid">
- <ul class="breadcrumb">
- <li>
- <a href="/scheduler">Home</a> <span class="divider">></span>
- </li>
- <li>
- <a href="/scheduler/$role$">Role: $role$</a> <span class="divider">></span>
- </li>
- <li>
- <a href="/scheduler/$role$/$environment$">Environment: $environment$</a>
- <span class="divider">></span>
- </li>
- <li class="active">Job: $job$</li>
- </ul>
- </div>
-
- $if(admin)$
- $if(is_cron)$
- (<a href="/structdump/cron/$role$/$environment$/$job$">dump</a>)
- $endif$
- $endif$
-
-
- $if(schedulingDetails)$
- <div class="row-fluid" style="margin-bottom: 20px;">
- <button id="scheduling-details-toggle-btn" class="btn">
- <i class="icon-plus" style="margin-right: 3px;"></i>
- <span class="btn-text">Show Details</span>
- </button>
- <div id="scheduling-details" class="collapse" style="width: 400px; margin-top: 10px;">
- <table class="table table-condensed" style="margin-bottom: 0px;">
- <thead>
- <th>Instances
- <th>Details
- </thead>
- <tbody>
- $schedulingDetails.keys:{ instances |
- <tr>
- <td style='text-align:center; vertical-align:middle'>$instances$
- <td>
- <table class="table table-condensed" style="margin-bottom: 0px;">
- <tbody>
- $schedulingDetails.(instances).details.keys: { name |
- <tr>
- <td>$name$
- <td>$schedulingDetails.(instances).details.(name)$
- </tr>
- }$
- </tbody>
- </table>
- </tr>
- }$
- </tbody>
- </table>
- </div>
- </div>
- $endif$
-
- <script>
- // The current Date, used in calls to printElapsed().
- var nowTimestamp = new Date().getTime();
- </script>
-
- $if(activeTasks)$
- <div style='clear:both'>
- <h3>Active Tasks</h3>
- <table class="table table-striped table-condensed table-hover">
- <thead>
- <th>Instance
- <th>Status
- $if(admin)$
- <th>Task ID
- $endif$
- <th>Host
- </thead>
- <tbody>
- $activeTasks:{ task |
- <tr>
- <td style='vertical-align:middle'>
- $task.instanceId$ $task.links.keys, task.links.values: { k, v | <a href='$v$'>$k$</a> }$
- <td style='vertical-align:middle'>
- <a href='#accordion-$task.taskId$' class='expand-btn' title='Show details'>
- <i class='icon-plus'></i>
- </a>
- <script>
- document.write(printElapsed(nowTimestamp, $task.statusTimestamp$) + ' ago ');
- </script>
- - <span class='task-status' status='$task.status$'>$task.status$ </span>
- $if(task.pendingReason)$ ($task.pendingReason$)$endif$
- <div class='collapse' id='accordion-$task.taskId$' style='margin-bottom: 0px'>
- <ul>
- $task.taskEvents:{ event |
- <li>
- <script>document.write(printDate($event.timestamp$));</script>
- - <span class='task-status' status='$event.status$'>$event.status$</span>
- $if(event.message)$: $event.message$$endif$
- </li>
- }$
- </ul>
- </div>
- $if(admin)$
- <td><a href='/structdump/task/$task.taskId$'>$task.taskId$</a>
- $endif$
- <td style='vertical-align:middle'>
- $if(task.slaveHost)$ <a href='$task.executorUri$'>$task.slaveHost$</a> $endif$
- </tr>
- }$
- </tbody>
- </table>
- </div>
- $endif$
-
- <br />
- <br />
-
- $if(completedTasks)$
- <h3>Completed Tasks</h3>
- <table class="table table-striped table-condensed table-hover">
- <thead>
- <th>Instance
- <th>Status
- <th>Running duration
- $if(admin)$
- <th>Task ID
- $endif$
- <th>Host
- </thead>
- <tbody>
- $completedTasks:{ task |
- <tr>
- <td>$task.instanceId$
- <td style='vertical-align:middle'>
- <a href='#collapse-$task.taskId$' class='expand-btn' title='Show details'>
- <i class='icon-plus'></i>
- </a>
- <script>
- document.write(printElapsed(nowTimestamp, $task.statusTimestamp$) + ' ago ');
- </script>
- - <span class='task-status' status='$task.status$'>$task.status$</span>
- $last(task.taskEvents):{ event |
- $if(event.message)$
- : $event.message$
- $endif$
- }$
-
- <div id='collapse-$task.taskId$' class='collapse'>
- <ul>
- $task.taskEvents:{ event |
- <li>
- <script>document.write(printDate($event.timestamp$));</script>
- - <span class='task-status' status='$event.status$'>$event.status$</span>
- $if(event.message)$: $event.message$$endif$
- </li>
- }$
- </ul>
- </div>
- <td>
- <script>
- var events = [
- $task.taskEvents:{ event | [ '$event.status$', $event.timestamp$ ], }$
- ];
- var aliveTimestamp = 0;
- for (var i = events.length - 1; i >= 0; i--) {
- if (events[i][0] == 'RUNNING') {
- aliveTimestamp = events[i][1];
- break;
- }
- }
- if (aliveTimestamp != 0) {
- document.write(printElapsed(events[events.length - 1][1], aliveTimestamp));
- }
- </script>
- $if(admin)$
- <td><a href='/structdump/task/$task.taskId$'>$task.taskId$</a>
- $endif$
- <td>$if(task.slaveHost)$ <a href='$task.executorUri$'>$task.slaveHost$</a> $endif$
- </tr>
- }$
- </tbody>
- </table>
- $endif$
-
- $if(exception)$
- <font color='red'>Exception: $exception$</font>
- $endif$
-
- <div class='pagination' style='text-align: center'>
- <ul>
- <li $if(!prevOffset)$ class='disabled' $endif$>
- $if(prevOffset)$
- <a href='/scheduler/$role$/$environment$/$job$?$if(status)$status=$status$$endif$&o=$prevOffset$'>«</a>
- $else$
- <span>«</span>
- $endif$
- </li>
- <li $if(!nextOffset)$ class='disabled' $endif$>
- $if(nextOffset)$
- <a href='/scheduler/$role$/$environment$/$job$?$if(status)$status=$status$$endif$&o=$nextOffset$'>»</a>
- $else$
- <span>»</span>
- $endif$
- </li>
- </ul>
- </div>
- </div>
-
- <script type="text/javascript">
- jQuery(function(){
- // Enable tooltips on all expansion buttons
- jQuery('.expand-btn').tooltip();
-
- // Bind click event to all expand buttons
- jQuery('.expand-btn').on('click', function(e){
- e.stopPropagation();
- e.preventDefault();
-
- jQuery(jQuery(this).attr('href')).collapse('toggle');
- });
-
- // Switch between + and - icons based on button state
- jQuery('div.collapse').on('show', function(){
- var parent = jQuery(this).parent();
- var icon = parent.find('i');
- var button = parent.find('.expand-btn');
-
- icon.addClass('icon-minus').removeClass('icon-plus');
- button.attr('data-original-title', 'Hide details').tooltip('hide');
- });
-
- jQuery('div.collapse').on('hide', function(){
- var parent = jQuery(this).parent();
- var icon = parent.find('i');
- var button = parent.find('.expand-btn');
-
- icon.addClass('icon-plus').removeClass('icon-minus');
- button.attr('data-original-title', 'View details');
- });
-
- // Attach tooltips to task states
- jQuery('.task-status').each(function() {
- jQuery(this).popover({
- placement: 'right',
- trigger: 'hover',
- title: jQuery(this).attr('status'),
- content: dictionary.states[jQuery(this).attr('status')]
- });
- });
- });
-
- // Bind click event to button
- jQuery('#scheduling-details-toggle-btn').on('click', function(e){
- e.stopPropagation();
- e.preventDefault();
- jQuery('#scheduling-details').collapse('toggle');
- });
-
- // Switch between + and - icons based on button state
- jQuery('#scheduling-details').on('show', function(){
- var parent = jQuery(this).parent();
- var icon = parent.find('i');
- var buttonText = parent.find('.btn-text');
-
- icon.addClass('icon-minus').removeClass('icon-plus');
- buttonText.html('Hide Details');
- });
-
- jQuery('#scheduling-details').on('hide', function(){
- var parent = jQuery(this).parent();
- var icon = parent.find('i');
- var buttonText = parent.find('.btn-text');
-
- icon.addClass('icon-plus').removeClass('icon-minus');
- buttonText.html('Show Details');
- });
- </script>
- </body>
- <div style='bottom: 10px; right: 10px; position: fixed;'>
- <a href='/scheduler/$role$/$environment$/$job$?admin$if(status)$&status=$status$$endif$&o=$offset$'>π</a>
- </div>
-</html>
-
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzrole.st
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzrole.st b/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzrole.st
deleted file mode 100644
index d568a7e..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/schedulerzrole.st
+++ /dev/null
@@ -1,265 +0,0 @@
-<!DOCTYPE html>
-<html>
- <head>
- <meta charset="UTF-8">
- <meta name="google" value="notranslate">
- <link rel="icon" href="/images/aurora.png" type="image/png" />
- <title>Aurora $cluster_name$ scheduler - Jobs for $role$</title>
- <link href="/css/bootstrap.min.css" rel="stylesheet" />
- <link href="/css/jquery.dataTables.css" rel="stylesheet" />
- <script type="text/javascript" src="/js/jquery.min.js"></script>
- <script type="text/javascript" src="/js/bootstrap.min.js"></script>
- <script type="text/javascript" src="/js/util.js"></script>
- <script type="text/javascript" src="/js/jquery.dataTables.min.js"></script>
- <script type="text/javascript" src="/js/dataTables.bootstrap.js"></script>
- <script type="text/javascript" src="/js/dataTables.localstorage.js"></script>
- <script type="text/javascript" src="/js/dataTables.htmlNumberType.js"></script>
- </head>
- <body>
- <div class="container-fluid">
- <div class="row-fluid">
- <center>
- <h2>Jobs for role $role$ $if(environment)$ and environment $environment$ $endif$ </h2>
- </center>
- </div>
-
- <div class="row-fluid">
- <ul class="breadcrumb">
- <li>
- <a href="/scheduler">Home</a> <span class="divider">></span>
- </li>
- $if(environment)$
- <li><a href="/scheduler/$role$">Role : $role$</a> <span class="divider">></span></li>
- <li>Environment: $environment$</li>
- $else$
- <li class="active">Role: $role$</li>
- $endif$
- </ul>
- </div>
-
- $if(jobs)$
- $if(environment)$
- <span/>
- $else$
- <div class="row-fluid" style="margin-bottom: 20px;">
- <button id="prod-resources-toggle-btn" class="btn">
- <i class="icon-plus" style="margin-right: 3px;"></i><span class="btn-text">Show Resource Consumption</span>
- </button>
- <div id="prod-resources" class="collapse" style="width: 600px; margin-top: 10px;">
- <table class="table table-striped table-condensed table-hover">
- <thead>
- <th>Resource
- <th><span class="label" title="Resources used by production jobs(counted against quota)">Production consumption</span>
- <th><span class="label" title="Quota allocated in production">Quota</span>
- <th><span class="label" title="Resources used by non-production jobs (not counted against your quota)">Non-Production consumption</span>
- </thead>
- <tbody>
- <tr>
- <td>CPU
- <td>$prodResourcesUsed.numCpus$ cores
- <td>$resourceQuota.numCpus$ cores
- <td>$nonProdResourcesUsed.numCpus$ cores
- </tr>
- <tr>
- <td>RAM
- <td><script>document.write(scaleMb($prodResourcesUsed.ramMb$));</script>
- <td><script>document.write(scaleMb($resourceQuota.ramMb$));</script>
- <td><script>document.write(scaleMb($nonProdResourcesUsed.ramMb$));</script>
- </tr>
- <tr>
- <td>Disk
- <td><script>document.write(scaleMb($prodResourcesUsed.diskMb$));</script>
- <td><script>document.write(scaleMb($resourceQuota.diskMb$));</script>
- <td><script>document.write(scaleMb($nonProdResourcesUsed.diskMb$));</script>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
- $endif$
-
- <div class="row-fluid" style="margin-bottom: 20px;">
- <table id="jobs-table" class="table table-bordered table-striped table-condensed table-hover">
- <thead>
- <th>Job Type
- <th>Environment
- <th>Job
- <th>Pending Tasks
- <th>Active Tasks
- <th>Finished Tasks
- <th>Failed Tasks
- </thead>
- <tbody>
- $jobs:{ job |
- <tr>
- <td>
- <span class="jobType">$job.type$</span>
- <td>
- <a href="/scheduler/$role$/$job.environment$">$job.environment$</a>
- <td>
- <a href="/scheduler/$role$/$job.environment$/$job.name$">$job.name$</a>
- <span class="job-status pull-right badge"></span>
- <span class="production-badge pull-right badge"></span>
- <input type="hidden" class="production" value="$job.production$" />
- <td>
- <a href="/scheduler/$role$/$job.environment$/$job.name$?status=pending">$job.pendingTaskCount$</a>
- <input type="hidden" class="pending-tasks" value="$job.pendingTaskCount$" />
- <td>
- <a href="/scheduler/$role$/$job.environment$/$job.name$?status=running">$job.activeTaskCount$</a>
- <input type="hidden" class="active-tasks" value="$job.activeTaskCount$" />
- <td>
- <a href="/scheduler/$role$/$job.environment$/$job.name$?status=finished">$job.finishedTaskCount$</a>
- <input type="hidden" class="finished-tasks" value="$job.finishedTaskCount$" />
- <td>
- <a href="/scheduler/$role$/$job.environment$/$job.name$?status=failed">$job.failedTaskCount$</a>
- <input type="hidden" class="failed-tasks" value="$job.failedTaskCount$" />
- <input type="hidden" class="recently-failed-tasks"
- value="$job.recentlyFailedTaskCount$" />
- </tr>
- }$
- </tbody>
- </table>
- </div>
- $endif$
-
- $if(cronJobs)$
- <div class="row-fluid">
- <h4>Cron jobs</h4>
- <div>
- <table id="cron-jobs-table" class="table table-bordered table-striped table-condensed table-hover">
- <thead>
- <th>Environment
- <th>Name
- <th>Tasks
- <th>Schedule
- <th>Next Run
- <th>Collision policy
- <th>Packages
- </thead>
- <tbody>
- $cronJobs:{ job |
- <tr>
- <td>$job.environment$
- <td>$job.name$
- <td>$job.pendingTaskCount$
- <td>$job.cronSchedule$
- <td><script>document.write(printDate($job.nextRun$));</script>
- <td>$job.cronCollisionPolicy$
- <td>$job.packages$
- </tr>
- }$
- </tbody>
- </table>
- </div>
- </div>
- $endif$
- </div>
-
- $if(exception)$
- <font color='red'>Exception: $exception$</font>
- $endif$
-
- <script type="text/javascript">
- jQuery(function(){
- jQuery('#jobs-table').dataTable({
- 'iDisplayLength': 25,
- 'aLengthMenu': [[10, 25, 50, 100, -1], [10, 25, 50, 100, "All"]],
- 'bStateSave': false,
- 'aoColumns': [
- {'bSearchable': true},
- {'bSearchable': false},
- {'bSearchable': true},
- {'bSearchable': false, 'sType': 'num-html'},
- {'bSearchable': false, 'sType': 'num-html'},
- {'bSearchable': false, 'sType': 'num-html'},
- {'bSearchable': false, 'sType': 'num-html'}
- ],
- 'fnDrawCallback': drawBadges
- });
- jQuery('#cron-jobs-table').dataTable({
- 'iDisplayLength': 25,
- 'aLengthMenu': [[10, 25, 50, 100, -1], [10, 25, 50, 100, "All"]],
- 'bStateSave': false,
- 'aoColumns': [
- {'bSearchable': false},
- {'bSearchable': true},
- {'bSearchable': false},
- {'bSearchable': false, 'bSortable': false},
- {'bSearchable': false},
- {'bSearchable': true},
- {'bSearchable': true, 'bSortable': true}
- ]
- });
-
- // Bind click event to button
- jQuery('#prod-resources-toggle-btn').on('click', function(e){
- e.stopPropagation();
- e.preventDefault();
- jQuery('#prod-resources').collapse('toggle');
- });
-
- // Switch between + and - icons based on button state
- jQuery('#prod-resources').on('show', function(){
- var parent = jQuery(this).parent();
- var icon = parent.find('i');
- var buttonText = parent.find('.btn-text');
-
- icon.addClass('icon-minus').removeClass('icon-plus');
- buttonText.html('Hide Resource Consumption');
- });
-
- jQuery('#prod-resources').on('hide', function(){
- var parent = jQuery(this).parent();
- var icon = parent.find('i');
- var buttonText = parent.find('.btn-text');
-
- icon.addClass('icon-plus').removeClass('icon-minus');
- buttonText.html('Show Resource Consumption');
- });
-
- function drawBadges() {
- jQuery('#jobs-table>tbody').find('tr').each(function(){
- var tr = jQuery(this);
- displayHealthBadge(tr);
- displayProductionBadge(tr);
- });
- }
-
- function displayHealthBadge(tr) {
- var activeTasks = Number(tr.find('.active-tasks').val());
- var recentlyFailedTasks = Number(tr.find('.recently-failed-tasks').val());
- var pendingTasks = Number(tr.find('.pending-tasks').val());
- var finishedTasks = Number(tr.find('.finished-tasks').val());
- var totalTasks = pendingTasks + activeTasks + finishedTasks;
- var jobStatus = tr.find('.job-status');
-
- if (recentlyFailedTasks >= (totalTasks / 2)) {
- jobStatus.addClass('label-important').html('Unstable');
- jobStatus.attr('title', 'More than half of total tasks have failed in last 6 hours');
- } else if (recentlyFailedTasks > 0) {
- jobStatus.addClass('label-warning').html('Warning');
- jobStatus.attr('title', 'One or more tasks have failed in last 6 hours');
- } else {
- jobStatus.addClass('label-success').html('Stable');
- jobStatus.attr('title', 'No failed tasks in last 6 hours');
- }
-
- jobStatus.tooltip();
- }
-
- function displayProductionBadge(tr) {
- var productionBadgeElem = tr.find('.production-badge');
- var production = tr.find('.production').val();
- if (production == "true") {
- productionBadgeElem.addClass('badge-important').html('P');
- productionBadgeElem.attr('title', 'This is a production job. This job gets higher priority over a non-production job.');
- } else {
- productionBadgeElem.hide();
- }
-
- productionBadgeElem.tooltip();
- }
- });
- </script>
- </body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/slaves.st
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/slaves.st b/src/main/resources/com/twitter/aurora/scheduler/http/slaves.st
deleted file mode 100644
index 0d28dd9..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/slaves.st
+++ /dev/null
@@ -1,30 +0,0 @@
-<html>
- <head>
- <title>Aurora $cluster_name$ slaves</title>
- <link href="/css/bootstrap.min.css" rel="stylesheet" />
- </head>
- <body>
- <h2>Slave Hosts</h2>
- <table class="table table-striped table-condensed table-hover">
- <thead>
- <th>Host
- <th>Slave ID
- <th>Maintenance mode
- <th>Attributes
- </thead>
- <tbody>
- $slaves:{ slave |
- <tr>
- <td><a href="http://$slave.host$:8081">$slave.host$</a>
- <td>$slave.id$
- <td>$slave.mode$
- <td>$slave.attributes$
- </tr>
- }$
- </tbody>
- </table>
- $if(exception)$
- <font color='red'>Exception: $exception$</font>
- $endif$
- </body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/structdump.st
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/structdump.st b/src/main/resources/com/twitter/aurora/scheduler/http/structdump.st
deleted file mode 100644
index e440990..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/structdump.st
+++ /dev/null
@@ -1,13 +0,0 @@
-<html>
- <title>Aurora Struct Dump</title>
- <head>$id$</head>
- <body>
- <pre>
- $structPretty$
- </pre>
-
- $if(exception)$
- <font color='red'>Exception: $exception$</font>
- $endif$
- </body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/utilization.st
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/utilization.st b/src/main/resources/com/twitter/aurora/scheduler/http/utilization.st
deleted file mode 100644
index 1e5c625..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/utilization.st
+++ /dev/null
@@ -1,35 +0,0 @@
-<html>
- <head>
- <title>Aurora $cluster_name$ slaves</title>
- <link href="/css/bootstrap.min.css" rel="stylesheet" />
- </head>
- <body>
- <h2>Cluster utilization</h2>
- <table class="table table-striped table-condensed table-hover">
- <thead>
- <th>Type
- <th>CPU
- <th>RAM
- <th>Disk
- </thead>
- <tbody>
- $metrics:{ metric |
- <tr>
- <td>
- $if(metric.link)$
- <a href="/utilization/$metric.link$">$metric.title$</a>
- $else$
- $metric.title$
- $endif$
- <td>$metric.cpu$
- <td>$metric.ramGb$ GB
- <td>$metric.diskGb$ GB
- </tr>
- }$
- </tbody>
- </table>
- $if(exception)$
- <font color='red'>Exception: $exception$</font>
- $endif$
- </body>
-</html>
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css
new file mode 100644
index 0000000..ee6da23
--- /dev/null
+++ b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/css/jquery.dataTables.css
@@ -0,0 +1,195 @@
+/*
+ * File: jquery.dataTables.css
+ * Version: 1.9.4
+ * Author: Allan Jardine (www.sprymedia.co.uk)
+ * Info: www.datatables.net
+ *
+ * Copyright 2008-2012 Allan Jardine, all rights reserved.
+ *
+ * This source file is free software, under either the GPL v2 license or a
+ * BSD style license, available at:
+ * http://datatables.net/license_gpl2
+ * http://datatables.net/license_bsd
+ *
+ * This source file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
+ */
+
+div.dataTables_length label {
+ float: left;
+ text-align: left;
+}
+
+div.dataTables_length select {
+ width: 75px;
+}
+
+div.dataTables_filter label {
+ float: right;
+}
+
+div.dataTables_info {
+ padding-top: 8px;
+}
+
+div.dataTables_paginate {
+ float: right;
+ margin: 0;
+}
+
+table.table {
+ clear: both;
+ margin-bottom: 6px !important;
+ max-width: none !important;
+}
+
+table.table thead .sorting,
+table.table thead .sorting_asc,
+table.table thead .sorting_desc,
+table.table thead .sorting_asc_disabled,
+table.table thead .sorting_desc_disabled {
+ cursor: pointer;
+ *cursor: hand;
+}
+
+table.table thead .sorting { background: url('/images/sort_both.png') no-repeat center right; }
+table.table thead .sorting_asc { background: url('/images/sort_asc.png') no-repeat center right; }
+table.table thead .sorting_desc { background: url('/images/sort_desc.png') no-repeat center right; }
+
+table.table thead .sorting_asc_disabled { background: url('/images/sort_asc_disabled.png') no-repeat center right; }
+table.table thead .sorting_desc_disabled { background: url('/images/sort_desc_disabled.png') no-repeat center right; }
+
+table.dataTable th:active {
+ outline: none;
+}
+
+/* Scrolling */
+div.dataTables_scrollHead table {
+ margin-bottom: 0 !important;
+ border-bottom-left-radius: 0;
+ border-bottom-right-radius: 0;
+}
+
+div.dataTables_scrollHead table thead tr:last-child th:first-child,
+div.dataTables_scrollHead table thead tr:last-child td:first-child {
+ border-bottom-left-radius: 0 !important;
+ border-bottom-right-radius: 0 !important;
+}
+
+div.dataTables_scrollBody table {
+ border-top: none;
+ margin-bottom: 0 !important;
+}
+
+div.dataTables_scrollBody tbody tr:first-child th,
+div.dataTables_scrollBody tbody tr:first-child td {
+ border-top: none;
+}
+
+div.dataTables_scrollFoot table {
+ border-top: none;
+}
+
+
+
+
+/*
+ * TableTools styles
+ */
+.table tbody tr.active td,
+.table tbody tr.active th {
+ background-color: #08C;
+ color: white;
+}
+
+.table tbody tr.active:hover td,
+.table tbody tr.active:hover th {
+ background-color: #0075b0 !important;
+}
+
+.table-striped tbody tr.active:nth-child(odd) td,
+.table-striped tbody tr.active:nth-child(odd) th {
+ background-color: #017ebc;
+}
+
+table.DTTT_selectable tbody tr {
+ cursor: pointer;
+ *cursor: hand;
+}
+
+div.DTTT .btn {
+ color: #333 !important;
+ font-size: 12px;
+}
+
+div.DTTT .btn:hover {
+ text-decoration: none !important;
+}
+
+
+ul.DTTT_dropdown.dropdown-menu a {
+ color: #333 !important; /* needed only when demo_page.css is included */
+}
+
+ul.DTTT_dropdown.dropdown-menu li:hover a {
+ background-color: #0088cc;
+ color: white !important;
+}
+
+/* TableTools information display */
+div.DTTT_print_info.modal {
+ height: 150px;
+ margin-top: -75px;
+ text-align: center;
+}
+
+div.DTTT_print_info h6 {
+ font-weight: normal;
+ font-size: 28px;
+ line-height: 28px;
+ margin: 1em;
+}
+
+div.DTTT_print_info p {
+ font-size: 14px;
+ line-height: 20px;
+}
+
+
+
+/*
+ * FixedColumns styles
+ */
+div.DTFC_LeftHeadWrapper table,
+div.DTFC_LeftFootWrapper table,
+table.DTFC_Cloned tr.even {
+ background-color: white;
+}
+
+div.DTFC_LeftHeadWrapper table {
+ margin-bottom: 0 !important;
+ border-top-right-radius: 0 !important;
+ border-bottom-left-radius: 0 !important;
+ border-bottom-right-radius: 0 !important;
+}
+
+div.DTFC_LeftHeadWrapper table thead tr:last-child th:first-child,
+div.DTFC_LeftHeadWrapper table thead tr:last-child td:first-child {
+ border-bottom-left-radius: 0 !important;
+ border-bottom-right-radius: 0 !important;
+}
+
+div.DTFC_LeftBodyWrapper table {
+ border-top: none;
+ margin-bottom: 0 !important;
+}
+
+div.DTFC_LeftBodyWrapper tbody tr:first-child th,
+div.DTFC_LeftBodyWrapper tbody tr:first-child td {
+ border-top: none;
+}
+
+div.DTFC_LeftFootWrapper table {
+ border-top: none;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_disabled.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_disabled.png
new file mode 100644
index 0000000..881de79
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_disabled.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled.png
new file mode 100644
index 0000000..c608682
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png
new file mode 100644
index 0000000..d300f10
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/back_enabled_hover.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_disabled.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_disabled.png
new file mode 100644
index 0000000..6a6ded7
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_disabled.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled.png
new file mode 100644
index 0000000..a4e6b53
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png
new file mode 100644
index 0000000..fc46c5e
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc.png
new file mode 100644
index 0000000..a88d797
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png
new file mode 100644
index 0000000..4e144cf
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_both.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_both.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_both.png
new file mode 100644
index 0000000..1867040
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_both.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc.png
new file mode 100644
index 0000000..def071e
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png
new file mode 100644
index 0000000..7824973
Binary files /dev/null and b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js
new file mode 100644
index 0000000..420e507
--- /dev/null
+++ b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js
@@ -0,0 +1,165 @@
+/*
+ * File: dataTables.bootstrap.js
+ * Version: 1.9.4
+ * Author: Allan Jardine (www.sprymedia.co.uk)
+ * Info: www.datatables.net
+ *
+ * Copyright 2008-2012 Allan Jardine, all rights reserved.
+ *
+ * This source file is free software, under either the GPL v2 license or a
+ * BSD style license, available at:
+ * http://datatables.net/license_gpl2
+ * http://datatables.net/license_bsd
+ *
+ * This source file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
+ */
+
+/* Set the defaults for DataTables initialisation */
+$.extend( true, $.fn.dataTable.defaults, {
+ "sDom": "<'row-fluid'<'span4'l><'span2'r><'span6'f>>t<'row-fluid'<'span6'i><'span6'p>>",
+ "sPaginationType": "bootstrap",
+ "oLanguage": {
+ "sLengthMenu": "_MENU_ records per page"
+ }
+} );
+
+
+/* Default class modification */
+$.extend( $.fn.dataTableExt.oStdClasses, {
+ "sWrapper": "dataTables_wrapper form-inline"
+} );
+
+
+/* API method to get paging information */
+$.fn.dataTableExt.oApi.fnPagingInfo = function ( oSettings )
+{
+ return {
+ "iStart": oSettings._iDisplayStart,
+ "iEnd": oSettings.fnDisplayEnd(),
+ "iLength": oSettings._iDisplayLength,
+ "iTotal": oSettings.fnRecordsTotal(),
+ "iFilteredTotal": oSettings.fnRecordsDisplay(),
+ "iPage": oSettings._iDisplayLength === -1 ?
+ 0 : Math.ceil( oSettings._iDisplayStart / oSettings._iDisplayLength ),
+ "iTotalPages": oSettings._iDisplayLength === -1 ?
+ 0 : Math.ceil( oSettings.fnRecordsDisplay() / oSettings._iDisplayLength )
+ };
+};
+
+
+/* Bootstrap style pagination control */
+$.extend( $.fn.dataTableExt.oPagination, {
+ "bootstrap": {
+ "fnInit": function( oSettings, nPaging, fnDraw ) {
+ var oLang = oSettings.oLanguage.oPaginate;
+ var fnClickHandler = function ( e ) {
+ e.preventDefault();
+ if ( oSettings.oApi._fnPageChange(oSettings, e.data.action) ) {
+ fnDraw( oSettings );
+ }
+ };
+
+ $(nPaging).addClass('pagination').append(
+ '<ul>'+
+ '<li class="prev disabled"><a href="#">← '+oLang.sPrevious+'</a></li>'+
+ '<li class="next disabled"><a href="#">'+oLang.sNext+' → </a></li>'+
+ '</ul>'
+ );
+ var els = $('a', nPaging);
+ $(els[0]).bind( 'click.DT', { action: "previous" }, fnClickHandler );
+ $(els[1]).bind( 'click.DT', { action: "next" }, fnClickHandler );
+ },
+
+ "fnUpdate": function ( oSettings, fnDraw ) {
+ var iListLength = 5;
+ var oPaging = oSettings.oInstance.fnPagingInfo();
+ var an = oSettings.aanFeatures.p;
+ var i, ien, j, sClass, iStart, iEnd, iHalf=Math.floor(iListLength/2);
+
+ if ( oPaging.iTotalPages < iListLength) {
+ iStart = 1;
+ iEnd = oPaging.iTotalPages;
+ }
+ else if ( oPaging.iPage <= iHalf ) {
+ iStart = 1;
+ iEnd = iListLength;
+ } else if ( oPaging.iPage >= (oPaging.iTotalPages-iHalf) ) {
+ iStart = oPaging.iTotalPages - iListLength + 1;
+ iEnd = oPaging.iTotalPages;
+ } else {
+ iStart = oPaging.iPage - iHalf + 1;
+ iEnd = iStart + iListLength - 1;
+ }
+
+ for ( i=0, ien=an.length ; i<ien ; i++ ) {
+ // Remove the middle elements
+ $('li:gt(0)', an[i]).filter(':not(:last)').remove();
+
+ // Add the new list items and their event handlers
+ for ( j=iStart ; j<=iEnd ; j++ ) {
+ sClass = (j==oPaging.iPage+1) ? 'class="active"' : '';
+ $('<li '+sClass+'><a href="#">'+j+'</a></li>')
+ .insertBefore( $('li:last', an[i])[0] )
+ .bind('click', function (e) {
+ e.preventDefault();
+ oSettings._iDisplayStart = (parseInt($('a', this).text(),10)-1) * oPaging.iLength;
+ fnDraw( oSettings );
+ } );
+ }
+
+ // Add / remove disabled classes from the static elements
+ if ( oPaging.iPage === 0 ) {
+ $('li:first', an[i]).addClass('disabled');
+ } else {
+ $('li:first', an[i]).removeClass('disabled');
+ }
+
+ if ( oPaging.iPage === oPaging.iTotalPages-1 || oPaging.iTotalPages === 0 ) {
+ $('li:last', an[i]).addClass('disabled');
+ } else {
+ $('li:last', an[i]).removeClass('disabled');
+ }
+ }
+ }
+ }
+} );
+
+
+/*
+ * TableTools Bootstrap compatibility
+ * Required TableTools 2.1+
+ */
+if ( $.fn.DataTable.TableTools ) {
+ // Set the classes that TableTools uses to something suitable for Bootstrap
+ $.extend( true, $.fn.DataTable.TableTools.classes, {
+ "container": "DTTT btn-group",
+ "buttons": {
+ "normal": "btn",
+ "disabled": "disabled"
+ },
+ "collection": {
+ "container": "DTTT_dropdown dropdown-menu",
+ "buttons": {
+ "normal": "",
+ "disabled": "disabled"
+ }
+ },
+ "print": {
+ "info": "DTTT_print_info modal"
+ },
+ "select": {
+ "row": "active"
+ }
+ } );
+
+ // Have the collection use a bootstrap compatible dropdown
+ $.extend( true, $.fn.DataTable.TableTools.DEFAULTS.oTags, {
+ "collection": {
+ "container": "ul",
+ "button": "li",
+ "liner": "a"
+ }
+ } );
+};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js
new file mode 100644
index 0000000..885ad34
--- /dev/null
+++ b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js
@@ -0,0 +1,67 @@
+/**
+ * DataTables custom plugin to extract numbers from HTML tags
+ * for sorting rows in numeric order.
+ *
+ * Ref: http://www.datatables.net/plug-ins/sorting
+ *
+ * Compatible with DataTables v1.9.4
+ *
+ */
+
+jQuery.extend(jQuery.fn.dataTableExt.oSort, {
+ "num-html-pre": function (a) {
+ // Extract the numeric value from the html by removing all tags.
+ var x = String(a).replace(/<[\s\S]*?>/g, "");
+ return parseFloat(x);
+ },
+
+ "num-html-asc": function (a, b) {
+ // Compare the extracted numeric values for asc sort.
+ return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+ },
+
+ "num-html-desc": function (a, b) {
+ // Compare the extracted numeric values for desc sort.
+ return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+ }
+});
+
+/*
+ * Type detection function for custom type "num-html".
+ *
+ * Ref: http://www.datatables.net/plug-ins/type-detection#how_to
+ */
+jQuery.fn.dataTableExt.aTypes.unshift(function (sData) {
+ sData = typeof sData.replace == "function" ?
+ sData.replace( /<[\s\S]*?>/g, "" ) : sData;
+ sData = $.trim(sData);
+
+ var sValidFirstChars = "0123456789-";
+ var sValidChars = "0123456789.";
+ var Char;
+ var bDecimal = false;
+
+ // Check for a valid first char (no period and allow negatives).
+ Char = sData.charAt(0);
+ if (sValidFirstChars.indexOf(Char) == -1) {
+ return null;
+ }
+
+ // Check all the other characters are valid.
+ for ( var i=1 ; i<sData.length ; i++ ) {
+ Char = sData.charAt(i);
+ if (sValidChars.indexOf(Char) == -1) {
+ return null;
+ }
+
+ // Only one decimal place allowed.
+ if (Char == ".") {
+ if (bDecimal) {
+ return null;
+ }
+ bDecimal = true;
+ }
+ }
+
+ return "num-html";
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js
new file mode 100644
index 0000000..3c0b39a
--- /dev/null
+++ b/src/main/resources/org/apache/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js
@@ -0,0 +1,18 @@
+/**
+ * Make browser's local storage as the default storage for Datatables plugin.
+ * Compatible with Datatables v1.9.4
+ *
+ * http://www.datatables.net/beta/1.9/examples/advanced_init/localstorage.html
+ *
+ * Always link this file after linking jquery.dataTables.min.js
+ */
+$.extend(true, $.fn.dataTable.defaults, {
+ "fnStateSave": function (oSettings, oData) {
+ localStorage.setItem("DataTables_" + window.location.pathname + "_" + oSettings.sTableId,
+ JSON.stringify(oData));
+ },
+ "fnStateLoad": function (oSettings) {
+ return JSON.parse(localStorage
+ .getItem("DataTables_" + window.location.pathname + "_" + oSettings.sTableId));
+ }
+});
\ No newline at end of file
[39/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/aop/AopModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/AopModule.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/AopModule.java
deleted file mode 100644
index 4afc263..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/AopModule.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-import java.util.List;
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.inject.AbstractModule;
-import com.google.inject.Binder;
-import com.google.inject.Key;
-import com.google.inject.PrivateModule;
-import com.google.inject.TypeLiteral;
-import com.google.inject.matcher.Matcher;
-import com.google.inject.matcher.Matchers;
-
-import org.aopalliance.intercept.MethodInterceptor;
-
-import com.twitter.aurora.GuiceUtils;
-import com.twitter.aurora.auth.CapabilityValidator;
-import com.twitter.aurora.gen.AuroraAdmin;
-import com.twitter.aurora.gen.AuroraSchedulerManager;
-import com.twitter.aurora.scheduler.thrift.auth.DecoratedThrift;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-
-/**
- * Binding module for AOP-style decorations of the thrift API.
- */
-public class AopModule extends AbstractModule {
-
- @CmdLine(name = "enable_job_updates", help = "Whether new job updates should be accepted.")
- private static final Arg<Boolean> ENABLE_UPDATES = Arg.create(true);
-
- @CmdLine(name = "enable_job_creation",
- help = "Allow new jobs to be created, if false all job creation requests will be denied.")
- private static final Arg<Boolean> ENABLE_JOB_CREATION = Arg.create(true);
-
- private static final Matcher<? super Class<?>> THRIFT_IFACE_MATCHER =
- Matchers.subclassesOf(AuroraAdmin.Iface.class)
- .and(Matchers.annotatedWith(DecoratedThrift.class));
-
- private final Map<String, Boolean> toggledMethods;
-
- public AopModule() {
- this(ImmutableMap.of(
- "createJob", ENABLE_JOB_CREATION.get(),
- "acquireLock", ENABLE_UPDATES.get()));
- }
-
- @VisibleForTesting
- AopModule(Map<String, Boolean> toggledMethods) {
- this.toggledMethods = ImmutableMap.copyOf(toggledMethods);
- }
-
- private static final Function<Method, String> GET_NAME = new Function<Method, String>() {
- @Override public String apply(Method method) {
- return method.getName();
- }
- };
-
- @Override
- protected void configure() {
- requireBinding(CapabilityValidator.class);
-
- // Layer ordering:
- // Log -> CapabilityValidator -> FeatureToggle -> StatsExporter -> APIVersion ->
- // SchedulerThriftInterface
-
- // TODO(Sathya): Consider using provider pattern for constructing interceptors to facilitate
- // unit testing without the creation of Guice injectors.
- bindThriftDecorator(new LoggingInterceptor());
-
- // Note: it's important that the capability interceptor is only applied to AuroraAdmin.Iface
- // methods, and does not pick up methods on AuroraSchedulerManager.Iface.
- MethodInterceptor authInterceptor = new UserCapabilityInterceptor();
- requestInjection(authInterceptor);
- bindInterceptor(
- THRIFT_IFACE_MATCHER,
- GuiceUtils.interfaceMatcher(AuroraAdmin.Iface.class, true),
- authInterceptor);
-
- install(new PrivateModule() {
- @Override protected void configure() {
- // Ensure that the provided methods exist on the decorated interface.
- List<Method> methods =
- ImmutableList.copyOf(AuroraSchedulerManager.Iface.class.getMethods());
- for (String toggledMethod : toggledMethods.keySet()) {
- Preconditions.checkArgument(
- Iterables.any(methods,
- Predicates.compose(Predicates.equalTo(toggledMethod), GET_NAME)),
- String.format("Method %s was not found in class %s",
- toggledMethod,
- AuroraSchedulerManager.Iface.class));
- }
-
- bind(new TypeLiteral<Map<String, Boolean>>() { }).toInstance(toggledMethods);
- bind(IsFeatureEnabled.class).in(Singleton.class);
- Key<Predicate<Method>> predicateKey = Key.get(new TypeLiteral<Predicate<Method>>() { });
- bind(predicateKey).to(IsFeatureEnabled.class);
- expose(predicateKey);
- }
- });
- bindThriftDecorator(new FeatureToggleInterceptor());
- bindThriftDecorator(new ThriftStatsExporterInterceptor());
- bindThriftDecorator(new APIVersionInterceptor());
- }
-
- private void bindThriftDecorator(MethodInterceptor interceptor) {
- bindThriftDecorator(binder(), THRIFT_IFACE_MATCHER, interceptor);
- }
-
- @VisibleForTesting
- static void bindThriftDecorator(
- Binder binder,
- Matcher<? super Class<?>> classMatcher,
- MethodInterceptor interceptor) {
-
- binder.bindInterceptor(classMatcher, Matchers.any(), interceptor);
- binder.requestInjection(interceptor);
- }
-
- private static class IsFeatureEnabled implements Predicate<Method> {
- private final Predicate<String> methodEnabled;
-
- @Inject
- IsFeatureEnabled(Map<String, Boolean> toggleMethods) {
- Predicate<String> builder = Predicates.alwaysTrue();
- for (Map.Entry<String, Boolean> toggleMethod : toggleMethods.entrySet()) {
- Predicate<String> enableMethod = Predicates.or(
- toggleMethod.getValue()
- ? Predicates.<String>alwaysTrue()
- : Predicates.<String>alwaysFalse(),
- Predicates.not(Predicates.equalTo(toggleMethod.getKey())));
- builder = Predicates.and(builder, enableMethod);
- }
- methodEnabled = builder;
- }
-
- @Override
- public boolean apply(Method method) {
- return methodEnabled.apply(method.getName());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
deleted file mode 100644
index 03c3d99..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Predicate;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-
-import com.twitter.aurora.gen.ResponseCode;
-
-/**
- * A method interceptor that blocks access to features based on a supplied predicate.
- */
-public class FeatureToggleInterceptor implements MethodInterceptor {
-
- @Inject private Predicate<Method> allowMethod;
-
- @Override
- public Object invoke(MethodInvocation invocation) throws Throwable {
- Method method = invocation.getMethod();
- if (!allowMethod.apply(method)) {
- return Interceptors.properlyTypedResponse(
- method,
- ResponseCode.ERROR,
- "The " + method.getName() + " feature is currently disabled on this scheduler.");
- } else {
- return invocation.proceed();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/aop/Interceptors.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/Interceptors.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/Interceptors.java
deleted file mode 100644
index d0cb9c1..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/Interceptors.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.logging.Logger;
-
-import com.google.common.base.Throwables;
-
-import com.twitter.aurora.gen.ResponseCode;
-
-/**
- * Utility class for functions useful when implementing an interceptor on the thrift interface.
- */
-final class Interceptors {
-
- private Interceptors() {
- // Utility class.
- }
-
- private static final Logger LOG = Logger.getLogger(Interceptors.class.getName());
-
- static Object properlyTypedResponse(Method method, ResponseCode responseCode, String message)
- throws IllegalAccessException, InstantiationException {
-
- Class<?> returnType = method.getReturnType();
- Object response = returnType.newInstance();
- invoke(returnType, response, "setResponseCode", ResponseCode.class, responseCode);
- invoke(returnType, response, "setMessage", String.class, message);
- return response;
- }
-
- private static <T> void invoke(
- Class<?> type,
- Object obj,
- String name,
- Class<T> parameterType,
- T argument) {
-
- Method method;
- try {
- method = type.getMethod(name, parameterType);
- } catch (NoSuchMethodException e) {
- LOG.severe(type + " does not support " + name);
- throw Throwables.propagate(e);
- }
- try {
- method.invoke(obj, argument);
- } catch (IllegalAccessException e) {
- LOG.severe("Method " + name + " is not accessible in " + type);
- throw Throwables.propagate(e);
- } catch (InvocationTargetException e) {
- LOG.severe("Failed to invoke " + name + " on " + type);
- throw Throwables.propagate(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/aop/LoggingInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/LoggingInterceptor.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/LoggingInterceptor.java
deleted file mode 100644
index 5f773cc..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/LoggingInterceptor.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.aop;
-
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-
-import com.twitter.aurora.auth.CapabilityValidator;
-import com.twitter.aurora.gen.ExecutorConfig;
-import com.twitter.aurora.gen.JobConfiguration;
-import com.twitter.aurora.gen.ResponseCode;
-import com.twitter.aurora.gen.SessionKey;
-
-import static com.twitter.aurora.scheduler.thrift.aop.Interceptors.properlyTypedResponse;
-
-/**
- * A method interceptor that logs all invocations as well as any unchecked exceptions thrown from
- * the underlying call.
- */
-class LoggingInterceptor implements MethodInterceptor {
-
- private static final Logger LOG = Logger.getLogger(LoggingInterceptor.class.getName());
-
- @Inject private CapabilityValidator validator;
-
- // TODO(wfarner): Scrub updateToken when it is identifiable by type.
- private final Map<Class<?>, Function<Object, String>> printFunctions =
- ImmutableMap.<Class<?>, Function<Object, String>>of(
- JobConfiguration.class,
- new Function<Object, String>() {
- @Override public String apply(Object input) {
- JobConfiguration configuration = ((JobConfiguration) input).deepCopy();
- if (configuration.isSetTaskConfig()) {
- configuration.getTaskConfig().setExecutorConfig(
- new ExecutorConfig("BLANKED", "BLANKED"));
- }
- return configuration.toString();
- }
- },
- SessionKey.class,
- new Function<Object, String>() {
- @Override public String apply(Object input) {
- SessionKey key = (SessionKey) input;
- return validator.toString(key);
- }
- }
- );
-
- @Override
- public Object invoke(MethodInvocation invocation) throws Throwable {
- List<String> argStrings = Lists.newArrayList();
- for (Object arg : invocation.getArguments()) {
- if (arg == null) {
- argStrings.add("null");
- } else {
- Function<Object, String> printFunction = printFunctions.get(arg.getClass());
- argStrings.add((printFunction == null) ? arg.toString() : printFunction.apply(arg));
- }
- }
- String methodName = invocation.getMethod().getName();
- String message = String.format("%s(%s)", methodName, Joiner.on(", ").join(argStrings));
- LOG.info(message);
- try {
- return invocation.proceed();
- } catch (RuntimeException e) {
- LOG.log(Level.WARNING, "Uncaught exception while handling " + message, e);
- return properlyTypedResponse(invocation.getMethod(), ResponseCode.ERROR, e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
deleted file mode 100644
index d700ab5..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-
-import com.twitter.common.stats.SlidingStats;
-import com.twitter.common.stats.Stats;
-
-/**
- * A method interceptor that exports counterStats about thrift calls.
- */
-class ThriftStatsExporterInterceptor implements MethodInterceptor {
-
- private final LoadingCache<Method, SlidingStats> stats =
- CacheBuilder.newBuilder().build(new CacheLoader<Method, SlidingStats>() {
- @Override public SlidingStats load(Method method) {
- return new SlidingStats(
- Stats.normalizeName(String.format("scheduler_thrift_%s", method.getName())),
- "nanos");
- }
- });
-
- @Override
- public Object invoke(MethodInvocation invocation) throws Throwable {
- SlidingStats stat = stats.get(invocation.getMethod());
- long start = System.nanoTime();
- try {
- return invocation.proceed();
- } finally {
- stat.accumulate(System.nanoTime() - start);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java b/src/main/java/com/twitter/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java
deleted file mode 100644
index d9240bc..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-import java.util.Arrays;
-import java.util.List;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-
-import com.twitter.aurora.auth.CapabilityValidator;
-import com.twitter.aurora.auth.CapabilityValidator.AuditCheck;
-import com.twitter.aurora.auth.CapabilityValidator.Capability;
-import com.twitter.aurora.auth.SessionValidator.AuthFailedException;
-import com.twitter.aurora.gen.ResponseCode;
-import com.twitter.aurora.gen.SessionKey;
-import com.twitter.aurora.scheduler.thrift.auth.Requires;
-
-/**
- * A method interceptor that will authenticate users identified by a {@link SessionKey} argument
- * to invoked methods.
- * <p>
- * Intercepted methods will require {@link Capability#ROOT}, but additional capabilities
- * may be specified by annotating methods with {@link Requires} and supplying a whitelist.
- */
-class UserCapabilityInterceptor implements MethodInterceptor {
- private static final Logger LOG = Logger.getLogger(UserCapabilityInterceptor.class.getName());
-
- @Inject private CapabilityValidator capabilityValidator;
-
- private static final Function<Object, SessionKey> CAST = new Function<Object, SessionKey>() {
- @Override public SessionKey apply(Object o) {
- return (SessionKey) o;
- }
- };
-
- @Override
- public Object invoke(MethodInvocation invocation) throws Throwable {
- Preconditions.checkNotNull(capabilityValidator, "Session validator has not yet been set.");
-
- // Ensure ROOT is always permitted.
- ImmutableList.Builder<Capability> whitelistBuilder =
- ImmutableList.<Capability>builder().add(Capability.ROOT);
-
- Method method = invocation.getMethod();
- Requires requires = method.getAnnotation(Requires.class);
- if (requires != null) {
- whitelistBuilder.add(requires.whitelist());
- }
-
- List<Capability> whitelist = whitelistBuilder.build();
- LOG.fine("Operation " + method.getName() + " may be performed by: " + whitelist);
- Optional<SessionKey> sessionKey = FluentIterable.from(Arrays.asList(invocation.getArguments()))
- .firstMatch(Predicates.instanceOf(SessionKey.class)).transform(CAST);
- if (!sessionKey.isPresent()) {
- LOG.severe("Interceptor should only be applied to methods accepting a SessionKey, but "
- + method + " does not.");
- return invocation.proceed();
- }
-
- String key = capabilityValidator.toString(sessionKey.get());
- for (Capability user : whitelist) {
- LOG.fine("Attempting to validate " + key + " against " + user);
- try {
- capabilityValidator.checkAuthorized(sessionKey.get(), user, AuditCheck.NONE);
-
- LOG.info("Permitting " + key + " to act as "
- + user + " and perform action " + method.getName());
- return invocation.proceed();
- } catch (AuthFailedException e) {
- LOG.fine("Auth failed: " + e);
- }
- }
-
- // User is not permitted to perform this operation.
- return Interceptors.properlyTypedResponse(
- method,
- ResponseCode.AUTH_FAILED,
- "Session identified by '" + key
- + "' does not have the required capability to perform this action: " + whitelist);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/auth/DecoratedThrift.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/auth/DecoratedThrift.java b/src/main/java/com/twitter/aurora/scheduler/thrift/auth/DecoratedThrift.java
deleted file mode 100644
index 4a667c5..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/auth/DecoratedThrift.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.auth;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.ElementType.TYPE;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Type annotation to apply to a thrift interface implementation that should be decorated with
- * additional functionality.
- */
-@Target({PARAMETER, TYPE}) @Retention(RUNTIME)
-public @interface DecoratedThrift {
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/auth/Requires.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/auth/Requires.java b/src/main/java/com/twitter/aurora/scheduler/thrift/auth/Requires.java
deleted file mode 100644
index 0fff3f6..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/auth/Requires.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.auth;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-
-import com.twitter.aurora.auth.CapabilityValidator.Capability;
-
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Annotation applied to a method that may allow users with non-ROOT capabilities to perform
- * an action.
- */
-@Target(METHOD) @Retention(RUNTIME)
-public @interface Requires {
- /**
- * The list of capabilities required to perform an action.
- */
- Capability[] whitelist() default { Capability.ROOT };
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/thrift/auth/ThriftAuthModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/thrift/auth/ThriftAuthModule.java b/src/main/java/com/twitter/aurora/scheduler/thrift/auth/ThriftAuthModule.java
deleted file mode 100644
index 66f9033..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/thrift/auth/ThriftAuthModule.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.thrift.auth;
-
-import java.util.Map;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.AbstractModule;
-import com.google.inject.TypeLiteral;
-
-import com.twitter.aurora.auth.CapabilityValidator;
-import com.twitter.aurora.auth.CapabilityValidator.Capability;
-import com.twitter.aurora.auth.SessionValidator;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.args.constraints.NotEmpty;
-
-/**
- * Binding module for authentication of users with special capabilities for admin functions.
- */
-public class ThriftAuthModule extends AbstractModule {
-
- private static final Map<Capability, String> DEFAULT_CAPABILITIES =
- ImmutableMap.of(Capability.ROOT, "mesos");
-
- @NotEmpty
- @CmdLine(name = "user_capabilities",
- help = "Concrete name mappings for administration capabilities.")
- private static final Arg<Map<Capability, String>> USER_CAPABILITIES =
- Arg.create(DEFAULT_CAPABILITIES);
-
- private Map<Capability, String> capabilities;
-
- public ThriftAuthModule() {
- this(USER_CAPABILITIES.get());
- }
-
- @VisibleForTesting
- public ThriftAuthModule(Map<Capability, String> capabilities) {
- this.capabilities = Preconditions.checkNotNull(capabilities);
- }
-
- @Override
- protected void configure() {
- Preconditions.checkArgument(
- capabilities.containsKey(Capability.ROOT),
- "A ROOT capability must be provided with --user_capabilities");
- bind(new TypeLiteral<Map<Capability, String>>() { }).toInstance(capabilities);
-
- requireBinding(SessionValidator.class);
- requireBinding(CapabilityValidator.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/GuiceUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/GuiceUtils.java b/src/main/java/org/apache/aurora/GuiceUtils.java
new file mode 100644
index 0000000..207535d
--- /dev/null
+++ b/src/main/java/org/apache/aurora/GuiceUtils.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.lang.reflect.Method;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.inject.Binder;
+import com.google.inject.BindingAnnotation;
+import com.google.inject.matcher.AbstractMatcher;
+import com.google.inject.matcher.Matcher;
+import com.google.inject.matcher.Matchers;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+
+import com.twitter.common.collections.Pair;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Utilities for guice configuration in aurora.
+ */
+public final class GuiceUtils {
+
+ private static final Logger LOG = Logger.getLogger(GuiceUtils.class.getName());
+
+ // Method annotation that allows a trapped interface to whitelist methods that may throw
+ // unchecked exceptions.
+ @BindingAnnotation
+ @Target(METHOD) @Retention(RUNTIME)
+ public @interface AllowUnchecked { }
+
+ private GuiceUtils() {
+ // utility
+ }
+
+ // No wildcards on the Class here because it upsets checkstyle - complains with:
+ // '>' is followed by whitespace.
+ private static final Function<Method, Pair<String, Class[]>> CANONICALIZE =
+ new Function<Method, Pair<String, Class[]>>() {
+ @Override public Pair<String, Class[]> apply(Method method) {
+ return Pair.of(method.getName(), (Class[]) method.getParameterTypes());
+ }
+ };
+
+ /**
+ * Creates a matcher that will match methods of an interface, optionally excluding inherited
+ * methods.
+ *
+ * @param matchInterface The interface to match.
+ * @param declaredMethodsOnly if {@code true} only methods directly declared in the interface
+ * will be matched, otherwise all methods on the interface are matched.
+ * @return A new matcher instance.
+ */
+ public static Matcher<Method> interfaceMatcher(
+ Class<?> matchInterface,
+ boolean declaredMethodsOnly) {
+
+ Method[] methods =
+ declaredMethodsOnly ? matchInterface.getDeclaredMethods() : matchInterface.getMethods();
+ final Set<Pair<String, Class[]>> interfaceMethods =
+ ImmutableSet.copyOf(Iterables.transform(ImmutableList.copyOf(methods), CANONICALIZE));
+ final LoadingCache<Method, Pair<String, Class[]>> cache = CacheBuilder.newBuilder()
+ .build(CacheLoader.from(CANONICALIZE));
+
+ return new AbstractMatcher<Method>() {
+ @Override public boolean matches(Method method) {
+ return interfaceMethods.contains(cache.getUnchecked(method));
+ }
+ };
+ }
+
+ /**
+ * Binds an interceptor that ensures the main ClassLoader is bound as the thread context
+ * {@link ClassLoader} during JNI callbacks from mesos. Some libraries require a thread
+ * context ClassLoader be set and this ensures those libraries work properly.
+ *
+ * @param binder The binder to use to register an interceptor with.
+ * @param wrapInterface Interface whose methods should wrapped.
+ */
+ public static void bindJNIContextClassLoader(Binder binder, Class<?> wrapInterface) {
+ final ClassLoader mainClassLoader = GuiceUtils.class.getClassLoader();
+ binder.bindInterceptor(
+ Matchers.subclassesOf(wrapInterface),
+ interfaceMatcher(wrapInterface, false),
+ new MethodInterceptor() {
+ @Override public Object invoke(MethodInvocation invocation) throws Throwable {
+ Thread currentThread = Thread.currentThread();
+ ClassLoader prior = currentThread.getContextClassLoader();
+ try {
+ currentThread.setContextClassLoader(mainClassLoader);
+ return invocation.proceed();
+ } finally {
+ currentThread.setContextClassLoader(prior);
+ }
+ }
+ });
+ }
+
+ private static final Predicate<Method> IS_WHITELISTED = new Predicate<Method>() {
+ @Override public boolean apply(Method method) {
+ return method.getAnnotation(AllowUnchecked.class) != null;
+ }
+ };
+
+ private static final Matcher<Method> WHITELIST_MATCHER = new AbstractMatcher<Method>() {
+ @Override public boolean matches(Method method) {
+ return IS_WHITELISTED.apply(method);
+ }
+ };
+
+ private static final Predicate<Method> VOID_METHOD = new Predicate<Method>() {
+ @Override public boolean apply(Method method) {
+ return method.getReturnType() == Void.TYPE;
+ }
+ };
+
+ /**
+ * Binds an exception trap on all interface methods of all classes bound against an interface.
+ * Individual methods may opt out of trapping by annotating with {@link AllowUnchecked}.
+ * Only void methods are allowed, any non-void interface methods must explicitly opt out.
+ *
+ * @param binder The binder to register an interceptor with.
+ * @param wrapInterface Interface whose methods should be wrapped.
+ * @throws IllegalArgumentException If any of the non-whitelisted interface methods are non-void.
+ */
+ public static void bindExceptionTrap(Binder binder, Class<?> wrapInterface)
+ throws IllegalArgumentException {
+
+ Set<Method> disallowed = ImmutableSet.copyOf(Iterables.filter(
+ ImmutableList.copyOf(wrapInterface.getMethods()),
+ Predicates.and(Predicates.not(IS_WHITELISTED), Predicates.not(VOID_METHOD))));
+ Preconditions.checkArgument(disallowed.isEmpty(),
+ "Non-void methods must be explicitly whitelisted with @AllowUnchecked: " + disallowed);
+
+ Matcher<Method> matcher =
+ Matchers.<Method>not(WHITELIST_MATCHER).and(interfaceMatcher(wrapInterface, false));
+ binder.bindInterceptor(Matchers.subclassesOf(wrapInterface), matcher,
+ new MethodInterceptor() {
+ @Override public Object invoke(MethodInvocation invocation) throws Throwable {
+ try {
+ return invocation.proceed();
+ } catch (RuntimeException e) {
+ LOG.log(Level.WARNING, "Trapped uncaught exception: " + e, e);
+ return null;
+ }
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/Protobufs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/Protobufs.java b/src/main/java/org/apache/aurora/Protobufs.java
new file mode 100644
index 0000000..c3ddea4
--- /dev/null
+++ b/src/main/java/org/apache/aurora/Protobufs.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora;
+
+import com.google.common.base.Function;
+import com.google.protobuf.Message;
+import com.google.protobuf.TextFormat;
+
+/**
+ * Utility functions that are useful for working with protocol buffer messages.
+ */
+public final class Protobufs {
+
+ private Protobufs() {
+ // Utility class.
+ }
+
+ /**
+ * Function to call {@link #toString(Message)} on message objects.
+ */
+ public static final Function<Message, String> SHORT_TOSTRING = new Function<Message, String>() {
+ @Override public String apply(Message message) {
+ return Protobufs.toString(message);
+ }
+ };
+
+ /**
+ * Alternative to the default protobuf toString implementation, which omits newlines.
+ *
+ * @param message Message to print.
+ * @return String representation of the message.
+ */
+ public static String toString(Message message) {
+ return TextFormat.shortDebugString(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/auth/CapabilityValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/auth/CapabilityValidator.java b/src/main/java/org/apache/aurora/auth/CapabilityValidator.java
new file mode 100644
index 0000000..05958f0
--- /dev/null
+++ b/src/main/java/org/apache/aurora/auth/CapabilityValidator.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.auth;
+
+import com.twitter.aurora.gen.SessionKey;
+
+/**
+ * A session validator that supports user capability matching.
+ * <p>
+ * This supports asking whether a user has been granted a specific administration capability.
+ */
+public interface CapabilityValidator extends SessionValidator {
+
+ enum Capability {
+ ROOT,
+ PROVISIONER
+ }
+
+ /**
+ * Audit check to perform for a given action.
+ */
+ enum AuditCheck {
+ /**
+ * Default. No audit checks will be performed.
+ */
+ NONE,
+
+ /**
+ * A check will be performed to verify if a given action has
+ * all necessary data to generate a valid audit trail.
+ */
+ REQUIRED
+ }
+
+ /**
+ * Checks whether a session key is authenticated, and has the specified capability.
+ *
+ * @param sessionKey Key to validate.
+ * @param capability User capability to authenticate against.
+ * @param check Auditing data presence check required.
+ * @return A {@link SessionContext} object that provides information about the validated session.
+ * @throws AuthFailedException If the key cannot be validated as the role or lacks
+ * the requested capability.
+ */
+ SessionContext checkAuthorized(SessionKey sessionKey, Capability capability, AuditCheck check)
+ throws AuthFailedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/auth/SessionValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/auth/SessionValidator.java b/src/main/java/org/apache/aurora/auth/SessionValidator.java
new file mode 100644
index 0000000..327c5ca
--- /dev/null
+++ b/src/main/java/org/apache/aurora/auth/SessionValidator.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.auth;
+
+import java.util.Set;
+
+import com.twitter.aurora.gen.SessionKey;
+
+/**
+ * Validator for RPC sessions with Aurora.
+ */
+public interface SessionValidator {
+
+ /**
+ * Checks whether a session key is authenticated, and has permission to act as all the roles
+ * provided. Authentication is successful only if the SessionKey is successfully validated against
+ * all the roles.
+ *
+ * @param sessionKey Key to validate.
+ * @param targetRoles A set of roles to validate the key against.
+ * @return A {@link SessionContext} object that provides information about the validated session.
+ * @throws AuthFailedException If the key cannot be validated against a role.
+ */
+ SessionContext checkAuthenticated(SessionKey sessionKey, Set<String> targetRoles)
+ throws AuthFailedException;
+
+ /**
+ * Translates a {@link SessionKey} to a string. Primarily provides a way for the binary data
+ * within a {@link SessionKey} to be decoded and converted into a string.
+ *
+ * @param sessionKey The session key to translate.
+ * @return A string representation of the {@link SessionKey}.
+ */
+ String toString(SessionKey sessionKey);
+
+ /**
+ * Provides information about a session.
+ */
+ interface SessionContext {
+
+ /**
+ * Provides the identity for a validated session.
+ *
+ * @return A string that identifies the session.
+ */
+ String getIdentity();
+ }
+
+ /**
+ * Thrown when authentication is not successful.
+ */
+ public static class AuthFailedException extends Exception {
+ public AuthFailedException(String msg) {
+ super(msg);
+ }
+
+ public AuthFailedException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/auth/UnsecureAuthModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/auth/UnsecureAuthModule.java b/src/main/java/org/apache/aurora/auth/UnsecureAuthModule.java
new file mode 100644
index 0000000..8fe9953
--- /dev/null
+++ b/src/main/java/org/apache/aurora/auth/UnsecureAuthModule.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.auth;
+
+import java.util.Set;
+import java.util.logging.Logger;
+
+import com.google.inject.AbstractModule;
+
+import com.twitter.aurora.gen.SessionKey;
+
+/**
+ * An authentication module that uses an {@link UnsecureSessionValidator}. This behavior
+ * can be overridden by binding a secure validator, querying an internal authentication system,
+ * to {@link SessionValidator}.
+ */
+public class UnsecureAuthModule extends AbstractModule {
+ private static final String UNSECURE = "UNSECURE";
+ private static final Logger LOG = Logger.getLogger(UnsecureAuthModule.class.getName());
+
+ @Override
+ protected void configure() {
+ LOG.info("Using default (UNSECURE!!!) authentication module.");
+ bind(SessionValidator.class).to(UnsecureSessionValidator.class);
+ bind(CapabilityValidator.class).to(UnsecureCapabilityValidator.class);
+ }
+
+ static class UnsecureSessionValidator implements SessionValidator {
+ @Override
+ public SessionContext checkAuthenticated(SessionKey key, Set<String> targetRoles)
+ throws AuthFailedException {
+
+ return new SessionContext() {
+ @Override public String getIdentity() {
+ return UNSECURE;
+ }
+ };
+ }
+
+ @Override
+ public String toString(SessionKey sessionKey) {
+ return sessionKey.toString();
+ }
+ }
+
+ static class UnsecureCapabilityValidator implements CapabilityValidator {
+ @Override
+ public SessionContext checkAuthorized(SessionKey key, Capability capability, AuditCheck check)
+ throws AuthFailedException {
+
+ return new SessionContext() {
+ @Override public String getIdentity() {
+ return UNSECURE;
+ }
+ };
+ }
+
+ @Override
+ public SessionContext checkAuthenticated(SessionKey key, Set<String> targetRoles)
+ throws AuthFailedException {
+
+ return new SessionContext() {
+ @Override public String getIdentity() {
+ return UNSECURE;
+ }
+ };
+ }
+
+ @Override
+ public String toString(SessionKey sessionKey) {
+ return sessionKey.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java b/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java
new file mode 100644
index 0000000..2443078
--- /dev/null
+++ b/src/main/java/org/apache/aurora/codec/ThriftBinaryCodec.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.codec;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.thrift.TBase;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
+
+/**
+ * Codec that works for thrift objects.
+ */
+public final class ThriftBinaryCodec {
+
+ /**
+ * Protocol factory used for all thrift encoding and decoding.
+ */
+ public static final TProtocolFactory PROTOCOL_FACTORY = new TBinaryProtocol.Factory();
+
+ private ThriftBinaryCodec() {
+ // Utility class.
+ }
+
+ /**
+ * Identical to {@link #decodeNonNull(Class, byte[])}, but allows for a null buffer.
+ *
+ * @param clazz Class to instantiate and deserialize to.
+ * @param buffer Buffer to decode.
+ * @param <T> Target type.
+ * @return A populated message, or {@code null} if the buffer was {@code null}.
+ * @throws CodingException If the message could not be decoded.
+ */
+ @Nullable
+ public static <T extends TBase<T, ?>> T decode(Class<T> clazz, @Nullable byte[] buffer)
+ throws CodingException {
+
+ if (buffer == null) {
+ return null;
+ }
+ return decodeNonNull(clazz, buffer);
+ }
+
+ /**
+ * Decodes a binary-encoded byte array into a target type.
+ *
+ * @param clazz Class to instantiate and deserialize to.
+ * @param buffer Buffer to decode.
+ * @param <T> Target type.
+ * @return A populated message.
+ * @throws CodingException If the message could not be decoded.
+ */
+ public static <T extends TBase<T, ?>> T decodeNonNull(Class<T> clazz, byte[] buffer)
+ throws CodingException {
+
+ Preconditions.checkNotNull(clazz);
+ Preconditions.checkNotNull(buffer);
+
+ try {
+ T t = clazz.newInstance();
+ new TDeserializer(PROTOCOL_FACTORY).deserialize(t, buffer);
+ return t;
+ } catch (IllegalAccessException e) {
+ throw new CodingException("Failed to access constructor for target type.", e);
+ } catch (InstantiationException e) {
+ throw new CodingException("Failed to instantiate target type.", e);
+ } catch (TException e) {
+ throw new CodingException("Failed to deserialize thrift object.", e);
+ }
+ }
+
+ /**
+ * Identical to {@link #encodeNonNull(TBase)}, but allows for a null input.
+ *
+ * @param tBase Object to encode.
+ * @return Encoded object, or {@code null} if the argument was {@code null}.
+ * @throws CodingException If the object could not be encoded.
+ */
+ @Nullable
+ public static byte[] encode(@Nullable TBase<?, ?> tBase) throws CodingException {
+ if (tBase == null) {
+ return null;
+ }
+ return encodeNonNull(tBase);
+ }
+
+ /**
+ * Encodes a thrift object into a binary array.
+ *
+ * @param tBase Object to encode.
+ * @return Encoded object.
+ * @throws CodingException If the object could not be encoded.
+ */
+ public static byte[] encodeNonNull(TBase<?, ?> tBase) throws CodingException {
+ Preconditions.checkNotNull(tBase);
+
+ try {
+ return new TSerializer(PROTOCOL_FACTORY).serialize(tBase);
+ } catch (TException e) {
+ throw new CodingException("Failed to serialize: " + tBase, e);
+ }
+ }
+
+ /**
+ * Thrown when serialization or deserialization failed.
+ */
+ public static class CodingException extends Exception {
+ public CodingException(String message) {
+ super(message);
+ }
+ public CodingException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/Driver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/Driver.java b/src/main/java/org/apache/aurora/scheduler/Driver.java
new file mode 100644
index 0000000..aa77887
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/Driver.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.mesos.Protos;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.Status;
+import org.apache.mesos.Protos.TaskInfo;
+import org.apache.mesos.SchedulerDriver;
+
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.StateMachine;
+
+import static org.apache.mesos.Protos.Status.DRIVER_RUNNING;
+
+/**
+ * Wraps the mesos Scheduler driver to ensure its used in a valid lifecycle; namely:
+ * <pre>
+ * (run -> kill*)? -> stop*
+ * </pre>
+ *
+ * Also ensures the driver is only asked for when needed.
+ */
+public interface Driver {
+
+ /**
+ * Launches a task.
+ *
+ * @param offerId ID of the resource offer to accept with the task.
+ * @param task Task to launch.
+ */
+ void launchTask(OfferID offerId, TaskInfo task);
+
+ /**
+ * Declines a resource offer.
+ *
+ * @param offerId ID of the offer to decline.
+ */
+ void declineOffer(OfferID offerId);
+
+ /**
+ * Sends a kill task request for the given {@code taskId} to the mesos master.
+ *
+ * @param taskId The id of the task to kill.
+ */
+ void killTask(String taskId);
+
+ /**
+ * Stops the underlying driver if it is running, otherwise does nothing.
+ */
+ void stop();
+
+ /**
+ * Starts the underlying driver. Can only be called once.
+ *
+ * @return The status of the underlying driver run request.
+ */
+ Protos.Status start();
+
+ /**
+ * Blocks until the underlying driver is stopped or aborted.
+ *
+ * @return The status of the underlying driver upon exit.
+ */
+ Protos.Status join();
+
+ /**
+ * Mesos driver implementation.
+ */
+ static class DriverImpl implements Driver {
+ private static final Logger LOG = Logger.getLogger(Driver.class.getName());
+
+ /**
+ * Driver states.
+ */
+ enum State {
+ INIT,
+ RUNNING,
+ STOPPED
+ }
+
+ private final StateMachine<State> stateMachine;
+ private final Supplier<Optional<SchedulerDriver>> driverSupplier;
+ private final AtomicLong killFailures = Stats.exportLong("scheduler_driver_kill_failures");
+
+ /**
+ * Creates a driver manager that will only ask for the underlying mesos driver when actually
+ * needed.
+ *
+ * @param driverSupplier A factory for the underlying driver.
+ */
+ @Inject
+ DriverImpl(Supplier<Optional<SchedulerDriver>> driverSupplier) {
+ this.driverSupplier = driverSupplier;
+ this.stateMachine =
+ StateMachine.<State>builder("scheduler_driver")
+ .initialState(State.INIT)
+ .addState(State.INIT, State.RUNNING, State.STOPPED)
+ .addState(State.RUNNING, State.STOPPED)
+ .logTransitions()
+ .throwOnBadTransition(true)
+ .build();
+ }
+
+ private synchronized SchedulerDriver get(State expected) {
+ // TODO(William Farner): Formalize the failure case here by throwing a checked exception.
+ stateMachine.checkState(expected);
+ // This will and should fail if the driver is not present.
+ return driverSupplier.get().get();
+ }
+
+ @Override
+ public void launchTask(OfferID offerId, TaskInfo task) {
+ get(State.RUNNING).launchTasks(offerId, ImmutableList.of(task));
+ }
+
+ @Override
+ public void declineOffer(OfferID offerId) {
+ get(State.RUNNING).declineOffer(offerId);
+ }
+
+ @Override
+ public Protos.Status start() {
+ SchedulerDriver driver = get(State.INIT);
+ stateMachine.transition(State.RUNNING);
+ return driver.start();
+ }
+
+ @Override
+ public Status join() {
+ return get(State.RUNNING).join();
+ }
+
+ @Override
+ public synchronized void stop() {
+ if (stateMachine.getState() == State.RUNNING) {
+ SchedulerDriver driver = get(State.RUNNING);
+ driver.stop(true /* failover */);
+ stateMachine.transition(State.STOPPED);
+ }
+ }
+
+ @Override
+ public void killTask(String taskId) {
+ SchedulerDriver driver = get(State.RUNNING);
+ Protos.Status status = driver.killTask(Protos.TaskID.newBuilder().setValue(taskId).build());
+
+ if (status != DRIVER_RUNNING) {
+ LOG.severe(String.format("Attempt to kill task %s failed with code %s",
+ taskId, status));
+ killFailures.incrementAndGet();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/DriverFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/DriverFactory.java b/src/main/java/org/apache/aurora/scheduler/DriverFactory.java
new file mode 100644
index 0000000..e39bb09
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/DriverFactory.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Provider;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.protobuf.ByteString;
+
+import org.apache.mesos.MesosSchedulerDriver;
+import org.apache.mesos.Protos.Credential;
+import org.apache.mesos.Protos.FrameworkID;
+import org.apache.mesos.Protos.FrameworkInfo;
+import org.apache.mesos.Scheduler;
+import org.apache.mesos.SchedulerDriver;
+
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.args.constraints.NotNull;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+
+/**
+ * Factory to create scheduler driver instances.
+ */
+public interface DriverFactory extends Function<String, SchedulerDriver> {
+
+ static class DriverFactoryImpl implements DriverFactory {
+ private static final Logger LOG = Logger.getLogger(DriverFactory.class.getName());
+
+ @NotNull
+ @CmdLine(name = "mesos_master_address",
+ help = "Address for the mesos master, can be a socket address or zookeeper path.")
+ private static final Arg<String> MESOS_MASTER_ADDRESS = Arg.create();
+
+ @VisibleForTesting
+ static final String PRINCIPAL_KEY = "aurora_authentication_principal";
+ @VisibleForTesting
+ static final String SECRET_KEY = "aurora_authentication_secret";
+ @CmdLine(name = "framework_authentication_file",
+ help = "Properties file which contains framework credentials to authenticate with Mesos"
+ + "master. Must contain the properties '" + PRINCIPAL_KEY + "' and "
+ + "'" + SECRET_KEY + "'.")
+ private static final Arg<File> FRAMEWORK_AUTHENTICATION_FILE = Arg.create();
+
+ @CmdLine(name = "framework_failover_timeout",
+ help = "Time after which a framework is considered deleted. SHOULD BE VERY HIGH.")
+ private static final Arg<Amount<Long, Time>> FRAMEWORK_FAILOVER_TIMEOUT =
+ Arg.create(Amount.of(21L, Time.DAYS));
+
+ /**
+ * Require Mesos slaves to have checkpointing enabled. Slaves with checkpointing enabled will
+ * attempt to write checkpoints when required by a task's framework. These checkpoints allow
+ * executors to be reattached rather than killed when a slave is restarted.
+ *
+ * This flag is dangerous! When enabled tasks will not launch on slaves without checkpointing
+ * enabled.
+ *
+ * Behavior is as follows:
+ * (Scheduler -require_slave_checkpoint=true, Slave --checkpoint=true):
+ * Tasks will launch. Checkpoints will be written.
+ * (Scheduler -require_slave_checkpoint=true, Slave --checkpoint=false):
+ * Tasks WILL NOT launch.
+ * (Scheduler -require_slave_checkpoint=false, Slave --checkpoint=true):
+ * Tasks will launch. Checkpoints will not be written.
+ * (Scheduler -require_slave_checkpoint=false, Slave --checkpoint=false):
+ * Tasks will launch. Checkpoints will not be written.
+ *
+ * TODO(ksweeney): Remove warning table after https://issues.apache.org/jira/browse/MESOS-444
+ * is resolved.
+ */
+ @CmdLine(name = "require_slave_checkpoint",
+ help = "DANGEROUS! Require Mesos slaves to have checkpointing enabled. When enabled a "
+ + "slave restart should not kill executors, but the scheduler will not be able to "
+ + "launch tasks on slaves without --checkpoint=true in their command lines. See "
+ + "DriverFactory.java for more information.")
+ private static final Arg<Boolean> REQUIRE_SLAVE_CHECKPOINT = Arg.create(false);
+
+ private static final String EXECUTOR_USER = "root";
+
+ private static final String TWITTER_FRAMEWORK_NAME = "TwitterScheduler";
+
+ private final Provider<Scheduler> scheduler;
+
+ @Inject
+ DriverFactoryImpl(Provider<Scheduler> scheduler) {
+ this.scheduler = Preconditions.checkNotNull(scheduler);
+ }
+
+ @VisibleForTesting
+ static Properties parseCredentials(InputStream credentialsStream) {
+ Properties properties = new Properties();
+ try {
+ properties.load(credentialsStream);
+ } catch (IOException e) {
+ LOG.severe("Unable to load authentication file");
+ throw Throwables.propagate(e);
+ }
+ Preconditions.checkState(properties.containsKey(PRINCIPAL_KEY),
+ "The framework authentication file is missing the key: %s", PRINCIPAL_KEY);
+ Preconditions.checkState(properties.containsKey(SECRET_KEY),
+ "The framework authentication file is missing the key: %s", SECRET_KEY);
+ return properties;
+ }
+
+ @Override
+ public SchedulerDriver apply(@Nullable String frameworkId) {
+ LOG.info("Connecting to mesos master: " + MESOS_MASTER_ADDRESS.get());
+
+ FrameworkInfo.Builder frameworkInfo = FrameworkInfo.newBuilder()
+ .setUser(EXECUTOR_USER)
+ .setName(TWITTER_FRAMEWORK_NAME)
+ .setCheckpoint(REQUIRE_SLAVE_CHECKPOINT.get())
+ .setFailoverTimeout(FRAMEWORK_FAILOVER_TIMEOUT.get().as(Time.SECONDS));
+
+ if (frameworkId != null) {
+ LOG.info("Found persisted framework ID: " + frameworkId);
+ frameworkInfo.setId(FrameworkID.newBuilder().setValue(frameworkId));
+ } else {
+ LOG.warning("Did not find a persisted framework ID, connecting as a new framework.");
+ }
+
+ if (FRAMEWORK_AUTHENTICATION_FILE.hasAppliedValue()) {
+ Properties properties;
+ try {
+ properties = parseCredentials(new FileInputStream(FRAMEWORK_AUTHENTICATION_FILE.get()));
+ } catch (FileNotFoundException e) {
+ LOG.severe("Authentication File not Found");
+ throw Throwables.propagate(e);
+ }
+
+ LOG.info(String.format("Connecting to master using authentication (principal: %s).",
+ properties.get(PRINCIPAL_KEY)));
+
+ Credential credential = Credential.newBuilder()
+ .setPrincipal(properties.getProperty(PRINCIPAL_KEY))
+ .setSecret(ByteString.copyFromUtf8(properties.getProperty(SECRET_KEY)))
+ .build();
+
+ return new MesosSchedulerDriver(
+ scheduler.get(),
+ frameworkInfo.build(),
+ MESOS_MASTER_ADDRESS.get(),
+ credential);
+ } else {
+ LOG.warning("Connecting to master without authentication!");
+ return new MesosSchedulerDriver(
+ scheduler.get(),
+ frameworkInfo.build(),
+ MESOS_MASTER_ADDRESS.get());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/MesosSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/MesosSchedulerImpl.java b/src/main/java/org/apache/aurora/scheduler/MesosSchedulerImpl.java
new file mode 100644
index 0000000..fb41405
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/MesosSchedulerImpl.java
@@ -0,0 +1,268 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.mesos.Protos.ExecutorID;
+import org.apache.mesos.Protos.FrameworkID;
+import org.apache.mesos.Protos.MasterInfo;
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.SlaveID;
+import org.apache.mesos.Protos.TaskInfo;
+import org.apache.mesos.Protos.TaskStatus;
+import org.apache.mesos.Scheduler;
+import org.apache.mesos.SchedulerDriver;
+
+import com.twitter.aurora.GuiceUtils.AllowUnchecked;
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.gen.comm.SchedulerMessage;
+import com.twitter.aurora.scheduler.base.Conversions;
+import com.twitter.aurora.scheduler.base.SchedulerException;
+import com.twitter.aurora.scheduler.configuration.Resources;
+import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.Event;
+import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
+import com.twitter.aurora.scheduler.state.SchedulerCore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.common.application.Lifecycle;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.stats.Stats;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Location for communication with mesos.
+ */
+class MesosSchedulerImpl implements Scheduler {
+ private static final Logger LOG = Logger.getLogger(MesosSchedulerImpl.class.getName());
+
+ private final AtomicLong resourceOffers = Stats.exportLong("scheduler_resource_offers");
+ private final AtomicLong failedOffers = Stats.exportLong("scheduler_failed_offers");
+ private final AtomicLong failedStatusUpdates = Stats.exportLong("scheduler_status_updates");
+ private final AtomicLong frameworkDisconnects =
+ Stats.exportLong("scheduler_framework_disconnects");
+ private final AtomicLong frameworkReregisters =
+ Stats.exportLong("scheduler_framework_reregisters");
+ private final AtomicLong lostExecutors = Stats.exportLong("scheduler_lost_executors");
+
+ private final List<TaskLauncher> taskLaunchers;
+
+ private final Storage storage;
+ private final SchedulerCore schedulerCore;
+ private final Lifecycle lifecycle;
+ private volatile boolean registered = false;
+
+ /**
+ * Creates a new scheduler.
+ *
+ * @param schedulerCore Core scheduler.
+ * @param lifecycle Application lifecycle manager.
+ * @param taskLaunchers Task launchers.
+ */
+ @Inject
+ public MesosSchedulerImpl(
+ Storage storage,
+ SchedulerCore schedulerCore,
+ final Lifecycle lifecycle,
+ List<TaskLauncher> taskLaunchers) {
+
+ this.storage = checkNotNull(storage);
+ this.schedulerCore = checkNotNull(schedulerCore);
+ this.lifecycle = checkNotNull(lifecycle);
+ this.taskLaunchers = checkNotNull(taskLaunchers);
+ }
+
+ @Override
+ public void slaveLost(SchedulerDriver schedulerDriver, SlaveID slaveId) {
+ LOG.info("Received notification of lost slave: " + slaveId);
+ }
+
+ @SendNotification(after = Event.DriverRegistered)
+ @Override
+ public void registered(
+ SchedulerDriver driver,
+ final FrameworkID frameworkId,
+ MasterInfo masterInfo) {
+
+ LOG.info("Registered with ID " + frameworkId + ", master: " + masterInfo);
+
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ storeProvider.getSchedulerStore().saveFrameworkId(frameworkId.getValue());
+ }
+ });
+ registered = true;
+ }
+
+ @SendNotification(after = Event.DriverDisconnected)
+ @Override
+ public void disconnected(SchedulerDriver schedulerDriver) {
+ LOG.warning("Framework disconnected.");
+ frameworkDisconnects.incrementAndGet();
+ }
+
+ @Override
+ public void reregistered(SchedulerDriver schedulerDriver, MasterInfo masterInfo) {
+ LOG.info("Framework re-registered with master " + masterInfo);
+ frameworkReregisters.incrementAndGet();
+ }
+
+ private static boolean fitsInOffer(TaskInfo task, Offer offer) {
+ return Resources.from(offer).greaterThanOrEqual(Resources.from(task.getResourcesList()));
+ }
+
+ @Timed("scheduler_resource_offers")
+ @Override
+ public void resourceOffers(SchedulerDriver driver, List<Offer> offers) {
+ Preconditions.checkState(registered, "Must be registered before receiving offers.");
+
+ for (final Offer offer : offers) {
+ log(Level.FINE, "Received offer: %s", offer);
+ resourceOffers.incrementAndGet();
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ storeProvider.getAttributeStore().saveHostAttributes(Conversions.getAttributes(offer));
+ }
+ });
+
+ // Ordering of task launchers is important here, since offers are consumed greedily.
+ // TODO(William Farner): Refactor this area of code now that the primary task launcher
+ // is asynchronous.
+ for (TaskLauncher launcher : taskLaunchers) {
+ Optional<TaskInfo> task = Optional.absent();
+ try {
+ task = launcher.createTask(offer);
+ } catch (SchedulerException e) {
+ LOG.log(Level.WARNING, "Failed to schedule offers.", e);
+ failedOffers.incrementAndGet();
+ }
+
+ if (task.isPresent()) {
+ if (fitsInOffer(task.get(), offer)) {
+ driver.launchTasks(offer.getId(), ImmutableList.of(task.get()));
+ break;
+ } else {
+ LOG.warning("Insufficient resources to launch task " + task);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void offerRescinded(SchedulerDriver schedulerDriver, OfferID offerId) {
+ LOG.info("Offer rescinded: " + offerId);
+ for (TaskLauncher launcher : taskLaunchers) {
+ launcher.cancelOffer(offerId);
+ }
+ }
+
+ @AllowUnchecked
+ @Timed("scheduler_status_update")
+ @Override
+ public void statusUpdate(SchedulerDriver driver, TaskStatus status) {
+ String info = status.hasData() ? status.getData().toStringUtf8() : null;
+ String infoMsg = info != null ? " with info " + info : "";
+ String coreMsg = status.hasMessage() ? " with core message " + status.getMessage() : "";
+ LOG.info("Received status update for task " + status.getTaskId().getValue()
+ + " in state " + status.getState() + infoMsg + coreMsg);
+
+ try {
+ for (TaskLauncher launcher : taskLaunchers) {
+ if (launcher.statusUpdate(status)) {
+ return;
+ }
+ }
+ } catch (SchedulerException e) {
+ LOG.log(Level.SEVERE, "Status update failed due to scheduler exception: " + e, e);
+ // We re-throw the exception here in an effort to NACK the status update and trigger an
+ // abort of the driver. Previously we directly issued driver.abort(), but the re-entrancy
+ // guarantees of that are uncertain (and we believe it was not working). However, this
+ // was difficult to discern since logging is unreliable during JVM shutdown and we would not
+ // see the above log message.
+ throw e;
+ }
+
+ LOG.warning("Unhandled status update " + status);
+ failedStatusUpdates.incrementAndGet();
+ }
+
+ @Override
+ public void error(SchedulerDriver driver, String message) {
+ LOG.severe("Received error message: " + message);
+ lifecycle.shutdown();
+ }
+
+ @Override
+ public void executorLost(SchedulerDriver schedulerDriver, ExecutorID executorID, SlaveID slaveID,
+ int status) {
+
+ LOG.info("Lost executor " + executorID);
+ lostExecutors.incrementAndGet();
+ }
+
+ @Timed("scheduler_framework_message")
+ @Override
+ public void frameworkMessage(SchedulerDriver driver, ExecutorID executor, SlaveID slave,
+ byte[] data) {
+
+ if (data == null) {
+ LOG.info("Received empty framework message.");
+ return;
+ }
+
+ try {
+ SchedulerMessage schedulerMsg = ThriftBinaryCodec.decode(SchedulerMessage.class, data);
+ if (schedulerMsg == null || !schedulerMsg.isSet()) {
+ LOG.warning("Received empty scheduler message.");
+ return;
+ }
+
+ switch (schedulerMsg.getSetField()) {
+ case DELETED_TASKS:
+ // TODO(William Farner): Refactor this to use a thinner interface here. As it stands
+ // it is odd that we route the registered() call to schedulerCore via the
+ // registeredListener and call the schedulerCore directly here.
+ schedulerCore.tasksDeleted(schedulerMsg.getDeletedTasks().getTaskIds());
+ break;
+
+ default:
+ LOG.warning("Received unhandled scheduler message type: " + schedulerMsg.getSetField());
+ break;
+ }
+ } catch (ThriftBinaryCodec.CodingException e) {
+ LOG.log(Level.SEVERE, "Failed to decode framework message.", e);
+ }
+ }
+
+ private static void log(Level level, String message, Object... args) {
+ if (LOG.isLoggable(level)) {
+ LOG.log(level, String.format(message, args));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/MesosTaskFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/MesosTaskFactory.java b/src/main/java/org/apache/aurora/scheduler/MesosTaskFactory.java
new file mode 100644
index 0000000..5f73f71
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/MesosTaskFactory.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.protobuf.ByteString;
+
+import org.apache.mesos.Protos.ExecutorID;
+import org.apache.mesos.Protos.ExecutorInfo;
+import org.apache.mesos.Protos.Resource;
+import org.apache.mesos.Protos.SlaveID;
+import org.apache.mesos.Protos.TaskID;
+import org.apache.mesos.Protos.TaskInfo;
+
+import com.twitter.aurora.Protobufs;
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.scheduler.base.CommandUtil;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.SchedulerException;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.Resources;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.quantity.Data;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * A factory to create mesos task objects.
+ */
+public interface MesosTaskFactory {
+
+ /**
+ * Creates a mesos task object.
+ *
+ * @param task Assigned task to translate into a task object.
+ * @param slaveId Id of the slave the task is being assigned to.
+ * @return A new task.
+ * @throws SchedulerException If the task could not be encoded.
+ */
+ TaskInfo createFrom(IAssignedTask task, SlaveID slaveId) throws SchedulerException;
+
+ static class ExecutorConfig {
+ private final String executorPath;
+
+ public ExecutorConfig(String executorPath) {
+ this.executorPath = checkNotBlank(executorPath);
+ }
+
+ String getExecutorPath() {
+ return executorPath;
+ }
+ }
+
+ static class MesosTaskFactoryImpl implements MesosTaskFactory {
+ private static final Logger LOG = Logger.getLogger(MesosTaskFactoryImpl.class.getName());
+ private static final String EXECUTOR_PREFIX = "thermos-";
+
+ /**
+ * Name to associate with task executors.
+ */
+ @VisibleForTesting
+ static final String EXECUTOR_NAME = "aurora.task";
+
+ private final String executorPath;
+
+ @Inject
+ MesosTaskFactoryImpl(ExecutorConfig executorConfig) {
+ this.executorPath = executorConfig.getExecutorPath();
+ }
+
+ @VisibleForTesting
+ static ExecutorID getExecutorId(String taskId) {
+ return ExecutorID.newBuilder().setValue(EXECUTOR_PREFIX + taskId).build();
+ }
+
+ public static String getJobSourceName(IJobKey jobkey) {
+ return String.format("%s.%s.%s", jobkey.getRole(), jobkey.getEnvironment(), jobkey.getName());
+ }
+
+ public static String getJobSourceName(ITaskConfig task) {
+ return getJobSourceName(JobKeys.from(task));
+ }
+
+ public static String getInstanceSourceName(ITaskConfig task, int instanceId) {
+ return String.format("%s.%s", getJobSourceName(task), instanceId);
+ }
+
+ @Override
+ public TaskInfo createFrom(IAssignedTask task, SlaveID slaveId) throws SchedulerException {
+ checkNotNull(task);
+ byte[] taskInBytes;
+ try {
+ taskInBytes = ThriftBinaryCodec.encode(task.newBuilder());
+ } catch (ThriftBinaryCodec.CodingException e) {
+ LOG.log(Level.SEVERE, "Unable to serialize task.", e);
+ throw new SchedulerException("Internal error.", e);
+ }
+
+ ITaskConfig config = task.getTask();
+ List<Resource> resources;
+ if (task.isSetAssignedPorts()) {
+ resources = Resources.from(config)
+ .toResourceList(ImmutableSet.copyOf(task.getAssignedPorts().values()));
+ } else {
+ resources = ImmutableList.of();
+ }
+
+ if (LOG.isLoggable(Level.FINE)) {
+ LOG.fine("Setting task resources to "
+ + Iterables.transform(resources, Protobufs.SHORT_TOSTRING));
+ }
+ TaskInfo.Builder taskBuilder =
+ TaskInfo.newBuilder()
+ .setName(JobKeys.toPath(Tasks.ASSIGNED_TO_JOB_KEY.apply(task)))
+ .setTaskId(TaskID.newBuilder().setValue(task.getTaskId()))
+ .setSlaveId(slaveId)
+ .addAllResources(resources)
+ .setData(ByteString.copyFrom(taskInBytes));
+
+ ExecutorInfo executor = ExecutorInfo.newBuilder()
+ .setCommand(CommandUtil.create(executorPath))
+ .setExecutorId(getExecutorId(task.getTaskId()))
+ .setName(EXECUTOR_NAME)
+ .setSource(getInstanceSourceName(config, task.getInstanceId()))
+ .addResources(Resources.makeMesosResource(Resources.CPUS, ResourceSlot.EXECUTOR_CPUS))
+ .addResources(
+ Resources.makeMesosResource(Resources.RAM_MB, ResourceSlot.EXECUTOR_RAM.as(Data.MB)))
+ .build();
+ return taskBuilder
+ .setExecutor(executor)
+ .build();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java b/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java
new file mode 100644
index 0000000..a9c14e6
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/ResourceSlot.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.Arrays;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+
+import com.twitter.aurora.scheduler.configuration.Resources;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+
+import static org.apache.mesos.Protos.Offer;
+
+/**
+ * Resource containing class that is aware of executor overhead.
+ */
+public final class ResourceSlot {
+
+ private final Resources resources;
+
+ /**
+ * CPU allocated for each executor.
+ */
+ @VisibleForTesting
+ static final double EXECUTOR_CPUS = 0.25;
+
+ /**
+ * RAM required for the executor. Executors in the wild have been observed using 48-54MB RSS,
+ * setting to 128MB to be extra vigilant initially.
+ */
+ @VisibleForTesting
+ static final Amount<Long, Data> EXECUTOR_RAM = Amount.of(128L, Data.MB);
+
+ private ResourceSlot(Resources r) {
+ this.resources = r;
+ }
+
+ public static ResourceSlot from(ITaskConfig task) {
+ double totalCPU = task.getNumCpus() + EXECUTOR_CPUS;
+ Amount<Long, Data> totalRAM = Amount.of(task.getRamMb() + EXECUTOR_RAM.as(Data.MB), Data.MB);
+ Amount<Long, Data> disk = Amount.of(task.getDiskMb(), Data.MB);
+ return new ResourceSlot(
+ new Resources(totalCPU, totalRAM, disk, task.getRequestedPorts().size()));
+ }
+
+ public static ResourceSlot from(Offer offer) {
+ return new ResourceSlot(Resources.from(offer));
+ }
+
+ public double getNumCpus() {
+ return resources.getNumCpus();
+ }
+
+ public Amount<Long, Data> getRam() {
+ return resources.getRam();
+ }
+
+ public Amount<Long, Data> getDisk() {
+ return resources.getDisk();
+ }
+
+ public int getNumPorts() {
+ return resources.getNumPorts();
+ }
+
+ @VisibleForTesting
+ public static ResourceSlot from(double cpu,
+ Amount<Long, Data> ram,
+ Amount<Long, Data> disk,
+ int ports) {
+ double totalCPU = cpu + EXECUTOR_CPUS;
+ Amount<Long, Data> totalRAM = Amount.of(ram.as(Data.MB) + EXECUTOR_RAM.as(Data.MB), Data.MB);
+
+ return new ResourceSlot(new Resources(totalCPU, totalRAM, disk, ports));
+ }
+
+ public static ResourceSlot sum(ResourceSlot... rs) {
+ return sum(Arrays.asList(rs));
+ }
+
+ public static ResourceSlot sum(Iterable<ResourceSlot> rs) {
+ Resources r = Resources.sum(Iterables.transform(rs, new Function<ResourceSlot, Resources>() {
+ @Override public Resources apply(ResourceSlot input) {
+ return input.resources;
+ }
+ }));
+
+ return new ResourceSlot(r);
+ }
+
+ public static final Ordering<ResourceSlot> ORDER = new Ordering<ResourceSlot>() {
+ @Override public int compare(ResourceSlot left, ResourceSlot right) {
+ return Resources.RESOURCE_ORDER.compare(left.resources, right.resources);
+ }
+ };
+}
[51/51] [partial] git commit: Rename twitter* and com.twitter to
apache and org.apache directories to preserve all file history before the
refactor.
Posted by wf...@apache.org.
Rename twitter* and com.twitter to apache and org.apache directories to
preserve all file history before the refactor.
Testing Done:
depends on Part 2, patch split up for easier review
Bugs closed: AURORA-3
Reviewed at https://reviews.apache.org/r/16543/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/bc1635df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/bc1635df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/bc1635df
Branch: refs/heads/master
Commit: bc1635df8943fb76157c2eaca49e68696e5d552b
Parents: e8a4db1
Author: Jake Farrell <jf...@apache.org>
Authored: Tue Dec 31 13:15:01 2013 -0800
Committer: Bill Farner <wf...@apache.org>
Committed: Tue Dec 31 13:15:01 2013 -0800
----------------------------------------------------------------------
.../java/com/twitter/aurora/GuiceUtils.java | 179 -
src/main/java/com/twitter/aurora/Protobufs.java | 49 -
.../aurora/auth/CapabilityValidator.java | 60 -
.../twitter/aurora/auth/SessionValidator.java | 74 -
.../twitter/aurora/auth/UnsecureAuthModule.java | 87 -
.../twitter/aurora/codec/ThriftBinaryCodec.java | 133 -
.../com/twitter/aurora/scheduler/Driver.java | 177 -
.../twitter/aurora/scheduler/DriverFactory.java | 180 -
.../aurora/scheduler/MesosSchedulerImpl.java | 268 -
.../aurora/scheduler/MesosTaskFactory.java | 160 -
.../twitter/aurora/scheduler/ResourceSlot.java | 114 -
.../aurora/scheduler/SchedulerLifecycle.java | 417 -
.../aurora/scheduler/SchedulerModule.java | 115 -
.../aurora/scheduler/TaskIdGenerator.java | 68 -
.../twitter/aurora/scheduler/TaskLauncher.java | 59 -
.../com/twitter/aurora/scheduler/TaskVars.java | 173 -
.../aurora/scheduler/UserTaskLauncher.java | 103 -
.../twitter/aurora/scheduler/app/AppModule.java | 184 -
.../aurora/scheduler/app/Log4jConfigurator.java | 71 -
.../twitter/aurora/scheduler/app/Modules.java | 69 -
.../aurora/scheduler/app/SchedulerMain.java | 256 -
.../aurora/scheduler/async/AsyncModule.java | 274 -
.../aurora/scheduler/async/HistoryPruner.java | 212 -
.../aurora/scheduler/async/OfferQueue.java | 341 -
.../aurora/scheduler/async/Preemptor.java | 411 -
.../scheduler/async/RescheduleCalculator.java | 188 -
.../aurora/scheduler/async/TaskGroup.java | 142 -
.../aurora/scheduler/async/TaskGroups.java | 294 -
.../aurora/scheduler/async/TaskScheduler.java | 257 -
.../aurora/scheduler/async/TaskTimeout.java | 269 -
.../aurora/scheduler/base/CommandUtil.java | 59 -
.../aurora/scheduler/base/Conversions.java | 135 -
.../twitter/aurora/scheduler/base/JobKeys.java | 166 -
.../twitter/aurora/scheduler/base/Numbers.java | 66 -
.../twitter/aurora/scheduler/base/Query.java | 364 -
.../scheduler/base/ScheduleException.java | 33 -
.../scheduler/base/SchedulerException.java | 31 -
.../twitter/aurora/scheduler/base/Tasks.java | 186 -
.../configuration/ConfigurationManager.java | 413 -
.../scheduler/configuration/Resources.java | 447 -
.../configuration/SanitizedConfiguration.java | 98 -
.../aurora/scheduler/cron/CronException.java | 33 -
.../aurora/scheduler/cron/CronPredictor.java | 31 -
.../aurora/scheduler/cron/CronScheduler.java | 82 -
.../scheduler/cron/noop/NoopCronModule.java | 40 -
.../scheduler/cron/noop/NoopCronPredictor.java | 33 -
.../scheduler/cron/noop/NoopCronScheduler.java | 83 -
.../scheduler/cron/testing/AbstractCronIT.java | 95 -
.../events/NotifyingMethodInterceptor.java | 66 -
.../events/NotifyingSchedulingFilter.java | 72 -
.../aurora/scheduler/events/PubsubEvent.java | 319 -
.../scheduler/events/PubsubEventModule.java | 147 -
.../scheduler/filter/AttributeFilter.java | 93 -
.../scheduler/filter/ConstraintFilter.java | 138 -
.../scheduler/filter/SchedulingFilter.java | 116 -
.../scheduler/filter/SchedulingFilterImpl.java | 294 -
.../aurora/scheduler/http/ClusterName.java | 33 -
.../com/twitter/aurora/scheduler/http/Cron.java | 58 -
.../aurora/scheduler/http/DisplayUtils.java | 57 -
.../aurora/scheduler/http/HttpStatsFilter.java | 73 -
.../scheduler/http/JerseyTemplateServlet.java | 49 -
.../aurora/scheduler/http/LeaderRedirect.java | 171 -
.../scheduler/http/LeaderRedirectFilter.java | 54 -
.../aurora/scheduler/http/Maintenance.java | 108 -
.../twitter/aurora/scheduler/http/Mname.java | 248 -
.../twitter/aurora/scheduler/http/Offers.java | 135 -
.../aurora/scheduler/http/PendingTasks.java | 52 -
.../twitter/aurora/scheduler/http/Quotas.java | 113 -
.../aurora/scheduler/http/SchedulerzHome.java | 148 -
.../aurora/scheduler/http/SchedulerzJob.java | 432 -
.../aurora/scheduler/http/SchedulerzRole.java | 393 -
.../aurora/scheduler/http/ServletModule.java | 225 -
.../twitter/aurora/scheduler/http/Slaves.java | 140 -
.../aurora/scheduler/http/StructDump.java | 128 -
.../scheduler/http/TransformationUtils.java | 59 -
.../aurora/scheduler/http/Utilization.java | 217 -
.../scheduler/local/FakeDriverFactory.java | 137 -
.../local/IsolatedSchedulerModule.java | 330 -
.../com/twitter/aurora/scheduler/log/Log.java | 122 -
.../scheduler/log/mesos/LogInterface.java | 42 -
.../aurora/scheduler/log/mesos/MesosLog.java | 379 -
.../log/mesos/MesosLogStreamModule.java | 203 -
.../aurora/scheduler/log/testing/FileLog.java | 164 -
.../log/testing/FileLogStreamModule.java | 48 -
.../scheduler/metadata/MetadataModule.java | 34 -
.../aurora/scheduler/metadata/NearestFit.java | 178 -
.../scheduler/periodic/GcExecutorLauncher.java | 195 -
.../scheduler/quota/QuotaComparisonResult.java | 90 -
.../aurora/scheduler/quota/QuotaFilter.java | 85 -
.../aurora/scheduler/quota/QuotaManager.java | 91 -
.../aurora/scheduler/quota/QuotaModule.java | 41 -
.../twitter/aurora/scheduler/quota/Quotas.java | 155 -
.../aurora/scheduler/state/CronJobManager.java | 472 -
.../scheduler/state/ImmediateJobManager.java | 61 -
.../aurora/scheduler/state/JobFilter.java | 119 -
.../aurora/scheduler/state/JobManager.java | 85 -
.../aurora/scheduler/state/LockManager.java | 66 -
.../aurora/scheduler/state/LockManagerImpl.java | 130 -
.../scheduler/state/MaintenanceController.java | 302 -
.../aurora/scheduler/state/SchedulerCore.java | 127 -
.../scheduler/state/SchedulerCoreImpl.java | 328 -
.../scheduler/state/SideEffectStorage.java | 169 -
.../aurora/scheduler/state/StateManager.java | 80 -
.../scheduler/state/StateManagerImpl.java | 458 -
.../aurora/scheduler/state/StateModule.java | 70 -
.../aurora/scheduler/state/TaskAssigner.java | 102 -
.../scheduler/state/TaskStateMachine.java | 622 --
.../aurora/scheduler/state/UUIDGenerator.java | 17 -
.../aurora/scheduler/state/WorkCommand.java | 33 -
.../scheduler/stats/AsyncStatsModule.java | 137 -
.../aurora/scheduler/stats/CachedCounters.java | 48 -
.../aurora/scheduler/stats/ResourceCounter.java | 207 -
.../aurora/scheduler/stats/SlotSizeCounter.java | 94 -
.../scheduler/stats/TaskStatCalculator.java | 46 -
.../scheduler/storage/AttributeStore.java | 96 -
.../storage/CallOrderEnforcingStorage.java | 144 -
.../storage/DistributedSnapshotStore.java | 34 -
.../scheduler/storage/ForwardingStore.java | 261 -
.../aurora/scheduler/storage/JobStore.java | 80 -
.../aurora/scheduler/storage/LockStore.java | 64 -
.../aurora/scheduler/storage/QuotaStore.java | 65 -
.../scheduler/storage/ReadWriteLockManager.java | 108 -
.../scheduler/storage/SchedulerStore.java | 40 -
.../aurora/scheduler/storage/SnapshotStore.java | 38 -
.../aurora/scheduler/storage/Storage.java | 322 -
.../scheduler/storage/StorageBackfill.java | 145 -
.../aurora/scheduler/storage/TaskStore.java | 99 -
.../scheduler/storage/backup/BackupModule.java | 145 -
.../scheduler/storage/backup/Recovery.java | 210 -
.../scheduler/storage/backup/StorageBackup.java | 209 -
.../storage/backup/TemporaryStorage.java | 102 -
.../aurora/scheduler/storage/log/Entries.java | 137 -
.../scheduler/storage/log/LogManager.java | 516 -
.../scheduler/storage/log/LogStorage.java | 715 --
.../scheduler/storage/log/LogStorageModule.java | 99 -
.../storage/log/SnapshotStoreImpl.java | 225 -
.../storage/log/testing/LogOpMatcher.java | 114 -
.../aurora/scheduler/storage/mem/Interner.java | 98 -
.../storage/mem/MemAttributeStore.java | 73 -
.../scheduler/storage/mem/MemJobStore.java | 107 -
.../scheduler/storage/mem/MemLockStore.java | 60 -
.../scheduler/storage/mem/MemQuotaStore.java | 66 -
.../storage/mem/MemSchedulerStore.java | 42 -
.../scheduler/storage/mem/MemStorage.java | 156 -
.../scheduler/storage/mem/MemStorageModule.java | 85 -
.../scheduler/storage/mem/MemTaskStore.java | 304 -
.../aurora/scheduler/storage/mem/Util.java | 53 -
.../storage/testing/StorageTestUtil.java | 135 -
.../scheduler/thrift/SchedulerAPIServlet.java | 19 -
.../thrift/SchedulerThriftInterface.java | 1000 --
.../scheduler/thrift/ThriftConfiguration.java | 41 -
.../aurora/scheduler/thrift/ThriftModule.java | 42 -
.../aurora/scheduler/thrift/ThriftServer.java | 107 -
.../scheduler/thrift/ThriftServerLauncher.java | 115 -
.../thrift/aop/APIVersionInterceptor.java | 20 -
.../aurora/scheduler/thrift/aop/AopModule.java | 169 -
.../thrift/aop/FeatureToggleInterceptor.java | 48 -
.../scheduler/thrift/aop/Interceptors.java | 71 -
.../thrift/aop/LoggingInterceptor.java | 95 -
.../aop/ThriftStatsExporterInterceptor.java | 54 -
.../thrift/aop/UserCapabilityInterceptor.java | 106 -
.../scheduler/thrift/auth/DecoratedThrift.java | 31 -
.../aurora/scheduler/thrift/auth/Requires.java | 36 -
.../scheduler/thrift/auth/ThriftAuthModule.java | 68 -
src/main/java/org/apache/aurora/GuiceUtils.java | 179 +
src/main/java/org/apache/aurora/Protobufs.java | 49 +
.../apache/aurora/auth/CapabilityValidator.java | 60 +
.../apache/aurora/auth/SessionValidator.java | 74 +
.../apache/aurora/auth/UnsecureAuthModule.java | 87 +
.../apache/aurora/codec/ThriftBinaryCodec.java | 133 +
.../org/apache/aurora/scheduler/Driver.java | 177 +
.../apache/aurora/scheduler/DriverFactory.java | 180 +
.../aurora/scheduler/MesosSchedulerImpl.java | 268 +
.../aurora/scheduler/MesosTaskFactory.java | 160 +
.../apache/aurora/scheduler/ResourceSlot.java | 114 +
.../aurora/scheduler/SchedulerLifecycle.java | 417 +
.../aurora/scheduler/SchedulerModule.java | 115 +
.../aurora/scheduler/TaskIdGenerator.java | 68 +
.../apache/aurora/scheduler/TaskLauncher.java | 59 +
.../org/apache/aurora/scheduler/TaskVars.java | 173 +
.../aurora/scheduler/UserTaskLauncher.java | 103 +
.../apache/aurora/scheduler/app/AppModule.java | 184 +
.../aurora/scheduler/app/Log4jConfigurator.java | 71 +
.../apache/aurora/scheduler/app/Modules.java | 69 +
.../aurora/scheduler/app/SchedulerMain.java | 256 +
.../aurora/scheduler/async/AsyncModule.java | 274 +
.../aurora/scheduler/async/HistoryPruner.java | 212 +
.../aurora/scheduler/async/OfferQueue.java | 341 +
.../aurora/scheduler/async/Preemptor.java | 411 +
.../scheduler/async/RescheduleCalculator.java | 188 +
.../aurora/scheduler/async/TaskGroup.java | 142 +
.../aurora/scheduler/async/TaskGroups.java | 294 +
.../aurora/scheduler/async/TaskScheduler.java | 257 +
.../aurora/scheduler/async/TaskTimeout.java | 269 +
.../aurora/scheduler/base/CommandUtil.java | 59 +
.../aurora/scheduler/base/Conversions.java | 135 +
.../apache/aurora/scheduler/base/JobKeys.java | 166 +
.../apache/aurora/scheduler/base/Numbers.java | 66 +
.../org/apache/aurora/scheduler/base/Query.java | 364 +
.../scheduler/base/ScheduleException.java | 33 +
.../scheduler/base/SchedulerException.java | 31 +
.../org/apache/aurora/scheduler/base/Tasks.java | 186 +
.../configuration/ConfigurationManager.java | 413 +
.../scheduler/configuration/Resources.java | 447 +
.../configuration/SanitizedConfiguration.java | 98 +
.../aurora/scheduler/cron/CronException.java | 33 +
.../aurora/scheduler/cron/CronPredictor.java | 31 +
.../aurora/scheduler/cron/CronScheduler.java | 82 +
.../scheduler/cron/noop/NoopCronModule.java | 40 +
.../scheduler/cron/noop/NoopCronPredictor.java | 33 +
.../scheduler/cron/noop/NoopCronScheduler.java | 83 +
.../scheduler/cron/testing/AbstractCronIT.java | 95 +
.../events/NotifyingMethodInterceptor.java | 66 +
.../events/NotifyingSchedulingFilter.java | 72 +
.../aurora/scheduler/events/PubsubEvent.java | 319 +
.../scheduler/events/PubsubEventModule.java | 147 +
.../scheduler/filter/AttributeFilter.java | 93 +
.../scheduler/filter/ConstraintFilter.java | 138 +
.../scheduler/filter/SchedulingFilter.java | 116 +
.../scheduler/filter/SchedulingFilterImpl.java | 294 +
.../aurora/scheduler/http/ClusterName.java | 33 +
.../org/apache/aurora/scheduler/http/Cron.java | 58 +
.../aurora/scheduler/http/DisplayUtils.java | 57 +
.../aurora/scheduler/http/HttpStatsFilter.java | 73 +
.../scheduler/http/JerseyTemplateServlet.java | 49 +
.../aurora/scheduler/http/LeaderRedirect.java | 171 +
.../scheduler/http/LeaderRedirectFilter.java | 54 +
.../aurora/scheduler/http/Maintenance.java | 108 +
.../org/apache/aurora/scheduler/http/Mname.java | 248 +
.../apache/aurora/scheduler/http/Offers.java | 135 +
.../aurora/scheduler/http/PendingTasks.java | 52 +
.../apache/aurora/scheduler/http/Quotas.java | 113 +
.../aurora/scheduler/http/SchedulerzHome.java | 148 +
.../aurora/scheduler/http/SchedulerzJob.java | 432 +
.../aurora/scheduler/http/SchedulerzRole.java | 393 +
.../aurora/scheduler/http/ServletModule.java | 225 +
.../apache/aurora/scheduler/http/Slaves.java | 140 +
.../aurora/scheduler/http/StructDump.java | 128 +
.../scheduler/http/TransformationUtils.java | 59 +
.../aurora/scheduler/http/Utilization.java | 217 +
.../scheduler/local/FakeDriverFactory.java | 137 +
.../local/IsolatedSchedulerModule.java | 330 +
.../org/apache/aurora/scheduler/log/Log.java | 122 +
.../scheduler/log/mesos/LogInterface.java | 42 +
.../aurora/scheduler/log/mesos/MesosLog.java | 379 +
.../log/mesos/MesosLogStreamModule.java | 203 +
.../aurora/scheduler/log/testing/FileLog.java | 164 +
.../log/testing/FileLogStreamModule.java | 48 +
.../scheduler/metadata/MetadataModule.java | 34 +
.../aurora/scheduler/metadata/NearestFit.java | 178 +
.../scheduler/periodic/GcExecutorLauncher.java | 195 +
.../scheduler/quota/QuotaComparisonResult.java | 90 +
.../aurora/scheduler/quota/QuotaFilter.java | 85 +
.../aurora/scheduler/quota/QuotaManager.java | 91 +
.../aurora/scheduler/quota/QuotaModule.java | 41 +
.../apache/aurora/scheduler/quota/Quotas.java | 155 +
.../aurora/scheduler/state/CronJobManager.java | 472 +
.../scheduler/state/ImmediateJobManager.java | 61 +
.../aurora/scheduler/state/JobFilter.java | 119 +
.../aurora/scheduler/state/JobManager.java | 85 +
.../aurora/scheduler/state/LockManager.java | 66 +
.../aurora/scheduler/state/LockManagerImpl.java | 130 +
.../scheduler/state/MaintenanceController.java | 302 +
.../aurora/scheduler/state/SchedulerCore.java | 127 +
.../scheduler/state/SchedulerCoreImpl.java | 328 +
.../scheduler/state/SideEffectStorage.java | 169 +
.../aurora/scheduler/state/StateManager.java | 80 +
.../scheduler/state/StateManagerImpl.java | 458 +
.../aurora/scheduler/state/StateModule.java | 70 +
.../aurora/scheduler/state/TaskAssigner.java | 102 +
.../scheduler/state/TaskStateMachine.java | 622 ++
.../aurora/scheduler/state/UUIDGenerator.java | 17 +
.../aurora/scheduler/state/WorkCommand.java | 33 +
.../scheduler/stats/AsyncStatsModule.java | 137 +
.../aurora/scheduler/stats/CachedCounters.java | 48 +
.../aurora/scheduler/stats/ResourceCounter.java | 207 +
.../aurora/scheduler/stats/SlotSizeCounter.java | 94 +
.../scheduler/stats/TaskStatCalculator.java | 46 +
.../scheduler/storage/AttributeStore.java | 96 +
.../storage/CallOrderEnforcingStorage.java | 144 +
.../storage/DistributedSnapshotStore.java | 34 +
.../scheduler/storage/ForwardingStore.java | 261 +
.../aurora/scheduler/storage/JobStore.java | 80 +
.../aurora/scheduler/storage/LockStore.java | 64 +
.../aurora/scheduler/storage/QuotaStore.java | 65 +
.../scheduler/storage/ReadWriteLockManager.java | 108 +
.../scheduler/storage/SchedulerStore.java | 40 +
.../aurora/scheduler/storage/SnapshotStore.java | 38 +
.../aurora/scheduler/storage/Storage.java | 322 +
.../scheduler/storage/StorageBackfill.java | 145 +
.../aurora/scheduler/storage/TaskStore.java | 99 +
.../scheduler/storage/backup/BackupModule.java | 145 +
.../scheduler/storage/backup/Recovery.java | 210 +
.../scheduler/storage/backup/StorageBackup.java | 209 +
.../storage/backup/TemporaryStorage.java | 102 +
.../aurora/scheduler/storage/log/Entries.java | 137 +
.../scheduler/storage/log/LogManager.java | 516 +
.../scheduler/storage/log/LogStorage.java | 715 ++
.../scheduler/storage/log/LogStorageModule.java | 99 +
.../storage/log/SnapshotStoreImpl.java | 225 +
.../storage/log/testing/LogOpMatcher.java | 114 +
.../aurora/scheduler/storage/mem/Interner.java | 98 +
.../storage/mem/MemAttributeStore.java | 73 +
.../scheduler/storage/mem/MemJobStore.java | 107 +
.../scheduler/storage/mem/MemLockStore.java | 60 +
.../scheduler/storage/mem/MemQuotaStore.java | 66 +
.../storage/mem/MemSchedulerStore.java | 42 +
.../scheduler/storage/mem/MemStorage.java | 156 +
.../scheduler/storage/mem/MemStorageModule.java | 85 +
.../scheduler/storage/mem/MemTaskStore.java | 304 +
.../aurora/scheduler/storage/mem/Util.java | 53 +
.../storage/testing/StorageTestUtil.java | 135 +
.../scheduler/thrift/SchedulerAPIServlet.java | 19 +
.../thrift/SchedulerThriftInterface.java | 1000 ++
.../scheduler/thrift/ThriftConfiguration.java | 41 +
.../aurora/scheduler/thrift/ThriftModule.java | 42 +
.../aurora/scheduler/thrift/ThriftServer.java | 107 +
.../scheduler/thrift/ThriftServerLauncher.java | 115 +
.../thrift/aop/APIVersionInterceptor.java | 20 +
.../aurora/scheduler/thrift/aop/AopModule.java | 169 +
.../thrift/aop/FeatureToggleInterceptor.java | 48 +
.../scheduler/thrift/aop/Interceptors.java | 71 +
.../thrift/aop/LoggingInterceptor.java | 95 +
.../aop/ThriftStatsExporterInterceptor.java | 54 +
.../thrift/aop/UserCapabilityInterceptor.java | 106 +
.../scheduler/thrift/auth/DecoratedThrift.java | 31 +
.../aurora/scheduler/thrift/auth/Requires.java | 36 +
.../scheduler/thrift/auth/ThriftAuthModule.java | 68 +
src/main/python/apache/__init__.py | 1 +
src/main/python/apache/aurora/BUILD | 0
src/main/python/apache/aurora/BUILD.thirdparty | 19 +
src/main/python/apache/aurora/__init__.py | 1 +
src/main/python/apache/aurora/admin/BUILD | 11 +
src/main/python/apache/aurora/admin/__init__.py | 0
.../apache/aurora/admin/mesos_maintenance.py | 113 +
src/main/python/apache/aurora/client/BUILD | 78 +
.../python/apache/aurora/client/__init__.py | 1 +
src/main/python/apache/aurora/client/api/BUILD | 104 +
.../python/apache/aurora/client/api/__init__.py | 190 +
.../apache/aurora/client/api/command_runner.py | 120 +
.../apache/aurora/client/api/disambiguator.py | 89 +
.../apache/aurora/client/api/health_check.py | 123 +
.../aurora/client/api/instance_watcher.py | 134 +
.../apache/aurora/client/api/job_monitor.py | 63 +
.../apache/aurora/client/api/restarter.py | 73 +
.../aurora/client/api/scheduler_client.py | 257 +
.../python/apache/aurora/client/api/updater.py | 410 +
.../apache/aurora/client/api/updater_util.py | 77 +
src/main/python/apache/aurora/client/base.py | 151 +
src/main/python/apache/aurora/client/bin/BUILD | 25 +
.../python/apache/aurora/client/bin/__init__.py | 0
.../apache/aurora/client/bin/aurora_admin.py | 23 +
.../apache/aurora/client/bin/aurora_client.py | 33 +
.../apache/aurora/client/binding_helper.py | 115 +
src/main/python/apache/aurora/client/cli/BUILD | 26 +
.../python/apache/aurora/client/cli/__init__.py | 213 +
.../python/apache/aurora/client/cli/context.py | 49 +
.../python/apache/aurora/client/cli/jobs.py | 115 +
.../python/apache/aurora/client/cli/options.py | 25 +
.../python/apache/aurora/client/commands/BUILD | 81 +
.../apache/aurora/client/commands/__init__.py | 0
.../apache/aurora/client/commands/admin.py | 406 +
.../apache/aurora/client/commands/core.py | 600 ++
.../apache/aurora/client/commands/help.py | 53 +
.../python/apache/aurora/client/commands/run.py | 40 +
.../python/apache/aurora/client/commands/ssh.py | 64 +
src/main/python/apache/aurora/client/config.py | 208 +
src/main/python/apache/aurora/client/factory.py | 27 +
.../python/apache/aurora/client/hooks/BUILD | 10 +
.../apache/aurora/client/hooks/__init__.py | 49 +
.../apache/aurora/client/hooks/hooked_api.py | 167 +
src/main/python/apache/aurora/client/options.py | 199 +
src/main/python/apache/aurora/common/BUILD | 63 +
.../python/apache/aurora/common/__init__.py | 0
.../apache/aurora/common/aurora_job_key.py | 97 +
src/main/python/apache/aurora/common/auth/BUILD | 9 +
.../apache/aurora/common/auth/__init__.py | 2 +
.../apache/aurora/common/auth/auth_module.py | 30 +
.../aurora/common/auth/auth_module_manager.py | 47 +
src/main/python/apache/aurora/common/cluster.py | 81 +
.../apache/aurora/common/cluster_option.py | 82 +
.../python/apache/aurora/common/clusters.py | 135 +
.../apache/aurora/common/http_signaler.py | 82 +
src/main/python/apache/aurora/config/BUILD | 43 +
.../python/apache/aurora/config/__init__.py | 271 +
src/main/python/apache/aurora/config/loader.py | 63 +
.../apache/aurora/config/port_resolver.py | 45 +
src/main/python/apache/aurora/config/recipes.py | 44 +
src/main/python/apache/aurora/config/repl.py | 8 +
.../python/apache/aurora/config/schema/BUILD | 9 +
.../apache/aurora/config/schema/__init__.py | 0
.../python/apache/aurora/config/schema/base.py | 122 +
src/main/python/apache/aurora/config/thrift.py | 259 +
src/main/python/apache/aurora/executor/BUILD | 139 +
.../python/apache/aurora/executor/__init__.py | 1 +
.../python/apache/aurora/executor/bin/BUILD | 47 +
.../apache/aurora/executor/bin/__init__.py | 0
.../aurora/executor/bin/gc_executor_main.py | 45 +
.../executor/bin/thermos_executor_main.py | 67 +
.../aurora/executor/bin/thermos_runner_main.py | 12 +
.../python/apache/aurora/executor/common/BUILD | 69 +
.../apache/aurora/executor/common/__init__.py | 0
.../aurora/executor/common/executor_timeout.py | 21 +
.../aurora/executor/common/health_checker.py | 99 +
.../aurora/executor/common/kill_manager.py | 19 +
.../apache/aurora/executor/common/sandbox.py | 72 +
.../aurora/executor/common/status_checker.py | 111 +
.../apache/aurora/executor/common/task_info.py | 90 +
.../aurora/executor/common/task_runner.py | 20 +
.../apache/aurora/executor/executor_base.py | 103 +
.../apache/aurora/executor/executor_detector.py | 77 +
.../apache/aurora/executor/executor_vars.py | 123 +
.../apache/aurora/executor/gc_executor.py | 499 +
.../apache/aurora/executor/status_manager.py | 39 +
.../apache/aurora/executor/thermos_executor.py | 289 +
.../apache/aurora/executor/thermos_runner.py | 138 +
.../aurora/executor/thermos_task_runner.py | 331 +
.../aurora/tools/java/organize_imports.py | 130 +
.../aurora/tools/java/thrift_wrapper_codegen.py | 475 +
src/main/python/apache/thermos/BUILD | 18 +
src/main/python/apache/thermos/BUILD.thirdparty | 34 +
src/main/python/apache/thermos/README.md | 461 +
src/main/python/apache/thermos/__init__.py | 1 +
src/main/python/apache/thermos/bin/BUILD | 24 +
src/main/python/apache/thermos/bin/__init__.py | 0
src/main/python/apache/thermos/bin/thermos.py | 653 ++
.../python/apache/thermos/bin/thermos_ckpt.py | 55 +
src/main/python/apache/thermos/common/BUILD | 45 +
.../python/apache/thermos/common/__init__.py | 0
src/main/python/apache/thermos/common/ckpt.py | 375 +
.../python/apache/thermos/common/options.py | 34 +
src/main/python/apache/thermos/common/path.py | 101 +
.../python/apache/thermos/common/planner.py | 304 +
src/main/python/apache/thermos/config/BUILD | 24 +
.../python/apache/thermos/config/__init__.py | 0
.../apache/thermos/config/bin/config_load.py | 47 +
.../apache/thermos/config/bin/config_repl.py | 3 +
src/main/python/apache/thermos/config/dsl.py | 0
src/main/python/apache/thermos/config/loader.py | 179 +
src/main/python/apache/thermos/config/schema.py | 2 +
.../python/apache/thermos/config/schema_base.py | 75 +
.../apache/thermos/config/schema_helpers.py | 247 +
src/main/python/apache/thermos/core/BUILD | 88 +
src/main/python/apache/thermos/core/__init__.py | 1 +
src/main/python/apache/thermos/core/helper.py | 387 +
.../python/apache/thermos/core/inspector.py | 105 +
src/main/python/apache/thermos/core/muxer.py | 142 +
src/main/python/apache/thermos/core/process.py | 371 +
src/main/python/apache/thermos/core/runner.py | 905 ++
src/main/python/apache/thermos/monitoring/BUILD | 97 +
.../apache/thermos/monitoring/__init__.py | 0
.../apache/thermos/monitoring/detector.py | 91 +
.../python/apache/thermos/monitoring/disk.py | 181 +
.../python/apache/thermos/monitoring/garbage.py | 183 +
.../python/apache/thermos/monitoring/monitor.py | 125 +
.../python/apache/thermos/monitoring/process.py | 49 +
.../monitoring/process_collector_psutil.py | 92 +
.../apache/thermos/monitoring/resource.py | 222 +
src/main/python/apache/thermos/observer/BUILD | 51 +
.../python/apache/thermos/observer/__init__.py | 0
.../python/apache/thermos/observer/bin/BUILD | 14 +
.../apache/thermos/observer/bin/__init__.py | 0
.../thermos/observer/bin/thermos_observer.py | 55 +
.../python/apache/thermos/observer/http/BUILD | 47 +
.../apache/thermos/observer/http/__init__.py | 0
.../thermos/observer/http/assets/bootstrap.css | 9 +
.../thermos/observer/http/assets/favicon.ico | Bin 0 -> 145652 bytes
.../thermos/observer/http/assets/jquery.js | 9404 ++++++++++++++++++
.../observer/http/assets/jquery.pailer.js | 317 +
.../observer/http/assets/mootools-core.js | 5789 +++++++++++
.../thermos/observer/http/assets/observer.js | 202 +
.../thermos/observer/http/file_browser.py | 124 +
.../thermos/observer/http/http_observer.py | 133 +
.../python/apache/thermos/observer/http/json.py | 56 +
.../thermos/observer/http/static_assets.py | 43 +
.../observer/http/templates/filebrowse.tpl | 81 +
.../observer/http/templates/filelist.tpl | 141 +
.../thermos/observer/http/templates/home.tpl | 58 +
.../thermos/observer/http/templates/index.tpl | 51 +
.../observer/http/templates/logbrowse.tpl | 80 +
.../thermos/observer/http/templates/main.tpl | 101 +
.../thermos/observer/http/templates/process.tpl | 109 +
.../thermos/observer/http/templates/rawtask.tpl | 28 +
.../thermos/observer/http/templates/task.tpl | 140 +
.../apache/thermos/observer/http/templating.py | 9 +
.../apache/thermos/observer/observed_task.py | 136 +
.../apache/thermos/observer/task_observer.py | 640 ++
src/main/python/apache/thermos/testing/BUILD | 14 +
.../python/apache/thermos/testing/__init__.py | 0
.../python/apache/thermos/testing/runner.py | 191 +
src/main/python/twitter/__init__.py | 1 -
src/main/python/twitter/aurora/BUILD | 0
src/main/python/twitter/aurora/BUILD.thirdparty | 19 -
src/main/python/twitter/aurora/__init__.py | 1 -
src/main/python/twitter/aurora/admin/BUILD | 11 -
.../python/twitter/aurora/admin/__init__.py | 0
.../twitter/aurora/admin/mesos_maintenance.py | 113 -
src/main/python/twitter/aurora/client/BUILD | 78 -
.../python/twitter/aurora/client/__init__.py | 1 -
src/main/python/twitter/aurora/client/api/BUILD | 104 -
.../twitter/aurora/client/api/__init__.py | 190 -
.../twitter/aurora/client/api/command_runner.py | 120 -
.../twitter/aurora/client/api/disambiguator.py | 89 -
.../twitter/aurora/client/api/health_check.py | 123 -
.../aurora/client/api/instance_watcher.py | 134 -
.../twitter/aurora/client/api/job_monitor.py | 63 -
.../twitter/aurora/client/api/restarter.py | 73 -
.../aurora/client/api/scheduler_client.py | 257 -
.../python/twitter/aurora/client/api/updater.py | 410 -
.../twitter/aurora/client/api/updater_util.py | 77 -
src/main/python/twitter/aurora/client/base.py | 151 -
src/main/python/twitter/aurora/client/bin/BUILD | 25 -
.../twitter/aurora/client/bin/__init__.py | 0
.../twitter/aurora/client/bin/aurora_admin.py | 23 -
.../twitter/aurora/client/bin/aurora_client.py | 33 -
.../twitter/aurora/client/binding_helper.py | 115 -
src/main/python/twitter/aurora/client/cli/BUILD | 26 -
.../twitter/aurora/client/cli/__init__.py | 213 -
.../python/twitter/aurora/client/cli/context.py | 49 -
.../python/twitter/aurora/client/cli/jobs.py | 115 -
.../python/twitter/aurora/client/cli/options.py | 25 -
.../python/twitter/aurora/client/commands/BUILD | 81 -
.../twitter/aurora/client/commands/__init__.py | 0
.../twitter/aurora/client/commands/admin.py | 406 -
.../twitter/aurora/client/commands/core.py | 600 --
.../twitter/aurora/client/commands/help.py | 53 -
.../twitter/aurora/client/commands/run.py | 40 -
.../twitter/aurora/client/commands/ssh.py | 64 -
src/main/python/twitter/aurora/client/config.py | 208 -
.../python/twitter/aurora/client/factory.py | 27 -
.../python/twitter/aurora/client/hooks/BUILD | 10 -
.../twitter/aurora/client/hooks/__init__.py | 49 -
.../twitter/aurora/client/hooks/hooked_api.py | 167 -
.../python/twitter/aurora/client/options.py | 199 -
src/main/python/twitter/aurora/common/BUILD | 63 -
.../python/twitter/aurora/common/__init__.py | 0
.../twitter/aurora/common/aurora_job_key.py | 97 -
.../python/twitter/aurora/common/auth/BUILD | 9 -
.../twitter/aurora/common/auth/__init__.py | 2 -
.../twitter/aurora/common/auth/auth_module.py | 30 -
.../aurora/common/auth/auth_module_manager.py | 47 -
.../python/twitter/aurora/common/cluster.py | 81 -
.../twitter/aurora/common/cluster_option.py | 82 -
.../python/twitter/aurora/common/clusters.py | 135 -
.../twitter/aurora/common/http_signaler.py | 82 -
src/main/python/twitter/aurora/config/BUILD | 43 -
.../python/twitter/aurora/config/__init__.py | 271 -
src/main/python/twitter/aurora/config/loader.py | 63 -
.../twitter/aurora/config/port_resolver.py | 45 -
.../python/twitter/aurora/config/recipes.py | 44 -
src/main/python/twitter/aurora/config/repl.py | 8 -
.../python/twitter/aurora/config/schema/BUILD | 9 -
.../twitter/aurora/config/schema/__init__.py | 0
.../python/twitter/aurora/config/schema/base.py | 122 -
src/main/python/twitter/aurora/config/thrift.py | 259 -
src/main/python/twitter/aurora/executor/BUILD | 139 -
.../python/twitter/aurora/executor/__init__.py | 1 -
.../python/twitter/aurora/executor/bin/BUILD | 47 -
.../twitter/aurora/executor/bin/__init__.py | 0
.../aurora/executor/bin/gc_executor_main.py | 45 -
.../executor/bin/thermos_executor_main.py | 67 -
.../aurora/executor/bin/thermos_runner_main.py | 12 -
.../python/twitter/aurora/executor/common/BUILD | 69 -
.../twitter/aurora/executor/common/__init__.py | 0
.../aurora/executor/common/executor_timeout.py | 21 -
.../aurora/executor/common/health_checker.py | 99 -
.../aurora/executor/common/kill_manager.py | 19 -
.../twitter/aurora/executor/common/sandbox.py | 72 -
.../aurora/executor/common/status_checker.py | 111 -
.../twitter/aurora/executor/common/task_info.py | 90 -
.../aurora/executor/common/task_runner.py | 20 -
.../twitter/aurora/executor/executor_base.py | 103 -
.../aurora/executor/executor_detector.py | 77 -
.../twitter/aurora/executor/executor_vars.py | 123 -
.../twitter/aurora/executor/gc_executor.py | 499 -
.../twitter/aurora/executor/status_manager.py | 39 -
.../twitter/aurora/executor/thermos_executor.py | 289 -
.../twitter/aurora/executor/thermos_runner.py | 138 -
.../aurora/executor/thermos_task_runner.py | 331 -
.../aurora/tools/java/organize_imports.py | 130 -
.../aurora/tools/java/thrift_wrapper_codegen.py | 475 -
src/main/python/twitter/thermos/BUILD | 18 -
.../python/twitter/thermos/BUILD.thirdparty | 34 -
src/main/python/twitter/thermos/README.md | 461 -
src/main/python/twitter/thermos/__init__.py | 1 -
src/main/python/twitter/thermos/bin/BUILD | 24 -
src/main/python/twitter/thermos/bin/__init__.py | 0
src/main/python/twitter/thermos/bin/thermos.py | 653 --
.../python/twitter/thermos/bin/thermos_ckpt.py | 55 -
src/main/python/twitter/thermos/common/BUILD | 45 -
.../python/twitter/thermos/common/__init__.py | 0
src/main/python/twitter/thermos/common/ckpt.py | 375 -
.../python/twitter/thermos/common/options.py | 34 -
src/main/python/twitter/thermos/common/path.py | 101 -
.../python/twitter/thermos/common/planner.py | 304 -
src/main/python/twitter/thermos/config/BUILD | 24 -
.../python/twitter/thermos/config/__init__.py | 0
.../twitter/thermos/config/bin/config_load.py | 47 -
.../twitter/thermos/config/bin/config_repl.py | 3 -
src/main/python/twitter/thermos/config/dsl.py | 0
.../python/twitter/thermos/config/loader.py | 179 -
.../python/twitter/thermos/config/schema.py | 2 -
.../twitter/thermos/config/schema_base.py | 75 -
.../twitter/thermos/config/schema_helpers.py | 247 -
src/main/python/twitter/thermos/core/BUILD | 88 -
.../python/twitter/thermos/core/__init__.py | 1 -
src/main/python/twitter/thermos/core/helper.py | 387 -
.../python/twitter/thermos/core/inspector.py | 105 -
src/main/python/twitter/thermos/core/muxer.py | 142 -
src/main/python/twitter/thermos/core/process.py | 371 -
src/main/python/twitter/thermos/core/runner.py | 905 --
.../python/twitter/thermos/monitoring/BUILD | 97 -
.../twitter/thermos/monitoring/__init__.py | 0
.../twitter/thermos/monitoring/detector.py | 91 -
.../python/twitter/thermos/monitoring/disk.py | 181 -
.../twitter/thermos/monitoring/garbage.py | 183 -
.../twitter/thermos/monitoring/monitor.py | 125 -
.../twitter/thermos/monitoring/process.py | 49 -
.../monitoring/process_collector_psutil.py | 92 -
.../twitter/thermos/monitoring/resource.py | 222 -
src/main/python/twitter/thermos/observer/BUILD | 51 -
.../python/twitter/thermos/observer/__init__.py | 0
.../python/twitter/thermos/observer/bin/BUILD | 14 -
.../twitter/thermos/observer/bin/__init__.py | 0
.../thermos/observer/bin/thermos_observer.py | 55 -
.../python/twitter/thermos/observer/http/BUILD | 47 -
.../twitter/thermos/observer/http/__init__.py | 0
.../thermos/observer/http/assets/bootstrap.css | 9 -
.../thermos/observer/http/assets/favicon.ico | Bin 145652 -> 0 bytes
.../thermos/observer/http/assets/jquery.js | 9404 ------------------
.../observer/http/assets/jquery.pailer.js | 317 -
.../observer/http/assets/mootools-core.js | 5789 -----------
.../thermos/observer/http/assets/observer.js | 202 -
.../thermos/observer/http/file_browser.py | 124 -
.../thermos/observer/http/http_observer.py | 133 -
.../twitter/thermos/observer/http/json.py | 56 -
.../thermos/observer/http/static_assets.py | 43 -
.../observer/http/templates/filebrowse.tpl | 81 -
.../observer/http/templates/filelist.tpl | 141 -
.../thermos/observer/http/templates/home.tpl | 58 -
.../thermos/observer/http/templates/index.tpl | 51 -
.../observer/http/templates/logbrowse.tpl | 80 -
.../thermos/observer/http/templates/main.tpl | 101 -
.../thermos/observer/http/templates/process.tpl | 109 -
.../thermos/observer/http/templates/rawtask.tpl | 28 -
.../thermos/observer/http/templates/task.tpl | 140 -
.../twitter/thermos/observer/http/templating.py | 9 -
.../twitter/thermos/observer/observed_task.py | 136 -
.../twitter/thermos/observer/task_observer.py | 640 --
src/main/python/twitter/thermos/testing/BUILD | 14 -
.../python/twitter/thermos/testing/__init__.py | 0
.../python/twitter/thermos/testing/runner.py | 191 -
.../assets/datatables/css/jquery.dataTables.css | 195 -
.../assets/datatables/images/back_disabled.png | Bin 1361 -> 0 bytes
.../assets/datatables/images/back_enabled.png | Bin 1379 -> 0 bytes
.../datatables/images/back_enabled_hover.png | Bin 1375 -> 0 bytes
.../datatables/images/forward_disabled.png | Bin 1363 -> 0 bytes
.../datatables/images/forward_enabled.png | Bin 1380 -> 0 bytes
.../datatables/images/forward_enabled_hover.png | Bin 1379 -> 0 bytes
.../http/assets/datatables/images/sort_asc.png | Bin 1118 -> 0 bytes
.../datatables/images/sort_asc_disabled.png | Bin 1050 -> 0 bytes
.../http/assets/datatables/images/sort_both.png | Bin 1136 -> 0 bytes
.../http/assets/datatables/images/sort_desc.png | Bin 1127 -> 0 bytes
.../datatables/images/sort_desc_disabled.png | Bin 1045 -> 0 bytes
.../datatables/js/dataTables.bootstrap.js | 165 -
.../datatables/js/dataTables.htmlNumberType.js | 67 -
.../datatables/js/dataTables.localstorage.js | 18 -
.../datatables/js/jquery.dataTables.min.js | 155 -
.../aurora/scheduler/http/assets/dictionary.js | 21 -
.../scheduler/http/assets/images/aurora.png | Bin 5874 -> 0 bytes
.../aurora/scheduler/http/assets/images/viz.png | Bin 14938 -> 0 bytes
.../aurora/scheduler/http/assets/util.js | 65 -
.../aurora/scheduler/http/schedulerzhome.st | 53 -
.../aurora/scheduler/http/schedulerzjob.st | 315 -
.../aurora/scheduler/http/schedulerzrole.st | 265 -
.../com/twitter/aurora/scheduler/http/slaves.st | 30 -
.../twitter/aurora/scheduler/http/structdump.st | 13 -
.../aurora/scheduler/http/utilization.st | 35 -
.../assets/datatables/css/jquery.dataTables.css | 195 +
.../assets/datatables/images/back_disabled.png | Bin 0 -> 1361 bytes
.../assets/datatables/images/back_enabled.png | Bin 0 -> 1379 bytes
.../datatables/images/back_enabled_hover.png | Bin 0 -> 1375 bytes
.../datatables/images/forward_disabled.png | Bin 0 -> 1363 bytes
.../datatables/images/forward_enabled.png | Bin 0 -> 1380 bytes
.../datatables/images/forward_enabled_hover.png | Bin 0 -> 1379 bytes
.../http/assets/datatables/images/sort_asc.png | Bin 0 -> 1118 bytes
.../datatables/images/sort_asc_disabled.png | Bin 0 -> 1050 bytes
.../http/assets/datatables/images/sort_both.png | Bin 0 -> 1136 bytes
.../http/assets/datatables/images/sort_desc.png | Bin 0 -> 1127 bytes
.../datatables/images/sort_desc_disabled.png | Bin 0 -> 1045 bytes
.../datatables/js/dataTables.bootstrap.js | 165 +
.../datatables/js/dataTables.htmlNumberType.js | 67 +
.../datatables/js/dataTables.localstorage.js | 18 +
.../datatables/js/jquery.dataTables.min.js | 155 +
.../aurora/scheduler/http/assets/dictionary.js | 21 +
.../scheduler/http/assets/images/aurora.png | Bin 0 -> 5874 bytes
.../aurora/scheduler/http/assets/images/viz.png | Bin 0 -> 14938 bytes
.../apache/aurora/scheduler/http/assets/util.js | 65 +
.../aurora/scheduler/http/schedulerzhome.st | 53 +
.../aurora/scheduler/http/schedulerzjob.st | 315 +
.../aurora/scheduler/http/schedulerzrole.st | 265 +
.../org/apache/aurora/scheduler/http/slaves.st | 30 +
.../apache/aurora/scheduler/http/structdump.st | 13 +
.../apache/aurora/scheduler/http/utilization.st | 35 +
src/main/thrift/com/twitter/aurora/gen/BUILD | 50 -
.../thrift/com/twitter/aurora/gen/api.thrift | 561 --
.../com/twitter/aurora/gen/internal_rpc.thrift | 66 -
.../com/twitter/aurora/gen/storage.thrift | 186 -
.../com/twitter/aurora/gen/storage_local.thrift | 23 -
.../thrift/com/twitter/aurora/gen/test.thrift | 258 -
src/main/thrift/com/twitter/thermos/BUILD | 11 -
.../com/twitter/thermos/thermos_internal.thrift | 84 -
src/main/thrift/org/apache/aurora/gen/BUILD | 50 +
.../thrift/org/apache/aurora/gen/api.thrift | 561 ++
.../org/apache/aurora/gen/internal_rpc.thrift | 66 +
.../thrift/org/apache/aurora/gen/storage.thrift | 186 +
.../org/apache/aurora/gen/storage_local.thrift | 23 +
.../thrift/org/apache/aurora/gen/test.thrift | 258 +
src/main/thrift/org/apache/thermos/BUILD | 11 +
.../org/apache/thermos/thermos_internal.thrift | 84 +
.../com/twitter/aurora/scheduler/http/BUILD | 4 -
.../root/checkpoints/failure/coordinator.p1 | Bin 162 -> 0 bytes
.../root/checkpoints/failure/coordinator.p2 | Bin 324 -> 0 bytes
.../root/checkpoints/failure/coordinator.p3 | Bin 486 -> 0 bytes
.../thermos/root/checkpoints/failure/runner | Bin 1383 -> 0 bytes
.../checkpoints/failure_limit/coordinator.a | Bin 159 -> 0 bytes
.../checkpoints/failure_limit/coordinator.b | Bin 159 -> 0 bytes
.../root/checkpoints/failure_limit/runner | Bin 589 -> 0 bytes
.../checkpoints/hello_world/coordinator.echo | Bin 168 -> 0 bytes
.../thermos/root/checkpoints/hello_world/runner | Bin 405 -> 0 bytes
.../checkpoints/ordering/coordinator.echo_one | Bin 180 -> 0 bytes
.../checkpoints/ordering/coordinator.echo_three | Bin 186 -> 0 bytes
.../checkpoints/ordering/coordinator.echo_two | Bin 180 -> 0 bytes
.../thermos/root/checkpoints/ordering/runner | Bin 870 -> 0 bytes
.../root/checkpoints/ports/coordinator.echo | Bin 168 -> 0 bytes
.../thermos/root/checkpoints/ports/runner | Bin 433 -> 0 bytes
.../checkpoints/sleep60-lost/coordinator.sleep | Bin 114 -> 0 bytes
.../root/checkpoints/sleep60-lost/runner | Bin 352 -> 0 bytes
.../root/checkpoints/sleep60/coordinator.sleep | Bin 114 -> 0 bytes
.../thermos/root/checkpoints/sleep60/runner | Bin 446 -> 0 bytes
.../thermos/root/logs/failure/p1/0/stderr | 0
.../thermos/root/logs/failure/p1/0/stdout | 2 -
.../thermos/root/logs/failure/p2/0/stderr | 0
.../thermos/root/logs/failure/p2/0/stdout | 2 -
.../thermos/root/logs/failure/p2/1/stderr | 0
.../thermos/root/logs/failure/p2/1/stdout | 2 -
.../thermos/root/logs/failure/p3/0/stderr | 0
.../thermos/root/logs/failure/p3/0/stdout | 2 -
.../thermos/root/logs/failure/p3/1/stderr | 0
.../thermos/root/logs/failure/p3/1/stdout | 2 -
.../thermos/root/logs/failure/p3/2/stderr | 0
.../thermos/root/logs/failure/p3/2/stdout | 2 -
.../thermos/root/logs/failure_limit/a/0/stderr | 0
.../thermos/root/logs/failure_limit/a/0/stdout | 1 -
.../thermos/root/logs/failure_limit/b/0/stderr | 0
.../thermos/root/logs/failure_limit/b/0/stdout | 0
.../thermos/root/logs/hello_world/echo/0/stderr | 0
.../thermos/root/logs/hello_world/echo/0/stdout | 1 -
.../root/logs/ordering/echo_one/0/stderr | 0
.../root/logs/ordering/echo_one/0/stdout | 1 -
.../root/logs/ordering/echo_three/0/stderr | 0
.../root/logs/ordering/echo_three/0/stdout | 1 -
.../root/logs/ordering/echo_two/0/stderr | 0
.../root/logs/ordering/echo_two/0/stdout | 1 -
.../thermos/root/logs/ports/echo/0/stderr | 0
.../thermos/root/logs/ports/echo/0/stdout | 1 -
.../root/logs/sleep60-lost/sleep/0/stderr | 0
.../root/logs/sleep60-lost/sleep/0/stdout | 0
.../thermos/root/logs/sleep60/sleep/0/stderr | 0
.../thermos/root/logs/sleep60/sleep/0/stdout | 0
.../com/twitter/thermos/root/sandbox/p1 | 1 -
.../com/twitter/thermos/root/sandbox/p2 | 2 -
.../com/twitter/thermos/root/sandbox/p3 | 3 -
.../thermos/root/tasks/active/sleep60-lost | 1 -
.../twitter/thermos/root/tasks/finished/failure | 1 -
.../thermos/root/tasks/finished/failure_limit | 1 -
.../thermos/root/tasks/finished/hello_world | 1 -
.../thermos/root/tasks/finished/ordering | 1 -
.../twitter/thermos/root/tasks/finished/ports | 1 -
.../twitter/thermos/root/tasks/finished/sleep60 | 1 -
.../org/apache/aurora/scheduler/http/BUILD | 4 +
.../root/checkpoints/failure/coordinator.p1 | Bin 0 -> 162 bytes
.../root/checkpoints/failure/coordinator.p2 | Bin 0 -> 324 bytes
.../root/checkpoints/failure/coordinator.p3 | Bin 0 -> 486 bytes
.../thermos/root/checkpoints/failure/runner | Bin 0 -> 1383 bytes
.../checkpoints/failure_limit/coordinator.a | Bin 0 -> 159 bytes
.../checkpoints/failure_limit/coordinator.b | Bin 0 -> 159 bytes
.../root/checkpoints/failure_limit/runner | Bin 0 -> 589 bytes
.../checkpoints/hello_world/coordinator.echo | Bin 0 -> 168 bytes
.../thermos/root/checkpoints/hello_world/runner | Bin 0 -> 405 bytes
.../checkpoints/ordering/coordinator.echo_one | Bin 0 -> 180 bytes
.../checkpoints/ordering/coordinator.echo_three | Bin 0 -> 186 bytes
.../checkpoints/ordering/coordinator.echo_two | Bin 0 -> 180 bytes
.../thermos/root/checkpoints/ordering/runner | Bin 0 -> 870 bytes
.../root/checkpoints/ports/coordinator.echo | Bin 0 -> 168 bytes
.../thermos/root/checkpoints/ports/runner | Bin 0 -> 433 bytes
.../checkpoints/sleep60-lost/coordinator.sleep | Bin 0 -> 114 bytes
.../root/checkpoints/sleep60-lost/runner | Bin 0 -> 352 bytes
.../root/checkpoints/sleep60/coordinator.sleep | Bin 0 -> 114 bytes
.../thermos/root/checkpoints/sleep60/runner | Bin 0 -> 446 bytes
.../thermos/root/logs/failure/p1/0/stderr | 0
.../thermos/root/logs/failure/p1/0/stdout | 2 +
.../thermos/root/logs/failure/p2/0/stderr | 0
.../thermos/root/logs/failure/p2/0/stdout | 2 +
.../thermos/root/logs/failure/p2/1/stderr | 0
.../thermos/root/logs/failure/p2/1/stdout | 2 +
.../thermos/root/logs/failure/p3/0/stderr | 0
.../thermos/root/logs/failure/p3/0/stdout | 2 +
.../thermos/root/logs/failure/p3/1/stderr | 0
.../thermos/root/logs/failure/p3/1/stdout | 2 +
.../thermos/root/logs/failure/p3/2/stderr | 0
.../thermos/root/logs/failure/p3/2/stdout | 2 +
.../thermos/root/logs/failure_limit/a/0/stderr | 0
.../thermos/root/logs/failure_limit/a/0/stdout | 1 +
.../thermos/root/logs/failure_limit/b/0/stderr | 0
.../thermos/root/logs/failure_limit/b/0/stdout | 0
.../thermos/root/logs/hello_world/echo/0/stderr | 0
.../thermos/root/logs/hello_world/echo/0/stdout | 1 +
.../root/logs/ordering/echo_one/0/stderr | 0
.../root/logs/ordering/echo_one/0/stdout | 1 +
.../root/logs/ordering/echo_three/0/stderr | 0
.../root/logs/ordering/echo_three/0/stdout | 1 +
.../root/logs/ordering/echo_two/0/stderr | 0
.../root/logs/ordering/echo_two/0/stdout | 1 +
.../thermos/root/logs/ports/echo/0/stderr | 0
.../thermos/root/logs/ports/echo/0/stdout | 1 +
.../root/logs/sleep60-lost/sleep/0/stderr | 0
.../root/logs/sleep60-lost/sleep/0/stdout | 0
.../thermos/root/logs/sleep60/sleep/0/stderr | 0
.../thermos/root/logs/sleep60/sleep/0/stdout | 0
.../org/apache/thermos/root/sandbox/p1 | 1 +
.../org/apache/thermos/root/sandbox/p2 | 2 +
.../org/apache/thermos/root/sandbox/p3 | 3 +
.../thermos/root/tasks/active/sleep60-lost | 1 +
.../apache/thermos/root/tasks/finished/failure | 1 +
.../thermos/root/tasks/finished/failure_limit | 1 +
.../thermos/root/tasks/finished/hello_world | 1 +
.../apache/thermos/root/tasks/finished/ordering | 1 +
.../apache/thermos/root/tasks/finished/ports | 1 +
.../apache/thermos/root/tasks/finished/sleep60 | 1 +
.../java/com/twitter/aurora/GuiceUtilsTest.java | 138 -
.../aurora/codec/ThriftBinaryCodecTest.java | 56 -
.../aurora/scheduler/DriverFactoryImplTest.java | 68 -
.../twitter/aurora/scheduler/DriverTest.java | 116 -
.../scheduler/MesosSchedulerImplTest.java | 338 -
.../scheduler/MesosTaskFactoryImplTest.java | 85 -
.../scheduler/SchedulerLifecycleTest.java | 138 -
.../twitter/aurora/scheduler/TaskVarsTest.java | 241 -
.../aurora/scheduler/UserTaskLauncherTest.java | 209 -
.../aurora/scheduler/app/SchedulerIT.java | 355 -
.../scheduler/async/HistoryPrunerTest.java | 431 -
.../scheduler/async/OfferQueueImplTest.java | 155 -
.../twitter/aurora/scheduler/async/Offers.java | 45 -
.../scheduler/async/PreemptorImplTest.java | 587 --
.../scheduler/async/TaskSchedulerImplTest.java | 277 -
.../scheduler/async/TaskSchedulerTest.java | 747 --
.../aurora/scheduler/async/TaskTimeoutTest.java | 322 -
.../aurora/scheduler/base/CommandUtilTest.java | 53 -
.../aurora/scheduler/base/NumbersTest.java | 44 -
.../configuration/ConfigurationManagerTest.java | 104 -
.../scheduler/configuration/ResourcesTest.java | 180 -
.../aurora/scheduler/cron/noop/NoopCronIT.java | 63 -
.../events/NotifyingMethodInterceptorTest.java | 102 -
.../events/NotifyingSchedulingFilterTest.java | 82 -
.../filter/SchedulingFilterImplTest.java | 621 --
.../scheduler/http/LeaderRedirectTest.java | 149 -
.../aurora/scheduler/http/MnameTest.java | 46 -
.../scheduler/http/SchedulerzRoleTest.java | 82 -
.../scheduler/log/mesos/MesosLogTest.java | 64 -
.../scheduler/log/testing/FileLogTest.java | 58 -
.../scheduler/metadata/NearestFitTest.java | 136 -
.../periodic/GcExecutorLauncherTest.java | 180 -
.../quota/QuotaComparisonResultTest.java | 90 -
.../aurora/scheduler/quota/QuotaFilterTest.java | 190 -
.../scheduler/quota/QuotaManagerImplTest.java | 204 -
.../state/BaseSchedulerCoreImplTest.java | 1472 ---
.../scheduler/state/CronJobManagerTest.java | 358 -
.../scheduler/state/LockManagerImplTest.java | 140 -
.../state/MaintenanceControllerImplTest.java | 224 -
.../aurora/scheduler/state/PubsubTestUtil.java | 69 -
.../scheduler/state/StateManagerImplTest.java | 300 -
.../scheduler/state/TaskStateMachineTest.java | 348 -
.../scheduler/stats/SlotSizeCounterTest.java | 123 -
.../storage/ReadWriteLockManagerTest.java | 109 -
.../scheduler/storage/StorageBackfillTest.java | 78 -
.../scheduler/storage/backup/RecoveryTest.java | 181 -
.../storage/backup/StorageBackupTest.java | 183 -
.../scheduler/storage/log/LogManagerTest.java | 550 -
.../scheduler/storage/log/LogStorageTest.java | 689 --
.../storage/log/SnapshotStoreImplTest.java | 135 -
.../scheduler/storage/mem/InternerTest.java | 122 -
.../scheduler/storage/mem/MemJobStoreTest.java | 108 -
.../scheduler/storage/mem/MemLockStoreTest.java | 80 -
.../mem/MemStorageSchedulerCoreImplTest.java | 27 -
.../scheduler/storage/mem/MemStorageTest.java | 214 -
.../scheduler/storage/mem/MemTaskStoreTest.java | 276 -
.../thrift/SchedulerThriftInterfaceTest.java | 1403 ---
.../aurora/scheduler/thrift/ThriftIT.java | 178 -
.../thrift/aop/APIVersionInterceptorTest.java | 80 -
.../scheduler/thrift/aop/AopModuleTest.java | 111 -
.../aop/FeatureToggleInterceptorTest.java | 87 -
.../scheduler/thrift/aop/ForwardingThrift.java | 232 -
.../thrift/aop/MockDecoratedThrift.java | 54 -
.../aop/ThriftStatsExporterInterceptorTest.java | 82 -
.../java/org/apache/aurora/GuiceUtilsTest.java | 138 +
.../aurora/codec/ThriftBinaryCodecTest.java | 56 +
.../aurora/scheduler/DriverFactoryImplTest.java | 68 +
.../org/apache/aurora/scheduler/DriverTest.java | 116 +
.../scheduler/MesosSchedulerImplTest.java | 338 +
.../scheduler/MesosTaskFactoryImplTest.java | 85 +
.../scheduler/SchedulerLifecycleTest.java | 138 +
.../apache/aurora/scheduler/TaskVarsTest.java | 241 +
.../aurora/scheduler/UserTaskLauncherTest.java | 209 +
.../aurora/scheduler/app/SchedulerIT.java | 355 +
.../scheduler/async/HistoryPrunerTest.java | 431 +
.../scheduler/async/OfferQueueImplTest.java | 155 +
.../apache/aurora/scheduler/async/Offers.java | 45 +
.../scheduler/async/PreemptorImplTest.java | 587 ++
.../scheduler/async/TaskSchedulerImplTest.java | 277 +
.../scheduler/async/TaskSchedulerTest.java | 747 ++
.../aurora/scheduler/async/TaskTimeoutTest.java | 322 +
.../aurora/scheduler/base/CommandUtilTest.java | 53 +
.../aurora/scheduler/base/NumbersTest.java | 44 +
.../configuration/ConfigurationManagerTest.java | 104 +
.../scheduler/configuration/ResourcesTest.java | 180 +
.../aurora/scheduler/cron/noop/NoopCronIT.java | 63 +
.../events/NotifyingMethodInterceptorTest.java | 102 +
.../events/NotifyingSchedulingFilterTest.java | 82 +
.../filter/SchedulingFilterImplTest.java | 621 ++
.../scheduler/http/LeaderRedirectTest.java | 149 +
.../apache/aurora/scheduler/http/MnameTest.java | 46 +
.../scheduler/http/SchedulerzRoleTest.java | 82 +
.../scheduler/log/mesos/MesosLogTest.java | 64 +
.../scheduler/log/testing/FileLogTest.java | 58 +
.../scheduler/metadata/NearestFitTest.java | 136 +
.../periodic/GcExecutorLauncherTest.java | 180 +
.../quota/QuotaComparisonResultTest.java | 90 +
.../aurora/scheduler/quota/QuotaFilterTest.java | 190 +
.../scheduler/quota/QuotaManagerImplTest.java | 204 +
.../state/BaseSchedulerCoreImplTest.java | 1472 +++
.../scheduler/state/CronJobManagerTest.java | 358 +
.../scheduler/state/LockManagerImplTest.java | 140 +
.../state/MaintenanceControllerImplTest.java | 224 +
.../aurora/scheduler/state/PubsubTestUtil.java | 69 +
.../scheduler/state/StateManagerImplTest.java | 300 +
.../scheduler/state/TaskStateMachineTest.java | 348 +
.../scheduler/stats/SlotSizeCounterTest.java | 123 +
.../storage/ReadWriteLockManagerTest.java | 109 +
.../scheduler/storage/StorageBackfillTest.java | 78 +
.../scheduler/storage/backup/RecoveryTest.java | 181 +
.../storage/backup/StorageBackupTest.java | 183 +
.../scheduler/storage/log/LogManagerTest.java | 550 +
.../scheduler/storage/log/LogStorageTest.java | 689 ++
.../storage/log/SnapshotStoreImplTest.java | 135 +
.../scheduler/storage/mem/InternerTest.java | 122 +
.../scheduler/storage/mem/MemJobStoreTest.java | 108 +
.../scheduler/storage/mem/MemLockStoreTest.java | 80 +
.../mem/MemStorageSchedulerCoreImplTest.java | 27 +
.../scheduler/storage/mem/MemStorageTest.java | 214 +
.../scheduler/storage/mem/MemTaskStoreTest.java | 276 +
.../thrift/SchedulerThriftInterfaceTest.java | 1403 +++
.../aurora/scheduler/thrift/ThriftIT.java | 178 +
.../thrift/aop/APIVersionInterceptorTest.java | 80 +
.../scheduler/thrift/aop/AopModuleTest.java | 111 +
.../aop/FeatureToggleInterceptorTest.java | 87 +
.../scheduler/thrift/aop/ForwardingThrift.java | 232 +
.../thrift/aop/MockDecoratedThrift.java | 54 +
.../aop/ThriftStatsExporterInterceptorTest.java | 82 +
src/test/python/apache/aurora/BUILD | 10 +
src/test/python/apache/aurora/admin/BUILD | 15 +
.../aurora/admin/test_mesos_maintenance.py | 87 +
src/test/python/apache/aurora/client/BUILD | 36 +
.../python/apache/aurora/client/__init__.py | 0
src/test/python/apache/aurora/client/api/BUILD | 70 +
.../aurora/client/api/test_disambiguator.py | 111 +
.../aurora/client/api/test_health_check.py | 174 +
.../aurora/client/api/test_instance_watcher.py | 188 +
.../aurora/client/api/test_job_monitor.py | 31 +
.../apache/aurora/client/api/test_restarter.py | 136 +
.../aurora/client/api/test_scheduler_client.py | 260 +
.../apache/aurora/client/api/test_updater.py | 672 ++
src/test/python/apache/aurora/client/cli/BUILD | 26 +
.../apache/aurora/client/cli/test_create.py | 200 +
.../apache/aurora/client/cli/test_kill.py | 114 +
.../python/apache/aurora/client/cli/util.py | 152 +
.../python/apache/aurora/client/commands/BUILD | 60 +
.../client/commands/test_cancel_update.py | 144 +
.../aurora/client/commands/test_create.py | 298 +
.../apache/aurora/client/commands/test_diff.py | 183 +
.../apache/aurora/client/commands/test_kill.py | 180 +
.../aurora/client/commands/test_listjobs.py | 71 +
.../aurora/client/commands/test_restart.py | 169 +
.../apache/aurora/client/commands/test_run.py | 116 +
.../apache/aurora/client/commands/test_ssh.py | 103 +
.../aurora/client/commands/test_status.py | 109 +
.../aurora/client/commands/test_update.py | 264 +
.../apache/aurora/client/commands/util.py | 116 +
.../aurora/client/fake_scheduler_proxy.py | 14 +
.../python/apache/aurora/client/hooks/BUILD | 24 +
.../aurora/client/hooks/test_hooked_api.py | 57 +
.../aurora/client/hooks/test_non_hooked_api.py | 92 +
.../apache/aurora/client/test_binding_helper.py | 132 +
.../python/apache/aurora/client/test_config.py | 152 +
src/test/python/apache/aurora/common/BUILD | 55 +
.../apache/aurora/common/test_aurora_job_key.py | 8 +
.../python/apache/aurora/common/test_cluster.py | 32 +
.../apache/aurora/common/test_cluster_option.py | 48 +
.../apache/aurora/common/test_clusters.py | 109 +
.../apache/aurora/common/test_http_signaler.py | 63 +
src/test/python/apache/aurora/config/BUILD | 43 +
.../python/apache/aurora/config/test_base.py | 196 +
.../aurora/config/test_constraint_parsing.py | 54 +
.../python/apache/aurora/config/test_loader.py | 90 +
.../python/apache/aurora/config/test_thrift.py | 216 +
src/test/python/apache/aurora/executor/BUILD | 86 +
.../python/apache/aurora/executor/common/BUILD | 47 +
.../executor/common/test_directory_sandbox.py | 42 +
.../executor/common/test_health_checker.py | 93 +
.../executor/common/test_status_checker.py | 60 +
.../aurora/executor/common/test_task_info.py | 48 +
.../aurora/executor/test_executor_builds.py | 21 +
.../aurora/executor/test_executor_detector.py | 66 +
.../aurora/executor/test_executor_vars.py | 40 +
.../apache/aurora/executor/test_gc_executor.py | 636 ++
.../aurora/executor/test_thermos_executor.py | 522 +
.../aurora/executor/test_thermos_task_runner.py | 168 +
src/test/python/apache/thermos/BUILD | 9 +
src/test/python/apache/thermos/common/BUILD | 32 +
.../apache/thermos/common/test_pathspec.py | 17 +
.../apache/thermos/common/test_planner.py | 70 +
.../apache/thermos/common/test_task_planner.py | 316 +
src/test/python/apache/thermos/config/BUILD | 13 +
.../python/apache/thermos/config/test_schema.py | 147 +
src/test/python/apache/thermos/core/BUILD | 101 +
.../python/apache/thermos/core/test_angry.py | 37 +
.../apache/thermos/core/test_ephemerals.py | 33 +
.../apache/thermos/core/test_failing_runner.py | 45 +
.../apache/thermos/core/test_failure_limit.py | 64 +
.../apache/thermos/core/test_finalization.py | 116 +
.../python/apache/thermos/core/test_helper.py | 98 +
.../python/apache/thermos/core/test_process.py | 158 +
.../thermos/core/test_runner_integration.py | 166 +
.../apache/thermos/core/test_staged_kill.py | 283 +
src/test/python/apache/thermos/monitoring/BUILD | 13 +
.../apache/thermos/monitoring/test_disk.py | 68 +
src/test/python/twitter/aurora/BUILD | 10 -
src/test/python/twitter/aurora/admin/BUILD | 15 -
.../aurora/admin/test_mesos_maintenance.py | 87 -
src/test/python/twitter/aurora/client/BUILD | 36 -
.../python/twitter/aurora/client/__init__.py | 0
src/test/python/twitter/aurora/client/api/BUILD | 70 -
.../aurora/client/api/test_disambiguator.py | 111 -
.../aurora/client/api/test_health_check.py | 174 -
.../aurora/client/api/test_instance_watcher.py | 188 -
.../aurora/client/api/test_job_monitor.py | 31 -
.../twitter/aurora/client/api/test_restarter.py | 136 -
.../aurora/client/api/test_scheduler_client.py | 260 -
.../twitter/aurora/client/api/test_updater.py | 672 --
src/test/python/twitter/aurora/client/cli/BUILD | 26 -
.../twitter/aurora/client/cli/test_create.py | 200 -
.../twitter/aurora/client/cli/test_kill.py | 114 -
.../python/twitter/aurora/client/cli/util.py | 152 -
.../python/twitter/aurora/client/commands/BUILD | 60 -
.../client/commands/test_cancel_update.py | 144 -
.../aurora/client/commands/test_create.py | 298 -
.../twitter/aurora/client/commands/test_diff.py | 183 -
.../twitter/aurora/client/commands/test_kill.py | 180 -
.../aurora/client/commands/test_listjobs.py | 71 -
.../aurora/client/commands/test_restart.py | 169 -
.../twitter/aurora/client/commands/test_run.py | 116 -
.../twitter/aurora/client/commands/test_ssh.py | 103 -
.../aurora/client/commands/test_status.py | 109 -
.../aurora/client/commands/test_update.py | 264 -
.../twitter/aurora/client/commands/util.py | 116 -
.../aurora/client/fake_scheduler_proxy.py | 14 -
.../python/twitter/aurora/client/hooks/BUILD | 24 -
.../aurora/client/hooks/test_hooked_api.py | 57 -
.../aurora/client/hooks/test_non_hooked_api.py | 92 -
.../aurora/client/test_binding_helper.py | 132 -
.../python/twitter/aurora/client/test_config.py | 152 -
src/test/python/twitter/aurora/common/BUILD | 55 -
.../aurora/common/test_aurora_job_key.py | 8 -
.../twitter/aurora/common/test_cluster.py | 32 -
.../aurora/common/test_cluster_option.py | 48 -
.../twitter/aurora/common/test_clusters.py | 109 -
.../twitter/aurora/common/test_http_signaler.py | 63 -
src/test/python/twitter/aurora/config/BUILD | 43 -
.../python/twitter/aurora/config/test_base.py | 196 -
.../aurora/config/test_constraint_parsing.py | 54 -
.../python/twitter/aurora/config/test_loader.py | 90 -
.../python/twitter/aurora/config/test_thrift.py | 216 -
src/test/python/twitter/aurora/executor/BUILD | 86 -
.../python/twitter/aurora/executor/common/BUILD | 47 -
.../executor/common/test_directory_sandbox.py | 42 -
.../executor/common/test_health_checker.py | 93 -
.../executor/common/test_status_checker.py | 60 -
.../aurora/executor/common/test_task_info.py | 48 -
.../aurora/executor/test_executor_builds.py | 21 -
.../aurora/executor/test_executor_detector.py | 66 -
.../aurora/executor/test_executor_vars.py | 40 -
.../twitter/aurora/executor/test_gc_executor.py | 636 --
.../aurora/executor/test_thermos_executor.py | 522 -
.../aurora/executor/test_thermos_task_runner.py | 168 -
src/test/python/twitter/thermos/BUILD | 9 -
src/test/python/twitter/thermos/common/BUILD | 32 -
.../twitter/thermos/common/test_pathspec.py | 17 -
.../twitter/thermos/common/test_planner.py | 70 -
.../twitter/thermos/common/test_task_planner.py | 316 -
src/test/python/twitter/thermos/config/BUILD | 13 -
.../twitter/thermos/config/test_schema.py | 147 -
src/test/python/twitter/thermos/core/BUILD | 101 -
.../python/twitter/thermos/core/test_angry.py | 37 -
.../twitter/thermos/core/test_ephemerals.py | 33 -
.../twitter/thermos/core/test_failing_runner.py | 45 -
.../twitter/thermos/core/test_failure_limit.py | 64 -
.../twitter/thermos/core/test_finalization.py | 116 -
.../python/twitter/thermos/core/test_helper.py | 98 -
.../python/twitter/thermos/core/test_process.py | 158 -
.../thermos/core/test_runner_integration.py | 166 -
.../twitter/thermos/core/test_staged_kill.py | 283 -
.../python/twitter/thermos/monitoring/BUILD | 13 -
.../twitter/thermos/monitoring/test_disk.py | 68 -
.../aurora/scheduler/app/AuroraTestKeyStore | Bin 1267 -> 0 bytes
.../aurora/scheduler/app/AuroraTestKeyStore | Bin 0 -> 1267 bytes
1124 files changed, 84483 insertions(+), 84483 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/GuiceUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/GuiceUtils.java b/src/main/java/com/twitter/aurora/GuiceUtils.java
deleted file mode 100644
index 207535d..0000000
--- a/src/main/java/com/twitter/aurora/GuiceUtils.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.lang.reflect.Method;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.inject.Binder;
-import com.google.inject.BindingAnnotation;
-import com.google.inject.matcher.AbstractMatcher;
-import com.google.inject.matcher.Matcher;
-import com.google.inject.matcher.Matchers;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-
-import com.twitter.common.collections.Pair;
-
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * Utilities for guice configuration in aurora.
- */
-public final class GuiceUtils {
-
- private static final Logger LOG = Logger.getLogger(GuiceUtils.class.getName());
-
- // Method annotation that allows a trapped interface to whitelist methods that may throw
- // unchecked exceptions.
- @BindingAnnotation
- @Target(METHOD) @Retention(RUNTIME)
- public @interface AllowUnchecked { }
-
- private GuiceUtils() {
- // utility
- }
-
- // No wildcards on the Class here because it upsets checkstyle - complains with:
- // '>' is followed by whitespace.
- private static final Function<Method, Pair<String, Class[]>> CANONICALIZE =
- new Function<Method, Pair<String, Class[]>>() {
- @Override public Pair<String, Class[]> apply(Method method) {
- return Pair.of(method.getName(), (Class[]) method.getParameterTypes());
- }
- };
-
- /**
- * Creates a matcher that will match methods of an interface, optionally excluding inherited
- * methods.
- *
- * @param matchInterface The interface to match.
- * @param declaredMethodsOnly if {@code true} only methods directly declared in the interface
- * will be matched, otherwise all methods on the interface are matched.
- * @return A new matcher instance.
- */
- public static Matcher<Method> interfaceMatcher(
- Class<?> matchInterface,
- boolean declaredMethodsOnly) {
-
- Method[] methods =
- declaredMethodsOnly ? matchInterface.getDeclaredMethods() : matchInterface.getMethods();
- final Set<Pair<String, Class[]>> interfaceMethods =
- ImmutableSet.copyOf(Iterables.transform(ImmutableList.copyOf(methods), CANONICALIZE));
- final LoadingCache<Method, Pair<String, Class[]>> cache = CacheBuilder.newBuilder()
- .build(CacheLoader.from(CANONICALIZE));
-
- return new AbstractMatcher<Method>() {
- @Override public boolean matches(Method method) {
- return interfaceMethods.contains(cache.getUnchecked(method));
- }
- };
- }
-
- /**
- * Binds an interceptor that ensures the main ClassLoader is bound as the thread context
- * {@link ClassLoader} during JNI callbacks from mesos. Some libraries require a thread
- * context ClassLoader be set and this ensures those libraries work properly.
- *
- * @param binder The binder to use to register an interceptor with.
- * @param wrapInterface Interface whose methods should wrapped.
- */
- public static void bindJNIContextClassLoader(Binder binder, Class<?> wrapInterface) {
- final ClassLoader mainClassLoader = GuiceUtils.class.getClassLoader();
- binder.bindInterceptor(
- Matchers.subclassesOf(wrapInterface),
- interfaceMatcher(wrapInterface, false),
- new MethodInterceptor() {
- @Override public Object invoke(MethodInvocation invocation) throws Throwable {
- Thread currentThread = Thread.currentThread();
- ClassLoader prior = currentThread.getContextClassLoader();
- try {
- currentThread.setContextClassLoader(mainClassLoader);
- return invocation.proceed();
- } finally {
- currentThread.setContextClassLoader(prior);
- }
- }
- });
- }
-
- private static final Predicate<Method> IS_WHITELISTED = new Predicate<Method>() {
- @Override public boolean apply(Method method) {
- return method.getAnnotation(AllowUnchecked.class) != null;
- }
- };
-
- private static final Matcher<Method> WHITELIST_MATCHER = new AbstractMatcher<Method>() {
- @Override public boolean matches(Method method) {
- return IS_WHITELISTED.apply(method);
- }
- };
-
- private static final Predicate<Method> VOID_METHOD = new Predicate<Method>() {
- @Override public boolean apply(Method method) {
- return method.getReturnType() == Void.TYPE;
- }
- };
-
- /**
- * Binds an exception trap on all interface methods of all classes bound against an interface.
- * Individual methods may opt out of trapping by annotating with {@link AllowUnchecked}.
- * Only void methods are allowed, any non-void interface methods must explicitly opt out.
- *
- * @param binder The binder to register an interceptor with.
- * @param wrapInterface Interface whose methods should be wrapped.
- * @throws IllegalArgumentException If any of the non-whitelisted interface methods are non-void.
- */
- public static void bindExceptionTrap(Binder binder, Class<?> wrapInterface)
- throws IllegalArgumentException {
-
- Set<Method> disallowed = ImmutableSet.copyOf(Iterables.filter(
- ImmutableList.copyOf(wrapInterface.getMethods()),
- Predicates.and(Predicates.not(IS_WHITELISTED), Predicates.not(VOID_METHOD))));
- Preconditions.checkArgument(disallowed.isEmpty(),
- "Non-void methods must be explicitly whitelisted with @AllowUnchecked: " + disallowed);
-
- Matcher<Method> matcher =
- Matchers.<Method>not(WHITELIST_MATCHER).and(interfaceMatcher(wrapInterface, false));
- binder.bindInterceptor(Matchers.subclassesOf(wrapInterface), matcher,
- new MethodInterceptor() {
- @Override public Object invoke(MethodInvocation invocation) throws Throwable {
- try {
- return invocation.proceed();
- } catch (RuntimeException e) {
- LOG.log(Level.WARNING, "Trapped uncaught exception: " + e, e);
- return null;
- }
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/Protobufs.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/Protobufs.java b/src/main/java/com/twitter/aurora/Protobufs.java
deleted file mode 100644
index c3ddea4..0000000
--- a/src/main/java/com/twitter/aurora/Protobufs.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora;
-
-import com.google.common.base.Function;
-import com.google.protobuf.Message;
-import com.google.protobuf.TextFormat;
-
-/**
- * Utility functions that are useful for working with protocol buffer messages.
- */
-public final class Protobufs {
-
- private Protobufs() {
- // Utility class.
- }
-
- /**
- * Function to call {@link #toString(Message)} on message objects.
- */
- public static final Function<Message, String> SHORT_TOSTRING = new Function<Message, String>() {
- @Override public String apply(Message message) {
- return Protobufs.toString(message);
- }
- };
-
- /**
- * Alternative to the default protobuf toString implementation, which omits newlines.
- *
- * @param message Message to print.
- * @return String representation of the message.
- */
- public static String toString(Message message) {
- return TextFormat.shortDebugString(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/auth/CapabilityValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/auth/CapabilityValidator.java b/src/main/java/com/twitter/aurora/auth/CapabilityValidator.java
deleted file mode 100644
index 05958f0..0000000
--- a/src/main/java/com/twitter/aurora/auth/CapabilityValidator.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.auth;
-
-import com.twitter.aurora.gen.SessionKey;
-
-/**
- * A session validator that supports user capability matching.
- * <p>
- * This supports asking whether a user has been granted a specific administration capability.
- */
-public interface CapabilityValidator extends SessionValidator {
-
- enum Capability {
- ROOT,
- PROVISIONER
- }
-
- /**
- * Audit check to perform for a given action.
- */
- enum AuditCheck {
- /**
- * Default. No audit checks will be performed.
- */
- NONE,
-
- /**
- * A check will be performed to verify if a given action has
- * all necessary data to generate a valid audit trail.
- */
- REQUIRED
- }
-
- /**
- * Checks whether a session key is authenticated, and has the specified capability.
- *
- * @param sessionKey Key to validate.
- * @param capability User capability to authenticate against.
- * @param check Auditing data presence check required.
- * @return A {@link SessionContext} object that provides information about the validated session.
- * @throws AuthFailedException If the key cannot be validated as the role or lacks
- * the requested capability.
- */
- SessionContext checkAuthorized(SessionKey sessionKey, Capability capability, AuditCheck check)
- throws AuthFailedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/auth/SessionValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/auth/SessionValidator.java b/src/main/java/com/twitter/aurora/auth/SessionValidator.java
deleted file mode 100644
index 327c5ca..0000000
--- a/src/main/java/com/twitter/aurora/auth/SessionValidator.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.auth;
-
-import java.util.Set;
-
-import com.twitter.aurora.gen.SessionKey;
-
-/**
- * Validator for RPC sessions with Aurora.
- */
-public interface SessionValidator {
-
- /**
- * Checks whether a session key is authenticated, and has permission to act as all the roles
- * provided. Authentication is successful only if the SessionKey is successfully validated against
- * all the roles.
- *
- * @param sessionKey Key to validate.
- * @param targetRoles A set of roles to validate the key against.
- * @return A {@link SessionContext} object that provides information about the validated session.
- * @throws AuthFailedException If the key cannot be validated against a role.
- */
- SessionContext checkAuthenticated(SessionKey sessionKey, Set<String> targetRoles)
- throws AuthFailedException;
-
- /**
- * Translates a {@link SessionKey} to a string. Primarily provides a way for the binary data
- * within a {@link SessionKey} to be decoded and converted into a string.
- *
- * @param sessionKey The session key to translate.
- * @return A string representation of the {@link SessionKey}.
- */
- String toString(SessionKey sessionKey);
-
- /**
- * Provides information about a session.
- */
- interface SessionContext {
-
- /**
- * Provides the identity for a validated session.
- *
- * @return A string that identifies the session.
- */
- String getIdentity();
- }
-
- /**
- * Thrown when authentication is not successful.
- */
- public static class AuthFailedException extends Exception {
- public AuthFailedException(String msg) {
- super(msg);
- }
-
- public AuthFailedException(String msg, Throwable cause) {
- super(msg, cause);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/auth/UnsecureAuthModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/auth/UnsecureAuthModule.java b/src/main/java/com/twitter/aurora/auth/UnsecureAuthModule.java
deleted file mode 100644
index 8fe9953..0000000
--- a/src/main/java/com/twitter/aurora/auth/UnsecureAuthModule.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.auth;
-
-import java.util.Set;
-import java.util.logging.Logger;
-
-import com.google.inject.AbstractModule;
-
-import com.twitter.aurora.gen.SessionKey;
-
-/**
- * An authentication module that uses an {@link UnsecureSessionValidator}. This behavior
- * can be overridden by binding a secure validator, querying an internal authentication system,
- * to {@link SessionValidator}.
- */
-public class UnsecureAuthModule extends AbstractModule {
- private static final String UNSECURE = "UNSECURE";
- private static final Logger LOG = Logger.getLogger(UnsecureAuthModule.class.getName());
-
- @Override
- protected void configure() {
- LOG.info("Using default (UNSECURE!!!) authentication module.");
- bind(SessionValidator.class).to(UnsecureSessionValidator.class);
- bind(CapabilityValidator.class).to(UnsecureCapabilityValidator.class);
- }
-
- static class UnsecureSessionValidator implements SessionValidator {
- @Override
- public SessionContext checkAuthenticated(SessionKey key, Set<String> targetRoles)
- throws AuthFailedException {
-
- return new SessionContext() {
- @Override public String getIdentity() {
- return UNSECURE;
- }
- };
- }
-
- @Override
- public String toString(SessionKey sessionKey) {
- return sessionKey.toString();
- }
- }
-
- static class UnsecureCapabilityValidator implements CapabilityValidator {
- @Override
- public SessionContext checkAuthorized(SessionKey key, Capability capability, AuditCheck check)
- throws AuthFailedException {
-
- return new SessionContext() {
- @Override public String getIdentity() {
- return UNSECURE;
- }
- };
- }
-
- @Override
- public SessionContext checkAuthenticated(SessionKey key, Set<String> targetRoles)
- throws AuthFailedException {
-
- return new SessionContext() {
- @Override public String getIdentity() {
- return UNSECURE;
- }
- };
- }
-
- @Override
- public String toString(SessionKey sessionKey) {
- return sessionKey.toString();
- }
- }
-}
[32/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/ImmediateJobManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/ImmediateJobManager.java b/src/main/java/org/apache/aurora/scheduler/state/ImmediateJobManager.java
new file mode 100644
index 0000000..a085d7d
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/ImmediateJobManager.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Job scheduler that accepts any job and executes it immediately.
+ */
+class ImmediateJobManager extends JobManager {
+
+ private static final Logger LOG = Logger.getLogger(ImmediateJobManager.class.getName());
+
+ private final StateManager stateManager;
+ private final Storage storage;
+
+ @Inject
+ ImmediateJobManager(StateManager stateManager, Storage storage) {
+ this.stateManager = checkNotNull(stateManager);
+ this.storage = checkNotNull(storage);
+ }
+
+ @Override
+ public String getUniqueKey() {
+ return "IMMEDIATE";
+ }
+
+ @Override
+ public boolean receiveJob(SanitizedConfiguration config) {
+ LOG.info("Launching " + config.getTaskConfigs().size() + " tasks.");
+ stateManager.insertPendingTasks(config.getTaskConfigs());
+ return true;
+ }
+
+ @Override
+ public boolean hasJob(final IJobKey jobKey) {
+ return !Storage.Util.consistentFetchTasks(storage, Query.jobScoped(jobKey).active()).isEmpty();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/JobFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/JobFilter.java b/src/main/java/org/apache/aurora/scheduler/state/JobFilter.java
new file mode 100644
index 0000000..3b8f5d8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/JobFilter.java
@@ -0,0 +1,119 @@
+package com.twitter.aurora.scheduler.state;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * An action that either accepts a configuration or rejects it with a reason.
+ */
+public interface JobFilter {
+ /**
+ * Accept the JobConfiguration or reject it with a reason.
+ *
+ * @param jobConfiguration The job configuration to filter.
+ * @return A result and the reason the result was reached.
+ */
+ JobFilterResult filter(IJobConfiguration jobConfiguration);
+
+ /**
+ * Accept the TaskConfig with the specified number of instances
+ * or reject it with a reason.
+ *
+ * @param template The task configuration to filter.
+ * @param instanceCount Number of instances to apply taskConfig to.
+ * @return A result and the reason the result was reached.
+ */
+ JobFilterResult filter(ITaskConfig template, int instanceCount);
+
+ /**
+ * An indication of whether a job passed a filter or not.
+ */
+ public static final class JobFilterResult {
+ private final boolean pass;
+ private final String reason;
+
+ private JobFilterResult(boolean pass, String reason) {
+ this.pass = pass;
+ this.reason = Preconditions.checkNotNull(reason);
+ }
+
+ /**
+ * Create a new result indicating the job has passed the filter.
+ *
+ * @return A result indicating the job has passed with a default reason.
+ * @see #fail(String)
+ */
+ public static JobFilterResult pass() {
+ return new JobFilterResult(true, "Accepted by filter.");
+ }
+
+ /**
+ * Create a new result indicating the job has passed the filter.
+ *
+ * @param reason A reason that the job has passed.
+ * @return A result indicating the job has passed because of the given reason.
+ * @throws NullPointerException if reason is {@code null}.
+ * @see #fail(String)
+ */
+ public static JobFilterResult pass(String reason) {
+ return new JobFilterResult(true, reason);
+ }
+
+ /**
+ * Create a new result indicating the job has failed the filter.
+ *
+ * @param reason A reason that the job has failed.
+ * @return A result indicating the job has failed because of the given reason.
+ * @throws NullPointerException if {@code reason} is {@code null}.
+ * @see #pass()
+ * @see #pass(String)
+ */
+ public static JobFilterResult fail(String reason) {
+ return new JobFilterResult(false, reason);
+ }
+
+ /**
+ * Indicates whether the result indicates a pass.
+ *
+ * @return {@code true} if the result indicates a pass.
+ */
+ public boolean isPass() {
+ return pass;
+ }
+
+ /**
+ * Indicates the reason for the result.
+ *
+ * @return The reason for the result.
+ */
+ public String getReason() {
+ return reason;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof JobFilterResult)) {
+ return false;
+ }
+ JobFilterResult that = (JobFilterResult) o;
+ return Objects.equal(pass, that.pass)
+ && Objects.equal(reason, that.reason);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(pass, reason);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("pass", pass)
+ .add("reason", reason)
+ .toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/JobManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/JobManager.java b/src/main/java/org/apache/aurora/scheduler/state/JobManager.java
new file mode 100644
index 0000000..4cb4ff6
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/JobManager.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Collections;
+
+import javax.inject.Inject;
+
+import com.twitter.aurora.scheduler.base.ScheduleException;
+import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+
+/**
+ * Interface for a job manager. A job manager is responsible for deciding whether and when to
+ * trigger execution of a job.
+ */
+public abstract class JobManager {
+
+ // TODO(Bill Farner): Remove this. It is only used since the CronJobManager and SchedulerCoreImpl
+ // have a circular dependency.
+ @Inject
+ protected SchedulerCore schedulerCore;
+
+ /**
+ * Gets a key that uniquely identifies this manager type, to distinguish from other schedulers.
+ * These keys end up being persisted, so they must be considered permanently immutable.
+ *
+ * @return Job manager key.
+ */
+ public abstract String getUniqueKey();
+
+ /**
+ * Submits a job to the manager. The job may be submitted to the job runner before this method
+ * returns or at any point in the future. This method will return false if the manager will not
+ * execute the job.
+ *
+ * @param config The job to schedule.
+ * @return {@code true} If the manager accepted the job, {@code false} otherwise.
+ * @throws ScheduleException If there is a problem with scheduling the job.
+ */
+ public abstract boolean receiveJob(SanitizedConfiguration config) throws ScheduleException;
+
+ /**
+ * Fetches the configured jobs that this manager is storing.
+ *
+ * @return Jobs stored by this job manager.
+ */
+ // TODO(ksweeney): Consider adding a Map<JobKey, JobConfiguration> to complement this.
+ public Iterable<IJobConfiguration> getJobs() {
+ return Collections.emptyList();
+ }
+
+ /**
+ * Checks whether this manager is storing a job with the given key.
+ *
+ * @param jobKey Job key.
+ * @return {@code true} if the manager has a matching job, {@code false} otherwise.
+ */
+ public abstract boolean hasJob(IJobKey jobKey);
+
+ /**
+ * Instructs the manager to delete any jobs with the given key.
+ *
+ * @param jobKey Job key.
+ * @return {@code true} if a matching job was deleted.
+ */
+ public boolean deleteJob(IJobKey jobKey) {
+ // Optionally overridden by implementing class.
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/LockManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/LockManager.java b/src/main/java/org/apache/aurora/scheduler/state/LockManager.java
new file mode 100644
index 0000000..58d7b0e
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/LockManager.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import com.google.common.base.Optional;
+
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.ILockKey;
+
+/**
+ * Defines all {@link ILock} primitives like: acquire, release, validate.
+ */
+public interface LockManager {
+ /**
+ * Creates, saves and returns a new {@link ILock} with the specified {@link ILockKey}.
+ * This method is not re-entrant, i.e. attempting to acquire a lock with the
+ * same key would throw a {@link LockException}.
+ *
+ * @param lockKey A key uniquely identify the lock to be created.
+ * @param user Name of the user requesting a lock.
+ * @return A new ILock instance.
+ * @throws LockException In case the lock with specified key already exists.
+ */
+ ILock acquireLock(ILockKey lockKey, String user) throws LockException;
+
+ /**
+ * Releases (removes) the specified {@link ILock} from the system.
+ *
+ * @param lock {@link ILock} to remove from the system.
+ */
+ void releaseLock(ILock lock);
+
+ /**
+ * Verifies if the provided lock instance is identical to the one stored in the scheduler
+ * ONLY if the operation context represented by the {@link ILockKey} is in fact locked.
+ * No validation will be performed in case there is no correspondent scheduler lock
+ * found for the provided context.
+ *
+ * @param context Operation context to validate with the provided lock.
+ * @param heldLock Lock to validate.
+ * @throws LockException If provided lock does not exist or not identical to the stored one.
+ */
+ void validateIfLocked(ILockKey context, Optional<ILock> heldLock) throws LockException;
+
+ /**
+ * Thrown when {@link ILock} related operation failed.
+ */
+ class LockException extends Exception {
+ public LockException(String msg) {
+ super(msg);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/LockManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/LockManagerImpl.java b/src/main/java/org/apache/aurora/scheduler/state/LockManagerImpl.java
new file mode 100644
index 0000000..4667f9f
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/LockManagerImpl.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Date;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Optional;
+
+import com.twitter.aurora.gen.Lock;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.storage.LockStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.ILockKey;
+import com.twitter.common.util.Clock;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Implements lock-related primitives required to provide mutual exclusion guarantees
+ * to the critical Scheduler state-mutating operations.
+ */
+class LockManagerImpl implements LockManager {
+ private final Storage storage;
+ private final Clock clock;
+ private final UUIDGenerator tokenGenerator;
+
+ @Inject
+ LockManagerImpl(Storage storage, Clock clock, UUIDGenerator tokenGenerator) {
+ this.storage = checkNotNull(storage);
+ this.clock = checkNotNull(clock);
+ this.tokenGenerator = checkNotNull(tokenGenerator);
+ }
+
+ @Override
+ public ILock acquireLock(final ILockKey lockKey, final String user) throws LockException {
+ return storage.write(new MutateWork<ILock, LockException>() {
+ @Override public ILock apply(Storage.MutableStoreProvider storeProvider)
+ throws LockException {
+
+ LockStore.Mutable lockStore = storeProvider.getLockStore();
+ Optional<ILock> existingLock = lockStore.fetchLock(lockKey);
+
+ if (existingLock.isPresent()) {
+ throw new LockException(String.format(
+ "Operation for: %s is already in progress. Started at: %s. Current owner: %s.",
+ formatLockKey(lockKey),
+ new Date(existingLock.get().getTimestampMs()).toString(),
+ existingLock.get().getUser()));
+ }
+
+ ILock lock = ILock.build(new Lock()
+ .setKey(lockKey.newBuilder())
+ .setToken(tokenGenerator.createNew().toString())
+ .setTimestampMs(clock.nowMillis())
+ .setUser(user));
+
+ lockStore.saveLock(lock);
+ return lock;
+ }
+ });
+ }
+
+ @Override
+ public void releaseLock(final ILock lock) {
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override public void execute(MutableStoreProvider storeProvider) {
+ storeProvider.getLockStore().removeLock(lock.getKey());
+ }
+ });
+ }
+
+ @Override
+ public synchronized void validateIfLocked(final ILockKey context, Optional<ILock> heldLock)
+ throws LockException {
+
+ Optional<ILock> stored = storage.consistentRead(new Work.Quiet<Optional<ILock>>() {
+ @Override public Optional<ILock> apply(StoreProvider storeProvider) {
+ return storeProvider.getLockStore().fetchLock(context);
+ }
+ });
+
+ // The implementation below assumes the following use cases:
+ // +-----------+-----------------+----------+
+ // | eq | held | not held |
+ // +-----------+-----------------+----------+
+ // |stored |(stored == held)?| invalid |
+ // +-----------+-----------------+----------+
+ // |not stored | invalid | valid |
+ // +-----------+-----------------+----------+
+ if (!stored.equals(heldLock)) {
+ if (stored.isPresent()) {
+ throw new LockException(String.format(
+ "Unable to perform operation for: %s. Use override/cancel option.",
+ formatLockKey(context)));
+ } else if (heldLock.isPresent()) {
+ throw new LockException(
+ String.format("Invalid operation context: %s", formatLockKey(context)));
+ }
+ }
+ }
+
+ private static String formatLockKey(ILockKey lockKey) {
+ switch (lockKey.getSetField()) {
+ case JOB:
+ return JobKeys.toPath(lockKey.getJob());
+ default:
+ return "Unknown lock key type: " + lockKey.getSetField();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java b/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
new file mode 100644
index 0000000..d447f52
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
@@ -0,0 +1,302 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Set;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.Subscribe;
+
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.HostStatus;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.events.PubsubEvent;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import com.twitter.aurora.scheduler.storage.AttributeStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.common.base.Closure;
+import com.twitter.common.base.Closures;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
+
+/**
+ * Logic that puts hosts into maintenance mode, and triggers draining of hosts upon request.
+ * All state-changing functions return their results. Additionally, all state-changing functions
+ * will ignore requests to change state of unknown hosts and subsequently omit these hosts from
+ * return values.
+ */
+public interface MaintenanceController {
+
+ /**
+ * Places hosts in maintenance mode.
+ * Hosts in maintenance mode are less-preferred for scheduling.
+ * No change will be made for hosts that are not recognized, and unrecognized hosts will not be
+ * included in the result.
+ *
+ * @param hosts Hosts to put into maintenance mode.
+ * @return The adjusted state of the hosts.
+ */
+ Set<HostStatus> startMaintenance(Set<String> hosts);
+
+ /**
+ * Initiate a drain of all active tasks on {@code hosts}.
+ *
+ * @param hosts Hosts to drain.
+ * @return The adjusted state of the hosts. Hosts without any active tasks will be immediately
+ * moved to DRAINED.
+ */
+ Set<HostStatus> drain(Set<String> hosts);
+
+ /**
+ * Fetches the current maintenance mode of {$code host}.
+ *
+ * @param host Host to fetch state for.
+ * @return Maintenance mode of host, {@link MaintenanceMode#NONE} if the host is not known.
+ */
+ MaintenanceMode getMode(String host);
+
+ /**
+ * Fetches the current state of {@code hosts}.
+ *
+ * @param hosts Hosts to fetch state for.
+ * @return The state of the hosts.
+ */
+ Set<HostStatus> getStatus(Set<String> hosts);
+
+ /**
+ * Moves {@code hosts} out of maintenance mode, returning them to mode NONE.
+ *
+ * @param hosts Hosts to move out of maintenance mode.
+ * @return The adjusted state of the hosts.
+ */
+ Set<HostStatus> endMaintenance(Set<String> hosts);
+
+ class MaintenanceControllerImpl implements MaintenanceController, EventSubscriber {
+ private final Storage storage;
+ private final StateManager stateManager;
+ private final Closure<PubsubEvent> eventSink;
+
+ @Inject
+ public MaintenanceControllerImpl(
+ Storage storage,
+ StateManager stateManager,
+ Closure<PubsubEvent> eventSink) {
+
+ this.storage = checkNotNull(storage);
+ this.stateManager = checkNotNull(stateManager);
+ this.eventSink = checkNotNull(eventSink);
+ }
+
+ private Set<HostStatus> watchDrainingTasks(
+ MutableStoreProvider store,
+ Set<String> hosts,
+ Closure<Query.Builder> callback) {
+
+ Set<String> emptyHosts = Sets.newHashSet();
+ for (String host : hosts) {
+ // If there are no tasks on the host, immediately transition to DRAINED.
+ Query.Builder query = Query.slaveScoped(host).active();
+ Set<String> activeTasks = FluentIterable.from(store.getTaskStore().fetchTasks(query))
+ .transform(Tasks.SCHEDULED_TO_ID)
+ .toSet();
+ if (activeTasks.isEmpty()) {
+ emptyHosts.add(host);
+ } else {
+ callback.execute(query);
+ }
+ }
+
+ return ImmutableSet.<HostStatus>builder()
+ .addAll(setMaintenanceMode(store, emptyHosts, DRAINED))
+ .addAll(setMaintenanceMode(store, Sets.difference(hosts, emptyHosts), DRAINING))
+ .build();
+ }
+
+ private Set<HostStatus> watchDrainingTasks(MutableStoreProvider store, Set<String> hosts) {
+ return watchDrainingTasks(store, hosts, Closures.<Query.Builder>noop());
+ }
+
+ private static final Predicate<HostAttributes> IS_DRAINING = new Predicate<HostAttributes>() {
+ @Override public boolean apply(HostAttributes attributes) {
+ return DRAINING == attributes.getMode();
+ }
+ };
+
+ /**
+ * Notifies the MaintenanceController that storage has started, and maintenance statuses are
+ * ready to be loaded.
+ *
+ * @param started Event.
+ */
+ @Subscribe
+ public void storageStarted(StorageStarted started) {
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ Set<String> drainingHosts =
+ FluentIterable.from(storeProvider.getAttributeStore().getHostAttributes())
+ .filter(IS_DRAINING)
+ .transform(HOST_NAME)
+ .toSet();
+ watchDrainingTasks(storeProvider, drainingHosts);
+ }
+ });
+ }
+
+ /**
+ * Notifies the MaintenanceController that a task has changed state
+ *
+ * @param change Event
+ */
+ @Subscribe
+ public void taskChangedState(final TaskStateChange change) {
+ if (Tasks.isTerminated(change.getNewState())) {
+ final String host = change.getTask().getAssignedTask().getSlaveHost();
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override public void execute(MutableStoreProvider store) {
+ // If the task _was_ associated with a draining host, and it was the last task on the
+ // host.
+ Optional<HostAttributes> attributes = store.getAttributeStore().getHostAttributes(host);
+ if (attributes.isPresent() && attributes.get().getMode() == DRAINING) {
+ Query.Builder builder = Query.slaveScoped(host).active();
+ if (store.getTaskStore().fetchTasks(builder).isEmpty()) {
+ setMaintenanceMode(store, ImmutableSet.of(host), DRAINED);
+ }
+ }
+ }
+ });
+ }
+ }
+
+ @Override
+ public Set<HostStatus> startMaintenance(final Set<String> hosts) {
+ return storage.write(new MutateWork.Quiet<Set<HostStatus>>() {
+ @Override public Set<HostStatus> apply(MutableStoreProvider storeProvider) {
+ return setMaintenanceMode(storeProvider, hosts, MaintenanceMode.SCHEDULED);
+ }
+ });
+ }
+
+ @VisibleForTesting
+ static final Optional<String> DRAINING_MESSAGE =
+ Optional.of("Draining machine for maintenance.");
+
+ @Override
+ public Set<HostStatus> drain(final Set<String> hosts) {
+ return storage.write(new MutateWork.Quiet<Set<HostStatus>>() {
+ @Override public Set<HostStatus> apply(MutableStoreProvider store) {
+ return watchDrainingTasks(store, hosts, new Closure<Query.Builder>() {
+ @Override public void execute(Query.Builder query) {
+ stateManager.changeState(query, ScheduleStatus.RESTARTING, DRAINING_MESSAGE);
+ }
+ });
+ }
+ });
+ }
+
+ private static final Function<HostAttributes, String> HOST_NAME =
+ new Function<HostAttributes, String>() {
+ @Override public String apply(HostAttributes attributes) {
+ return attributes.getHost();
+ }
+ };
+
+ private static final Function<HostAttributes, HostStatus> ATTRS_TO_STATUS =
+ new Function<HostAttributes, HostStatus>() {
+ @Override public HostStatus apply(HostAttributes attributes) {
+ return new HostStatus().setHost(attributes.getHost()).setMode(attributes.getMode());
+ }
+ };
+
+ private static final Function<HostStatus, MaintenanceMode> GET_MODE =
+ new Function<HostStatus, MaintenanceMode>() {
+ @Override public MaintenanceMode apply(HostStatus status) {
+ return status.getMode();
+ }
+ };
+
+ @Override
+ public MaintenanceMode getMode(final String host) {
+ return storage.weaklyConsistentRead(new Work.Quiet<MaintenanceMode>() {
+ @Override public MaintenanceMode apply(StoreProvider storeProvider) {
+ return storeProvider.getAttributeStore().getHostAttributes(host)
+ .transform(ATTRS_TO_STATUS)
+ .transform(GET_MODE)
+ .or(MaintenanceMode.NONE);
+ }
+ });
+ }
+
+ @Override
+ public Set<HostStatus> getStatus(final Set<String> hosts) {
+ return storage.weaklyConsistentRead(new Work.Quiet<Set<HostStatus>>() {
+ @Override public Set<HostStatus> apply(StoreProvider storeProvider) {
+ // Warning - this is filtering _all_ host attributes. If using this to frequently query
+ // for a small set of hosts, a getHostAttributes variant should be added.
+ return FluentIterable.from(storeProvider.getAttributeStore().getHostAttributes())
+ .filter(Predicates.compose(Predicates.in(hosts), HOST_NAME))
+ .transform(ATTRS_TO_STATUS).toSet();
+ }
+ });
+ }
+
+ @Override
+ public Set<HostStatus> endMaintenance(final Set<String> hosts) {
+ return storage.write(new MutateWork.Quiet<Set<HostStatus>>() {
+ @Override public Set<HostStatus> apply(MutableStoreProvider storeProvider) {
+ return setMaintenanceMode(storeProvider, hosts, MaintenanceMode.NONE);
+ }
+ });
+ }
+
+ private Set<HostStatus> setMaintenanceMode(
+ MutableStoreProvider storeProvider,
+ Set<String> hosts,
+ MaintenanceMode mode) {
+
+ AttributeStore.Mutable store = storeProvider.getAttributeStore();
+ ImmutableSet.Builder<HostStatus> statuses = ImmutableSet.builder();
+ for (String host : hosts) {
+ if (store.setMaintenanceMode(host, mode)) {
+ HostStatus status = new HostStatus().setHost(host).setMode(mode);
+ eventSink.execute(new PubsubEvent.HostMaintenanceStateChange(status.deepCopy()));
+ statuses.add(status);
+ }
+ }
+ return statuses.build();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/SchedulerCore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/SchedulerCore.java b/src/main/java/org/apache/aurora/scheduler/state/SchedulerCore.java
new file mode 100644
index 0000000..2db01c0
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/SchedulerCore.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Set;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.ScheduleException;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
+import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * Scheduling core, stores scheduler state and makes decisions about which tasks to schedule when
+ * a resource offer is made.
+ *
+ * When a job is submitted to the scheduler core, it will store the job configuration and offer
+ * the job to all configured scheduler modules, which are responsible for triggering execution of
+ * the job. Until a job is triggered by a scheduler module, it is retained in the scheduler core
+ * in the PENDING state.
+ */
+public interface SchedulerCore {
+
+ /**
+ * Creates a new job, whose tasks will become candidates for scheduling.
+ *
+ * @param sanitizedConfiguration The configuration of the job to create tasks for.
+ * @throws ScheduleException If there was an error scheduling a cron job.
+ * @throws TaskDescriptionException If an invalid task description was given.
+ */
+ void createJob(SanitizedConfiguration sanitizedConfiguration)
+ throws ScheduleException, TaskDescriptionException;
+
+ /**
+ * Adds new instances specified by the instances set.
+ * <p>
+ * Provided instance IDs should be disjoint from the instance IDs active in the job.
+ *
+ * @param jobKey IJobKey identifying the parent job.
+ * @param instanceIds Set of instance IDs to be added to the job.
+ * @param config ITaskConfig to use with new instances.
+ * @throws ScheduleException If any of the existing instance IDs already exist.
+ */
+ void addInstances(IJobKey jobKey, ImmutableSet<Integer> instanceIds, ITaskConfig config)
+ throws ScheduleException;
+
+ /**
+ * Validates the new job configuration passes resource filters.
+ *
+ * @param sanitizedConfiguration Job configuration to validate.
+ * @throws ScheduleException If job resources do not pass filters.
+ */
+ void validateJobResources(SanitizedConfiguration sanitizedConfiguration) throws ScheduleException;
+
+ /**
+ * Starts a cron job immediately.
+ *
+ * @param jobKey Job key.
+ * @throws ScheduleException If the specified job does not exist, or is not a cron job.
+ * @throws TaskDescriptionException If the parsing of the job failed.
+ */
+ void startCronJob(IJobKey jobKey) throws ScheduleException, TaskDescriptionException;
+
+ /**
+ * Assigns a new state to tasks.
+ *
+ * @param query Builder for a query to identify tasks
+ * @param status The new state of the tasks.
+ * @param message Additional information about the state transition.
+ */
+ void setTaskStatus(Query.Builder query, ScheduleStatus status, Optional<String> message);
+
+ /**
+ * Kills a specific set of tasks.
+ *
+ * @param query Builder for a query to identify tasks
+ * @param user Name of the user performing the kill.
+ * @throws ScheduleException If a problem occurs with the kill request.
+ */
+ void killTasks(Query.Builder query, String user) throws ScheduleException;
+
+ /**
+ * Initiates a restart of shards within an active job.
+ *
+ * @param jobKey Key of job to be restarted.
+ * @param shards Shards to be restarted.
+ * @param requestingUser User performing the restart action.
+ * @throws ScheduleException If there are no matching active shards.
+ */
+ void restartShards(IJobKey jobKey, Set<Integer> shards, String requestingUser)
+ throws ScheduleException;
+
+ /**
+ * Preempts a task in favor of another.
+ *
+ * @param task Task being preempted.
+ * @param preemptingTask Task we are preempting in favor of.
+ * @throws ScheduleException If a problem occurs while trying to perform the preemption.
+ */
+ void preemptTask(IAssignedTask task, IAssignedTask preemptingTask) throws ScheduleException;
+
+ /**
+ * Indicates to the scheduler that tasks were deleted on the assigned host.
+ *
+ * @param taskIds IDs of tasks that were deleted.
+ */
+ void tasksDeleted(Set<String> taskIds);
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/SchedulerCoreImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/SchedulerCoreImpl.java b/src/main/java/org/apache/aurora/scheduler/state/SchedulerCoreImpl.java
new file mode 100644
index 0000000..cf74e49
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/SchedulerCoreImpl.java
@@ -0,0 +1,328 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+
+import java.util.Set;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Functions;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.TaskIdGenerator;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.ScheduleException;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
+import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.args.constraints.Positive;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
+import static com.twitter.aurora.gen.ScheduleStatus.RESTARTING;
+import static com.twitter.aurora.scheduler.base.Tasks.ACTIVE_STATES;
+
+/**
+ * Implementation of the scheduler core.
+ */
+class SchedulerCoreImpl implements SchedulerCore {
+ @Positive
+ @CmdLine(name = "max_tasks_per_job", help = "Maximum number of allowed tasks in a single job.")
+ public static final Arg<Integer> MAX_TASKS_PER_JOB = Arg.create(4000);
+
+ private static final Logger LOG = Logger.getLogger(SchedulerCoreImpl.class.getName());
+
+ private final Storage storage;
+
+ private final CronJobManager cronScheduler;
+
+ // Schedulers that are responsible for triggering execution of jobs.
+ private final ImmutableList<JobManager> jobManagers;
+
+ // TODO(Bill Farner): Avoid using StateManagerImpl.
+ // State manager handles persistence of task modifications and state transitions.
+ private final StateManagerImpl stateManager;
+
+ private final TaskIdGenerator taskIdGenerator;
+ private final JobFilter jobFilter;
+
+ /**
+ * Creates a new core scheduler.
+ *
+ * @param storage Backing store implementation.
+ * @param cronScheduler Cron scheduler.
+ * @param immediateScheduler Immediate scheduler.
+ * @param stateManager Persistent state manager.
+ * @param taskIdGenerator Task ID generator.
+ * @param jobFilter Job filter.
+ */
+ @Inject
+ public SchedulerCoreImpl(
+ Storage storage,
+ CronJobManager cronScheduler,
+ ImmediateJobManager immediateScheduler,
+ StateManagerImpl stateManager,
+ TaskIdGenerator taskIdGenerator,
+ JobFilter jobFilter) {
+
+ this.storage = checkNotNull(storage);
+
+ // The immediate scheduler will accept any job, so it's important that other schedulers are
+ // placed first.
+ this.jobManagers = ImmutableList.of(cronScheduler, immediateScheduler);
+ this.cronScheduler = cronScheduler;
+ this.stateManager = checkNotNull(stateManager);
+ this.taskIdGenerator = checkNotNull(taskIdGenerator);
+ this.jobFilter = checkNotNull(jobFilter);
+ }
+
+ private boolean hasActiveJob(IJobConfiguration job) {
+ return Iterables.any(jobManagers, managerHasJob(job));
+ }
+
+ @Override
+ public synchronized void tasksDeleted(Set<String> taskIds) {
+ setTaskStatus(Query.taskScoped(taskIds), ScheduleStatus.UNKNOWN, Optional.<String>absent());
+ }
+
+ @Override
+ public synchronized void createJob(SanitizedConfiguration sanitizedConfiguration)
+ throws ScheduleException {
+
+ IJobConfiguration job = sanitizedConfiguration.getJobConfig();
+ if (hasActiveJob(job)) {
+ throw new ScheduleException("Job already exists: " + JobKeys.toPath(job));
+ }
+
+ runJobFilters(job.getKey(), job.getTaskConfig(), job.getInstanceCount(), false);
+
+ boolean accepted = false;
+ for (final JobManager manager : jobManagers) {
+ if (manager.receiveJob(sanitizedConfiguration)) {
+ LOG.info("Job accepted by manager: " + manager.getUniqueKey());
+ accepted = true;
+ break;
+ }
+ }
+
+ if (!accepted) {
+ LOG.severe("Job was not accepted by any of the configured schedulers, discarding.");
+ LOG.severe("Discarded job: " + job);
+ throw new ScheduleException("Job not accepted, discarding.");
+ }
+ }
+
+ // This number is derived from the maximum file name length limit on most UNIX systems, less
+ // the number of characters we've observed being added by mesos for the executor ID, prefix, and
+ // delimiters.
+ @VisibleForTesting
+ static final int MAX_TASK_ID_LENGTH = 255 - 90;
+
+ // TODO(maximk): Consider a better approach to quota checking. MESOS-4476.
+ private void runJobFilters(IJobKey jobKey, ITaskConfig task, int count, boolean incremental)
+ throws ScheduleException {
+
+ int instanceCount = count;
+ if (incremental) {
+ instanceCount +=
+ Storage.Util.weaklyConsistentFetchTasks(storage, Query.jobScoped(jobKey).active()).size();
+ }
+
+ // TODO(maximk): This is a short-term hack to stop the bleeding from
+ // https://issues.apache.org/jira/browse/MESOS-691
+ if (taskIdGenerator.generate(task, instanceCount).length() > MAX_TASK_ID_LENGTH) {
+ throw new ScheduleException(
+ "Task ID is too long, please shorten your role or job name.");
+ }
+
+ JobFilter.JobFilterResult filterResult = jobFilter.filter(task, instanceCount);
+ // TODO(maximk): Consider deprecating JobFilterResult in favor of custom exception.
+ if (!filterResult.isPass()) {
+ throw new ScheduleException(filterResult.getReason());
+ }
+
+ if (instanceCount > MAX_TASKS_PER_JOB.get()) {
+ throw new ScheduleException("Job exceeds task limit of " + MAX_TASKS_PER_JOB.get());
+ }
+ }
+
+ @Override
+ public void validateJobResources(SanitizedConfiguration sanitizedConfiguration)
+ throws ScheduleException {
+
+ IJobConfiguration job = sanitizedConfiguration.getJobConfig();
+ runJobFilters(job.getKey(), job.getTaskConfig(), job.getInstanceCount(), false);
+ }
+
+ @Override
+ public void addInstances(
+ final IJobKey jobKey,
+ final ImmutableSet<Integer> instanceIds,
+ final ITaskConfig config) throws ScheduleException {
+
+ runJobFilters(jobKey, config, instanceIds.size(), true);
+ storage.write(new MutateWork.NoResult<ScheduleException>() {
+ @Override
+ protected void execute(MutableStoreProvider storeProvider)
+ throws ScheduleException {
+
+ ImmutableSet<IScheduledTask> tasks =
+ storeProvider.getTaskStore().fetchTasks(Query.jobScoped(jobKey).active());
+
+ Set<Integer> existingInstanceIds =
+ FluentIterable.from(tasks).transform(Tasks.SCHEDULED_TO_INSTANCE_ID).toSet();
+ if (!Sets.intersection(existingInstanceIds, instanceIds).isEmpty()) {
+ throw new ScheduleException("Instance ID collision detected.");
+ }
+
+ stateManager.insertPendingTasks(Maps.asMap(instanceIds, Functions.constant(config)));
+ }
+ });
+ }
+
+ @Override
+ public synchronized void startCronJob(IJobKey jobKey)
+ throws ScheduleException, TaskDescriptionException {
+
+ checkNotNull(jobKey);
+
+ if (!cronScheduler.hasJob(jobKey)) {
+ throw new ScheduleException("Cron job does not exist for " + JobKeys.toPath(jobKey));
+ }
+
+ cronScheduler.startJobNow(jobKey);
+ }
+
+ /**
+ * Creates a predicate that will determine whether a job manager has a job matching a job key.
+ *
+ * @param job Job to match.
+ * @return A new predicate matching the job owner and name given.
+ */
+ private static Predicate<JobManager> managerHasJob(final IJobConfiguration job) {
+ return new Predicate<JobManager>() {
+ @Override public boolean apply(JobManager manager) {
+ return manager.hasJob(job.getKey());
+ }
+ };
+ }
+
+ @Override
+ public synchronized void setTaskStatus(
+ Query.Builder query,
+ final ScheduleStatus status,
+ Optional<String> message) {
+
+ checkNotNull(query);
+ checkNotNull(status);
+
+ stateManager.changeState(query, status, message);
+ }
+
+ @Override
+ public synchronized void killTasks(Query.Builder query, String user) throws ScheduleException {
+ checkNotNull(query);
+ LOG.info("Killing tasks matching " + query);
+
+ boolean jobDeleted = false;
+
+ if (Query.isOnlyJobScoped(query)) {
+ // If this looks like a query for all tasks in a job, instruct the scheduler modules to
+ // delete the job.
+ IJobKey jobKey = JobKeys.from(query).get();
+ for (JobManager manager : jobManagers) {
+ if (manager.deleteJob(jobKey)) {
+ jobDeleted = true;
+ }
+ }
+ }
+
+ // Unless statuses were specifically supplied, only attempt to kill active tasks.
+ Query.Builder taskQuery = query.get().isSetStatuses() ? query.byStatus(ACTIVE_STATES) : query;
+
+ int tasksAffected =
+ stateManager.changeState(taskQuery, KILLING, Optional.of("Killed by " + user));
+ if (!jobDeleted && (tasksAffected == 0)) {
+ throw new ScheduleException("No jobs to kill");
+ }
+ }
+
+ @Override
+ public void restartShards(
+ IJobKey jobKey,
+ final Set<Integer> shards,
+ final String requestingUser) throws ScheduleException {
+
+ if (!JobKeys.isValid(jobKey)) {
+ throw new ScheduleException("Invalid job key: " + jobKey);
+ }
+
+ if (shards.isEmpty()) {
+ throw new ScheduleException("At least one shard must be specified.");
+ }
+
+ final Query.Builder query = Query.instanceScoped(jobKey, shards).active();
+ storage.write(new MutateWork.NoResult<ScheduleException>() {
+ @Override protected void execute(MutableStoreProvider storeProvider)
+ throws ScheduleException {
+
+ Set<IScheduledTask> matchingTasks = storeProvider.getTaskStore().fetchTasks(query);
+ if (matchingTasks.size() != shards.size()) {
+ throw new ScheduleException("Not all requested shards are active.");
+ }
+ LOG.info("Restarting shards matching " + query);
+ stateManager.changeState(
+ Query.taskScoped(Tasks.ids(matchingTasks)),
+ RESTARTING,
+ Optional.of("Restarted by " + requestingUser));
+ }
+ });
+ }
+
+
+ @Override
+ public synchronized void preemptTask(IAssignedTask task, IAssignedTask preemptingTask) {
+ checkNotNull(task);
+ checkNotNull(preemptingTask);
+ // TODO(William Farner): Throw SchedulingException if either task doesn't exist, etc.
+
+ stateManager.changeState(Query.taskScoped(task.getTaskId()), ScheduleStatus.PREEMPTING,
+ Optional.of("Preempting in favor of " + preemptingTask.getTaskId()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/SideEffectStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/SideEffectStorage.java b/src/main/java/org/apache/aurora/scheduler/state/SideEffectStorage.java
new file mode 100644
index 0000000..23ffff9
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/SideEffectStorage.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+import com.twitter.aurora.scheduler.events.PubsubEvent;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.StorageException;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.common.base.Closure;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Wrapper around the persistent storage and mutable state.
+ */
+class SideEffectStorage {
+
+ private final Queue<PubsubEvent> events = Lists.newLinkedList();
+ @VisibleForTesting
+ Queue<PubsubEvent> getEvents() {
+ return events;
+ }
+
+ private AtomicBoolean inOperation = new AtomicBoolean(false);
+
+ private final Storage storage;
+ private final OperationFinalizer operationFinalizer;
+ private final Closure<PubsubEvent> taskEventSink;
+
+ interface OperationFinalizer {
+ /**
+ * Performs any work necessary to complete the operation.
+ * This is executed in the context of a write operation, immediately after the work
+ * executes normally.
+ * NOTE: At present, this is executed for every nesting level of operations, rather than
+ * at the completion of the top-level operation.
+ * See comment in {@link #SideEffectStorage#executeSideEffectsAfter(SideEffectWork)}
+ * for more detail.
+ *
+ * @param work Work to finalize.
+ * @param storeProvider Mutable store reference.
+ */
+ void finalize(SideEffectWork<?, ?> work, MutableStoreProvider storeProvider);
+ }
+
+ SideEffectStorage(
+ Storage storage,
+ OperationFinalizer operationFinalizer,
+ Closure<PubsubEvent> taskEventSink) {
+
+ this.storage = checkNotNull(storage);
+ this.operationFinalizer = checkNotNull(operationFinalizer);
+ this.taskEventSink = checkNotNull(taskEventSink);
+ }
+
+ /**
+ * Perform a unit of work in a mutating operation. This supports nesting/reentrancy.
+ *
+ * @param work Work to perform.
+ * @param <T> Work return type
+ * @param <E> Work exception type.
+ * @return The work return value.
+ * @throws E The work exception.
+ */
+ <T, E extends Exception> T write(SideEffectWork<T, E> work) throws E {
+ return storage.write(executeSideEffectsAfter(work));
+ }
+
+ <T, E extends Exception> T consistentRead(Work<T, E> work) throws StorageException, E {
+ return storage.consistentRead(work);
+ }
+
+ /**
+ * Work that has side effects external to the storage system.
+ * Work may add side effect and pubsub events, which will be executed/sent upon normal
+ * completion of the operation.
+ *
+ * @param <T> Work return type.
+ * @param <E> Work exception type.
+ */
+ abstract class SideEffectWork<T, E extends Exception> implements MutateWork<T, E> {
+ protected final void addTaskEvent(PubsubEvent notice) {
+ Preconditions.checkState(inOperation.get());
+ events.add(Preconditions.checkNotNull(notice));
+ }
+ }
+
+ /**
+ * Work with side effects which does not throw checked exceptions.
+ *
+ * @param <T> Work return type.
+ */
+ abstract class QuietSideEffectWork<T> extends SideEffectWork<T, RuntimeException> {
+ }
+
+ /**
+ * Work with side effects that does not have a return value.
+ *
+ * @param <E> Work exception type.
+ */
+ abstract class NoResultSideEffectWork<E extends Exception> extends SideEffectWork<Void, E> {
+
+ @Override public final Void apply(MutableStoreProvider storeProvider) throws E {
+ execute(storeProvider);
+ return null;
+ }
+
+ abstract void execute(MutableStoreProvider storeProvider) throws E;
+ }
+
+ /**
+ * Work with side effects which does not throw checked exceptions or have a return
+ * value.
+ */
+ abstract class NoResultQuietSideEffectWork extends NoResultSideEffectWork<RuntimeException> {
+ }
+
+ private <T, E extends Exception> MutateWork<T, E> executeSideEffectsAfter(
+ final SideEffectWork<T, E> work) {
+
+ return new MutateWork<T, E>() {
+ @Override public T apply(MutableStoreProvider storeProvider) throws E {
+ boolean topLevelOperation = inOperation.compareAndSet(false, true);
+
+ try {
+ T result = work.apply(storeProvider);
+
+ // TODO(William Farner): Maintaining this since it matches prior behavior, but this
+ // seems wrong. Double-check whether this is necessary, or if only the top-level
+ // operation should be executing the finalizer. Update doc on OperationFinalizer
+ // once this is assessed.
+ operationFinalizer.finalize(work, storeProvider);
+ if (topLevelOperation) {
+ while (!events.isEmpty()) {
+ taskEventSink.execute(events.remove());
+ }
+ }
+ return result;
+ } finally {
+ if (topLevelOperation) {
+ inOperation.set(false);
+ }
+ }
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/StateManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/StateManager.java b/src/main/java/org/apache/aurora/scheduler/state/StateManager.java
new file mode 100644
index 0000000..099ec70
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/StateManager.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.base.Optional;
+
+import org.apache.mesos.Protos.SlaveID;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * Thin interface for the state manager.
+ */
+public interface StateManager {
+
+ /**
+ * Performs a simple state change, transitioning all tasks matching a query to the given
+ * state and applying the given audit message.
+ * TODO(William Farner): Consider removing the return value.
+ *
+ * @param query Builder of the query to perform, the results of which will be modified.
+ * @param newState State to move the resulting tasks into.
+ * @param auditMessage Audit message to apply along with the state change.
+ * @return the number of successful state changes.
+ */
+ int changeState(Query.Builder query, ScheduleStatus newState, Optional<String> auditMessage);
+
+ /**
+ * Assigns a task to a specific slave.
+ * This will modify the task record to reflect the host assignment and return the updated record.
+ *
+ * @param taskId ID of the task to mutate.
+ * @param slaveHost Host name that the task is being assigned to.
+ * @param slaveId ID of the slave that the task is being assigned to.
+ * @param assignedPorts Ports on the host that are being assigned to the task.
+ * @return The updated task record, or {@code null} if the task was not found.
+ */
+ IAssignedTask assignTask(
+ String taskId,
+ String slaveHost,
+ SlaveID slaveId,
+ Set<Integer> assignedPorts);
+
+ /**
+ * Inserts new tasks into the store. Tasks will immediately move into PENDING and will be eligible
+ * for scheduling.
+ *
+ * @param tasks Tasks to insert, mapped by their instance IDs.
+ */
+ void insertPendingTasks(Map<Integer, ITaskConfig> tasks);
+
+ /**
+ * Deletes records of tasks from the task store.
+ * This will not perform any state checking or state transitions, but will immediately remove
+ * the tasks from the store. It will also silently ignore attempts to delete task IDs that do
+ * not exist.
+ *
+ * @param taskIds IDs of tasks to delete.
+ */
+ void deleteTasks(final Set<String> taskIds);
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/StateManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/StateManagerImpl.java b/src/main/java/org/apache/aurora/scheduler/state/StateManagerImpl.java
new file mode 100644
index 0000000..37d13f4
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/StateManagerImpl.java
@@ -0,0 +1,458 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Atomics;
+
+import org.apache.mesos.Protos.SlaveID;
+
+import com.twitter.aurora.gen.AssignedTask;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.ScheduledTask;
+import com.twitter.aurora.scheduler.Driver;
+import com.twitter.aurora.scheduler.TaskIdGenerator;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.events.PubsubEvent;
+import com.twitter.aurora.scheduler.state.SideEffectStorage.SideEffectWork;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.TaskStore;
+import com.twitter.aurora.scheduler.storage.TaskStore.Mutable.TaskMutation;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.base.Closure;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Iterables.transform;
+
+import static com.twitter.aurora.gen.ScheduleStatus.INIT;
+import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
+import static com.twitter.aurora.gen.ScheduleStatus.UNKNOWN;
+import static com.twitter.aurora.scheduler.state.SideEffectStorage.OperationFinalizer;
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * Manager of all persistence-related operations for the scheduler. Acts as a controller for
+ * persisted state machine transitions, and their side-effects.
+ *
+ * TODO(William Farner): Re-evaluate thread safety here, specifically risk of races that
+ * modify managerState.
+ */
+public class StateManagerImpl implements StateManager {
+ private static final Logger LOG = Logger.getLogger(StateManagerImpl.class.getName());
+
+ private final SideEffectStorage storage;
+ @VisibleForTesting
+ SideEffectStorage getStorage() {
+ return storage;
+ }
+
+ private final TaskIdGenerator taskIdGenerator;
+
+ // Work queue to receive state machine side effect work.
+ // Items are sorted to place DELETE entries last. This is to ensure that within an operation,
+ // a delete is always processed after a state transition.
+ private final Queue<WorkEntry> workQueue = new PriorityQueue<>(10,
+ new Comparator<WorkEntry>() {
+ @Override public int compare(WorkEntry a, WorkEntry b) {
+ if ((a.command == WorkCommand.DELETE) != (b.command == WorkCommand.DELETE)) {
+ return (a.command == WorkCommand.DELETE) ? 1 : -1;
+ } else {
+ return 0;
+ }
+ }
+ });
+
+ // Adapt the work queue into a sink.
+ private final TaskStateMachine.WorkSink workSink = new TaskStateMachine.WorkSink() {
+ @Override public void addWork(
+ WorkCommand work,
+ TaskStateMachine stateMachine,
+ Function<IScheduledTask, IScheduledTask> mutation) {
+
+ workQueue.add(new WorkEntry(work, stateMachine, mutation));
+ }
+ };
+
+ private final Function<Map.Entry<Integer, ITaskConfig>, IScheduledTask> taskCreator =
+ new Function<Map.Entry<Integer, ITaskConfig>, IScheduledTask>() {
+ @Override public IScheduledTask apply(Map.Entry<Integer, ITaskConfig> entry) {
+ ITaskConfig task = entry.getValue();
+ AssignedTask assigned = new AssignedTask()
+ .setTaskId(taskIdGenerator.generate(task, entry.getKey()))
+ .setInstanceId(entry.getKey())
+ .setTask(task.newBuilder());
+ return IScheduledTask.build(new ScheduledTask()
+ .setStatus(INIT)
+ .setAssignedTask(assigned));
+ }
+ };
+
+ private final Driver driver;
+ private final Clock clock;
+
+ /**
+ * An item of work on the work queue.
+ */
+ private static class WorkEntry {
+ private final WorkCommand command;
+ private final TaskStateMachine stateMachine;
+ private final Function<IScheduledTask, IScheduledTask> mutation;
+
+ WorkEntry(
+ WorkCommand command,
+ TaskStateMachine stateMachine,
+ Function<IScheduledTask, IScheduledTask> mutation) {
+
+ this.command = command;
+ this.stateMachine = stateMachine;
+ this.mutation = mutation;
+ }
+ }
+
+ @Inject
+ StateManagerImpl(
+ final Storage storage,
+ final Clock clock,
+ Driver driver,
+ TaskIdGenerator taskIdGenerator,
+ Closure<PubsubEvent> taskEventSink) {
+
+ checkNotNull(storage);
+ this.clock = checkNotNull(clock);
+
+ OperationFinalizer finalizer = new OperationFinalizer() {
+ @Override public void finalize(SideEffectWork<?, ?> work, MutableStoreProvider store) {
+ processWorkQueueInWriteOperation(work, store);
+ }
+ };
+
+ this.storage = new SideEffectStorage(storage, finalizer, taskEventSink);
+
+ this.driver = checkNotNull(driver);
+ this.taskIdGenerator = checkNotNull(taskIdGenerator);
+
+ Stats.exportSize("work_queue_depth", workQueue);
+ }
+
+ @Override
+ public void insertPendingTasks(final Map<Integer, ITaskConfig> tasks) {
+ checkNotNull(tasks);
+
+ // Done outside the write transaction to minimize the work done inside a transaction.
+ final Set<IScheduledTask> scheduledTasks =
+ ImmutableSet.copyOf(transform(tasks.entrySet(), taskCreator));
+
+ storage.write(storage.new NoResultQuietSideEffectWork() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ storeProvider.getUnsafeTaskStore().saveTasks(scheduledTasks);
+
+ for (IScheduledTask task : scheduledTasks) {
+ createStateMachine(task).updateState(PENDING);
+ }
+ }
+ });
+ }
+
+ @Override
+ public int changeState(
+ Query.Builder query,
+ final ScheduleStatus newState,
+ final Optional<String> auditMessage) {
+
+ return changeState(query, new Function<TaskStateMachine, Boolean>() {
+ @Override public Boolean apply(TaskStateMachine stateMachine) {
+ return stateMachine.updateState(newState, auditMessage);
+ }
+ });
+ }
+
+ @Override
+ public IAssignedTask assignTask(
+ String taskId,
+ String slaveHost,
+ SlaveID slaveId,
+ Set<Integer> assignedPorts) {
+
+ checkNotBlank(taskId);
+ checkNotBlank(slaveHost);
+ checkNotNull(assignedPorts);
+
+ TaskAssignMutation mutation = assignHost(slaveHost, slaveId, assignedPorts);
+ changeState(Query.taskScoped(taskId), mutation);
+
+ return mutation.getAssignedTask();
+ }
+
+ private int changeStateInWriteOperation(
+ Set<String> taskIds,
+ Function<TaskStateMachine, Boolean> stateChange) {
+
+ int count = 0;
+ for (TaskStateMachine stateMachine : getStateMachines(taskIds).values()) {
+ if (stateChange.apply(stateMachine)) {
+ ++count;
+ }
+ }
+ return count;
+ }
+
+ private int changeState(
+ final Query.Builder query,
+ final Function<TaskStateMachine, Boolean> stateChange) {
+
+ return storage.write(storage.new QuietSideEffectWork<Integer>() {
+ @Override public Integer apply(MutableStoreProvider storeProvider) {
+ Set<String> ids = FluentIterable.from(storeProvider.getTaskStore().fetchTasks(query))
+ .transform(Tasks.SCHEDULED_TO_ID)
+ .toSet();
+ return changeStateInWriteOperation(ids, stateChange);
+ }
+ });
+ }
+
+ private interface TaskAssignMutation extends Function<TaskStateMachine, Boolean> {
+ IAssignedTask getAssignedTask();
+ }
+
+ private static Map<String, Integer> getNameMappedPorts(
+ Set<String> portNames,
+ Set<Integer> allocatedPorts) {
+
+ Preconditions.checkNotNull(portNames);
+
+ // Expand ports.
+ Map<String, Integer> ports = Maps.newHashMap();
+ Set<Integer> portsRemaining = Sets.newHashSet(allocatedPorts);
+ Iterator<Integer> portConsumer = Iterables.consumingIterable(portsRemaining).iterator();
+
+ for (String portName : portNames) {
+ Preconditions.checkArgument(portConsumer.hasNext(),
+ "Allocated ports %s were not sufficient to expand task.", allocatedPorts);
+ int portNumber = portConsumer.next();
+ ports.put(portName, portNumber);
+ }
+
+ if (!portsRemaining.isEmpty()) {
+ LOG.warning("Not all allocated ports were used to map ports!");
+ }
+
+ return ports;
+ }
+
+ private TaskAssignMutation assignHost(
+ final String slaveHost,
+ final SlaveID slaveId,
+ final Set<Integer> assignedPorts) {
+
+ final TaskMutation mutation = new TaskMutation() {
+ @Override public IScheduledTask apply(IScheduledTask task) {
+ ScheduledTask builder = task.newBuilder();
+ AssignedTask assigned = builder.getAssignedTask();
+ assigned.setAssignedPorts(
+ getNameMappedPorts(assigned.getTask().getRequestedPorts(), assignedPorts));
+ assigned.setSlaveHost(slaveHost)
+ .setSlaveId(slaveId.getValue());
+ return IScheduledTask.build(builder);
+ }
+ };
+
+ return new TaskAssignMutation() {
+ private AtomicReference<IAssignedTask> assignedTask = Atomics.newReference();
+ @Override public IAssignedTask getAssignedTask() {
+ return assignedTask.get();
+ }
+
+ @Override public Boolean apply(final TaskStateMachine stateMachine) {
+ TaskMutation wrapper = new TaskMutation() {
+ @Override public IScheduledTask apply(IScheduledTask task) {
+ IScheduledTask mutated = mutation.apply(task);
+ Preconditions.checkState(
+ assignedTask.compareAndSet(null, mutated.getAssignedTask()),
+ "More than one result was found for an identity query.");
+ return mutated;
+ }
+ };
+ return stateMachine.updateState(ScheduleStatus.ASSIGNED, wrapper);
+ }
+ };
+ }
+
+ private void processWorkQueueInWriteOperation(
+ SideEffectWork<?, ?> sideEffectWork,
+ MutableStoreProvider storeProvider) {
+
+ for (final WorkEntry work : Iterables.consumingIterable(workQueue)) {
+ final TaskStateMachine stateMachine = work.stateMachine;
+
+ if (work.command == WorkCommand.KILL) {
+ driver.killTask(stateMachine.getTaskId());
+ } else {
+ TaskStore.Mutable taskStore = storeProvider.getUnsafeTaskStore();
+ String taskId = stateMachine.getTaskId();
+ Query.Builder idQuery = Query.taskScoped(taskId);
+
+ switch (work.command) {
+ case RESCHEDULE:
+ ScheduledTask builder =
+ Iterables.getOnlyElement(taskStore.fetchTasks(idQuery)).newBuilder();
+ builder.getAssignedTask().unsetSlaveId();
+ builder.getAssignedTask().unsetSlaveHost();
+ builder.getAssignedTask().unsetAssignedPorts();
+ builder.unsetTaskEvents();
+ builder.setAncestorId(taskId);
+ String newTaskId = taskIdGenerator.generate(
+ ITaskConfig.build(builder.getAssignedTask().getTask()),
+ builder.getAssignedTask().getInstanceId());
+ builder.getAssignedTask().setTaskId(newTaskId);
+
+ LOG.info("Task being rescheduled: " + taskId);
+
+ IScheduledTask task = IScheduledTask.build(builder);
+ taskStore.saveTasks(ImmutableSet.of(task));
+
+ createStateMachine(task).updateState(PENDING, Optional.of("Rescheduled"));
+ ITaskConfig taskInfo = task.getAssignedTask().getTask();
+ sideEffectWork.addTaskEvent(
+ new PubsubEvent.TaskRescheduled(
+ taskInfo.getOwner().getRole(),
+ taskInfo.getJobName(),
+ task.getAssignedTask().getInstanceId()));
+ break;
+
+ case UPDATE_STATE:
+ taskStore.mutateTasks(idQuery, new TaskMutation() {
+ @Override public IScheduledTask apply(IScheduledTask task) {
+ return work.mutation.apply(
+ IScheduledTask.build(task.newBuilder().setStatus(stateMachine.getState())));
+ }
+ });
+ sideEffectWork.addTaskEvent(
+ new PubsubEvent.TaskStateChange(
+ Iterables.getOnlyElement(taskStore.fetchTasks(idQuery)),
+ stateMachine.getPreviousState()));
+ break;
+
+ case DELETE:
+ deleteTasks(ImmutableSet.of(taskId));
+ break;
+
+ case INCREMENT_FAILURES:
+ taskStore.mutateTasks(idQuery, new TaskMutation() {
+ @Override public IScheduledTask apply(IScheduledTask task) {
+ return IScheduledTask.build(
+ task.newBuilder().setFailureCount(task.getFailureCount() + 1));
+ }
+ });
+ break;
+
+ default:
+ LOG.severe("Unrecognized work command type " + work.command);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void deleteTasks(final Set<String> taskIds) {
+ storage.write(storage.new NoResultQuietSideEffectWork() {
+ @Override protected void execute(final MutableStoreProvider storeProvider) {
+ TaskStore.Mutable taskStore = storeProvider.getUnsafeTaskStore();
+ Iterable<IScheduledTask> tasks = taskStore.fetchTasks(Query.taskScoped(taskIds));
+ addTaskEvent(new PubsubEvent.TasksDeleted(ImmutableSet.copyOf(tasks)));
+ taskStore.deleteTasks(taskIds);
+ }
+ });
+ }
+
+ private Map<String, TaskStateMachine> getStateMachines(final Set<String> taskIds) {
+ return storage.consistentRead(new Work.Quiet<Map<String, TaskStateMachine>>() {
+ @Override public Map<String, TaskStateMachine> apply(StoreProvider storeProvider) {
+ Map<String, IScheduledTask> existingTasks = Maps.uniqueIndex(
+ storeProvider.getTaskStore().fetchTasks(Query.taskScoped(taskIds)),
+ new Function<IScheduledTask, String>() {
+ @Override public String apply(IScheduledTask input) {
+ return input.getAssignedTask().getTaskId();
+ }
+ });
+
+ ImmutableMap.Builder<String, TaskStateMachine> builder = ImmutableMap.builder();
+ for (String taskId : taskIds) {
+ // Pass null get() values through.
+ builder.put(taskId, getStateMachine(taskId, existingTasks.get(taskId)));
+ }
+ return builder.build();
+ }
+ });
+ }
+
+ private TaskStateMachine getStateMachine(String taskId, @Nullable IScheduledTask task) {
+ if (task != null) {
+ return createStateMachine(task, task.getStatus());
+ }
+
+ // The task is unknown, not present in storage.
+ TaskStateMachine stateMachine = new TaskStateMachine(
+ taskId,
+ null,
+ workSink,
+ clock,
+ INIT);
+ stateMachine.updateState(UNKNOWN);
+ return stateMachine;
+ }
+
+ private TaskStateMachine createStateMachine(IScheduledTask task) {
+ return createStateMachine(task, INIT);
+ }
+
+ private TaskStateMachine createStateMachine(IScheduledTask task, ScheduleStatus initialState) {
+ return new TaskStateMachine(
+ Tasks.id(task),
+ task,
+ workSink,
+ clock,
+ initialState);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/StateModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/StateModule.java b/src/main/java/org/apache/aurora/scheduler/state/StateModule.java
new file mode 100644
index 0000000..870d085
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/StateModule.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import javax.inject.Singleton;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.AbstractModule;
+import com.google.inject.Binder;
+
+import com.twitter.aurora.scheduler.MesosTaskFactory;
+import com.twitter.aurora.scheduler.MesosTaskFactory.MesosTaskFactoryImpl;
+import com.twitter.aurora.scheduler.events.PubsubEventModule;
+import com.twitter.aurora.scheduler.state.MaintenanceController.MaintenanceControllerImpl;
+import com.twitter.aurora.scheduler.state.TaskAssigner.TaskAssignerImpl;
+import com.twitter.aurora.scheduler.state.UUIDGenerator.UUIDGeneratorImpl;
+
+/**
+ * Binding module for scheduling logic and higher-level state management.
+ */
+public class StateModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(TaskAssigner.class).to(TaskAssignerImpl.class);
+ bind(TaskAssignerImpl.class).in(Singleton.class);
+ bind(MesosTaskFactory.class).to(MesosTaskFactoryImpl.class);
+
+ bind(SchedulerCore.class).to(SchedulerCoreImpl.class).in(Singleton.class);
+
+ bind(StateManager.class).to(StateManagerImpl.class);
+ bind(StateManagerImpl.class).in(Singleton.class);
+
+ bind(UUIDGenerator.class).to(UUIDGeneratorImpl.class);
+ bind(UUIDGeneratorImpl.class).in(Singleton.class);
+ bind(LockManager.class).to(LockManagerImpl.class);
+ bind(LockManagerImpl.class).in(Singleton.class);
+
+ bindCronJobManager(binder());
+ bind(ImmediateJobManager.class).in(Singleton.class);
+
+ bindMaintenanceController(binder());
+ }
+
+ @VisibleForTesting
+ static void bindCronJobManager(Binder binder) {
+ binder.bind(CronJobManager.class).in(Singleton.class);
+ PubsubEventModule.bindSubscriber(binder, CronJobManager.class);
+ }
+
+ @VisibleForTesting
+ static void bindMaintenanceController(Binder binder) {
+ binder.bind(MaintenanceController.class).to(MaintenanceControllerImpl.class);
+ binder.bind(MaintenanceControllerImpl.class).in(Singleton.class);
+ PubsubEventModule.bindSubscriber(binder, MaintenanceControllerImpl.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/TaskAssigner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/TaskAssigner.java b/src/main/java/org/apache/aurora/scheduler/state/TaskAssigner.java
new file mode 100644
index 0000000..c37feae
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/TaskAssigner.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Set;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Optional;
+
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.TaskInfo;
+
+import com.twitter.aurora.scheduler.MesosTaskFactory;
+import com.twitter.aurora.scheduler.ResourceSlot;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.Resources;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Responsible for matching a task against an offer.
+ */
+public interface TaskAssigner {
+
+ /**
+ * Tries to match a task against an offer. If a match is found, the assigner should
+ * make the appropriate changes to the task and provide a non-empty result.
+ *
+ * @param offer The resource offer.
+ * @param task The task to match against and optionally assign.
+ * @return Instructions for launching the task if matching and assignment were successful.
+ */
+ Optional<TaskInfo> maybeAssign(Offer offer, IScheduledTask task);
+
+ class TaskAssignerImpl implements TaskAssigner {
+ private static final Logger LOG = Logger.getLogger(TaskAssignerImpl.class.getName());
+
+ private final StateManager stateManager;
+ private final SchedulingFilter filter;
+ private final MesosTaskFactory taskFactory;
+
+ @Inject
+ public TaskAssignerImpl(
+ StateManager stateManager,
+ SchedulingFilter filter,
+ MesosTaskFactory taskFactory) {
+
+ this.stateManager = checkNotNull(stateManager);
+ this.filter = checkNotNull(filter);
+ this.taskFactory = checkNotNull(taskFactory);
+ }
+
+ private TaskInfo assign(Offer offer, IScheduledTask task) {
+ String host = offer.getHostname();
+ Set<Integer> selectedPorts =
+ Resources.getPorts(offer, task.getAssignedTask().getTask().getRequestedPorts().size());
+ IAssignedTask assigned = stateManager.assignTask(
+ Tasks.id(task),
+ host,
+ offer.getSlaveId(),
+ selectedPorts);
+ LOG.info(String.format("Offer on slave %s (id %s) is being assigned task for %s.",
+ host, offer.getSlaveId(), Tasks.id(task)));
+ return taskFactory.createFrom(assigned, offer.getSlaveId());
+ }
+
+ @Override
+ public Optional<TaskInfo> maybeAssign(Offer offer, IScheduledTask task) {
+ Set<Veto> vetoes = filter.filter(
+ ResourceSlot.from(offer),
+ offer.getHostname(),
+ task.getAssignedTask().getTask(),
+ Tasks.id(task));
+ if (vetoes.isEmpty()) {
+ return Optional.of(assign(offer, task));
+ } else {
+ LOG.fine("Slave " + offer.getHostname() + " vetoed task " + Tasks.id(task)
+ + ": " + vetoes);
+ return Optional.absent();
+ }
+ }
+ }
+}
[41/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java
deleted file mode 100644
index 74e8c07..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/Entries.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.log;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-import java.util.zip.DeflaterOutputStream;
-import java.util.zip.InflaterInputStream;
-
-import com.google.common.base.Preconditions;
-import com.google.common.io.ByteStreams;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.storage.LogEntry;
-import com.twitter.aurora.gen.storage.LogEntry._Fields;
-import com.twitter.common.stats.Stats;
-
-/**
- * Utility class for working with log entries.
- */
-final class Entries {
-
- private static final Logger LOG = Logger.getLogger(Entries.class.getName());
-
- private static final AtomicLong COMPRESSION_BYTES_SAVED =
- Stats.exportLong("log_compressed_entry_bytes_saved");
-
- private Entries() {
- // Utility class.
- }
-
- /**
- * Deflates a log entry and wraps it in a deflated entry.
- * <p>
- * This will encode the entry using the thrift binary codec, and will apply deflate compression to
- * the resulting encoded data.
- * <p>
- * This operation is symmetric with {@link #inflate(LogEntry)}.
- *
- * @param entry Entry to deflate.
- * @return An entry with the {@code deflatedEntry} field set with the deflated serialized value
- * of the original entry.
- * @throws CodingException If the value could not be encoded or deflated.
- */
- static LogEntry deflate(LogEntry entry) throws CodingException {
- byte[] data = thriftBinaryEncode(entry);
- int initialLength = data.length;
- LOG.info("Deflating log entry of size " + initialLength);
- ByteArrayOutputStream deflated = new ByteArrayOutputStream();
- DeflaterOutputStream deflater = new DeflaterOutputStream(deflated);
- try {
- deflater.write(data);
- deflater.flush();
- deflater.close();
- byte[] deflatedData = deflated.toByteArray();
- int bytesSaved = initialLength - deflatedData.length;
- if (bytesSaved < 0) {
- LOG.warning("Deflated entry is larger than original by " + (bytesSaved * -1) + " bytes");
- } else {
- LOG.info("Deflated log entry size: " + deflatedData.length + " (saved " + bytesSaved + ")");
- }
-
- COMPRESSION_BYTES_SAVED.addAndGet(bytesSaved);
- return LogEntry.deflatedEntry(ByteBuffer.wrap(deflatedData));
- } catch (IOException e) {
- throw new CodingException("Failed to deflate snapshot: " + e, e);
- }
- }
-
- /**
- * Inflates and deserializes a deflated log entry.
- * <p>
- * This requires that the {@code deflatedEntry} field is set on the provided {@code entry}.
- * The encoded value will be inflated and deserialized as a {@link LogEntry}.
- *
- * @param entry Entry to inflate, which must be a deflated entry.
- * @return The inflated entry.
- * @throws CodingException If the value could not be inflated or decoded.
- */
- static LogEntry inflate(LogEntry entry) throws CodingException {
- Preconditions.checkArgument(entry.isSet(_Fields.DEFLATED_ENTRY));
-
- ByteArrayOutputStream inflated = new ByteArrayOutputStream();
- ByteBuffer data = entry.bufferForDeflatedEntry();
- LOG.info("Inflating deflated log entry of size " + data.remaining());
- InflaterInputStream inflater = new InflaterInputStream(
- new ByteArrayInputStream(data.array(), data.position(), data.remaining()));
- try {
- ByteStreams.copy(inflater, inflated);
- byte[] inflatedData = inflated.toByteArray();
- LOG.info("Inflated log entry size: " + inflatedData.length);
- return thriftBinaryDecode(inflatedData);
- } catch (IOException e) {
- throw new CodingException("Failed to inflate compressed log entry.", e);
- }
- }
-
- /**
- * Thrift binary-encodes a log entry.
- *
- * @param entry The entry to encode.
- * @return The serialized entry value.
- * @throws CodingException If the entry could not be encoded.
- */
- static byte[] thriftBinaryEncode(LogEntry entry) throws CodingException {
- return ThriftBinaryCodec.encodeNonNull(entry);
- }
-
- /**
- * Decodes a byte array containing thrift binary-encoded data.
- *
- * @param contents The data to decode.
- * @return The deserialized entry.
- * @throws CodingException If the entry could not be deserialized.
- */
- static LogEntry thriftBinaryDecode(byte[] contents) throws CodingException {
- return ThriftBinaryCodec.decodeNonNull(LogEntry.class, contents);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java
deleted file mode 100644
index da29401..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogManager.java
+++ /dev/null
@@ -1,516 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.log;
-
-import java.io.IOException;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.primitives.Bytes;
-import com.google.inject.BindingAnnotation;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.ScheduledTask;
-import com.twitter.aurora.gen.storage.Frame;
-import com.twitter.aurora.gen.storage.FrameChunk;
-import com.twitter.aurora.gen.storage.FrameHeader;
-import com.twitter.aurora.gen.storage.LogEntry;
-import com.twitter.aurora.gen.storage.LogEntry._Fields;
-import com.twitter.aurora.gen.storage.Op;
-import com.twitter.aurora.gen.storage.RemoveTasks;
-import com.twitter.aurora.gen.storage.SaveHostAttributes;
-import com.twitter.aurora.gen.storage.SaveTasks;
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.gen.storage.Transaction;
-import com.twitter.aurora.gen.storage.storageConstants;
-import com.twitter.aurora.scheduler.log.Log;
-import com.twitter.aurora.scheduler.log.Log.Entry;
-import com.twitter.aurora.scheduler.log.Log.Position;
-import com.twitter.aurora.scheduler.log.Log.Stream;
-import com.twitter.aurora.scheduler.log.Log.Stream.InvalidPositionException;
-import com.twitter.aurora.scheduler.log.Log.Stream.StreamAccessException;
-import com.twitter.common.application.ShutdownRegistry;
-import com.twitter.common.base.Closure;
-import com.twitter.common.base.ExceptionalCommand;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Data;
-import com.twitter.common.stats.Stats;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Manages opening, reading from and writing to a {@link Log}.
- */
-public final class LogManager {
-
- /**
- * Identifies the maximum log entry size to permit before chunking entries into frames.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ ElementType.PARAMETER, ElementType.METHOD })
- @BindingAnnotation
- public @interface MaxEntrySize { }
-
- /**
- * Binding annotation for settings regarding the way snapshots are written.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ ElementType.PARAMETER, ElementType.METHOD })
- @BindingAnnotation
- public @interface SnapshotSetting { }
-
- private static final Logger LOG = Logger.getLogger(LogManager.class.getName());
-
- private final Log log;
- private final Amount<Integer, Data> maxEntrySize;
- private final boolean deflateSnapshots;
- private final ShutdownRegistry shutdownRegistry;
-
- @Inject
- LogManager(
- Log log,
- @MaxEntrySize Amount<Integer, Data> maxEntrySize,
- @SnapshotSetting boolean deflateSnapshots,
- ShutdownRegistry shutdownRegistry) {
-
- this.log = checkNotNull(log);
- this.maxEntrySize = checkNotNull(maxEntrySize);
- this.deflateSnapshots = deflateSnapshots;
- this.shutdownRegistry = checkNotNull(shutdownRegistry);
- }
-
- /**
- * Opens the log for reading and writing.
- *
- * @return A stream manager that can be used to manipulate the log stream.
- * @throws IOException If there is a problem opening the log.
- */
- public StreamManager open() throws IOException {
- final Stream stream = log.open();
- shutdownRegistry.addAction(new ExceptionalCommand<IOException>() {
- @Override public void execute() throws IOException {
- stream.close();
- }
- });
- return new StreamManager(stream, deflateSnapshots, maxEntrySize);
- }
-
- /**
- * Manages interaction with the log stream. Log entries can be
- * {@link #readFromBeginning(com.twitter.common.base.Closure) read from} the beginning,
- * a {@link #startTransaction() transaction} consisting of one or more local storage
- * operations can be committed atomically, or the log can be compacted by
- * {@link #snapshot(com.twitter.aurora.gen.storage.Snapshot) snapshotting}.
- */
- public static class StreamManager {
-
- private static MessageDigest createDigest() {
- try {
- return MessageDigest.getInstance("MD5");
- } catch (NoSuchAlgorithmException e) {
- throw new IllegalStateException("Could not find provider for standard algorithm 'MD5'", e);
- }
- }
-
- private static class Vars {
- private final AtomicInteger unSnapshottedTransactions =
- Stats.exportInt("scheduler_log_un_snapshotted_transactions");
- private final AtomicLong bytesWritten = Stats.exportLong("scheduler_log_bytes_written");
- private final AtomicLong entriesWritten = Stats.exportLong("scheduler_log_entries_written");
- private final AtomicLong badFramesRead = Stats.exportLong("scheduler_log_bad_frames_read");
- private final AtomicLong bytesRead = Stats.exportLong("scheduler_log_bytes_read");
- private final AtomicLong entriesRead = Stats.exportLong("scheduler_log_entries_read");
- private final AtomicLong deflatedEntriesRead =
- Stats.exportLong("scheduler_log_deflated_entries_read");
- private final AtomicLong snapshots = Stats.exportLong("scheduler_log_snapshots");
- }
- private final Vars vars = new Vars();
-
- private final Object writeMutex = new Object();
- private final Stream stream;
- private final boolean deflateSnapshots;
- private final MessageDigest digest;
- private final EntrySerializer entrySerializer;
-
- StreamManager(Stream stream, boolean deflateSnapshots, Amount<Integer, Data> maxEntrySize) {
- this.stream = checkNotNull(stream);
- this.deflateSnapshots = deflateSnapshots;
- digest = createDigest();
- entrySerializer = new EntrySerializer(digest, maxEntrySize);
- }
-
- /**
- * Reads all entries in the log stream after the given position. If the position
- * supplied is {@code null} then all log entries in the stream will be read.
- *
- * @param reader A reader that will be handed log entries decoded from the stream.
- * @throws CodingException if there was a problem decoding a log entry from the stream.
- * @throws InvalidPositionException if the given position is not found in the log.
- * @throws StreamAccessException if there is a problem reading from the log.
- */
- public void readFromBeginning(Closure<LogEntry> reader)
- throws CodingException, InvalidPositionException, StreamAccessException {
-
- Iterator<Entry> entries = stream.readAll();
-
- while (entries.hasNext()) {
- LogEntry logEntry = decodeLogEntry(entries.next());
- while (logEntry != null && isFrame(logEntry)) {
- logEntry = tryDecodeFrame(logEntry.getFrame(), entries);
- }
- if (logEntry != null) {
- if (logEntry.isSet(_Fields.DEFLATED_ENTRY)) {
- logEntry = Entries.inflate(logEntry);
- vars.deflatedEntriesRead.incrementAndGet();
- }
-
- reader.execute(logEntry);
- vars.entriesRead.incrementAndGet();
- }
- }
- }
-
- @Nullable
- private LogEntry tryDecodeFrame(Frame frame, Iterator<Entry> entries) throws CodingException {
- if (!isHeader(frame)) {
- LOG.warning("Found a frame with no preceding header, skipping.");
- return null;
- }
- FrameHeader header = frame.getHeader();
- byte[][] chunks = new byte[header.chunkCount][];
-
- digest.reset();
- for (int i = 0; i < header.chunkCount; i++) {
- if (!entries.hasNext()) {
- logBadFrame(header, i);
- return null;
- }
- LogEntry logEntry = decodeLogEntry(entries.next());
- if (!isFrame(logEntry)) {
- logBadFrame(header, i);
- return logEntry;
- }
- Frame chunkFrame = logEntry.getFrame();
- if (!isChunk(chunkFrame)) {
- logBadFrame(header, i);
- return logEntry;
- }
- byte[] chunkData = chunkFrame.getChunk().getData();
- digest.update(chunkData);
- chunks[i] = chunkData;
- }
- if (!Arrays.equals(header.getChecksum(), digest.digest())) {
- throw new CodingException("Read back a framed log entry that failed its checksum");
- }
- return Entries.thriftBinaryDecode(Bytes.concat(chunks));
- }
-
- private static boolean isFrame(LogEntry logEntry) {
- return logEntry.getSetField() == LogEntry._Fields.FRAME;
- }
-
- private static boolean isChunk(Frame frame) {
- return frame.getSetField() == Frame._Fields.CHUNK;
- }
-
- private static boolean isHeader(Frame frame) {
- return frame.getSetField() == Frame._Fields.HEADER;
- }
-
- private void logBadFrame(FrameHeader header, int chunkIndex) {
- LOG.info(String.format("Found an aborted transaction, required %d frames and found %d",
- header.chunkCount, chunkIndex));
- vars.badFramesRead.incrementAndGet();
- }
-
- private LogEntry decodeLogEntry(Entry entry) throws CodingException {
- byte[] contents = entry.contents();
- vars.bytesRead.addAndGet(contents.length);
- return Entries.thriftBinaryDecode(contents);
- }
-
- /**
- * Truncates all entries in the log stream occuring before the given position. The entry at the
- * given position becomes the first entry in the stream when this call completes.
- *
- * @param position The last position to keep in the stream.
- * @throws InvalidPositionException if the specified position does not exist in this log.
- * @throws StreamAccessException if the stream could not be truncated.
- */
- void truncateBefore(Position position) {
- stream.truncateBefore(position);
- }
-
- /**
- * Starts a transaction that can be used to commit a series of {@link Op}s to the log stream
- * atomically.
- *
- * @return StreamTransaction A transaction manager to handle batching up commits to the
- * underlying stream.
- */
- StreamTransaction startTransaction() {
- return new StreamTransaction();
- }
-
- /**
- * Adds a snapshot to the log and if successful, truncates the log entries preceding the
- * snapshot.
- *
- * @param snapshot The snapshot to add.
- * @throws CodingException if the was a problem encoding the snapshot into a log entry.
- * @throws InvalidPositionException if there was a problem truncating before the snapshot.
- * @throws StreamAccessException if there was a problem appending the snapshot to the log.
- */
- @Timed("log_manager_snapshot")
- void snapshot(Snapshot snapshot)
- throws CodingException, InvalidPositionException, StreamAccessException {
-
- LogEntry entry = LogEntry.snapshot(snapshot);
- if (deflateSnapshots) {
- entry = Entries.deflate(entry);
- }
-
- Position position = appendAndGetPosition(entry);
- vars.snapshots.incrementAndGet();
- vars.unSnapshottedTransactions.set(0);
- stream.truncateBefore(position);
- }
-
- @Timed("log_manager_append")
- private Position appendAndGetPosition(LogEntry logEntry) throws CodingException {
- Position firstPosition = null;
- byte[][] entries = entrySerializer.serialize(logEntry);
- synchronized (writeMutex) { // ensure all sub-entries are written as a unit
- for (byte[] entry : entries) {
- Position position = stream.append(entry);
- if (firstPosition == null) {
- firstPosition = position;
- }
- vars.bytesWritten.addAndGet(entry.length);
- }
- }
- vars.entriesWritten.incrementAndGet();
- return firstPosition;
- }
-
- @VisibleForTesting
- public static class EntrySerializer {
- private final MessageDigest digest;
- private final int maxEntrySizeBytes;
-
- private EntrySerializer(MessageDigest digest, Amount<Integer, Data> maxEntrySize) {
- this.digest = checkNotNull(digest);
- maxEntrySizeBytes = maxEntrySize.as(Data.BYTES);
- }
-
- public EntrySerializer(Amount<Integer, Data> maxEntrySize) {
- this(createDigest(), maxEntrySize);
- }
-
- /**
- * Serializes a log entry and splits it into chunks no larger than {@code maxEntrySizeBytes}.
- *
- * @param logEntry The log entry to serialize.
- * @return Serialized and chunked log entry.
- * @throws CodingException If the entry could not be serialized.
- */
- @VisibleForTesting
- public byte[][] serialize(LogEntry logEntry) throws CodingException {
- byte[] entry = Entries.thriftBinaryEncode(logEntry);
- if (entry.length <= maxEntrySizeBytes) {
- return new byte[][] {entry};
- }
-
- int chunks = (int) Math.ceil(entry.length / (double) maxEntrySizeBytes);
- byte[][] frames = new byte[chunks + 1][];
-
- frames[0] = encode(Frame.header(new FrameHeader(chunks, ByteBuffer.wrap(checksum(entry)))));
- for (int i = 0; i < chunks; i++) {
- int offset = i * maxEntrySizeBytes;
- ByteBuffer chunk =
- ByteBuffer.wrap(entry, offset, Math.min(maxEntrySizeBytes, entry.length - offset));
- frames[i + 1] = encode(Frame.chunk(new FrameChunk(chunk)));
- }
- return frames;
- }
-
- private byte[] checksum(byte[] data) {
- digest.reset();
- return digest.digest(data);
- }
-
- private static byte[] encode(Frame frame) throws CodingException {
- return Entries.thriftBinaryEncode(LogEntry.frame(frame));
- }
- }
-
- /**
- * Manages a single log stream append transaction. Local storage ops can be added to the
- * transaction and then later committed as an atomic unit.
- */
- final class StreamTransaction {
- private final Transaction transaction =
- new Transaction().setSchemaVersion(storageConstants.CURRENT_SCHEMA_VERSION);
- private final AtomicBoolean committed = new AtomicBoolean(false);
-
- private StreamTransaction() {
- // supplied by factory method
- }
-
- /**
- * Appends any ops that have been added to this transaction to the log stream in a single
- * atomic record.
- *
- * @return The position of the log entry committed in this transaction, if any.
- * @throws CodingException If there was a problem encoding a log entry for commit.
- */
- Position commit() throws CodingException {
- Preconditions.checkState(!committed.getAndSet(true),
- "Can only call commit once per transaction.");
-
- if (!transaction.isSetOps()) {
- return null;
- }
-
- Position position = appendAndGetPosition(LogEntry.transaction(transaction));
- vars.unSnapshottedTransactions.incrementAndGet();
- return position;
- }
-
- /**
- * Adds a local storage operation to this transaction.
- *
- * @param op The local storage op to add.
- */
- void add(Op op) {
- Preconditions.checkState(!committed.get());
-
- Op prior = transaction.isSetOps() ? Iterables.getLast(transaction.getOps(), null) : null;
- if (prior == null || !coalesce(prior, op)) {
- transaction.addToOps(op);
- }
- }
-
- /**
- * Tries to coalesce a new op into the prior to compact the binary representation and increase
- * batching.
- *
- * <p>Its recommended that as new {@code Op}s are added, they be treated here although they
- * need not be</p>
- *
- * @param prior The previous op.
- * @param next The next op to be added.
- * @return {@code true} if the next op was coalesced into the prior, {@code false} otherwise.
- */
- private boolean coalesce(Op prior, Op next) {
- if (!prior.isSet() && !next.isSet()) {
- return false;
- }
-
- Op._Fields priorType = prior.getSetField();
- if (!priorType.equals(next.getSetField())) {
- return false;
- }
-
- switch (priorType) {
- case SAVE_FRAMEWORK_ID:
- prior.setSaveFrameworkId(next.getSaveFrameworkId());
- return true;
-
- case SAVE_ACCEPTED_JOB:
- case REMOVE_JOB:
- case SAVE_QUOTA:
- case REMOVE_QUOTA:
- return false;
-
- case SAVE_TASKS:
- coalesce(prior.getSaveTasks(), next.getSaveTasks());
- return true;
- case REMOVE_TASKS:
- coalesce(prior.getRemoveTasks(), next.getRemoveTasks());
- return true;
- case SAVE_HOST_ATTRIBUTES:
- return coalesce(prior.getSaveHostAttributes(), next.getSaveHostAttributes());
- default:
- LOG.warning("Unoptimized op: " + priorType);
- return false;
- }
- }
-
- private void coalesce(SaveTasks prior, SaveTasks next) {
- if (next.isSetTasks()) {
- if (prior.isSetTasks()) {
- // It is an expected invariant that an operation may reference a task (identified by
- // task ID) no more than one time. Therefore, to coalesce two SaveTasks operations,
- // the most recent task definition overrides the prior operation.
- Map<String, ScheduledTask> coalesced = Maps.newHashMap();
- for (ScheduledTask task : prior.getTasks()) {
- coalesced.put(task.getAssignedTask().getTaskId(), task);
- }
- for (ScheduledTask task : next.getTasks()) {
- coalesced.put(task.getAssignedTask().getTaskId(), task);
- }
- prior.setTasks(ImmutableSet.copyOf(coalesced.values()));
- } else {
- prior.setTasks(next.getTasks());
- }
- }
- }
-
- private void coalesce(RemoveTasks prior, RemoveTasks next) {
- if (next.isSetTaskIds()) {
- if (prior.isSetTaskIds()) {
- prior.setTaskIds(ImmutableSet.<String>builder()
- .addAll(prior.getTaskIds())
- .addAll(next.getTaskIds())
- .build());
- } else {
- prior.setTaskIds(next.getTaskIds());
- }
- }
- }
-
- private boolean coalesce(SaveHostAttributes prior, SaveHostAttributes next) {
- if (prior.getHostAttributes().getHost().equals(next.getHostAttributes().getHost())) {
- prior.getHostAttributes().setAttributes(next.getHostAttributes().getAttributes());
- return true;
- }
- return false;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorage.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorage.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorage.java
deleted file mode 100644
index 74f06aa..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorage.java
+++ /dev/null
@@ -1,715 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.log;
-
-import java.io.IOException;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.Date;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-import com.google.inject.BindingAnnotation;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.gen.storage.LogEntry;
-import com.twitter.aurora.gen.storage.Op;
-import com.twitter.aurora.gen.storage.RemoveJob;
-import com.twitter.aurora.gen.storage.RemoveLock;
-import com.twitter.aurora.gen.storage.RemoveQuota;
-import com.twitter.aurora.gen.storage.RemoveTasks;
-import com.twitter.aurora.gen.storage.RewriteTask;
-import com.twitter.aurora.gen.storage.SaveAcceptedJob;
-import com.twitter.aurora.gen.storage.SaveFrameworkId;
-import com.twitter.aurora.gen.storage.SaveHostAttributes;
-import com.twitter.aurora.gen.storage.SaveLock;
-import com.twitter.aurora.gen.storage.SaveQuota;
-import com.twitter.aurora.gen.storage.SaveTasks;
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.SchedulerException;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.log.Log.Stream.InvalidPositionException;
-import com.twitter.aurora.scheduler.log.Log.Stream.StreamAccessException;
-import com.twitter.aurora.scheduler.storage.AttributeStore;
-import com.twitter.aurora.scheduler.storage.DistributedSnapshotStore;
-import com.twitter.aurora.scheduler.storage.ForwardingStore;
-import com.twitter.aurora.scheduler.storage.JobStore;
-import com.twitter.aurora.scheduler.storage.LockStore;
-import com.twitter.aurora.scheduler.storage.QuotaStore;
-import com.twitter.aurora.scheduler.storage.SchedulerStore;
-import com.twitter.aurora.scheduler.storage.SnapshotStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
-import com.twitter.aurora.scheduler.storage.TaskStore;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.ILockKey;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.aurora.scheduler.storage.log.LogManager.StreamManager;
-import com.twitter.aurora.scheduler.storage.log.LogManager.StreamManager.StreamTransaction;
-import com.twitter.common.application.ShutdownRegistry;
-import com.twitter.common.base.Closure;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.util.concurrent.ExecutorServiceShutdown;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * A storage implementation that ensures committed transactions are written to a log.
- *
- * <p>In the classic write-ahead log usage we'd perform mutations as follows:
- * <ol>
- * <li>write op to log</li>
- * <li>perform op locally</li>
- * <li>*checkpoint</li>
- * </ol>
- *
- * <p>Writing the operation to the log provides us with a fast persistence mechanism to ensure we
- * have a record of our mutation in case we should need to recover state later after a crash or on
- * a new host (assuming the log is distributed). We then apply the mutation to a local (in-memory)
- * data structure for serving fast read requests and then optionally write down the position of the
- * log entry we wrote in the first step to stable storage to allow for quicker recovery after a
- * crash. Instead of reading the whole log, we can read all entries past the checkpoint. This
- * design implies that all mutations must be idempotent and free from constraint and thus
- * replayable over newer operations when recovering from an old checkpoint.
- *
- * <p>The important detail in our case is the possibility of writing an op to the log, and then
- * failing to commit locally since we use a local database instead of an in-memory data structure.
- * If we die after such a failure, then another instance can read and apply the logged op
- * erroneously.
- *
- * <p>This implementation leverages a local transaction to handle this:
- * <ol>
- * <li>start local transaction</li>
- * <li>perform op locally (uncommitted!)</li>
- * <li>write op to log</li>
- * <li>commit local transaction</li>
- * <li>*checkpoint</li>
- * </ol>
- *
- * <p>If the op fails to apply to local storage we will never write the op to the log and if the op
- * fails to apply to the log, it'll throw and abort the local storage transaction as well.
- */
-public class LogStorage extends ForwardingStore
- implements NonVolatileStorage, DistributedSnapshotStore {
-
- /**
- * A service that can schedule an action to be executed periodically.
- */
- @VisibleForTesting
- interface SchedulingService {
-
- /**
- * Schedules an action to execute periodically.
- *
- * @param interval The time period to wait until running the {@code action} again.
- * @param action The action to execute periodically.
- */
- void doEvery(Amount<Long, Time> interval, Runnable action);
- }
-
- private static class ScheduledExecutorSchedulingService implements SchedulingService {
- private final ScheduledExecutorService scheduledExecutor;
-
- ScheduledExecutorSchedulingService(ShutdownRegistry shutdownRegistry,
- Amount<Long, Time> shutdownGracePeriod) {
- scheduledExecutor = Executors.newSingleThreadScheduledExecutor();
- shutdownRegistry.addAction(
- new ExecutorServiceShutdown(scheduledExecutor, shutdownGracePeriod));
- }
-
- @Override
- public void doEvery(Amount<Long, Time> interval, Runnable action) {
- checkNotNull(interval);
- checkNotNull(action);
-
- long delay = interval.getValue();
- TimeUnit timeUnit = interval.getUnit().getTimeUnit();
- scheduledExecutor.scheduleWithFixedDelay(action, delay, delay, timeUnit);
- }
- }
-
- private static final Logger LOG = Logger.getLogger(LogStorage.class.getName());
-
- private final LogManager logManager;
- private final SchedulingService schedulingService;
- private final SnapshotStore<Snapshot> snapshotStore;
- private final Amount<Long, Time> snapshotInterval;
-
- private StreamManager streamManager;
-
- private boolean recovered = false;
- private StreamTransaction transaction = null;
-
- private final MutableStoreProvider logStoreProvider = new MutableStoreProvider() {
- @Override public SchedulerStore.Mutable getSchedulerStore() {
- return LogStorage.this;
- }
-
- @Override public JobStore.Mutable getJobStore() {
- return LogStorage.this;
- }
-
- @Override public TaskStore getTaskStore() {
- return LogStorage.this;
- }
-
- @Override public TaskStore.Mutable getUnsafeTaskStore() {
- return LogStorage.this;
- }
-
- @Override public LockStore.Mutable getLockStore() {
- return LogStorage.this;
- }
-
- @Override public QuotaStore.Mutable getQuotaStore() {
- return LogStorage.this;
- }
-
- @Override public AttributeStore.Mutable getAttributeStore() {
- return LogStorage.this;
- }
- };
-
- /**
- * Identifies the grace period to give in-process snapshots and checkpoints to complete during
- * shutdown.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ ElementType.PARAMETER, ElementType.METHOD })
- @BindingAnnotation
- public @interface ShutdownGracePeriod { }
-
- /**
- * Identifies the interval between snapshots of local storage truncating the log.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ ElementType.PARAMETER, ElementType.METHOD })
- @BindingAnnotation
- public @interface SnapshotInterval { }
-
- /**
- * Identifies a local storage layer that is written to only after first ensuring the write
- * operation is persisted in the log.
- */
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ ElementType.PARAMETER, ElementType.METHOD })
- @BindingAnnotation
- public @interface WriteBehind { }
-
- @Inject
- LogStorage(LogManager logManager,
- ShutdownRegistry shutdownRegistry,
- @ShutdownGracePeriod Amount<Long, Time> shutdownGracePeriod,
- SnapshotStore<Snapshot> snapshotStore,
- @SnapshotInterval Amount<Long, Time> snapshotInterval,
- @WriteBehind Storage storage,
- @WriteBehind SchedulerStore.Mutable schedulerStore,
- @WriteBehind JobStore.Mutable jobStore,
- @WriteBehind TaskStore.Mutable taskStore,
- @WriteBehind LockStore.Mutable lockStore,
- @WriteBehind QuotaStore.Mutable quotaStore,
- @WriteBehind AttributeStore.Mutable attributeStore) {
-
- this(logManager,
- new ScheduledExecutorSchedulingService(shutdownRegistry, shutdownGracePeriod),
- snapshotStore,
- snapshotInterval,
- storage,
- schedulerStore,
- jobStore,
- taskStore,
- lockStore,
- quotaStore,
- attributeStore);
- }
-
- @VisibleForTesting
- LogStorage(LogManager logManager,
- SchedulingService schedulingService,
- SnapshotStore<Snapshot> snapshotStore,
- Amount<Long, Time> snapshotInterval,
- Storage storage,
- SchedulerStore.Mutable schedulerStore,
- JobStore.Mutable jobStore,
- TaskStore.Mutable taskStore,
- LockStore.Mutable lockStore,
- QuotaStore.Mutable quotaStore,
- AttributeStore.Mutable attributeStore) {
-
- super(storage, schedulerStore, jobStore, taskStore, lockStore, quotaStore, attributeStore);
- this.logManager = checkNotNull(logManager);
- this.schedulingService = checkNotNull(schedulingService);
- this.snapshotStore = checkNotNull(snapshotStore);
- this.snapshotInterval = checkNotNull(snapshotInterval);
- }
-
- @Override
- public synchronized void prepare() {
- // Open the log to make a log replica available to the scheduler group.
- try {
- streamManager = logManager.open();
- } catch (IOException e) {
- throw new IllegalStateException("Failed to open the log, cannot continue", e);
- }
-
- // TODO(John Sirois): start incremental recovery here from the log and do a final recovery
- // catchup in start after shutting down the incremental syncer.
- }
-
- @Override
- public synchronized void start(final MutateWork.NoResult.Quiet initializationLogic) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- // Must have the underlying storage started so we can query it for the last checkpoint.
- // We replay these entries in the forwarded storage system's transactions but not ours - we
- // do not want to re-record these ops to the log.
- recover();
- recovered = true;
-
- // Now that we're recovered we should let any mutations done in initializationLogic append
- // to the log, so run it in one of our transactions.
- write(initializationLogic);
- }
- });
-
- scheduleSnapshots();
- }
-
- @Override
- public void stop() {
- // No-op.
- }
-
- @Timed("scheduler_log_recover")
- void recover() throws RecoveryFailedException {
- try {
- streamManager.readFromBeginning(new Closure<LogEntry>() {
- @Override public void execute(LogEntry logEntry) {
- replay(logEntry);
- }
- });
- } catch (CodingException | InvalidPositionException | StreamAccessException e) {
- throw new RecoveryFailedException(e);
- }
- }
-
- private static final class RecoveryFailedException extends SchedulerException {
- private RecoveryFailedException(Throwable cause) {
- super(cause);
- }
- }
-
- void replay(final LogEntry logEntry) {
- switch (logEntry.getSetField()) {
- case SNAPSHOT:
- Snapshot snapshot = logEntry.getSnapshot();
- LOG.info("Applying snapshot taken on " + new Date(snapshot.getTimestamp()));
- snapshotStore.applySnapshot(snapshot);
- break;
-
- case TRANSACTION:
- for (Op op : logEntry.getTransaction().getOps()) {
- replayOp(op);
- }
- break;
-
- case NOOP:
- // Nothing to do here
- break;
-
- case DEFLATED_ENTRY:
- throw new IllegalArgumentException("Deflated entries are not handled at this layer.");
-
- case FRAME:
- throw new IllegalArgumentException("Framed entries are not handled at this layer.");
-
- default:
- throw new IllegalStateException("Unknown log entry type: " + logEntry);
- }
- }
-
- private void replayOp(Op op) {
- switch (op.getSetField()) {
- case SAVE_FRAMEWORK_ID:
- saveFrameworkId(op.getSaveFrameworkId().getId());
- break;
-
- case SAVE_ACCEPTED_JOB:
- SaveAcceptedJob acceptedJob = op.getSaveAcceptedJob();
- saveAcceptedJob(
- acceptedJob.getManagerId(),
- IJobConfiguration.build(acceptedJob.getJobConfig()));
- break;
-
- case REMOVE_JOB:
- removeJob(IJobKey.build(op.getRemoveJob().getJobKey()));
- break;
-
- case SAVE_TASKS:
- saveTasks(IScheduledTask.setFromBuilders(op.getSaveTasks().getTasks()));
- break;
-
- case REWRITE_TASK:
- RewriteTask rewriteTask = op.getRewriteTask();
- unsafeModifyInPlace(rewriteTask.getTaskId(), ITaskConfig.build(rewriteTask.getTask()));
- break;
-
- case REMOVE_TASKS:
- deleteTasks(op.getRemoveTasks().getTaskIds());
- break;
-
- case SAVE_QUOTA:
- SaveQuota saveQuota = op.getSaveQuota();
- saveQuota(saveQuota.getRole(), IQuota.build(saveQuota.getQuota()));
- break;
-
- case REMOVE_QUOTA:
- removeQuota(op.getRemoveQuota().getRole());
- break;
-
- case SAVE_HOST_ATTRIBUTES:
- saveHostAttributes(op.getSaveHostAttributes().hostAttributes);
- break;
-
- case SAVE_LOCK:
- saveLock(ILock.build(op.getSaveLock().getLock()));
- break;
-
- case REMOVE_LOCK:
- removeLock(ILockKey.build(op.getRemoveLock().getLockKey()));
- break;
-
- default:
- throw new IllegalStateException("Unknown transaction op: " + op);
- }
- }
-
- private void scheduleSnapshots() {
- if (snapshotInterval.getValue() > 0) {
- schedulingService.doEvery(snapshotInterval, new Runnable() {
- @Override public void run() {
- try {
- snapshot();
- } catch (StorageException e) {
- if (e.getCause() != null) {
- LOG.log(Level.WARNING, e.getMessage(), e.getCause());
- } else {
- LOG.log(Level.WARNING, "StorageException when attempting to snapshot.", e);
- }
- }
- }
- });
- }
- }
-
- /**
- * Forces a snapshot of the storage state.
- *
- * @throws CodingException If there is a problem encoding the snapshot.
- * @throws InvalidPositionException If the log stream cursor is invalid.
- * @throws StreamAccessException If there is a problem writing the snapshot to the log stream.
- */
- @Timed("scheduler_log_snapshot")
- void doSnapshot() throws CodingException, InvalidPositionException, StreamAccessException {
- super.write(new MutateWork.NoResult<CodingException>() {
- @Override protected void execute(MutableStoreProvider unused)
- throws CodingException, InvalidPositionException, StreamAccessException {
-
- persist(snapshotStore.createSnapshot());
- }
- });
- }
-
- @Timed("scheduler_log_snapshot_persist")
- @Override
- public void persist(Snapshot snapshot)
- throws CodingException, InvalidPositionException, StreamAccessException {
-
- streamManager.snapshot(snapshot);
- }
-
- @Override
- public synchronized <T, E extends Exception> T write(final MutateWork<T, E> work)
- throws StorageException, E {
-
- // We don't want to use the log when recovering from it, we just want to update the underlying
- // store - so pass mutations straight through to the underlying storage.
- if (!recovered) {
- return super.write(work);
- }
-
- // The log stream transaction has already been set up so we just need to delegate with our
- // store provider so any mutations performed by work get logged.
- if (transaction != null) {
- return super.write(new MutateWork<T, E>() {
- @Override public T apply(MutableStoreProvider unused) throws E {
- return work.apply(logStoreProvider);
- }
- });
- }
-
- transaction = streamManager.startTransaction();
- try {
- return super.write(new MutateWork<T, E>() {
- @Override public T apply(MutableStoreProvider unused) throws E {
- T result = work.apply(logStoreProvider);
- try {
- transaction.commit();
- } catch (CodingException e) {
- throw new IllegalStateException(
- "Problem encoding transaction operations to the log stream", e);
- } catch (StreamAccessException e) {
- throw new StorageException(
- "There was a problem committing the transaction to the log.", e);
- }
- return result;
- }
- });
- } finally {
- transaction = null;
- }
- }
-
- @Timed("scheduler_log_save_framework_id")
- @Override
- public void saveFrameworkId(final String frameworkId) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.saveFrameworkId(new SaveFrameworkId(frameworkId)));
- LogStorage.super.saveFrameworkId(frameworkId);
- }
- });
- }
-
- @Timed("scheduler_log_job_save")
- @Override
- public void saveAcceptedJob(final String managerId, final IJobConfiguration jobConfig) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.saveAcceptedJob(new SaveAcceptedJob(managerId, jobConfig.newBuilder())));
- LogStorage.super.saveAcceptedJob(managerId, jobConfig);
- }
- });
- }
-
- @Timed("scheduler_log_job_remove")
- @Override
- public void removeJob(final IJobKey jobKey) {
- checkNotNull(jobKey);
-
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.removeJob(new RemoveJob().setJobKey(jobKey.newBuilder())));
- LogStorage.super.removeJob(jobKey);
- }
- });
- }
-
- @Timed("scheduler_log_tasks_save")
- @Override
- public void saveTasks(final Set<IScheduledTask> newTasks) throws IllegalStateException {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.saveTasks(new SaveTasks(IScheduledTask.toBuildersSet(newTasks))));
- LogStorage.super.saveTasks(newTasks);
- }
- });
- }
-
- @Override
- public void deleteAllTasks() {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- Query.Builder query = Query.unscoped();
- Set<String> ids = FluentIterable.from(storeProvider.getTaskStore().fetchTasks(query))
- .transform(Tasks.SCHEDULED_TO_ID)
- .toSet();
- deleteTasks(ids);
- }
- });
- }
-
- @Timed("scheduler_log_tasks_remove")
- @Override
- public void deleteTasks(final Set<String> taskIds) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.removeTasks(new RemoveTasks(taskIds)));
- LogStorage.super.deleteTasks(taskIds);
- }
- });
- }
-
- @Timed("scheduler_log_tasks_mutate")
- @Override
- public ImmutableSet<IScheduledTask> mutateTasks(
- final Query.Builder query,
- final Function<IScheduledTask, IScheduledTask> mutator) {
-
- return write(new MutateWork.Quiet<ImmutableSet<IScheduledTask>>() {
- @Override public ImmutableSet<IScheduledTask> apply(MutableStoreProvider unused) {
- ImmutableSet<IScheduledTask> mutated = LogStorage.super.mutateTasks(query, mutator);
-
- Map<String, IScheduledTask> tasksById = Tasks.mapById(mutated);
- if (LOG.isLoggable(Level.FINE)) {
- LOG.fine("Storing updated tasks to log: "
- + Maps.transformValues(tasksById, Tasks.GET_STATUS));
- }
-
- // TODO(William Farner): Avoid writing an op when mutated is empty.
- log(Op.saveTasks(new SaveTasks(IScheduledTask.toBuildersSet(mutated))));
- return mutated;
- }
- });
- }
-
- @Timed("scheduler_log_unsafe_modify_in_place")
- @Override
- public boolean unsafeModifyInPlace(final String taskId, final ITaskConfig taskConfiguration) {
- return write(new MutateWork.Quiet<Boolean>() {
- @Override public Boolean apply(MutableStoreProvider storeProvider) {
- boolean mutated = LogStorage.super.unsafeModifyInPlace(taskId, taskConfiguration);
- if (mutated) {
- log(Op.rewriteTask(new RewriteTask(taskId, taskConfiguration.newBuilder())));
- }
- return mutated;
- }
- });
- }
-
- @Timed("scheduler_log_quota_remove")
- @Override
- public void removeQuota(final String role) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.removeQuota(new RemoveQuota(role)));
- LogStorage.super.removeQuota(role);
- }
- });
- }
-
- @Timed("scheduler_log_quota_save")
- @Override
- public void saveQuota(final String role, final IQuota quota) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.saveQuota(new SaveQuota(role, quota.newBuilder())));
- LogStorage.super.saveQuota(role, quota);
- }
- });
- }
-
- @Timed("scheduler_save_host_attribute")
- @Override
- public void saveHostAttributes(final HostAttributes attrs) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- // Pass the updated attributes upstream, and then check if the stored value changes.
- // We do this since different parts of the system write partial HostAttributes objects
- // and they are merged together internally.
- // TODO(William Farner): Split out a separate method
- // saveAttributes(String host, Iterable<Attributes>) to simplify this.
- Optional<HostAttributes> saved = LogStorage.super.getHostAttributes(attrs.getHost());
- LogStorage.super.saveHostAttributes(attrs);
- Optional<HostAttributes> updated = LogStorage.super.getHostAttributes(attrs.getHost());
- if (!saved.equals(updated)) {
- log(Op.saveHostAttributes(new SaveHostAttributes(updated.get())));
- }
- }
- });
- }
-
- @Timed("scheduler_lock_save")
- @Override
- public void saveLock(final ILock lock) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.saveLock(new SaveLock(lock.newBuilder())));
- LogStorage.super.saveLock(lock);
- }
- });
- }
-
- @Timed("scheduler_lock_remove")
- @Override
- public void removeLock(final ILockKey lockKey) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- log(Op.removeLock(new RemoveLock(lockKey.newBuilder())));
- LogStorage.super.removeLock(lockKey);
- }
- });
- }
-
- @Override
- public boolean setMaintenanceMode(final String host, final MaintenanceMode mode) {
- write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider unused) {
- Optional<HostAttributes> saved = LogStorage.super.getHostAttributes(host);
- if (saved.isPresent()) {
- HostAttributes attributes = saved.get().setMode(mode);
- log(Op.saveHostAttributes(new SaveHostAttributes(attributes)));
- LogStorage.super.saveHostAttributes(attributes);
- }
- }
- });
- return false;
- }
-
- @Override
- public void snapshot() throws StorageException {
- try {
- doSnapshot();
- } catch (CodingException e) {
- throw new StorageException("Failed to encode a snapshot", e);
- } catch (InvalidPositionException e) {
- throw new StorageException("Saved snapshot but failed to truncate entries preceding it", e);
- } catch (StreamAccessException e) {
- throw new StorageException("Failed to create a snapshot", e);
- }
- super.snapshot();
- }
-
- private void log(Op op) {
- if (recovered) {
- transaction.add(op);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorageModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorageModule.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorageModule.java
deleted file mode 100644
index 92568c8..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/LogStorageModule.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.log;
-
-import java.lang.annotation.Annotation;
-
-import javax.inject.Singleton;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.inject.AbstractModule;
-import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
-
-import com.twitter.aurora.scheduler.log.Log;
-import com.twitter.aurora.scheduler.storage.CallOrderEnforcingStorage;
-import com.twitter.aurora.scheduler.storage.DistributedSnapshotStore;
-import com.twitter.aurora.scheduler.storage.log.LogManager.MaxEntrySize;
-import com.twitter.aurora.scheduler.storage.log.LogManager.SnapshotSetting;
-import com.twitter.aurora.scheduler.storage.log.LogStorage.ShutdownGracePeriod;
-import com.twitter.aurora.scheduler.storage.log.LogStorage.SnapshotInterval;
-import com.twitter.common.application.ShutdownRegistry;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Data;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.util.Clock;
-
-/**
- * Bindings for scheduler distributed log based storage.
- * <p/>
- * Requires bindings for:
- * <ul>
- * <li>{@link Clock}</li>
- * <li>{@link ShutdownRegistry}</li>
- * <li>The concrete {@link Log} implementation.</li>
- * </ul>
- * <p/>
- */
-public class LogStorageModule extends AbstractModule {
-
- @CmdLine(name = "dlog_shutdown_grace_period",
- help = "Specifies the maximum time to wait for scheduled checkpoint and snapshot "
- + "actions to complete before forcibly shutting down.")
- private static final Arg<Amount<Long, Time>> SHUTDOWN_GRACE_PERIOD =
- Arg.create(Amount.of(2L, Time.SECONDS));
-
- @CmdLine(name = "dlog_snapshot_interval",
- help = "Specifies the frequency at which snapshots of local storage are taken and "
- + "written to the log.")
- private static final Arg<Amount<Long, Time>> SNAPSHOT_INTERVAL =
- Arg.create(Amount.of(1L, Time.HOURS));
-
- @CmdLine(name = "dlog_max_entry_size",
- help = "Specifies the maximum entry size to append to the log. Larger entries will be "
- + "split across entry Frames.")
- @VisibleForTesting
- public static final Arg<Amount<Integer, Data>> MAX_LOG_ENTRY_SIZE =
- Arg.create(Amount.of(512, Data.KB));
-
- @CmdLine(name = "deflate_snapshots", help = "Whether snapshots should be deflate-compressed.")
- private static final Arg<Boolean> DEFLATE_SNAPSHOTS = Arg.create(true);
-
- @Override
- protected void configure() {
- requireBinding(Log.class);
- requireBinding(Clock.class);
- requireBinding(ShutdownRegistry.class);
-
- bindInterval(ShutdownGracePeriod.class, SHUTDOWN_GRACE_PERIOD);
- bindInterval(SnapshotInterval.class, SNAPSHOT_INTERVAL);
-
- bind(new TypeLiteral<Amount<Integer, Data>>() { }).annotatedWith(MaxEntrySize.class)
- .toInstance(MAX_LOG_ENTRY_SIZE.get());
- bind(LogManager.class).in(Singleton.class);
- bind(Boolean.class).annotatedWith(SnapshotSetting.class).toInstance(DEFLATE_SNAPSHOTS.get());
-
- bind(LogStorage.class).in(Singleton.class);
- install(CallOrderEnforcingStorage.wrappingModule(LogStorage.class));
- bind(DistributedSnapshotStore.class).to(LogStorage.class);
- }
-
- private void bindInterval(Class<? extends Annotation> key, Arg<Amount<Long, Time>> value) {
- bind(Key.get(new TypeLiteral<Amount<Long, Time>>() { }, key)).toInstance(value.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java
deleted file mode 100644
index df6b899..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/SnapshotStoreImpl.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.log;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Properties;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.collect.ImmutableSet;
-
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.Lock;
-import com.twitter.aurora.gen.storage.QuotaConfiguration;
-import com.twitter.aurora.gen.storage.SchedulerMetadata;
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.gen.storage.StoredJob;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.storage.SnapshotStore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Volatile;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.util.BuildInfo;
-import com.twitter.common.util.Clock;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
-
-/**
- * Snapshot store implementation that delegates to underlying snapshot stores by
- * extracting/applying fields in a snapshot thrift struct.
- */
-public class SnapshotStoreImpl implements SnapshotStore<Snapshot> {
-
- private static final Logger LOG = Logger.getLogger(SnapshotStoreImpl.class.getName());
-
- private static final SnapshotField ATTRIBUTE_FIELD = new SnapshotField() {
- @Override public void saveToSnapshot(StoreProvider storeProvider, Snapshot snapshot) {
- snapshot.setHostAttributes(storeProvider.getAttributeStore().getHostAttributes());
- }
-
- @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
- store.getAttributeStore().deleteHostAttributes();
-
- if (snapshot.isSetHostAttributes()) {
- for (HostAttributes attributes : snapshot.getHostAttributes()) {
- store.getAttributeStore().saveHostAttributes(attributes);
- }
- }
- }
- };
-
- private static final Iterable<SnapshotField> SNAPSHOT_FIELDS = Arrays.asList(
- ATTRIBUTE_FIELD,
- new SnapshotField() {
- @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
- snapshot.setTasks(
- IScheduledTask.toBuildersSet(store.getTaskStore().fetchTasks(Query.unscoped())));
- }
-
- @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
- store.getUnsafeTaskStore().deleteAllTasks();
-
- if (snapshot.isSetTasks()) {
- store.getUnsafeTaskStore().saveTasks(
- IScheduledTask.setFromBuilders(snapshot.getTasks()));
- }
- }
- },
- new SnapshotField() {
- @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
- ImmutableSet.Builder<StoredJob> jobs = ImmutableSet.builder();
- for (String managerId : store.getJobStore().fetchManagerIds()) {
- for (IJobConfiguration config : store.getJobStore().fetchJobs(managerId)) {
- jobs.add(new StoredJob(managerId, config.newBuilder()));
- }
- }
- snapshot.setJobs(jobs.build());
- }
-
- @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
- store.getJobStore().deleteJobs();
-
- if (snapshot.isSetJobs()) {
- for (StoredJob job : snapshot.getJobs()) {
- store.getJobStore().saveAcceptedJob(
- job.getJobManagerId(),
- IJobConfiguration.build(job.getJobConfiguration()));
- }
- }
- }
- },
- new SnapshotField() {
- @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
- Properties props = new BuildInfo().getProperties();
-
- snapshot.setSchedulerMetadata(
- new SchedulerMetadata()
- .setFrameworkId(store.getSchedulerStore().fetchFrameworkId())
- .setRevision(props.getProperty(BuildInfo.Key.GIT_REVISION.value))
- .setTag(props.getProperty(BuildInfo.Key.GIT_TAG.value))
- .setTimestamp(props.getProperty(BuildInfo.Key.TIMESTAMP.value))
- .setUser(props.getProperty(BuildInfo.Key.USER.value))
- .setMachine(props.getProperty(BuildInfo.Key.MACHINE.value))
- .setVersion(CURRENT_API_VERSION));
- }
-
- @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
- if (snapshot.isSetSchedulerMetadata()) {
- // No delete necessary here since this is a single value.
-
- store.getSchedulerStore()
- .saveFrameworkId(snapshot.getSchedulerMetadata().getFrameworkId());
- }
- }
- },
- new SnapshotField() {
- @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
- ImmutableSet.Builder<QuotaConfiguration> quotas = ImmutableSet.builder();
- for (Map.Entry<String, IQuota> entry : store.getQuotaStore().fetchQuotas().entrySet()) {
- quotas.add(new QuotaConfiguration(entry.getKey(), entry.getValue().newBuilder()));
- }
-
- snapshot.setQuotaConfigurations(quotas.build());
- }
-
- @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
- store.getQuotaStore().deleteQuotas();
-
- if (snapshot.isSetQuotaConfigurations()) {
- for (QuotaConfiguration quota : snapshot.getQuotaConfigurations()) {
- store.getQuotaStore().saveQuota(quota.getRole(), IQuota.build(quota.getQuota()));
- }
- }
- }
- },
- new SnapshotField() {
- @Override public void saveToSnapshot(StoreProvider store, Snapshot snapshot) {
- snapshot.setLocks(ILock.toBuildersSet(store.getLockStore().fetchLocks()));
- }
-
- @Override public void restoreFromSnapshot(MutableStoreProvider store, Snapshot snapshot) {
- store.getLockStore().deleteLocks();
-
- if (snapshot.isSetLocks()) {
- for (Lock lock : snapshot.getLocks()) {
- store.getLockStore().saveLock(ILock.build(lock));
- }
- }
- }
- }
- );
-
- private final Clock clock;
- private final Storage storage;
-
- @Inject
- public SnapshotStoreImpl(Clock clock, @Volatile Storage storage) {
- this.clock = checkNotNull(clock);
- this.storage = checkNotNull(storage);
- }
-
- @Timed("snapshot_create")
- @Override public Snapshot createSnapshot() {
- return storage.consistentRead(new Work.Quiet<Snapshot>() {
- @Override public Snapshot apply(StoreProvider storeProvider) {
- Snapshot snapshot = new Snapshot();
-
- // Capture timestamp to signify the beginning of a snapshot operation, apply after in case
- // one of the field closures is mean and tries to apply a timestamp.
- long timestamp = clock.nowMillis();
- for (SnapshotField field : SNAPSHOT_FIELDS) {
- field.saveToSnapshot(storeProvider, snapshot);
- }
- snapshot.setTimestamp(timestamp);
- return snapshot;
- }
- });
- }
-
- @Timed("snapshot_apply")
- @Override public void applySnapshot(final Snapshot snapshot) {
- checkNotNull(snapshot);
-
- storage.write(new MutateWork.NoResult.Quiet() {
- @Override protected void execute(MutableStoreProvider storeProvider) {
- LOG.info("Restoring snapshot.");
-
- for (SnapshotField field : SNAPSHOT_FIELDS) {
- field.restoreFromSnapshot(storeProvider, snapshot);
- }
- }
- });
- }
-
- private interface SnapshotField {
- void saveToSnapshot(StoreProvider storeProvider, Snapshot snapshot);
-
- void restoreFromSnapshot(MutableStoreProvider storeProvider, Snapshot snapshot);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java b/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java
deleted file mode 100644
index a4c0126..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/log/testing/LogOpMatcher.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.log.testing;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import org.easymock.EasyMock;
-import org.easymock.IArgumentMatcher;
-import org.easymock.IExpectationSetters;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.storage.LogEntry;
-import com.twitter.aurora.gen.storage.Op;
-import com.twitter.aurora.gen.storage.Snapshot;
-import com.twitter.aurora.gen.storage.Transaction;
-import com.twitter.aurora.gen.storage.storageConstants;
-import com.twitter.aurora.scheduler.log.Log.Position;
-import com.twitter.aurora.scheduler.log.Log.Stream;
-
-import static org.easymock.EasyMock.expect;
-
-/**
- * A junit argument matcher that detects same-value {@link LogEntry} objects in a more human
- * readable way than byte array comparison.
- */
-public class LogOpMatcher implements IArgumentMatcher {
- private final LogEntry expected;
-
- public LogOpMatcher(LogEntry expected) {
- this.expected = expected;
- }
-
- @Override
- public boolean matches(Object argument) {
- try {
- return expected.equals(ThriftBinaryCodec.decodeNonNull(LogEntry.class, (byte[]) argument));
- } catch (CodingException e) {
- return false;
- }
- }
-
- @Override
- public void appendTo(StringBuffer buffer) {
- buffer.append(expected);
- }
-
- /**
- * Creates a stream matcher that will set expectations on the provided {@code stream}.
- *
- * @param stream Mocked stream.
- * @return A stream matcher to set expectations against {@code stream}.
- */
- public static StreamMatcher matcherFor(Stream stream) {
- return new StreamMatcher(stream);
- }
-
- public static final class StreamMatcher {
- private final Stream stream;
-
- private StreamMatcher(Stream stream) {
- this.stream = Preconditions.checkNotNull(stream);
- }
-
- /**
- * Sets an expectation for a stream transaction containing the provided {@code ops}.
- *
- * @param ops Operations to expect in the transaction.
- * @return An expectation setter.
- */
- public IExpectationSetters<Position> expectTransaction(Op...ops) {
- LogEntry entry = LogEntry.transaction(
- new Transaction(ImmutableList.copyOf(ops), storageConstants.CURRENT_SCHEMA_VERSION));
- return expect(stream.append(sameEntry(entry)));
- }
-
- /**
- * Sets an expectation for a snapshot.
- *
- * @param snapshot Expected snapshot.
- * @return An expectation setter.
- */
- public IExpectationSetters<Position> expectSnapshot(Snapshot snapshot) {
- LogEntry entry = LogEntry.snapshot(snapshot);
- return expect(stream.append(sameEntry(entry)));
- }
- }
-
- /**
- * Creates a matcher that supports value matching between a serialized {@link LogEntry} byte array
- * and a log entry object.
- *
- * @param entry Entry to match against.
- * @return {@code null}, return value included for easymock-style embedding.
- */
- private static byte[] sameEntry(LogEntry entry) {
- EasyMock.reportMatcher(new LogOpMatcher(entry));
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/Interner.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/Interner.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/Interner.java
deleted file mode 100644
index bf05caa..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/Interner.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-/**
- * An interning pool that can be used to retrieve the canonical instances of objects, while
- * maintaining a reference count to the canonical instances.
- *
- * @param <T> The interned object type.
- * @param <A> The type used for maintaining associations.
- */
-class Interner<T, A> {
-
- private final Map<T, InternEntry<A, T>> pool = Maps.newHashMap();
-
- /**
- * Retrieves the canonical instance of {@code t} and maintains {@code association} with the
- * interned value. If {@code t} was not previously interned, the provided instance is stored.
- *
- * @param t The object to intern, or get the previously-interned value for.
- * @param association A value to associate with {@code t}.
- * @return The interned value, which may be reference-equivalent to {@code t}.
- */
- synchronized T addAssociation(T t, A association) {
- InternEntry<A, T> entry = pool.get(t);
- if (entry == null) {
- entry = new InternEntry<>(t, association);
- pool.put(t, entry);
- } else {
- entry.associations.add(association);
- }
- return entry.interned;
- }
-
- /**
- * Removes an association with an interned value, effectively decrementing the reference count.
- *
- * @param t The interned value that {@code association} was associated with.
- * @param association The association to remove.
- */
- synchronized void removeAssociation(T t, A association) {
- InternEntry<A, T> entry = pool.get(t);
- if (entry != null) {
- entry.associations.remove(association);
- if (entry.associations.isEmpty()) {
- pool.remove(t);
- }
- }
- }
-
- /**
- * Removes all interned values and associations.
- */
- synchronized void clear() {
- pool.clear();
- }
-
- @VisibleForTesting
- synchronized boolean isInterned(T t) {
- return pool.containsKey(t);
- }
-
- @VisibleForTesting
- synchronized Set<A> getAssociations(T t) {
- return ImmutableSet.copyOf(pool.get(t).associations);
- }
-
- private static class InternEntry<A, T> {
- private final T interned;
- private final Set<A> associations = Sets.newHashSet();
-
- InternEntry(T interned, A initialAssociation) {
- this.interned = interned;
- associations.add(initialAssociation);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemAttributeStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemAttributeStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemAttributeStore.java
deleted file mode 100644
index 6c383c7..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemAttributeStore.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.scheduler.storage.AttributeStore.Mutable;
-
-/**
- * An in-memory attribute store.
- */
-class MemAttributeStore implements Mutable {
- private final ConcurrentMap<String, HostAttributes> hostAttributes = Maps.newConcurrentMap();
-
- @Override
- public void deleteHostAttributes() {
- hostAttributes.clear();
- }
-
- @Override
- public void saveHostAttributes(HostAttributes attributes) {
- hostAttributes.putIfAbsent(attributes.getHost(), attributes);
-
- HostAttributes stored = hostAttributes.get(attributes.getHost());
- if (!stored.isSetMode()) {
- stored.setMode(attributes.isSetMode() ? attributes.getMode() : MaintenanceMode.NONE);
- }
- stored.setAttributes(attributes.isSetAttributes()
- ? attributes.getAttributes() : ImmutableSet.<Attribute>of());
- }
-
- @Override
- public boolean setMaintenanceMode(String host, MaintenanceMode mode) {
- HostAttributes stored = hostAttributes.get(host);
- if (stored != null) {
- stored.setMode(mode);
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- public Optional<HostAttributes> getHostAttributes(String host) {
- return Optional.fromNullable(hostAttributes.get(host));
- }
-
- @Override
- public Set<HostAttributes> getHostAttributes() {
- return ImmutableSet.copyOf(hostAttributes.values());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemJobStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemJobStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemJobStore.java
deleted file mode 100644
index 8268e8d..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemJobStore.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Nullable;
-
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.storage.JobStore;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * An in-memory job store.
- */
-class MemJobStore implements JobStore.Mutable {
-
- private final LoadingCache<String, Manager> managers = CacheBuilder.newBuilder()
- .build(new CacheLoader<String, Manager>() {
- @Override public Manager load(String key) {
- return new Manager();
- }
- });
-
- @Override
- public void saveAcceptedJob(String managerId, IJobConfiguration jobConfig) {
- checkNotNull(managerId);
- checkNotNull(jobConfig);
-
- IJobKey key = JobKeys.assertValid(jobConfig.getKey());
- managers.getUnchecked(managerId).jobs.put(key, jobConfig);
- }
-
- @Override
- public void removeJob(IJobKey jobKey) {
- checkNotNull(jobKey);
-
- for (Manager manager : managers.asMap().values()) {
- manager.jobs.remove(jobKey);
- }
- }
-
- @Override
- public void deleteJobs() {
- managers.invalidateAll();
- }
-
- @Override
- public Iterable<IJobConfiguration> fetchJobs(String managerId) {
- checkNotNull(managerId);
-
- @Nullable Manager manager = managers.getIfPresent(managerId);
- if (manager == null) {
- return ImmutableSet.of();
- }
-
- synchronized (manager.jobs) {
- return ImmutableSet.copyOf(manager.jobs.values());
- }
- }
-
- @Override
- public Optional<IJobConfiguration> fetchJob(String managerId, IJobKey jobKey) {
- checkNotNull(managerId);
- checkNotNull(jobKey);
-
- Optional<Manager> manager = Optional.fromNullable(managers.getIfPresent(managerId));
- if (!manager.isPresent()) {
- return Optional.absent();
- } else {
- return Optional.fromNullable(manager.get().jobs.get(jobKey));
- }
- }
-
- @Override
- public Set<String> fetchManagerIds() {
- return ImmutableSet.copyOf(managers.asMap().keySet());
- }
-
- private static class Manager {
- private final Map<IJobKey, IJobConfiguration> jobs = Maps.newConcurrentMap();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemLockStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemLockStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemLockStore.java
deleted file mode 100644
index 1ced973..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemLockStore.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-
-import com.twitter.aurora.scheduler.storage.LockStore;
-import com.twitter.aurora.scheduler.storage.entities.ILock;
-import com.twitter.aurora.scheduler.storage.entities.ILockKey;
-
-/**
- * An in-memory lock store.
- */
-class MemLockStore implements LockStore.Mutable {
-
- private final Map<ILockKey, ILock> locks = Maps.newConcurrentMap();
-
- @Override
- public void saveLock(ILock lock) {
- locks.put(lock.getKey(), lock);
- }
-
- @Override
- public void removeLock(ILockKey lockKey) {
- locks.remove(lockKey);
- }
-
- @Override
- public void deleteLocks() {
- locks.clear();
- }
-
- @Override
- public Set<ILock> fetchLocks() {
- return ImmutableSet.copyOf(locks.values());
- }
-
- @Override
- public Optional<ILock> fetchLock(ILockKey lockKey) {
- return Optional.fromNullable(locks.get(lockKey));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemQuotaStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemQuotaStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemQuotaStore.java
deleted file mode 100644
index 855b39c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemQuotaStore.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.Map;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-
-import com.twitter.aurora.scheduler.storage.QuotaStore;
-import com.twitter.aurora.scheduler.storage.entities.IQuota;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * An in-memory quota store.
- */
-class MemQuotaStore implements QuotaStore.Mutable {
-
- private final Map<String, IQuota> quotas = Maps.newConcurrentMap();
-
- @Override
- public void deleteQuotas() {
- quotas.clear();
- }
-
- @Override
- public void removeQuota(String role) {
- checkNotNull(role);
-
- quotas.remove(role);
- }
-
- @Override
- public void saveQuota(String role, IQuota quota) {
- checkNotNull(role);
- checkNotNull(quota);
-
- quotas.put(role, quota);
- }
-
- @Override
- public Optional<IQuota> fetchQuota(String role) {
- checkNotNull(role);
- return Optional.fromNullable(quotas.get(role));
- }
-
- @Override
- public Map<String, IQuota> fetchQuotas() {
- return ImmutableMap.copyOf(quotas);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemSchedulerStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemSchedulerStore.java b/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemSchedulerStore.java
deleted file mode 100644
index 19bf4ec..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/storage/mem/MemSchedulerStore.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.storage.mem;
-
-import java.util.concurrent.atomic.AtomicReference;
-
-import javax.annotation.Nullable;
-
-import com.google.common.util.concurrent.Atomics;
-
-import com.twitter.aurora.scheduler.storage.SchedulerStore;
-
-/**
- * An in-memory scheduler store.
- */
-class MemSchedulerStore implements SchedulerStore.Mutable {
- private final AtomicReference<String> frameworkId = Atomics.newReference();
-
- @Override
- public void saveFrameworkId(String newFrameworkId) {
- frameworkId.set(newFrameworkId);
- }
-
- @Nullable
- @Override
- public String fetchFrameworkId() {
- return frameworkId.get();
- }
-}
[26/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/bin/aurora_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/aurora_client.py b/src/main/python/apache/aurora/client/bin/aurora_client.py
new file mode 100644
index 0000000..8228b88
--- /dev/null
+++ b/src/main/python/apache/aurora/client/bin/aurora_client.py
@@ -0,0 +1,33 @@
+from twitter.aurora.client.base import generate_terse_usage
+from twitter.common import app
+from twitter.common.log.options import LogOptions
+
+# These are are side-effecting imports in that they register commands via
+# app.command. This is a poor code practice and should be fixed long-term
+# with the creation of twitter.common.cli that allows for argparse-style CLI
+# composition.
+from twitter.aurora.client.commands import (
+ core,
+ help,
+ run,
+ ssh,
+)
+from twitter.aurora.client.options import add_verbosity_options
+
+app.register_commands_from(core, run, ssh)
+app.register_commands_from(help)
+add_verbosity_options()
+
+
+def main():
+ app.help()
+
+
+LogOptions.set_stderr_log_level('INFO')
+LogOptions.disable_disk_logging()
+app.set_name('aurora-client')
+app.set_usage(generate_terse_usage())
+
+
+def proxy_main():
+ app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/binding_helper.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/binding_helper.py b/src/main/python/apache/aurora/client/binding_helper.py
new file mode 100644
index 0000000..fa03b17
--- /dev/null
+++ b/src/main/python/apache/aurora/client/binding_helper.py
@@ -0,0 +1,115 @@
+from abc import abstractmethod, abstractproperty
+import inspect
+import os
+import sys
+
+from twitter.common.lang import Interface
+
+__all__ = (
+ 'BindingHelper',
+ 'CachingBindingHelper',
+ 'apply_all',
+ 'clear_binding_caches',
+ 'unregister_all',
+)
+
+
+# The registry for binding helpers.
+_BINDING_HELPERS = []
+
+
+# TODO(wickman) Update the pydocs to remove references to common_internal components.
+class BindingHelper(Interface):
+ """A component which resolves some set of pseudo-bindings in a config.
+
+ Many bindings are too complex to resolve with bindings using the standard mechanisms,
+ because they require some python computation to determine how to bind them. For example,
+ for references like {{packer[role][pkg][version]}}, we need to talk to the packer to figure
+ out the correct packer call for the desired cluster.
+
+ A BindingHelper is responsible for resolving one of these types of pseudo-bindings.
+ PackerBindingHelper will resolve "packer" bindings; BuildBindingHelper will resolve "build"
+ bindings, JenkinsBindingHelper will resolve "jenkins" bindings, etc.
+
+ A BindingHelper can be registered by calling "BindingHelper.register(Helper)". Instead of
+ explicitly calling "inject" methods in populate_namespaces, it will compute the set of open
+ bindings, and then call the appropriate helpers for each.
+
+ The bindings can be computed either from scratch, or from a binding dictionary. A binding
+ dictionary can be computed from live data, and then passed over an RPC connection, so that
+ the bindings can be recomputed on the server.
+
+ Each helper is responsible for computing its own binding dict. The data in the dict should
+ meet two requirements: it should be enough data to allow it to produce exactly the same
+ result as the scratch binding, and the data should provide information that makes the
+ binding comprehensible for a human debugging a job.
+
+ For example, a packer helper's binding dict should provide enough information to identify
+ the HDFS file that should be used, but also the version number of the binary in packer,
+ (because a human reader wants to know the version of the package, not the meaningless
+ HDFS URL.
+ """
+ @classmethod
+ def register(cls):
+ _BINDING_HELPERS.append(cls())
+
+ def apply(self, config, env=None, binding_dict=None):
+ for match in self.matcher.match(config.raw()):
+ self.bind(config, match, env, binding_dict or config.binding_dicts[self.name])
+
+ @abstractproperty
+ def name(self):
+ """Returns the name of this BindingHelper. Typically it is the first component of
+ the matcher, e.g. if the matcher matches {{git[sha]}}, return "git"."""
+
+ @abstractproperty
+ def matcher(self):
+ """Returns the pystachio matcher for refs that this binding helper binds."""
+
+ @abstractmethod
+ def bind(self, config, match, env, binding_dict):
+ """Resolves a ref, adding a binding to the config."""
+
+
+class CachingBindingHelper(BindingHelper):
+ """A binding helper implementation that caches binding results"""
+ def __init__(self):
+ self.cache = {}
+
+ def flush_cache(self):
+ self.cache = {}
+
+ def bind(self, config, match, env, binding_dict):
+ if match not in self.cache:
+ self.cache[match] = self.uncached_bind(config, match, env, binding_dict)
+ config.bind(self.cache[match])
+
+ @abstractmethod
+ def uncached_bind(self, config, match, env, binding_dict):
+ """Compute the binding for a ref that hasn't been seen before."""
+
+
+def unregister_all():
+ _BINDING_HELPERS[:] = []
+
+
+def apply_all(config, env=None, binding_dict=None):
+ """Computes a set of bindings and applies them to the config.
+
+ :param config: the config whose bindings need to be computed.
+ :param env: the python environment where the configuration was evaluated.
+ :param binding_dict: an optional dictionary containing data to be used to compute the
+ bindings. If this is provided, then data from the dictionary should be used in
+ preference over live data.
+ :return: a binding dictionary with data that can be used to recompute the bindings. The
+ config is updated in-place.
+ """
+ for helper in _BINDING_HELPERS:
+ helper.apply(config, env, binding_dict or config.binding_dicts[helper.name])
+
+
+def clear_binding_caches():
+ """Clear the binding helper's caches for testing."""
+ for helper in _BINDING_HELPERS:
+ if isinstance(helper, CachingBindingHelper):
+ helper.flush_cache()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/cli/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/BUILD b/src/main/python/apache/aurora/client/cli/BUILD
new file mode 100644
index 0000000..0a44a21
--- /dev/null
+++ b/src/main/python/apache/aurora/client/cli/BUILD
@@ -0,0 +1,26 @@
+
+python_binary(
+ name='client',
+ entry_point = 'twitter.aurora.cli:main',
+ dependencies = [ pants(':cli') ],
+ )
+
+python_library(
+ name='cli',
+ sources = [ '__init__.py', 'context.py', 'jobs.py', 'options.py' ],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/python'),
+ pants('src/main/python/twitter/aurora/client/api:command_runner'),
+ pants('src/main/python/twitter/aurora/client/api:disambiguator'),
+ pants('src/main/python/twitter/aurora/client/api:job_monitor'),
+ pants('src/main/python/twitter/aurora/client/api:updater'),
+ pants('src/main/python/twitter/aurora/client/hooks'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/python/twitter/aurora/client:config'),
+ pants('src/main/python/twitter/aurora/client:factory'),
+ pants('src/main/python/twitter/aurora/client:options'),
+ pants('src/main/python/twitter/aurora/common'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ pants('src/main/python/twitter/aurora:argparse')
+ ]
+ )
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/cli/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/__init__.py b/src/main/python/apache/aurora/client/cli/__init__.py
new file mode 100644
index 0000000..2c08cf9
--- /dev/null
+++ b/src/main/python/apache/aurora/client/cli/__init__.py
@@ -0,0 +1,213 @@
+'''Command-line tooling infrastructure for aurora client v2.
+
+This provides a framework for a noun/verb command-line application. The application is structured
+around a collection of basic objects (nouns) that can be manipulated by the command line, where
+each type of object provides a collection of operations (verbs). Every command invocation
+consists of the name of the noun, followed by one of the verbs for that noun, followed by other
+arguments needed by the verb.
+
+For example:
+- To create a job, the noun is "job", the verb is "create":
+ $ aurora job create us-west/www/prod/server server.aurora
+
+- To find out the resource quota for a specific user, the noun is "user" and the verb is
+ "get_quota":
+ $ aurora user get_quota mchucarroll
+'''
+
+from __future__ import print_function
+
+from abc import abstractmethod
+import argparse
+import sys
+
+
+# Constants for standard return codes.
+EXIT_OK = 0
+EXIT_INVALID_CONFIGURATION = 3
+EXIT_COMMAND_FAILURE = 4
+EXIT_INVALID_COMMAND = 5
+EXIT_INVALID_PARAMETER = 6
+EXIT_NETWORK_ERROR = 7
+EXIT_PERMISSION_VIOLATION = 8
+EXIT_TIMEOUT = 9
+EXIT_UNKNOWN_ERROR = 20
+
+
+class Context(object):
+ class Error(Exception): pass
+
+ class ArgumentException(Error): pass
+
+ class CommandError(Error):
+ def __init__(self, code, msg):
+ super(Context.CommandError, self).__init__(msg)
+ self.msg = msg
+ self.code = code
+
+ def set_options(self, options):
+ """Add the options object to a context.
+ This is separated from the constructor to make patching tests easier.
+ """
+ self.options = options
+
+
+class CommandOption(object):
+ """A lightweight encapsulation of an argparse option specification, which can be used to
+ define options that can be reused by multiple commands."""
+
+ def __init__(self, *args, **kwargs):
+ self.args = args
+ self.kwargs = kwargs
+
+ def add_to_parser(self, parser):
+ parser.add_argument(*self.args, **self.kwargs)
+
+
+class AuroraCommand(object):
+ def setup_options_parser(self, argparser):
+ """Set up command line options parsing for this command.
+ This is a thin veneer over the standard python argparse system.
+ :param argparser: the argument parser where this command can add its arguments.
+ """
+ pass
+
+ def add_option(self, argparser, option):
+ """Add a predefined argument encapsulated an a CommandOption to an argument parser."""
+ if not isinstance(option, CommandOption):
+ raise TypeError('Command option object must be an instance of CommandOption')
+ option.add_to_parser(argparser)
+
+ @property
+ def help(self):
+ """The help message for a command that will be used in the argparse help message"""
+
+ @property
+ def name(self):
+ """The command name"""
+
+
+class CommandLine(object):
+ """The top-level object implementing a command-line application."""
+
+ def __init__(self):
+ self.nouns = {}
+ self.parser = None
+
+ def register_noun(self, noun):
+ """Add a noun to the application"""
+ if not isinstance(noun, Noun):
+ raise TypeError('register_noun requires a Noun argument')
+ self.nouns[noun.name] = noun
+
+ def setup_options_parser(self):
+ """ Build the options parsing for the application."""
+ self.parser = argparse.ArgumentParser()
+ subparser = self.parser.add_subparsers(dest='noun')
+ for (name, noun) in self.nouns.items():
+ noun_parser = subparser.add_parser(name, help=noun.help)
+ noun.internal_setup_options_parser(noun_parser)
+
+ def register_nouns(self):
+ """This method should overridden by applications to register the collection of nouns
+ that they can manipulate.
+ """
+ pass
+
+ def execute(self, args):
+ """Execute a command.
+ :param args: the command-line arguments for the command. This only includes arguments
+ that should be parsed by the application; it does not include sys.argv[0].
+ """
+ self.register_nouns()
+ self.setup_options_parser()
+ options = self.parser.parse_args(args)
+ if options.noun not in self.nouns:
+ raise ValueError('Unknown command: %s' % options.noun)
+ noun = self.nouns[options.noun]
+ context = noun.create_context()
+ context.set_options(options)
+ try:
+ return noun.execute(context)
+ except Context.CommandError as c:
+ print('Error executing command: %s' % c.msg, file=sys.stderr)
+ return c.code
+
+
+class Noun(AuroraCommand):
+ """A type of object manipulated by a command line application"""
+ class InvalidVerbException(Exception): pass
+
+ def __init__(self):
+ super(Noun, self).__init__()
+ self.verbs = {}
+
+ def register_verb(self, verb):
+ """Add an operation supported for this noun."""
+ if not isinstance(verb, Verb):
+ raise TypeError('register_verb requires a Verb argument')
+ self.verbs[verb.name] = verb
+ verb._register(self)
+
+ def internal_setup_options_parser(self, argparser):
+ """Internal driver for the options processing framework."""
+ self.setup_options_parser(argparser)
+ subparser = argparser.add_subparsers(dest='verb')
+ for (name, verb) in self.verbs.items():
+ vparser = subparser.add_parser(name, help=verb.help)
+ verb.setup_options_parser(vparser)
+
+ @classmethod
+ def create_context(cls):
+ """Commands access state through a context object. The noun specifies what kind
+ of context should be created for this noun's required state.
+ """
+ pass
+
+ @abstractmethod
+ def setup_options_parser(self, argparser):
+ pass
+
+ def execute(self, context):
+ if context.options.verb not in self.verbs:
+ raise self.InvalidVerbException('Noun %s does not have a verb %s' %
+ (self.name, context.options.verb))
+ self.verbs[context.options.verb].execute(context)
+
+
+class Verb(AuroraCommand):
+ """An operation for a noun. Most application logic will live in verbs."""
+
+ def _register(self, noun):
+ """Create a link from a verb to its noun."""
+ self.noun = noun
+
+ @abstractmethod
+ def setup_options_parser(self, argparser):
+ pass
+
+ def execute(self, context):
+ pass
+
+
+class AuroraCommandLine(CommandLine):
+ """ An example implementation of a command line application using this framework.
+ This should probably eventually get moved in to its own source file.
+ """
+
+ @classmethod
+ def get_description(cls):
+ return 'Aurora client command line'
+
+ def register_nouns(self):
+ from .jobs import Job
+ self.register_noun(Job())
+
+
+def main():
+ cmd = AuroraCommandLine()
+ cmd.execute(sys.argv[1:])
+
+
+if __name__ == '__main__':
+ main(sys.argv)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/cli/context.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/context.py b/src/main/python/apache/aurora/client/cli/context.py
new file mode 100644
index 0000000..2ae92ec
--- /dev/null
+++ b/src/main/python/apache/aurora/client/cli/context.py
@@ -0,0 +1,49 @@
+
+from twitter.aurora.client.base import synthesize_url
+from twitter.aurora.client.cli import Context, EXIT_NETWORK_ERROR
+from twitter.aurora.client.config import get_config
+from twitter.aurora.client.factory import make_client
+from twitter.common import log
+
+from gen.twitter.aurora.ttypes import ResponseCode
+
+
+class AuroraCommandContext(Context):
+ """A context object used by Aurora commands to manage command processing state
+ and common operations.
+ """
+
+ def get_api(self, cluster):
+ """Creates an API object for a specified cluster"""
+ return make_client(cluster)
+
+ def get_job_config(self, job_key, config_file):
+ """Loads a job configuration from a config file"""
+ jobname = job_key.name
+ return get_config(
+ jobname,
+ config_file,
+ self.options.json,
+ self.options.bindings,
+ select_cluster=job_key.cluster,
+ select_role=job_key.role,
+ select_env=job_key.env)
+
+ def open_page(self, url):
+ import webbrowser
+ webbrowser.open_new_tab(url)
+
+ def open_job_page(self, api, config):
+ self.open_page(synthesize_url(api.scheduler.scheduler().url, config.role(),
+ config.environment(), config.name()))
+
+ def handle_open(self, api):
+ if self.options.open_browser:
+ self.open_page(synthesize_url(api.scheduler.scheduler().url,
+ self.options.jobspec.role, self.options.jobspec.env, self.options.jobspec.name))
+
+ def check_and_log_response(self, resp):
+ log.info('Response from scheduler: %s (message: %s)'
+ % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
+ if resp.responseCode != ResponseCode.OK:
+ raise self.CommandError(EXIT_NETWORK_ERROR, resp.message)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/cli/jobs.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/jobs.py b/src/main/python/apache/aurora/client/cli/jobs.py
new file mode 100644
index 0000000..e66f181
--- /dev/null
+++ b/src/main/python/apache/aurora/client/cli/jobs.py
@@ -0,0 +1,115 @@
+from twitter.aurora.client.api.job_monitor import JobMonitor
+from twitter.aurora.client.cli import (
+ EXIT_INVALID_CONFIGURATION,
+ Noun,
+ Verb
+)
+from twitter.aurora.client.cli.context import AuroraCommandContext
+from twitter.aurora.client.cli.options import (
+ BIND_OPTION,
+ BROWSER_OPTION,
+ CONFIG_OPTION,
+ JOBSPEC_OPTION,
+ JSON_OPTION
+)
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+
+from pystachio.config import Config
+
+
+def parse_instances(instances):
+ """Parse lists of instances or instance ranges into a set().
+
+ Examples:
+ 0-2
+ 0,1-3,5
+ 1,3,5
+ """
+ if instances is None or instances == '':
+ return None
+ result = set()
+ for part in instances.split(','):
+ x = part.split('-')
+ result.update(range(int(x[0]), int(x[-1]) + 1))
+ return sorted(result)
+
+
+class CreateJobCommand(Verb):
+ @property
+ def name(self):
+ return 'create'
+
+ @property
+ def help(self):
+ return 'Create a job using aurora'
+
+ CREATE_STATES = ('PENDING', 'RUNNING', 'FINISHED')
+
+ def setup_options_parser(self, parser):
+ self.add_option(parser, BIND_OPTION)
+ self.add_option(parser, BROWSER_OPTION)
+ self.add_option(parser, JSON_OPTION)
+ parser.add_argument('--wait_until', choices=self.CREATE_STATES,
+ default='PENDING',
+ help=('Block the client until all the tasks have transitioned into the requested state. '
+ 'Default: PENDING'))
+ self.add_option(parser, JOBSPEC_OPTION)
+ self.add_option(parser, CONFIG_OPTION)
+
+ def execute(self, context):
+ try:
+ config = context.get_job_config(context.options.jobspec, context.options.config_file)
+ except Config.InvalidConfigError as e:
+ raise context.CommandError(EXIT_INVALID_CONFIGURATION,
+ 'Error loading job configuration: %s' % e)
+ api = context.get_api(config.cluster())
+ monitor = JobMonitor(api, config.role(), config.environment(), config.name())
+ resp = api.create_job(config)
+ context.check_and_log_response(resp)
+ if context.options.open_browser:
+ context.open_job_page(api, config)
+ if context.options.wait_until == 'RUNNING':
+ monitor.wait_until(monitor.running_or_finished)
+ elif context.options.wait_until == 'FINISHED':
+ monitor.wait_until(monitor.terminal)
+
+
+class KillJobCommand(Verb):
+ @property
+ def name(self):
+ return 'kill'
+
+ def setup_options_parser(self, parser):
+ self.add_option(parser, BROWSER_OPTION)
+ parser.add_argument('--instances', type=parse_instances, dest='instances', default=None,
+ help='A list of instance ids to act on. Can either be a comma-separated list (e.g. 0,1,2) '
+ 'or a range (e.g. 0-2) or any combination of the two (e.g. 0-2,5,7-9). If not set, '
+ 'all instances will be acted on.')
+ parser.add_argument('--config', type=str, default=None, dest='config',
+ help='Config file for the job, possibly containing hooks')
+ self.add_option(parser, JOBSPEC_OPTION)
+
+ def execute(self, context):
+ api = context.get_api(context.options.jobspec.cluster)
+ resp = api.kill_job(context.options.jobspec, context.options.instances)
+ context.check_and_log_response(resp)
+ context.handle_open(api)
+
+
+class Job(Noun):
+ @property
+ def name(self):
+ return 'job'
+
+ @property
+ def help(self):
+ return "Work with an aurora job"
+
+ @classmethod
+ def create_context(cls):
+ return AuroraCommandContext()
+
+ def __init__(self):
+ super(Job, self).__init__()
+ self.register_verb(CreateJobCommand())
+ self.register_verb(KillJobCommand())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/cli/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/cli/options.py b/src/main/python/apache/aurora/client/cli/options.py
new file mode 100644
index 0000000..64a58f8
--- /dev/null
+++ b/src/main/python/apache/aurora/client/cli/options.py
@@ -0,0 +1,25 @@
+from twitter.aurora.client.cli import CommandOption
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+
+
+BIND_OPTION = CommandOption('--bind', type=str, default=[], dest='bindings',
+ action='append',
+ help='Bind a thermos mustache variable name to a value. '
+ 'Multiple flags may be used to specify multiple values.')
+
+
+BROWSER_OPTION = CommandOption('--open-browser', default=False, dest='open_browser',
+ action='store_true',
+ help='open browser to view job page after job is created')
+
+
+CONFIG_OPTION = CommandOption('config_file', type='str', dest='config_file',
+ help='pathname of the aurora configuration file contain the job specification')
+
+
+JOBSPEC_OPTION = CommandOption('jobspec', type=AuroraJobKey.from_path,
+ help='Fully specified job key, in CLUSTER/ROLE/ENV/NAME format')
+
+
+JSON_OPTION = CommandOption('--json', default=False, dest='json', action='store_true',
+ help='Read job configuration in json format')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/commands/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/BUILD b/src/main/python/apache/aurora/client/commands/BUILD
new file mode 100644
index 0000000..3bcb6fd
--- /dev/null
+++ b/src/main/python/apache/aurora/client/commands/BUILD
@@ -0,0 +1,81 @@
+python_library(
+ name = 'all',
+ dependencies = [
+ pants(':core'),
+ pants(':help'),
+ pants(':run'),
+ pants(':ssh'),
+ ]
+)
+
+python_library(
+ name = 'admin',
+ sources = ['admin.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/aurora/admin:mesos_maintenance'),
+ pants('src/main/python/twitter/aurora/client/api'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/python/twitter/aurora/common:clusters'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'core',
+ sources = ['core.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/python'),
+ pants('src/main/python/twitter/aurora/client/api:command_runner'),
+ pants('src/main/python/twitter/aurora/client/api:disambiguator'),
+ pants('src/main/python/twitter/aurora/client/api:job_monitor'),
+ pants('src/main/python/twitter/aurora/client/api:updater'),
+ pants('src/main/python/twitter/aurora/client/hooks'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/python/twitter/aurora/client:config'),
+ pants('src/main/python/twitter/aurora/client:factory'),
+ pants('src/main/python/twitter/aurora/client:options'),
+ pants('src/main/python/twitter/aurora/common'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'help',
+ sources = ['help.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ ]
+)
+
+python_library(
+ name = 'run',
+ sources = ['run.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('src/main/python/twitter/aurora/client/api:command_runner'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/python/twitter/aurora/client:options'),
+ pants('src/main/python/twitter/aurora/common:aurora_job_key'),
+ pants('src/main/python/twitter/aurora/common:clusters'),
+ ]
+)
+
+python_library(
+ name = 'ssh',
+ sources = ['ssh.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('src/main/python/twitter/aurora/client/api:command_runner'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/python/twitter/aurora/client:factory'),
+ pants('src/main/python/twitter/aurora/client:options'),
+ pants('src/main/python/twitter/aurora/common:aurora_job_key'),
+ pants('src/main/python/twitter/aurora/common:clusters'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/commands/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/__init__.py b/src/main/python/apache/aurora/client/commands/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/commands/admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/admin.py b/src/main/python/apache/aurora/client/commands/admin.py
new file mode 100644
index 0000000..c1c9c42
--- /dev/null
+++ b/src/main/python/apache/aurora/client/commands/admin.py
@@ -0,0 +1,406 @@
+from __future__ import print_function
+
+"""Command-line client for managing admin-only interactions with the aurora scheduler.
+"""
+
+import os
+import optparse
+import subprocess
+
+from twitter.aurora.admin.mesos_maintenance import MesosMaintenance
+from twitter.aurora.client.api import AuroraClientAPI
+from twitter.aurora.client.base import check_and_log_response, die, requires
+from twitter.aurora.common.clusters import CLUSTERS
+from twitter.common import app, log
+from twitter.common.quantity import Amount, Data
+from twitter.common.quantity.parse_simple import parse_data
+
+from gen.twitter.aurora.constants import ACTIVE_STATES, TERMINAL_STATES
+from gen.twitter.aurora.ttypes import (
+ ResponseCode,
+ ScheduleStatus,
+ TaskQuery,
+)
+
+
+GROUPING_OPTION = optparse.Option(
+ '--grouping',
+ type='choice',
+ choices=MesosMaintenance.GROUPING_FUNCTIONS.keys(),
+ metavar='GROUPING',
+ default=MesosMaintenance.DEFAULT_GROUPING,
+ dest='grouping',
+ help='Grouping function to use to group hosts. Options: %s. Default: %%default' % (
+ ', '.join(MesosMaintenance.GROUPING_FUNCTIONS.keys())))
+
+
+def parse_hosts(options):
+ if not (options.filename or options.hosts):
+ die('Please specify either --filename or --hosts')
+ if options.filename:
+ with open(options.filename, 'r') as hosts:
+ hosts = [hostname.strip() for hostname in hosts]
+ elif options.hosts:
+ hosts = [hostname.strip() for hostname in options.hosts.split(",")]
+ if not hosts:
+ die('No valid hosts found.')
+ return hosts
+
+
+@app.command
+@app.command_option('--force', dest='force', default=False, action='store_true',
+ help='Force expensive queries to run.')
+@app.command_option('--shards', dest='shards', default=None,
+ help='Only match given shards of a job.')
+@app.command_option('--states', dest='states', default='RUNNING',
+ help='Only match tasks with given state(s).')
+@app.command_option('-l', '--listformat', dest='listformat',
+ default="%role%/%jobName%/%instanceId% %status%",
+ help='Format string of job/task items to print out.')
+# TODO(ksweeney): Allow query by environment here.
+def query(args, options):
+ """usage: query [--shards=N[,N,...]]
+ [--states=State[,State,...]]
+ cluster [role [job]]
+
+ Query Mesos about jobs and tasks.
+ """
+ def _convert_fmt_string(fmtstr):
+ import re
+ def convert(match):
+ return "%%(%s)s" % match.group(1)
+ return re.sub(r'%(\w+)%', convert, fmtstr)
+
+ def flatten_task(t, d={}):
+ for key in t.__dict__.keys():
+ val = getattr(t, key)
+ try:
+ val.__dict__.keys()
+ except AttributeError:
+ d[key] = val
+ else:
+ flatten_task(val, d)
+
+ return d
+
+ def map_values(d):
+ default_value = lambda v: v
+ mapping = {
+ 'status': lambda v: ScheduleStatus._VALUES_TO_NAMES[v],
+ }
+ return dict(
+ (k, mapping.get(k, default_value)(v)) for (k, v) in d.items()
+ )
+
+ for state in options.states.split(','):
+ if state not in ScheduleStatus._NAMES_TO_VALUES:
+ msg = "Unknown state '%s' specified. Valid states are:\n" % state
+ msg += ','.join(ScheduleStatus._NAMES_TO_VALUES.keys())
+ die(msg)
+
+ # Role, Job, Instances, States, and the listformat
+ if len(args) == 0:
+ die('Must specify at least cluster.')
+
+ cluster = args[0]
+ role = args[1] if len(args) > 1 else None
+ job = args[2] if len(args) > 2 else None
+ instances = set(map(int, options.shards.split(','))) if options.shards else set()
+
+ if options.states:
+ states = set(map(ScheduleStatus._NAMES_TO_VALUES.get, options.states.split(',')))
+ else:
+ states = ACTIVE_STATES | TERMINAL_STATES
+ listformat = _convert_fmt_string(options.listformat)
+
+ # Figure out "expensive" queries here and bone if they do not have --force
+ # - Does not specify role
+ if role is None and not options.force:
+ die('--force is required for expensive queries (no role specified)')
+
+ # - Does not specify job
+ if job is None and not options.force:
+ die('--force is required for expensive queries (no job specified)')
+
+ # - Specifies status outside of ACTIVE_STATES
+ if not (states <= ACTIVE_STATES) and not options.force:
+ die('--force is required for expensive queries (states outside ACTIVE states')
+
+ api = AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
+ query_info = api.query(api.build_query(role, job, instances=instances, statuses=states))
+ tasks = query_info.result.scheduleStatusResult.tasks
+ if query_info.responseCode != ResponseCode.OK:
+ die('Failed to query scheduler: %s' % query_info.message)
+ if tasks is None:
+ return
+
+ try:
+ for task in tasks:
+ d = flatten_task(task)
+ print(listformat % map_values(d))
+ except KeyError:
+ msg = "Unknown key in format string. Valid keys are:\n"
+ msg += ','.join(d.keys())
+ die(msg)
+
+
+@app.command
+@requires.exactly('cluster', 'role', 'cpu', 'ramMb', 'diskMb')
+def set_quota(cluster, role, cpu_str, ram_mb_str, disk_mb_str):
+ """usage: set_quota cluster role cpu ramMb diskMb
+
+ Alters the amount of production quota allocated to a user.
+ """
+ try:
+ cpu = float(cpu_str)
+ ram_mb = int(ram_mb_str)
+ disk_mb = int(disk_mb_str)
+ except ValueError:
+ log.error('Invalid value')
+
+ options = app.get_options()
+ resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).set_quota(role, cpu, ram_mb, disk_mb)
+ check_and_log_response(resp)
+
+
+@app.command
+@app.command_option('--filename', dest='filename', default=None,
+ help='Name of the file with hostnames')
+@app.command_option('--hosts', dest='hosts', default=None,
+ help='Comma separated list of hosts')
+@requires.exactly('cluster')
+def start_maintenance_hosts(cluster):
+ """usage: start_maintenance_hosts cluster [--filename=filename]
+ [--hosts=hosts]
+ """
+ options = app.get_options()
+ MesosMaintenance(CLUSTERS[cluster], options.verbosity).start_maintenance(parse_hosts(options))
+
+
+@app.command
+@app.command_option('--filename', dest='filename', default=None,
+ help='Name of the file with hostnames')
+@app.command_option('--hosts', dest='hosts', default=None,
+ help='Comma separated list of hosts')
+@requires.exactly('cluster')
+def end_maintenance_hosts(cluster):
+ """usage: end_maintenance_hosts cluster [--filename=filename]
+ [--hosts=hosts]
+ """
+ options = app.get_options()
+ MesosMaintenance(CLUSTERS[cluster], options.verbosity).end_maintenance(parse_hosts(options))
+
+
+@app.command
+@app.command_option('--filename', dest='filename', default=None,
+ help='Name of the file with hostnames')
+@app.command_option('--hosts', dest='hosts', default=None,
+ help='Comma separated list of hosts')
+@app.command_option('--batch_size', dest='batch_size', default=1,
+ help='Number of groups to operate on at a time.')
+@app.command_option('--post_drain_script', dest='post_drain_script', default=None,
+ help='Path to a script to run for each host.')
+@app.command_option(GROUPING_OPTION)
+@requires.exactly('cluster')
+def perform_maintenance_hosts(cluster):
+ """usage: perform_maintenance cluster [--filename=filename]
+ [--hosts=hosts]
+ [--batch_size=num]
+ [--post_drain_script=path]
+ [--grouping=function]
+
+ Asks the scheduler to remove any running tasks from the machine and remove it
+ from service temporarily, perform some action on them, then return the machines
+ to service.
+ """
+ options = app.get_options()
+ drainable_hosts = parse_hosts(options)
+
+ if options.post_drain_script:
+ if not os.path.exists(options.post_drain_script):
+ die("No such file: %s" % options.post_drain_script)
+ cmd = os.path.abspath(options.post_drain_script)
+ drained_callback = lambda host: subprocess.Popen([cmd, host])
+ else:
+ drained_callback = None
+
+ MesosMaintenance(CLUSTERS[cluster], options.verbosity).perform_maintenance(
+ drainable_hosts,
+ batch_size=int(options.batch_size),
+ callback=drained_callback,
+ grouping_function=options.grouping)
+
+
+@app.command
+@app.command_option('--filename', dest='filename', default=None,
+ help='Name of the file with hostnames')
+@app.command_option('--hosts', dest='hosts', default=None,
+ help='Comma separated list of hosts')
+@requires.exactly('cluster')
+def host_maintenance_status(cluster):
+ """usage: host_maintenance_status cluster [--filename=filename]
+ [--hosts=hosts]
+
+ Check on the schedulers maintenance status for a list of hosts in the cluster.
+ """
+ options = app.get_options()
+ checkable_hosts = parse_hosts(options)
+ statuses = MesosMaintenance(CLUSTERS[cluster], options.verbosity).check_status(checkable_hosts)
+ for pair in statuses:
+ log.info("%s is in state: %s" % pair)
+
+
+@app.command
+@requires.exactly('cluster', 'role', 'cpu', 'ram', 'disk')
+def increase_quota(cluster, role, cpu_str, ram_str, disk_str):
+ """usage: increase_quota cluster role cpu ram[unit] disk[unit]
+
+ Increases the amount of production quota allocated to a user.
+ """
+ cpu = float(cpu_str)
+ ram = parse_data(ram_str)
+ disk = parse_data(disk_str)
+
+ options = app.get_options()
+ client = AuroraClientAPI(CLUSTERS[cluster], options.verbosity == 'verbose')
+ resp = client.get_quota(role)
+ quota = resp.result.getQuotaResult.quota
+ log.info('Current quota for %s:\n\tCPU\t%s\n\tRAM\t%s MB\n\tDisk\t%s MB' %
+ (role, quota.numCpus, quota.ramMb, quota.diskMb))
+
+ new_cpu = cpu + quota.numCpus
+ new_ram = ram + Amount(quota.ramMb, Data.MB)
+ new_disk = disk + Amount(quota.diskMb, Data.MB)
+
+ log.info('Attempting to update quota for %s to\n\tCPU\t%s\n\tRAM\t%s MB\n\tDisk\t%s MB' %
+ (role, new_cpu, new_ram.as_(Data.MB), new_disk.as_(Data.MB)))
+
+ resp = client.set_quota(role, new_cpu, new_ram.as_(Data.MB), new_disk.as_(Data.MB))
+ check_and_log_response(resp)
+
+
+@app.command
+@requires.exactly('cluster')
+def scheduler_backup_now(cluster):
+ """usage: scheduler_backup_now cluster
+
+ Immediately initiates a full storage backup.
+ """
+ options = app.get_options()
+ check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity).perform_backup())
+
+
+@app.command
+@requires.exactly('cluster')
+def scheduler_list_backups(cluster):
+ """usage: scheduler_list_backups cluster
+
+ Lists backups available for recovery.
+ """
+ options = app.get_options()
+ resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).list_backups()
+ check_and_log_response(resp)
+ backups = resp.result.listBackupsResult.backups
+ print('%s available backups:' % len(backups))
+ for backup in backups:
+ print(backup)
+
+
+@app.command
+@requires.exactly('cluster', 'backup_id')
+def scheduler_stage_recovery(cluster, backup_id):
+ """usage: scheduler_stage_recovery cluster backup_id
+
+ Stages a backup for recovery.
+ """
+ options = app.get_options()
+ check_and_log_response(
+ AuroraClientAPI(CLUSTERS[cluster], options.verbosity).stage_recovery(backup_id))
+
+
+@app.command
+@requires.exactly('cluster')
+def scheduler_print_recovery_tasks(cluster):
+ """usage: scheduler_print_recovery_tasks cluster
+
+ Prints all active tasks in a staged recovery.
+ """
+ options = app.get_options()
+ resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).query_recovery(
+ TaskQuery(statuses=ACTIVE_STATES))
+ check_and_log_response(resp)
+ log.info('Role\tJob\tShard\tStatus\tTask ID')
+ for task in resp.tasks:
+ assigned = task.assignedTask
+ conf = assigned.task
+ log.info('\t'.join((conf.owner.role,
+ conf.jobName,
+ str(assigned.instanceId),
+ ScheduleStatus._VALUES_TO_NAMES[task.status],
+ assigned.taskId)))
+
+
+@app.command
+@requires.exactly('cluster', 'task_ids')
+def scheduler_delete_recovery_tasks(cluster, task_ids):
+ """usage: scheduler_delete_recovery_tasks cluster task_ids
+
+ Deletes a comma-separated list of task IDs from a staged recovery.
+ """
+ ids = set(task_ids.split(','))
+ options = app.get_options()
+ check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
+ .delete_recovery_tasks(TaskQuery(taskIds=ids)))
+
+
+@app.command
+@requires.exactly('cluster')
+def scheduler_commit_recovery(cluster):
+ """usage: scheduler_commit_recovery cluster
+
+ Commits a staged recovery.
+ """
+ options = app.get_options()
+ check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
+ .commit_recovery())
+
+
+@app.command
+@requires.exactly('cluster')
+def scheduler_unload_recovery(cluster):
+ """usage: scheduler_unload_recovery cluster
+
+ Unloads a staged recovery.
+ """
+ options = app.get_options()
+ check_and_log_response(AuroraClientAPI(CLUSTERS[cluster], options.verbosity)
+ .unload_recovery())
+
+
+@app.command
+@requires.exactly('cluster')
+def scheduler_list_job_updates(cluster):
+ """usage: scheduler_list_job_updates cluster
+
+ Lists in-flight job updates.
+ """
+ options = app.get_options()
+ resp = AuroraClientAPI(CLUSTERS[cluster], options.verbosity).get_job_updates()
+ check_and_log_response(resp)
+ print('Role\tEnv\tJob')
+ for update in resp.jobUpdates:
+ print('%s\t%s\t%s' % (
+ update.jobKey.role if update.jobKey else update.roleDeprecated,
+ update.jobKey.environment if update.jobKey else None,
+ update.jobKey.name if update.jobKey else update.jobDeprecated))
+
+
+@app.command
+@requires.exactly('cluster')
+def scheduler_snapshot(cluster):
+ """usage: scheduler_snapshot cluster
+
+ Request that the scheduler perform a storage snapshot and block until complete.
+ """
+ options = app.get_options()
+ check_and_log_response(AuroraClientAPI(CLUSTERS['cluster'], options.verbosity).snapshot())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/commands/core.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/core.py b/src/main/python/apache/aurora/client/commands/core.py
new file mode 100644
index 0000000..712c7ea
--- /dev/null
+++ b/src/main/python/apache/aurora/client/commands/core.py
@@ -0,0 +1,600 @@
+"""Command-line client for managing jobs with the Aurora scheduler.
+"""
+
+from __future__ import print_function
+
+import collections
+from datetime import datetime
+import json
+import os
+import pprint
+import subprocess
+import sys
+import time
+from tempfile import NamedTemporaryFile
+
+from twitter.common import app, log
+from twitter.common.python.pex import PexInfo
+from twitter.common.python.dirwrapper import PythonDirectoryWrapper
+
+from twitter.aurora.client.base import (
+ check_and_log_response,
+ deprecation_warning,
+ die,
+ handle_open,
+ requires,
+ synthesize_url)
+from twitter.aurora.client.api.disambiguator import LiveJobDisambiguator
+from twitter.aurora.client.api.job_monitor import JobMonitor
+from twitter.aurora.client.api.updater_util import UpdaterConfig
+from twitter.aurora.client.config import get_config
+from twitter.aurora.client.factory import make_client, make_client_factory
+from twitter.aurora.client.options import (
+ CLUSTER_CONFIG_OPTION,
+ CLUSTER_INVOKE_OPTION,
+ ENV_CONFIG_OPTION,
+ ENVIRONMENT_BIND_OPTION,
+ FROM_JOBKEY_OPTION,
+ HEALTH_CHECK_INTERVAL_SECONDS_OPTION,
+ JSON_OPTION,
+ OPEN_BROWSER_OPTION,
+ SHARDS_OPTION,
+ WAIT_UNTIL_OPTION)
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+
+from gen.twitter.aurora.constants import ACTIVE_STATES, CURRENT_API_VERSION, AURORA_EXECUTOR_NAME
+from gen.twitter.aurora.ttypes import ExecutorConfig, ResponseCode, ScheduleStatus
+
+
+def get_job_config(job_spec, config_file, options):
+ try:
+ job_key = AuroraJobKey.from_path(job_spec)
+ select_cluster = job_key.cluster
+ select_env = job_key.env
+ select_role = job_key.role
+ jobname = job_key.name
+ except AuroraJobKey.Error:
+ deprecation_warning('Please refer to your job in CLUSTER/ROLE/ENV/NAME format.')
+ select_cluster = options.cluster if options.cluster else None
+ select_env = options.env
+ select_role = None
+ jobname = job_spec
+ try:
+ json_option = options.json
+ except AttributeError:
+ json_option = False
+ try:
+ bindings = options.bindings
+ except AttributeError:
+ bindings = ()
+ return get_config(
+ jobname,
+ config_file,
+ json_option,
+ bindings,
+ select_cluster=select_cluster,
+ select_role=select_role,
+ select_env=select_env)
+
+@app.command
+def version(args):
+ """usage: version
+
+ Prints information about the version of the aurora client being run.
+ """
+ try:
+ pexpath = sys.argv[0]
+ pex_info = PexInfo.from_pex(PythonDirectoryWrapper.get(pexpath))
+ print("Aurora client build info:")
+ print("\tsha: %s" % pex_info.build_properties['sha'])
+ print("\tdate: %s" % pex_info.build_properties['date'])
+ except (IOError, PythonDirectoryWrapper.Error):
+ print("Aurora client build info not available")
+ print("Aurora API version: %s" % CURRENT_API_VERSION)
+
+
+@app.command
+@app.command_option(ENVIRONMENT_BIND_OPTION)
+@app.command_option(OPEN_BROWSER_OPTION)
+@app.command_option(CLUSTER_CONFIG_OPTION)
+@app.command_option(ENV_CONFIG_OPTION)
+@app.command_option(JSON_OPTION)
+@app.command_option(WAIT_UNTIL_OPTION)
+@requires.exactly('cluster/role/env/job', 'config')
+def create(job_spec, config_file):
+ """usage: create cluster/role/env/job config
+
+ Creates a job based on a configuration file.
+ """
+ options = app.get_options()
+ try:
+ config = get_job_config(job_spec, config_file, options)
+ except ValueError as v:
+ print("Error: %s" % v)
+ sys.exit(1)
+ api = make_client(config.cluster())
+ monitor = JobMonitor(api, config.role(), config.environment(), config.name())
+ resp = api.create_job(config)
+ check_and_log_response(resp)
+ handle_open(api.scheduler.scheduler().url, config.role(), config.environment(), config.name())
+ if options.wait_until == 'RUNNING':
+ monitor.wait_until(monitor.running_or_finished)
+ elif options.wait_until == 'FINISHED':
+ monitor.wait_until(monitor.terminal)
+
+
+@app.command
+@app.command_option(ENVIRONMENT_BIND_OPTION)
+@app.command_option(CLUSTER_CONFIG_OPTION)
+@app.command_option(ENV_CONFIG_OPTION)
+@app.command_option(JSON_OPTION)
+@app.command_option(FROM_JOBKEY_OPTION)
+@requires.exactly('cluster/role/env/job', 'config')
+def diff(job_spec, config_file):
+ """usage: diff cluster/role/env/job config
+
+ Compares a job configuration against a running job.
+ By default the diff will be displayed using 'diff', though you may choose an alternate
+ diff program by specifying the DIFF_VIEWER environment variable."""
+ options = app.get_options()
+ config = get_job_config(job_spec, config_file, options)
+ if options.rename_from:
+ cluster, role, env, name = options.rename_from
+ else:
+ cluster = config.cluster()
+ role = config.role()
+ env = config.environment()
+ name = config.name()
+ api = make_client(cluster)
+ resp = api.query(api.build_query(role, name, statuses=ACTIVE_STATES, env=env))
+ if resp.responseCode != ResponseCode.OK:
+ die('Request failed, server responded with "%s"' % resp.message)
+ remote_tasks = [t.assignedTask.task for t in resp.result.scheduleStatusResult.tasks]
+ resp = api.populate_job_config(config)
+ if resp.responseCode != ResponseCode.OK:
+ die('Request failed, server responded with "%s"' % resp.message)
+ local_tasks = resp.result.populateJobResult.populated
+
+ pp = pprint.PrettyPrinter(indent=2)
+ def pretty_print_task(task):
+ # The raw configuration is not interesting - we only care about what gets parsed.
+ task.configuration = None
+ task.executorConfig = ExecutorConfig(
+ name=AURORA_EXECUTOR_NAME,
+ data=json.loads(task.executorConfig.data))
+ return pp.pformat(vars(task))
+
+ def pretty_print_tasks(tasks):
+ return ',\n'.join([pretty_print_task(t) for t in tasks])
+
+ def dump_tasks(tasks, out_file):
+ out_file.write(pretty_print_tasks(tasks))
+ out_file.write('\n')
+ out_file.flush()
+
+ diff_program = os.environ.get('DIFF_VIEWER', 'diff')
+ with NamedTemporaryFile() as local:
+ dump_tasks(local_tasks, local)
+ with NamedTemporaryFile() as remote:
+ dump_tasks(remote_tasks, remote)
+ result = subprocess.call([diff_program, remote.name, local.name])
+ # Unlike most commands, diff doesn't return zero on success; it returns
+ # 1 when a successful diff is non-empty.
+ if result != 0 and result != 1:
+ return result
+ else:
+ return 0
+
+
+@app.command(name='open')
+def do_open(args, _):
+ """usage: open cluster[/role[/env/job]]
+
+ Opens the scheduler page for a cluster, role or job in the default web browser.
+ """
+ cluster_name = role = env = job = None
+ args = args[0].split("/")
+ if len(args) > 0:
+ cluster_name = args[0]
+ if len(args) > 1:
+ role = args[1]
+ if len(args) > 2:
+ env = args[2]
+ if len(args) > 3:
+ job = args[3]
+ else:
+ # TODO(ksweeney): Remove this after MESOS-2945 is completed.
+ die('env scheduler pages are not yet implemented, please specify job')
+
+ if not cluster_name:
+ die('cluster is required')
+
+ api = make_client(cluster_name)
+
+ import webbrowser
+ webbrowser.open_new_tab(synthesize_url(api.scheduler.scheduler().url, role, env, job))
+
+
+@app.command
+@app.command_option('--local', dest='local', default=False, action='store_true',
+ help='Inspect the configuration as would be created by the "spawn" command.')
+@app.command_option('--raw', dest='raw', default=False, action='store_true',
+ help='Show the raw configuration.')
+@app.command_option(ENVIRONMENT_BIND_OPTION)
+@app.command_option(CLUSTER_CONFIG_OPTION)
+@app.command_option(ENV_CONFIG_OPTION)
+@app.command_option(JSON_OPTION)
+@requires.exactly('cluster/role/env/job', 'config')
+def inspect(job_spec, config_file):
+ """usage: inspect cluster/role/env/job config
+
+ Verifies that a job can be parsed from a configuration file, and displays
+ the parsed configuration.
+ """
+ options = app.get_options()
+ config = get_job_config(job_spec, config_file, options)
+ if options.raw:
+ print('Parsed job config: %s' % config.job())
+ return
+
+ job_thrift = config.job()
+ job = config.raw()
+ job_thrift = config.job()
+ print('Job level information')
+ print(' name: %s' % job.name())
+ print(' role: %s' % job.role())
+ print(' contact: %s' % job.contact())
+ print(' cluster: %s' % job.cluster())
+ print(' instances: %s' % job.instances())
+ if job.has_cron_schedule():
+ print(' cron:')
+ print(' schedule: %s' % job.cron_schedule())
+ print(' policy: %s' % job.cron_collision_policy())
+ if job.has_constraints():
+ print(' constraints:')
+ for constraint, value in job.constraints().get().items():
+ print(' %s: %s' % (constraint, value))
+ print(' service: %s' % job_thrift.taskConfig.isService)
+ print(' production: %s' % bool(job.production().get()))
+ print()
+
+ task = job.task()
+ print('Task level information')
+ print(' name: %s' % task.name())
+ if len(task.constraints().get()) > 0:
+ print(' constraints:')
+ for constraint in task.constraints():
+ print(' %s' % (' < '.join(st.get() for st in constraint.order())))
+ print()
+
+ processes = task.processes()
+ for process in processes:
+ print('Process %s:' % process.name())
+ if process.daemon().get():
+ print(' daemon')
+ if process.ephemeral().get():
+ print(' ephemeral')
+ if process.final().get():
+ print(' final')
+ print(' cmdline:')
+ for line in process.cmdline().get().splitlines():
+ print(' ' + line)
+ print()
+
+
+@app.command
+@app.command_option(CLUSTER_INVOKE_OPTION)
+@app.command_option(OPEN_BROWSER_OPTION)
+def start_cron(args, options):
+ """usage: start_cron cluster/role/env/job
+
+ Invokes a cron job immediately, out of its normal cron cycle.
+ This does not affect the cron cycle in any way.
+ """
+
+ api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
+ args, options, make_client_factory())
+ config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
+ resp = api.start_cronjob(job_key, config=config)
+ check_and_log_response(resp)
+ handle_open(api.scheduler.scheduler().url, job_key.role, job_key.env, job_key.name)
+
+
+@app.command
+@app.command_option(
+ '--pretty',
+ dest='pretty',
+ default=False,
+ action='store_true',
+ help='Show job information in prettyprinted format')
+@app.command_option(
+ '--show-cron',
+ '-c',
+ dest='show_cron_schedule',
+ default=False,
+ action='store_true',
+ help='List jobs registered with the Aurora scheduler')
+@requires.exactly('cluster/role')
+def list_jobs(cluster_and_role):
+ """usage: list_jobs [--show-cron] cluster/role/env/job
+
+ Shows all jobs that match the job-spec known by the scheduler.
+ If --show-cron is specified, then also shows the registered cron schedule.
+ """
+ def show_job_simple(job):
+ if options.show_cron_schedule:
+ print(('{0}/{1.key.role}/{1.key.environment}/{1.key.name}' +
+ '\t\'{1.cronSchedule}\'\t{1.cronCollisionPolicy}').format(cluster, job))
+ else:
+ print('{0}/{1.key.role}/{1.key.environment}/{1.key.name}'.format(cluster, job))
+
+ def show_job_pretty(job):
+ print("Job %s/%s/%s/%s:" %
+ (cluster, job.key.role, job.key.environment, job.key.name))
+ print('\tcron schedule: %s' % job.cronSchedule)
+ print('\tcron policy: %s' % job.cronCollisionPolicy)
+
+ options = app.get_options()
+ if options.show_cron_schedule and options.pretty:
+ print_fn = show_job_pretty
+ else:
+ print_fn = show_job_simple
+ # Take the cluster_and_role parameter, and split it into its two components.
+ if cluster_and_role.count('/') != 1:
+ die('list_jobs parameter must be in cluster/role format')
+ (cluster,role) = cluster_and_role.split('/')
+ api = make_client(cluster)
+ resp = api.get_jobs(role)
+ check_and_log_response(resp)
+ for job in resp.result.getJobsResult.configs:
+ print_fn(job)
+
+
+@app.command
+@app.command_option(CLUSTER_INVOKE_OPTION)
+@app.command_option(OPEN_BROWSER_OPTION)
+@app.command_option(SHARDS_OPTION)
+def kill(args, options):
+ """usage: kill cluster/role/env/job
+
+ Kills a running job, blocking until all tasks have terminated.
+
+ Default behaviour is to kill all shards in the job, but the kill
+ can be limited to specific shards with the --shards option
+ """
+ api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
+ args, options, make_client_factory())
+ options = app.get_options()
+ config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
+ resp = api.kill_job(job_key, options.shards, config=config)
+ check_and_log_response(resp)
+ handle_open(api.scheduler.scheduler().url, job_key.role, job_key.env, job_key.name)
+
+
+@app.command
+@app.command_option(CLUSTER_INVOKE_OPTION)
+def status(args, options):
+ """usage: status cluster/role/env/job
+
+ Fetches and prints information about the active tasks in a job.
+ """
+ def is_active(task):
+ return task.status in ACTIVE_STATES
+
+ def print_task(scheduled_task):
+ assigned_task = scheduled_task.assignedTask
+ taskInfo = assigned_task.task
+ taskString = ''
+ if taskInfo:
+ taskString += '''cpus: %s, ram: %s MB, disk: %s MB''' % (taskInfo.numCpus,
+ taskInfo.ramMb,
+ taskInfo.diskMb)
+ if assigned_task.assignedPorts:
+ taskString += '\n\tports: %s' % assigned_task.assignedPorts
+ taskString += '\n\tfailure count: %s (max %s)' % (scheduled_task.failureCount,
+ taskInfo.maxTaskFailures)
+ taskString += '\n\tevents:'
+ for event in scheduled_task.taskEvents:
+ taskString += '\n\t\t %s %s: %s' % (datetime.fromtimestamp(event.timestamp / 1000),
+ ScheduleStatus._VALUES_TO_NAMES[event.status],
+ event.message)
+ taskString += '\n\tpackages:'
+ for pkg in assigned_task.task.packages:
+ taskString += ('\n\t\trole: %s, package: %s, version: %s' % (pkg.role, pkg.name, pkg.version))
+
+ return taskString
+
+ def print_tasks(tasks):
+ for task in tasks:
+ taskString = print_task(task)
+
+ log.info('role: %s, env: %s, name: %s, shard: %s, status: %s on %s\n%s' %
+ (task.assignedTask.task.owner.role,
+ task.assignedTask.task.environment,
+ task.assignedTask.task.jobName,
+ task.assignedTask.instanceId,
+ ScheduleStatus._VALUES_TO_NAMES[task.status],
+ task.assignedTask.slaveHost,
+ taskString))
+ for pkg in task.assignedTask.task.packages:
+ log.info('\tpackage %s/%s/%s' % (pkg.role, pkg.name, pkg.version))
+
+ api, job_key, _ = LiveJobDisambiguator.disambiguate_args_or_die(
+ args, options, make_client_factory())
+ resp = api.check_status(job_key)
+ check_and_log_response(resp)
+
+ tasks = resp.result.scheduleStatusResult.tasks
+ if tasks:
+ active_tasks = filter(is_active, tasks)
+ log.info('Active Tasks (%s)' % len(active_tasks))
+ print_tasks(active_tasks)
+ inactive_tasks = filter(lambda x: not is_active(x), tasks)
+ log.info('Inactive Tasks (%s)' % len(inactive_tasks))
+ print_tasks(inactive_tasks)
+ else:
+ log.info('No tasks found.')
+
+
+@app.command
+@app.command_option(SHARDS_OPTION)
+@app.command_option(ENVIRONMENT_BIND_OPTION)
+@app.command_option(CLUSTER_CONFIG_OPTION)
+@app.command_option(ENV_CONFIG_OPTION)
+@app.command_option(JSON_OPTION)
+@app.command_option(HEALTH_CHECK_INTERVAL_SECONDS_OPTION)
+@app.command_option(
+ '--force',
+ dest='force',
+ default=True, # TODO(maximk): Temporary bandaid for MESOS-4310 until a better fix is available.
+ action='store_true',
+ help='Turn off warning message that the update looks large enough to be disruptive.')
+@requires.exactly('cluster/role/env/job', 'config')
+def update(job_spec, config_file):
+ """usage: update cluster/role/env/job config
+
+ Performs a rolling upgrade on a running job, using the update configuration
+ within the config file as a control for update velocity and failure tolerance.
+
+ Updates are fully controlled client-side, so aborting an update halts the
+ update and leaves the job in a 'locked' state on the scheduler.
+ Subsequent update attempts will fail until the update is 'unlocked' using the
+ 'cancel_update' command.
+
+ The updater only takes action on shards in a job that have changed, meaning
+ that changing a single shard will only induce a restart on the changed shard.
+
+ You may want to consider using the 'diff' subcommand before updating,
+ to preview what changes will take effect.
+ """
+ def warn_if_dangerous_change(api, job_spec, config):
+ # Get the current job status, so that we can check if there's anything
+ # dangerous about this update.
+ job_key = AuroraJobKey(config.cluster(), config.role(), config.environment(), config.name())
+ resp = api.query(api.build_query(config.role(), config.name(),
+ statuses=ACTIVE_STATES, env=config.environment()))
+ if resp.responseCode != ResponseCode.OK:
+ die('Could not get job status from server for comparison: %s' % resp.message)
+ remote_tasks = [t.assignedTask.task for t in resp.result.scheduleStatusResult.tasks]
+ resp = api.populate_job_config(config)
+ if resp.responseCode != ResponseCode.OK:
+ die('Server could not populate job config for comparison: %s' % resp.message)
+ local_task_count = len(resp.result.populateJobResult.populated)
+ remote_task_count = len(remote_tasks)
+ if (local_task_count >= 4 * remote_task_count or local_task_count <= 4 * remote_task_count
+ or local_task_count == 0):
+ print('Warning: this update is a large change. Press ^c within 5 seconds to abort')
+ time.sleep(5)
+
+ options = app.get_options()
+ config = get_job_config(job_spec, config_file, options)
+ api = make_client(config.cluster())
+ if not options.force:
+ warn_if_dangerous_change(api, job_spec, config)
+ resp = api.update_job(config, options.health_check_interval_seconds, options.shards)
+ check_and_log_response(resp)
+
+
+@app.command
+@app.command_option(CLUSTER_INVOKE_OPTION)
+@app.command_option(HEALTH_CHECK_INTERVAL_SECONDS_OPTION)
+@app.command_option(OPEN_BROWSER_OPTION)
+@app.command_option(SHARDS_OPTION)
+@app.command_option(
+ '--batch_size',
+ dest='batch_size',
+ type=int,
+ default=1,
+ help='Number of shards to be restarted in one iteration.')
+@app.command_option(
+ '--max_per_shard_failures',
+ dest='max_per_shard_failures',
+ type=int,
+ default=0,
+ help='Maximum number of restarts per shard during restart. Increments total failure count when '
+ 'this limit is exceeded.')
+@app.command_option(
+ '--max_total_failures',
+ dest='max_total_failures',
+ type=int,
+ default=0,
+ help='Maximum number of shard failures to be tolerated in total during restart.')
+@app.command_option(
+ '--restart_threshold',
+ dest='restart_threshold',
+ type=int,
+ default=60,
+ help='Maximum number of seconds before a shard must move into the RUNNING state before '
+ 'considered a failure.')
+@app.command_option(
+ '--watch_secs',
+ dest='watch_secs',
+ type=int,
+ default=30,
+ help='Minimum number of seconds a shard must remain in RUNNING state before considered a '
+ 'success.')
+def restart(args, options):
+ """usage: restart cluster/role/env/job
+ [--shards=SHARDS]
+ [--batch_size=INT]
+ [--updater_health_check_interval_seconds=SECONDS]
+ [--max_per_shard_failures=INT]
+ [--max_total_failures=INT]
+ [--restart_threshold=INT]
+ [--watch_secs=SECONDS]
+
+ Performs a rolling restart of shards within a job.
+
+ Restarts are fully controlled client-side, so aborting halts the restart.
+ """
+ api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
+ args, options, make_client_factory())
+ config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
+ updater_config = UpdaterConfig(
+ options.batch_size,
+ options.restart_threshold,
+ options.watch_secs,
+ options.max_per_shard_failures,
+ options.max_total_failures)
+ resp = api.restart(job_key, options.shards, updater_config,
+ options.health_check_interval_seconds, config=config)
+ check_and_log_response(resp)
+ handle_open(api.scheduler.scheduler().url, job_key.role, job_key.env, job_key.name)
+
+
+@app.command
+@app.command_option(CLUSTER_INVOKE_OPTION)
+def cancel_update(args, options):
+ """usage: cancel_update cluster/role/env/job
+
+ Unlocks a job for updates.
+ A job may be locked if a client's update session terminated abnormally,
+ or if another user is actively updating the job. This command should only
+ be used when the user is confident that they are not conflicting with another user.
+ """
+ api, job_key, config_file = LiveJobDisambiguator.disambiguate_args_or_die(
+ args, options, make_client_factory())
+ config = get_job_config(job_key.to_path(), config_file, options) if config_file else None
+ resp = api.cancel_update(job_key, config=config)
+ check_and_log_response(resp)
+
+
+@app.command
+@app.command_option(CLUSTER_INVOKE_OPTION)
+@requires.exactly('role')
+def get_quota(role):
+ """usage: get_quota --cluster=CLUSTER role
+
+ Prints the production quota that has been allocated to a user.
+ """
+ options = app.get_options()
+ resp = make_client(options.cluster).get_quota(role)
+ quota = resp.result.getQuotaResult.quota
+
+ quota_fields = [
+ ('CPU', quota.numCpus),
+ ('RAM', '%f GB' % (float(quota.ramMb) / 1024)),
+ ('Disk', '%f GB' % (float(quota.diskMb) / 1024))
+ ]
+ log.info('Quota for %s:\n\t%s' %
+ (role, '\n\t'.join(['%s\t%s' % (k, v) for (k, v) in quota_fields])))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/commands/help.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/help.py b/src/main/python/apache/aurora/client/commands/help.py
new file mode 100644
index 0000000..a74c607
--- /dev/null
+++ b/src/main/python/apache/aurora/client/commands/help.py
@@ -0,0 +1,53 @@
+from __future__ import print_function
+
+import collections
+import sys
+
+from twitter.aurora.client.base import die
+from twitter.common import app
+
+
+def make_commands_str(commands):
+ commands.sort()
+ if len(commands) == 1:
+ return str(commands[0])
+ elif len(commands) == 2:
+ return '%s (or %s)' % (str(commands[0]), str(commands[1]))
+ else:
+ return '%s (or any of: %s)' % (str(commands[0]), ' '.join(map(str, commands[1:])))
+
+
+def generate_full_usage():
+ docs_to_commands = collections.defaultdict(list)
+ for (command, doc) in app.get_commands_and_docstrings():
+ if doc is not None:
+ docs_to_commands[doc].append(command)
+ def make_docstring(item):
+ (doc_text, commands) = item
+ def format_line(line):
+ return ' %s\n' % line.lstrip()
+ stripped = ''.join(map(format_line, doc_text.splitlines()))
+ return '%s\n%s' % (make_commands_str(commands), stripped)
+ usage = sorted(map(make_docstring, docs_to_commands.items()))
+ return 'Available commands:\n\n' + '\n'.join(usage)
+
+
+@app.command
+def help(args):
+ """usage: help [subcommand]
+
+ Prints help for using the aurora client, or one of its specific subcommands.
+ """
+ if not args:
+ print(generate_full_usage())
+ sys.exit(0)
+
+ if len(args) > 1:
+ die('Please specify at most one subcommand.')
+
+ subcmd = args[0]
+ if subcmd in app.get_commands():
+ app.command_parser(subcmd).print_help()
+ else:
+ print('Subcommand %s not found.' % subcmd)
+ sys.exit(1)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/commands/run.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/run.py b/src/main/python/apache/aurora/client/commands/run.py
new file mode 100644
index 0000000..494ce47
--- /dev/null
+++ b/src/main/python/apache/aurora/client/commands/run.py
@@ -0,0 +1,40 @@
+from twitter.common import app
+from twitter.aurora.client.base import die
+from twitter.aurora.client.options import (
+ EXECUTOR_SANDBOX_OPTION,
+ SSH_USER_OPTION,
+)
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+from twitter.aurora.common.clusters import CLUSTERS
+from twitter.aurora.client.api.command_runner import DistributedCommandRunner
+
+
+@app.command
+@app.command_option('-t', '--threads', type=int, default=1, dest='num_threads',
+ help='The number of threads to use.')
+@app.command_option(SSH_USER_OPTION)
+@app.command_option(EXECUTOR_SANDBOX_OPTION)
+def run(args, options):
+ """usage: run cluster/role/env/job cmd
+
+ Runs a shell command on all machines currently hosting shards of a single job.
+
+ This feature supports the same command line wildcards that are used to
+ populate a job's commands.
+
+ This means anything in the {{mesos.*}} and {{thermos.*}} namespaces.
+ """
+ # TODO(William Farner): Add support for invoking on individual shards.
+ # TODO(Kevin Sweeney): Restore the ability to run across jobs with globs (See MESOS-3010).
+ if not args:
+ die('job path is required')
+ job_path = args.pop(0)
+ try:
+ cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
+ except AuroraJobKey.Error as e:
+ die('Invalid job path "%s": %s' % (job_path, e))
+
+ command = ' '.join(args)
+ cluster = CLUSTERS[cluster_name]
+ dcr = DistributedCommandRunner(cluster, role, env, [name], options.ssh_user)
+ dcr.run(command, parallelism=options.num_threads, executor_sandbox=options.executor_sandbox)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/commands/ssh.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/commands/ssh.py b/src/main/python/apache/aurora/client/commands/ssh.py
new file mode 100644
index 0000000..109229f
--- /dev/null
+++ b/src/main/python/apache/aurora/client/commands/ssh.py
@@ -0,0 +1,64 @@
+import subprocess
+
+from twitter.common import app
+from twitter.aurora.client.base import check_and_log_response, die
+from twitter.aurora.client.factory import make_client
+from twitter.aurora.client.options import (
+ EXECUTOR_SANDBOX_OPTION,
+ SSH_USER_OPTION,
+)
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+from twitter.aurora.client.api.command_runner import DistributedCommandRunner
+
+
+@app.command
+@app.command_option(EXECUTOR_SANDBOX_OPTION)
+@app.command_option(SSH_USER_OPTION)
+@app.command_option('-L', dest='tunnels', action='append', metavar='PORT:NAME',
+ default=[],
+ help="Add tunnel from local port PORT to remote named port NAME.")
+def ssh(args, options):
+ """usage: ssh cluster/role/env/job shard [args...]
+
+ Initiate an SSH session on the machine that a shard is running on.
+ """
+ if not args:
+ die('Job path is required')
+ job_path = args.pop(0)
+ try:
+ cluster_name, role, env, name = AuroraJobKey.from_path(job_path)
+ except AuroraJobKey.Error as e:
+ die('Invalid job path "%s": %s' % (job_path, e))
+ if not args:
+ die('Shard is required')
+ try:
+ shard = int(args.pop(0))
+ except ValueError:
+ die('Shard must be an integer')
+ api = make_client(cluster_name)
+ resp = api.query(api.build_query(role, name, set([int(shard)]), env=env))
+ check_and_log_response(resp)
+
+ first_task = resp.result.scheduleStatusResult.tasks[0]
+ remote_cmd = 'bash' if not args else ' '.join(args)
+ command = DistributedCommandRunner.substitute(remote_cmd, first_task,
+ api.cluster, executor_sandbox=options.executor_sandbox)
+
+ ssh_command = ['ssh', '-t']
+
+ role = first_task.assignedTask.task.owner.role
+ slave_host = first_task.assignedTask.slaveHost
+
+ for tunnel in options.tunnels:
+ try:
+ port, name = tunnel.split(':')
+ port = int(port)
+ except ValueError:
+ die('Could not parse tunnel: %s. Must be of form PORT:NAME' % tunnel)
+ if name not in first_task.assignedTask.assignedPorts:
+ die('Task %s has no port named %s' % (first_task.assignedTask.taskId, name))
+ ssh_command += [
+ '-L', '%d:%s:%d' % (port, slave_host, first_task.assignedTask.assignedPorts[name])]
+
+ ssh_command += ['%s@%s' % (options.ssh_user or role, slave_host), command]
+ return subprocess.call(ssh_command)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/config.py b/src/main/python/apache/aurora/client/config.py
new file mode 100644
index 0000000..32df4eb
--- /dev/null
+++ b/src/main/python/apache/aurora/client/config.py
@@ -0,0 +1,208 @@
+'''Library of utilities called by the mesos client binary
+'''
+
+from __future__ import print_function
+
+import functools
+import math
+import posixpath
+import re
+import sys
+
+from twitter.common import app, log
+
+from twitter.aurora.client import binding_helper
+from twitter.aurora.client.base import deprecation_warning, die
+from twitter.aurora.config import AuroraConfig
+from twitter.thermos.config.schema_helpers import Tasks
+
+from gen.twitter.aurora.constants import DEFAULT_ENVIRONMENT
+
+from pystachio import Empty, Ref
+
+
+APPAPP_DEPRECATION_WARNING = """
+The use of app-app is deprecated. Please reach out to mesos-team@twitter.com for advice on
+migrating your application away from app-app layouts to an alternative packaging solution.
+"""
+
+
+def _warn_on_appapp_layouts(config):
+ if config.raw().has_layout():
+ deprecation_warning(APPAPP_DEPRECATION_WARNING)
+
+
+CRON_DEPRECATION_WARNING = """
+The "cron_policy" parameter to Jobs has been renamed to "cron_collision_policy".
+Please update your Jobs accordingly.
+"""
+
+
+def _warn_on_deprecated_cron_policy(config):
+ if config.raw().cron_policy() is not Empty:
+ deprecation_warning(CRON_DEPRECATION_WARNING)
+
+
+DAEMON_DEPRECATION_WARNING = """
+The "daemon" parameter to Jobs is deprecated in favor of the "service" parameter.
+Please update your Job to set "service = True" instead of "daemon = True", or use
+the top-level Service() instead of Job().
+"""
+
+
+def _warn_on_deprecated_daemon_job(config):
+ if config.raw().daemon() is not Empty:
+ deprecation_warning(DAEMON_DEPRECATION_WARNING)
+
+
+HEALTH_CHECK_INTERVAL_SECS_DEPRECATION_WARNING = """
+The "health_check_interval_secs" parameter to Jobs is deprecated in favor of the
+"health_check_config" parameter. Please update your Job to set the parameter by creating a new
+HealthCheckConfig.
+
+See the HealthCheckConfig section of the Configuration Reference page for more information:
+http://go/auroraconfig/#Aurora%2BThermosConfigurationReference-HealthCheckConfig
+"""
+
+
+def _warn_on_deprecated_health_check_interval_secs(config):
+ if config.raw().health_check_interval_secs() is not Empty:
+ deprecation_warning(HEALTH_CHECK_INTERVAL_SECS_DEPRECATION_WARNING)
+
+
+ANNOUNCE_WARNING = """
+Announcer specified primary port as '%(primary_port)s' but no processes have bound that port.
+If you would like to utilize this port, you should listen on {{thermos.ports[%(primary_port)s]}}
+from some Process bound to your task.
+"""
+
+
+def _validate_announce_configuration(config):
+ if not config.raw().has_announce():
+ return
+
+ primary_port = config.raw().announce().primary_port().get()
+ if primary_port not in config.ports():
+ print(ANNOUNCE_WARNING % {'primary_port': primary_port}, file=sys.stderr)
+
+ if config.raw().has_announce() and not config.raw().has_constraints() or (
+ 'dedicated' not in config.raw().constraints()):
+ for port in config.raw().announce().portmap().get().values():
+ try:
+ port = int(port)
+ except ValueError:
+ continue
+ raise ValueError('Job must be dedicated in order to specify static ports!')
+
+
+STAGING_RE = re.compile(r'^staging\d*$')
+
+
+def _validate_environment_name(config):
+ env_name = str(config.raw().environment())
+ if STAGING_RE.match(env_name):
+ return
+ if env_name not in ('prod', 'devel', 'test'):
+ raise ValueError('Environment name should be one of "prod", "devel", "test" or '
+ 'staging<number>! Got %s' % env_name)
+
+
+UPDATE_CONFIG_MAX_FAILURES_ERROR = '''
+max_total_failures in update_config must be lesser than the job size.
+Based on your job size (%s) you should use max_total_failures <= %s.
+
+See http://go/auroraconfig for details.
+'''
+
+
+UPDATE_CONFIG_DEDICATED_THRESHOLD_ERROR = '''
+Since this is a dedicated job, you must set your max_total_failures in
+your update configuration to no less than 2%% of your job size.
+Based on your job size (%s) you should use max_total_failures >= %s.
+
+See http://go/auroraconfig for details.
+'''
+
+
+def _validate_update_config(config):
+ job_size = config.instances()
+ max_failures = config.update_config().max_total_failures().get()
+
+ if max_failures >= job_size:
+ die(UPDATE_CONFIG_MAX_FAILURES_ERROR % (job_size, job_size - 1))
+
+ if config.is_dedicated():
+ min_failure_threshold = int(math.floor(job_size * 0.02))
+ if max_failures < min_failure_threshold:
+ die(UPDATE_CONFIG_DEDICATED_THRESHOLD_ERROR % (job_size, min_failure_threshold))
+
+
+HEALTH_CHECK_INTERVAL_SECS_ERROR = '''
+health_check_interval_secs paramater to Job has been deprecated. Please specify health_check_config
+only.
+
+See http://go/auroraconfig/#Aurora%2BThermosConfigurationReference-HealthCheckConfig
+'''
+
+
+def _validate_health_check_config(config):
+ # TODO(Sathya): Remove this check after health_check_interval_secs deprecation cycle is complete.
+ if config.raw().has_health_check_interval_secs() and config.raw().has_health_check_config():
+ die(HEALTH_CHECK_INTERVAL_SECS_ERROR)
+
+
+DEFAULT_ENVIRONMENT_WARNING = '''
+Job did not specify environment, auto-populating to "%s".
+'''
+
+
+def _inject_default_environment(config):
+ if not config.raw().has_environment():
+ print(DEFAULT_ENVIRONMENT_WARNING % DEFAULT_ENVIRONMENT, file=sys.stderr)
+ config.update_job(config.raw()(environment=DEFAULT_ENVIRONMENT))
+
+
+def validate_config(config, env=None):
+ _validate_update_config(config)
+ _validate_health_check_config(config)
+ _validate_announce_configuration(config)
+ _validate_environment_name(config)
+
+
+def populate_namespaces(config, env=None):
+ _inject_default_environment(config)
+ _warn_on_deprecated_cron_policy(config)
+ _warn_on_deprecated_daemon_job(config)
+ _warn_on_deprecated_health_check_interval_secs(config)
+ _warn_on_appapp_layouts(config)
+ return config
+
+
+def inject_hooks(config, env=None):
+ config.hooks = (env or {}).get('hooks', [])
+
+
+class AnnotatedAuroraConfig(AuroraConfig):
+ @classmethod
+ def plugins(cls):
+ return (inject_hooks,
+ functools.partial(binding_helper.apply_all),
+ functools.partial(populate_namespaces),
+ validate_config)
+
+
+def get_config(jobname,
+ config_file,
+ json=False,
+ bindings=(),
+ select_cluster=None,
+ select_role=None,
+ select_env=None):
+ """Creates and returns a config object contained in the provided file."""
+ loader = AnnotatedAuroraConfig.load_json if json else AnnotatedAuroraConfig.load
+ return loader(config_file,
+ jobname,
+ bindings,
+ select_cluster=select_cluster,
+ select_role=select_role,
+ select_env=select_env)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/factory.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/factory.py b/src/main/python/apache/aurora/client/factory.py
new file mode 100644
index 0000000..7a44e3b
--- /dev/null
+++ b/src/main/python/apache/aurora/client/factory.py
@@ -0,0 +1,27 @@
+import functools
+
+from twitter.aurora.client.hooks.hooked_api import HookedAuroraClientAPI
+from twitter.aurora.common.cluster import Cluster
+from twitter.aurora.common.clusters import CLUSTERS
+from twitter.common import app
+
+from .base import die
+
+
+# TODO(wickman) Kill make_client and make_client_factory as part of MESOS-3801.
+# These are currently necessary indirections for the LiveJobDisambiguator among
+# other things but can go away once those are scrubbed.
+
+def make_client_factory():
+ verbose = getattr(app.get_options(), 'verbosity', 'normal') == 'verbose'
+ class TwitterAuroraClientAPI(HookedAuroraClientAPI):
+ def __init__(self, cluster, *args, **kw):
+ if cluster not in CLUSTERS:
+ die('Unknown cluster: %s' % cluster)
+ super(TwitterAuroraClientAPI, self).__init__(CLUSTERS[cluster], *args, **kw)
+ return functools.partial(TwitterAuroraClientAPI, verbose=verbose)
+
+
+def make_client(cluster):
+ factory = make_client_factory()
+ return factory(cluster.name if isinstance(cluster, Cluster) else cluster)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/hooks/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/hooks/BUILD b/src/main/python/apache/aurora/client/hooks/BUILD
new file mode 100644
index 0000000..c3d0a1a
--- /dev/null
+++ b/src/main/python/apache/aurora/client/hooks/BUILD
@@ -0,0 +1,10 @@
+python_library(
+ name = 'hooks',
+ sources = ['__init__.py', 'hooked_api.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/client:api'),
+ pants('src/main/python/twitter/aurora/common:aurora_job_key'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/hooks/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/hooks/__init__.py b/src/main/python/apache/aurora/client/hooks/__init__.py
new file mode 100644
index 0000000..3091f67
--- /dev/null
+++ b/src/main/python/apache/aurora/client/hooks/__init__.py
@@ -0,0 +1,49 @@
+"""
+A hooks implementation for the Aurora client.
+
+The Hook protocol is the following:
+ Any object may be passed in as a hook.
+
+ If the object has pre_<api method name> defined that is callable, it will be called with:
+ method(*args, **kw)
+
+ where *args and **kw are the arguments and keyword arguments passed into
+ the original APi call. This is done prior to the invocation of the API
+ call. If this method returns Falsy, the API call will be aborted.
+
+ If the object has an err_<api method name> defined that is callable, it will be called with:
+ method(exc, *args, **kw)
+
+ If the object has a post_<api method name> defined that is callable, it will be called with:
+ method(result, *args, **kw)
+
+ These methods are called after the respective API call has been made. The
+ return codes of err and post methods are ignored.
+
+If the object does not have any of these attributes, it will instead delegate to the
+'generic_hook' method, if available. The method signature for generic_hook is:
+
+ generic_hook(hook_config, event, method_name, result_or_err, args, kw)
+
+Where hook_config is a namedtuple of 'config' and 'job_key', event is one of
+'pre', 'err', 'post', method_name is the API method name, and args, kw are
+the arguments / keyword arguments. result_or_err is a tri_state:
+ - None for pre hooks
+ - result for post hooks
+ - exc for err hooks
+
+Examples:
+
+ class Logger(object):
+ '''Just logs every at all point for all API calls'''
+ def generic_hook(self, hook_config, event, method_name, result_or_err, *args, **kw)
+ log.info('%s: %s_%s of %s' % (self.__class__.__name__, event, method_name, job_key))
+
+ class KillConfirmer(object):
+ def confirm(self, msg):
+ return True if raw_input(msg).lower() == 'yes' else False
+
+ def pre_kill(self, job_key, shards=None):
+ shards = ('shards %s' % shards) if shards is not None else 'all shards'
+ return self.confirm('Are you sure you want to kill %s? (yes/no): ' % (job_key, shards))
+"""
[24/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/BUILD b/src/main/python/apache/aurora/executor/common/BUILD
new file mode 100644
index 0000000..9c932d1
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/BUILD
@@ -0,0 +1,69 @@
+python_library(
+ name = 'status_checker',
+ sources = ['status_checker.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
+ ]
+)
+
+python_library(
+ name = 'task_runner',
+ sources = ['task_runner.py'],
+ dependencies = [
+ pants(':status_checker'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ ]
+)
+
+python_library(
+ name = 'health_checker',
+ sources = ['health_checker.py'],
+ dependencies = [
+ pants(':status_checker'),
+ pants(':task_info'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('src/main/python/twitter/aurora/common:http_signaler'),
+ ]
+)
+
+python_library(
+ name = 'executor_timeout',
+ sources = ['executor_timeout.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ ]
+)
+
+python_library(
+ name = 'kill_manager',
+ sources = ['kill_manager.py'],
+ dependencies = [
+ pants(':status_checker'),
+ ]
+)
+
+python_library(
+ name = 'sandbox',
+ sources = ['sandbox.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ ]
+)
+
+python_library(
+ name = 'task_info',
+ sources = ['task_info.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/config'),
+ pants('src/main/python/twitter/aurora/config/schema'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/__init__.py b/src/main/python/apache/aurora/executor/common/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/executor_timeout.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/executor_timeout.py b/src/main/python/apache/aurora/executor/common/executor_timeout.py
new file mode 100644
index 0000000..2828973
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/executor_timeout.py
@@ -0,0 +1,21 @@
+from twitter.common import log
+from twitter.common.quantity import Amount, Time
+from twitter.common.exceptions import ExceptionalThread
+
+
+class ExecutorTimeout(ExceptionalThread):
+ DEFAULT_TIMEOUT = Amount(10, Time.SECONDS)
+
+ def __init__(self, event, driver, logger=log.error, timeout=DEFAULT_TIMEOUT):
+ self._event = event
+ self._logger = logger
+ self._driver = driver
+ self._timeout = timeout
+ super(ExecutorTimeout, self).__init__()
+ self.daemon = True
+
+ def run(self):
+ self._event.wait(self._timeout.as_(Time.SECONDS))
+ if not self._event.is_set():
+ self._logger('Executor timing out.')
+ self._driver.stop()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/health_checker.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/health_checker.py b/src/main/python/apache/aurora/executor/common/health_checker.py
new file mode 100644
index 0000000..3b25f0e
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/health_checker.py
@@ -0,0 +1,99 @@
+import threading
+import time
+
+from twitter.aurora.common.http_signaler import HttpSignaler
+from twitter.common import log
+from twitter.common.exceptions import ExceptionalThread
+
+from .status_checker import (
+ ExitState,
+ StatusChecker,
+ StatusCheckerProvider,
+ StatusResult,
+)
+from .task_info import mesos_task_instance_from_assigned_task, resolve_ports
+
+
+class HealthCheckerThread(StatusChecker, ExceptionalThread):
+ """Generic, StatusChecker-conforming thread for arbitrary periodic health checks
+
+ health_checker should be a callable returning a tuple of (boolean, reason), indicating
+ respectively the health of the service and the reason for its failure (or None if the service is
+ still healthy).
+ """
+ def __init__(self,
+ health_checker,
+ interval_secs=30,
+ initial_interval_secs=None,
+ max_consecutive_failures=0,
+ clock=time):
+ self._checker = health_checker
+ self._interval = interval_secs
+ if initial_interval_secs is not None:
+ self._initial_interval = initial_interval_secs
+ else:
+ self._initial_interval = interval_secs * 2
+ self._current_consecutive_failures = 0
+ self._max_consecutive_failures = max_consecutive_failures
+ self._dead = threading.Event()
+ if self._initial_interval > 0:
+ self._healthy, self._reason = True, None
+ else:
+ self._healthy, self._reason = self._checker()
+ self._clock = clock
+ super(HealthCheckerThread, self).__init__()
+ self.daemon = True
+
+ @property
+ def status(self):
+ if not self._healthy:
+ return StatusResult('Failed health check! %s' % self._reason, ExitState.FAILED)
+
+ def run(self):
+ log.debug('Health checker thread started.')
+ self._clock.sleep(self._initial_interval)
+ log.debug('Initial interval expired.')
+ while not self._dead.is_set():
+ self._maybe_update_failure_count(*self._checker())
+ self._clock.sleep(self._interval)
+
+ def _maybe_update_failure_count(self, is_healthy, reason):
+ if not is_healthy:
+ log.warning('Health check failure: %s' % reason)
+ self._current_consecutive_failures += 1
+ if self._current_consecutive_failures > self._max_consecutive_failures:
+ log.warning('Reached consecutive failure limit.')
+ self._healthy = False
+ self._reason = reason
+ else:
+ if self._current_consecutive_failures > 0:
+ log.debug('Reset consecutive failures counter.')
+ self._current_consecutive_failures = 0
+
+ def start(self):
+ StatusChecker.start(self)
+ ExceptionalThread.start(self)
+
+ def stop(self):
+ log.debug('Health checker thread stopped.')
+ self._dead.set()
+
+
+class HealthCheckerProvider(StatusCheckerProvider):
+ def from_assigned_task(self, assigned_task, _):
+ mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
+ portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
+
+ if 'health' not in portmap:
+ return None
+
+ health_check_config = mesos_task.health_check_config().get()
+ http_signaler = HttpSignaler(
+ portmap['health'],
+ timeout_secs=health_check_config.get('timeout_secs'))
+ health_checker = HealthCheckerThread(
+ http_signaler.health,
+ interval_secs=health_check_config.get('interval_secs'),
+ initial_interval_secs=health_check_config.get('initial_interval_secs'),
+ max_consecutive_failures=health_check_config.get('max_consecutive_failures'))
+ return health_checker
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/kill_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/kill_manager.py b/src/main/python/apache/aurora/executor/common/kill_manager.py
new file mode 100644
index 0000000..70ab733
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/kill_manager.py
@@ -0,0 +1,19 @@
+from .status_checker import ExitState, StatusChecker, StatusResult
+
+
+class KillManager(StatusChecker):
+ """
+ A health interface that provides a kill-switch for a task monitored by the status manager.
+ """
+ def __init__(self):
+ self._killed = False
+ self._reason = None
+
+ @property
+ def status(self):
+ if self._killed:
+ return StatusResult(self._reason, ExitState.KILLED)
+
+ def kill(self, reason):
+ self._reason = reason
+ self._killed = True
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/sandbox.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/sandbox.py b/src/main/python/apache/aurora/executor/common/sandbox.py
new file mode 100644
index 0000000..551d360
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/sandbox.py
@@ -0,0 +1,72 @@
+from abc import abstractmethod, abstractproperty
+import getpass
+import grp
+import os
+import pwd
+
+from twitter.common import log
+from twitter.common.dirutil import safe_mkdir, safe_rmtree
+from twitter.common.lang import Interface
+
+
+class SandboxInterface(Interface):
+ class Error(Exception): pass
+ class CreationError(Error): pass
+ class DeletionError(Error): pass
+
+ @abstractproperty
+ def root(self):
+ """Return the root path of the sandbox."""
+
+ @abstractproperty
+ def chrooted(self):
+ """Returns whether or not the sandbox is a chroot."""
+
+ @abstractmethod
+ def exists(self):
+ """Returns true if the sandbox appears to exist."""
+
+ @abstractmethod
+ def create(self, *args, **kw):
+ """Create the sandbox."""
+
+ @abstractmethod
+ def destroy(self, *args, **kw):
+ """Destroy the sandbox."""
+
+
+class SandboxProvider(Interface):
+ @abstractmethod
+ def from_assigned_task(self, assigned_task):
+ """Return the appropriate Sandbox implementation from an AssignedTask."""
+
+
+class DirectorySandbox(SandboxInterface):
+ """ Basic sandbox implementation using a directory on the filesystem """
+ def __init__(self, root, user=getpass.getuser()):
+ self._root = root
+ self._user = user
+
+ @property
+ def root(self):
+ return self._root
+
+ @property
+ def chrooted(self):
+ return False
+
+ def exists(self):
+ return os.path.exists(self.root)
+
+ def create(self):
+ log.debug('DirectorySandbox: mkdir %s' % self.root)
+ safe_mkdir(self.root)
+ pwent = pwd.getpwnam(self._user)
+ grent = grp.getgrgid(pwent.pw_gid)
+ log.debug('DirectorySandbox: chown %s:%s %s' % (self._user, grent.gr_name, self.root))
+ os.chown(self.root, pwent.pw_uid, pwent.pw_gid)
+ log.debug('DirectorySandbox: chmod 700 %s' % self.root)
+ os.chmod(self.root, 0700)
+
+ def destroy(self):
+ safe_rmtree(self.root)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/status_checker.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/status_checker.py b/src/main/python/apache/aurora/executor/common/status_checker.py
new file mode 100644
index 0000000..efffa8f
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/status_checker.py
@@ -0,0 +1,111 @@
+from abc import abstractmethod, abstractproperty
+
+from twitter.common import log
+from twitter.common.lang import Interface
+from twitter.common.metrics import NamedGauge, Observable
+
+
+# This mirrors mesos_pb2 TaskStatus without explicitly depending upon it.
+#
+# The dependency is a 30MB egg, so for smaller applications that just need
+# the status text, we proxy them. The actual conversion betwen ExitState
+# and TaskStatus is done in the StatusManager.
+class ExitState(object):
+ FAILED = object()
+ FINISHED = object()
+ KILLED = object()
+ LOST = object()
+
+ ALL_STATES = {
+ FAILED: 'FAILED',
+ FINISHED: 'FINISHED',
+ KILLED: 'KILLED',
+ LOST: 'LOST',
+ }
+
+
+class StatusResult(object):
+ """
+ Encapsulates a reason for failure and an optional reason which defaults to
+ ExitState.FAILED.
+ """
+
+ def __init__(self, reason, status):
+ self._reason = reason
+ if status not in ExitState.ALL_STATES:
+ raise ValueError('Unknown task state: %r' % status)
+ self._status = status
+
+ @property
+ def reason(self):
+ return self._reason
+
+ @property
+ def status(self):
+ return self._status
+
+ def __repr__(self):
+ return '%s(%r, status=%r)' % (
+ self.__class__.__name__,
+ self._reason,
+ ExitState.ALL_STATES[self._status])
+
+
+class StatusChecker(Observable, Interface):
+ """Interface to pluggable status checkers for the Aurora Executor."""
+
+ @abstractproperty
+ def status(self):
+ """Return None under normal operations. Return StatusResult to indicate status proposal."""
+
+ def start(self):
+ """Invoked once the task has been started."""
+ self.metrics.register(NamedGauge('enabled', 1))
+
+ def stop(self):
+ """Invoked once a non-None status has been reported."""
+ pass
+
+
+class StatusCheckerProvider(Interface):
+ @abstractmethod
+ def from_assigned_task(self, assigned_task, sandbox):
+ pass
+
+
+class Healthy(StatusChecker):
+ @property
+ def status(self):
+ return None
+
+
+class ChainedStatusChecker(StatusChecker):
+ def __init__(self, status_checkers):
+ self._status_checkers = status_checkers
+ self._status = None
+ if not all(isinstance(h_i, StatusChecker) for h_i in status_checkers):
+ raise TypeError('ChainedStatusChecker must take an iterable of StatusCheckers.')
+ super(ChainedStatusChecker, self).__init__()
+
+ @property
+ def status(self):
+ if self._status is None:
+ for status_checker in self._status_checkers:
+ log.debug('Checking status from %s' % status_checker.__class__.__name__)
+ status_checker_status = status_checker.status
+ if status_checker_status is not None:
+ log.info('%s reported %s' % (status_checker.__class__.__name__, status_checker_status))
+ if not isinstance(status_checker_status, StatusResult):
+ raise TypeError('StatusChecker returned something other than a StatusResult: got %s' %
+ type(status_checker_status))
+ self._status = status_checker_status
+ break
+ return self._status
+
+ def start(self):
+ for status_checker in self._status_checkers:
+ status_checker.start()
+
+ def stop(self):
+ for status_checker in self._status_checkers:
+ status_checker.stop()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/task_info.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/task_info.py b/src/main/python/apache/aurora/executor/common/task_info.py
new file mode 100644
index 0000000..1a7cf27
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/task_info.py
@@ -0,0 +1,90 @@
+import json
+
+from twitter.aurora.config.port_resolver import PortResolver
+from twitter.aurora.config.schema.base import MesosJob, MesosTaskInstance
+from twitter.aurora.config.thrift import task_instance_from_job
+from twitter.common import log
+
+from gen.twitter.aurora.ttypes import AssignedTask
+
+from pystachio import Ref
+from thrift.Thrift import TException
+from thrift.TSerialization import deserialize as thrift_deserialize
+
+
+def assigned_task_from_mesos_task(task):
+ """Deserialize AssignedTask from a launchTask task protocol buffer."""
+ try:
+ assigned_task = thrift_deserialize(AssignedTask(), task.data)
+ except (EOFError, TException) as e:
+ raise ValueError('Could not deserialize task! %s' % e)
+ return assigned_task
+
+
+def mesos_job_from_assigned_task(assigned_task):
+ """Deserialize a MesosJob pystachio struct from an AssignedTask."""
+ thermos_task = assigned_task.task.executorConfig.data
+ try:
+ json_blob = json.loads(thermos_task)
+ except (TypeError, ValueError):
+ return None
+ if 'instance' in json_blob:
+ # This is a MesosTaskInstance so we cannot get a MesosJob from this assigned_task
+ return None
+ return MesosJob.json_loads(thermos_task)
+
+
+def mesos_task_instance_from_assigned_task(assigned_task):
+ """Deserialize MesosTaskInstance from an AssignedTask thrift."""
+ thermos_task = assigned_task.task.executorConfig.data
+
+ if not thermos_task:
+ raise ValueError('Task did not have a thermos config!')
+
+ try:
+ json_blob = json.loads(thermos_task)
+ except (TypeError, ValueError) as e:
+ raise ValueError('Could not deserialize thermos config: %s' % e)
+
+ # As part of the transition for MESOS-2133, we can send either a MesosTaskInstance
+ # or we can be sending a MesosJob. So handle both possible cases. Once everyone
+ # is using MesosJob, then we can begin to leverage additional information that
+ # becomes available such as cluster.
+ if 'instance' in json_blob:
+ return MesosTaskInstance.json_loads(thermos_task)
+
+ # This is a MesosJob
+ mti, refs = task_instance_from_job(MesosJob.json_loads(thermos_task), assigned_task.instanceId)
+ for ref in refs:
+ # If the ref is {{thermos.task_id}} or a subscope of
+ # {{thermos.ports}}, it currently gets bound by the Thermos Runner,
+ # so we must leave them unbound.
+ #
+ # {{thermos.user}} is a legacy binding which we can safely ignore.
+ #
+ # TODO(wickman) These should be rewritten by the mesos client to use
+ # %%style%% replacements in order to allow us to better type-check configs
+ # client-side.
+ if ref == Ref.from_address('thermos.task_id'):
+ continue
+ if Ref.subscope(Ref.from_address('thermos.ports'), ref):
+ continue
+ if ref == Ref.from_address('thermos.user'):
+ continue
+ raise ValueError('Unexpected unbound refs: %s' % ' '.join(map(str, refs)))
+ return mti
+
+
+def resolve_ports(mesos_task, portmap):
+ """Given a MesosTaskInstance and the portmap of resolved ports from the scheduler,
+ create a fully resolved map of port name => port number for the thermos
+ runner and discovery manager."""
+ task_portmap = mesos_task.announce().portmap().get() if mesos_task.has_announce() else {}
+ task_portmap.update(portmap)
+ task_portmap = PortResolver.resolve(task_portmap)
+
+ for name, port in task_portmap.items():
+ if not isinstance(port, int):
+ log.warning('Task has unmapped port: %s => %s' % (name, port))
+
+ return dict((name, port) for (name, port) in task_portmap.items() if isinstance(port, int))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/common/task_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/common/task_runner.py b/src/main/python/apache/aurora/executor/common/task_runner.py
new file mode 100644
index 0000000..c7a57cb
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/common/task_runner.py
@@ -0,0 +1,20 @@
+from abc import abstractmethod
+
+from twitter.common.lang import Interface
+
+from .status_checker import StatusChecker
+
+
+class TaskError(Exception):
+ pass
+
+
+class TaskRunner(StatusChecker):
+ # For now, TaskRunner should just maintain the StatusChecker API.
+ pass
+
+
+class TaskRunnerProvider(Interface):
+ @abstractmethod
+ def from_assigned_task(self, assigned_task, sandbox):
+ pass
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/executor_base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/executor_base.py b/src/main/python/apache/aurora/executor/executor_base.py
new file mode 100644
index 0000000..4fd9aa4
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/executor_base.py
@@ -0,0 +1,103 @@
+from twitter.common import log
+
+from gen.twitter.aurora.constants import TERMINAL_STATES as AURORA_TERMINAL_STATES
+from gen.twitter.aurora.ttypes import ScheduleStatus
+from gen.twitter.thermos.ttypes import TaskState
+
+import mesos
+import mesos_pb2 as mesos_pb
+
+
+class ThermosExecutorBase(mesos.Executor):
+ # Statuses are hard, let's go shopping.
+ MESOS_STATES = {
+ mesos_pb.TASK_STARTING: 'STARTING',
+ mesos_pb.TASK_RUNNING: 'RUNNING',
+ mesos_pb.TASK_FINISHED: 'FINISHED',
+ mesos_pb.TASK_FAILED: 'FAILED',
+ mesos_pb.TASK_KILLED: 'KILLED',
+ mesos_pb.TASK_LOST: 'LOST',
+ }
+
+ THERMOS_TO_MESOS_STATES = {
+ TaskState.ACTIVE: mesos_pb.TASK_RUNNING,
+ TaskState.SUCCESS: mesos_pb.TASK_FINISHED,
+ TaskState.FAILED: mesos_pb.TASK_FAILED,
+ TaskState.KILLED: mesos_pb.TASK_KILLED,
+ TaskState.LOST: mesos_pb.TASK_LOST,
+ }
+
+ THERMOS_TO_TWITTER_STATES = {
+ TaskState.ACTIVE: ScheduleStatus.RUNNING,
+ TaskState.CLEANING: ScheduleStatus.RUNNING,
+ TaskState.FINALIZING: ScheduleStatus.RUNNING,
+ TaskState.SUCCESS: ScheduleStatus.FINISHED,
+ TaskState.FAILED: ScheduleStatus.FAILED,
+ TaskState.KILLED: ScheduleStatus.KILLED,
+ TaskState.LOST: ScheduleStatus.LOST,
+ }
+
+ @staticmethod
+ def twitter_status_is_terminal(status):
+ return status in AURORA_TERMINAL_STATES
+
+ @staticmethod
+ def mesos_status_is_terminal(status):
+ return status in (
+ mesos_pb.TASK_FAILED,
+ mesos_pb.TASK_FINISHED,
+ mesos_pb.TASK_KILLED,
+ mesos_pb.TASK_LOST,
+ )
+
+ @staticmethod
+ def thermos_status_is_terminal(status):
+ return status in (
+ TaskState.FAILED,
+ TaskState.KILLED,
+ TaskState.LOST,
+ TaskState.SUCCESS,
+ )
+
+ def __init__(self):
+ self._slave_id = None
+
+ def log(self, msg):
+ log.info('Executor [%s]: %s' % (self._slave_id, msg))
+
+ def registered(self, driver, executor_info, framework_info, slave_info):
+ self.log('registered() called with:')
+ self.log(' ExecutorInfo: %s' % executor_info)
+ self.log(' FrameworkInfo: %s' % framework_info)
+ self.log(' SlaveInfo: %s' % slave_info)
+ self._driver = driver
+ self._executor_info = executor_info
+ self._framework_info = framework_info
+ self._slave_info = slave_info
+
+ def reregistered(self, driver, slave_info):
+ self.log('reregistered() called with:')
+ self.log(' SlaveInfo: %s' % slave_info)
+
+ def disconnected(self, driver):
+ self.log('disconnected() called')
+
+ def send_update(self, driver, task_id, state, message=None):
+ update = mesos_pb.TaskStatus()
+ if not isinstance(state, int):
+ raise TypeError('Invalid state type %s, should be int.' % type(state))
+ if state not in self.MESOS_STATES:
+ raise ValueError('Invalid state: %s' % state)
+ update.state = state
+ update.task_id.value = task_id
+ if message:
+ update.message = str(message)
+ self.log('Updating %s => %s' % (task_id, self.MESOS_STATES[state]))
+ self.log(' Reason: %s' % message)
+ driver.sendStatusUpdate(update)
+
+ def frameworkMessage(self, driver, message):
+ self.log('frameworkMessage() got message: %s, ignoring.' % message)
+
+ def error(self, driver, message):
+ self.log('Received error message: %s' % message)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/executor_detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/executor_detector.py b/src/main/python/apache/aurora/executor/executor_detector.py
new file mode 100644
index 0000000..c26c00e
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/executor_detector.py
@@ -0,0 +1,77 @@
+import os
+from glob import glob
+
+from twitter.common.string import ScanfParser
+
+
+# TODO(wickman) MESOS-2805 This makes an assumption about the directory
+# layout Mesos provides. Ideally we never need to do this and we should
+# work with the Mesos core team to make it unnecessary.
+class ExecutorDetector(object):
+ class Error(Exception): pass
+ class CannotFindRoot(Error): pass
+
+ LOG_PATH = 'executor_logs'
+ RESOURCE_PATH = 'resource_usage.recordio'
+ VARS_PATH = 'executor_vars.json'
+ PATTERN = [
+ '%(root)s',
+ 'slaves',
+ '%(slave_id)s',
+ 'frameworks',
+ '%(framework_id)s',
+ 'executors',
+ '%(executor_id)s',
+ 'runs',
+ '%(run)s']
+ EXTRACTOR = ScanfParser(os.path.join(*PATTERN))
+
+ @classmethod
+ def find_root(cls, path):
+ """Does this path appear to match the executor directory pattern?"""
+
+ def root_from_path(path):
+ path = os.path.normpath(path)
+ path_vector = path.split(os.path.sep)
+ pattern_vector = cls.PATTERN
+ if len(path_vector) < len(pattern_vector):
+ return None
+ for pattern, path_component in zip(reversed(pattern_vector), reversed(path_vector)):
+ if pattern.startswith('%'):
+ continue
+ if path_component != pattern:
+ return None
+ matched_path = os.path.join(*path_vector[-len(pattern_vector) + 1:])
+ return os.path.normpath(path[:-len(matched_path)])
+
+ while path != os.path.dirname(path):
+ root = root_from_path(path)
+ if root:
+ return root
+ path = os.path.dirname(path)
+
+ @classmethod
+ def match(cls, path):
+ try:
+ return cls.EXTRACTOR.parse(path)
+ except ScanfParser.ParseError:
+ return None
+
+ @classmethod
+ def path(cls, result):
+ return os.path.join(*cls.PATTERN) % result.groups()
+
+ @classmethod
+ def find(cls, root, slave_id='*', framework_id='*', executor_id='*', run='*'):
+ mixins = dict(
+ root=root, slave_id=slave_id, framework_id=framework_id, executor_id=executor_id, run=run)
+ return filter(None, map(cls.match, glob(os.path.join(*cls.PATTERN) % mixins)))
+
+ def __init__(self, root=None):
+ self.root = root or self.find_root(os.getcwd())
+ if self.root is None:
+ raise self.CannotFindRoot('Not a valid executor root!')
+
+ def __iter__(self):
+ for extraction in self.find(root=self.root):
+ yield extraction
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/executor_vars.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/executor_vars.py b/src/main/python/apache/aurora/executor/executor_vars.py
new file mode 100644
index 0000000..7f54d8f
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/executor_vars.py
@@ -0,0 +1,123 @@
+import os
+import time
+
+from twitter.common.exceptions import ExceptionalThread
+from twitter.common.metrics import (
+ LambdaGauge,
+ MutatorGauge,
+ NamedGauge,
+ Observable)
+from twitter.common.python.dirwrapper import PythonDirectoryWrapper
+from twitter.common.python.pex import PexInfo
+from twitter.common.quantity import Amount, Time
+from twitter.common.string.scanf import ScanfParser
+
+import psutil
+
+
+class ExecutorVars(Observable, ExceptionalThread):
+ """
+ Executor exported /vars wrapper.
+
+ Currently writes to disk and communicates through the Aurora Observer,
+ pending MESOS-433.
+ """
+ MUTATOR_METRICS = ('rss', 'cpu', 'thermos_pss', 'thermos_cpu')
+ RELEASE_TAG_FORMAT = ScanfParser('%(project)s_R%(release)d')
+ DEPLOY_TAG_FORMAT = ScanfParser('%(project)s_%(environment)s_%(release)d_R%(deploy)d')
+ PROJECT_NAMES = ('thermos', 'thermos_executor')
+ COLLECTION_INTERVAL = Amount(1, Time.MINUTES)
+
+ @classmethod
+ def get_release_from_tag(cls, tag):
+ def parse_from(parser):
+ try:
+ scanf = parser.parse(tag)
+ if scanf and scanf.project in cls.PROJECT_NAMES:
+ return scanf.release
+ except ScanfParser.ParseError:
+ pass
+ release = parse_from(cls.RELEASE_TAG_FORMAT)
+ if release is None:
+ release = parse_from(cls.DEPLOY_TAG_FORMAT)
+ if release is None:
+ release = 'UNKNOWN'
+ return release
+
+ @classmethod
+ def get_release_from_binary(cls, binary):
+ try:
+ pex_info = PexInfo.from_pex(PythonDirectoryWrapper.get(binary))
+ return cls.get_release_from_tag(pex_info.build_properties.get('tag', ''))
+ except PythonDirectoryWrapper.Error:
+ return 'UNKNOWN'
+
+ def __init__(self, clock=time):
+ self._clock = clock
+ self._self = psutil.Process(os.getpid())
+ if hasattr(self._self, 'getcwd'):
+ self._version = self.get_release_from_binary(
+ os.path.join(self._self.getcwd(), self._self.cmdline[1]))
+ else:
+ self._version = 'UNKNOWN'
+ self.metrics.register(NamedGauge('version', self._version))
+ self._orphan = False
+ self.metrics.register(LambdaGauge('orphan', lambda: int(self._orphan)))
+ self._metrics = dict((metric, MutatorGauge(metric, 0)) for metric in self.MUTATOR_METRICS)
+ for metric in self._metrics.values():
+ self.metrics.register(metric)
+ ExceptionalThread.__init__(self)
+ self.daemon = True
+
+ def write_metric(self, metric, value):
+ self._metrics[metric].write(value)
+
+ @classmethod
+ def thermos_children(cls, parent):
+ try:
+ for child in parent.get_children():
+ yield child # thermos_runner
+ try:
+ for grandchild in child.get_children():
+ yield grandchild # thermos_coordinator
+ except psutil.Error:
+ continue
+ except psutil.Error:
+ return
+
+ @classmethod
+ def aggregate_memory(cls, process, attribute='pss'):
+ try:
+ return sum(getattr(mmap, attribute) for mmap in process.get_memory_maps())
+ except (psutil.Error, AttributeError):
+ # psutil on OS X does not support get_memory_maps
+ return 0
+
+ @classmethod
+ def cpu_rss_pss(cls, process):
+ return (process.get_cpu_percent(0),
+ process.get_memory_info().rss,
+ cls.aggregate_memory(process, attribute='pss'))
+
+ def run(self):
+ while True:
+ self._clock.sleep(self.COLLECTION_INTERVAL.as_(Time.SECONDS))
+ self.sample()
+
+ def sample(self):
+ try:
+ executor_cpu, executor_rss, _ = self.cpu_rss_pss(self._self)
+ self.write_metric('cpu', executor_cpu)
+ self.write_metric('rss', executor_rss)
+ self._orphan = self._self.ppid == 1
+ except psutil.Error:
+ return False
+
+ try:
+ child_stats = map(self.cpu_rss_pss, self.thermos_children(self._self))
+ self.write_metric('thermos_cpu', sum(stat[0] for stat in child_stats))
+ self.write_metric('thermos_pss', sum(stat[2] for stat in child_stats))
+ except psutil.Error:
+ pass
+
+ return True
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/gc_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/gc_executor.py b/src/main/python/apache/aurora/executor/gc_executor.py
new file mode 100644
index 0000000..f17d0a2
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/gc_executor.py
@@ -0,0 +1,499 @@
+"""Thermos garbage-collection (GC) executor
+
+This module containts the Thermos GC executor, responsible for garbage collecting old tasks and
+reconciling task states with the Mesos scheduler. It is intended to be run periodically on Mesos
+slaves utilising the Thermos executor.
+
+"""
+
+import os
+import threading
+import time
+
+from twitter.common.collections import OrderedDict
+from twitter.common.exceptions import ExceptionalThread
+from twitter.common.metrics import Observable
+from twitter.common.metrics.gauge import AtomicGauge
+from twitter.common.quantity import Amount, Time
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+from twitter.thermos.common.path import TaskPath
+from twitter.thermos.core.inspector import CheckpointInspector
+from twitter.thermos.core.helper import TaskKiller
+from twitter.thermos.monitoring.detector import TaskDetector
+from twitter.thermos.monitoring.garbage import TaskGarbageCollector
+
+from gen.twitter.aurora.comm.ttypes import (
+ AdjustRetainedTasks,
+ DeletedTasks,
+ SchedulerMessage)
+from gen.twitter.aurora.ttypes import ScheduleStatus
+
+from .common.sandbox import DirectorySandbox, SandboxInterface
+from .executor_base import ThermosExecutorBase
+from .executor_detector import ExecutorDetector
+
+import mesos_pb2 as mesos_pb
+import psutil
+from thrift.TSerialization import deserialize as thrift_deserialize
+from thrift.TSerialization import serialize as thrift_serialize
+
+
+class ThermosGCExecutor(ThermosExecutorBase, ExceptionalThread, Observable):
+ """
+ Thermos GC Executor, responsible for:
+ - garbage collecting old tasks to make sure they don't clutter up the system
+ - state reconciliation with the scheduler (in case it thinks we're running
+ something we're not or vice versa.)
+ """
+ MAX_PID_TIME_DRIFT = Amount(10, Time.SECONDS)
+ MAX_CHECKPOINT_TIME_DRIFT = Amount(1, Time.HOURS) # maximum runner disconnection time
+
+ # how old a task must be before we're willing to kill it, assuming that there could be
+ # slight races in the following scenario:
+ # launch gc with retained_tasks={t1, t2, t3}
+ # launch task t4
+ MINIMUM_KILL_AGE = Amount(10, Time.MINUTES)
+
+ # wait time between checking for new GC events from the slave and/or cleaning orphaned tasks
+ POLL_WAIT = Amount(5, Time.MINUTES)
+
+ # maximum amount of time the executor will wait with no tasks before it exits.
+ MAXIMUM_EXECUTOR_WAIT = Amount(15, Time.MINUTES)
+
+ # maximum lifetime of this executor. this is to prevent older GC executor binaries from
+ # running forever
+ MAXIMUM_EXECUTOR_LIFETIME = Amount(1, Time.DAYS)
+
+ PERSISTENCE_WAIT = Amount(5, Time.SECONDS)
+
+ def __init__(self,
+ checkpoint_root,
+ verbose=True,
+ task_killer=TaskKiller,
+ executor_detector=ExecutorDetector,
+ task_garbage_collector=TaskGarbageCollector,
+ clock=time):
+ ThermosExecutorBase.__init__(self)
+ ExceptionalThread.__init__(self)
+ self.daemon = True
+ self._stop_event = threading.Event()
+ self._gc_task_queue = OrderedDict() # mapping of task_id => (TaskInfo, AdjustRetainedTasks), in
+ # the order in which they were received via a launchTask.
+ self._driver = None # cache the ExecutorDriver provided by the slave, so we can use it
+ # out of band from slave-initiated callbacks. This should be supplied by
+ # ThermosExecutorBase.registered() when the executor first registers with
+ # the slave.
+ self._slave_id = None # cache the slave ID provided by the slave
+ self._task_id = None # the task_id currently being executed by the ThermosGCExecutor, if any
+ self._start_time = None # the start time of a task currently being executed, if any
+ self._detector = executor_detector()
+ self._collector = task_garbage_collector(root=checkpoint_root)
+ self._clock = clock
+ self._task_killer = task_killer
+ self._checkpoint_root = checkpoint_root
+ self._dropped_tasks = AtomicGauge('dropped_tasks')
+ self.metrics.register(self._dropped_tasks)
+
+ def _runner_ckpt(self, task_id):
+ """Return the runner checkpoint file for a given task_id."""
+ return TaskPath(root=self._checkpoint_root, task_id=task_id).getpath('runner_checkpoint')
+
+ def _terminate_task(self, task_id, kill=True):
+ """Terminate a task using the associated task killer. Returns a boolean indicating success."""
+ killer = self._task_killer(task_id, self._checkpoint_root)
+ self.log('Terminating %s...' % task_id)
+ runner_terminate = killer.kill if kill else killer.lose
+ try:
+ runner_terminate(force=True)
+ return True
+ except Exception as e:
+ self.log('Could not terminate: %s' % e)
+ return False
+
+ def partition_tasks(self):
+ """Return active/finished tasks as discovered from the checkpoint root."""
+ detector = TaskDetector(root=self._checkpoint_root)
+ active_tasks = set(t_id for _, t_id in detector.get_task_ids(state='active'))
+ finished_tasks = set(t_id for _, t_id in detector.get_task_ids(state='finished'))
+ return active_tasks, finished_tasks
+
+ def get_states(self, task_id):
+ """Returns the (timestamp, status) tuples of the task or [] if could not replay."""
+ statuses = CheckpointDispatcher.iter_statuses(self._runner_ckpt(task_id))
+ try:
+ return [(state.timestamp_ms / 1000.0, state.state) for state in statuses]
+ except CheckpointDispatcher.ErrorRecoveringState:
+ return []
+
+ def get_sandbox(self, task_id):
+ """Returns the sandbox of the task, or None if it has not yet been initialized."""
+ try:
+ for update in CheckpointDispatcher.iter_updates(self._runner_ckpt(task_id)):
+ if update.runner_header and update.runner_header.sandbox:
+ return update.runner_header.sandbox
+ except CheckpointDispatcher.ErrorRecoveringState:
+ return None
+
+ def maybe_terminate_unknown_task(self, task_id):
+ """Terminate a task if we believe the scheduler doesn't know about it.
+
+ It's possible for the scheduler to queue a GC and launch a task afterwards, in which
+ case we may see actively running tasks that the scheduler did not report in the
+ AdjustRetainedTasks message.
+
+ Returns:
+ boolean indicating whether the task was terminated
+ """
+ states = self.get_states(task_id)
+ if states:
+ task_start_time, _ = states[0]
+ if self._start_time - task_start_time > self.MINIMUM_KILL_AGE.as_(Time.SECONDS):
+ return self._terminate_task(task_id)
+ return False
+
+ def should_gc_task(self, task_id):
+ """Check if a possibly-corrupt task should be locally GCed
+
+ A task should be GCed if its checkpoint stream appears to be corrupted and the kill age
+ threshold is exceeded.
+
+ Returns:
+ set, containing the task_id if it should be marked for local GC, or empty otherwise
+ """
+ runner_ckpt = self._runner_ckpt(task_id)
+ if not os.path.exists(runner_ckpt):
+ return set()
+ latest_update = os.path.getmtime(runner_ckpt)
+ if self._start_time - latest_update > self.MINIMUM_KILL_AGE.as_(Time.SECONDS):
+ self.log('Got corrupt checkpoint file for %s - marking for local GC' % task_id)
+ return set([task_id])
+ else:
+ self.log('Checkpoint file unreadable, but not yet beyond MINIMUM_KILL_AGE threshold')
+ return set()
+
+ def reconcile_states(self, driver, retained_tasks):
+ """Reconcile states that the scheduler thinks tasks are in vs what they really are in.
+
+ Local vs Scheduler => Action
+ ===================================
+ ACTIVE ACTIVE => no-op
+ ACTIVE STARTING => no-op
+ ACTIVE TERMINAL => maybe kill task*
+ ACTIVE !EXISTS => maybe kill task*
+ TERMINAL ACTIVE => send actual status**
+ TERMINAL STARTING => send actual status**
+ TERMINAL TERMINAL => no-op
+ TERMINAL !EXISTS => gc locally
+ !EXISTS ACTIVE => send LOST**
+ !EXISTS STARTING => no-op
+ !EXISTS TERMINAL => gc remotely
+
+ * - Only kill if this does not appear to be a race condition.
+ ** - These appear to have no effect
+
+ Side effecting operations:
+ ACTIVE | (TERMINAL / !EXISTS) => maybe kill
+ TERMINAL | !EXISTS => delete
+ !EXISTS | TERMINAL => delete
+
+ Returns tuple of (local_gc, remote_gc, updates), where:
+ local_gc - set of task_ids to be GCed locally
+ remote_gc - set of task_ids to be deleted on the scheduler
+ updates - dictionary of updates sent to the scheduler (task_id: ScheduleStatus)
+ """
+ def partition(rt):
+ active, starting, finished = set(), set(), set()
+ for task_id, schedule_status in rt.items():
+ if self.twitter_status_is_terminal(schedule_status):
+ finished.add(task_id)
+ elif (schedule_status == ScheduleStatus.STARTING or
+ schedule_status == ScheduleStatus.ASSIGNED):
+ starting.add(task_id)
+ else:
+ active.add(task_id)
+ return active, starting, finished
+
+ local_active, local_finished = self.partition_tasks()
+ sched_active, sched_starting, sched_finished = partition(retained_tasks)
+ local_task_ids = local_active | local_finished
+ sched_task_ids = sched_active | sched_starting | sched_finished
+ all_task_ids = local_task_ids | sched_task_ids
+
+ self.log('Told to retain the following task ids:')
+ for task_id, schedule_status in retained_tasks.items():
+ self.log(' => %s as %s' % (task_id, ScheduleStatus._VALUES_TO_NAMES[schedule_status]))
+
+ self.log('Local active tasks:')
+ for task_id in local_active:
+ self.log(' => %s' % task_id)
+
+ self.log('Local finished tasks:')
+ for task_id in local_finished:
+ self.log(' => %s' % task_id)
+
+ local_gc, remote_gc = set(), set()
+ updates = {}
+
+ for task_id in all_task_ids:
+ if task_id in local_active and task_id not in (sched_active | sched_starting):
+ self.log('Inspecting task %s for termination.' % task_id)
+ if not self.maybe_terminate_unknown_task(task_id):
+ local_gc.update(self.should_gc_task(task_id))
+ if task_id in local_finished and task_id not in sched_task_ids:
+ self.log('Queueing task %s for local deletion.' % task_id)
+ local_gc.add(task_id)
+ if task_id in local_finished and task_id in (sched_active | sched_starting):
+ self.log('Task %s finished but scheduler thinks active/starting.' % task_id)
+ states = self.get_states(task_id)
+ if states:
+ _, last_state = states[-1]
+ updates[task_id] = self.THERMOS_TO_TWITTER_STATES.get(last_state, ScheduleStatus.UNKNOWN)
+ self.send_update(
+ driver,
+ task_id,
+ self.THERMOS_TO_MESOS_STATES.get(last_state, mesos_pb.TASK_LOST),
+ 'Task finish detected by GC executor.')
+ else:
+ local_gc.update(self.should_gc_task(task_id))
+ if task_id in sched_finished and task_id not in local_task_ids:
+ self.log('Queueing task %s for remote deletion.' % task_id)
+ remote_gc.add(task_id)
+ if task_id not in local_task_ids and task_id in sched_active:
+ self.log('Know nothing about task %s, telling scheduler of LOSS.' % task_id)
+ updates[task_id] = ScheduleStatus.LOST
+ self.send_update(driver, task_id, mesos_pb.TASK_LOST, 'GC executor found no trace of task.')
+ if task_id not in local_task_ids and task_id in sched_starting:
+ self.log('Know nothing about task %s, but scheduler says STARTING - passing' % task_id)
+
+ return local_gc, remote_gc, updates
+
+ def clean_orphans(self, driver):
+ """Inspect checkpoints for trees that have been kill -9'ed but not properly cleaned up."""
+ self.log('Checking for orphaned tasks')
+ active_tasks, _ = self.partition_tasks()
+ updates = {}
+
+ inspector = CheckpointInspector(self._checkpoint_root)
+
+ def is_our_process(process, uid, timestamp):
+ if process.uids.real != uid:
+ return False
+ estimated_start_time = self._clock.time() - process.create_time
+ return abs(timestamp - estimated_start_time) < self.MAX_PID_TIME_DRIFT.as_(Time.SECONDS)
+
+ for task_id in active_tasks:
+ self.log('Inspecting running task: %s' % task_id)
+ inspection = inspector.inspect(task_id)
+ if not inspection:
+ self.log(' - Error inspecting task runner')
+ continue
+ latest_runner = inspection.runner_processes[-1]
+ # Assume that it has not yet started?
+ if not latest_runner:
+ self.log(' - Task has no registered runners.')
+ continue
+ runner_pid, runner_uid, timestamp_ms = latest_runner
+ try:
+ runner_process = psutil.Process(runner_pid)
+ if is_our_process(runner_process, runner_uid, timestamp_ms / 1000.0):
+ self.log(' - Runner appears healthy.')
+ continue
+ except psutil.NoSuchProcess:
+ # Runner is dead
+ pass
+ except psutil.Error as err:
+ self.log(' - Error sampling runner process [pid=%s]: %s' % (runner_pid, err))
+ continue
+ try:
+ latest_update = os.path.getmtime(self._runner_ckpt(task_id))
+ except (IOError, OSError) as err:
+ self.log(' - Error accessing runner ckpt: %s' % err)
+ continue
+ if self._clock.time() - latest_update < self.MAX_CHECKPOINT_TIME_DRIFT.as_(Time.SECONDS):
+ self.log(' - Runner is dead but under LOST threshold.')
+ continue
+ self.log(' - Runner is dead but beyond LOST threshold: %.1fs' % (
+ self._clock.time() - latest_update))
+ if self._terminate_task(task_id, kill=False):
+ updates[task_id] = ScheduleStatus.LOST
+ self.send_update(
+ driver, task_id, mesos_pb.TASK_LOST, 'GC executor detected failed task runner.')
+
+ return updates
+
+ def _erase_sandbox(self, task_id):
+ # TODO(wickman) Only mesos should be in the business of garbage collecting sandboxes.
+ header_sandbox = self.get_sandbox(task_id)
+ directory_sandbox = DirectorySandbox(header_sandbox) if header_sandbox else None
+ if directory_sandbox and directory_sandbox.exists():
+ self.log('Destroying DirectorySandbox for %s' % task_id)
+ directory_sandbox.destroy()
+ else:
+ self.log('Found no sandboxes for %s' % task_id)
+
+ def _gc(self, task_id):
+ """Erase the sandbox, logs and metadata of the given task."""
+ self.log('Erasing sandbox for %s' % task_id)
+ self._erase_sandbox(task_id)
+ self.log('Erasing logs for %s' % task_id)
+ self._collector.erase_logs(task_id)
+ self.log('Erasing metadata for %s' % task_id)
+ self._collector.erase_metadata(task_id)
+
+ def garbage_collect(self, force_delete=frozenset()):
+ """Garbage collect tasks on the system no longer active or in the supplied force_delete.
+
+ Return a set of task_ids representing the tasks that were garbage collected.
+ """
+ active_tasks, finished_tasks = self.partition_tasks()
+ retained_executors = set(iter(self.linked_executors))
+ self.log('Executor sandboxes retained by Mesos:')
+ if retained_executors:
+ for r_e in sorted(retained_executors):
+ self.log(' %s' % r_e)
+ else:
+ self.log(' None')
+ for task_id in (active_tasks - retained_executors):
+ self.log('ERROR: Active task %s had its executor sandbox pulled.' % task_id)
+ gc_tasks = (finished_tasks - retained_executors) | force_delete
+ for gc_task in gc_tasks:
+ self._gc(gc_task)
+ return gc_tasks
+
+ @property
+ def linked_executors(self):
+ """Generator yielding the executor sandboxes detected on the system."""
+ thermos_executor_prefix = 'thermos-'
+ for executor in self._detector:
+ # It's possible for just the 'latest' symlink to be present but no run directories.
+ # This indicates that the task has been fully garbage collected.
+ if executor.executor_id.startswith(thermos_executor_prefix) and executor.run != 'latest':
+ yield executor.executor_id[len(thermos_executor_prefix):]
+
+ def _run_gc(self, task, retain_tasks, retain_start):
+ """
+ Reconcile the set of tasks to retain (provided by the scheduler) with the current state of
+ executors on this system. Garbage collect tasks/executors as appropriate.
+
+ Not re-entrant! Previous executions must complete (and clear self._task_id) before this can be
+ invoked.
+
+ Potentially blocking (e.g. on I/O) in self.garbage_collect()
+
+ Args:
+ task: TaskInfo provided by the slave
+ retain_tasks: mapping of task_id => ScheduleStatus, describing what the scheduler thinks is
+ running on this system
+ retain_start: the time at which the retain_tasks message is effective -- this means that
+ tasks started after the retain_tasks message is effective are skipped
+ until future GC runs.
+ """
+ task_id = task.task_id.value
+ if self._task_id is not None:
+ raise RuntimeError('_run_gc() called [task_id=%s], but already running [task_id=%s]'
+ % (task_id, self._task_id))
+ self._task_id = task_id
+ self.log('Launching garbage collection [task_id=%s]' % task_id)
+ self._start_time = retain_start
+ local_gc, remote_gc, _ = self.reconcile_states(self._driver, retain_tasks)
+ deleted_tasks = set(retain_tasks).intersection(self.garbage_collect(local_gc)) | remote_gc
+ if deleted_tasks:
+ self._driver.sendFrameworkMessage(thrift_serialize(
+ SchedulerMessage(deletedTasks=DeletedTasks(taskIds=deleted_tasks))))
+ self.send_update(
+ self._driver, task.task_id.value, mesos_pb.TASK_FINISHED, 'Garbage collection finished.')
+ self.log('Garbage collection complete [task_id=%s]' % task_id)
+ self._task_id = self._start_time = None
+
+ def run(self):
+ """Main GC executor event loop.
+
+ Periodically perform state reconciliation with the set of tasks provided
+ by the slave, and garbage collect orphaned tasks on the system.
+ """
+ run_start = self._clock.time()
+ last_gc_run = self._clock.time()
+
+ def should_terminate():
+ now = self._clock.time()
+ if now > run_start + self.MAXIMUM_EXECUTOR_LIFETIME.as_(Time.SECONDS):
+ return True
+ if now > last_gc_run + self.MAXIMUM_EXECUTOR_WAIT.as_(Time.SECONDS):
+ return True
+ return self._stop_event.is_set()
+
+ while not should_terminate():
+ try:
+ _, (task, retain_tasks, retain_start) = self._gc_task_queue.popitem(0)
+ last_gc_run = retain_start
+ self._run_gc(task, retain_tasks, retain_start)
+ except KeyError: # no enqueued GC tasks
+ pass
+ if self._driver is not None:
+ self.clean_orphans(self._driver)
+ self._stop_event.wait(self.POLL_WAIT.as_(Time.SECONDS))
+
+ # shutdown
+ if self._driver is not None:
+ try:
+ prev_task_id, _ = self._gc_task_queue.popitem(0)
+ except KeyError: # no enqueued GC tasks
+ pass
+ else:
+ self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
+ 'Garbage collection skipped - GC executor shutting down')
+ # TODO(jon) Remove this once external MESOS-243 is resolved.
+ self.log('Sleeping briefly to mitigate https://issues.apache.org/jira/browse/MESOS-243')
+ self._clock.sleep(self.PERSISTENCE_WAIT.as_(Time.SECONDS))
+
+ self._driver.stop()
+
+ """ Mesos Executor API methods follow """
+
+ def launchTask(self, driver, task):
+ """Queue a new garbage collection run, and drop any currently-enqueued runs."""
+ if self._slave_id is None:
+ self._slave_id = task.slave_id.value
+ task_id = task.task_id.value
+ self.log('launchTask() got task_id: %s' % task_id)
+ if task_id == self._task_id:
+ self.log('=> GC with task_id %s currently running - ignoring' % task_id)
+ return
+ elif task_id in self._gc_task_queue:
+ self.log('=> Already have task_id %s queued - ignoring' % task_id)
+ return
+ try:
+ art = thrift_deserialize(AdjustRetainedTasks(), task.data)
+ except Exception as err:
+ self.log('Error deserializing task: %s' % err)
+ self.send_update(
+ self._driver, task_id, mesos_pb.TASK_FAILED, 'Deserialization of GC task failed')
+ return
+ try:
+ prev_task_id, _ = self._gc_task_queue.popitem(0)
+ except KeyError: # no enqueued GC tasks - reset counter
+ self._dropped_tasks.write(0)
+ else:
+ self.log('=> Dropping previously queued GC with task_id %s' % prev_task_id)
+ self._dropped_tasks.increment()
+ self.log('=> Updating scheduler')
+ self.send_update(self._driver, prev_task_id, mesos_pb.TASK_FINISHED,
+ 'Garbage collection skipped - GC executor received another task')
+ self.log('=> Adding %s to GC queue' % task_id)
+ self._gc_task_queue[task_id] = (task, art.retainedTasks, self._clock.time())
+
+ def killTask(self, driver, task_id):
+ """Remove the specified task from the queue, if it's not yet run. Otherwise, no-op."""
+ self.log('killTask() got task_id: %s' % task_id)
+ task = self._gc_task_queue.pop(task_id, None)
+ if task is not None:
+ self.log('=> Removed %s from queued GC tasks' % task_id)
+ elif task_id == self._task_id:
+ self.log('=> GC with task_id %s currently running - ignoring' % task_id)
+ else:
+ self.log('=> Unknown task_id %s - ignoring' % task_id)
+
+ def shutdown(self, driver):
+ """Trigger the Executor to shut down as soon as the current GC run is finished."""
+ self.log('shutdown() called - setting stop event')
+ self._stop_event.set()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/status_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/status_manager.py b/src/main/python/apache/aurora/executor/status_manager.py
new file mode 100644
index 0000000..c974597
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/status_manager.py
@@ -0,0 +1,39 @@
+import time
+
+from twitter.common import log
+from twitter.common.exceptions import ExceptionalThread
+from twitter.common.quantity import Amount, Time
+
+from .common.status_checker import StatusChecker
+
+
+class StatusManager(ExceptionalThread):
+ """
+ An agent that periodically checks the health of a task via StatusCheckers that
+ provide HTTP health checking, resource consumption, etc.
+
+ If any of the status interfaces return a status, the Status Manager
+ invokes the user-supplied callback with the status.
+ """
+ POLL_WAIT = Amount(500, Time.MILLISECONDS)
+
+ def __init__(self, status_checker, callback, clock=time):
+ if not isinstance(status_checker, StatusChecker):
+ raise TypeError('status_checker must be a StatusChecker, got %s' % type(status_checker))
+ if not callable(callback):
+ raise TypeError('callback needs to be callable!')
+ self._status_checker = status_checker
+ self._callback = callback
+ self._clock = clock
+ super(StatusManager, self).__init__()
+ self.daemon = True
+
+ def run(self):
+ while True:
+ status_result = self._status_checker.status
+ if status_result is not None:
+ log.info('Status manager got %s' % status_result)
+ self._callback(status_result)
+ break
+ else:
+ self._clock.sleep(self.POLL_WAIT.as_(Time.SECONDS))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/thermos_executor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/thermos_executor.py b/src/main/python/apache/aurora/executor/thermos_executor.py
new file mode 100644
index 0000000..52b514a
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/thermos_executor.py
@@ -0,0 +1,289 @@
+import os
+import threading
+import time
+import traceback
+
+from twitter.common import log
+from twitter.common.concurrent import deadline, defer, Timeout
+from twitter.common.metrics import Observable
+from twitter.common.quantity import Amount, Time
+
+from .common.kill_manager import KillManager
+from .common.sandbox import DirectorySandbox, SandboxProvider
+from .common.status_checker import ChainedStatusChecker, ExitState
+from .common.task_info import (
+ assigned_task_from_mesos_task,
+ mesos_task_instance_from_assigned_task,
+ resolve_ports,
+)
+from .common.task_runner import (
+ TaskError,
+ TaskRunner,
+ TaskRunnerProvider,
+)
+from .executor_base import ThermosExecutorBase
+from .status_manager import StatusManager
+
+import mesos_pb2 as mesos_pb
+
+
+class DefaultSandboxProvider(SandboxProvider):
+ SANDBOX_NAME = 'sandbox'
+
+ def from_assigned_task(self, assigned_task):
+ mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
+ return DirectorySandbox(
+ os.path.realpath(self.SANDBOX_NAME),
+ mesos_task.role().get())
+
+
+class ThermosExecutor(Observable, ThermosExecutorBase):
+ PERSISTENCE_WAIT = Amount(5, Time.SECONDS)
+ SANDBOX_INITIALIZATION_TIMEOUT = Amount(10, Time.MINUTES)
+ START_TIMEOUT = Amount(2, Time.MINUTES)
+ STOP_TIMEOUT = Amount(2, Time.MINUTES)
+ STOP_WAIT = Amount(5, Time.SECONDS)
+
+ def __init__(self,
+ runner_provider,
+ status_manager_class=StatusManager,
+ sandbox_provider=DefaultSandboxProvider,
+ status_providers=(),
+ clock=time):
+
+ ThermosExecutorBase.__init__(self)
+ if not isinstance(runner_provider, TaskRunnerProvider):
+ raise TypeError('runner_provider must be a TaskRunnerProvider, got %s' %
+ type(runner_provider))
+ self._runner = None
+ self._runner_provider = runner_provider
+ self._clock = clock
+ self._task_id = None
+ self._status_providers = status_providers
+ self._status_manager = None
+ self._status_manager_class = status_manager_class
+ self._sandbox = None
+ self._sandbox_provider = sandbox_provider()
+ self._kill_manager = KillManager()
+ # Events that are exposed for interested entities
+ self.runner_aborted = threading.Event()
+ self.runner_started = threading.Event()
+ self.sandbox_initialized = threading.Event()
+ self.sandbox_created = threading.Event()
+ self.terminated = threading.Event()
+ self.launched = threading.Event()
+
+ @property
+ def runner(self):
+ return self._runner
+
+ def _die(self, driver, status, msg):
+ log.fatal(msg)
+ self.send_update(driver, self._task_id, status, msg)
+ defer(driver.stop, delay=self.STOP_WAIT)
+
+ def _run(self, driver, assigned_task, mesos_task):
+ """
+ Commence running a Task.
+ - Initialize the sandbox
+ - Start the ThermosTaskRunner (fork the Thermos TaskRunner)
+ - Set up necessary HealthCheckers
+ - Set up DiscoveryManager, if applicable
+ - Set up ResourceCheckpointer
+ - Set up StatusManager, and attach HealthCheckers
+ """
+ self.send_update(driver, self._task_id, mesos_pb.TASK_STARTING, 'Initializing sandbox.')
+
+ if not self._initialize_sandbox(driver, assigned_task):
+ return
+
+ # Fully resolve the portmap
+ portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
+
+ # start the process on a separate thread and give the message processing thread back
+ # to the driver
+ try:
+ self._runner = self._runner_provider.from_assigned_task(assigned_task, self._sandbox)
+ except TaskError as e:
+ self._die(driver, mesos_pb.TASK_FAILED, str(e))
+ return
+
+ if not isinstance(self._runner, TaskRunner):
+ self._die(driver, mesos_pb.TASK_FAILED, 'Unrecognized task!')
+ return
+
+ if not self._start_runner(driver, assigned_task, mesos_task, portmap):
+ return
+
+ self.send_update(driver, self._task_id, mesos_pb.TASK_RUNNING)
+
+ self._start_status_manager(driver, assigned_task, mesos_task, portmap)
+
+ def _initialize_sandbox(self, driver, assigned_task):
+ self._sandbox = self._sandbox_provider.from_assigned_task(assigned_task)
+ self.sandbox_initialized.set()
+ try:
+ deadline(self._sandbox.create, timeout=self.SANDBOX_INITIALIZATION_TIMEOUT,
+ daemon=True, propagate=True)
+ except Timeout:
+ self._die(driver, mesos_pb.TASK_FAILED, 'Timed out waiting for sandbox to initialize!')
+ return
+ except self._sandbox.Error as e:
+ self._die(driver, mesos_pb.TASK_FAILED, 'Failed to initialize sandbox: %s' % e)
+ return
+ self.sandbox_created.set()
+ return True
+
+ def _start_runner(self, driver, assigned_task, mesos_task, portmap):
+ if self.runner_aborted.is_set():
+ self._die(driver, mesos_pb.TASK_KILLED, 'Task killed during initialization.')
+
+ try:
+ deadline(self._runner.start, timeout=self.START_TIMEOUT, propagate=True)
+ except TaskError as e:
+ self._die(driver, mesos_pb.TASK_FAILED, 'Task initialization failed: %s' % e)
+ return False
+ except Timeout:
+ self._die(driver, mesos_pb.TASK_LOST, 'Timed out waiting for task to start!')
+ return False
+
+ self.runner_started.set()
+ log.debug('Task started.')
+
+ return True
+
+ def _start_status_manager(self, driver, assigned_task, mesos_task, portmap):
+ status_checkers = [self._kill_manager]
+ self.metrics.register_observable('kill_manager', self._kill_manager)
+
+ for status_provider in self._status_providers:
+ status_checker = status_provider.from_assigned_task(assigned_task, self._sandbox)
+ if status_checker is None:
+ continue
+ status_checkers.append(status_checker)
+ # self.metrics.register_observable()
+
+ self._chained_checker = ChainedStatusChecker(status_checkers)
+ self._chained_checker.start()
+
+ # chain the runner to the other checkers, but do not chain .start()/.stop()
+ complete_checker = ChainedStatusChecker([self._runner, self._chained_checker])
+ self._status_manager = self._status_manager_class(
+ complete_checker, self._shutdown, clock=self._clock)
+ self._status_manager.start()
+
+ def _signal_kill_manager(self, driver, task_id, reason):
+ if self._task_id is None:
+ log.error('Was asked to kill task but no task running!')
+ return
+ if task_id != self._task_id:
+ log.error('Asked to kill a task other than what we are running!')
+ return
+ if not self.sandbox_created.is_set():
+ log.error('Asked to kill task with incomplete sandbox - aborting runner start')
+ self.runner_aborted.set()
+ return
+ self.log('Activating kill manager.')
+ self._kill_manager.kill(reason)
+
+ @classmethod
+ def translate_exit_state_to_mesos(cls, exit_state):
+ # Put into executor_base?
+ if exit_state == ExitState.FAILED:
+ return mesos_pb.TASK_FAILED
+ elif exit_state == ExitState.KILLED:
+ return mesos_pb.TASK_KILLED
+ elif exit_state == ExitState.FINISHED:
+ return mesos_pb.TASK_FINISHED
+ elif exit_state == ExitState.LOST:
+ return mesos_pb.TASK_LOST
+ log.error('Unknown exit state, defaulting to TASK_FINISHED.')
+ return mesos_pb.TASK_FINISHED
+
+ def _shutdown(self, status_result):
+ runner_status = self._runner.status
+
+ try:
+ deadline(self._runner.stop, timeout=self.STOP_TIMEOUT)
+ except Timeout:
+ log.error('Failed to stop runner within deadline.')
+
+ try:
+ deadline(self._chained_checker.stop, timeout=self.STOP_TIMEOUT)
+ except Timeout:
+ log.error('Failed to stop all checkers within deadline.')
+
+ # If the runner was alive when _shutdown was called, defer to the status_result,
+ # otherwise the runner's terminal state is the preferred state.
+ exit_status = runner_status or status_result
+
+ self.send_update(
+ self._driver,
+ self._task_id,
+ self.translate_exit_state_to_mesos(exit_status.status),
+ status_result.reason)
+
+ self.terminated.set()
+ defer(self._driver.stop, delay=self.PERSISTENCE_WAIT)
+
+ """ Mesos Executor API methods follow """
+
+ def launchTask(self, driver, task):
+ """
+ Invoked when a task has been launched on this executor (initiated via Scheduler::launchTasks).
+ Note that this task can be realized with a thread, a process, or some simple computation,
+ however, no other callbacks will be invoked on this executor until this callback has returned.
+ """
+ self.launched.set()
+ self.log('launchTask got task: %s:%s' % (task.name, task.task_id.value))
+
+ # TODO(wickman) Update the tests to call registered(), then remove this line and issue
+ # an assert if self._driver is not populated.
+ self._driver = driver
+
+ if self._runner:
+ log.error('Already running a task! %s' % self._task_id)
+ self.send_update(driver, task.task_id.value, mesos_pb.TASK_LOST,
+ "Task already running on this executor: %s" % self._task_id)
+ return
+
+ self._slave_id = task.slave_id.value
+ self._task_id = task.task_id.value
+
+ try:
+ assigned_task = assigned_task_from_mesos_task(task)
+ mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
+ except Exception as e:
+ log.fatal('Could not deserialize AssignedTask')
+ log.fatal(traceback.format_exc())
+ self.send_update(
+ driver, self._task_id, mesos_pb.TASK_FAILED, "Could not deserialize task: %s" % e)
+ defer(driver.stop, delay=self.STOP_WAIT)
+ return
+
+ defer(lambda: self._run(driver, assigned_task, mesos_task))
+
+ def killTask(self, driver, task_id):
+ """
+ Invoked when a task running within this executor has been killed (via
+ SchedulerDriver::killTask). Note that no status update will be sent on behalf of the executor,
+ the executor is responsible for creating a new TaskStatus (i.e., with TASK_KILLED) and invoking
+ ExecutorDriver::sendStatusUpdate.
+ """
+ self.log('killTask got task_id: %s' % task_id)
+ self._signal_kill_manager(driver, task_id.value, "Instructed to kill task.")
+ self.log('killTask returned.')
+
+ def shutdown(self, driver):
+ """
+ Invoked when the executor should terminate all of its currently running tasks. Note that after
+ Mesos has determined that an executor has terminated any tasks that the executor did not send
+ terminal status updates for (e.g., TASK_KILLED, TASK_FINISHED, TASK_FAILED, etc) a TASK_LOST
+ status update will be created.
+
+ """
+ self.log('shutdown called')
+ if self._task_id:
+ self.log('shutting down %s' % self._task_id)
+ self._signal_kill_manager(driver, self._task_id, "Told to shut down executor.")
+ self.log('shutdown returned')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/thermos_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/thermos_runner.py b/src/main/python/apache/aurora/executor/thermos_runner.py
new file mode 100644
index 0000000..0c9af05
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/thermos_runner.py
@@ -0,0 +1,138 @@
+import functools
+import os
+import signal
+import sys
+import traceback
+
+from twitter.common import app, log
+from twitter.thermos.common.options import add_port_to
+from twitter.thermos.common.planner import TaskPlanner
+from twitter.thermos.config.loader import ThermosConfigLoader
+from twitter.thermos.core.runner import TaskRunner
+
+
+app.add_option(
+ "--thermos_json",
+ dest="thermos_json",
+ default=None,
+ help="read a thermos Task from a serialized json blob")
+
+
+app.add_option(
+ "--sandbox",
+ dest="sandbox",
+ metavar="PATH",
+ default=None,
+ help="the sandbox in which this task should run")
+
+
+app.add_option(
+ "--checkpoint_root",
+ dest="checkpoint_root",
+ metavar="PATH",
+ default=None,
+ help="the path where we will store checkpoints")
+
+
+app.add_option(
+ "--task_id",
+ dest="task_id",
+ metavar="STRING",
+ default=None,
+ help="The id to which this task should be bound, created if it does not exist.")
+
+
+app.add_option(
+ "--setuid",
+ dest="setuid",
+ metavar="USER",
+ default=None,
+ help="setuid tasks to this user, requires superuser privileges.")
+
+
+app.add_option(
+ "--enable_chroot",
+ dest="chroot",
+ default=False,
+ action='store_true',
+ help="chroot tasks to the sandbox before executing them.")
+
+
+app.add_option(
+ "--port",
+ type='string',
+ nargs=1,
+ action='callback',
+ callback=add_port_to('prebound_ports'),
+ dest='prebound_ports',
+ default={},
+ metavar = "NAME:PORT",
+ help="bind a numbered port PORT to name NAME")
+
+
+def get_task_from_options(opts):
+ tasks = ThermosConfigLoader.load_json(opts.thermos_json)
+ if len(tasks.tasks()) == 0:
+ app.error("No tasks specified!")
+ if len(tasks.tasks()) > 1:
+ app.error("Multiple tasks in config but no task name specified!")
+ task = tasks.tasks()[0]
+ if not task.task.check().ok():
+ app.error(task.task.check().message())
+ return task
+
+
+def runner_teardown(runner, sig=signal.SIGUSR1, frame=None):
+ """Destroy runner on SIGUSR1 (kill) or SIGUSR2 (lose)"""
+ op = 'kill' if sig == signal.SIGUSR1 else 'lose'
+ log.info('Thermos runner got signal %s, shutting down.' % sig)
+ log.info('Interrupted frame:')
+ if frame:
+ for line in ''.join(traceback.format_stack(frame)).splitlines():
+ log.info(line)
+ runner.close_ckpt()
+ log.info('Calling runner.%s()' % op)
+ getattr(runner, op)()
+ sys.exit(0)
+
+
+class CappedTaskPlanner(TaskPlanner):
+ TOTAL_RUN_LIMIT = 100
+
+
+def proxy_main(args, opts):
+ assert opts.thermos_json and os.path.exists(opts.thermos_json)
+ assert opts.sandbox
+ assert opts.checkpoint_root
+
+ thermos_task = get_task_from_options(opts)
+ prebound_ports = opts.prebound_ports
+ missing_ports = set(thermos_task.ports()) - set(prebound_ports)
+
+ if missing_ports:
+ app.error('ERROR! Unbound ports: %s' % ' '.join(port for port in missing_ports))
+
+ task_runner = TaskRunner(
+ thermos_task.task,
+ opts.checkpoint_root,
+ opts.sandbox,
+ task_id=opts.task_id,
+ user=opts.setuid,
+ portmap=prebound_ports,
+ chroot=opts.chroot,
+ planner_class=CappedTaskPlanner
+ )
+
+ for sig in (signal.SIGUSR1, signal.SIGUSR2):
+ signal.signal(sig, functools.partial(runner_teardown, task_runner))
+
+ try:
+ task_runner.run()
+ except TaskRunner.InternalError as err:
+ app.error('Internal error: %s' % err)
+ except TaskRunner.InvalidTask as err:
+ app.error(str(err))
+ except TaskRunner.StateError:
+ app.error('Task appears to already be in a terminal state.')
+ except KeyboardInterrupt:
+ runner_teardown(task_runner)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/executor/thermos_task_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/executor/thermos_task_runner.py b/src/main/python/apache/aurora/executor/thermos_task_runner.py
new file mode 100644
index 0000000..dba99d2
--- /dev/null
+++ b/src/main/python/apache/aurora/executor/thermos_task_runner.py
@@ -0,0 +1,331 @@
+import errno
+import getpass
+import os
+import signal
+import subprocess
+import threading
+import time
+
+from twitter.aurora.common.http_signaler import HttpSignaler
+from twitter.common import log
+from twitter.common.dirutil import chmod_plus_x, safe_mkdtemp
+from twitter.common.log.options import LogOptions
+from twitter.common.quantity import Amount, Time
+from twitter.thermos.common.path import TaskPath
+from twitter.thermos.config.loader import ThermosTaskWrapper
+from twitter.thermos.core import runner as core
+from twitter.thermos.monitoring.monitor import TaskMonitor
+
+from gen.twitter.thermos.ttypes import TaskState
+
+from .common.status_checker import ExitState, StatusResult
+from .common.task_info import (
+ mesos_task_instance_from_assigned_task,
+ resolve_ports,
+)
+from .common.task_runner import (
+ TaskError,
+ TaskRunner,
+ TaskRunnerProvider,
+)
+
+
+class ThermosTaskRunner(TaskRunner):
+ ESCALATION_WAIT = Amount(5, Time.SECONDS)
+ EXIT_STATE_MAP = {
+ TaskState.ACTIVE: StatusResult('Runner died while task was active.', ExitState.LOST),
+ TaskState.FAILED: StatusResult('Task failed.', ExitState.FAILED),
+ TaskState.KILLED: StatusResult('Task killed.', ExitState.KILLED),
+ TaskState.LOST: StatusResult('Task lost.', ExitState.LOST),
+ TaskState.SUCCESS: StatusResult('Task finished.', ExitState.FINISHED),
+ }
+ MAX_WAIT = Amount(1, Time.MINUTES)
+ PEX_NAME = 'thermos_runner.pex'
+ POLL_INTERVAL = Amount(500, Time.MILLISECONDS)
+ THERMOS_PREEMPTION_WAIT = Amount(1, Time.MINUTES)
+
+ def __init__(self,
+ runner_pex,
+ task_id,
+ task,
+ role,
+ portmap,
+ sandbox,
+ checkpoint_root=None,
+ artifact_dir=None,
+ clock=time):
+ """
+ runner_pex location of the thermos_runner pex that this task runner should use
+ task_id task_id assigned by scheduler
+ task thermos pystachio Task object
+ role role to run the task under
+ portmap { name => port } dictionary
+ sandbox the sandbox object
+ checkpoint_root the checkpoint root for the thermos runner
+ artifact_dir scratch space for the thermos runner (basically cwd of thermos.pex)
+ clock clock
+ """
+ self._runner_pex = runner_pex
+ self._task_id = task_id
+ self._task = task
+ self._popen = None
+ self._monitor = None
+ self._status = None
+ self._ports = portmap
+ self._root = sandbox.root
+ self._checkpoint_root = checkpoint_root or TaskPath.DEFAULT_CHECKPOINT_ROOT
+ self._enable_chroot = sandbox.chrooted
+ self._role = role
+ self._clock = clock
+ self._artifact_dir = artifact_dir or safe_mkdtemp()
+
+ # wait events
+ self._dead = threading.Event()
+ self._kill_signal = threading.Event()
+ self.forking = threading.Event()
+ self.forked = threading.Event()
+
+ try:
+ with open(os.path.join(self._artifact_dir, 'task.json'), 'w') as fp:
+ self._task_filename = fp.name
+ ThermosTaskWrapper(self._task).to_file(self._task_filename)
+ except ThermosTaskWrapper.InvalidTask as e:
+ raise TaskError('Failed to load task: %s' % e)
+
+ def _terminate_http(self):
+ if 'health' not in self._ports:
+ return
+
+ http_signaler = HttpSignaler(self._ports['health'])
+
+ # pass 1
+ http_signaler.quitquitquit()
+ self._clock.sleep(self.ESCALATION_WAIT.as_(Time.SECONDS))
+ if self.status is not None:
+ return True
+
+ # pass 2
+ http_signaler.abortabortabort()
+ self._clock.sleep(self.ESCALATION_WAIT.as_(Time.SECONDS))
+ if self.status is not None:
+ return True
+
+ @property
+ def artifact_dir(self):
+ return self._artifact_dir
+
+ def task_state(self):
+ return self._monitor.task_state() if self._monitor else None
+
+ @property
+ def is_alive(self):
+ """
+ Is the process underlying the Thermos task runner alive?
+ """
+ if not self._popen:
+ return False
+ if self._dead.is_set():
+ return False
+
+ # N.B. You cannot mix this code and any code that relies upon os.wait
+ # mechanisms with blanket child process collection. One example is the
+ # Thermos task runner which calls os.wait4 -- without refactoring, you
+ # should not mix a Thermos task runner in the same process as this
+ # thread.
+ try:
+ pid, _ = os.waitpid(self._popen.pid, os.WNOHANG)
+ if pid == 0:
+ return True
+ else:
+ log.info('Detected runner termination: pid=%s' % pid)
+ except OSError as e:
+ log.error('is_alive got OSError: %s' % e)
+ if e.errno != errno.ECHILD:
+ raise
+
+ self._dead.set()
+ return False
+
+ def compute_status(self):
+ if self.is_alive:
+ return None
+ exit_state = self.EXIT_STATE_MAP.get(self.task_state())
+ if exit_state is None:
+ log.error('Received unexpected exit state from TaskMonitor.')
+ return exit_state
+
+ def terminate_runner(self, as_loss=False):
+ """
+ Terminate the underlying runner process, if it exists.
+ """
+ if self._kill_signal.is_set():
+ log.warning('Duplicate kill/lose signal received, ignoring.')
+ return
+ self._kill_signal.set()
+ if self.is_alive:
+ sig = 'SIGUSR2' if as_loss else 'SIGUSR1'
+ log.info('Runner is alive, sending %s' % sig)
+ try:
+ self._popen.send_signal(getattr(signal, sig))
+ except OSError as e:
+ log.error('Got OSError sending %s: %s' % (sig, e))
+ else:
+ log.info('Runner is dead, skipping kill.')
+
+ def kill(self):
+ self.terminate_runner()
+
+ def lose(self):
+ self.terminate_runner(as_loss=True)
+
+ def quitquitquit(self):
+ """Bind to the process tree of a Thermos task and kill it with impunity."""
+ try:
+ runner = core.TaskRunner.get(self._task_id, self._checkpoint_root)
+ if runner:
+ log.info('quitquitquit calling runner.kill')
+ # Right now preemption wait is hardcoded, though it may become configurable in the future.
+ runner.kill(force=True, preemption_wait=self.THERMOS_PREEMPTION_WAIT)
+ else:
+ log.error('Could not instantiate runner!')
+ except core.TaskRunner.Error as e:
+ log.error('Could not quitquitquit runner: %s' % e)
+
+ def _cmdline(self):
+ params = dict(log_dir=LogOptions.log_dir(),
+ log_to_disk='DEBUG',
+ checkpoint_root=self._checkpoint_root,
+ sandbox=self._root,
+ task_id=self._task_id,
+ thermos_json=self._task_filename)
+
+ if getpass.getuser() == 'root':
+ params.update(setuid=self._role)
+
+ cmdline_args = [self._runner_pex]
+ cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items())
+ if self._enable_chroot:
+ cmdline_args.extend(['--enable_chroot'])
+ for name, port in self._ports.items():
+ cmdline_args.extend(['--port=%s:%s' % (name, port)])
+ return cmdline_args
+
+ # --- public interface
+ def start(self, timeout=MAX_WAIT):
+ """Fork the task runner and return once the underlying task is running, up to timeout."""
+ self.forking.set()
+
+ try:
+ chmod_plus_x(self._runner_pex)
+ except OSError as e:
+ if e.errno != errno.EPERM:
+ raise TaskError('Failed to chmod +x runner: %s' % e)
+
+ self._monitor = TaskMonitor(TaskPath(root=self._checkpoint_root), self._task_id)
+
+ cmdline_args = self._cmdline()
+ log.info('Forking off runner with cmdline: %s' % ' '.join(cmdline_args))
+
+ try:
+ self._popen = subprocess.Popen(cmdline_args)
+ except OSError as e:
+ raise TaskError(e)
+
+ self.forked.set()
+
+ log.debug('Waiting for task to start.')
+
+ def is_started():
+ return self._monitor and (self._monitor.active or self._monitor.finished)
+
+ waited = Amount(0, Time.SECONDS)
+ while not is_started() and waited < timeout:
+ log.debug(' - sleeping...')
+ self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
+ waited += self.POLL_INTERVAL
+
+ if not is_started():
+ log.error('Task did not start with in deadline, forcing loss.')
+ self.lose()
+ raise TaskError('Task did not start within deadline.')
+
+ def stop(self, timeout=MAX_WAIT):
+ """Stop the runner. If it's already completed, no-op. If it's still running, issue a kill."""
+ log.info('ThermosTaskRunner is shutting down.')
+
+ if not self.forking.is_set():
+ raise TaskError('Failed to call TaskRunner.start.')
+
+ log.info('Invoking runner HTTP teardown.')
+ self._terminate_http()
+
+ log.info('Invoking runner.kill')
+ self.kill()
+
+ waited = Amount(0, Time.SECONDS)
+ while self.is_alive and waited < timeout:
+ self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
+ waited += self.POLL_INTERVAL
+
+ if not self.is_alive and self.task_state() != TaskState.ACTIVE:
+ return
+
+ log.info('Thermos task did not shut down cleanly, rebinding to kill.')
+ self.quitquitquit()
+
+ while not self._monitor.finished and waited < timeout:
+ self._clock.sleep(self.POLL_INTERVAL.as_(Time.SECONDS))
+ waited += self.POLL_INTERVAL
+
+ if not self._monitor.finished:
+ raise TaskError('Task did not stop within deadline.')
+
+ @property
+ def status(self):
+ """Return the StatusResult of this task runner. This returns None as
+ long as no terminal state is reached."""
+ if self._status is None:
+ self._status = self.compute_status()
+ return self._status
+
+
+class DefaultThermosTaskRunnerProvider(TaskRunnerProvider):
+ def __init__(self,
+ pex_location,
+ checkpoint_root=None,
+ artifact_dir=None,
+ task_runner_class=ThermosTaskRunner,
+ max_wait=Amount(1, Time.MINUTES),
+ preemption_wait=Amount(1, Time.MINUTES),
+ poll_interval=Amount(500, Time.MILLISECONDS),
+ clock=time):
+ self._artifact_dir = artifact_dir or safe_mkdtemp()
+ self._checkpoint_root = checkpoint_root
+ self._clock = clock
+ self._max_wait = max_wait
+ self._pex_location = pex_location
+ self._poll_interval = poll_interval
+ self._preemption_wait = preemption_wait
+ self._task_runner_class = task_runner_class
+
+ def from_assigned_task(self, assigned_task, sandbox):
+ task_id = assigned_task.taskId
+ role = assigned_task.task.owner.role
+ mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
+ mesos_ports = resolve_ports(mesos_task, assigned_task.assignedPorts)
+
+ class ProvidedThermosTaskRunner(self._task_runner_class):
+ MAX_WAIT = self._max_wait
+ POLL_INTERVAL = self._poll_interval
+ THERMOS_PREEMPTION_WAIT = self._preemption_wait
+
+ return ProvidedThermosTaskRunner(
+ self._pex_location,
+ task_id,
+ mesos_task.task(),
+ role,
+ mesos_ports,
+ sandbox,
+ checkpoint_root=self._checkpoint_root,
+ artifact_dir=self._artifact_dir,
+ clock=self._clock)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/tools/java/organize_imports.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/tools/java/organize_imports.py b/src/main/python/apache/aurora/tools/java/organize_imports.py
new file mode 100644
index 0000000..48a0821
--- /dev/null
+++ b/src/main/python/apache/aurora/tools/java/organize_imports.py
@@ -0,0 +1,130 @@
+#!/bin/env python
+'''
+Organizes a java source file's import statements in a way that pleases Twitter's checkstyle
+configuration.
+This expects exactly one argument: the name of the file to modify with preferred import
+ordering.
+'''
+
+from __future__ import print_function
+
+import re
+import sys
+from collections import defaultdict
+
+
+IMPORT_RE = re.compile('import(?: static)? (.*);')
+def get_group(import_statement):
+ matcher = IMPORT_RE.match(import_statement)
+ assert matcher, 'Could not parse import statement: %s' % import_statement
+ class_name = matcher.group(1)
+ if class_name.startswith('com.twitter'):
+ group = 'com.twitter'
+ else:
+ group = class_name.split('.')[0]
+ return group
+
+
+def index_by_group(import_statements):
+ groups = defaultdict(list)
+ for line in import_statements:
+ groups[get_group(line)].append(line)
+ return groups
+
+
+IMPORT_CLASS_RE = re.compile(
+ 'import(?: static)? (?P<outer>[^A-Z]*[A-Z]\w+)(?:\.(?P<inners>[\w][^;]*))?')
+def get_all_group_lines(import_groups):
+ if not import_groups:
+ return []
+
+ def get_group_lines(group):
+ def comparator(x, y):
+ # These shenanigans are used to properly order imports for inner classes.
+ # So we get ordering like:
+ # import com.foo.Bar;
+ # import com.foo.Bar.Baz;
+ # (this is not lexicographical, so normal sort won't suffice)
+ x_m = IMPORT_CLASS_RE.match(x)
+ y_m = IMPORT_CLASS_RE.match(y)
+ if x_m.group('outer') == y_m.group('outer'):
+ return cmp(x_m.group('inners'), y_m.group('inners'))
+ else:
+ return cmp(x, y)
+ lines = sorted(import_groups[group], comparator)
+ lines.append('')
+ return lines
+
+ all_lines = []
+ explicit_groups = ['java', 'javax', 'scala', 'com', 'net', 'org', 'com.twitter']
+ for group in explicit_groups:
+ if group in import_groups:
+ all_lines += get_group_lines(group)
+
+ # Gather remaining groups.
+ remaining_groups = sorted(set(import_groups.keys()) - set(explicit_groups))
+ for group in remaining_groups:
+ all_lines += get_group_lines(group)
+ return all_lines
+
+
+if len(sys.argv) != 2:
+ print('usage: %s FILE' % sys.argv[0])
+ sys.exit(1)
+
+BEFORE_IMPORTS = 'before_imports'
+IMPORTS = 'imports'
+STATIC_IMPORTS = 'static_imports'
+AFTER_IMPORTS = 'after_imports'
+
+print('Organizing imports in %s' % sys.argv[1])
+lines_before_imports = []
+import_lines = []
+static_import_lines = []
+lines_after_imports = []
+with open(sys.argv[1], 'r') as f:
+ position = BEFORE_IMPORTS
+ for line in f:
+ line = line.rstrip()
+ if position == BEFORE_IMPORTS:
+ if line.startswith('import'):
+ position = IMPORTS
+ else:
+ lines_before_imports.append(line)
+ if position == IMPORTS:
+ if line.startswith('import static'):
+ position = STATIC_IMPORTS
+ elif line.startswith('import'):
+ import_lines.append(line)
+ elif line.strip():
+ position = AFTER_IMPORTS
+ if position == STATIC_IMPORTS:
+ if line.startswith('import static'):
+ static_import_lines.append(line)
+ elif line.strip():
+ position = AFTER_IMPORTS
+ if position == AFTER_IMPORTS:
+ lines_after_imports.append(line)
+
+import_groups = index_by_group(import_lines)
+static_import_groups = index_by_group(static_import_lines)
+
+def ensure_line_padding(lines):
+ if lines and lines[-1] != '':
+ lines.append('')
+ return lines
+
+file_lines = lines_before_imports
+if import_groups:
+ ensure_line_padding(file_lines)
+ file_lines += get_all_group_lines(import_groups)
+if static_import_groups:
+ ensure_line_padding(file_lines)
+ file_lines += get_all_group_lines(static_import_groups)
+if lines_after_imports:
+ ensure_line_padding(file_lines)
+ file_lines += lines_after_imports
+
+with open(sys.argv[1], 'w') as f:
+ for line in file_lines:
+ print(line, file=f)
[27/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/auth/ThriftAuthModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/auth/ThriftAuthModule.java b/src/main/java/org/apache/aurora/scheduler/thrift/auth/ThriftAuthModule.java
new file mode 100644
index 0000000..66f9033
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/auth/ThriftAuthModule.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.auth;
+
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.AbstractModule;
+import com.google.inject.TypeLiteral;
+
+import com.twitter.aurora.auth.CapabilityValidator;
+import com.twitter.aurora.auth.CapabilityValidator.Capability;
+import com.twitter.aurora.auth.SessionValidator;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.args.constraints.NotEmpty;
+
+/**
+ * Binding module for authentication of users with special capabilities for admin functions.
+ */
+public class ThriftAuthModule extends AbstractModule {
+
+ private static final Map<Capability, String> DEFAULT_CAPABILITIES =
+ ImmutableMap.of(Capability.ROOT, "mesos");
+
+ @NotEmpty
+ @CmdLine(name = "user_capabilities",
+ help = "Concrete name mappings for administration capabilities.")
+ private static final Arg<Map<Capability, String>> USER_CAPABILITIES =
+ Arg.create(DEFAULT_CAPABILITIES);
+
+ private Map<Capability, String> capabilities;
+
+ public ThriftAuthModule() {
+ this(USER_CAPABILITIES.get());
+ }
+
+ @VisibleForTesting
+ public ThriftAuthModule(Map<Capability, String> capabilities) {
+ this.capabilities = Preconditions.checkNotNull(capabilities);
+ }
+
+ @Override
+ protected void configure() {
+ Preconditions.checkArgument(
+ capabilities.containsKey(Capability.ROOT),
+ "A ROOT capability must be provided with --user_capabilities");
+ bind(new TypeLiteral<Map<Capability, String>>() { }).toInstance(capabilities);
+
+ requireBinding(SessionValidator.class);
+ requireBinding(CapabilityValidator.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/__init__.py b/src/main/python/apache/__init__.py
new file mode 100644
index 0000000..de40ea7
--- /dev/null
+++ b/src/main/python/apache/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/BUILD b/src/main/python/apache/aurora/BUILD
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/BUILD.thirdparty
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/BUILD.thirdparty b/src/main/python/apache/aurora/BUILD.thirdparty
new file mode 100644
index 0000000..7a43aca
--- /dev/null
+++ b/src/main/python/apache/aurora/BUILD.thirdparty
@@ -0,0 +1,19 @@
+def make_dep(name, version, dependency_name=None):
+ """Build a target from a specified dependency tuple.
+
+ name is the target name, specified in other BUILD files.
+ version is a hardcoded version string
+ dependency_name is used to identify the specific binary to resolve
+ """
+ dependency_name = dependency_name or name
+ versioned_name = "%s==%s" % (dependency_name, version)
+ python_requirement(requirement=versioned_name, name=name)
+
+make_dep('argparse', '1.2.1')
+make_dep('mesos-core', '0.15.0-rc4', 'mesos')
+make_dep('mock', '1.0.1')
+make_dep('mox', '0.5.3')
+make_dep('psutil', '1.1.2')
+make_dep('pystachio', '0.7.2')
+make_dep('pyyaml', '3.10', 'PyYAML')
+make_dep('thrift', '0.9.1')
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/__init__.py b/src/main/python/apache/aurora/__init__.py
new file mode 100644
index 0000000..b0d6433
--- /dev/null
+++ b/src/main/python/apache/aurora/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/admin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/BUILD b/src/main/python/apache/aurora/admin/BUILD
new file mode 100644
index 0000000..c8089b4
--- /dev/null
+++ b/src/main/python/apache/aurora/admin/BUILD
@@ -0,0 +1,11 @@
+python_library(
+ name = 'mesos_maintenance',
+ sources = 'mesos_maintenance.py',
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/aurora/client:api'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/admin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/__init__.py b/src/main/python/apache/aurora/admin/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/admin/mesos_maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/mesos_maintenance.py b/src/main/python/apache/aurora/admin/mesos_maintenance.py
new file mode 100644
index 0000000..4e14436
--- /dev/null
+++ b/src/main/python/apache/aurora/admin/mesos_maintenance.py
@@ -0,0 +1,113 @@
+from collections import defaultdict
+import time
+
+from twitter.common import log
+from twitter.common.quantity import Amount, Time
+
+from twitter.aurora.client.api import AuroraClientAPI
+from twitter.aurora.client.base import check_and_log_response
+
+from gen.twitter.aurora.ttypes import Hosts, MaintenanceMode
+
+
+def group_by_host(hostname):
+ return hostname
+
+
+class MesosMaintenance(object):
+ """This class provides more methods to interact with the mesos cluster and perform
+ maintenance.
+ """
+
+ DEFAULT_GROUPING = 'by_host'
+ GROUPING_FUNCTIONS = {
+ 'by_host': group_by_host,
+ }
+ START_MAINTENANCE_DELAY = Amount(30, Time.SECONDS)
+
+ @classmethod
+ def group_hosts(cls, hostnames, grouping_function=DEFAULT_GROUPING):
+ try:
+ grouping_function = cls.GROUPING_FUNCTIONS[grouping_function]
+ except KeyError:
+ raise ValueError('Unknown grouping function %s!' % grouping_function)
+ groups = defaultdict(set)
+ for hostname in hostnames:
+ groups[grouping_function(hostname)].add(hostname)
+ return groups
+
+ @classmethod
+ def iter_batches(cls, hostnames, batch_size, grouping_function=DEFAULT_GROUPING):
+ if batch_size <= 0:
+ raise ValueError('Batch size must be > 0!')
+ groups = cls.group_hosts(hostnames, grouping_function)
+ groups = sorted(groups.items(), key=lambda v: v[0])
+ for k in range(0, len(groups), batch_size):
+ yield Hosts(set.union(*(hostset for (key, hostset) in groups[k:k+batch_size])))
+
+ def __init__(self, cluster, verbosity):
+ self._client = AuroraClientAPI(cluster, verbosity == 'verbose')
+
+ def _drain_hosts(self, drainable_hosts, clock=time):
+ """This will actively turn down tasks running on hosts."""
+ check_and_log_response(self._client.drain_hosts(drainable_hosts))
+ not_ready_hosts = [hostname for hostname in drainable_hosts.hostNames]
+ while not_ready_hosts:
+ log.info("Sleeping for %s." % self.START_MAINTENANCE_DELAY)
+ clock.sleep(self.START_MAINTENANCE_DELAY.as_(Time.SECONDS))
+ resp = self._client.maintenance_status(Hosts(not_ready_hosts))
+ #TODO(jsmith): Workaround until scheduler responds with unknown slaves in MESOS-3454
+ if not resp.result.maintenanceStatusResult.statuses:
+ not_ready_hosts = None
+ for host_status in resp.result.maintenanceStatusResult.statuses:
+ if host_status.mode != MaintenanceMode.DRAINED:
+ log.warning('%s is currently in status %s' %
+ (host_status.host, MaintenanceMode._VALUES_TO_NAMES[host_status.mode]))
+ else:
+ not_ready_hosts.remove(host_status.host)
+
+ def _complete_maintenance(self, drained_hosts):
+ """End the maintenance status for a give set of hosts."""
+ check_and_log_response(self._client.end_maintenance(drained_hosts))
+ resp = self._client.maintenance_status(drained_hosts)
+ for host_status in resp.result.maintenanceStatusResult.statuses:
+ if host_status.mode != MaintenanceMode.NONE:
+ log.warning('%s is DRAINING or in DRAINED' % host_status.host)
+
+ def _operate_on_hosts(self, drained_hosts, callback):
+ """Perform a given operation on a list of hosts that are ready for maintenance."""
+ for host in drained_hosts.hostNames:
+ callback(host)
+
+ def end_maintenance(self, hosts):
+ """Pull a list of hosts out of maintenance mode."""
+ self._complete_maintenance(Hosts(set(hosts)))
+
+ def start_maintenance(self, hosts):
+ """Put a list of hosts into maintenance mode, to de-prioritize scheduling."""
+ check_and_log_response(self._client.start_maintenance(Hosts(set(hosts))))
+
+ def perform_maintenance(self, hosts, batch_size=1, grouping_function=DEFAULT_GROUPING,
+ callback=None):
+ """The wrap a callback in between sending hosts into maintenance mode and back.
+
+ Walk through the process of putting hosts into maintenance, draining them of tasks,
+ performing an action on them once drained, then removing them from maintenance mode
+ so tasks can schedule.
+ """
+ self._complete_maintenance(Hosts(set(hosts)))
+ self.start_maintenance(hosts)
+
+ for hosts in self.iter_batches(hosts, batch_size, grouping_function):
+ self._drain_hosts(hosts)
+ if callback:
+ self._operate_on_hosts(hosts, callback)
+ self._complete_maintenance(hosts)
+
+ def check_status(self, hosts):
+ resp = self._client.maintenance_status(Hosts(set(hosts)))
+ check_and_log_response(resp)
+ statuses = []
+ for host_status in resp.result.maintenanceStatusResult.statuses:
+ statuses.append((host_status.host, MaintenanceMode._VALUES_TO_NAMES[host_status.mode]))
+ return statuses
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/BUILD b/src/main/python/apache/aurora/client/BUILD
new file mode 100644
index 0000000..70eef2e
--- /dev/null
+++ b/src/main/python/apache/aurora/client/BUILD
@@ -0,0 +1,78 @@
+import os
+
+# Create an alias for the previous target
+python_library(
+ name = 'api',
+ dependencies = [
+ pants('src/main/python/twitter/aurora/client/api')
+ ]
+)
+
+python_library(
+ name = 'base',
+ sources = ['base.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'binding_helper',
+ sources = ['binding_helper.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ ]
+)
+
+python_library(
+ name = 'config',
+ sources = ['config.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ pants(':base'),
+ pants(':binding_helper'),
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/config'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'factory',
+ sources = ['factory.py'],
+ dependencies = [
+ pants(':base'),
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('src/main/python/twitter/aurora/client/hooks'),
+ pants('src/main/python/twitter/aurora/common:cluster'),
+ pants('src/main/python/twitter/aurora/common:clusters'),
+ ]
+)
+
+python_library(
+ name = 'options',
+ sources = ['options.py'],
+ dependencies = [
+ pants('src/main/python/twitter/thermos/common:options'),
+ pants('src/main/python/twitter/aurora/common:aurora_job_key'),
+ ]
+)
+
+python_library(
+ name = 'client-packaged',
+ dependencies = [
+ pants('src/main/python/twitter/aurora/common'),
+ pants('src/main/python/twitter/aurora/config'),
+ pants('src/main/python/twitter/thermos/common'),
+ ],
+ provides = setup_py(
+ name = 'twitter.aurora.client',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ ).with_binaries(
+ aurora_admin = pants('src/main/python/twitter/aurora/client/bin:aurora_admin'),
+ aurora_client = pants('src/main/python/twitter/aurora/client/bin:aurora_client'),
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/__init__.py b/src/main/python/apache/aurora/client/__init__.py
new file mode 100644
index 0000000..b0d6433
--- /dev/null
+++ b/src/main/python/apache/aurora/client/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/BUILD b/src/main/python/apache/aurora/client/api/BUILD
new file mode 100644
index 0000000..4935b8a
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/BUILD
@@ -0,0 +1,104 @@
+python_library(
+ name = 'api',
+ sources = ['__init__.py'],
+ dependencies = [
+ pants(':restarter'),
+ pants(':scheduler_client'),
+ pants(':updater'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/common'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'command_runner',
+ sources = ['command_runner.py'],
+ dependencies = [
+ pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
+ pants(':api'),
+ pants('src/main/python/twitter/thermos/config:schema'),
+ pants('src/main/python/twitter/aurora/common:cluster'),
+ pants('src/main/python/twitter/aurora/config:schema'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'disambiguator',
+ sources = ['disambiguator.py'],
+ dependencies = [
+ pants(':api'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/python/twitter/aurora/common'),
+ ]
+)
+
+python_library(
+ name = 'job_monitor',
+ sources = ['job_monitor.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'restarter',
+ sources = ['restarter.py'],
+ dependencies = [
+ pants(':instance_watcher'),
+ pants(':updater_util'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'scheduler_client',
+ sources = ['scheduler_client.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('aurora/twitterdeps/src/python/twitter/common/rpc/transports:tsslsocket'),
+ pants('aurora/twitterdeps/src/python/twitter/common/zookeeper/serverset:kazoo_serverset'),
+ pants('aurora/twitterdeps/src/python/twitter/common/zookeeper:kazoo_client'),
+ pants('src/main/python/twitter/aurora/common/auth'),
+ pants('src/main/python/twitter/aurora/common:cluster'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'instance_watcher',
+ sources = ['instance_watcher.py', 'health_check.py'],
+ dependencies = [
+ pants(':scheduler_client'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/common:http_signaler'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'updater',
+ sources = ['updater.py'],
+ dependencies = [
+ pants(':scheduler_client'),
+ pants(':instance_watcher'),
+ pants(':updater_util'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'updater_util',
+ sources = ['updater_util.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/__init__.py b/src/main/python/apache/aurora/client/api/__init__.py
new file mode 100644
index 0000000..60f4011
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/__init__.py
@@ -0,0 +1,190 @@
+from twitter.common import log
+
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+from twitter.aurora.common.auth import make_session_key
+from twitter.aurora.common.cluster import Cluster
+
+from gen.twitter.aurora.constants import LIVE_STATES
+from gen.twitter.aurora.ttypes import (
+ Response,
+ Identity,
+ Quota,
+ ResponseCode,
+ TaskQuery)
+
+from .restarter import Restarter
+from .scheduler_client import SchedulerProxy
+from .updater import Updater
+
+
+class AuroraClientAPI(object):
+ """This class provides the API to talk to the twitter scheduler"""
+
+ class Error(Exception): pass
+ class TypeError(Error, TypeError): pass
+ class ClusterMismatch(Error, ValueError): pass
+
+ def __init__(self, cluster, verbose=False, session_key_factory=make_session_key):
+ if not isinstance(cluster, Cluster):
+ raise TypeError('AuroraClientAPI expects instance of Cluster for "cluster", got %s' %
+ type(cluster))
+ self._scheduler = SchedulerProxy(
+ cluster, verbose=verbose, session_key_factory=session_key_factory)
+ self._cluster = cluster
+
+ @property
+ def cluster(self):
+ return self._cluster
+
+ @property
+ def scheduler(self):
+ return self._scheduler
+
+ def create_job(self, config, lock=None):
+ log.info('Creating job %s' % config.name())
+ log.debug('Full configuration: %s' % config.job())
+ log.debug('Lock %s' % lock)
+ return self._scheduler.createJob(config.job(), lock)
+
+ def populate_job_config(self, config, validation=None):
+ return self._scheduler.populateJobConfig(config.job(), validation)
+
+ def start_cronjob(self, job_key):
+ self._assert_valid_job_key(job_key)
+
+ log.info("Starting cron job: %s" % job_key)
+ return self._scheduler.startCronJob(job_key.to_thrift())
+
+ def get_jobs(self, role):
+ log.info("Retrieving jobs for role %s" % role)
+ return self._scheduler.getJobs(role)
+
+ def kill_job(self, job_key, instances=None, lock=None):
+ log.info("Killing tasks for job: %s" % job_key)
+ if not isinstance(job_key, AuroraJobKey):
+ raise TypeError('Expected type of job_key %r to be %s but got %s instead'
+ % (job_key, AuroraJobKey.__name__, job_key.__class__.__name__))
+
+ # Leave query.owner.user unset so the query doesn't filter jobs only submitted by a particular
+ # user.
+ # TODO(wfarner): Refactor this when Identity is removed from TaskQuery.
+ query = job_key.to_thrift_query()
+ if instances is not None:
+ log.info("Instances to be killed: %s" % instances)
+ query.instanceIds = frozenset([int(s) for s in instances])
+ return self._scheduler.killTasks(query, lock)
+
+ def check_status(self, job_key):
+ self._assert_valid_job_key(job_key)
+
+ log.info("Checking status of %s" % job_key)
+ return self.query(job_key.to_thrift_query())
+
+ @classmethod
+ def build_query(cls, role, job, instances=None, statuses=LIVE_STATES, env=None):
+ return TaskQuery(owner=Identity(role=role),
+ jobName=job,
+ statuses=statuses,
+ instanceIds=instances,
+ environment=env)
+
+ def query(self, query):
+ return self._scheduler.getTasksStatus(query)
+
+ def update_job(self, config, health_check_interval_seconds=3, instances=None):
+ """Run a job update for a given config, for the specified instances. If
+ instances is left unspecified, update all instances. Returns whether or not
+ the update was successful."""
+
+ log.info("Updating job: %s" % config.name())
+ updater = Updater(config, health_check_interval_seconds, self._scheduler)
+
+ return updater.update(instances)
+
+ def cancel_update(self, job_key):
+ """Cancel the update represented by job_key. Returns whether or not the cancellation was
+ successful."""
+ self._assert_valid_job_key(job_key)
+
+ log.info("Canceling update on job %s" % job_key)
+ resp = Updater.cancel_update(self._scheduler, job_key)
+ if resp.responseCode != ResponseCode.OK:
+ log.error('Error cancelling the update: %s' % resp.message)
+ return resp
+
+ def restart(self, job_key, instances, updater_config, health_check_interval_seconds):
+ """Perform a rolling restart of the job. If instances is None or [], restart all instances. Returns
+ the scheduler response for the last restarted batch of instances (which allows the client to
+ show the job URL), or the status check response if no tasks were active.
+ """
+ self._assert_valid_job_key(job_key)
+
+ return Restarter(job_key, updater_config, health_check_interval_seconds, self._scheduler
+ ).restart(instances)
+
+ def start_maintenance(self, hosts):
+ log.info("Starting maintenance for: %s" % hosts.hostNames)
+ return self._scheduler.startMaintenance(hosts)
+
+ def drain_hosts(self, hosts):
+ log.info("Draining tasks on: %s" % hosts.hostNames)
+ return self._scheduler.drainHosts(hosts)
+
+ def maintenance_status(self, hosts):
+ log.info("Maintenance status for: %s" % hosts.hostNames)
+ return self._scheduler.maintenanceStatus(hosts)
+
+ def end_maintenance(self, hosts):
+ log.info("Ending maintenance for: %s" % hosts.hostNames)
+ return self._scheduler.endMaintenance(hosts)
+
+ def get_quota(self, role):
+ log.info("Getting quota for: %s" % role)
+ return self._scheduler.getQuota(role)
+
+ def set_quota(self, role, cpu, ram_mb, disk_mb):
+ log.info("Setting quota for user:%s cpu:%f ram_mb:%d disk_mb: %d"
+ % (role, cpu, ram_mb, disk_mb))
+ return self._scheduler.setQuota(role, Quota(cpu, ram_mb, disk_mb))
+
+ def force_task_state(self, task_id, status):
+ log.info("Requesting that task %s transition to state %s" % (task_id, status))
+ return self._scheduler.forceTaskState(task_id, status)
+
+ def perform_backup(self):
+ return self._scheduler.performBackup()
+
+ def list_backups(self):
+ return self._scheduler.listBackups()
+
+ def stage_recovery(self, backup_id):
+ return self._scheduler.stageRecovery(backup_id)
+
+ def query_recovery(self, query):
+ return self._scheduler.queryRecovery(query)
+
+ def delete_recovery_tasks(self, query):
+ return self._scheduler.deleteRecoveryTasks(query)
+
+ def commit_recovery(self):
+ return self._scheduler.commitRecovery()
+
+ def unload_recovery(self):
+ return self._scheduler.unloadRecovery()
+
+ def get_job_updates(self):
+ return self._scheduler.getJobUpdates()
+
+ def snapshot(self):
+ return self._scheduler.snapshot()
+
+ def unsafe_rewrite_config(self, rewrite_request):
+ return self._scheduler.rewriteConfigs(rewrite_request)
+
+ def _assert_valid_job_key(self, job_key):
+ if not isinstance(job_key, AuroraJobKey):
+ raise self.TypeError('Invalid job_key %r: expected %s but got %s'
+ % (job_key, AuroraJobKey.__name__, job_key.__class__.__name__))
+ if job_key.cluster != self.cluster.name:
+ raise self.ClusterMismatch('job %s does not belong to cluster %s' % (job_key,
+ self.cluster.name))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/command_runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/command_runner.py b/src/main/python/apache/aurora/client/api/command_runner.py
new file mode 100644
index 0000000..0a8ad33
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/command_runner.py
@@ -0,0 +1,120 @@
+from multiprocessing.pool import ThreadPool
+import posixpath
+import subprocess
+
+from twitter.common import log
+
+from twitter.aurora.client.api import AuroraClientAPI
+from twitter.aurora.config.schema.base import MesosContext
+from twitter.aurora.common.cluster import Cluster
+from twitter.thermos.config.schema import ThermosContext
+
+from gen.twitter.aurora.constants import LIVE_STATES
+from gen.twitter.aurora.ttypes import (
+ Identity,
+ ResponseCode,
+ TaskQuery)
+
+from pystachio import Environment, Required, String
+
+
+class CommandRunnerTrait(Cluster.Trait):
+ slave_root = Required(String)
+ slave_run_directory = Required(String)
+
+
+class DistributedCommandRunner(object):
+ @staticmethod
+ def execute(args):
+ hostname, role, command = args
+ ssh_command = ['ssh', '-n', '-q', '%s@%s' % (role, hostname), command]
+ po = subprocess.Popen(ssh_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ output = po.communicate()
+ return '\n'.join('%s: %s' % (hostname, line) for line in output[0].splitlines())
+
+ @classmethod
+ def make_executor_path(cls, cluster, executor_name):
+ parameters = cls.sandbox_args(cluster)
+ parameters.update(executor_name=executor_name)
+ return posixpath.join(
+ '%(slave_root)s',
+ 'slaves/*/frameworks/*/executors/%(executor_name)s/runs',
+ '%(slave_run_directory)s'
+ ) % parameters
+
+ @classmethod
+ def thermos_sandbox(cls, cluster, executor_sandbox=False):
+ sandbox = cls.make_executor_path(cluster, 'thermos-{{thermos.task_id}}')
+ return sandbox if executor_sandbox else posixpath.join(sandbox, 'sandbox')
+
+ @classmethod
+ def sandbox_args(cls, cluster):
+ cluster = cluster.with_trait(CommandRunnerTrait)
+ return {'slave_root': cluster.slave_root, 'slave_run_directory': cluster.slave_run_directory}
+
+ @classmethod
+ def substitute_thermos(cls, command, task, cluster, **kw):
+ prefix_command = 'cd %s;' % cls.thermos_sandbox(cluster, **kw)
+ thermos_namespace = ThermosContext(
+ task_id=task.assignedTask.taskId,
+ ports=task.assignedTask.assignedPorts)
+ mesos_namespace = MesosContext(instance=task.assignedTask.instanceId)
+ command = String(prefix_command + command) % Environment(
+ thermos=thermos_namespace,
+ mesos=mesos_namespace)
+ return command.get()
+
+ @classmethod
+ def aurora_sandbox(cls, cluster, executor_sandbox=False):
+ if executor_sandbox:
+ return cls.make_executor_path(cluster, 'twitter')
+ else:
+ return '/var/run/nexus/%task_id%/sandbox'
+
+ @classmethod
+ def substitute_aurora(cls, command, task, cluster, **kw):
+ command = ('cd %s;' % cls.aurora_sandbox(cluster, **kw)) + command
+ command = command.replace('%shard_id%', str(task.assignedTask.instanceId))
+ command = command.replace('%task_id%', task.assignedTask.taskId)
+ for name, port in task.assignedTask.assignedPorts.items():
+ command = command.replace('%port:' + name + '%', str(port))
+ return command
+
+ @classmethod
+ def substitute(cls, command, task, cluster, **kw):
+ if task.assignedTask.task.executorConfig:
+ return cls.substitute_thermos(command, task, cluster, **kw)
+ else:
+ return cls.substitute_aurora(command, task, cluster, **kw)
+
+ @classmethod
+ def query_from(cls, role, env, job):
+ return TaskQuery(statuses=LIVE_STATES, owner=Identity(role), jobName=job, environment=env)
+
+ def __init__(self, cluster, role, env, jobs, ssh_user=None):
+ self._cluster = cluster
+ self._api = AuroraClientAPI(cluster=cluster)
+ self._role = role
+ self._env = env
+ self._jobs = jobs
+ self._ssh_user = ssh_user if ssh_user else self._role
+
+ def resolve(self):
+ for job in self._jobs:
+ resp = self._api.query(self.query_from(self._role, self._env, job))
+ if resp.responseCode != ResponseCode.OK:
+ log.error('Failed to query job: %s' % job)
+ continue
+ for task in resp.result.scheduleStatusResult.tasks:
+ yield task
+
+ def process_arguments(self, command, **kw):
+ for task in self.resolve():
+ host = task.assignedTask.slaveHost
+ role = task.assignedTask.task.owner.role
+ yield (host, self._ssh_user, self.substitute(command, task, self._cluster, **kw))
+
+ def run(self, command, parallelism=1, **kw):
+ threadpool = ThreadPool(processes=parallelism)
+ for result in threadpool.imap_unordered(self.execute, self.process_arguments(command, **kw)):
+ print result
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/disambiguator.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/disambiguator.py b/src/main/python/apache/aurora/client/api/disambiguator.py
new file mode 100644
index 0000000..4693574
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/disambiguator.py
@@ -0,0 +1,89 @@
+from twitter.common import log
+
+from twitter.aurora.client.api import AuroraClientAPI
+from twitter.aurora.client.base import check_and_log_response, deprecation_warning, die
+from twitter.aurora.common.aurora_job_key import AuroraJobKey
+
+
+class LiveJobDisambiguator(object):
+ """
+ Disambiguates a job-specification into concrete AuroraJobKeys by querying the scheduler API.
+ """
+
+ def __init__(self, client, role, env, name):
+ if not isinstance(client, AuroraClientAPI):
+ raise TypeError("client must be a AuroraClientAPI")
+ self._client = client
+
+ if not role:
+ raise ValueError("role is required")
+ self._role = role
+ if not name:
+ raise ValueError("name is required")
+ self._name = name
+ self._env = env
+
+ @property
+ def ambiguous(self):
+ return not all((self._role, self._env, self._name))
+
+ def query_matches(self):
+ resp = self._client.get_jobs(self._role)
+ check_and_log_response(resp)
+ return set(AuroraJobKey(self._client.cluster.name, j.key.role, j.key.environment, j.key.name)
+ for j in resp.result.getJobsResult.configs if j.key.name == self._name)
+
+ @classmethod
+ def _disambiguate_or_die(cls, client, role, env, name):
+ # Returns a single AuroraJobKey if one can be found given the args, potentially
+ # querying the scheduler. Calls die() with an appropriate error message otherwise.
+ try:
+ disambiguator = cls(client, role, env, name)
+ except ValueError as e:
+ die(e)
+
+ if not disambiguator.ambiguous:
+ return AuroraJobKey(client.cluster.name, role, env, name)
+
+ deprecation_warning("Job ambiguously specified - querying the scheduler to disambiguate")
+ matches = disambiguator.query_matches()
+ if len(matches) == 1:
+ (match,) = matches
+ log.info("Found job %s" % match)
+ return match
+ elif len(matches) == 0:
+ die("No jobs found")
+ else:
+ die("Multiple jobs match (%s) - disambiguate by using the CLUSTER/ROLE/ENV/NAME form"
+ % ",".join(str(m) for m in matches))
+
+ @classmethod
+ def disambiguate_args_or_die(cls, args, options, client_factory=AuroraClientAPI):
+ """
+ Returns a (AuroraClientAPI, AuroraJobKey, AuroraConfigFile:str) tuple
+ if one can be found given the args, potentially querying the scheduler with the returned client.
+ Calls die() with an appropriate error message otherwise.
+
+ Arguments:
+ args: args from app command invocation.
+ options: options from app command invocation. must have env and cluster attributes.
+ client_factory: a callable (cluster) -> AuroraClientAPI.
+ """
+ if not len(args) > 0:
+ die('job path is required')
+ try:
+ job_key = AuroraJobKey.from_path(args[0])
+ client = client_factory(job_key.cluster)
+ config_file = args[1] if len(args) > 1 else None # the config for hooks
+ return client, job_key, config_file
+ except AuroraJobKey.Error:
+ log.warning("Failed to parse job path, falling back to compatibility mode")
+ role = args[0] if len(args) > 0 else None
+ name = args[1] if len(args) > 1 else None
+ env = None
+ config_file = None # deprecated form does not support hooks functionality
+ cluster = options.cluster
+ if not cluster:
+ die('cluster is required')
+ client = client_factory(cluster)
+ return client, cls._disambiguate_or_die(client, role, env, name), config_file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/health_check.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/health_check.py b/src/main/python/apache/aurora/client/api/health_check.py
new file mode 100644
index 0000000..b75ee6e
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/health_check.py
@@ -0,0 +1,123 @@
+from abc import abstractmethod
+
+from twitter.common import log
+from twitter.common.lang import Interface
+
+from twitter.aurora.common.http_signaler import HttpSignaler
+
+from gen.twitter.aurora.ttypes import ScheduleStatus
+
+
+class HealthCheck(Interface):
+ @abstractmethod
+ def health(self, task):
+ """Checks health of the task and returns a (healthy, retriable) pair."""
+
+
+class HealthStatus(object):
+ @classmethod
+ def alive(cls):
+ return cls(True).health()
+
+ @classmethod
+ def dead(cls):
+ return cls(False).health()
+
+ def __init__(self, retry, health):
+ self._retry = retry
+ self._health = health
+
+ def health(self):
+ return (self._health, self._retry)
+
+
+class NotRetriable(HealthStatus):
+ def __init__(self, health):
+ super(NotRetriable, self).__init__(False, health)
+
+
+class Retriable(HealthStatus):
+ def __init__(self, health):
+ super(Retriable, self).__init__(True, health)
+
+
+class StatusHealthCheck(HealthCheck):
+ """Verifies the health of a task based on the task status. A task is healthy iff,
+ 1. A task is in state RUNNING
+ 2. A task that satisfies (1) and is already known has the same task id.
+ """
+ def __init__(self):
+ self._task_ids = {}
+
+ def health(self, task):
+ task_id = task.assignedTask.taskId
+ instance_id = task.assignedTask.instanceId
+ status = task.status
+
+ if status == ScheduleStatus.RUNNING:
+ if instance_id in self._task_ids:
+ return Retriable.alive() if task_id == self._task_ids.get(instance_id) else NotRetriable.dead()
+ else:
+ log.info('Detected RUNNING instance %s' % instance_id)
+ self._task_ids[instance_id] = task_id
+ return Retriable.alive()
+ else:
+ return Retriable.dead()
+
+
+class HttpHealthCheck(HealthCheck):
+ """Verifies the health of a task based on http health checks. A new http signaler is created for a
+ task iff,
+ 1. The instance id of the task is unknown.
+ 2. The instance id is known but the (host, port) is different for the task.
+ """
+ def __init__(self, http_signaler_factory=HttpSignaler):
+ self._http_signalers = {}
+ self._http_signaler_factory = http_signaler_factory
+
+ def health(self, task):
+ assigned_task = task.assignedTask
+ instance_id = assigned_task.instanceId
+ host_port = (assigned_task.slaveHost, assigned_task.assignedPorts['health'])
+ http_signaler = None
+ if instance_id in self._http_signalers:
+ checker_host_port, signaler = self._http_signalers.get(instance_id)
+ # Only reuse the health checker if it is for the same destination.
+ if checker_host_port == host_port:
+ http_signaler = signaler
+ if not http_signaler:
+ http_signaler = self._http_signaler_factory(host_port[1], host_port[0])
+ self._http_signalers[instance_id] = (host_port, http_signaler)
+ return Retriable.alive() if http_signaler.health()[0] else Retriable.dead()
+
+
+class ChainedHealthCheck(HealthCheck):
+ """Delegates health checks to configured health checkers."""
+ def __init__(self, *health_checkers):
+ self._health_checkers = health_checkers
+
+ def health(self, task):
+ for checker in self._health_checkers:
+ healthy, retriable = checker.health(task)
+ if not healthy:
+ return (healthy, retriable)
+ return Retriable.alive()
+
+
+class InstanceWatcherHealthCheck(HealthCheck):
+ """Makes the decision: if a task has health port, then use Status+HTTP, else use only status.
+ Caveat: Only works if either ALL tasks have a health port or none of them have a health port.
+ """
+ # TODO(atollenaere) Refactor the code to use the executor StatusChecker/HealthChecker instead
+
+ def __init__(self, http_signaler_factory=HttpSignaler):
+ self._has_health_port = False
+ self._health_checker = StatusHealthCheck()
+ self._http_signaler_factory = http_signaler_factory
+
+ def health(self, task):
+ if not self._has_health_port and 'health' in task.assignedTask.assignedPorts:
+ log.debug('Health port detected, enabling HTTP checks')
+ self._health_checker = ChainedHealthCheck(self._health_checker, HttpHealthCheck(self._http_signaler_factory))
+ self._has_health_port = True
+ return self._health_checker.health(task)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/instance_watcher.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/instance_watcher.py b/src/main/python/apache/aurora/client/api/instance_watcher.py
new file mode 100644
index 0000000..5c69487
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/instance_watcher.py
@@ -0,0 +1,134 @@
+import time
+
+from twitter.common import log
+from .health_check import InstanceWatcherHealthCheck
+
+from gen.twitter.aurora.ttypes import (
+ Identity,
+ ResponseCode,
+ ScheduleStatus,
+ TaskQuery,
+)
+
+class Instance(object):
+ def __init__(self, birthday=None, finished=False):
+ self.birthday = birthday
+ self.finished = finished
+ self.healthy = False
+
+ def set_healthy(self, value):
+ self.healthy = value
+ self.finished = True
+
+ def __str__(self):
+ return ('[birthday=%s, healthy=%s, finished=%s]' % (self.birthday, self.healthy, self.finished))
+
+
+class InstanceWatcher(object):
+ def __init__(self,
+ scheduler,
+ job_key,
+ restart_threshold,
+ watch_secs,
+ health_check_interval_seconds,
+ clock=time):
+
+ self._scheduler = scheduler
+ self._job_key = job_key
+ self._restart_threshold = restart_threshold
+ self._watch_secs = watch_secs
+ self._health_check_interval_seconds = health_check_interval_seconds
+ self._clock = clock
+
+ def watch(self, instance_ids, health_check=None):
+ """Watches a set of instances and detects failures based on a delegated health check.
+
+ Arguments:
+ instance_ids -- set of instances to watch.
+
+ Returns a set of instances that are considered failed.
+ """
+ log.info('Watching instances: %s' % instance_ids)
+ instance_ids = set(instance_ids)
+ health_check = health_check or InstanceWatcherHealthCheck()
+ now = self._clock.time()
+ expected_healthy_by = now + self._restart_threshold
+ max_time = now + self._restart_threshold + self._watch_secs
+
+ instance_states = {}
+
+ def finished_instances():
+ return dict((s_id, s) for s_id, s in instance_states.items() if s.finished)
+
+ def set_instance_healthy(instance_id, now):
+ if instance_id not in instance_states:
+ instance_states[instance_id] = Instance(now)
+ instance = instance_states.get(instance_id)
+ if now > (instance.birthday + self._watch_secs):
+ log.info('Instance %s has been up and healthy for at least %d seconds' % (
+ instance_id, self._watch_secs))
+ instance.set_healthy(True)
+
+ def maybe_set_instance_unhealthy(instance_id, retriable):
+ # An instance that was previously healthy and currently unhealthy has failed.
+ if instance_id in instance_states:
+ log.info('Instance %s is unhealthy' % instance_id)
+ instance_states[instance_id].set_healthy(False)
+ # If the restart threshold has expired or if the instance cannot be retried it is unhealthy.
+ elif now > expected_healthy_by or not retriable:
+ log.info('Instance %s was not reported healthy within %d seconds' % (
+ instance_id, self._restart_threshold))
+ instance_states[instance_id] = Instance(finished=True)
+
+ while True:
+ running_tasks = self._get_tasks_by_instance_id(instance_ids)
+ now = self._clock.time()
+ tasks_by_instance = dict((task.assignedTask.instanceId, task) for task in running_tasks)
+ for instance_id in instance_ids:
+ if instance_id not in finished_instances():
+ running_task = tasks_by_instance.get(instance_id)
+ if running_task is not None:
+ task_healthy, retriable = health_check.health(running_task)
+ if task_healthy:
+ set_instance_healthy(instance_id, now)
+ else:
+ maybe_set_instance_unhealthy(instance_id, retriable)
+ else:
+ # Set retriable=True since an instance should be retried if it has not been healthy.
+ maybe_set_instance_unhealthy(instance_id, retriable=True)
+
+ log.debug('Instances health: %s' % ['%s: %s' % val for val in instance_states.items()])
+
+ # Return if all tasks are finished.
+ if set(finished_instances().keys()) == instance_ids:
+ return set([s_id for s_id, s in instance_states.items() if not s.healthy])
+
+ # Return if time is up.
+ if now > max_time:
+ return set([s_id for s_id in instance_ids if s_id not in instance_states
+ or not instance_states[s_id].healthy])
+
+ self._clock.sleep(self._health_check_interval_seconds)
+
+ def _get_tasks_by_instance_id(self, instance_ids):
+ log.debug('Querying instance statuses.')
+ query = TaskQuery()
+ query.owner = Identity(role=self._job_key.role)
+ query.environment = self._job_key.environment
+ query.jobName = self._job_key.name
+ query.statuses = set([ScheduleStatus.RUNNING])
+
+ query.instanceIds = instance_ids
+ try:
+ resp = self._scheduler.getTasksStatus(query)
+ except IOError as e:
+ log.error('IO Exception during scheduler call: %s' % e)
+ return []
+
+ tasks = []
+ if resp.responseCode == ResponseCode.OK:
+ tasks = resp.result.scheduleStatusResult.tasks
+
+ log.debug('Response from scheduler: %s (message: %s)'
+ % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
+ return tasks
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/job_monitor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/job_monitor.py b/src/main/python/apache/aurora/client/api/job_monitor.py
new file mode 100644
index 0000000..45edd1a
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/job_monitor.py
@@ -0,0 +1,63 @@
+import time
+
+from twitter.common.quantity import Amount, Time
+
+from gen.twitter.aurora.constants import (
+ LIVE_STATES,
+ TERMINAL_STATES
+)
+from gen.twitter.aurora.ttypes import (
+ Identity,
+ TaskQuery
+)
+
+from thrift.transport import TTransport
+
+
+class JobMonitor(object):
+ MIN_POLL_INTERVAL = Amount(10, Time.SECONDS)
+ MAX_POLL_INTERVAL = Amount(2, Time.MINUTES)
+
+ @classmethod
+ def running_or_finished(cls, status):
+ return status in (LIVE_STATES | TERMINAL_STATES)
+
+ @classmethod
+ def terminal(cls, status):
+ return status in TERMINAL_STATES
+
+ # TODO(ksweeney): Make this use the AuroraJobKey
+ def __init__(self, client, role, env, jobname):
+ self._client = client
+ self._query = TaskQuery(owner=Identity(role=role), environment=env, jobName=jobname)
+ self._initial_tasks = set()
+ self._initial_tasks = set(task.assignedTask.taskId for task in self.iter_query())
+
+ def iter_query(self):
+ try:
+ res = self._client.scheduler.getTasksStatus(self._query)
+ except TTransport.TTransportException as e:
+ print('Failed to query slaves from scheduler: %s' % e)
+ return
+ if res is None or res.result is None:
+ return
+ for task in res.result.scheduleStatusResult.tasks:
+ if task.assignedTask.taskId not in self._initial_tasks:
+ yield task
+
+ def states(self):
+ states = {}
+ for task in self.iter_query():
+ status, instance_id = task.status, task.assignedTask.instanceId
+ first_timestamp = task.taskEvents[0].timestamp
+ if instance_id not in states or first_timestamp > states[instance_id][0]:
+ states[instance_id] = (first_timestamp, status)
+ return dict((instance_id, status[1]) for (instance_id, status) in states.items())
+
+ def wait_until(self, predicate):
+ """Given a predicate (from ScheduleStatus => Boolean), return once all tasks
+ return true for that predicate."""
+ poll_interval = self.MIN_POLL_INTERVAL
+ while not all(predicate(state) for state in self.states().values()):
+ time.sleep(poll_interval.as_(Time.SECONDS))
+ poll_interval = min(self.MAX_POLL_INTERVAL, 2 * poll_interval)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/restarter.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/restarter.py b/src/main/python/apache/aurora/client/api/restarter.py
new file mode 100644
index 0000000..bf196e1
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/restarter.py
@@ -0,0 +1,73 @@
+from twitter.common import log
+
+from gen.twitter.aurora.constants import ACTIVE_STATES
+from gen.twitter.aurora.ttypes import ResponseCode
+
+from .instance_watcher import InstanceWatcher
+from .updater_util import FailureThreshold
+
+
+class Restarter(object):
+ def __init__(self,
+ job_key,
+ update_config,
+ health_check_interval_seconds,
+ scheduler,
+ instance_watcher=None,
+ lock=None):
+ self._job_key = job_key
+ self._update_config = update_config
+ self.health_check_interval_seconds = health_check_interval_seconds
+ self._scheduler = scheduler
+ self._lock = lock
+ self._instance_watcher = instance_watcher or InstanceWatcher(
+ scheduler,
+ job_key.to_thrift(),
+ update_config.restart_threshold,
+ update_config.watch_secs,
+ health_check_interval_seconds)
+
+ def restart(self, instances):
+ failure_threshold = FailureThreshold(
+ self._update_config.max_per_instance_failures,
+ self._update_config.max_total_failures)
+
+ if not instances:
+ query = self._job_key.to_thrift_query()
+ query.statuses = ACTIVE_STATES
+ status = self._scheduler.getTasksStatus(query)
+
+ if status.responseCode != ResponseCode.OK:
+ return status
+
+ tasks = status.result.scheduleStatusResult.tasks
+
+ instances = sorted(task.assignedTask.instanceId for task in tasks)
+ if not instances:
+ log.info("No instances specified, and no active instances found in job %s" % self._job_key)
+ log.info("Nothing to do.")
+ return status
+
+ log.info("Performing rolling restart of job %s (instances: %s)" % (self._job_key, instances))
+
+ while instances and not failure_threshold.is_failed_update():
+ batch = instances[:self._update_config.batch_size]
+ instances = instances[self._update_config.batch_size:]
+
+ log.info("Restarting instances: %s", batch)
+
+ resp = self._scheduler.restartShards(self._job_key.to_thrift(), batch, self._lock)
+ if resp.responseCode != ResponseCode.OK:
+ log.error('Error restarting instances: %s', resp.message)
+ return resp
+
+ failed_instances = self._instance_watcher.watch(batch)
+ instances += failed_instances
+ failure_threshold.update_failure_counts(failed_instances)
+
+ if failure_threshold.is_failed_update():
+ log.info("Restart failures threshold reached. Aborting")
+ else:
+ log.info("All instances were restarted successfully")
+
+ return resp
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/scheduler_client.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/scheduler_client.py b/src/main/python/apache/aurora/client/api/scheduler_client.py
new file mode 100644
index 0000000..ffec604
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/scheduler_client.py
@@ -0,0 +1,257 @@
+import functools
+import threading
+import time
+
+from twitter.aurora.common.auth import make_session_key, SessionKeyError
+from twitter.aurora.common.cluster import Cluster
+from twitter.common import log
+from twitter.common.quantity import Amount, Time
+from twitter.common.rpc.transports.tsslsocket import DelayedHandshakeTSSLSocket
+from twitter.common.zookeeper.kazoo_client import TwitterKazooClient
+from twitter.common.zookeeper.serverset import ServerSet
+
+from gen.twitter.aurora import AuroraAdmin
+from gen.twitter.aurora.constants import CURRENT_API_VERSION
+
+from thrift.protocol import TBinaryProtocol
+from thrift.transport import TSocket, TTransport
+from pystachio import Boolean, Default, Integer, String
+
+
+class SchedulerClientTrait(Cluster.Trait):
+ zk = String
+ zk_port = Default(Integer, 2181)
+ scheduler_zk_path = String
+ scheduler_uri = String
+ proxy_url = String
+ auth_mechanism = Default(String, 'UNAUTHENTICATED')
+ use_thrift_ssl = Default(Boolean, False)
+
+
+class SchedulerClient(object):
+ THRIFT_RETRIES = 5
+ RETRY_TIMEOUT = Amount(1, Time.SECONDS)
+
+ class CouldNotConnect(Exception): pass
+
+ # TODO(wickman) Refactor per MESOS-3005 into two separate classes with separate traits:
+ # ZookeeperClientTrait
+ # DirectClientTrait
+ @classmethod
+ def get(cls, cluster, **kwargs):
+ if not isinstance(cluster, Cluster):
+ raise TypeError('"cluster" must be an instance of Cluster, got %s' % type(cluster))
+ cluster = cluster.with_trait(SchedulerClientTrait)
+ if cluster.zk:
+ return ZookeeperSchedulerClient(
+ cluster, port=cluster.zk_port, ssl=cluster.use_thrift_ssl, **kwargs)
+ elif cluster.scheduler_uri:
+ try:
+ host, port = cluster.scheduler_uri.split(':', 2)
+ port = int(port)
+ except ValueError:
+ raise ValueError('Malformed Cluster scheduler_uri: %s' % cluster.scheduler_uri)
+ return DirectSchedulerClient(host, port, ssl=cluster.use_thrift_ssl)
+ else:
+ raise ValueError('"cluster" does not specify zk or scheduler_uri')
+
+ def __init__(self, verbose=False, ssl=False):
+ self._client = None
+ self._verbose = verbose
+ self._ssl = ssl
+
+ def get_thrift_client(self):
+ if self._client is None:
+ self._client = self._connect()
+ return self._client
+
+ # per-class implementation -- mostly meant to set up a valid host/port
+ # pair and then delegate the opening to SchedulerClient._connect_scheduler
+ def _connect(self):
+ return None
+
+ @staticmethod
+ def _connect_scheduler(host, port, with_ssl=False):
+ if with_ssl:
+ socket = DelayedHandshakeTSSLSocket(host, port, delay_handshake=True, validate=False)
+ else:
+ socket = TSocket.TSocket(host, port)
+ transport = TTransport.TBufferedTransport(socket)
+ protocol = TBinaryProtocol.TBinaryProtocol(transport)
+ schedulerClient = AuroraAdmin.Client(protocol)
+ for _ in range(SchedulerClient.THRIFT_RETRIES):
+ try:
+ transport.open()
+ return schedulerClient
+ except TTransport.TTransportException:
+ time.sleep(SchedulerClient.RETRY_TIMEOUT.as_(Time.SECONDS))
+ continue
+ except Exception as e:
+ # Monkey-patched proxies, like socks, can generate a proxy error here.
+ # without adding a dependency, we can't catch those in a more specific way.
+ raise SchedulerClient.CouldNotConnect('Connection to scheduler failed: %s' % e)
+ raise SchedulerClient.CouldNotConnect('Could not connect to %s:%s' % (host, port))
+
+
+class ZookeeperSchedulerClient(SchedulerClient):
+ SERVERSET_TIMEOUT = Amount(10, Time.SECONDS)
+
+ @classmethod
+ def get_scheduler_serverset(cls, cluster, port=2181, verbose=False, **kw):
+ if cluster.zk is None:
+ raise ValueError('Cluster has no associated zookeeper ensemble!')
+ if cluster.scheduler_zk_path is None:
+ raise ValueError('Cluster has no defined scheduler path, must specify scheduler_zk_path '
+ 'in your cluster config!')
+ zk = TwitterKazooClient.make(str('%s:%s' % (cluster.zk, port)), verbose=verbose)
+ return zk, ServerSet(zk, cluster.scheduler_zk_path, **kw)
+
+ def __init__(self, cluster, port=2181, ssl=False, verbose=False):
+ SchedulerClient.__init__(self, verbose=verbose, ssl=ssl)
+ self._cluster = cluster
+ self._zkport = port
+ self._endpoint = None
+
+ def _connect(self):
+ joined = threading.Event()
+ def on_join(elements):
+ joined.set()
+ zk, serverset = self.get_scheduler_serverset(self._cluster, verbose=self._verbose,
+ port=self._zkport, on_join=on_join)
+ joined.wait(timeout=self.SERVERSET_TIMEOUT.as_(Time.SECONDS))
+ serverset_endpoints = list(serverset)
+ if len(serverset_endpoints) == 0:
+ raise self.CouldNotConnect('No schedulers detected in %s!' % self._cluster.name)
+ instance = serverset_endpoints[0]
+ self._endpoint = instance.service_endpoint
+ self._http = instance.additional_endpoints.get('http')
+ zk.stop()
+ return self._connect_scheduler(self._endpoint.host, self._endpoint.port, self._ssl)
+
+ @property
+ def url(self):
+ proxy_url = self._cluster.proxy_url
+ if proxy_url:
+ return proxy_url
+ if self._http:
+ return 'http://%s:%s' % (self._http.host, self._http.port)
+
+
+class DirectSchedulerClient(SchedulerClient):
+ def __init__(self, host, port, ssl=False):
+ SchedulerClient.__init__(self, verbose=True, ssl=ssl)
+ self._host = host
+ self._port = port
+
+ def _connect(self):
+ return self._connect_scheduler(self._host, self._port, with_ssl=self._ssl)
+
+ @property
+ def url(self):
+ # TODO(wickman) This is broken -- make this tunable in MESOS-3005
+ return 'http://%s:8081' % self._host
+
+
+class SchedulerProxy(object):
+ """
+ This class is responsible for creating a reliable thrift client to the
+ twitter scheduler. Basically all the dirty work needed by the
+ AuroraClientAPI.
+ """
+ CONNECT_MAXIMUM_WAIT = Amount(1, Time.MINUTES)
+ RPC_RETRY_INTERVAL = Amount(5, Time.SECONDS)
+ RPC_MAXIMUM_WAIT = Amount(10, Time.MINUTES)
+ UNAUTHENTICATED_RPCS = frozenset([
+ 'populateJobConfig',
+ 'getTasksStatus',
+ 'getJobs',
+ 'getQuota',
+ 'getVersion',
+ ])
+
+ class Error(Exception): pass
+ class TimeoutError(Error): pass
+ class AuthenticationError(Error): pass
+ class APIVersionError(Error): pass
+
+ def __init__(self, cluster, verbose=False, session_key_factory=make_session_key):
+ """A callable session_key_factory should be provided for authentication"""
+ self.cluster = cluster
+ # TODO(Sathya): Make this a part of cluster trait when authentication is pushed to the transport
+ # layer.
+ self._session_key_factory = session_key_factory
+ self._client = self._scheduler = None
+ self.verbose = verbose
+
+ def with_scheduler(method):
+ """Decorator magic to make sure a connection is made to the scheduler"""
+ def _wrapper(self, *args, **kwargs):
+ if not self._scheduler:
+ self._construct_scheduler()
+ return method(self, *args, **kwargs)
+ return _wrapper
+
+ def invalidate(self):
+ self._client = self._scheduler = None
+
+ @with_scheduler
+ def client(self):
+ return self._client
+
+ @with_scheduler
+ def scheduler(self):
+ return self._scheduler
+
+ def session_key(self):
+ try:
+ return self._session_key_factory(self.cluster.auth_mechanism)
+ except SessionKeyError as e:
+ raise self.AuthenticationError('Unable to create session key %s' % e)
+
+ def _construct_scheduler(self):
+ """
+ Populates:
+ self._scheduler
+ self._client
+ """
+ self._scheduler = SchedulerClient.get(self.cluster, verbose=self.verbose)
+ assert self._scheduler, "Could not find scheduler (cluster = %s)" % self.cluster.name
+ start = time.time()
+ while (time.time() - start) < self.CONNECT_MAXIMUM_WAIT.as_(Time.SECONDS):
+ try:
+ self._client = self._scheduler.get_thrift_client()
+ break
+ except SchedulerClient.CouldNotConnect as e:
+ log.warning('Could not connect to scheduler: %s' % e)
+ if not self._client:
+ raise self.TimeoutError('Timed out trying to connect to scheduler at %s' % self.cluster.name)
+
+ server_version = self._client.getVersion().result.getVersionResult
+ if server_version != CURRENT_API_VERSION:
+ raise self.APIVersionError("Client Version: %s, Server Version: %s" %
+ (CURRENT_API_VERSION, server_version))
+
+ def __getattr__(self, method_name):
+ # If the method does not exist, getattr will return AttributeError for us.
+ method = getattr(AuroraAdmin.Client, method_name)
+ if not callable(method):
+ return method
+
+ @functools.wraps(method)
+ def method_wrapper(*args):
+ start = time.time()
+ while (time.time() - start) < self.RPC_MAXIMUM_WAIT.as_(Time.SECONDS):
+ auth_args = () if method_name in self.UNAUTHENTICATED_RPCS else (self.session_key(),)
+ try:
+ method = getattr(self.client(), method_name)
+ if not callable(method):
+ return method
+ return method(*(args + auth_args))
+ except (TTransport.TTransportException, self.TimeoutError) as e:
+ log.warning('Connection error with scheduler: %s, reconnecting...' % e)
+ self.invalidate()
+ time.sleep(self.RPC_RETRY_INTERVAL.as_(Time.SECONDS))
+ raise self.TimeoutError('Timed out attempting to issue %s to %s' % (
+ method_name, self.cluster.name))
+
+ return method_wrapper
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/updater.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/updater.py b/src/main/python/apache/aurora/client/api/updater.py
new file mode 100644
index 0000000..6e8e8a9
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/updater.py
@@ -0,0 +1,410 @@
+from collections import namedtuple
+from difflib import unified_diff
+
+from twitter.common import log
+
+from gen.twitter.aurora.constants import ACTIVE_STATES
+
+from gen.twitter.aurora.ttypes import (
+ AddInstancesConfig,
+ JobConfigValidation,
+ JobKey,
+ Identity,
+ Lock,
+ LockKey,
+ LockValidation,
+ Response,
+ ResponseCode,
+ TaskQuery,
+)
+from .updater_util import FailureThreshold, UpdaterConfig
+from .instance_watcher import InstanceWatcher
+from .scheduler_client import SchedulerProxy
+
+
+class Updater(object):
+ """Update the instances of a job in batches."""
+
+ class Error(Exception): pass
+
+ InstanceState = namedtuple('InstanceState', ['instance_id', 'is_updated'])
+ OperationConfigs = namedtuple('OperationConfigs', ['from_config', 'to_config'])
+ InstanceConfigs = namedtuple(
+ 'InstanceConfigs',
+ ['remote_config_map', 'local_config_map', 'instances_to_process']
+ )
+
+ def __init__(self, config, health_check_interval_seconds, scheduler=None, instance_watcher=None):
+ self._config = config
+ self._job_key = JobKey(role=config.role(), environment=config.environment(), name=config.name())
+ self._health_check_interval_seconds = health_check_interval_seconds
+ self._scheduler = scheduler or SchedulerProxy(config.cluster())
+ try:
+ self._update_config = UpdaterConfig(**config.update_config().get())
+ except ValueError as e:
+ raise self.Error(str(e))
+ self._lock = None
+ self._watcher = instance_watcher or InstanceWatcher(
+ self._scheduler,
+ self._job_key,
+ self._update_config.restart_threshold,
+ self._update_config.watch_secs,
+ self._health_check_interval_seconds)
+
+ def _start(self):
+ """Starts an update by applying an exclusive lock on a job being updated.
+
+ Returns:
+ Response instance from the scheduler call.
+ """
+ resp = self._scheduler.acquireLock(LockKey(job=self._job_key))
+ if resp.responseCode == ResponseCode.OK:
+ self._lock = resp.result.acquireLockResult.lock
+ return resp
+
+ def _finish(self):
+ """Finishes an update by removing an exclusive lock on an updated job.
+
+ Returns:
+ Response instance from the scheduler call.
+ """
+ resp = self._scheduler.releaseLock(self._lock, LockValidation.CHECKED)
+
+ if resp.responseCode == ResponseCode.OK:
+ self._lock = None
+ else:
+ log.error('There was an error finalizing the update: %s' % resp.message)
+ return resp
+
+ def _update(self, instance_configs):
+ """Drives execution of the update logic. Performs a batched update/rollback for all instances
+ affected by the current update request.
+
+ Arguments:
+ instance_configs -- list of instance update configurations to go through.
+
+ Returns the set of instances that failed to update.
+ """
+ failure_threshold = FailureThreshold(
+ self._update_config.max_per_instance_failures,
+ self._update_config.max_total_failures
+ )
+
+ instance_operation = self.OperationConfigs(
+ from_config=instance_configs.remote_config_map,
+ to_config=instance_configs.local_config_map
+ )
+
+ remaining_instances = [
+ self.InstanceState(instance_id, is_updated=False)
+ for instance_id in instance_configs.instances_to_process
+ ]
+
+ log.info('Starting job update.')
+ while remaining_instances and not failure_threshold.is_failed_update():
+ batch_instances = remaining_instances[0 : self._update_config.batch_size]
+ remaining_instances = list(set(remaining_instances) - set(batch_instances))
+ instances_to_restart = [s.instance_id for s in batch_instances if s.is_updated]
+ instances_to_update = [s.instance_id for s in batch_instances if not s.is_updated]
+
+ instances_to_watch = []
+ if instances_to_restart:
+ instances_to_watch += self._restart_instances(instances_to_restart)
+
+ if instances_to_update:
+ instances_to_watch += self._update_instances(instances_to_update, instance_operation)
+
+ failed_instances = self._watcher.watch(instances_to_watch) if instances_to_watch else set()
+
+ if failed_instances:
+ log.error('Failed instances: %s' % failed_instances)
+
+ unretryable_instances = failure_threshold.update_failure_counts(failed_instances)
+ if unretryable_instances:
+ log.warn('Not restarting failed instances %s, which exceeded '
+ 'maximum allowed instance failure limit of %s' %
+ (unretryable_instances, self._update_config.max_per_instance_failures))
+ retryable_instances = list(set(failed_instances) - set(unretryable_instances))
+ remaining_instances += [
+ self.InstanceState(instance_id, is_updated=True) for instance_id in retryable_instances
+ ]
+ remaining_instances.sort(key=lambda tup: tup.instance_id)
+
+ if failure_threshold.is_failed_update():
+ untouched_instances = [s.instance_id for s in remaining_instances if not s.is_updated]
+ instances_to_rollback = list(
+ set(instance_configs.instances_to_process) - set(untouched_instances)
+ )
+ self._rollback(instances_to_rollback, instance_configs)
+
+ return not failure_threshold.is_failed_update()
+
+ def _rollback(self, instances_to_rollback, instance_configs):
+ """Performs a rollback operation for the failed instances.
+
+ Arguments:
+ instances_to_rollback -- instance ids to rollback.
+ instance_configs -- instance configuration to use for rollback.
+ """
+ log.info('Reverting update for %s' % instances_to_rollback)
+ instance_operation = self.OperationConfigs(
+ from_config=instance_configs.local_config_map,
+ to_config=instance_configs.remote_config_map
+ )
+ instances_to_rollback.sort(reverse=True)
+ failed_instances = []
+ while instances_to_rollback:
+ batch_instances = instances_to_rollback[0 : self._update_config.batch_size]
+ instances_to_rollback = list(set(instances_to_rollback) - set(batch_instances))
+ instances_to_rollback.sort(reverse=True)
+ instances_to_watch = self._update_instances(batch_instances, instance_operation)
+ failed_instances += self._watcher.watch(instances_to_watch)
+
+ if failed_instances:
+ log.error('Rollback failed for instances: %s' % failed_instances)
+
+ def _create_kill_add_lists(self, instance_ids, operation_configs):
+ """Determines a particular action (kill or add) to use for every instance in instance_ids.
+
+ Arguments:
+ instance_ids -- current batch of IDs to process.
+ operation_configs -- OperationConfigs with update details.
+
+ Returns lists of instances to kill and to add.
+ """
+ to_kill = []
+ to_add = []
+ for instance_id in instance_ids:
+ from_config = operation_configs.from_config.get(instance_id)
+ to_config = operation_configs.to_config.get(instance_id)
+
+ if from_config and to_config:
+ # Sort internal dicts before comparing to rule out differences due to hashing.
+ diff_output = ''.join(unified_diff(
+ str(sorted(from_config.__dict__.items(), key=lambda x: x[0])),
+ str(sorted(to_config.__dict__.items(), key=lambda x: x[0]))))
+ if diff_output:
+ log.debug('Task configuration changed for instance [%s]:\n%s' % (instance_id, diff_output))
+ to_kill.append(instance_id)
+ to_add.append(instance_id)
+ elif from_config and not to_config:
+ to_kill.append(instance_id)
+ elif not from_config and to_config:
+ to_add.append(instance_id)
+ else:
+ raise self.Error('Instance %s is outside of supported range' % instance_id)
+
+ return to_kill, to_add
+
+ def _update_instances(self, instance_ids, operation_configs):
+ """Applies kill/add actions for the specified batch instances.
+
+ Arguments:
+ instance_ids -- current batch of IDs to process.
+ operation_configs -- OperationConfigs with update details.
+
+ Returns a list of added instances.
+ """
+ log.info('Examining instances: %s' % instance_ids)
+
+ to_kill, to_add = self._create_kill_add_lists(instance_ids, operation_configs)
+
+ unchanged = list(set(instance_ids) - set(to_kill + to_add))
+ if unchanged:
+ log.info('Skipping unchanged instances: %s' % unchanged)
+
+ # Kill is a blocking call in scheduler -> no need to watch it yet.
+ self._kill_instances(to_kill)
+ self._add_instances(to_add, operation_configs.to_config)
+ return to_add
+
+ def _kill_instances(self, instance_ids):
+ """Instructs the scheduler to kill instances and waits for completion.
+
+ Arguments:
+ instance_ids -- list of IDs to kill.
+ """
+ if instance_ids:
+ log.info('Killing instances: %s' % instance_ids)
+ query = self._create_task_query(instanceIds=frozenset(int(s) for s in instance_ids))
+ self._check_and_log_response(self._scheduler.killTasks(query, self._lock))
+ log.info('Instances killed')
+
+ def _add_instances(self, instance_ids, to_config):
+ """Instructs the scheduler to add instances.
+
+ Arguments:
+ instance_ids -- list of IDs to add.
+ to_config -- OperationConfigs with update details.
+ """
+ if instance_ids:
+ log.info('Adding instances: %s' % instance_ids)
+ add_config = AddInstancesConfig(
+ key=self._job_key,
+ taskConfig=to_config[instance_ids[0]], # instance_ids will always have at least 1 item.
+ instanceIds=frozenset(int(s) for s in instance_ids))
+ self._check_and_log_response(self._scheduler.addInstances(add_config, self._lock))
+ log.info('Instances added')
+
+ def _restart_instances(self, instance_ids):
+ """Instructs the scheduler to restart instances.
+
+ Arguments:
+ instance_ids -- set of instances to be restarted by the scheduler.
+ """
+ log.info('Restarting instances: %s' % instance_ids)
+ resp = self._scheduler.restartShards(self._job_key, instance_ids, self._lock)
+ self._check_and_log_response(resp)
+ return instance_ids
+
+ def _get_update_instructions(self, instances=None):
+ """Loads, validates and populates update working set.
+
+ Arguments:
+ instances -- (optional) set of instances to update.
+
+ Returns:
+ InstanceConfigs with the following data:
+ remote_config_map -- dictionary of {key:instance_id, value:task_config} from scheduler.
+ local_config_map -- dictionary of {key:instance_id, value:task_config} with local
+ task configs validated and populated with default values.
+ instances_to_process -- list of instance IDs to go over in update.
+ """
+ # Load existing tasks and populate remote config map and instance list.
+ assigned_tasks = self._get_existing_tasks()
+ remote_config_map = {}
+ remote_instances = []
+ for assigned_task in assigned_tasks:
+ remote_config_map[assigned_task.instanceId] = assigned_task.task
+ remote_instances.append(assigned_task.instanceId)
+
+ # Validate local job config and populate local task config.
+ local_task_config = self._validate_and_populate_local_config()
+
+ # Union of local and remote instance IDs.
+ job_config_instances = list(range(self._config.instances()))
+ instance_superset = sorted(list(set(remote_instances) | set(job_config_instances)))
+
+ # Calculate the update working set.
+ if instances is None:
+ # Full job update -> union of remote and local instances
+ instances_to_process = instance_superset
+ else:
+ # Partial job update -> validate all instances are recognized
+ instances_to_process = instances
+ unrecognized = list(set(instances) - set(instance_superset))
+ if unrecognized:
+ raise self.Error('Instances %s are outside of supported range' % unrecognized)
+
+ # Populate local config map
+ local_config_map = dict.fromkeys(job_config_instances, local_task_config)
+
+ return self.InstanceConfigs(remote_config_map, local_config_map, instances_to_process)
+
+ def _get_existing_tasks(self):
+ """Loads all existing tasks from the scheduler.
+
+ Returns a list of AssignedTasks.
+ """
+ resp = self._scheduler.getTasksStatus(self._create_task_query())
+ self._check_and_log_response(resp)
+ return [t.assignedTask for t in resp.result.scheduleStatusResult.tasks]
+
+ def _validate_and_populate_local_config(self):
+ """Validates local job configuration and populates local task config with default values.
+
+ Returns a TaskConfig populated with default values.
+ """
+ resp = self._scheduler.populateJobConfig(self._config.job(), JobConfigValidation.RUN_FILTERS)
+ self._check_and_log_response(resp)
+
+ # Safe to take the first element as Scheduler would throw in case zero instances provided.
+ return list(resp.result.populateJobResult.populated)[0]
+
+ def _replace_template_if_cron(self):
+ """Checks if the provided job config represents a cron job and if so, replaces it.
+
+ Returns True if job is cron and False otherwise.
+ """
+ if self._config.job().cronSchedule:
+ resp = self._scheduler.replaceCronTemplate(self._config.job(), self._lock)
+ self._check_and_log_response(resp)
+ return True
+ else:
+ return False
+
+ def _create_task_query(self, instanceIds=None):
+ return TaskQuery(
+ owner=Identity(role=self._job_key.role),
+ environment=self._job_key.environment,
+ jobName=self._job_key.name,
+ statuses=ACTIVE_STATES,
+ instanceIds=instanceIds)
+
+ def _failed_response(self, message):
+ return Response(responseCode=ResponseCode.ERROR, message=message)
+
+ def update(self, instances=None):
+ """Performs the job update, blocking until it completes.
+ A rollback will be performed if the update was considered a failure based on the
+ update configuration.
+
+ Arguments:
+ instances -- (optional) instances to update. If not specified, all instances will be updated.
+
+ Returns a response object with update result status.
+ """
+ resp = self._start()
+ if resp.responseCode != ResponseCode.OK:
+ return resp
+
+ try:
+ # Handle cron jobs separately from other jobs.
+ if self._replace_template_if_cron():
+ log.info('Cron template updated, next run will reflect changes')
+ return self._finish()
+ else:
+ try:
+ instance_configs = self._get_update_instructions(instances)
+ except self.Error as e:
+ # Safe to release the lock acquired above as no job mutation has happened yet.
+ self._finish()
+ return self._failed_response('Unable to start job update: %s' % e)
+
+ if not self._update(instance_configs):
+ log.warn('Update failures threshold reached')
+ self._finish()
+ return self._failed_response('Update reverted')
+ else:
+ log.info('Update successful')
+ return self._finish()
+ except self.Error as e:
+ return self._failed_response('Aborting update without rollback! Fatal error: %s' % e)
+
+ @classmethod
+ def cancel_update(cls, scheduler, job_key):
+ """Cancels an update process by removing an exclusive lock on a provided job.
+
+ Arguments:
+ scheduler -- scheduler instance to use.
+ job_key -- job key to cancel update for.
+
+ Returns a response object with cancel update result status.
+ """
+ return scheduler.releaseLock(
+ Lock(key=LockKey(job=job_key.to_thrift())),
+ LockValidation.UNCHECKED)
+
+ def _check_and_log_response(self, resp):
+ """Checks scheduler return status, raises Error in case of unexpected response.
+
+ Arguments:
+ resp -- scheduler response object.
+
+ Raises Error in case of unexpected response status.
+ """
+ name, message = ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message
+ if resp.responseCode == ResponseCode.OK:
+ log.debug('Response from scheduler: %s (message: %s)' % (name, message))
+ else:
+ raise self.Error(message)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/api/updater_util.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/api/updater_util.py b/src/main/python/apache/aurora/client/api/updater_util.py
new file mode 100644
index 0000000..db9f053
--- /dev/null
+++ b/src/main/python/apache/aurora/client/api/updater_util.py
@@ -0,0 +1,77 @@
+import collections
+
+from twitter.common import log
+
+class UpdaterConfig(object):
+ """
+ For updates involving a health check,
+
+ UPDATE INSTANCE HEALTHY REMAIN HEALTHY
+ ----------------------------------------|-----------------------|
+ \--------------------------------------/ \----------------------/
+ restart_thresold watch_secs
+
+ When an update is initiated, an instance is expected to be "healthy" before restart_threshold.
+ An instance is also expected to remain healthy for at least watch_secs. If these conditions are
+ not satisfied, the instance is deemed unhealthy.
+ """
+ def __init__(self,
+ batch_size,
+ restart_threshold,
+ watch_secs,
+ max_per_shard_failures,
+ max_total_failures):
+
+ if batch_size <= 0:
+ raise ValueError('Batch size should be greater than 0')
+ if restart_threshold <= 0:
+ raise ValueError('Restart Threshold should be greater than 0')
+ if watch_secs <= 0:
+ raise ValueError('Watch seconds should be greater than 0')
+ self.batch_size = batch_size
+ self.restart_threshold = restart_threshold
+ self.watch_secs = watch_secs
+ self.max_total_failures = max_total_failures
+ self.max_per_instance_failures = max_per_shard_failures
+
+
+class FailureThreshold(object):
+ def __init__(self, max_per_instance_failures, max_total_failures):
+ self._max_per_instance_failures = max_per_instance_failures
+ self._max_total_failures = max_total_failures
+ self._failures_by_instance = collections.defaultdict(int)
+
+
+ def update_failure_counts(self, failed_instances):
+ """Update the failure counts metrics based upon a batch of failed instances.
+
+ Arguments:
+ failed_instances - list of failed instances
+
+ Returns a list of instances with failure counts exceeding _max_per_instance_failures.
+ """
+ exceeded_failure_count_instances = []
+ for instance in failed_instances:
+ self._failures_by_instance[instance] += 1
+ if self._failures_by_instance[instance] > self._max_per_instance_failures:
+ exceeded_failure_count_instances.append(instance)
+
+ return exceeded_failure_count_instances
+
+ def is_failed_update(self):
+ total_failed_instances = self._exceeded_instance_fail_count()
+ is_failed = total_failed_instances > self._max_total_failures
+
+ if is_failed:
+ log.error('%s failed instances observed, maximum allowed is %s' % (total_failed_instances,
+ self._max_total_failures))
+ for instance, failure_count in self._failures_by_instance.items():
+ if failure_count > self._max_per_instance_failures:
+ log.error('%s instance failures for instance %s, maximum allowed is %s' %
+ (failure_count, instance, self._max_per_instance_failures))
+ return is_failed
+
+ def _exceeded_instance_fail_count(self):
+ """Checks if the per instance failure is greater than a threshold."""
+ return sum(count > self._max_per_instance_failures
+ for count in self._failures_by_instance.values())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/base.py b/src/main/python/apache/aurora/client/base.py
new file mode 100644
index 0000000..0936743
--- /dev/null
+++ b/src/main/python/apache/aurora/client/base.py
@@ -0,0 +1,151 @@
+from collections import defaultdict
+import functools
+import sys
+from urlparse import urljoin
+
+from twitter.common import app, log
+
+from gen.twitter.aurora.ttypes import ResponseCode
+
+LOCKED_WARNING = """
+Note: if the scheduler detects that a job update is in progress (or was not
+properly completed) it will reject subsequent updates. This is because your
+job is likely in a partially-updated state. You should only begin another
+update if you are confident that nobody is updating this job, and that
+the job is in a state suitable for an update.
+
+After checking on the above, you may release the update lock on the job by
+invoking cancel_update.
+"""
+
+def die(msg):
+ log.fatal(msg)
+ sys.exit(1)
+
+def check_and_log_response(resp):
+ log.info('Response from scheduler: %s (message: %s)'
+ % (ResponseCode._VALUES_TO_NAMES[resp.responseCode], resp.message))
+ if resp.responseCode != ResponseCode.OK:
+ check_and_log_locked_response(resp)
+ sys.exit(1)
+
+def check_and_log_locked_response(resp):
+ if resp.responseCode == ResponseCode.LOCK_ERROR:
+ log.info(LOCKED_WARNING)
+
+def deprecation_warning(text):
+ log.warning('')
+ log.warning('*' * 80)
+ log.warning('* The command you ran is deprecated and will soon break!')
+ for line in text.split('\n'):
+ log.warning('* %s' % line)
+ log.warning('*' * 80)
+ log.warning('')
+
+
+class requires(object):
+ @staticmethod
+ def wrap_function(fn, fnargs, comparator):
+ @functools.wraps(fn)
+ def wrapped_function(args):
+ if not comparator(args, fnargs):
+ help = 'Incorrect parameters for %s' % fn.__name__
+ if fn.__doc__:
+ help = '%s\n\nsee the help subcommand for more details.' % fn.__doc__.split('\n')[0]
+ die(help)
+ return fn(*args)
+ return wrapped_function
+
+ @staticmethod
+ def exactly(*args):
+ def wrap(fn):
+ return requires.wrap_function(fn, args, (lambda want, got: len(want) == len(got)))
+ return wrap
+
+ @staticmethod
+ def at_least(*args):
+ def wrap(fn):
+ return requires.wrap_function(fn, args, (lambda want, got: len(want) >= len(got)))
+ return wrap
+
+ @staticmethod
+ def nothing(fn):
+ @functools.wraps(fn)
+ def real_fn(line):
+ return fn(*line)
+ return real_fn
+
+
+def synthesize_url(scheduler_url, role=None, env=None, job=None):
+ if not scheduler_url:
+ log.warning("Unable to find scheduler web UI!")
+ return None
+
+ if env and not role:
+ die('If env specified, must specify role')
+ if job and not (role and env):
+ die('If job specified, must specify role and env')
+
+ scheduler_url = urljoin(scheduler_url, 'scheduler')
+ if role:
+ scheduler_url += '/' + role
+ if env:
+ scheduler_url += '/' + env
+ if job:
+ scheduler_url += '/' + job
+ return scheduler_url
+
+
+def handle_open(scheduler_url, role, env, job):
+ url = synthesize_url(scheduler_url, role, env, job)
+ if url:
+ log.info('Job url: %s' % url)
+ if app.get_options().open_browser:
+ import webbrowser
+ webbrowser.open_new_tab(url)
+
+
+def make_commands_str(command_aliases):
+ """Format a string representation of a number of command aliases."""
+ commands = command_aliases[:]
+ commands.sort()
+ if len(commands) == 1:
+ return str(commands[0])
+ elif len(commands) == 2:
+ return '%s (or %s)' % (str(commands[0]), str(commands[1]))
+ else:
+ return '%s (or any of: %s)' % (str(commands[0]), ' '.join(map(str, commands[1:])))
+
+
+# TODO(wickman) This likely belongs in twitter.common.app (or split out as
+# part of a possible twitter.common.cli)
+def generate_full_usage():
+ """Generate verbose application usage from all registered
+ twitter.common.app commands and return as a string."""
+ docs_to_commands = defaultdict(list)
+ for (command, doc) in app.get_commands_and_docstrings():
+ docs_to_commands[doc].append(command)
+ def make_docstring(item):
+ (doc_text, commands) = item
+ def format_line(line):
+ return ' %s\n' % line.lstrip()
+ stripped = ''.join(map(format_line, doc_text.splitlines()))
+ return '%s\n%s' % (make_commands_str(commands), stripped)
+ usage = sorted(map(make_docstring, docs_to_commands.items()))
+ return 'Available commands:\n\n' + '\n'.join(usage)
+
+
+def generate_terse_usage():
+ """Generate minimal application usage from all registered
+ twitter.common.app commands and return as a string."""
+ docs_to_commands = defaultdict(list)
+ for (command, doc) in app.get_commands_and_docstrings():
+ docs_to_commands[doc].append(command)
+ usage = '\n '.join(sorted(map(make_commands_str, docs_to_commands.values())))
+ return """
+Available commands:
+ %s
+
+For more help on an individual command:
+ %s help <command>
+""" % (usage, app.name())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/BUILD b/src/main/python/apache/aurora/client/bin/BUILD
new file mode 100644
index 0000000..7802245
--- /dev/null
+++ b/src/main/python/apache/aurora/client/bin/BUILD
@@ -0,0 +1,25 @@
+python_binary(
+ name = 'aurora_client',
+ source = 'aurora_client.py',
+ entry_point = 'twitter.aurora.client.bin.aurora_client:proxy_main',
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/client/commands:all'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ ]
+)
+
+python_binary(
+ name = 'aurora_admin',
+ source = 'aurora_admin.py',
+ entry_point = 'twitter.aurora.client.bin.aurora_admin:proxy_main',
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/aurora/client/commands:admin'),
+ pants('src/main/python/twitter/aurora/client/commands:help'),
+ pants('src/main/python/twitter/aurora/client:base'),
+ pants('src/main/python/twitter/aurora/client:options'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/__init__.py b/src/main/python/apache/aurora/client/bin/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/aurora/client/bin/aurora_admin.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/bin/aurora_admin.py b/src/main/python/apache/aurora/client/bin/aurora_admin.py
new file mode 100644
index 0000000..53fd882
--- /dev/null
+++ b/src/main/python/apache/aurora/client/bin/aurora_admin.py
@@ -0,0 +1,23 @@
+from twitter.aurora.client.base import generate_terse_usage
+from twitter.aurora.client.commands import admin, help
+from twitter.aurora.client.options import add_verbosity_options
+from twitter.common import app
+from twitter.common.log.options import LogOptions
+
+
+app.register_commands_from(admin, help)
+add_verbosity_options()
+
+
+def main():
+ app.help()
+
+
+LogOptions.set_stderr_log_level('INFO')
+LogOptions.disable_disk_logging()
+app.set_name('aurora-admin')
+app.set_usage(generate_terse_usage())
+
+
+def proxy_main():
+ app.main()
[38/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java b/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java
new file mode 100644
index 0000000..f90869d
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/SchedulerLifecycle.java
@@ -0,0 +1,417 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
+import com.google.common.eventbus.Subscribe;
+import com.google.common.util.concurrent.Atomics;
+
+import org.apache.mesos.Protos;
+import org.apache.mesos.SchedulerDriver;
+
+import com.twitter.aurora.scheduler.events.PubsubEvent.DriverRegistered;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.StorageBackfill;
+import com.twitter.common.application.Lifecycle;
+import com.twitter.common.base.Closure;
+import com.twitter.common.base.Closures;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.StatImpl;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+import com.twitter.common.util.StateMachine;
+import com.twitter.common.util.StateMachine.Transition;
+import com.twitter.common.zookeeper.Group.JoinException;
+import com.twitter.common.zookeeper.ServerSet;
+import com.twitter.common.zookeeper.SingletonService.LeaderControl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.common.zookeeper.SingletonService.LeadershipListener;
+
+/**
+ * The central driver of the scheduler runtime lifecycle. Handles the transitions from startup and
+ * initialization through acting as a standby scheduler / log replica and finally to becoming the
+ * scheduler leader.
+ * <p>
+ * The (enforced) call order to be used with this class is:
+ * <ol>
+ * <li>{@link #prepare()}, to initialize the storage system.</li>
+ * <li>{@link LeadershipListener#onLeading(LeaderControl) onLeading()} on the
+ * {@link LeadershipListener LeadershipListener}
+ * returned from {@link #prepare()}, signaling that this process has exclusive control of the
+ * cluster.</li>
+ * <li>{@link #registered(DriverRegistered) registered()},
+ * indicating that registration with the mesos master has succeeded.
+ * At this point, the scheduler's presence will be announced via
+ * {@link LeaderControl#advertise() advertise()}.</li>
+ * </ol>
+ * If this call order is broken, calls will fail by throwing
+ * {@link java.lang.IllegalStateException}.
+ * <p>
+ * At any point in the lifecycle, the scheduler will respond to
+ * {@link LeadershipListener#onDefeated(com.twitter.common.zookeeper.ServerSet.EndpointStatus)
+ * onDefeated()} by initiating a clean shutdown using {@link Lifecycle#shutdown() shutdown()}.
+ * A clean shutdown will also be initiated if control actions fail during normal state transitions.
+ */
+public class SchedulerLifecycle implements EventSubscriber {
+
+ private static final Logger LOG = Logger.getLogger(SchedulerLifecycle.class.getName());
+
+ private enum State {
+ IDLE,
+ PREPARING_STORAGE,
+ STORAGE_PREPARED,
+ LEADER_AWAITING_REGISTRATION,
+ REGISTERED_LEADER,
+ RUNNING,
+ DEAD
+ }
+
+ private static final Predicate<Transition<State>> IS_DEAD = new Predicate<Transition<State>>() {
+ @Override public boolean apply(Transition<State> state) {
+ return state.getTo() == State.DEAD;
+ }
+ };
+
+ private static final Predicate<Transition<State>> NOT_DEAD = Predicates.not(IS_DEAD);
+
+ private final LeadershipListener leadershipListener;
+ private final AtomicBoolean registrationAcked = new AtomicBoolean(false);
+ private final AtomicReference<LeaderControl> leaderControl = Atomics.newReference();
+ private final StateMachine<State> stateMachine;
+
+ @Inject
+ SchedulerLifecycle(
+ final DriverFactory driverFactory,
+ final NonVolatileStorage storage,
+ final Lifecycle lifecycle,
+ final Driver driver,
+ final DriverReference driverRef,
+ final LeadingOptions leadingOptions,
+ final ScheduledExecutorService executorService,
+ final Clock clock) {
+
+ this(
+ driverFactory,
+ storage,
+ lifecycle,
+ driver,
+ driverRef,
+ new DelayedActions() {
+ @Override public void blockingDriverJoin(Runnable runnable) {
+ executorService.execute(runnable);
+ }
+
+ @Override public void onAutoFailover(Runnable runnable) {
+ executorService.schedule(
+ runnable,
+ leadingOptions.leadingTimeLimit.getValue(),
+ leadingOptions.leadingTimeLimit.getUnit().getTimeUnit());
+ }
+
+ @Override public void onRegistrationTimeout(Runnable runnable) {
+ LOG.info(
+ "Giving up on registration in " + leadingOptions.registrationDelayLimit);
+ executorService.schedule(
+ runnable,
+ leadingOptions.registrationDelayLimit.getValue(),
+ leadingOptions.registrationDelayLimit.getUnit().getTimeUnit());
+ }
+ },
+ clock);
+ }
+
+ @VisibleForTesting
+ SchedulerLifecycle(
+ final DriverFactory driverFactory,
+ final NonVolatileStorage storage,
+ final Lifecycle lifecycle,
+ // TODO(wfarner): The presence of Driver and DriverReference is quite confusing. Figure out
+ // a clean way to collapse the duties of DriverReference into DriverImpl.
+ final Driver driver,
+ final DriverReference driverRef,
+ final DelayedActions delayedActions,
+ final Clock clock) {
+
+ Stats.export(new StatImpl<Integer>("framework_registered") {
+ @Override public Integer read() {
+ return registrationAcked.get() ? 1 : 0;
+ }
+ });
+ for (final State state : State.values()) {
+ Stats.export(new StatImpl<Integer>("scheduler_lifecycle_" + state) {
+ @Override public Integer read() {
+ return (state == stateMachine.getState()) ? 1 : 0;
+ }
+ });
+ }
+
+ final Closure<Transition<State>> prepareStorage = new Closure<Transition<State>>() {
+ @Override public void execute(Transition<State> transition) {
+ try {
+ storage.prepare();
+ stateMachine.transition(State.STORAGE_PREPARED);
+ } catch (RuntimeException e) {
+ stateMachine.transition(State.DEAD);
+ throw e;
+ }
+ }
+ };
+
+ final Closure<Transition<State>> handleLeading = new Closure<Transition<State>>() {
+ @Override public void execute(Transition<State> transition) {
+ LOG.info("Elected as leading scheduler!");
+ storage.start(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ StorageBackfill.backfill(storeProvider, clock);
+ }
+ });
+
+ @Nullable final String frameworkId = storage.consistentRead(
+ new Work.Quiet<String>() {
+ @Override public String apply(StoreProvider storeProvider) {
+ return storeProvider.getSchedulerStore().fetchFrameworkId();
+ }
+ });
+ driverRef.set(driverFactory.apply(frameworkId));
+
+ delayedActions.onRegistrationTimeout(
+ new Runnable() {
+ @Override public void run() {
+ if (!registrationAcked.get()) {
+ LOG.severe(
+ "Framework has not been registered within the tolerated delay.");
+ stateMachine.transition(State.DEAD);
+ }
+ }
+ });
+
+ delayedActions.onAutoFailover(
+ new Runnable() {
+ @Override public void run() {
+ LOG.info("Triggering automatic failover.");
+ stateMachine.transition(State.DEAD);
+ }
+ });
+
+ Protos.Status status = driver.start();
+ LOG.info("Driver started with code " + status);
+ delayedActions.blockingDriverJoin(new Runnable() {
+ @Override public void run() {
+ // Blocks until driver exits.
+ driver.join();
+ stateMachine.transition(State.DEAD);
+ }
+ });
+ }
+ };
+
+ final Closure<Transition<State>> handleRegistered = new Closure<Transition<State>>() {
+ @Override public void execute(Transition<State> transition) {
+ registrationAcked.set(true);
+ try {
+ leaderControl.get().advertise();
+ } catch (JoinException e) {
+ LOG.log(Level.SEVERE, "Failed to advertise leader, shutting down.", e);
+ stateMachine.transition(State.DEAD);
+ } catch (InterruptedException e) {
+ LOG.log(Level.SEVERE, "Interrupted while advertising leader, shutting down.", e);
+ stateMachine.transition(State.DEAD);
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+
+ final Closure<Transition<State>> shutDown = new Closure<Transition<State>>() {
+ private final AtomicBoolean invoked = new AtomicBoolean(false);
+ @Override public void execute(Transition<State> transition) {
+ if (!invoked.compareAndSet(false, true)) {
+ LOG.info("Shutdown already invoked, ignoring extra call.");
+ return;
+ }
+
+ // TODO(wfarner): Consider using something like guava's Closer to abstractly tear down
+ // resources here.
+ try {
+ LeaderControl control = leaderControl.get();
+ if (control != null) {
+ try {
+ control.leave();
+ } catch (JoinException e) {
+ LOG.log(Level.WARNING, "Failed to leave leadership: " + e, e);
+ } catch (ServerSet.UpdateException e) {
+ LOG.log(Level.WARNING, "Failed to leave server set: " + e, e);
+ }
+ }
+
+ // TODO(wfarner): Re-evaluate tear-down ordering here. Should the top-level shutdown
+ // be invoked first, or the underlying critical components?
+ driver.stop();
+ storage.stop();
+ } finally {
+ lifecycle.shutdown();
+ }
+ }
+ };
+
+ stateMachine = StateMachine.<State>builder("SchedulerLifecycle")
+ .initialState(State.IDLE)
+ .logTransitions()
+ .addState(
+ Closures.filter(NOT_DEAD, prepareStorage),
+ State.IDLE,
+ State.PREPARING_STORAGE, State.DEAD)
+ .addState(
+ State.PREPARING_STORAGE,
+ State.STORAGE_PREPARED, State.DEAD)
+ .addState(
+ Closures.filter(NOT_DEAD, handleLeading),
+ State.STORAGE_PREPARED,
+ State.LEADER_AWAITING_REGISTRATION, State.DEAD)
+ .addState(
+ Closures.filter(NOT_DEAD, handleRegistered),
+ State.LEADER_AWAITING_REGISTRATION,
+ State.REGISTERED_LEADER, State.DEAD)
+ .addState(
+ State.REGISTERED_LEADER,
+ State.RUNNING, State.DEAD)
+ .addState(
+ State.RUNNING,
+ State.DEAD)
+ .addState(
+ State.DEAD,
+ // Allow cycles in DEAD to prevent throwing and avoid the need for call-site checking.
+ State.DEAD
+ )
+ .onAnyTransition(
+ Closures.filter(IS_DEAD, shutDown))
+ .build();
+
+ this.leadershipListener = new SchedulerCandidateImpl(stateMachine, leaderControl);
+ }
+
+ /**
+ * Prepares a scheduler to offer itself as a leader candidate. After this call the scheduler will
+ * host a live log replica and start syncing data from the leader via the log until it gets called
+ * upon to lead.
+ *
+ * @return A listener that can be offered for leadership of a distributed election.
+ */
+ public LeadershipListener prepare() {
+ stateMachine.transition(State.PREPARING_STORAGE);
+ return leadershipListener;
+ }
+
+ @Subscribe
+ public void registered(DriverRegistered event) {
+ stateMachine.transition(State.REGISTERED_LEADER);
+ }
+
+ /**
+ * Maintains a reference to the driver.
+ */
+ static class DriverReference implements Supplier<Optional<SchedulerDriver>> {
+ private final AtomicReference<SchedulerDriver> driver = Atomics.newReference();
+
+ @Override public Optional<SchedulerDriver> get() {
+ return Optional.fromNullable(driver.get());
+ }
+
+ private void set(SchedulerDriver ref) {
+ driver.set(ref);
+ }
+ }
+
+ private static class SchedulerCandidateImpl implements LeadershipListener {
+ private final StateMachine<State> stateMachine;
+ private final AtomicReference<LeaderControl> leaderControl;
+
+ SchedulerCandidateImpl(
+ StateMachine<State> stateMachine,
+ AtomicReference<LeaderControl> leaderControl) {
+
+ this.stateMachine = stateMachine;
+ this.leaderControl = leaderControl;
+ }
+
+ @Override public void onLeading(LeaderControl control) {
+ leaderControl.set(control);
+ stateMachine.transition(State.LEADER_AWAITING_REGISTRATION);
+ }
+
+ @Override public void onDefeated(@Nullable ServerSet.EndpointStatus status) {
+ LOG.severe("Lost leadership, committing suicide.");
+ stateMachine.transition(State.DEAD);
+ }
+ }
+
+ public static class LeadingOptions {
+ private final Amount<Long, Time> registrationDelayLimit;
+ private final Amount<Long, Time> leadingTimeLimit;
+
+ /**
+ * Creates a new collection of options for tuning leadership behavior.
+ *
+ * @param registrationDelayLimit Maximum amount of time to wait for framework registration to
+ * complete.
+ * @param leadingTimeLimit Maximum amount of time to serve as leader before abdicating.
+ */
+ public LeadingOptions(
+ Amount<Long, Time> registrationDelayLimit,
+ Amount<Long, Time> leadingTimeLimit) {
+
+ Preconditions.checkArgument(
+ registrationDelayLimit.getValue() >= 0,
+ "Registration delay limit must be positive.");
+ Preconditions.checkArgument(
+ leadingTimeLimit.getValue() >= 0,
+ "Leading time limit must be positive.");
+
+ this.registrationDelayLimit = checkNotNull(registrationDelayLimit);
+ this.leadingTimeLimit = checkNotNull(leadingTimeLimit);
+ }
+ }
+
+ @VisibleForTesting
+ interface DelayedActions {
+ void blockingDriverJoin(Runnable runnable);
+
+ void onAutoFailover(Runnable runnable);
+
+ void onRegistrationTimeout(Runnable runnable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java
new file mode 100644
index 0000000..be4c2b1
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
+import javax.inject.Singleton;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.AbstractModule;
+import com.google.inject.PrivateModule;
+import com.google.inject.Provides;
+import com.google.inject.TypeLiteral;
+
+import org.apache.mesos.Scheduler;
+import org.apache.mesos.SchedulerDriver;
+
+import com.twitter.aurora.scheduler.Driver.DriverImpl;
+import com.twitter.aurora.scheduler.SchedulerLifecycle.DriverReference;
+import com.twitter.aurora.scheduler.SchedulerLifecycle.LeadingOptions;
+import com.twitter.aurora.scheduler.TaskIdGenerator.TaskIdGeneratorImpl;
+import com.twitter.aurora.scheduler.events.PubsubEventModule;
+import com.twitter.aurora.scheduler.periodic.GcExecutorLauncher;
+import com.twitter.aurora.scheduler.periodic.GcExecutorLauncher.GcExecutorSettings;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+
+/**
+ * Binding module for top-level scheduling logic.
+ */
+public class SchedulerModule extends AbstractModule {
+
+ @CmdLine(name = "executor_gc_interval",
+ help = "Max interval on which to run the GC executor on a host to clean up dead tasks.")
+ private static final Arg<Amount<Long, Time>> EXECUTOR_GC_INTERVAL =
+ Arg.create(Amount.of(1L, Time.HOURS));
+
+ @CmdLine(name = "gc_executor_path", help = "Path to the gc executor launch script.")
+ private static final Arg<String> GC_EXECUTOR_PATH = Arg.create(null);
+
+ @CmdLine(name = "max_registration_delay",
+ help = "Max allowable delay to allow the driver to register before aborting")
+ private static final Arg<Amount<Long, Time>> MAX_REGISTRATION_DELAY =
+ Arg.create(Amount.of(1L, Time.MINUTES));
+
+ @CmdLine(name = "max_leading_duration",
+ help = "After leading for this duration, the scheduler should commit suicide.")
+ private static final Arg<Amount<Long, Time>> MAX_LEADING_DURATION =
+ Arg.create(Amount.of(1L, Time.DAYS));
+
+ @Override
+ protected void configure() {
+ bind(Driver.class).to(DriverImpl.class);
+ bind(DriverImpl.class).in(Singleton.class);
+ bind(new TypeLiteral<Supplier<Optional<SchedulerDriver>>>() { }).to(DriverReference.class);
+ bind(DriverReference.class).in(Singleton.class);
+
+ bind(Scheduler.class).to(MesosSchedulerImpl.class);
+ bind(MesosSchedulerImpl.class).in(Singleton.class);
+
+ bind(TaskIdGenerator.class).to(TaskIdGeneratorImpl.class);
+
+ bind(GcExecutorSettings.class).toInstance(new GcExecutorSettings(
+ EXECUTOR_GC_INTERVAL.get(),
+ Optional.fromNullable(GC_EXECUTOR_PATH.get())));
+
+ bind(GcExecutorLauncher.class).in(Singleton.class);
+ bind(UserTaskLauncher.class).in(Singleton.class);
+
+ install(new PrivateModule() {
+ @Override protected void configure() {
+ bind(LeadingOptions.class).toInstance(
+ new LeadingOptions(MAX_REGISTRATION_DELAY.get(), MAX_LEADING_DURATION.get()));
+ final ScheduledExecutorService executor = Executors.newScheduledThreadPool(
+ 1,
+ new ThreadFactoryBuilder().setNameFormat("Lifecycle-%d").setDaemon(true).build());
+ bind(ScheduledExecutorService.class).toInstance(executor);
+ bind(SchedulerLifecycle.class).in(Singleton.class);
+ expose(SchedulerLifecycle.class);
+ }
+ });
+
+ PubsubEventModule.bindSubscriber(binder(), SchedulerLifecycle.class);
+ PubsubEventModule.bindSubscriber(binder(), TaskVars.class);
+ }
+
+ @Provides
+ @Singleton
+ List<TaskLauncher> provideTaskLaunchers(
+ GcExecutorLauncher gcLauncher,
+ UserTaskLauncher userTaskLauncher) {
+
+ return ImmutableList.of(gcLauncher, userTaskLauncher);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/TaskIdGenerator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/TaskIdGenerator.java b/src/main/java/org/apache/aurora/scheduler/TaskIdGenerator.java
new file mode 100644
index 0000000..240649e
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/TaskIdGenerator.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.UUID;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Preconditions;
+
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.util.Clock;
+
+/**
+ * A function that generates universally-unique (not guaranteed, but highly confident) task IDs.
+ */
+public interface TaskIdGenerator {
+
+ /**
+ * Generates a universally-unique ID for the task. This is not necessarily a repeatable
+ * operation, two subsequent invocations with the same object need not return the same value.
+ *
+ * @param task Configuration of the task to create an ID for.
+ * @param instanceId Instance ID for the task.
+ * @return A universally-unique ID for the task.
+ */
+ String generate(ITaskConfig task, int instanceId);
+
+ class TaskIdGeneratorImpl implements TaskIdGenerator {
+ private final Clock clock;
+
+ @Inject
+ TaskIdGeneratorImpl(Clock clock) {
+ this.clock = Preconditions.checkNotNull(clock);
+ }
+
+ @Override
+ public String generate(ITaskConfig task, int instanceId) {
+ String sep = "-";
+ return new StringBuilder()
+ .append(clock.nowMillis()) // Allows chronological sorting.
+ .append(sep)
+ .append(task.getOwner().getRole()) // Identification and collision prevention.
+ .append(sep)
+ .append(task.getEnvironment())
+ .append(sep)
+ .append(task.getJobName())
+ .append(sep)
+ .append(instanceId) // Collision prevention within job.
+ .append(sep)
+ .append(UUID.randomUUID()) // Just-in-case collision prevention.
+ .toString().replaceAll("[^\\w-]", sep); // Constrain character set.
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/TaskLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/TaskLauncher.java b/src/main/java/org/apache/aurora/scheduler/TaskLauncher.java
new file mode 100644
index 0000000..aade6da
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/TaskLauncher.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import com.google.common.base.Optional;
+
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.TaskInfo;
+import org.apache.mesos.Protos.TaskStatus;
+
+/**
+ * A receiver of resource offers and task status updates.
+ */
+public interface TaskLauncher {
+
+ /**
+ * Grants a resource offer to the task launcher, which will be passed to any subsequent task
+ * launchers if this one does not accept.
+ * <p>
+ * A task launcher may choose to retain an offer for later use. Any retained offers must be
+ * cleaned up with {@link #cancelOffer(OfferID)}.
+ *
+ * @param offer The resource offer.
+ * @return A task, absent if the launcher chooses not to accept the offer.
+ */
+ Optional<TaskInfo> createTask(Offer offer);
+
+ /**
+ * Informs the launcher that a status update has been received for a task. If the task is not
+ * associated with the launcher, it should return {@code false} so that another launcher may
+ * receive it.
+ *
+ * @param status The status update.
+ * @return {@code true} if the status is relevant to the launcher and should not be delivered to
+ * other launchers, {@code false} otherwise.
+ */
+ boolean statusUpdate(TaskStatus status);
+
+ /**
+ * Informs the launcher that a previously-advertised offer is canceled and may not be used.
+ *
+ * @param offer The canceled offer.
+ */
+ void cancelOffer(OfferID offer);
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/TaskVars.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/TaskVars.java b/src/main/java/org/apache/aurora/scheduler/TaskVars.java
new file mode 100644
index 0000000..5574631
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/TaskVars.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.Subscribe;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
+import com.twitter.aurora.scheduler.storage.AttributeStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.stats.StatsProvider;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A container that tracks and exports stat counters for tasks.
+ */
+class TaskVars implements EventSubscriber {
+ private static final Logger LOG = Logger.getLogger(TaskVars.class.getName());
+
+ // Used to ignore pubsub events sent before storage has completely started. This avoids a
+ // miscount where a StorageStarted consumer is invoked before storageStarted is invoked here,
+ // and pubsub events are fired for tasks that we have not yet counted. For example, if
+ // tasksDeleted is invoked, we would end up with a negative count.
+ private volatile boolean storageStarted = false;
+
+ private final LoadingCache<String, AtomicLong> countersByStatus;
+ private final LoadingCache<String, AtomicLong> countersByRack;
+
+ private final Storage storage;
+
+ @Inject
+ TaskVars(Storage storage, final StatsProvider statProvider) {
+ this.storage = checkNotNull(storage);
+ checkNotNull(statProvider);
+ countersByStatus = CacheBuilder.newBuilder().build(new CacheLoader<String, AtomicLong>() {
+ @Override public AtomicLong load(String statName) {
+ return statProvider.makeCounter(statName);
+ }
+ });
+ countersByRack = CacheBuilder.newBuilder().build(new CacheLoader<String, AtomicLong>() {
+ @Override public AtomicLong load(String rack) {
+ return statProvider.makeCounter(rackStatName(rack));
+ }
+ });
+ }
+
+ @VisibleForTesting
+ static String getVarName(ScheduleStatus status) {
+ return "task_store_" + status;
+ }
+
+ @VisibleForTesting
+ static String rackStatName(String rack) {
+ return "tasks_lost_rack_" + rack;
+ }
+
+ private static final Predicate<Attribute> IS_RACK = new Predicate<Attribute>() {
+ @Override public boolean apply(Attribute attr) {
+ return "rack".equals(attr.getName());
+ }
+ };
+
+ private static final Function<Attribute, String> ATTR_VALUE = new Function<Attribute, String>() {
+ @Override public String apply(Attribute attr) {
+ return Iterables.getOnlyElement(attr.getValues());
+ }
+ };
+
+ private AtomicLong getCounter(ScheduleStatus status) {
+ return countersByStatus.getUnchecked(getVarName(status));
+ }
+
+ private void incrementCount(ScheduleStatus status) {
+ getCounter(status).incrementAndGet();
+ }
+
+ private void decrementCount(ScheduleStatus status) {
+ getCounter(status).decrementAndGet();
+ }
+
+ @Subscribe
+ public void taskChangedState(TaskStateChange stateChange) {
+ if (!storageStarted) {
+ return;
+ }
+
+ IScheduledTask task = stateChange.getTask();
+ if (stateChange.getOldState() != ScheduleStatus.INIT) {
+ decrementCount(stateChange.getOldState());
+ }
+ incrementCount(task.getStatus());
+
+ if (stateChange.getNewState() == ScheduleStatus.LOST) {
+ final String host = stateChange.getTask().getAssignedTask().getSlaveHost();
+ Optional<String> rack = storage.consistentRead(new Work.Quiet<Optional<String>>() {
+ @Override public Optional<String> apply(StoreProvider storeProvider) {
+ Optional<Attribute> rack = FluentIterable
+ .from(AttributeStore.Util.attributesOrNone(storeProvider, host))
+ .firstMatch(IS_RACK);
+ return rack.transform(ATTR_VALUE);
+ }
+ });
+
+ if (rack.isPresent()) {
+ countersByRack.getUnchecked(rack.get()).incrementAndGet();
+ } else {
+ LOG.warning("Failed to find rack attribute associated with host " + host);
+ }
+ }
+ }
+
+ @Subscribe
+ public void storageStarted(StorageStarted event) {
+ for (IScheduledTask task : Storage.Util.consistentFetchTasks(storage, Query.unscoped())) {
+ incrementCount(task.getStatus());
+ }
+
+ // Dummy read the counter for each status counter. This is important to guarantee a stat with
+ // value zero is present for each state, even if all states are not represented in the task
+ // store.
+ for (ScheduleStatus status : ScheduleStatus.values()) {
+ getCounter(status);
+ }
+ storageStarted = true;
+ }
+
+ @Subscribe
+ public void tasksDeleted(final TasksDeleted event) {
+ if (!storageStarted) {
+ return;
+ }
+
+ for (IScheduledTask task : event.getTasks()) {
+ decrementCount(task.getStatus());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/UserTaskLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/UserTaskLauncher.java b/src/main/java/org/apache/aurora/scheduler/UserTaskLauncher.java
new file mode 100644
index 0000000..0fb3bbb
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/UserTaskLauncher.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.TaskInfo;
+import org.apache.mesos.Protos.TaskStatus;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.async.OfferQueue;
+import com.twitter.aurora.scheduler.base.Conversions;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.SchedulerException;
+import com.twitter.aurora.scheduler.state.StateManager;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A task launcher that matches resource offers against user tasks.
+ */
+class UserTaskLauncher implements TaskLauncher {
+
+ private static final Logger LOG = Logger.getLogger(UserTaskLauncher.class.getName());
+
+ @VisibleForTesting
+ static final String MEMORY_LIMIT_EXCEEDED = "MEMORY STATISTICS";
+
+ @VisibleForTesting
+ static final String MEMORY_LIMIT_DISPLAY = "Task used more memory than requested.";
+
+ private final OfferQueue offerQueue;
+ private final StateManager stateManager;
+
+ @Inject
+ UserTaskLauncher(OfferQueue offerQueue, StateManager stateManager) {
+ this.offerQueue = checkNotNull(offerQueue);
+ this.stateManager = checkNotNull(stateManager);
+ }
+
+ @Override
+ public Optional<TaskInfo> createTask(Offer offer) {
+ checkNotNull(offer);
+
+ offerQueue.addOffer(offer);
+ return Optional.absent();
+ }
+
+ @Override
+ public synchronized boolean statusUpdate(TaskStatus status) {
+ @Nullable String message = null;
+ if (status.hasMessage()) {
+ message = status.getMessage();
+ }
+
+ try {
+ ScheduleStatus translatedState = Conversions.convertProtoState(status.getState());
+ // TODO(William Farner): Remove this hack once Mesos API change is done.
+ // Tracked by: https://issues.apache.org/jira/browse/MESOS-343
+ if ((translatedState == ScheduleStatus.FAILED)
+ && (message != null)
+ && (message.contains(MEMORY_LIMIT_EXCEEDED))) {
+ message = MEMORY_LIMIT_DISPLAY;
+ }
+
+ stateManager.changeState(
+ Query.taskScoped(status.getTaskId().getValue()),
+ translatedState,
+ Optional.fromNullable(message));
+ } catch (SchedulerException e) {
+ LOG.log(Level.WARNING, "Failed to update status for: " + status, e);
+ throw e;
+ }
+ return true;
+ }
+
+ @Override
+ public void cancelOffer(OfferID offer) {
+ offerQueue.cancelOffer(offer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
new file mode 100644
index 0000000..24702b0
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.app;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterables;
+import com.google.inject.AbstractModule;
+import com.google.inject.Key;
+import com.google.inject.Provides;
+
+import org.apache.mesos.Scheduler;
+import org.apache.zookeeper.data.ACL;
+
+import com.twitter.aurora.GuiceUtils;
+import com.twitter.aurora.scheduler.SchedulerModule;
+import com.twitter.aurora.scheduler.async.AsyncModule;
+import com.twitter.aurora.scheduler.events.PubsubEventModule;
+import com.twitter.aurora.scheduler.filter.SchedulingFilterImpl;
+import com.twitter.aurora.scheduler.http.ClusterName;
+import com.twitter.aurora.scheduler.http.ServletModule;
+import com.twitter.aurora.scheduler.metadata.MetadataModule;
+import com.twitter.aurora.scheduler.quota.QuotaModule;
+import com.twitter.aurora.scheduler.state.StateModule;
+import com.twitter.aurora.scheduler.stats.AsyncStatsModule;
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.application.modules.LifecycleModule;
+import com.twitter.common.base.Command;
+import com.twitter.common.inject.TimedInterceptor;
+import com.twitter.common.net.pool.DynamicHostSet;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.stats.StatsProvider;
+import com.twitter.common.util.Clock;
+import com.twitter.common.zookeeper.ServerSet;
+import com.twitter.common.zookeeper.ServerSetImpl;
+import com.twitter.common.zookeeper.SingletonService;
+import com.twitter.common.zookeeper.ZooKeeperClient;
+import com.twitter.common.zookeeper.ZooKeeperClient.Credentials;
+import com.twitter.common.zookeeper.ZooKeeperUtils;
+import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule.ClientConfig;
+import com.twitter.thrift.ServiceInstance;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * Binding module for the aurora scheduler application.
+ */
+class AppModule extends AbstractModule {
+ private static final Logger LOG = Logger.getLogger(AppModule.class.getName());
+
+ private final String clusterName;
+ private final String serverSetPath;
+ private final ClientConfig zkClientConfig;
+
+ AppModule(String clusterName, String serverSetPath, ClientConfig zkClientConfig) {
+ this.clusterName = checkNotBlank(clusterName);
+ this.serverSetPath = checkNotBlank(serverSetPath);
+ this.zkClientConfig = checkNotNull(zkClientConfig);
+ }
+
+ @Override
+ protected void configure() {
+ // Enable intercepted method timings and context classloader repair.
+ TimedInterceptor.bind(binder());
+ GuiceUtils.bindJNIContextClassLoader(binder(), Scheduler.class);
+ GuiceUtils.bindExceptionTrap(binder(), Scheduler.class);
+
+ bind(Clock.class).toInstance(Clock.SYSTEM_CLOCK);
+
+ bind(Key.get(String.class, ClusterName.class)).toInstance(clusterName);
+
+ // Filter layering: notifier filter -> base impl
+ PubsubEventModule.bind(binder(), SchedulingFilterImpl.class);
+ bind(SchedulingFilterImpl.class).in(Singleton.class);
+
+ LifecycleModule.bindStartupAction(binder(), RegisterShutdownStackPrinter.class);
+
+ install(new AsyncModule());
+ install(new AsyncStatsModule());
+ install(new MetadataModule());
+ install(new QuotaModule());
+ install(new ServletModule());
+ install(new SchedulerModule());
+ install(new StateModule());
+
+ bind(StatsProvider.class).toInstance(Stats.STATS_PROVIDER);
+ }
+
+ /**
+ * Command to register a thread stack printer that identifies initiator of a shutdown.
+ */
+ private static class RegisterShutdownStackPrinter implements Command {
+ private static final Function<StackTraceElement, String> STACK_ELEM_TOSTRING =
+ new Function<StackTraceElement, String>() {
+ @Override public String apply(StackTraceElement element) {
+ return element.getClassName() + "." + element.getMethodName()
+ + String.format("(%s:%s)", element.getFileName(), element.getLineNumber());
+ }
+ };
+
+ private final ShutdownRegistry shutdownRegistry;
+
+ @Inject
+ RegisterShutdownStackPrinter(ShutdownRegistry shutdownRegistry) {
+ this.shutdownRegistry = shutdownRegistry;
+ }
+
+ @Override
+ public void execute() {
+ shutdownRegistry.addAction(new Command() {
+ @Override public void execute() {
+ Thread thread = Thread.currentThread();
+ String message = new StringBuilder()
+ .append("Thread: ").append(thread.getName())
+ .append(" (id ").append(thread.getId()).append(")")
+ .append("\n")
+ .append(Joiner.on("\n ").join(
+ Iterables.transform(Arrays.asList(thread.getStackTrace()), STACK_ELEM_TOSTRING)))
+ .toString();
+
+ LOG.info("Shutdown initiated by: " + message);
+ }
+ });
+ }
+ }
+
+ @Provides
+ @Singleton
+ List<ACL> provideAcls() {
+ if (zkClientConfig.credentials == Credentials.NONE) {
+ LOG.warning("Running without ZooKeeper digest credentials. ZooKeeper ACLs are disabled.");
+ return ZooKeeperUtils.OPEN_ACL_UNSAFE;
+ } else {
+ return ZooKeeperUtils.EVERYONE_READ_CREATOR_ALL;
+ }
+ }
+
+ @Provides
+ @Singleton
+ ServerSet provideServerSet(ZooKeeperClient client, List<ACL> zooKeeperAcls) {
+ return new ServerSetImpl(client, zooKeeperAcls, serverSetPath);
+ }
+
+ @Provides
+ @Singleton
+ DynamicHostSet<ServiceInstance> provideSchedulerHostSet(ServerSet serverSet) {
+ // Used for a type re-binding of the serverset.
+ return serverSet;
+ }
+
+ @Provides
+ @Singleton
+ SingletonService provideSingletonService(
+ ZooKeeperClient client,
+ ServerSet serverSet,
+ List<ACL> zookeeperAcls) {
+
+ return new SingletonService(
+ serverSet,
+ SingletonService.createSingletonCandidate(client, serverSetPath, zookeeperAcls));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/app/Log4jConfigurator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/app/Log4jConfigurator.java b/src/main/java/org/apache/aurora/scheduler/app/Log4jConfigurator.java
new file mode 100644
index 0000000..0ea2204
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/app/Log4jConfigurator.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.app;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.twitter.common.logging.RootLogConfig;
+import com.twitter.common.logging.RootLogConfig.Configuration;
+import com.twitter.common.logging.log4j.GlogLayout;
+
+/**
+ * Configures log4j logging.
+ */
+final class Log4jConfigurator {
+ private static final java.util.logging.Logger LOG =
+ java.util.logging.Logger.getLogger(Log4jConfigurator.class.getName());
+
+ /**
+ * Configures log4j to log to stderr with a glog format.
+ *
+ * @param glogConfig The glog configuration in effect.
+ */
+ static void configureConsole(Configuration glogConfig) {
+ Preconditions.checkNotNull(glogConfig);
+
+ BasicConfigurator.configure(
+ new ConsoleAppender(new GlogLayout(), ConsoleAppender.SYSTEM_ERR));
+ Logger.getRootLogger().setLevel(getLevel(glogConfig));
+ }
+
+ private static Level getLevel(RootLogConfig.Configuration logConfig) {
+ switch (logConfig.getVlog()) {
+ case FINEST: // fall through
+ case FINER: // fall through
+ case FINE: // fall through
+ case CONFIG:
+ return Level.TRACE;
+ case INFO:
+ return Level.INFO;
+ case WARNING:
+ return Level.WARN;
+ case SEVERE:
+ return Level.ERROR;
+ default:
+ LOG.warning("Mapping unexpected vlog value of " + logConfig.getVlog() + " to log4j TRACE");
+ return Level.TRACE;
+ }
+ }
+
+ private Log4jConfigurator() {
+ // Utility class.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/app/Modules.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/app/Modules.java b/src/main/java/org/apache/aurora/scheduler/app/Modules.java
new file mode 100644
index 0000000..72a80e1
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/app/Modules.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.app;
+
+import com.google.inject.Module;
+import com.google.inject.PrivateModule;
+
+/**
+ * A utility class for managing guice modules.
+ */
+final class Modules {
+
+ private Modules() {
+ // Utility class
+ }
+
+ private static Module instantiateModule(final Class<? extends Module> moduleClass) {
+ try {
+ return moduleClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Failed to instantiate module %s. Are you sure it has a no-arg constructor?",
+ moduleClass.getName()),
+ e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Failed to instantiate module %s. Are you sure it's public?",
+ moduleClass.getName()),
+ e);
+ }
+ }
+
+ // Defensively wrap each module provided on the command-line in a PrivateModule that only
+ // exposes requested classes to ensure that we don't depend on surprise extra bindings across
+ // different implementations.
+ static Module wrapInPrivateModule(
+ Class<? extends Module> moduleClass,
+ final Iterable<Class<?>> exposedClasses) {
+
+ final Module module = instantiateModule(moduleClass);
+ return new PrivateModule() {
+ @Override protected void configure() {
+ install(module);
+ for (Class<?> klass : exposedClasses) {
+ expose(klass);
+ }
+ }
+ };
+ }
+
+ static Module getModule(Class<? extends Module> moduleClass) {
+ return instantiateModule(moduleClass);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
new file mode 100644
index 0000000..693c364
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.app;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.logging.Logger;
+
+import javax.annotation.Nonnegative;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+import com.google.inject.AbstractModule;
+import com.google.inject.Module;
+
+import com.twitter.aurora.auth.CapabilityValidator;
+import com.twitter.aurora.auth.SessionValidator;
+import com.twitter.aurora.auth.UnsecureAuthModule;
+import com.twitter.aurora.scheduler.DriverFactory;
+import com.twitter.aurora.scheduler.DriverFactory.DriverFactoryImpl;
+import com.twitter.aurora.scheduler.MesosTaskFactory.ExecutorConfig;
+import com.twitter.aurora.scheduler.SchedulerLifecycle;
+import com.twitter.aurora.scheduler.cron.CronPredictor;
+import com.twitter.aurora.scheduler.cron.CronScheduler;
+import com.twitter.aurora.scheduler.cron.noop.NoopCronModule;
+import com.twitter.aurora.scheduler.local.IsolatedSchedulerModule;
+import com.twitter.aurora.scheduler.log.mesos.MesosLogStreamModule;
+import com.twitter.aurora.scheduler.storage.backup.BackupModule;
+import com.twitter.aurora.scheduler.storage.log.LogStorage;
+import com.twitter.aurora.scheduler.storage.log.LogStorageModule;
+import com.twitter.aurora.scheduler.storage.log.SnapshotStoreImpl;
+import com.twitter.aurora.scheduler.storage.mem.MemStorageModule;
+import com.twitter.aurora.scheduler.thrift.ThriftConfiguration;
+import com.twitter.aurora.scheduler.thrift.ThriftModule;
+import com.twitter.aurora.scheduler.thrift.auth.ThriftAuthModule;
+import com.twitter.common.application.AbstractApplication;
+import com.twitter.common.application.AppLauncher;
+import com.twitter.common.application.Lifecycle;
+import com.twitter.common.application.modules.HttpModule;
+import com.twitter.common.application.modules.LocalServiceRegistry;
+import com.twitter.common.application.modules.LogModule;
+import com.twitter.common.application.modules.StatsModule;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.args.constraints.NotEmpty;
+import com.twitter.common.args.constraints.NotNull;
+import com.twitter.common.inject.Bindings;
+import com.twitter.common.logging.RootLogConfig;
+import com.twitter.common.zookeeper.Group;
+import com.twitter.common.zookeeper.SingletonService;
+import com.twitter.common.zookeeper.SingletonService.LeadershipListener;
+import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule;
+import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule.ClientConfig;
+import com.twitter.common.zookeeper.guice.client.flagged.FlaggedClientConfig;
+
+/**
+ * Launcher for the aurora scheduler.
+ */
+public class SchedulerMain extends AbstractApplication {
+
+ private static final Logger LOG = Logger.getLogger(SchedulerMain.class.getName());
+
+ @CmdLine(name = "testing_isolated_scheduler",
+ help = "If true, run in a testing mode with the scheduler isolated from other components.")
+ private static final Arg<Boolean> ISOLATED_SCHEDULER = Arg.create(false);
+
+ @NotNull
+ @CmdLine(name = "cluster_name", help = "Name to identify the cluster being served.")
+ private static final Arg<String> CLUSTER_NAME = Arg.create();
+
+ @NotNull
+ @NotEmpty
+ @CmdLine(name = "serverset_path", help = "ZooKeeper ServerSet path to register at.")
+ private static final Arg<String> SERVERSET_PATH = Arg.create();
+
+ @CmdLine(name = "mesos_ssl_keyfile",
+ help = "JKS keyfile for operating the Mesos Thrift-over-SSL interface.")
+ private static final Arg<File> MESOS_SSL_KEY_FILE = Arg.create();
+
+ @Nonnegative
+ @CmdLine(name = "thrift_port", help = "Thrift server port.")
+ private static final Arg<Integer> THRIFT_PORT = Arg.create(0);
+
+ @NotNull
+ @CmdLine(name = "thermos_executor_path", help = "Path to the thermos executor launch script.")
+ private static final Arg<String> THERMOS_EXECUTOR_PATH = Arg.create();
+
+ @CmdLine(name = "auth_module",
+ help = "A Guice module to provide auth bindings. NOTE: The default is unsecure.")
+ private static final Arg<? extends Class<? extends Module>> AUTH_MODULE =
+ Arg.create(UnsecureAuthModule.class);
+
+ private static final Iterable<Class<?>> AUTH_MODULE_CLASSES = ImmutableList.<Class<?>>builder()
+ .add(SessionValidator.class)
+ .add(CapabilityValidator.class)
+ .build();
+
+ @CmdLine(name = "cron_module",
+ help = "A Guice module to provide cron bindings. NOTE: The default is a no-op.")
+ private static final Arg<? extends Class<? extends Module>> CRON_MODULE =
+ Arg.create(NoopCronModule.class);
+
+ private static final Iterable<Class<?>> CRON_MODULE_CLASSES = ImmutableList.<Class<?>>builder()
+ .add(CronPredictor.class)
+ .add(CronScheduler.class)
+ .build();
+
+ // TODO(Suman Karumuri): Pass in AUTH and CRON modules as extra modules
+ @CmdLine(name = "extra_modules",
+ help = "A list of modules that provide additional functionality.")
+ private static final Arg<List<Class<? extends Module>>> EXTRA_MODULES =
+ Arg.create((List<Class<? extends Module>>) ImmutableList.<Class<? extends Module>>of());
+
+ @Inject private SingletonService schedulerService;
+ @Inject private LocalServiceRegistry serviceRegistry;
+ @Inject private SchedulerLifecycle schedulerLifecycle;
+ @Inject private Lifecycle appLifecycle;
+ @Inject private Optional<RootLogConfig.Configuration> glogConfig;
+
+ private static Iterable<? extends Module> getSystemModules() {
+ return ImmutableList.of(
+ new LogModule(),
+ new HttpModule(),
+ new StatsModule()
+ );
+ }
+
+ private static Iterable<? extends Module> getExtraModules() {
+ Builder<Module> modules = ImmutableList.builder();
+ modules.add(Modules.wrapInPrivateModule(AUTH_MODULE.get(), AUTH_MODULE_CLASSES))
+ .add(Modules.wrapInPrivateModule(CRON_MODULE.get(), CRON_MODULE_CLASSES));
+
+ for (Class<? extends Module> moduleClass : EXTRA_MODULES.get()) {
+ modules.add(Modules.getModule(moduleClass));
+ }
+
+ return modules.build();
+ }
+
+ static Iterable<? extends Module> getModules(
+ String clusterName,
+ String serverSetPath,
+ ClientConfig zkClientConfig) {
+
+ return ImmutableList.<Module>builder()
+ .addAll(getSystemModules())
+ .add(new AppModule(clusterName, serverSetPath, zkClientConfig))
+ .addAll(getExtraModules())
+ .add(new LogStorageModule())
+ .add(new MemStorageModule(Bindings.annotatedKeyFactory(LogStorage.WriteBehind.class)))
+ .add(new ThriftModule())
+ .add(new ThriftAuthModule())
+ .build();
+ }
+
+ @Override
+ public Iterable<? extends Module> getModules() {
+ Module additional;
+ final ClientConfig zkClientConfig = FlaggedClientConfig.create();
+ if (ISOLATED_SCHEDULER.get()) {
+ additional = new IsolatedSchedulerModule();
+ } else {
+ // TODO(Kevin Sweeney): Push these bindings down into a "production" module.
+ additional = new AbstractModule() {
+ @Override protected void configure() {
+ bind(DriverFactory.class).to(DriverFactoryImpl.class);
+ bind(DriverFactoryImpl.class).in(Singleton.class);
+ install(new MesosLogStreamModule(zkClientConfig));
+ }
+ };
+ }
+
+ Module configModule = new AbstractModule() {
+ @Override protected void configure() {
+ bind(ThriftConfiguration.class).toInstance(new ThriftConfiguration() {
+ @Override public Optional<InputStream> getSslKeyStream() throws FileNotFoundException {
+ if (MESOS_SSL_KEY_FILE.hasAppliedValue()) {
+ return Optional.<InputStream>of(new FileInputStream(MESOS_SSL_KEY_FILE.get()));
+ } else {
+ return Optional.absent();
+ }
+ }
+
+ @Override public int getServingPort() {
+ return THRIFT_PORT.get();
+ }
+ });
+ bind(ExecutorConfig.class).toInstance(new ExecutorConfig(THERMOS_EXECUTOR_PATH.get()));
+ }
+ };
+
+ return ImmutableList.<Module>builder()
+ .add(new BackupModule(SnapshotStoreImpl.class))
+ .addAll(getModules(CLUSTER_NAME.get(), SERVERSET_PATH.get(), zkClientConfig))
+ .add(new ZooKeeperClientModule(zkClientConfig))
+ .add(configModule)
+ .add(additional)
+ .build();
+ }
+
+ @Override
+ public void run() {
+ if (glogConfig.isPresent()) {
+ // Setup log4j to match our jul glog config in order to pick up zookeeper logging.
+ Log4jConfigurator.configureConsole(glogConfig.get());
+ } else {
+ LOG.warning("Running without expected glog configuration.");
+ }
+
+ LeadershipListener leaderListener = schedulerLifecycle.prepare();
+
+ Optional<InetSocketAddress> primarySocket = serviceRegistry.getPrimarySocket();
+ if (!primarySocket.isPresent()) {
+ throw new IllegalStateException("No primary service registered with LocalServiceRegistry.");
+ }
+
+ try {
+ schedulerService.lead(
+ primarySocket.get(),
+ serviceRegistry.getAuxiliarySockets(),
+ leaderListener);
+ } catch (Group.WatchException e) {
+ throw new IllegalStateException("Failed to watch group and lead service.", e);
+ } catch (Group.JoinException e) {
+ throw new IllegalStateException("Failed to join scheduler service group.", e);
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("Interrupted while joining scheduler service group.", e);
+ }
+
+ appLifecycle.awaitShutdown();
+ }
+
+ public static void main(String[] args) {
+ AppLauncher.launch(SchedulerMain.class, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java b/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java
new file mode 100644
index 0000000..faf3269
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java
@@ -0,0 +1,274 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.logging.Logger;
+
+import javax.inject.Singleton;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.RateLimiter;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.AbstractModule;
+import com.google.inject.Binder;
+import com.google.inject.BindingAnnotation;
+import com.google.inject.Key;
+import com.google.inject.PrivateModule;
+import com.google.inject.TypeLiteral;
+
+import com.twitter.aurora.scheduler.async.OfferQueue.OfferQueueImpl;
+import com.twitter.aurora.scheduler.async.OfferQueue.OfferReturnDelay;
+import com.twitter.aurora.scheduler.async.RescheduleCalculator.RescheduleCalculatorImpl;
+import com.twitter.aurora.scheduler.async.TaskGroups.TaskGroupsSettings;
+import com.twitter.aurora.scheduler.async.TaskScheduler.TaskSchedulerImpl;
+import com.twitter.aurora.scheduler.events.PubsubEventModule;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.StatImpl;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.stats.StatsProvider;
+import com.twitter.common.util.Random;
+import com.twitter.common.util.TruncatedBinaryBackoff;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import static com.twitter.aurora.scheduler.async.HistoryPruner.PruneThreshold;
+import static com.twitter.aurora.scheduler.async.Preemptor.PreemptorImpl;
+import static com.twitter.aurora.scheduler.async.Preemptor.PreemptorImpl.PreemptionDelay;
+import static com.twitter.aurora.scheduler.async.TaskScheduler.TaskSchedulerImpl.ReservationDuration;
+
+/**
+ * Binding module for async task management.
+ */
+public class AsyncModule extends AbstractModule {
+
+ private static final Logger LOG = Logger.getLogger(AsyncModule.class.getName());
+
+ @CmdLine(name = "async_worker_threads",
+ help = "The number of worker threads to process async task operations with.")
+ private static final Arg<Integer> ASYNC_WORKER_THREADS = Arg.create(1);
+
+ @CmdLine(name = "transient_task_state_timeout",
+ help = "The amount of time after which to treat a task stuck in a transient state as LOST.")
+ private static final Arg<Amount<Long, Time>> TRANSIENT_TASK_STATE_TIMEOUT =
+ Arg.create(Amount.of(5L, Time.MINUTES));
+
+ @CmdLine(name = "initial_schedule_delay",
+ help = "Initial amount of time to wait before attempting to schedule a PENDING task.")
+ private static final Arg<Amount<Long, Time>> INITIAL_SCHEDULE_DELAY =
+ Arg.create(Amount.of(1L, Time.SECONDS));
+
+ @CmdLine(name = "max_schedule_delay",
+ help = "Maximum delay between attempts to schedule a PENDING tasks.")
+ private static final Arg<Amount<Long, Time>> MAX_SCHEDULE_DELAY =
+ Arg.create(Amount.of(30L, Time.SECONDS));
+
+ @CmdLine(name = "min_offer_hold_time",
+ help = "Minimum amount of time to hold a resource offer before declining.")
+ private static final Arg<Amount<Integer, Time>> MIN_OFFER_HOLD_TIME =
+ Arg.create(Amount.of(5, Time.MINUTES));
+
+ @CmdLine(name = "history_prune_threshold",
+ help = "Time after which the scheduler will prune terminated task history.")
+ private static final Arg<Amount<Long, Time>> HISTORY_PRUNE_THRESHOLD =
+ Arg.create(Amount.of(2L, Time.DAYS));
+
+ @CmdLine(name = "max_schedule_attempts_per_sec",
+ help = "Maximum number of scheduling attempts to make per second.")
+ private static final Arg<Double> MAX_SCHEDULE_ATTEMPTS_PER_SEC = Arg.create(10D);
+
+ @CmdLine(name = "flapping_task_threshold",
+ help = "A task that repeatedly runs for less than this time is considered to be flapping.")
+ private static final Arg<Amount<Long, Time>> FLAPPING_THRESHOLD =
+ Arg.create(Amount.of(5L, Time.MINUTES));
+
+ @CmdLine(name = "initial_flapping_task_delay",
+ help = "Initial amount of time to wait before attempting to schedule a flapping task.")
+ private static final Arg<Amount<Long, Time>> INITIAL_FLAPPING_DELAY =
+ Arg.create(Amount.of(30L, Time.SECONDS));
+
+ @CmdLine(name = "max_flapping_task_delay",
+ help = "Maximum delay between attempts to schedule a flapping task.")
+ private static final Arg<Amount<Long, Time>> MAX_FLAPPING_DELAY =
+ Arg.create(Amount.of(5L, Time.MINUTES));
+
+ @CmdLine(name = "max_reschedule_task_delay_on_startup",
+ help = "Upper bound of random delay for pending task rescheduling on scheduler startup.")
+ private static final Arg<Amount<Integer, Time>> MAX_RESCHEDULING_DELAY =
+ Arg.create(Amount.of(30, Time.SECONDS));
+
+ @CmdLine(name = "preemption_delay",
+ help = "Time interval after which a pending task becomes eligible to preempt other tasks")
+ private static final Arg<Amount<Long, Time>> PREEMPTION_DELAY =
+ Arg.create(Amount.of(10L, Time.MINUTES));
+
+ @CmdLine(name = "enable_preemptor",
+ help = "Enable the preemptor and preemption")
+ private static final Arg<Boolean> ENABLE_PREEMPTOR = Arg.create(true);
+
+ private static final Preemptor NULL_PREEMPTOR = new Preemptor() {
+ @Override public Optional<String> findPreemptionSlotFor(String taskId) {
+ return Optional.absent();
+ }
+ };
+
+ @CmdLine(name = "offer_reservation_duration", help = "Time to reserve a slave's offers while "
+ + "trying to satisfy a task preempting another.")
+ private static final Arg<Amount<Long, Time>> RESERVATION_DURATION =
+ Arg.create(Amount.of(3L, Time.MINUTES));
+
+ @BindingAnnotation
+ @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+ private @interface PreemptionBinding { }
+
+ @VisibleForTesting
+ static final Key<Preemptor> PREEMPTOR_KEY = Key.get(Preemptor.class, PreemptionBinding.class);
+
+ @Override
+ protected void configure() {
+ // Don't worry about clean shutdown, these can be daemon and cleanup-free.
+ final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
+ ASYNC_WORKER_THREADS.get(),
+ new ThreadFactoryBuilder().setNameFormat("AsyncProcessor-%d").setDaemon(true).build());
+ Stats.exportSize("timeout_queue_size", executor.getQueue());
+ Stats.export(new StatImpl<Long>("async_tasks_completed") {
+ @Override public Long read() {
+ return executor.getCompletedTaskCount();
+ }
+ });
+
+ // AsyncModule itself is not a subclass of PrivateModule because TaskEventModule internally uses
+ // a MultiBinder, which cannot span multiple injectors.
+ binder().install(new PrivateModule() {
+ @Override protected void configure() {
+ bind(new TypeLiteral<Amount<Long, Time>>() { })
+ .toInstance(TRANSIENT_TASK_STATE_TIMEOUT.get());
+ bind(ScheduledExecutorService.class).toInstance(executor);
+
+ bind(TaskTimeout.class).in(Singleton.class);
+ requireBinding(StatsProvider.class);
+ expose(TaskTimeout.class);
+ }
+ });
+ PubsubEventModule.bindSubscriber(binder(), TaskTimeout.class);
+
+ binder().install(new PrivateModule() {
+ @Override protected void configure() {
+ bind(TaskGroupsSettings.class).toInstance(new TaskGroupsSettings(
+ new TruncatedBinaryBackoff(INITIAL_SCHEDULE_DELAY.get(), MAX_SCHEDULE_DELAY.get()),
+ RateLimiter.create(MAX_SCHEDULE_ATTEMPTS_PER_SEC.get())));
+
+ bind(RescheduleCalculatorImpl.RescheduleCalculatorSettings.class)
+ .toInstance(new RescheduleCalculatorImpl.RescheduleCalculatorSettings(
+ new TruncatedBinaryBackoff(INITIAL_FLAPPING_DELAY.get(), MAX_FLAPPING_DELAY.get()),
+ FLAPPING_THRESHOLD.get(),
+ MAX_RESCHEDULING_DELAY.get()));
+
+ bind(RescheduleCalculator.class).to(RescheduleCalculatorImpl.class).in(Singleton.class);
+ if (ENABLE_PREEMPTOR.get()) {
+ bind(PREEMPTOR_KEY).to(PreemptorImpl.class);
+ bind(PreemptorImpl.class).in(Singleton.class);
+ LOG.info("Preemptor Enabled.");
+ } else {
+ bind(PREEMPTOR_KEY).toInstance(NULL_PREEMPTOR);
+ LOG.warning("Preemptor Disabled.");
+ }
+ expose(PREEMPTOR_KEY);
+ bind(new TypeLiteral<Amount<Long, Time>>() {
+ }).annotatedWith(PreemptionDelay.class)
+ .toInstance(PREEMPTION_DELAY.get());
+ bind(TaskGroups.class).in(Singleton.class);
+ expose(TaskGroups.class);
+ }
+ });
+ bindTaskScheduler(binder(), PREEMPTOR_KEY, RESERVATION_DURATION.get());
+ PubsubEventModule.bindSubscriber(binder(), TaskGroups.class);
+
+ binder().install(new PrivateModule() {
+ @Override protected void configure() {
+ bind(OfferReturnDelay.class).to(RandomJitterReturnDelay.class);
+ bind(ScheduledExecutorService.class).toInstance(executor);
+ bind(OfferQueue.class).to(OfferQueueImpl.class);
+ bind(OfferQueueImpl.class).in(Singleton.class);
+ expose(OfferQueue.class);
+ }
+ });
+ PubsubEventModule.bindSubscriber(binder(), OfferQueue.class);
+
+ binder().install(new PrivateModule() {
+ @Override protected void configure() {
+ // TODO(ksweeney): Create a configuration validator module so this can be injected.
+ // TODO(William Farner): Revert this once large task counts is cheap ala hierarchichal store
+ bind(Integer.class).annotatedWith(PruneThreshold.class).toInstance(100);
+ bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(PruneThreshold.class)
+ .toInstance(HISTORY_PRUNE_THRESHOLD.get());
+ bind(ScheduledExecutorService.class).toInstance(executor);
+
+ bind(HistoryPruner.class).in(Singleton.class);
+ expose(HistoryPruner.class);
+ }
+ });
+ PubsubEventModule.bindSubscriber(binder(), HistoryPruner.class);
+ }
+
+ /**
+ * This method exists because we want to test the wiring up of TaskSchedulerImpl class to the
+ * PubSub system in the TaskSchedulerImplTest class. The method has a complex signature because
+ * the binding of the TaskScheduler and friends occurs in a PrivateModule which does not interact
+ * well with the MultiBinder that backs the PubSub system.
+ */
+ @VisibleForTesting
+ static void bindTaskScheduler(
+ Binder binder,
+ final Key<Preemptor> preemptorKey,
+ final Amount<Long, Time> reservationDuration) {
+ binder.install(new PrivateModule() {
+ @Override protected void configure() {
+ bind(Preemptor.class).to(preemptorKey);
+ bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(ReservationDuration.class)
+ .toInstance(reservationDuration);
+ bind(TaskScheduler.class).to(TaskSchedulerImpl.class);
+ bind(TaskSchedulerImpl.class).in(Singleton.class);
+ expose(TaskScheduler.class);
+ }
+ });
+ PubsubEventModule.bindSubscriber(binder, TaskScheduler.class);
+ }
+
+ /**
+ * Returns offers after a random duration within a fixed window.
+ */
+ private static class RandomJitterReturnDelay implements OfferReturnDelay {
+ private static final int JITTER_WINDOW_MS = Amount.of(1, Time.MINUTES).as(Time.MILLISECONDS);
+
+ private final int minHoldTimeMs = MIN_OFFER_HOLD_TIME.get().as(Time.MILLISECONDS);
+ private final Random random = new Random.SystemRandom(new java.util.Random());
+
+ @Override public Amount<Integer, Time> get() {
+ return Amount.of(minHoldTimeMs + random.nextInt(JITTER_WINDOW_MS), Time.MILLISECONDS);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/HistoryPruner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/HistoryPruner.java b/src/main/java/org/apache/aurora/scheduler/async/HistoryPruner.java
new file mode 100644
index 0000000..9af6d36
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/HistoryPruner.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.eventbus.Subscribe;
+import com.google.inject.BindingAnnotation;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.state.StateManager;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.util.Clock;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.scheduler.base.Tasks.LATEST_ACTIVITY;
+import static com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import static com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
+import static com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import static com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
+
+/**
+ * Prunes tasks in a job based on per-job history and an inactive time threshold by observing tasks
+ * transitioning into one of the inactive states.
+ */
+public class HistoryPruner implements EventSubscriber {
+ private static final Logger LOG = Logger.getLogger(HistoryPruner.class.getName());
+
+ @VisibleForTesting
+ static final Query.Builder INACTIVE_QUERY = Query.unscoped().terminal();
+
+ private final Multimap<IJobKey, String> tasksByJob =
+ Multimaps.synchronizedSetMultimap(LinkedHashMultimap.<IJobKey, String>create());
+ @VisibleForTesting
+ Multimap<IJobKey, String> getTasksByJob() {
+ return tasksByJob;
+ }
+
+ private final ScheduledExecutorService executor;
+ private final Storage storage;
+ private final StateManager stateManager;
+ private final Clock clock;
+ private final long pruneThresholdMillis;
+ private final int perJobHistoryGoal;
+ private final Map<String, Future<?>> taskIdToFuture = Maps.newConcurrentMap();
+
+ @BindingAnnotation
+ @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+ public @interface PruneThreshold { }
+
+ @Inject
+ HistoryPruner(
+ final ScheduledExecutorService executor,
+ final Storage storage,
+ final StateManager stateManager,
+ final Clock clock,
+ @PruneThreshold Amount<Long, Time> inactivePruneThreshold,
+ @PruneThreshold int perJobHistoryGoal) {
+
+ this.executor = checkNotNull(executor);
+ this.storage = checkNotNull(storage);
+ this.stateManager = checkNotNull(stateManager);
+ this.clock = checkNotNull(clock);
+ this.pruneThresholdMillis = inactivePruneThreshold.as(Time.MILLISECONDS);
+ this.perJobHistoryGoal = perJobHistoryGoal;
+ }
+
+ @VisibleForTesting
+ long calculateTimeout(long taskEventTimestampMillis) {
+ return pruneThresholdMillis - Math.max(0, clock.nowMillis() - taskEventTimestampMillis);
+ }
+
+ /**
+ * When triggered, records an inactive task state change.
+ *
+ * @param change Event when a task changes state.
+ */
+ @Subscribe
+ public void recordStateChange(TaskStateChange change) {
+ if (Tasks.isTerminated(change.getNewState())) {
+ registerInactiveTask(
+ Tasks.SCHEDULED_TO_JOB_KEY.apply(change.getTask()),
+ change.getTaskId(),
+ calculateTimeout(clock.nowMillis()));
+ }
+ }
+
+ /**
+ * When triggered, iterates through inactive tasks in the system and prunes tasks that
+ * exceed the history goal for a job or are beyond the time threshold.
+ *
+ * @param event A new StorageStarted event.
+ */
+ @Subscribe
+ public void storageStarted(StorageStarted event) {
+ for (IScheduledTask task
+ : LATEST_ACTIVITY.sortedCopy(Storage.Util.consistentFetchTasks(storage, INACTIVE_QUERY))) {
+
+ registerInactiveTask(
+ Tasks.SCHEDULED_TO_JOB_KEY.apply(task),
+ Tasks.id(task),
+ calculateTimeout(Iterables.getLast(task.getTaskEvents()).getTimestamp()));
+ }
+ }
+
+ private void deleteTasks(Set<String> taskIds) {
+ LOG.info("Pruning inactive tasks " + taskIds);
+ stateManager.deleteTasks(taskIds);
+ }
+
+ /**
+ * When triggered, removes the tasks scheduled for pruning and cancels any existing future.
+ *
+ * @param event A new TasksDeleted event.
+ */
+ @Subscribe
+ public void tasksDeleted(final TasksDeleted event) {
+ for (IScheduledTask task : event.getTasks()) {
+ String id = Tasks.id(task);
+ tasksByJob.remove(Tasks.SCHEDULED_TO_JOB_KEY.apply(task), id);
+ Future<?> future = taskIdToFuture.remove(id);
+ if (future != null) {
+ future.cancel(false);
+ }
+ }
+ }
+
+ private void registerInactiveTask(
+ final IJobKey jobKey,
+ final String taskId,
+ long timeRemaining) {
+
+ LOG.fine("Prune task " + taskId + " in " + timeRemaining + " ms.");
+ // Insert the latest inactive task at the tail.
+ tasksByJob.put(jobKey, taskId);
+ Runnable runnable = new Runnable() {
+ @Override public void run() {
+ LOG.info("Pruning expired inactive task " + taskId);
+ tasksByJob.remove(jobKey, taskId);
+ taskIdToFuture.remove(taskId);
+ deleteTasks(ImmutableSet.of(taskId));
+ }
+ };
+ taskIdToFuture.put(taskId, executor.schedule(runnable, timeRemaining, TimeUnit.MILLISECONDS));
+
+ ImmutableSet.Builder<String> pruneTaskIds = ImmutableSet.builder();
+ Collection<String> tasks = tasksByJob.get(jobKey);
+ // From Multimaps javadoc: "It is imperative that the user manually synchronize on the returned
+ // multimap when accessing any of its collection views".
+ synchronized (tasksByJob) {
+ Iterator<String> iterator = tasks.iterator();
+ while (tasks.size() > perJobHistoryGoal) {
+ // Pick oldest task from the head. Guaranteed by LinkedHashMultimap based on insertion
+ // order.
+ String id = iterator.next();
+ iterator.remove();
+ pruneTaskIds.add(id);
+ Future<?> future = taskIdToFuture.remove(id);
+ if (future != null) {
+ future.cancel(false);
+ }
+ }
+ }
+
+ Set<String> ids = pruneTaskIds.build();
+ if (!ids.isEmpty()) {
+ deleteTasks(ids);
+ }
+ }
+}
[18/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/assets/jquery.pailer.js
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/assets/jquery.pailer.js b/src/main/python/apache/thermos/observer/http/assets/jquery.pailer.js
new file mode 100644
index 0000000..5962f9e
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/assets/jquery.pailer.js
@@ -0,0 +1,317 @@
+// A jQuery plugin for PAging and taILing data (i.e., a
+// 'PAILer'). Paging occurs when scrolling reaches the "top" and
+// tailing occurs when scrolling has reached the "bottom".
+
+// A 'read' function must be provided for reading the data (in
+// bytes). This function should expect an "options" object with the
+// fields 'offset' and 'length' set for reading the data. The result
+// from of the function should be a "promise" like value which has a
+// 'success' and 'error' callback which each take a function. An
+// object with at least two fields defined ('offset' and 'length') is
+// expected on success. A third field, 'data' may be provided if
+// 'length' is greater than 0. If the offset requested is greater than
+// the available offset, the result should be an object with the
+// 'offset' field set to the available offset (i.e., the length of the
+// data) and the 'length' field set to 0.
+
+// The plugin prepends, appends, and updates the "html" component of
+// the elements specified in the jQuery selector (e.g., doing
+// $('#data').pailer(...) means that data will be updated within
+// $('#data') via $('#data').prepend(...) and $('#data').append(...)
+// and $('#data').html(...) calls).
+
+// An indicator paragraph element (i.e., <p>) can be specified that
+// the plugin will write text to describing any status/errors that
+// have been encountered.
+
+// Data will automagically get truncated at some specified length,
+// configurable via the 'truncate-length' option. Likewise, the amount
+// of data paged in at a time can be configured via the 'page-size'
+// option.
+
+// Example:
+// HTML:
+// <div id="data" style="white-space:pre-wrap;"></div>
+//
+// <div style="position: absolute; left: 5px; top: 0px;">
+// <p id="indicator"></p>
+// </div>
+// Javascript:
+// $('#data').pailer({
+// 'read': function(options) {
+// var settings = $.extend({
+// 'offset': -1,
+// 'length': -1
+// }, options);
+// var url = '/url/for/data'
+// + '?offset=' + settings.offset
+// + '&length=' + settings.length;
+// return $.getJSON(url);
+// },
+// 'indicator': $('#indicator')
+// });
+
+(function($) {
+ function Pailer(read, element, indicator, page_size, truncate_length) {
+ var this_ = this;
+
+ this_.read = read;
+ this_.element = element;
+ this_.indicator = indicator;
+ this_.initialized = false;
+ this_.start = -1;
+ this_.end = -1;
+ this_.paging = false;
+ this_.tailing = true;
+
+ page_size || $.error('Expecting page_size to be defined');
+ truncate_length || $.error('Expecting truncate_length to be defined');
+
+ this_.page_size = page_size;
+ this_.truncate_length = truncate_length;
+
+ this_.element.css('overflow', 'auto');
+
+ this_.element.scroll(function () {
+ var scrollTop = this_.element.scrollTop();
+ var height = this_.element.height();
+ var scrollHeight = this_.element[0].scrollHeight;
+
+ if (scrollTop == 0) {
+ this_.page();
+ } else if (scrollTop + height >= scrollHeight) {
+ if (!this_.tailing) {
+ this_.tailing = true;
+ this_.tail();
+ }
+ } else {
+ this_.tailing = false;
+ }
+ });
+ }
+
+
+ Pailer.prototype.initialize = function() {
+ var this_ = this;
+
+ // Set an indicator while we load the data.
+ this_.indicate('(LOADING)');
+
+ this_.read({'offset': -1})
+ .success(function(data) {
+ this_.indicate('');
+
+ // Get the last page of data.
+ if (data.offset > this_.page_size) {
+ this_.start = this_.end = data.offset - this_.page_size;
+ } else {
+ this_.start = this_.end = 0;
+ }
+
+ this_.initialized = true;
+ this_.element.html('');
+ setTimeout(function() { this_.tail(); }, 0);
+ })
+ .error(function() {
+ this_.indicate('(FAILED TO INITIALIZE ... RETRYING)');
+ setTimeout(function() {
+ this_.indicate('');
+ this_.initialize();
+ }, 1000);
+ });
+ }
+
+
+ Pailer.prototype.page = function() {
+ var this_ = this;
+
+ if (!this_.initialized) {
+ return;
+ }
+
+ if (this_.paging) {
+ return;
+ }
+
+ this_.paging = true;
+ this_.indicate('(PAGING)');
+
+ if (this_.start == 0) {
+ this_.paging = false;
+ this_.indicate('(AT BEGINNING OF FILE)');
+ setTimeout(function() { this_.indicate(''); }, 1000);
+ return;
+ }
+
+ var offset = this_.start - this_.page_size;
+ var length = this_.page_size;
+
+ if (offset < 0) {
+ offset = 0;
+ length = this_.start;
+ }
+
+ // Buffer the data in case what gets read is less than 'length'.
+ var buffer = '';
+
+ var read = function(offset, length) {
+ this_.read({'offset': offset, 'length': length})
+ .success(function(data) {
+ if (data.length < length) {
+ buffer += data.data;
+ read(offset + data.length, length - data.length);
+ } else if (data.length > 0) {
+ this_.indicate('(PAGED)');
+ setTimeout(function() { this_.indicate(''); }, 1000);
+
+ // Prepend buffer onto data.
+ data.offset -= buffer.length;
+ data.data = buffer + data.data;
+ data.length = data.data.length;
+
+ // Truncate to the first newline (unless this is the beginning).
+ if (data.offset != 0) {
+ var index = data.data.indexOf('\n') + 1;
+ data.offset += index;
+ data.data = data.data.substring(index);
+ data.length -= index;
+ }
+
+ this_.start = data.offset;
+
+ var scrollTop = this_.element.scrollTop();
+ var scrollHeight = this_.element[0].scrollHeight;
+
+ this_.element.prepend(data.data);
+
+ scrollTop += this_.element[0].scrollHeight - scrollHeight;
+ this_.element.scrollTop(scrollTop);
+
+ this_.paging = false;
+ }
+ })
+ .error(function() {
+ this_.indicate('(FAILED TO PAGE ... RETRYING)');
+ setTimeout(function() {
+ this_.indicate('');
+ this_.page();
+ }, 1000);
+ });
+ }
+
+ read(offset, length);
+ }
+
+
+ Pailer.prototype.tail = function() {
+ var this_ = this;
+
+ if (!this_.initialized) {
+ return;
+ }
+
+ this_.read({'offset': this_.end, 'length': this_.truncate_length})
+ .success(function(data) {
+ var scrollTop = this_.element.scrollTop();
+ var height = this_.element.height();
+ var scrollHeight = this_.element[0].scrollHeight;
+
+ // Check if we are still at the bottom (since this event might
+ // have fired before the scroll event has been dispatched).
+ if (scrollTop + height < scrollHeight) {
+ this_.tailing = false;
+ return;
+ }
+
+ if (data.length > 0) {
+ // Truncate to the first newline if this is the first time
+ // (and we aren't reading from the beginning of the log).
+ if (this_.start == this_.end && data.offset != 0) {
+ var index = data.data.indexOf('\n') + 1;
+ data.offset += index;
+ data.data = data.data.substring(index);
+ data.length -= index;
+ this_.start = data.offset; // Adjust the actual start too!
+ }
+
+ this_.end = data.offset + data.length;
+
+ this_.element.append(data.data);
+
+ scrollTop += this_.element[0].scrollHeight - scrollHeight;
+ this_.element.scrollTop(scrollTop);
+
+ // Also, only if we're at the bottom, truncate data so that we
+ // don't consume too much memory. TODO(benh): Only do
+ // truncations if we've been at the bottom for a while.
+ this_.truncate();
+ }
+
+ // Tail immediately if we got as much data as requested (since
+ // this probably means we've waited around a while). The
+ // alternative here would be to not get data in chunks, but the
+ // potential issue here is that we might end up requesting GB of
+ // log data at a time ... the right solution here might be to do
+ // a request to determine the new ending offset and then request
+ // the proper length.
+ if (data.length == this_.truncate_length) {
+ setTimeout(function() { this_.tail(); }, 0);
+ } else {
+ setTimeout(function() { this_.tail(); }, 1000);
+ }
+ })
+ .error(function() {
+ this_.indicate('(FAILED TO TAIL ... RETRYING)');
+ this_.initialized = false;
+ setTimeout(function() {
+ this_.indicate('');
+ this_.initialize();
+ }, 1000);
+ });
+ }
+
+
+ Pailer.prototype.indicate = function(text) {
+ var this_ = this;
+
+ if (this_.indicator) {
+ this_.indicator.text(text);
+ }
+ }
+
+
+ Pailer.prototype.truncate = function() {
+ var this_ = this;
+
+ var length = this_.element.html().length;
+ if (length >= this_.truncate_length) {
+ var index = length - this_.truncate_length;
+ this_.start = this_.end - this_.truncate_length;
+ this_.element.html(this_.element.html().substring(index));
+ }
+ }
+
+ $.fn.pailer = function(options) {
+ var settings = $.extend({
+ 'read': function() { return {
+ 'success': function() {},
+ 'error': function(f) { f(); }
+ }},
+ 'page_size': 8 * 4096, // 8 "pages".
+ 'truncate_length': 50000
+ }, options);
+
+ this.each(function() {
+ var pailer = $.data(this, 'pailer');
+ if (!pailer) {
+ pailer = new Pailer(settings.read,
+ $(this),
+ settings.indicator,
+ settings.page_size,
+ settings.truncate_length);
+ $.data(this, 'pailer', pailer);
+ pailer.initialize();
+ }
+ });
+ }
+})(jQuery);
[33/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/log/Log.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/Log.java b/src/main/java/org/apache/aurora/scheduler/log/Log.java
new file mode 100644
index 0000000..c1036e0
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/log/Log.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.log;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Represents an append only log that can be read after and truncated before a known
+ * {@link Position}.
+ *
+ * <p>Logs are accessed by {@link #open() opening} a {@link Stream}. All stream
+ * access occurs with references to log entry {@link Position positions} in the stream.
+ */
+public interface Log {
+
+ /**
+ * An opaque ordered handle to a log entry's position in the log stream.
+ */
+ interface Position extends Comparable<Position> {
+ }
+
+ /**
+ * Represents a single entry in the log stream.
+ */
+ interface Entry {
+
+ /**
+ * Gets the contents of the log entry.
+ *
+ * @return the data stored in this log entry
+ */
+ byte[] contents();
+ }
+
+ /**
+ * An interface to the live {@link Log} stream that allows for appending, reading and writing
+ * entries.
+ */
+ interface Stream extends Closeable {
+
+ /**
+ * Indicates a {@link Position} that is not (currently) contained in this log stream.
+ * This might indicate the position id from a different log or that the position was from this
+ * log but has been truncated.
+ */
+ class InvalidPositionException extends RuntimeException {
+ public InvalidPositionException(String message) {
+ super(message);
+ }
+ public InvalidPositionException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
+ /**
+ * Indicates a {@link Stream} could not be read from, written to or truncated due to some
+ * underlying IO error.
+ */
+ class StreamAccessException extends RuntimeException {
+ public StreamAccessException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
+ /**
+ * Appends an {@link Entry} to the end of the log stream.
+ *
+ * @param contents the data to store in the appended entry
+ * @return the posiiton of the appended entry
+ * @throws StreamAccessException if contents could not be appended to the stream
+ */
+ Position append(byte[] contents) throws StreamAccessException;
+
+ /**
+ * Allows reading all entries from the log stream. Implementations may
+ * materialize all entries at once or the may provide some form of streaming to back the
+ * returned entry iterator. If the implementation does use some form of streaming or batching,
+ * it may throw a
+ * {@code StreamAccessException} on any call to {@link Iterator#hasNext()} or
+ * {@link Iterator#next()}.
+ *
+ * @return an iterator that ranges from the entry from the given {@code position} to the last
+ * entry in the log.
+ * @throws InvalidPositionException if the specified position does not exist in this log
+ * @throws StreamAccessException if the stream could not be read from
+ */
+ Iterator<Entry> readAll() throws InvalidPositionException, StreamAccessException;
+
+ /**
+ * Removes all log entries preceding the log entry at the given {@code position}.
+ *
+ * @param position the position of the latest entry to remove
+ * @throws InvalidPositionException if the specified position does not exist in this log
+ * @throws StreamAccessException if the stream could not be truncated
+ */
+ void truncateBefore(Position position) throws InvalidPositionException, StreamAccessException;
+ }
+
+ /**
+ * Opens the log stream for reading writing and truncation. Clients should ensure the stream is
+ * closed when they are done using it.
+ *
+ * @return the log stream
+ * @throws IOException if there was a problem opening the log stream
+ */
+ Stream open() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/log/mesos/LogInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/mesos/LogInterface.java b/src/main/java/org/apache/aurora/scheduler/log/mesos/LogInterface.java
new file mode 100644
index 0000000..9ae4685
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/log/mesos/LogInterface.java
@@ -0,0 +1,42 @@
+package com.twitter.aurora.scheduler.log.mesos;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.mesos.Log;
+import org.apache.mesos.Log.Entry;
+import org.apache.mesos.Log.Position;
+
+/**
+ * An interface for {@link Log}, since the mesos Java API doesn't provide one.
+ * <p>
+ * This is needed because a static initializer that loads a native library, prevents us from
+ * mocking the Mesos Log API in tests. These wrapper interfaces and their corresponding
+ * implementations that delegate the calls to the underlying Mesos Log objects will help
+ * us mock the Log API.
+ * <p>
+ * TODO(Suman Karumuri): Remove this interface after https://issues.apache.org/jira/browse/MESOS-796
+ * is resolved.
+ */
+interface LogInterface {
+
+ Position position(byte[] identity);
+
+ interface ReaderInterface {
+ List<Entry> read(Position from, Position to, long timeout, TimeUnit unit)
+ throws TimeoutException, Log.OperationFailedException;
+
+ Position beginning();
+
+ Position ending();
+ }
+
+ interface WriterInterface {
+ Position append(byte[] data, long timeout, TimeUnit unit)
+ throws TimeoutException, Log.WriterFailedException;
+
+ Position truncate(Position to, long timeout, TimeUnit unit)
+ throws TimeoutException, Log.WriterFailedException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
new file mode 100644
index 0000000..d625734
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
@@ -0,0 +1,379 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.log.mesos;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.UnmodifiableIterator;
+import com.google.common.primitives.Longs;
+import com.google.inject.BindingAnnotation;
+
+import org.apache.mesos.Log;
+
+import com.twitter.aurora.scheduler.log.mesos.LogInterface.ReaderInterface;
+import com.twitter.aurora.scheduler.log.mesos.LogInterface.WriterInterface;
+import com.twitter.common.base.Function;
+import com.twitter.common.base.MorePreconditions;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.SlidingStats;
+import com.twitter.common.stats.Stats;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * A {@code Log} implementation backed by a true distributed log in mesos core.
+ */
+public class MesosLog implements com.twitter.aurora.scheduler.log.Log {
+
+ private static final Logger LOG = Logger.getLogger(MesosLog.class.getName());
+
+ /**
+ * Binding annotation for the opaque value of a log noop entry.
+ */
+ @BindingAnnotation
+ @Retention(RUNTIME)
+ @Target({ PARAMETER, METHOD })
+ public @interface NoopEntry { }
+
+ /**
+ * Binding annotation for log read timeouts.
+ */
+ @BindingAnnotation
+ @Retention(RUNTIME)
+ @Target({ PARAMETER, METHOD })
+ public @interface ReadTimeout { }
+
+ /**
+ * Binding annotation for log write timeouts - used for truncates and appends.
+ */
+ @BindingAnnotation
+ @Retention(RUNTIME)
+ @Target({ PARAMETER, METHOD })
+ public @interface WriteTimeout { }
+
+ private final Provider<LogInterface> logFactory;
+
+ private final Provider<ReaderInterface> readerFactory;
+ private final Amount<Long, Time> readTimeout;
+
+ private final Provider<WriterInterface> writerFactory;
+ private final Amount<Long, Time> writeTimeout;
+
+ private final byte[] noopEntry;
+
+ /**
+ * Creates a new mesos log.
+ *
+ * @param logFactory Factory to provide access to log.
+ * @param readerFactory Factory to provide access to log readers.
+ * @param readTimeout Log read timeout.
+ * @param writerFactory Factory to provide access to log writers.
+ * @param writeTimeout Log write timeout.
+ * @param noopEntry A no-op log entry blob.
+ */
+ @Inject
+ public MesosLog(
+ Provider<LogInterface> logFactory,
+ Provider<ReaderInterface> readerFactory,
+ @ReadTimeout Amount<Long, Time> readTimeout,
+ Provider<WriterInterface> writerFactory,
+ @WriteTimeout Amount<Long, Time> writeTimeout,
+ @NoopEntry byte[] noopEntry) {
+
+ this.logFactory = Preconditions.checkNotNull(logFactory);
+
+ this.readerFactory = Preconditions.checkNotNull(readerFactory);
+ this.readTimeout = readTimeout;
+
+ this.writerFactory = Preconditions.checkNotNull(writerFactory);
+ this.writeTimeout = writeTimeout;
+
+ this.noopEntry = Preconditions.checkNotNull(noopEntry);
+ }
+
+ @Override
+ public Stream open() {
+ return new LogStream(
+ logFactory.get(), readerFactory.get(), readTimeout, writerFactory, writeTimeout, noopEntry);
+ }
+
+ @VisibleForTesting
+ static class LogStream implements com.twitter.aurora.scheduler.log.Log.Stream {
+ @VisibleForTesting
+ static final class OpStats {
+ private final String opName;
+ private final SlidingStats timing;
+ private final AtomicLong timeouts;
+ private final AtomicLong failures;
+
+ OpStats(String opName) {
+ this.opName = MorePreconditions.checkNotBlank(opName);
+ timing = new SlidingStats("scheduler_log_native_" + opName, "nanos");
+ timeouts = exportLongStat("scheduler_log_native_%s_timeouts", opName);
+ failures = exportLongStat("scheduler_log_native_%s_failures", opName);
+ }
+
+ private static AtomicLong exportLongStat(String template, Object... args) {
+ return Stats.exportLong(String.format(template, args));
+ }
+ }
+
+ private static final Function<Log.Entry, LogEntry> MESOS_ENTRY_TO_ENTRY =
+ new Function<Log.Entry, LogEntry>() {
+ @Override public LogEntry apply(Log.Entry entry) {
+ return new LogEntry(entry);
+ }
+ };
+
+ private final OpStats read = new OpStats("read");
+ private final OpStats append = new OpStats("append");
+ private final OpStats truncate = new OpStats("truncate");
+ private final AtomicLong entriesSkipped =
+ Stats.exportLong("scheduler_log_native_native_entries_skipped");
+
+ private final LogInterface log;
+
+ private final ReaderInterface reader;
+ private final long readTimeout;
+ private final TimeUnit readTimeUnit;
+
+ private final Provider<WriterInterface> writerFactory;
+ private final long writeTimeout;
+ private final TimeUnit writeTimeUnit;
+
+ private final byte[] noopEntry;
+
+ private WriterInterface writer;
+
+ LogStream(LogInterface log, ReaderInterface reader, Amount<Long, Time> readTimeout,
+ Provider<WriterInterface> writerFactory, Amount<Long, Time> writeTimeout,
+ byte[] noopEntry) {
+
+ this.log = log;
+
+ this.reader = reader;
+ this.readTimeout = readTimeout.getValue();
+ this.readTimeUnit = readTimeout.getUnit().getTimeUnit();
+
+ this.writerFactory = writerFactory;
+ this.writeTimeout = writeTimeout.getValue();
+ this.writeTimeUnit = writeTimeout.getUnit().getTimeUnit();
+
+ this.noopEntry = noopEntry;
+ }
+
+ @Override
+ public Iterator<Entry> readAll() throws StreamAccessException {
+ // TODO(John Sirois): Currently we must be the coordinator to ensure we get the 'full read'
+ // of log entries expected by the users of the com.twitter.aurora.scheduler.log.Log interface.
+ // Switch to another method of ensuring this when it becomes available in mesos' log
+ // interface.
+ try {
+ append(noopEntry);
+ } catch (StreamAccessException e) {
+ throw new StreamAccessException("Error writing noop prior to a read", e);
+ }
+
+ final Log.Position from = reader.beginning();
+ final Log.Position to = end().unwrap();
+
+ // Reading all the entries at once may cause large garbage collections. Instead, we
+ // lazily read the entries one by one as they are requested.
+ // TODO(Benjamin Hindman): Eventually replace this functionality with functionality
+ // from the Mesos Log.
+ return new UnmodifiableIterator<Entry>() {
+ private long position = Longs.fromByteArray(from.identity());
+ private final long endPosition = Longs.fromByteArray(to.identity());
+ private Entry entry = null;
+
+ @Override
+ public boolean hasNext() {
+ if (entry != null) {
+ return true;
+ }
+
+ while (position <= endPosition) {
+ long start = System.nanoTime();
+ try {
+ Log.Position p = log.position(Longs.toByteArray(position));
+ if (LOG.isLoggable(Level.FINE)) {
+ LOG.fine("Reading position " + position + " from the log");
+ }
+ List<Log.Entry> entries = reader.read(p, p, readTimeout, readTimeUnit);
+
+ // N.B. HACK! There is currently no way to "increment" a position. Until the Mesos
+ // Log actually provides a way to "stream" the log, we approximate as much by
+ // using longs via Log.Position.identity and Log.position.
+ position++;
+
+ // Reading positions in this way means it's possible that we get an "invalid" entry
+ // (e.g., in the underlying log terminology this would be anything but an append)
+ // which will be removed from the returned entries resulting in an empty list.
+ // We skip these.
+ if (entries.isEmpty()) {
+ entriesSkipped.getAndIncrement();
+ } else {
+ entry = MESOS_ENTRY_TO_ENTRY.apply(Iterables.getOnlyElement(entries));
+ return true;
+ }
+ } catch (TimeoutException e) {
+ read.timeouts.getAndIncrement();
+ throw new StreamAccessException("Timeout reading from log.", e);
+ } catch (Log.OperationFailedException e) {
+ read.failures.getAndIncrement();
+ throw new StreamAccessException("Problem reading from log", e);
+ } finally {
+ read.timing.accumulate(System.nanoTime() - start);
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public Entry next() {
+ if (entry == null && !hasNext()) {
+ throw new NoSuchElementException();
+ }
+
+ Entry result = Preconditions.checkNotNull(entry);
+ entry = null;
+ return result;
+ }
+ };
+ }
+
+ @Override
+ public LogPosition append(final byte[] contents) throws StreamAccessException {
+ Preconditions.checkNotNull(contents);
+
+ Log.Position position = mutate(append, new Mutation<Log.Position>() {
+ @Override public Log.Position apply(WriterInterface logWriter)
+ throws TimeoutException, Log.WriterFailedException {
+ return logWriter.append(contents, writeTimeout, writeTimeUnit);
+ }
+ });
+ return LogPosition.wrap(position);
+ }
+
+ @Timed("scheduler_log_native_truncate_before")
+ @Override
+ public void truncateBefore(com.twitter.aurora.scheduler.log.Log.Position position)
+ throws StreamAccessException {
+
+ Preconditions.checkArgument(position instanceof LogPosition);
+
+ final Log.Position before = ((LogPosition) position).unwrap();
+ mutate(truncate, new Mutation<Void>() {
+ @Override public Void apply(WriterInterface logWriter)
+ throws TimeoutException, Log.WriterFailedException {
+ logWriter.truncate(before, writeTimeout, writeTimeUnit);
+ return null;
+ }
+ });
+ }
+
+ @VisibleForTesting
+ interface Mutation<T> {
+ T apply(WriterInterface writer) throws TimeoutException, Log.WriterFailedException;
+ }
+
+ @VisibleForTesting
+ synchronized <T> T mutate(OpStats stats, Mutation<T> mutation) {
+ long start = System.nanoTime();
+ if (writer == null) {
+ writer = writerFactory.get();
+ }
+ try {
+ return mutation.apply(writer);
+ } catch (TimeoutException e) {
+ stats.timeouts.getAndIncrement();
+ throw new StreamAccessException("Timeout performing log " + stats.opName, e);
+ } catch (Log.WriterFailedException e) {
+ stats.failures.getAndIncrement();
+
+ // We must throw away a writer on any write failure - this could be because of a coordinator
+ // election in which case we must trigger a new election.
+ writer = null;
+
+ throw new StreamAccessException("Problem performing log" + stats.opName, e);
+ } finally {
+ stats.timing.accumulate(System.nanoTime() - start);
+ }
+ }
+
+ private LogPosition end() {
+ return LogPosition.wrap(reader.ending());
+ }
+
+ @Override
+ public void close() {
+ // noop
+ }
+
+ private static class LogPosition implements com.twitter.aurora.scheduler.log.Log.Position {
+ private final Log.Position underlying;
+
+ LogPosition(Log.Position underlying) {
+ this.underlying = underlying;
+ }
+
+ static LogPosition wrap(Log.Position position) {
+ return new LogPosition(position);
+ }
+
+ Log.Position unwrap() {
+ return underlying;
+ }
+
+ @Override public int compareTo(Position o) {
+ Preconditions.checkArgument(o instanceof LogPosition);
+ return underlying.compareTo(((LogPosition) o).underlying);
+ }
+ }
+
+ private static class LogEntry implements com.twitter.aurora.scheduler.log.Log.Entry {
+ private final Log.Entry underlying;
+
+ public LogEntry(Log.Entry entry) {
+ this.underlying = entry;
+ }
+
+ @Override
+ public byte[] contents() {
+ return underlying.data;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java
new file mode 100644
index 0000000..7063cb7
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.log.mesos;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import javax.inject.Singleton;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.inject.PrivateModule;
+import com.google.inject.Provides;
+import com.google.inject.TypeLiteral;
+
+import org.apache.mesos.Log;
+import org.apache.zookeeper.common.PathUtils;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.gen.storage.LogEntry;
+import com.twitter.aurora.scheduler.log.mesos.LogInterface.ReaderInterface;
+import com.twitter.aurora.scheduler.log.mesos.LogInterface.WriterInterface;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.net.InetSocketAddressHelper;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule.ClientConfig;
+
+/**
+ * Binds a native mesos Log implementation.
+ *
+ * <p>Exports the following bindings:
+ * <ul>
+ * <li>{@link Log} - a log backed by the mesos native distributed log</li>
+ * </ul>
+ */
+public class MesosLogStreamModule extends PrivateModule {
+ @CmdLine(name = "native_log_quorum_size",
+ help = "The size of the quorum required for all log mutations.")
+ private static final Arg<Integer> QUORUM_SIZE = Arg.create(1);
+
+ @CmdLine(name = "native_log_file_path",
+ help = "Path to a file to store the native log data in. If the parent directory does"
+ + "not exist it will be created.")
+ private static final Arg<File> LOG_PATH = Arg.create(null);
+
+ @CmdLine(name = "native_log_zk_group_path",
+ help = "A zookeeper node for use by the native log to track the master coordinator.")
+ private static final Arg<String> ZK_LOG_GROUP_PATH = Arg.create(null);
+
+ /*
+ * This timeout includes the time to get a quorum to promise leadership to the coordinator and
+ * the time to fill any holes in the coordinator's log.
+ */
+ @CmdLine(name = "native_log_election_timeout",
+ help = "The timeout for a single attempt to obtain a new log writer.")
+ private static final Arg<Amount<Long, Time>> COORDINATOR_ELECTION_TIMEOUT =
+ Arg.create(Amount.of(15L, Time.SECONDS));
+
+ /*
+ * Normally retries would not be expected to help much - however in the small replica set where
+ * a few down replicas doom a coordinator election attempt, retrying effectively gives us a wider
+ * window in which to await a live quorum before giving up and thrashing the global election
+ * process. Observed log replica recovery times as of 4/6/2012 can be ~45 seconds so giving a
+ * window >= 2x this should support 1-round election events (that possibly use several retries in
+ * the single round).
+ */
+ @CmdLine(name = "native_log_election_retries",
+ help = "The maximum number of attempts to obtain a new log writer.")
+ private static final Arg<Integer> COORDINATOR_ELECTION_RETRIES = Arg.create(20);
+
+ @CmdLine(name = "native_log_read_timeout",
+ help = "The timeout for doing log reads.")
+ private static final Arg<Amount<Long, Time>> READ_TIMEOUT =
+ Arg.create(Amount.of(5L, Time.SECONDS));
+
+ @CmdLine(name = "native_log_write_timeout",
+ help = "The timeout for doing log appends and truncations.")
+ private static final Arg<Amount<Long, Time>> WRITE_TIMEOUT =
+ Arg.create(Amount.of(3L, Time.SECONDS));
+
+ private final ClientConfig zkClientConfig;
+
+ public MesosLogStreamModule(ClientConfig zkClientConfig) {
+ this.zkClientConfig = Preconditions.checkNotNull(zkClientConfig);
+ }
+
+ @Override
+ protected void configure() {
+ bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.ReadTimeout.class)
+ .toInstance(READ_TIMEOUT.get());
+ bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.WriteTimeout.class)
+ .toInstance(WRITE_TIMEOUT.get());
+
+ bind(com.twitter.aurora.scheduler.log.Log.class).to(MesosLog.class);
+ bind(MesosLog.class).in(Singleton.class);
+ expose(com.twitter.aurora.scheduler.log.Log.class);
+ }
+
+ @Provides
+ @Singleton
+ Log provideLog() {
+ File logPath = LOG_PATH.get();
+ File parentDir = logPath.getParentFile();
+ if (!parentDir.exists() && !parentDir.mkdirs()) {
+ addError("Failed to create parent directory to store native log at: %s", parentDir);
+ }
+
+ String zkConnectString = Joiner.on(',').join(
+ Iterables.transform(zkClientConfig.servers, InetSocketAddressHelper.INET_TO_STR));
+
+ PathUtils.validatePath(ZK_LOG_GROUP_PATH.get());
+ return new Log(
+ QUORUM_SIZE.get(),
+ logPath.getAbsolutePath(),
+ zkConnectString,
+ zkClientConfig.sessionTimeout.getValue(),
+ zkClientConfig.sessionTimeout.getUnit().getTimeUnit(),
+ ZK_LOG_GROUP_PATH.get(),
+ zkClientConfig.credentials.scheme(),
+ zkClientConfig.credentials.authToken());
+ }
+
+ @Provides
+ Log.Reader provideReader(Log log) {
+ return new Log.Reader(log);
+ }
+
+ @Provides
+ Log.Writer provideWriter(Log log) {
+ Amount<Long, Time> electionTimeout = COORDINATOR_ELECTION_TIMEOUT.get();
+ return new Log.Writer(log, electionTimeout.getValue(), electionTimeout.getUnit().getTimeUnit(),
+ COORDINATOR_ELECTION_RETRIES.get());
+ }
+
+ @Provides
+ LogInterface provideLogInterface(final Log log) {
+ return new LogInterface() {
+ @Override public Log.Position position(byte[] identity) {
+ return log.position(identity);
+ }
+ };
+ }
+
+ @Provides
+ ReaderInterface provideReaderInterface(final Log.Reader reader) {
+ return new ReaderInterface() {
+ @Override
+ public List<Log.Entry> read(Log.Position from, Log.Position to, long timeout, TimeUnit unit)
+ throws TimeoutException, Log.OperationFailedException {
+
+ return reader.read(from, to, timeout, unit);
+ }
+
+ @Override public Log.Position beginning() {
+ return reader.beginning();
+ }
+
+ @Override public Log.Position ending() {
+ return reader.ending();
+ }
+ };
+ }
+
+ @Provides
+ WriterInterface provideWriterInterface(final Log.Writer writer) {
+ return new WriterInterface() {
+ @Override public Log.Position append(byte[] data, long timeout, TimeUnit unit)
+ throws TimeoutException, Log.WriterFailedException {
+ return writer.append(data, timeout, unit);
+ }
+
+ @Override public Log.Position truncate(Log.Position to, long timeout, TimeUnit unit)
+ throws TimeoutException, Log.WriterFailedException {
+ return writer.truncate(to, timeout, unit);
+ }
+ };
+ }
+
+ @Provides
+ @Singleton
+ @MesosLog.NoopEntry
+ byte[] provideNoopEntry() throws ThriftBinaryCodec.CodingException {
+ return ThriftBinaryCodec.encodeNonNull(LogEntry.noop(true));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/log/testing/FileLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/testing/FileLog.java b/src/main/java/org/apache/aurora/scheduler/log/testing/FileLog.java
new file mode 100644
index 0000000..3dff4cf
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/log/testing/FileLog.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.log.testing;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Throwables;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+import com.google.common.io.Files;
+import com.google.common.primitives.Longs;
+
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.test.FileLogContents;
+import com.twitter.aurora.gen.test.LogRecord;
+import com.twitter.aurora.scheduler.log.Log;
+import com.twitter.common.base.Closure;
+
+/**
+ * A log implementation that reads from and writes to a local file.
+ * <p>
+ * This should never be used in a production setting, it is only intended for local testing.
+ * TODO(wfarner): Bind/inject a settable flag that indicates we are running with test settings.
+ * Surface this in a banner on the web UI.
+ */
+class FileLog implements Log {
+
+ private final File logFile;
+
+ @Inject
+ FileLog(File logFile) {
+ this.logFile = Preconditions.checkNotNull(logFile);
+ }
+
+ @Override
+ public Stream open() throws IOException {
+ try {
+ FileLogContents logContents;
+ // Treat an empty file as a new file.
+ // NOTE: we can't use logFile.length == 0 to test for an empty file, since empty unicode files
+ // contain a byte order mark (BOM) header that is 255 bytes in length.
+ if (logFile.createNewFile()
+ || Files.readFirstLine(logFile, Charset.defaultCharset()) == null) {
+ logContents = new FileLogContents(Maps.<Long, LogRecord>newHashMap());
+ } else {
+ logContents = ThriftBinaryCodec.decode(FileLogContents.class, Files.toByteArray(logFile));
+ }
+ Closure<FileLogContents> logWriter = new Closure<FileLogContents>() {
+ @Override public void execute(FileLogContents logContents) {
+ try {
+ Files.write(ThriftBinaryCodec.encode(logContents), logFile);
+ } catch (IOException | CodingException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ };
+ return new FileStream(logContents, logWriter);
+ } catch (CodingException e) {
+ throw new IOException("Failed to interpret log contents: " + e, e);
+ }
+ }
+
+ private static class FileStream implements Stream {
+ private final FileLogContents logContents;
+ private final Closure<FileLogContents> logWriter;
+ private long nextPosition;
+
+ FileStream(FileLogContents logContents, Closure<FileLogContents> logWriter) {
+ this.logContents = logContents;
+ this.logWriter = logWriter;
+ nextPosition = logContents.getRecords().isEmpty()
+ ? 1
+ : Ordering.natural().max(logContents.getRecords().keySet()) + 1;
+ }
+
+ @Override
+ public Position append(byte[] contents) throws StreamAccessException {
+ logContents.getRecords().put(nextPosition, new LogRecord(ByteBuffer.wrap(contents)));
+ Position position = new CounterPosition(nextPosition);
+ logWriter.execute(logContents);
+ nextPosition++;
+ return position;
+ }
+
+ private static final Function<LogRecord, Entry> TO_ENTRY = new Function<LogRecord, Entry>() {
+ @Override public Entry apply(final LogRecord record) {
+ return new Entry() {
+ @Override public byte[] contents() {
+ return record.getContents();
+ }
+ };
+ }
+ };
+
+ @Override
+ public Iterator<Entry> readAll() throws InvalidPositionException, StreamAccessException {
+ return FluentIterable.from(Ordering.natural().sortedCopy(logContents.getRecords().keySet()))
+ .transform(Functions.forMap(logContents.getRecords()))
+ .transform(TO_ENTRY)
+ .iterator();
+ }
+
+ @Override
+ public void truncateBefore(Position position)
+ throws InvalidPositionException, StreamAccessException {
+
+ if (!(position instanceof CounterPosition)) {
+ throw new InvalidPositionException("Unrecognized position " + position);
+ }
+
+ final long truncateBefore = ((CounterPosition) position).value;
+ Iterables.removeIf(logContents.getRecords().keySet(), new Predicate<Long>() {
+ @Override public boolean apply(Long recordPosition) {
+ return recordPosition < truncateBefore;
+ }
+ });
+ logWriter.execute(logContents);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // No-op.
+ }
+
+ private static class CounterPosition implements Position {
+ private final long value;
+
+ CounterPosition(long value) {
+ this.value = value;
+ }
+
+ @Override
+ public int compareTo(Position position) {
+ return Longs.compare(value, ((CounterPosition) position).value);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/log/testing/FileLogStreamModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/testing/FileLogStreamModule.java b/src/main/java/org/apache/aurora/scheduler/log/testing/FileLogStreamModule.java
new file mode 100644
index 0000000..5ef3e8e
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/log/testing/FileLogStreamModule.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.log.testing;
+
+import java.io.File;
+
+import javax.inject.Singleton;
+
+import com.google.common.base.Preconditions;
+import com.google.inject.PrivateModule;
+
+import com.twitter.aurora.scheduler.log.Log;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+
+/**
+ * Binding module that uses a local log file, intended for testing.
+ */
+public class FileLogStreamModule extends PrivateModule {
+
+ // TODO(William Farner): Make this a required argument and ensure it is not included in production
+ // builds (MESOS-471).
+ //@NotNull
+ @CmdLine(name = "testing_log_file_path", help = "Path to a file to store local log file data in.")
+ private static final Arg<File> LOG_PATH = Arg.create(null);
+
+ @Override
+ protected void configure() {
+ Preconditions.checkNotNull(LOG_PATH.get());
+ bind(File.class).toInstance(LOG_PATH.get());
+ bind(Log.class).to(FileLog.class);
+ bind(FileLog.class).in(Singleton.class);
+ expose(Log.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/metadata/MetadataModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/metadata/MetadataModule.java b/src/main/java/org/apache/aurora/scheduler/metadata/MetadataModule.java
new file mode 100644
index 0000000..08377ae
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/metadata/MetadataModule.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.metadata;
+
+import javax.inject.Singleton;
+
+import com.google.inject.AbstractModule;
+
+import com.twitter.aurora.scheduler.events.PubsubEventModule;
+
+/**
+ * Binding module for scheduler metadata management.
+ */
+public class MetadataModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(NearestFit.class).in(Singleton.class);
+ PubsubEventModule.bindSubscriber(binder(), NearestFit.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java b/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
new file mode 100644
index 0000000..47c1f5c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/metadata/NearestFit.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.metadata;
+
+import java.util.Set;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Ticker;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.Subscribe;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
+import com.twitter.aurora.scheduler.events.PubsubEvent.Vetoed;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+
+/**
+ * Tracks vetoes against scheduling decisions and maintains the closest fit among all the vetoes
+ * for a task.
+ */
+public class NearestFit implements EventSubscriber {
+ @VisibleForTesting
+ static final Amount<Long, Time> EXPIRATION = Amount.of(10L, Time.MINUTES);
+
+ @VisibleForTesting
+ static final ImmutableSet<Veto> NO_VETO = ImmutableSet.of();
+
+ private final LoadingCache<String, Fit> fitByTask;
+
+ @VisibleForTesting
+ NearestFit(Ticker ticker) {
+ fitByTask = CacheBuilder.newBuilder()
+ .expireAfterWrite(EXPIRATION.getValue(), EXPIRATION.getUnit().getTimeUnit())
+ .ticker(ticker)
+ .build(new CacheLoader<String, Fit>() {
+ @Override public Fit load(String taskId) {
+ return new Fit();
+ }
+ });
+ }
+
+ @Inject
+ NearestFit() {
+ this(Ticker.systemTicker());
+ }
+
+ /**
+ * Gets the vetoes that represent the nearest fit for the given task.
+ *
+ * @param taskId The task to look up.
+ * @return The nearest fit vetoes for the given task. This will return an empty set if
+ * no vetoes have been recorded for the task.
+ */
+ public synchronized ImmutableSet<Veto> getNearestFit(String taskId) {
+ Fit fit = fitByTask.getIfPresent(taskId);
+ return (fit == null) ? NO_VETO : fit.vetoes;
+ }
+
+ /**
+ * Records a task deletion event.
+ *
+ * @param deletedEvent Task deleted event.
+ */
+ @Subscribe
+ public synchronized void remove(TasksDeleted deletedEvent) {
+ fitByTask.invalidateAll(Tasks.ids(deletedEvent.getTasks()));
+ }
+
+ /**
+ * Records a task state change event.
+ * This will ignore any events where the previous state is not {@link ScheduleStatus#PENDING}.
+ *
+ * @param stateChangeEvent Task state change.
+ */
+ @Subscribe
+ public synchronized void stateChanged(TaskStateChange stateChangeEvent) {
+ if (stateChangeEvent.getOldState() == ScheduleStatus.PENDING) {
+ fitByTask.invalidate(stateChangeEvent.getTaskId());
+ }
+ }
+
+ private static final Predicate<Veto> IS_CONSTRAINT_MISMATCH = new Predicate<Veto>() {
+ @Override public boolean apply(Veto veto) {
+ return veto.isConstraintMismatch();
+ }
+ };
+
+ /**
+ * Records a task veto event.
+ * This will ignore any veto events where any veto returns {@code true} from
+ * {@link Veto#isConstraintMismatch()}.
+ *
+ * @param vetoEvent Veto event.
+ */
+ @Subscribe
+ public synchronized void vetoed(Vetoed vetoEvent) {
+ Preconditions.checkNotNull(vetoEvent);
+ fitByTask.getUnchecked(vetoEvent.getTaskId()).maybeUpdate(vetoEvent.getVetoes());
+ }
+
+ private static class Fit {
+ private ImmutableSet<Veto> vetoes;
+
+ private static int score(Iterable<Veto> vetoes) {
+ int total = 0;
+ for (Veto veto : vetoes) {
+ total += veto.getScore();
+ }
+ return total;
+ }
+
+ private void update(Iterable<Veto> newVetoes) {
+ vetoes = ImmutableSet.copyOf(newVetoes);
+ }
+
+ /**
+ * Updates the nearest fit if the provided vetoes represents a closer fit than the current
+ * best fit.
+ * <p>
+ * There are two classes of vetoes: those with and without constraint mismatches. A set of
+ * vetoes without a constraint mismatch is always a better fit than a set with constraint
+ * mismatches.
+ * <p>
+ * If two sets are equivalent in that they do or do not have constraint mismatches, they are
+ * compared by the following criteria:
+ * <ul>
+ * <li> the one with fewer vetoes is a better fit, irrespective of scores
+ * <li> if the veto count is equal, the one with the smaller aggregate score is a better fit
+ * </ul>
+ *
+ * @param newVetoes The vetoes for the scheduling assignment with {@code newHost}.
+ */
+ void maybeUpdate(Set<Veto> newVetoes) {
+ if (vetoes == null) {
+ update(newVetoes);
+ return;
+ }
+
+ boolean valueMismatchOld = Iterables.any(vetoes, IS_CONSTRAINT_MISMATCH);
+ boolean valueMismatchNew = Iterables.any(newVetoes, IS_CONSTRAINT_MISMATCH);
+ if (valueMismatchOld == valueMismatchNew) {
+ if (newVetoes.size() < vetoes.size()) {
+ update(newVetoes);
+ } else if ((newVetoes.size() == vetoes.size()) && (score(newVetoes) < score(vetoes))) {
+ update(newVetoes);
+ }
+ } else if (valueMismatchOld) {
+ update(newVetoes);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/periodic/GcExecutorLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/periodic/GcExecutorLauncher.java b/src/main/java/org/apache/aurora/scheduler/periodic/GcExecutorLauncher.java
new file mode 100644
index 0000000..df15854
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/periodic/GcExecutorLauncher.java
@@ -0,0 +1,195 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.periodic;
+
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Maps;
+import com.google.protobuf.ByteString;
+
+import org.apache.mesos.Protos.ExecutorID;
+import org.apache.mesos.Protos.ExecutorInfo;
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.TaskID;
+import org.apache.mesos.Protos.TaskInfo;
+import org.apache.mesos.Protos.TaskStatus;
+
+import com.twitter.aurora.Protobufs;
+import com.twitter.aurora.codec.ThriftBinaryCodec;
+import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
+import com.twitter.aurora.gen.comm.AdjustRetainedTasks;
+import com.twitter.aurora.scheduler.TaskLauncher;
+import com.twitter.aurora.scheduler.base.CommandUtil;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.Resources;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+import com.twitter.common.util.Random;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A task launcher that periodically initiates garbage collection on a host, re-using a single
+ * garbage collection executor
+ */
+public class GcExecutorLauncher implements TaskLauncher {
+ private static final Logger LOG = Logger.getLogger(GcExecutorLauncher.class.getName());
+
+ private final AtomicLong tasksCreated = Stats.exportLong("scheduler_gc_tasks_created");
+
+ @VisibleForTesting
+ static final Resources TOTAL_GC_EXECUTOR_RESOURCES =
+ new Resources(0.2, Amount.of(128L, Data.MB), Amount.of(16L, Data.MB), 0);
+
+ // An epsilon is used because we are required to supply executor and task resources.
+ @VisibleForTesting
+ static final Resources EPSILON =
+ new Resources(0.01, Amount.of(1L, Data.MB), Amount.of(1L, Data.MB), 0);
+
+ private static final Resources GC_EXECUTOR_RESOURCES =
+ Resources.subtract(TOTAL_GC_EXECUTOR_RESOURCES, EPSILON);
+
+ private static final String SYSTEM_TASK_PREFIX = "system-gc-";
+ private static final String EXECUTOR_NAME = "aurora.gc";
+
+ private final GcExecutorSettings settings;
+ private final Storage storage;
+ private final Clock clock;
+ private final Cache<String, Long> pulses;
+
+ @Inject
+ GcExecutorLauncher(
+ GcExecutorSettings settings,
+ Storage storage,
+ Clock clock) {
+
+ this.settings = checkNotNull(settings);
+ this.storage = checkNotNull(storage);
+ this.clock = checkNotNull(clock);
+
+ this.pulses = CacheBuilder.newBuilder()
+ .expireAfterWrite(settings.getMaxGcInterval(), TimeUnit.MILLISECONDS)
+ .build();
+ }
+
+ @Override
+ public Optional<TaskInfo> createTask(Offer offer) {
+ if (!settings.getGcExecutorPath().isPresent()
+ || !Resources.from(offer).greaterThanOrEqual(TOTAL_GC_EXECUTOR_RESOURCES)
+ || isAlive(offer.getHostname())) {
+ return Optional.absent();
+ }
+
+ Set<IScheduledTask> tasksOnHost =
+ Storage.Util.weaklyConsistentFetchTasks(storage, Query.slaveScoped(offer.getHostname()));
+ AdjustRetainedTasks message = new AdjustRetainedTasks()
+ .setRetainedTasks(Maps.transformValues(Tasks.mapById(tasksOnHost), Tasks.GET_STATUS));
+ byte[] data;
+ try {
+ data = ThriftBinaryCodec.encode(message);
+ } catch (CodingException e) {
+ LOG.severe("Failed to encode retained tasks message: " + message);
+ return Optional.absent();
+ }
+
+ tasksCreated.incrementAndGet();
+ pulses.put(offer.getHostname(), clock.nowMillis() + settings.getDelayMs());
+
+ ExecutorInfo.Builder executor = ExecutorInfo.newBuilder()
+ .setExecutorId(ExecutorID.newBuilder().setValue(EXECUTOR_NAME))
+ .setName(EXECUTOR_NAME)
+ .setSource(offer.getHostname())
+ .addAllResources(GC_EXECUTOR_RESOURCES.toResourceList())
+ .setCommand(CommandUtil.create(settings.getGcExecutorPath().get()));
+
+ return Optional.of(TaskInfo.newBuilder().setName("system-gc")
+ .setTaskId(TaskID.newBuilder().setValue(SYSTEM_TASK_PREFIX + UUID.randomUUID().toString()))
+ .setSlaveId(offer.getSlaveId())
+ .setData(ByteString.copyFrom(data))
+ .setExecutor(executor)
+ .addAllResources(EPSILON.toResourceList())
+ .build());
+ }
+
+ @Override
+ public boolean statusUpdate(TaskStatus status) {
+ if (status.getTaskId().getValue().startsWith(SYSTEM_TASK_PREFIX)) {
+ LOG.info("Received status update for GC task: " + Protobufs.toString(status));
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void cancelOffer(OfferID offer) {
+ // No-op.
+ }
+
+ private boolean isAlive(String hostname) {
+ Optional<Long> timestamp = Optional.fromNullable(pulses.getIfPresent(hostname));
+ return timestamp.isPresent() && clock.nowMillis() < timestamp.get();
+ }
+
+ /**
+ * Wraps configuration values for the {@code GcExecutorLauncher}.
+ */
+ public static class GcExecutorSettings {
+ private final Amount<Long, Time> gcInterval;
+ private final Optional<String> gcExecutorPath;
+ private final Random rand = new Random.SystemRandom(new java.util.Random());
+
+ public GcExecutorSettings(
+ Amount<Long, Time> gcInterval,
+ Optional<String> gcExecutorPath) {
+
+ this.gcInterval = checkNotNull(gcInterval);
+ this.gcExecutorPath = checkNotNull(gcExecutorPath);
+ }
+
+ @VisibleForTesting
+ long getMaxGcInterval() {
+ return gcInterval.as(Time.MILLISECONDS);
+ }
+
+ @VisibleForTesting
+ int getDelayMs() {
+ return rand.nextInt(gcInterval.as(Time.MILLISECONDS).intValue());
+ }
+
+ @VisibleForTesting
+ Optional<String> getGcExecutorPath() {
+ return gcExecutorPath;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/quota/QuotaComparisonResult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/quota/QuotaComparisonResult.java b/src/main/java/org/apache/aurora/scheduler/quota/QuotaComparisonResult.java
new file mode 100644
index 0000000..2ba2db8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/quota/QuotaComparisonResult.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.quota;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+
+/**
+ * Calculates and formats detailed quota comparison result.
+ */
+class QuotaComparisonResult {
+
+ private enum Resource {
+ CPU("core(s)"),
+ RAM("MB"),
+ DISK("MB");
+
+ private final String unit;
+ private Resource(String unit) {
+ this.unit = unit;
+ }
+ }
+
+ enum Result {
+ SUFFICIENT_QUOTA,
+ INSUFFICIENT_QUOTA
+ }
+
+ private final String details;
+ private final Result result;
+
+ @VisibleForTesting
+ QuotaComparisonResult(Result result, String details) {
+ this.result = result;
+ this.details = details;
+ }
+
+ Result result() {
+ return result;
+ }
+
+ String details() {
+ return details;
+ }
+
+ static QuotaComparisonResult greaterOrEqual(IQuota a, IQuota b) {
+ StringBuilder details = new StringBuilder();
+ boolean result = compare(a.getNumCpus(), b.getNumCpus(), Resource.CPU, details)
+ & compare(a.getRamMb(), b.getRamMb(), Resource.RAM, details)
+ & compare(a.getDiskMb(), b.getDiskMb(), Resource.DISK, details);
+
+ return new QuotaComparisonResult(
+ result ? Result.SUFFICIENT_QUOTA : Result.INSUFFICIENT_QUOTA,
+ details.toString());
+ }
+
+ private static boolean compare(
+ double a,
+ double b,
+ Resource resource,
+ StringBuilder details) {
+
+ boolean result = a >= b;
+ if (!result) {
+ details
+ .append(details.length() > 0 ? "; " : "")
+ .append(resource)
+ .append(" quota exceeded by ")
+ .append(String.format("%.2f", b - a))
+ .append(" ")
+ .append(resource.unit);
+ }
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/quota/QuotaFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/quota/QuotaFilter.java b/src/main/java/org/apache/aurora/scheduler/quota/QuotaFilter.java
new file mode 100644
index 0000000..cc67ba5
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/quota/QuotaFilter.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.quota;
+
+import javax.inject.Inject;
+
+import com.google.common.collect.Iterables;
+
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.quota.QuotaManager.QuotaManagerImpl;
+import com.twitter.aurora.scheduler.state.JobFilter;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.scheduler.quota.QuotaComparisonResult.Result.INSUFFICIENT_QUOTA;
+
+/**
+ * A filter that fails production jobs for roles that do not have sufficient quota to run them.
+ */
+class QuotaFilter implements JobFilter {
+ private final QuotaManagerImpl quotaManager;
+ private final Storage storage;
+
+ @Inject
+ QuotaFilter(QuotaManagerImpl quotaManager, Storage storage) {
+ this.quotaManager = checkNotNull(quotaManager);
+ this.storage = checkNotNull(storage);
+ }
+
+ @Override
+ public JobFilterResult filter(final IJobConfiguration job) {
+ return filterByTask(job.getKey(), job.getTaskConfig(), job.getInstanceCount());
+ }
+
+ @Override
+ public JobFilterResult filter(ITaskConfig template, int instanceCount) {
+ return filterByTask(JobKeys.from(template), template, instanceCount);
+ }
+
+ private synchronized JobFilterResult filterByTask(
+ IJobKey jobKey,
+ ITaskConfig template,
+ int instanceCount) {
+
+ if (!template.isProduction()) {
+ return JobFilterResult.pass();
+ }
+
+ IQuota currentUsage = Quotas.fromProductionTasks(
+ Iterables.transform(
+ Storage.Util.consistentFetchTasks(storage, Query.jobScoped(jobKey).active()),
+ Tasks.SCHEDULED_TO_INFO));
+
+ IQuota additionalRequested =
+ Quotas.subtract(Quotas.fromTasks(template, instanceCount), currentUsage);
+ QuotaComparisonResult comparisonResult =
+ quotaManager.checkQuota(jobKey.getRole(), additionalRequested);
+
+ if (comparisonResult.result() == INSUFFICIENT_QUOTA) {
+ return JobFilterResult.fail("Insufficient resource quota: " + comparisonResult.details());
+ }
+
+ return JobFilterResult.pass();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java b/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java
new file mode 100644
index 0000000..a2abedb
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.quota;
+
+import com.google.common.collect.Iterables;
+import com.google.inject.Inject;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.Storage.Work.Quiet;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * Allows access to resource quotas, and tracks quota consumption.
+ */
+public interface QuotaManager {
+ /**
+ * Fetches the current resource usage for the role.
+ *
+ * @param role to fetch quota usage for.
+ * @return Resource quota used by {@code role}.
+ */
+ IQuota getConsumption(String role);
+
+ /**
+ * Quota provider that stores quotas in the canonical {@link Storage} system.
+ */
+ static class QuotaManagerImpl implements QuotaManager {
+ private final Storage storage;
+
+ @Inject
+ public QuotaManagerImpl(Storage storage) {
+ this.storage = checkNotNull(storage);
+ }
+
+ @Override
+ public IQuota getConsumption(final String role) {
+ checkNotBlank(role);
+
+ final Query.Builder query = Query.roleScoped(role).active();
+
+ return storage.consistentRead(
+ new Work.Quiet<IQuota>() {
+ @Override public IQuota apply(StoreProvider storeProvider) {
+ return Quotas.fromProductionTasks(Iterables.transform(
+ storeProvider.getTaskStore().fetchTasks(query), Tasks.SCHEDULED_TO_INFO));
+ }
+ });
+ }
+
+ /**
+ * Tests whether the role has at least the specified amount of quota available.
+ *
+ * @param role Role to consume quota for.
+ * @param quota Quota amount to check for availability.
+ * @return QuotaComparisonResult with {@code result()} returning {@code true} if the role
+ * currently has at least {@code quota} quota remaining, {@code false} otherwise.
+ */
+ QuotaComparisonResult checkQuota(final String role, final IQuota quota) {
+ checkNotBlank(role);
+ checkNotNull(quota);
+
+ return storage.consistentRead(new Quiet<QuotaComparisonResult>() {
+ @Override public QuotaComparisonResult apply(StoreProvider storeProvider) {
+ IQuota reserved = storeProvider.getQuotaStore().fetchQuota(role).or(Quotas.noQuota());
+ return Quotas.greaterOrEqual(reserved, Quotas.add(getConsumption(role), quota));
+ }
+ });
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/quota/QuotaModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/quota/QuotaModule.java b/src/main/java/org/apache/aurora/scheduler/quota/QuotaModule.java
new file mode 100644
index 0000000..63e8888
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/quota/QuotaModule.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.quota;
+
+import javax.inject.Singleton;
+
+import com.google.inject.AbstractModule;
+
+import com.twitter.aurora.scheduler.quota.QuotaManager.QuotaManagerImpl;
+import com.twitter.aurora.scheduler.state.JobFilter;
+import com.twitter.aurora.scheduler.storage.Storage;
+
+/**
+ * Guice module for the quota package.
+ */
+public class QuotaModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ requireBinding(Storage.class);
+
+ bind(QuotaManager.class).to(QuotaManagerImpl.class);
+ bind(QuotaManagerImpl.class).in(Singleton.class);
+
+ bind(JobFilter.class).to(QuotaFilter.class);
+ bind(QuotaFilter.class).in(Singleton.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/quota/Quotas.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/quota/Quotas.java b/src/main/java/org/apache/aurora/scheduler/quota/Quotas.java
new file mode 100644
index 0000000..f7b6591
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/quota/Quotas.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.quota;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+
+import com.twitter.aurora.gen.Quota;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Convenience class for normalizing resource measures between tasks and offers.
+ */
+public final class Quotas {
+ private static final IQuota NO_QUOTA = IQuota.build(new Quota(0, 0, 0));
+
+ private Quotas() {
+ // Utility class.
+ }
+
+ /**
+ * Returns a quota with all resource vectors zeroed.
+ *
+ * @return A quota with all resource vectors zeroed.
+ */
+ public static IQuota noQuota() {
+ return NO_QUOTA;
+ }
+
+ /**
+ * Determines the amount of quota required for a set of job tasks.
+ *
+ * @param taskConfig Task template to count quota from.
+ * @return Quota requirement to run {@code job}.
+ */
+ public static IQuota fromTasks(ITaskConfig taskConfig, int instanceCount) {
+ return scale(fromProductionTasks(ImmutableSet.of(taskConfig)), instanceCount);
+ }
+
+ // TODO(Suman Karumuri): Refactor this function in to a new class.
+ // TODO(Suman Karumuri): Rename Quota to something more meaningful (ex: ResourceAggregate)
+ /**
+ * Determines the amount of quota required for production tasks among {@code tasks}.
+ *
+ * @param tasks Tasks to count quota from.
+ * @return Quota requirement to run {@code tasks}.
+ */
+ public static IQuota fromProductionTasks(Iterable<ITaskConfig> tasks) {
+ checkNotNull(tasks);
+
+ return fromTasks(Iterables.filter(tasks, Tasks.IS_PRODUCTION));
+ }
+
+ /**
+ * Determines the amount of quota required for the given tasks.
+ *
+ * @param tasks Tasks to count quota from.
+ * @return Quota requirement to run {@code tasks}.
+ */
+ public static IQuota fromTasks(Iterable<ITaskConfig> tasks) {
+ double cpu = 0;
+ int ramMb = 0;
+ int diskMb = 0;
+ for (ITaskConfig task : tasks) {
+ cpu += task.getNumCpus();
+ ramMb += task.getRamMb();
+ diskMb += task.getDiskMb();
+ }
+
+ return IQuota.build(new Quota()
+ .setNumCpus(cpu)
+ .setRamMb(ramMb)
+ .setDiskMb(diskMb));
+ }
+
+ /**
+ * a >= b
+ */
+ public static QuotaComparisonResult greaterOrEqual(IQuota a, IQuota b) {
+ return QuotaComparisonResult.greaterOrEqual(a, b);
+ }
+
+ /**
+ * a + b
+ */
+ public static IQuota add(IQuota a, IQuota b) {
+ return IQuota.build(new Quota()
+ .setNumCpus(a.getNumCpus() + b.getNumCpus())
+ .setRamMb(a.getRamMb() + b.getRamMb())
+ .setDiskMb(a.getDiskMb() + b.getDiskMb()));
+ }
+
+ /**
+ * a - b
+ */
+ public static IQuota subtract(IQuota a, IQuota b) {
+ return IQuota.build(new Quota()
+ .setNumCpus(a.getNumCpus() - b.getNumCpus())
+ .setRamMb(a.getRamMb() - b.getRamMb())
+ .setDiskMb(a.getDiskMb() - b.getDiskMb()));
+ }
+
+ /**
+ * a * m
+ */
+ public static IQuota scale(IQuota a, int m) {
+ return IQuota.build(new Quota()
+ .setNumCpus(a.getNumCpus() * m)
+ .setRamMb(a.getRamMb() * m)
+ .setDiskMb(a.getDiskMb() * m));
+ }
+
+ /**
+ * a / b
+ * <p>
+ * This calculates how many times {@code b} "fits into" {@code a}. Behavior is undefined when
+ * {@code b} contains resources with a value of zero.
+ */
+ public static int divide(IQuota a, IQuota b) {
+ return Ordering.natural().min(
+ a.getNumCpus() / b.getNumCpus(),
+ (double) a.getRamMb() / b.getRamMb(),
+ (double) a.getDiskMb() / b.getDiskMb()
+ ).intValue();
+ }
+
+ /**
+ * sum(qs)
+ */
+ public static IQuota sum(Iterable<IQuota> qs) {
+ IQuota sum = noQuota();
+ for (IQuota q : qs) {
+ sum = Quotas.add(sum, q);
+ }
+ return sum;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/state/CronJobManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/CronJobManager.java b/src/main/java/org/apache/aurora/scheduler/state/CronJobManager.java
new file mode 100644
index 0000000..d47c4af
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/state/CronJobManager.java
@@ -0,0 +1,472 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.state;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+import com.google.common.eventbus.Subscribe;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.twitter.aurora.gen.CronCollisionPolicy;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.ScheduleException;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
+import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
+import com.twitter.aurora.scheduler.cron.CronException;
+import com.twitter.aurora.scheduler.cron.CronScheduler;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.base.ExceptionalCommand;
+import com.twitter.common.base.Supplier;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.BackoffHelper;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
+
+/**
+ * A job scheduler that receives jobs that should be run periodically on a cron schedule.
+ */
+public class CronJobManager extends JobManager implements EventSubscriber {
+
+ public static final String MANAGER_KEY = "CRON";
+
+ @VisibleForTesting
+ static final String CRON_USER = "cron";
+
+ private static final Logger LOG = Logger.getLogger(CronJobManager.class.getName());
+
+ @CmdLine(name = "cron_start_initial_backoff", help =
+ "Initial backoff delay while waiting for a previous cron run to start.")
+ private static final Arg<Amount<Long, Time>> CRON_START_INITIAL_BACKOFF =
+ Arg.create(Amount.of(1L, Time.SECONDS));
+
+ @CmdLine(name = "cron_start_max_backoff", help =
+ "Max backoff delay while waiting for a previous cron run to start.")
+ private static final Arg<Amount<Long, Time>> CRON_START_MAX_BACKOFF =
+ Arg.create(Amount.of(1L, Time.MINUTES));
+
+ private final AtomicLong cronJobsTriggered = Stats.exportLong("cron_jobs_triggered");
+ private final AtomicLong cronJobLaunchFailures = Stats.exportLong("cron_job_launch_failures");
+
+ // Maps from the unique job identifier to the unique identifier used internally by the cron
+ // scheduler.
+ private final Map<IJobKey, String> scheduledJobs =
+ Collections.synchronizedMap(Maps.<IJobKey, String>newHashMap());
+
+ // Prevents runs from dogpiling while waiting for a run to transition out of the KILLING state.
+ // This is necessary because killing a job (if dictated by cron collision policy) is an
+ // asynchronous operation.
+ private final Map<IJobKey, SanitizedConfiguration> pendingRuns =
+ Collections.synchronizedMap(Maps.<IJobKey, SanitizedConfiguration>newHashMap());
+
+ private final StateManager stateManager;
+ private final Storage storage;
+ private final CronScheduler cron;
+ private final ShutdownRegistry shutdownRegistry;
+ private final BackoffHelper delayedStartBackoff;
+ private final Executor delayedRunExecutor;
+
+ @Inject
+ CronJobManager(
+ StateManager stateManager,
+ Storage storage,
+ CronScheduler cron,
+ ShutdownRegistry shutdownRegistry) {
+
+ this(
+ stateManager,
+ storage,
+ cron,
+ shutdownRegistry,
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("CronDelay-%d").build()));
+ }
+
+ @VisibleForTesting
+ CronJobManager(
+ StateManager stateManager,
+ Storage storage,
+ CronScheduler cron,
+ ShutdownRegistry shutdownRegistry,
+ Executor delayedRunExecutor) {
+
+ this.stateManager = checkNotNull(stateManager);
+ this.storage = checkNotNull(storage);
+ this.cron = checkNotNull(cron);
+ this.shutdownRegistry = checkNotNull(shutdownRegistry);
+ this.delayedStartBackoff =
+ new BackoffHelper(CRON_START_INITIAL_BACKOFF.get(), CRON_START_MAX_BACKOFF.get());
+ this.delayedRunExecutor = checkNotNull(delayedRunExecutor);
+
+ Stats.exportSize("cron_num_pending_runs", pendingRuns);
+ }
+
+ private void mapScheduledJob(IJobConfiguration job, String scheduledJobKey) {
+ IJobKey jobKey = job.getKey();
+ synchronized (scheduledJobs) {
+ Preconditions.checkState(
+ !scheduledJobs.containsKey(jobKey),
+ "Illegal state - cron schedule already exists for " + JobKeys.toPath(jobKey));
+ scheduledJobs.put(jobKey, scheduledJobKey);
+ }
+ }
+
+ /**
+ * Notifies the cron job manager that storage is started, and job configurations are ready to
+ * load.
+ *
+ * @param storageStarted Event.
+ */
+ @Subscribe
+ public void storageStarted(StorageStarted storageStarted) {
+ cron.start();
+ shutdownRegistry.addAction(new ExceptionalCommand<CronException>() {
+ @Override public void execute() throws CronException {
+ cron.stop();
+ }
+ });
+
+ Iterable<IJobConfiguration> crons =
+ storage.consistentRead(new Work.Quiet<Iterable<IJobConfiguration>>() {
+ @Override public Iterable<IJobConfiguration> apply(Storage.StoreProvider storeProvider) {
+ return storeProvider.getJobStore().fetchJobs(MANAGER_KEY);
+ }
+ });
+
+ for (IJobConfiguration job : crons) {
+ try {
+ mapScheduledJob(job, scheduleJob(SanitizedConfiguration.fromUnsanitized(job)));
+ } catch (ScheduleException | TaskDescriptionException e) {
+ logLaunchFailure(job, e);
+ }
+ }
+ }
+
+ private void logLaunchFailure(IJobConfiguration job, Exception e) {
+ cronJobLaunchFailures.incrementAndGet();
+ LOG.log(Level.SEVERE, "Scheduling failed for recovered job " + job, e);
+ }
+
+ /**
+ * Triggers execution of a job.
+ *
+ * @param jobKey Key of the job to start.
+ */
+ public void startJobNow(IJobKey jobKey) throws TaskDescriptionException {
+ checkNotNull(jobKey);
+
+ Optional<IJobConfiguration> jobConfig = fetchJob(jobKey);
+ checkArgument(jobConfig.isPresent(), "No such cron job " + JobKeys.toPath(jobKey));
+
+ cronTriggered(SanitizedConfiguration.fromUnsanitized(jobConfig.get()));
+ }
+
+ private void delayedRun(final Query.Builder query, final SanitizedConfiguration config) {
+ IJobConfiguration job = config.getJobConfig();
+ final String jobPath = JobKeys.toPath(job);
+ final IJobKey jobKey = job.getKey();
+ LOG.info("Waiting for job to terminate before launching cron job " + jobPath);
+ if (pendingRuns.put(jobKey, config) == null) {
+ LOG.info("Launching a task to wait for job to finish: " + jobPath);
+ // There was no run already pending for this job, launch a task to delay launch until the
+ // existing run has terminated.
+ delayedRunExecutor.execute(new Runnable() {
+ @Override public void run() {
+ runWhenTerminated(query, jobKey);
+ }
+ });
+ }
+ }
+
+ private void runWhenTerminated(final Query.Builder query, final IJobKey jobKey) {
+ try {
+ delayedStartBackoff.doUntilSuccess(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ if (!hasTasks(query)) {
+ LOG.info("Initiating delayed launch of cron " + jobKey);
+ SanitizedConfiguration config = pendingRuns.remove(jobKey);
+ checkNotNull(config, "Failed to fetch job for delayed run of " + jobKey);
+ LOG.info("Launching " + config.getTaskConfigs().size() + " tasks.");
+ stateManager.insertPendingTasks(config.getTaskConfigs());
+ return true;
+ } else {
+ LOG.info("Not yet safe to run cron " + jobKey);
+ return false;
+ }
+ }
+ });
+ } catch (InterruptedException e) {
+ LOG.log(Level.WARNING, "Interrupted while trying to launch cron " + jobKey, e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private boolean hasTasks(Query.Builder query) {
+ return !Storage.Util.consistentFetchTasks(storage, query).isEmpty();
+ }
+
+ public static CronCollisionPolicy orDefault(@Nullable CronCollisionPolicy policy) {
+ return Optional.fromNullable(policy).or(CronCollisionPolicy.KILL_EXISTING);
+ }
+
+ /**
+ * Triggers execution of a cron job, depending on the cron collision policy for the job.
+ *
+ * @param config The config of the job to be triggered.
+ */
+ @VisibleForTesting
+ void cronTriggered(SanitizedConfiguration config) {
+ IJobConfiguration job = config.getJobConfig();
+ LOG.info(String.format("Cron triggered for %s at %s with policy %s",
+ JobKeys.toPath(job), new Date(), job.getCronCollisionPolicy()));
+ cronJobsTriggered.incrementAndGet();
+
+ ImmutableMap.Builder<Integer, ITaskConfig> builder = ImmutableMap.builder();
+ final Query.Builder activeQuery = Query.jobScoped(job.getKey()).active();
+ Set<IScheduledTask> activeTasks = Storage.Util.consistentFetchTasks(storage, activeQuery);
+
+ if (activeTasks.isEmpty()) {
+ builder.putAll(config.getTaskConfigs());
+ } else {
+ // Assign a default collision policy.
+ CronCollisionPolicy collisionPolicy = orDefault(job.getCronCollisionPolicy());
+
+ switch (collisionPolicy) {
+ case KILL_EXISTING:
+ try {
+ schedulerCore.killTasks(activeQuery, CRON_USER);
+ // Check immediately if the tasks are gone. This could happen if the existing tasks
+ // were pending.
+ if (!hasTasks(activeQuery)) {
+ builder.putAll(config.getTaskConfigs());
+ } else {
+ delayedRun(activeQuery, config);
+ }
+ } catch (ScheduleException e) {
+ LOG.log(Level.SEVERE, "Failed to kill job.", e);
+ }
+ break;
+
+ case CANCEL_NEW:
+ break;
+
+ case RUN_OVERLAP:
+ Map<Integer, IScheduledTask> byInstance =
+ Maps.uniqueIndex(activeTasks, Tasks.SCHEDULED_TO_INSTANCE_ID);
+ Map<Integer, ScheduleStatus> existingTasks =
+ Maps.transformValues(byInstance, Tasks.GET_STATUS);
+ if (existingTasks.isEmpty()) {
+ builder.putAll(config.getTaskConfigs());
+ } else if (Iterables.any(existingTasks.values(), Predicates.equalTo(PENDING))) {
+ LOG.info("Job " + JobKeys.toPath(job) + " has pending tasks, suppressing run.");
+ } else {
+ // To safely overlap this run, we need to adjust the instance IDs of the overlapping
+ // run (maintaining the role/job/instance UUID invariant).
+ int instanceOffset = Ordering.natural().max(existingTasks.keySet()) + 1;
+ LOG.info("Adjusting instance IDs of " + JobKeys.toPath(job) + " by " + instanceOffset
+ + " for overlapping cron run.");
+ for (Map.Entry<Integer, ITaskConfig> entry : config.getTaskConfigs().entrySet()) {
+ builder.put(entry.getKey() + instanceOffset, entry.getValue());
+ }
+ }
+ break;
+
+ default:
+ LOG.severe("Unrecognized cron collision policy: " + job.getCronCollisionPolicy());
+ }
+ }
+
+ Map<Integer, ITaskConfig> newTasks = builder.build();
+ if (!newTasks.isEmpty()) {
+ stateManager.insertPendingTasks(newTasks);
+ }
+ }
+
+ /**
+ * Updates (re-schedules) the existing cron job.
+ *
+ * @param config New job configuration to update to.
+ * @throws ScheduleException If non-cron job confuration provided.
+ */
+ public void updateJob(SanitizedConfiguration config) throws ScheduleException {
+ IJobConfiguration job = config.getJobConfig();
+ if (!hasCronSchedule(job)) {
+ throw new ScheduleException("A cron job may not be updated to a non-cron job.");
+ }
+ String key = scheduledJobs.remove(job.getKey());
+ checkNotNull(key, "Attempted to update unknown job " + JobKeys.toPath(job));
+ cron.deschedule(key);
+ checkArgument(receiveJob(config));
+ }
+
+ @Override
+ public String getUniqueKey() {
+ return MANAGER_KEY;
+ }
+
+ private static boolean hasCronSchedule(IJobConfiguration job) {
+ checkNotNull(job);
+ return !StringUtils.isEmpty(job.getCronSchedule());
+ }
+
+ @Override
+ public boolean receiveJob(SanitizedConfiguration config) throws ScheduleException {
+ final IJobConfiguration job = config.getJobConfig();
+ if (!hasCronSchedule(job)) {
+ return false;
+ }
+
+ String scheduledJobKey = scheduleJob(config);
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(Storage.MutableStoreProvider storeProvider) {
+ storeProvider.getJobStore().saveAcceptedJob(MANAGER_KEY, job);
+ }
+ });
+ mapScheduledJob(job, scheduledJobKey);
+
+ return true;
+ }
+
+ private String scheduleJob(final SanitizedConfiguration config) throws ScheduleException {
+ final IJobConfiguration job = config.getJobConfig();
+ final String jobPath = JobKeys.toPath(job);
+ if (!hasCronSchedule(job)) {
+ throw new ScheduleException(
+ String.format("Not a valid cronjob, %s has no cron schedule", jobPath));
+ }
+
+ if (!cron.isValidSchedule(job.getCronSchedule())) {
+ throw new ScheduleException("Invalid cron schedule: " + job.getCronSchedule());
+ }
+
+ LOG.info(String.format("Scheduling cron job %s: %s", jobPath, job.getCronSchedule()));
+ try {
+ return cron.schedule(job.getCronSchedule(), new Runnable() {
+ @Override public void run() {
+ // TODO(William Farner): May want to record information about job runs.
+ LOG.info("Running cron job: " + jobPath);
+ cronTriggered(config);
+ }
+ });
+ } catch (CronException e) {
+ throw new ScheduleException("Failed to schedule cron job: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public Iterable<IJobConfiguration> getJobs() {
+ return storage.consistentRead(new Work.Quiet<Iterable<IJobConfiguration>>() {
+ @Override
+ public Iterable<IJobConfiguration> apply(Storage.StoreProvider storeProvider) {
+ return storeProvider.getJobStore().fetchJobs(MANAGER_KEY);
+ }
+ });
+ }
+
+ @Override
+ public boolean hasJob(IJobKey jobKey) {
+ return fetchJob(jobKey).isPresent();
+ }
+
+ private Optional<IJobConfiguration> fetchJob(final IJobKey jobKey) {
+ checkNotNull(jobKey);
+ return storage.consistentRead(new Work.Quiet<Optional<IJobConfiguration>>() {
+ @Override public Optional<IJobConfiguration> apply(Storage.StoreProvider storeProvider) {
+ return storeProvider.getJobStore().fetchJob(MANAGER_KEY, jobKey);
+ }
+ });
+ }
+
+ @Override
+ public boolean deleteJob(final IJobKey jobKey) {
+ checkNotNull(jobKey);
+
+ if (!hasJob(jobKey)) {
+ return false;
+ }
+
+ String scheduledJobKey = scheduledJobs.remove(jobKey);
+ if (scheduledJobKey != null) {
+ cron.deschedule(scheduledJobKey);
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(Storage.MutableStoreProvider storeProvider) {
+ storeProvider.getJobStore().removeJob(jobKey);
+ }
+ });
+ LOG.info("Successfully deleted cron job " + jobKey);
+ }
+ return true;
+ }
+
+ private final Function<String, String> keyToSchedule = new Function<String, String>() {
+ @Override public String apply(String key) {
+ return cron.getSchedule(key).or("Not found.");
+ }
+ };
+
+ public Map<IJobKey, String> getScheduledJobs() {
+ synchronized (scheduledJobs) {
+ return ImmutableMap.copyOf(Maps.transformValues(scheduledJobs, keyToSchedule));
+ }
+ }
+
+ public Set<IJobKey> getPendingRuns() {
+ synchronized (pendingRuns) {
+ return ImmutableSet.copyOf(pendingRuns.keySet());
+ }
+ }
+}
[49/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/app/Modules.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/app/Modules.java b/src/main/java/com/twitter/aurora/scheduler/app/Modules.java
deleted file mode 100644
index 72a80e1..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/app/Modules.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.app;
-
-import com.google.inject.Module;
-import com.google.inject.PrivateModule;
-
-/**
- * A utility class for managing guice modules.
- */
-final class Modules {
-
- private Modules() {
- // Utility class
- }
-
- private static Module instantiateModule(final Class<? extends Module> moduleClass) {
- try {
- return moduleClass.newInstance();
- } catch (InstantiationException e) {
- throw new IllegalArgumentException(
- String.format(
- "Failed to instantiate module %s. Are you sure it has a no-arg constructor?",
- moduleClass.getName()),
- e);
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException(
- String.format(
- "Failed to instantiate module %s. Are you sure it's public?",
- moduleClass.getName()),
- e);
- }
- }
-
- // Defensively wrap each module provided on the command-line in a PrivateModule that only
- // exposes requested classes to ensure that we don't depend on surprise extra bindings across
- // different implementations.
- static Module wrapInPrivateModule(
- Class<? extends Module> moduleClass,
- final Iterable<Class<?>> exposedClasses) {
-
- final Module module = instantiateModule(moduleClass);
- return new PrivateModule() {
- @Override protected void configure() {
- install(module);
- for (Class<?> klass : exposedClasses) {
- expose(klass);
- }
- }
- };
- }
-
- static Module getModule(Class<? extends Module> moduleClass) {
- return instantiateModule(moduleClass);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/app/SchedulerMain.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/app/SchedulerMain.java b/src/main/java/com/twitter/aurora/scheduler/app/SchedulerMain.java
deleted file mode 100644
index 693c364..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/app/SchedulerMain.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.app;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.logging.Logger;
-
-import javax.annotation.Nonnegative;
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableList.Builder;
-import com.google.inject.AbstractModule;
-import com.google.inject.Module;
-
-import com.twitter.aurora.auth.CapabilityValidator;
-import com.twitter.aurora.auth.SessionValidator;
-import com.twitter.aurora.auth.UnsecureAuthModule;
-import com.twitter.aurora.scheduler.DriverFactory;
-import com.twitter.aurora.scheduler.DriverFactory.DriverFactoryImpl;
-import com.twitter.aurora.scheduler.MesosTaskFactory.ExecutorConfig;
-import com.twitter.aurora.scheduler.SchedulerLifecycle;
-import com.twitter.aurora.scheduler.cron.CronPredictor;
-import com.twitter.aurora.scheduler.cron.CronScheduler;
-import com.twitter.aurora.scheduler.cron.noop.NoopCronModule;
-import com.twitter.aurora.scheduler.local.IsolatedSchedulerModule;
-import com.twitter.aurora.scheduler.log.mesos.MesosLogStreamModule;
-import com.twitter.aurora.scheduler.storage.backup.BackupModule;
-import com.twitter.aurora.scheduler.storage.log.LogStorage;
-import com.twitter.aurora.scheduler.storage.log.LogStorageModule;
-import com.twitter.aurora.scheduler.storage.log.SnapshotStoreImpl;
-import com.twitter.aurora.scheduler.storage.mem.MemStorageModule;
-import com.twitter.aurora.scheduler.thrift.ThriftConfiguration;
-import com.twitter.aurora.scheduler.thrift.ThriftModule;
-import com.twitter.aurora.scheduler.thrift.auth.ThriftAuthModule;
-import com.twitter.common.application.AbstractApplication;
-import com.twitter.common.application.AppLauncher;
-import com.twitter.common.application.Lifecycle;
-import com.twitter.common.application.modules.HttpModule;
-import com.twitter.common.application.modules.LocalServiceRegistry;
-import com.twitter.common.application.modules.LogModule;
-import com.twitter.common.application.modules.StatsModule;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.args.constraints.NotEmpty;
-import com.twitter.common.args.constraints.NotNull;
-import com.twitter.common.inject.Bindings;
-import com.twitter.common.logging.RootLogConfig;
-import com.twitter.common.zookeeper.Group;
-import com.twitter.common.zookeeper.SingletonService;
-import com.twitter.common.zookeeper.SingletonService.LeadershipListener;
-import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule;
-import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule.ClientConfig;
-import com.twitter.common.zookeeper.guice.client.flagged.FlaggedClientConfig;
-
-/**
- * Launcher for the aurora scheduler.
- */
-public class SchedulerMain extends AbstractApplication {
-
- private static final Logger LOG = Logger.getLogger(SchedulerMain.class.getName());
-
- @CmdLine(name = "testing_isolated_scheduler",
- help = "If true, run in a testing mode with the scheduler isolated from other components.")
- private static final Arg<Boolean> ISOLATED_SCHEDULER = Arg.create(false);
-
- @NotNull
- @CmdLine(name = "cluster_name", help = "Name to identify the cluster being served.")
- private static final Arg<String> CLUSTER_NAME = Arg.create();
-
- @NotNull
- @NotEmpty
- @CmdLine(name = "serverset_path", help = "ZooKeeper ServerSet path to register at.")
- private static final Arg<String> SERVERSET_PATH = Arg.create();
-
- @CmdLine(name = "mesos_ssl_keyfile",
- help = "JKS keyfile for operating the Mesos Thrift-over-SSL interface.")
- private static final Arg<File> MESOS_SSL_KEY_FILE = Arg.create();
-
- @Nonnegative
- @CmdLine(name = "thrift_port", help = "Thrift server port.")
- private static final Arg<Integer> THRIFT_PORT = Arg.create(0);
-
- @NotNull
- @CmdLine(name = "thermos_executor_path", help = "Path to the thermos executor launch script.")
- private static final Arg<String> THERMOS_EXECUTOR_PATH = Arg.create();
-
- @CmdLine(name = "auth_module",
- help = "A Guice module to provide auth bindings. NOTE: The default is unsecure.")
- private static final Arg<? extends Class<? extends Module>> AUTH_MODULE =
- Arg.create(UnsecureAuthModule.class);
-
- private static final Iterable<Class<?>> AUTH_MODULE_CLASSES = ImmutableList.<Class<?>>builder()
- .add(SessionValidator.class)
- .add(CapabilityValidator.class)
- .build();
-
- @CmdLine(name = "cron_module",
- help = "A Guice module to provide cron bindings. NOTE: The default is a no-op.")
- private static final Arg<? extends Class<? extends Module>> CRON_MODULE =
- Arg.create(NoopCronModule.class);
-
- private static final Iterable<Class<?>> CRON_MODULE_CLASSES = ImmutableList.<Class<?>>builder()
- .add(CronPredictor.class)
- .add(CronScheduler.class)
- .build();
-
- // TODO(Suman Karumuri): Pass in AUTH and CRON modules as extra modules
- @CmdLine(name = "extra_modules",
- help = "A list of modules that provide additional functionality.")
- private static final Arg<List<Class<? extends Module>>> EXTRA_MODULES =
- Arg.create((List<Class<? extends Module>>) ImmutableList.<Class<? extends Module>>of());
-
- @Inject private SingletonService schedulerService;
- @Inject private LocalServiceRegistry serviceRegistry;
- @Inject private SchedulerLifecycle schedulerLifecycle;
- @Inject private Lifecycle appLifecycle;
- @Inject private Optional<RootLogConfig.Configuration> glogConfig;
-
- private static Iterable<? extends Module> getSystemModules() {
- return ImmutableList.of(
- new LogModule(),
- new HttpModule(),
- new StatsModule()
- );
- }
-
- private static Iterable<? extends Module> getExtraModules() {
- Builder<Module> modules = ImmutableList.builder();
- modules.add(Modules.wrapInPrivateModule(AUTH_MODULE.get(), AUTH_MODULE_CLASSES))
- .add(Modules.wrapInPrivateModule(CRON_MODULE.get(), CRON_MODULE_CLASSES));
-
- for (Class<? extends Module> moduleClass : EXTRA_MODULES.get()) {
- modules.add(Modules.getModule(moduleClass));
- }
-
- return modules.build();
- }
-
- static Iterable<? extends Module> getModules(
- String clusterName,
- String serverSetPath,
- ClientConfig zkClientConfig) {
-
- return ImmutableList.<Module>builder()
- .addAll(getSystemModules())
- .add(new AppModule(clusterName, serverSetPath, zkClientConfig))
- .addAll(getExtraModules())
- .add(new LogStorageModule())
- .add(new MemStorageModule(Bindings.annotatedKeyFactory(LogStorage.WriteBehind.class)))
- .add(new ThriftModule())
- .add(new ThriftAuthModule())
- .build();
- }
-
- @Override
- public Iterable<? extends Module> getModules() {
- Module additional;
- final ClientConfig zkClientConfig = FlaggedClientConfig.create();
- if (ISOLATED_SCHEDULER.get()) {
- additional = new IsolatedSchedulerModule();
- } else {
- // TODO(Kevin Sweeney): Push these bindings down into a "production" module.
- additional = new AbstractModule() {
- @Override protected void configure() {
- bind(DriverFactory.class).to(DriverFactoryImpl.class);
- bind(DriverFactoryImpl.class).in(Singleton.class);
- install(new MesosLogStreamModule(zkClientConfig));
- }
- };
- }
-
- Module configModule = new AbstractModule() {
- @Override protected void configure() {
- bind(ThriftConfiguration.class).toInstance(new ThriftConfiguration() {
- @Override public Optional<InputStream> getSslKeyStream() throws FileNotFoundException {
- if (MESOS_SSL_KEY_FILE.hasAppliedValue()) {
- return Optional.<InputStream>of(new FileInputStream(MESOS_SSL_KEY_FILE.get()));
- } else {
- return Optional.absent();
- }
- }
-
- @Override public int getServingPort() {
- return THRIFT_PORT.get();
- }
- });
- bind(ExecutorConfig.class).toInstance(new ExecutorConfig(THERMOS_EXECUTOR_PATH.get()));
- }
- };
-
- return ImmutableList.<Module>builder()
- .add(new BackupModule(SnapshotStoreImpl.class))
- .addAll(getModules(CLUSTER_NAME.get(), SERVERSET_PATH.get(), zkClientConfig))
- .add(new ZooKeeperClientModule(zkClientConfig))
- .add(configModule)
- .add(additional)
- .build();
- }
-
- @Override
- public void run() {
- if (glogConfig.isPresent()) {
- // Setup log4j to match our jul glog config in order to pick up zookeeper logging.
- Log4jConfigurator.configureConsole(glogConfig.get());
- } else {
- LOG.warning("Running without expected glog configuration.");
- }
-
- LeadershipListener leaderListener = schedulerLifecycle.prepare();
-
- Optional<InetSocketAddress> primarySocket = serviceRegistry.getPrimarySocket();
- if (!primarySocket.isPresent()) {
- throw new IllegalStateException("No primary service registered with LocalServiceRegistry.");
- }
-
- try {
- schedulerService.lead(
- primarySocket.get(),
- serviceRegistry.getAuxiliarySockets(),
- leaderListener);
- } catch (Group.WatchException e) {
- throw new IllegalStateException("Failed to watch group and lead service.", e);
- } catch (Group.JoinException e) {
- throw new IllegalStateException("Failed to join scheduler service group.", e);
- } catch (InterruptedException e) {
- throw new IllegalStateException("Interrupted while joining scheduler service group.", e);
- }
-
- appLifecycle.awaitShutdown();
- }
-
- public static void main(String[] args) {
- AppLauncher.launch(SchedulerMain.class, args);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java b/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java
deleted file mode 100644
index faf3269..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/AsyncModule.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.logging.Logger;
-
-import javax.inject.Singleton;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.RateLimiter;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.inject.AbstractModule;
-import com.google.inject.Binder;
-import com.google.inject.BindingAnnotation;
-import com.google.inject.Key;
-import com.google.inject.PrivateModule;
-import com.google.inject.TypeLiteral;
-
-import com.twitter.aurora.scheduler.async.OfferQueue.OfferQueueImpl;
-import com.twitter.aurora.scheduler.async.OfferQueue.OfferReturnDelay;
-import com.twitter.aurora.scheduler.async.RescheduleCalculator.RescheduleCalculatorImpl;
-import com.twitter.aurora.scheduler.async.TaskGroups.TaskGroupsSettings;
-import com.twitter.aurora.scheduler.async.TaskScheduler.TaskSchedulerImpl;
-import com.twitter.aurora.scheduler.events.PubsubEventModule;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.StatImpl;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.stats.StatsProvider;
-import com.twitter.common.util.Random;
-import com.twitter.common.util.TruncatedBinaryBackoff;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import static com.twitter.aurora.scheduler.async.HistoryPruner.PruneThreshold;
-import static com.twitter.aurora.scheduler.async.Preemptor.PreemptorImpl;
-import static com.twitter.aurora.scheduler.async.Preemptor.PreemptorImpl.PreemptionDelay;
-import static com.twitter.aurora.scheduler.async.TaskScheduler.TaskSchedulerImpl.ReservationDuration;
-
-/**
- * Binding module for async task management.
- */
-public class AsyncModule extends AbstractModule {
-
- private static final Logger LOG = Logger.getLogger(AsyncModule.class.getName());
-
- @CmdLine(name = "async_worker_threads",
- help = "The number of worker threads to process async task operations with.")
- private static final Arg<Integer> ASYNC_WORKER_THREADS = Arg.create(1);
-
- @CmdLine(name = "transient_task_state_timeout",
- help = "The amount of time after which to treat a task stuck in a transient state as LOST.")
- private static final Arg<Amount<Long, Time>> TRANSIENT_TASK_STATE_TIMEOUT =
- Arg.create(Amount.of(5L, Time.MINUTES));
-
- @CmdLine(name = "initial_schedule_delay",
- help = "Initial amount of time to wait before attempting to schedule a PENDING task.")
- private static final Arg<Amount<Long, Time>> INITIAL_SCHEDULE_DELAY =
- Arg.create(Amount.of(1L, Time.SECONDS));
-
- @CmdLine(name = "max_schedule_delay",
- help = "Maximum delay between attempts to schedule a PENDING tasks.")
- private static final Arg<Amount<Long, Time>> MAX_SCHEDULE_DELAY =
- Arg.create(Amount.of(30L, Time.SECONDS));
-
- @CmdLine(name = "min_offer_hold_time",
- help = "Minimum amount of time to hold a resource offer before declining.")
- private static final Arg<Amount<Integer, Time>> MIN_OFFER_HOLD_TIME =
- Arg.create(Amount.of(5, Time.MINUTES));
-
- @CmdLine(name = "history_prune_threshold",
- help = "Time after which the scheduler will prune terminated task history.")
- private static final Arg<Amount<Long, Time>> HISTORY_PRUNE_THRESHOLD =
- Arg.create(Amount.of(2L, Time.DAYS));
-
- @CmdLine(name = "max_schedule_attempts_per_sec",
- help = "Maximum number of scheduling attempts to make per second.")
- private static final Arg<Double> MAX_SCHEDULE_ATTEMPTS_PER_SEC = Arg.create(10D);
-
- @CmdLine(name = "flapping_task_threshold",
- help = "A task that repeatedly runs for less than this time is considered to be flapping.")
- private static final Arg<Amount<Long, Time>> FLAPPING_THRESHOLD =
- Arg.create(Amount.of(5L, Time.MINUTES));
-
- @CmdLine(name = "initial_flapping_task_delay",
- help = "Initial amount of time to wait before attempting to schedule a flapping task.")
- private static final Arg<Amount<Long, Time>> INITIAL_FLAPPING_DELAY =
- Arg.create(Amount.of(30L, Time.SECONDS));
-
- @CmdLine(name = "max_flapping_task_delay",
- help = "Maximum delay between attempts to schedule a flapping task.")
- private static final Arg<Amount<Long, Time>> MAX_FLAPPING_DELAY =
- Arg.create(Amount.of(5L, Time.MINUTES));
-
- @CmdLine(name = "max_reschedule_task_delay_on_startup",
- help = "Upper bound of random delay for pending task rescheduling on scheduler startup.")
- private static final Arg<Amount<Integer, Time>> MAX_RESCHEDULING_DELAY =
- Arg.create(Amount.of(30, Time.SECONDS));
-
- @CmdLine(name = "preemption_delay",
- help = "Time interval after which a pending task becomes eligible to preempt other tasks")
- private static final Arg<Amount<Long, Time>> PREEMPTION_DELAY =
- Arg.create(Amount.of(10L, Time.MINUTES));
-
- @CmdLine(name = "enable_preemptor",
- help = "Enable the preemptor and preemption")
- private static final Arg<Boolean> ENABLE_PREEMPTOR = Arg.create(true);
-
- private static final Preemptor NULL_PREEMPTOR = new Preemptor() {
- @Override public Optional<String> findPreemptionSlotFor(String taskId) {
- return Optional.absent();
- }
- };
-
- @CmdLine(name = "offer_reservation_duration", help = "Time to reserve a slave's offers while "
- + "trying to satisfy a task preempting another.")
- private static final Arg<Amount<Long, Time>> RESERVATION_DURATION =
- Arg.create(Amount.of(3L, Time.MINUTES));
-
- @BindingAnnotation
- @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
- private @interface PreemptionBinding { }
-
- @VisibleForTesting
- static final Key<Preemptor> PREEMPTOR_KEY = Key.get(Preemptor.class, PreemptionBinding.class);
-
- @Override
- protected void configure() {
- // Don't worry about clean shutdown, these can be daemon and cleanup-free.
- final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
- ASYNC_WORKER_THREADS.get(),
- new ThreadFactoryBuilder().setNameFormat("AsyncProcessor-%d").setDaemon(true).build());
- Stats.exportSize("timeout_queue_size", executor.getQueue());
- Stats.export(new StatImpl<Long>("async_tasks_completed") {
- @Override public Long read() {
- return executor.getCompletedTaskCount();
- }
- });
-
- // AsyncModule itself is not a subclass of PrivateModule because TaskEventModule internally uses
- // a MultiBinder, which cannot span multiple injectors.
- binder().install(new PrivateModule() {
- @Override protected void configure() {
- bind(new TypeLiteral<Amount<Long, Time>>() { })
- .toInstance(TRANSIENT_TASK_STATE_TIMEOUT.get());
- bind(ScheduledExecutorService.class).toInstance(executor);
-
- bind(TaskTimeout.class).in(Singleton.class);
- requireBinding(StatsProvider.class);
- expose(TaskTimeout.class);
- }
- });
- PubsubEventModule.bindSubscriber(binder(), TaskTimeout.class);
-
- binder().install(new PrivateModule() {
- @Override protected void configure() {
- bind(TaskGroupsSettings.class).toInstance(new TaskGroupsSettings(
- new TruncatedBinaryBackoff(INITIAL_SCHEDULE_DELAY.get(), MAX_SCHEDULE_DELAY.get()),
- RateLimiter.create(MAX_SCHEDULE_ATTEMPTS_PER_SEC.get())));
-
- bind(RescheduleCalculatorImpl.RescheduleCalculatorSettings.class)
- .toInstance(new RescheduleCalculatorImpl.RescheduleCalculatorSettings(
- new TruncatedBinaryBackoff(INITIAL_FLAPPING_DELAY.get(), MAX_FLAPPING_DELAY.get()),
- FLAPPING_THRESHOLD.get(),
- MAX_RESCHEDULING_DELAY.get()));
-
- bind(RescheduleCalculator.class).to(RescheduleCalculatorImpl.class).in(Singleton.class);
- if (ENABLE_PREEMPTOR.get()) {
- bind(PREEMPTOR_KEY).to(PreemptorImpl.class);
- bind(PreemptorImpl.class).in(Singleton.class);
- LOG.info("Preemptor Enabled.");
- } else {
- bind(PREEMPTOR_KEY).toInstance(NULL_PREEMPTOR);
- LOG.warning("Preemptor Disabled.");
- }
- expose(PREEMPTOR_KEY);
- bind(new TypeLiteral<Amount<Long, Time>>() {
- }).annotatedWith(PreemptionDelay.class)
- .toInstance(PREEMPTION_DELAY.get());
- bind(TaskGroups.class).in(Singleton.class);
- expose(TaskGroups.class);
- }
- });
- bindTaskScheduler(binder(), PREEMPTOR_KEY, RESERVATION_DURATION.get());
- PubsubEventModule.bindSubscriber(binder(), TaskGroups.class);
-
- binder().install(new PrivateModule() {
- @Override protected void configure() {
- bind(OfferReturnDelay.class).to(RandomJitterReturnDelay.class);
- bind(ScheduledExecutorService.class).toInstance(executor);
- bind(OfferQueue.class).to(OfferQueueImpl.class);
- bind(OfferQueueImpl.class).in(Singleton.class);
- expose(OfferQueue.class);
- }
- });
- PubsubEventModule.bindSubscriber(binder(), OfferQueue.class);
-
- binder().install(new PrivateModule() {
- @Override protected void configure() {
- // TODO(ksweeney): Create a configuration validator module so this can be injected.
- // TODO(William Farner): Revert this once large task counts is cheap ala hierarchichal store
- bind(Integer.class).annotatedWith(PruneThreshold.class).toInstance(100);
- bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(PruneThreshold.class)
- .toInstance(HISTORY_PRUNE_THRESHOLD.get());
- bind(ScheduledExecutorService.class).toInstance(executor);
-
- bind(HistoryPruner.class).in(Singleton.class);
- expose(HistoryPruner.class);
- }
- });
- PubsubEventModule.bindSubscriber(binder(), HistoryPruner.class);
- }
-
- /**
- * This method exists because we want to test the wiring up of TaskSchedulerImpl class to the
- * PubSub system in the TaskSchedulerImplTest class. The method has a complex signature because
- * the binding of the TaskScheduler and friends occurs in a PrivateModule which does not interact
- * well with the MultiBinder that backs the PubSub system.
- */
- @VisibleForTesting
- static void bindTaskScheduler(
- Binder binder,
- final Key<Preemptor> preemptorKey,
- final Amount<Long, Time> reservationDuration) {
- binder.install(new PrivateModule() {
- @Override protected void configure() {
- bind(Preemptor.class).to(preemptorKey);
- bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(ReservationDuration.class)
- .toInstance(reservationDuration);
- bind(TaskScheduler.class).to(TaskSchedulerImpl.class);
- bind(TaskSchedulerImpl.class).in(Singleton.class);
- expose(TaskScheduler.class);
- }
- });
- PubsubEventModule.bindSubscriber(binder, TaskScheduler.class);
- }
-
- /**
- * Returns offers after a random duration within a fixed window.
- */
- private static class RandomJitterReturnDelay implements OfferReturnDelay {
- private static final int JITTER_WINDOW_MS = Amount.of(1, Time.MINUTES).as(Time.MILLISECONDS);
-
- private final int minHoldTimeMs = MIN_OFFER_HOLD_TIME.get().as(Time.MILLISECONDS);
- private final Random random = new Random.SystemRandom(new java.util.Random());
-
- @Override public Amount<Integer, Time> get() {
- return Amount.of(minHoldTimeMs + random.nextInt(JITTER_WINDOW_MS), Time.MILLISECONDS);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/HistoryPruner.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/HistoryPruner.java b/src/main/java/com/twitter/aurora/scheduler/async/HistoryPruner.java
deleted file mode 100644
index 9af6d36..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/HistoryPruner.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.LinkedHashMultimap;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-import com.google.common.eventbus.Subscribe;
-import com.google.inject.BindingAnnotation;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.state.StateManager;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.util.Clock;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.scheduler.base.Tasks.LATEST_ACTIVITY;
-import static com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import static com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
-import static com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import static com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
-
-/**
- * Prunes tasks in a job based on per-job history and an inactive time threshold by observing tasks
- * transitioning into one of the inactive states.
- */
-public class HistoryPruner implements EventSubscriber {
- private static final Logger LOG = Logger.getLogger(HistoryPruner.class.getName());
-
- @VisibleForTesting
- static final Query.Builder INACTIVE_QUERY = Query.unscoped().terminal();
-
- private final Multimap<IJobKey, String> tasksByJob =
- Multimaps.synchronizedSetMultimap(LinkedHashMultimap.<IJobKey, String>create());
- @VisibleForTesting
- Multimap<IJobKey, String> getTasksByJob() {
- return tasksByJob;
- }
-
- private final ScheduledExecutorService executor;
- private final Storage storage;
- private final StateManager stateManager;
- private final Clock clock;
- private final long pruneThresholdMillis;
- private final int perJobHistoryGoal;
- private final Map<String, Future<?>> taskIdToFuture = Maps.newConcurrentMap();
-
- @BindingAnnotation
- @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
- public @interface PruneThreshold { }
-
- @Inject
- HistoryPruner(
- final ScheduledExecutorService executor,
- final Storage storage,
- final StateManager stateManager,
- final Clock clock,
- @PruneThreshold Amount<Long, Time> inactivePruneThreshold,
- @PruneThreshold int perJobHistoryGoal) {
-
- this.executor = checkNotNull(executor);
- this.storage = checkNotNull(storage);
- this.stateManager = checkNotNull(stateManager);
- this.clock = checkNotNull(clock);
- this.pruneThresholdMillis = inactivePruneThreshold.as(Time.MILLISECONDS);
- this.perJobHistoryGoal = perJobHistoryGoal;
- }
-
- @VisibleForTesting
- long calculateTimeout(long taskEventTimestampMillis) {
- return pruneThresholdMillis - Math.max(0, clock.nowMillis() - taskEventTimestampMillis);
- }
-
- /**
- * When triggered, records an inactive task state change.
- *
- * @param change Event when a task changes state.
- */
- @Subscribe
- public void recordStateChange(TaskStateChange change) {
- if (Tasks.isTerminated(change.getNewState())) {
- registerInactiveTask(
- Tasks.SCHEDULED_TO_JOB_KEY.apply(change.getTask()),
- change.getTaskId(),
- calculateTimeout(clock.nowMillis()));
- }
- }
-
- /**
- * When triggered, iterates through inactive tasks in the system and prunes tasks that
- * exceed the history goal for a job or are beyond the time threshold.
- *
- * @param event A new StorageStarted event.
- */
- @Subscribe
- public void storageStarted(StorageStarted event) {
- for (IScheduledTask task
- : LATEST_ACTIVITY.sortedCopy(Storage.Util.consistentFetchTasks(storage, INACTIVE_QUERY))) {
-
- registerInactiveTask(
- Tasks.SCHEDULED_TO_JOB_KEY.apply(task),
- Tasks.id(task),
- calculateTimeout(Iterables.getLast(task.getTaskEvents()).getTimestamp()));
- }
- }
-
- private void deleteTasks(Set<String> taskIds) {
- LOG.info("Pruning inactive tasks " + taskIds);
- stateManager.deleteTasks(taskIds);
- }
-
- /**
- * When triggered, removes the tasks scheduled for pruning and cancels any existing future.
- *
- * @param event A new TasksDeleted event.
- */
- @Subscribe
- public void tasksDeleted(final TasksDeleted event) {
- for (IScheduledTask task : event.getTasks()) {
- String id = Tasks.id(task);
- tasksByJob.remove(Tasks.SCHEDULED_TO_JOB_KEY.apply(task), id);
- Future<?> future = taskIdToFuture.remove(id);
- if (future != null) {
- future.cancel(false);
- }
- }
- }
-
- private void registerInactiveTask(
- final IJobKey jobKey,
- final String taskId,
- long timeRemaining) {
-
- LOG.fine("Prune task " + taskId + " in " + timeRemaining + " ms.");
- // Insert the latest inactive task at the tail.
- tasksByJob.put(jobKey, taskId);
- Runnable runnable = new Runnable() {
- @Override public void run() {
- LOG.info("Pruning expired inactive task " + taskId);
- tasksByJob.remove(jobKey, taskId);
- taskIdToFuture.remove(taskId);
- deleteTasks(ImmutableSet.of(taskId));
- }
- };
- taskIdToFuture.put(taskId, executor.schedule(runnable, timeRemaining, TimeUnit.MILLISECONDS));
-
- ImmutableSet.Builder<String> pruneTaskIds = ImmutableSet.builder();
- Collection<String> tasks = tasksByJob.get(jobKey);
- // From Multimaps javadoc: "It is imperative that the user manually synchronize on the returned
- // multimap when accessing any of its collection views".
- synchronized (tasksByJob) {
- Iterator<String> iterator = tasks.iterator();
- while (tasks.size() > perJobHistoryGoal) {
- // Pick oldest task from the head. Guaranteed by LinkedHashMultimap based on insertion
- // order.
- String id = iterator.next();
- iterator.remove();
- pruneTaskIds.add(id);
- Future<?> future = taskIdToFuture.remove(id);
- if (future != null) {
- future.cancel(false);
- }
- }
- }
-
- Set<String> ids = pruneTaskIds.build();
- if (!ids.isEmpty()) {
- deleteTasks(ids);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/OfferQueue.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/OfferQueue.java b/src/main/java/com/twitter/aurora/scheduler/async/OfferQueue.java
deleted file mode 100644
index 95334ff..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/OfferQueue.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListSet;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Supplier;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-import com.google.common.eventbus.Subscribe;
-
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.TaskInfo;
-
-import com.twitter.aurora.gen.HostStatus;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.scheduler.Driver;
-import com.twitter.aurora.scheduler.events.PubsubEvent.DriverDisconnected;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.HostMaintenanceStateChange;
-import com.twitter.aurora.scheduler.state.MaintenanceController;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
-import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
-import static com.twitter.aurora.gen.MaintenanceMode.NONE;
-import static com.twitter.aurora.gen.MaintenanceMode.SCHEDULED;
-
-/**
- * Tracks the Offers currently known by the scheduler
- */
-public interface OfferQueue extends EventSubscriber {
-
- /**
- * Notifies the scheduler of a new resource offer.
- *
- * @param offer Newly-available resource offer.
- */
- void addOffer(Offer offer);
-
- /**
- * Invalidates an offer. This indicates that the scheduler should not attempt to match any
- * tasks against the offer.
- *
- * @param offer Canceled offer.
- */
- void cancelOffer(OfferID offer);
-
- /**
- * Launches the first task that satisfies the {@code acceptor} by returning a {@link TaskInfo}.
- *
- * @param acceptor Function that determines if an offer is accepted.
- * @return {@code true} if the task was launched, {@code false} if no offers satisfied the
- * {@code acceptor}.
- * @throws LaunchException If the acceptor accepted an offer, but there was an error launching the
- * task.
- */
- boolean launchFirst(Function<Offer, Optional<TaskInfo>> acceptor) throws LaunchException;
-
- /**
- * Notifies the offer queue that a host has changed state.
- *
- * @param change State change notification.
- */
- void hostChangedState(HostMaintenanceStateChange change);
-
- /**
- * Gets the offers that the scheduler is holding.
- *
- * @return A snapshot of the offers that the scheduler is currently holding.
- */
- Iterable<Offer> getOffers();
-
- /**
- * Calculates the amount of time before an offer should be 'returned' by declining it.
- * The delay is calculated for each offer that is received, so the return delay may be
- * fixed or variable.
- */
- public interface OfferReturnDelay extends Supplier<Amount<Integer, Time>> {
- }
-
- /**
- * Thrown when there was an unexpected failure trying to launch a task.
- */
- static class LaunchException extends Exception {
- LaunchException(String msg) {
- super(msg);
- }
-
- LaunchException(String msg, Throwable cause) {
- super(msg, cause);
- }
- }
-
- class OfferQueueImpl implements OfferQueue {
- private static final Logger LOG = Logger.getLogger(OfferQueueImpl.class.getName());
-
- static final Comparator<HostOffer> PREFERENCE_COMPARATOR =
- // Currently, the only preference is based on host maintenance status.
- Ordering.explicit(NONE, SCHEDULED, DRAINING, DRAINED)
- .onResultOf(new Function<HostOffer, MaintenanceMode>() {
- @Override public MaintenanceMode apply(HostOffer offer) {
- return offer.mode;
- }
- })
- .compound(Ordering.arbitrary());
-
- private final Set<HostOffer> hostOffers = new ConcurrentSkipListSet<>(PREFERENCE_COMPARATOR);
- private final AtomicLong offerRaces = Stats.exportLong("offer_accept_races");
-
- private final Driver driver;
- private final OfferReturnDelay returnDelay;
- private final ScheduledExecutorService executor;
- private final MaintenanceController maintenance;
-
- @Inject
- OfferQueueImpl(Driver driver,
- OfferReturnDelay returnDelay,
- ScheduledExecutorService executor,
- MaintenanceController maintenance) {
-
- this.driver = driver;
- this.returnDelay = returnDelay;
- this.executor = executor;
- this.maintenance = maintenance;
- // Potential gotcha - since this is now a ConcurrentSkipListSet, size() is more expensive.
- // Could track this separately if it turns out to pose problems.
- Stats.exportSize("outstanding_offers", hostOffers);
- }
-
- @Override
- public void addOffer(final Offer offer) {
- // We run a slight risk of a race here, which is acceptable. The worst case is that we
- // temporarily hold two offers for the same host, which should be corrected when we return
- // them after the return delay.
- // There's also a chance that we return an offer for compaction ~simultaneously with the
- // same-host offer being canceled/returned. This is also fine.
- List<HostOffer> sameSlave = FluentIterable.from(hostOffers)
- .filter(new Predicate<HostOffer>() {
- @Override public boolean apply(HostOffer hostOffer) {
- return hostOffer.offer.getSlaveId().equals(offer.getSlaveId());
- }
- })
- .toList();
- if (sameSlave.isEmpty()) {
- hostOffers.add(new HostOffer(offer, maintenance.getMode(offer.getHostname())));
- executor.schedule(
- new Runnable() {
- @Override public void run() {
- removeAndDecline(offer.getId());
- }
- },
- returnDelay.get().as(Time.MILLISECONDS),
- TimeUnit.MILLISECONDS);
- } else {
- // If there are existing offers for the slave, decline all of them so the master can
- // compact all of those offers into a single offer and send them back.
- LOG.info("Returning " + (sameSlave.size() + 1)
- + " offers for " + offer.getSlaveId().getValue() + " for compaction.");
- decline(offer.getId());
- for (HostOffer sameSlaveOffer : sameSlave) {
- removeAndDecline(sameSlaveOffer.offer.getId());
- }
- }
- }
-
- void removeAndDecline(OfferID id) {
- if (removeFromHostOffers(id)) {
- decline(id);
- }
- }
-
- void decline(OfferID id) {
- LOG.fine("Declining offer " + id);
- driver.declineOffer(id);
- }
-
- @Override
- public void cancelOffer(final OfferID offerId) {
- removeFromHostOffers(offerId);
- }
-
- private boolean removeFromHostOffers(final OfferID offerId) {
- Preconditions.checkNotNull(offerId);
-
- // The small risk of inconsistency is acceptable here - if we have an accept/remove race
- // on an offer, the master will mark the task as LOST and it will be retried.
- return Iterables.removeIf(hostOffers,
- new Predicate<HostOffer>() {
- @Override public boolean apply(HostOffer input) {
- return input.offer.getId().equals(offerId);
- }
- });
- }
-
- @Override
- public Iterable<Offer> getOffers() {
- return Iterables.unmodifiableIterable(
- FluentIterable.from(hostOffers)
- .transform(new Function<HostOffer, Offer>() {
- @Override public Offer apply(HostOffer offer) {
- return offer.offer;
- }
- }));
- }
-
- /**
- * Updates the preference of a host's offers.
- *
- * @param change Host change notification.
- */
- @Subscribe
- public void hostChangedState(HostMaintenanceStateChange change) {
- final HostStatus hostStatus = change.getStatus();
-
- // Remove and re-add a host's offers to re-sort based on its new hostStatus
- Set<HostOffer> changedOffers = FluentIterable.from(hostOffers)
- .filter(new Predicate<HostOffer>() {
- @Override public boolean apply(HostOffer hostOffer) {
- return hostOffer.offer.getHostname().equals(hostStatus.getHost());
- }
- })
- .toSet();
- hostOffers.removeAll(changedOffers);
- hostOffers.addAll(
- FluentIterable.from(changedOffers)
- .transform(new Function<HostOffer, HostOffer>() {
- @Override public HostOffer apply(HostOffer hostOffer) {
- return new HostOffer(hostOffer.offer, hostStatus.getMode());
- }
- })
- .toSet());
- }
-
- /**
- * Notifies the queue that the driver is disconnected, and all the stored offers are now
- * invalid.
- * <p>
- * The queue takes this as a signal to flush its queue.
- *
- * @param event Disconnected event.
- */
- @Subscribe
- public void driverDisconnected(DriverDisconnected event) {
- LOG.info("Clearing stale offers since the driver is disconnected.");
- hostOffers.clear();
- }
-
- /**
- * Encapsulate an offer from a host, and the host's maintenance mode.
- */
- private static class HostOffer {
- private final Offer offer;
- private final MaintenanceMode mode;
-
- HostOffer(Offer offer, MaintenanceMode mode) {
- this.offer = offer;
- this.mode = mode;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof HostOffer)) {
- return false;
- }
- HostOffer other = (HostOffer) o;
- return Objects.equal(offer, other.offer) && (mode == other.mode);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(offer, mode);
- }
- }
-
- @Override
- public boolean launchFirst(Function<Offer, Optional<TaskInfo>> acceptor)
- throws LaunchException {
-
- // It's important that this method is not called concurrently - doing so would open up the
- // possibility of a race between the same offers being accepted by different threads.
-
- for (HostOffer hostOffer : hostOffers) {
- Optional<TaskInfo> assignment = acceptor.apply(hostOffer.offer);
- if (assignment.isPresent()) {
- // Guard against an offer being removed after we grabbed it from the iterator.
- // If that happens, the offer will not exist in hostOffers, and we can immediately
- // send it back to LOST for quick reschedule.
- if (hostOffers.remove(hostOffer)) {
- try {
- driver.launchTask(hostOffer.offer.getId(), assignment.get());
- return true;
- } catch (IllegalStateException e) {
- // TODO(William Farner): Catch only the checked exception produced by Driver
- // once it changes from throwing IllegalStateException when the driver is not yet
- // registered.
- throw new LaunchException("Failed to launch task.", e);
- }
- } else {
- offerRaces.incrementAndGet();
- throw new LaunchException(
- "Accepted offer no longer exists in offer queue, likely data race.");
- }
- }
- }
-
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/Preemptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/Preemptor.java b/src/main/java/com/twitter/aurora/scheduler/async/Preemptor.java
deleted file mode 100644
index a01790c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/Preemptor.java
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.Sets;
-import com.google.inject.BindingAnnotation;
-
-import com.twitter.aurora.scheduler.ResourceSlot;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.ScheduleException;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter;
-import com.twitter.aurora.scheduler.state.SchedulerCore;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.Clock;
-
-import static java.lang.annotation.ElementType.FIELD;
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-import org.apache.mesos.Protos.Offer;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
-import static com.twitter.aurora.gen.ScheduleStatus.PREEMPTING;
-import static com.twitter.aurora.scheduler.base.Tasks.SCHEDULED_TO_ASSIGNED;
-
-/**
- * Preempts active tasks in favor of higher priority tasks.
- */
-public interface Preemptor {
-
- /**
- * Preempts active tasks in favor of the input task.
- *
- * @param taskId ID of the preempting task.
- * @return ID of the slave where preemption occured.
- */
- Optional<String> findPreemptionSlotFor(String taskId);
-
- /**
- * A task preemptor that tries to find tasks that are waiting to be scheduled, which are of higher
- * priority than tasks that are currently running.
- *
- * To avoid excessive churn, the preemptor requires that a task is PENDING for a duration
- * (dictated by {@link #preemptionCandidacyDelay}) before it becomes eligible to preempt other
- * tasks.
- */
- class PreemptorImpl implements Preemptor {
-
- /**
- * Binding annotation for the time interval after which a pending task becomes eligible to
- * preempt other tasks.
- */
- @BindingAnnotation
- @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
- @interface PreemptionDelay { }
-
- @VisibleForTesting
- static final Query.Builder CANDIDATE_QUERY = Query.statusScoped(
- EnumSet.copyOf(Sets.difference(Tasks.ACTIVE_STATES, EnumSet.of(PENDING, PREEMPTING))));
-
- private static final Logger LOG = Logger.getLogger(PreemptorImpl.class.getName());
-
- private static final Function<IAssignedTask, Integer> GET_PRIORITY =
- new Function<IAssignedTask, Integer>() {
- @Override public Integer apply(IAssignedTask task) {
- return task.getTask().getPriority();
- }
- };
-
- private final AtomicLong tasksPreempted = Stats.exportLong("preemptor_tasks_preempted");
- private final AtomicLong failedPreemptions = Stats.exportLong("preemptor_failed_preemptions");
- // Incremented every time the preemptor is invoked and finds tasks pending and preemptable tasks
- private final AtomicLong attemptedPreemptions = Stats.exportLong("preemptor_attempts");
- // Incremented every time we fail to find tasks to preempt for a pending task.
- private final AtomicLong noSlotsFound = Stats.exportLong("preemptor_no_slots_found");
-
- private final Predicate<IScheduledTask> isIdleTask = new Predicate<IScheduledTask>() {
- @Override public boolean apply(IScheduledTask task) {
- return (clock.nowMillis() - Iterables.getLast(task.getTaskEvents()).getTimestamp())
- >= preemptionCandidacyDelay.as(Time.MILLISECONDS);
- }
- };
-
- private final Storage storage;
- private final SchedulerCore scheduler;
- private final OfferQueue offerQueue;
- private final SchedulingFilter schedulingFilter;
- private final Amount<Long, Time> preemptionCandidacyDelay;
- private final Clock clock;
-
- /**
- * Creates a new preemptor.
- *
- * @param storage Backing store for tasks.
- * @param scheduler Scheduler to fetch task information from, and instruct when preempting
- * tasks.
- * @param offerQueue Queue that contains available Mesos resource offers.
- * @param schedulingFilter Filter to identify whether tasks may reside on given slaves.
- * @param preemptionCandidacyDelay Time a task must be PENDING before it may preempt other
- * tasks.
- * @param clock Clock to check current time.
- */
- @Inject
- PreemptorImpl(
- Storage storage,
- SchedulerCore scheduler,
- OfferQueue offerQueue,
- SchedulingFilter schedulingFilter,
- @PreemptionDelay Amount<Long, Time> preemptionCandidacyDelay,
- Clock clock) {
-
- this.storage = checkNotNull(storage);
- this.scheduler = checkNotNull(scheduler);
- this.offerQueue = checkNotNull(offerQueue);
- this.schedulingFilter = checkNotNull(schedulingFilter);
- this.preemptionCandidacyDelay = checkNotNull(preemptionCandidacyDelay);
- this.clock = checkNotNull(clock);
- }
-
- private List<IAssignedTask> fetch(Query.Builder query, Predicate<IScheduledTask> filter) {
- return Lists.newArrayList(Iterables.transform(Iterables.filter(
- Storage.Util.consistentFetchTasks(storage, query), filter),
- SCHEDULED_TO_ASSIGNED));
- }
-
- private List<IAssignedTask> fetch(Query.Builder query) {
- return fetch(query, Predicates.<IScheduledTask>alwaysTrue());
- }
-
- private static final Function<IAssignedTask, String> TASK_TO_SLAVE_ID =
- new Function<IAssignedTask, String>() {
- @Override public String apply(IAssignedTask input) {
- return input.getSlaveId();
- }
- };
-
- private static final Function<IAssignedTask, String> TASK_TO_HOST =
- new Function<IAssignedTask, String>() {
- @Override public String apply(IAssignedTask input) {
- return input.getSlaveHost();
- }
- };
-
- private static Predicate<IAssignedTask> canPreempt(final IAssignedTask pending) {
- return new Predicate<IAssignedTask>() {
- @Override public boolean apply(IAssignedTask possibleVictim) {
- return preemptionFilter(possibleVictim).apply(pending);
- }
- };
- }
-
- private static final Function<IAssignedTask, ResourceSlot> TASK_TO_RESOURCES =
- new Function<IAssignedTask, ResourceSlot>() {
- @Override public ResourceSlot apply(IAssignedTask input) {
- return ResourceSlot.from(input.getTask());
- }
- };
-
- private static final Function<Offer, ResourceSlot> OFFER_TO_RESOURCE_SLOT =
- new Function<Offer, ResourceSlot>() {
- @Override public ResourceSlot apply(Offer offer) {
- return ResourceSlot.from(offer);
- }
- };
-
- private static final Function<Offer, String> OFFER_TO_HOST =
- new Function<Offer, String>() {
- @Override public String apply(Offer offer) {
- return offer.getHostname();
- }
- };
-
- // TODO(zmanji) Consider using Dominant Resource Fairness for ordering instead of the vector
- // ordering
- private static final Ordering<IAssignedTask> RESOURCE_ORDER =
- ResourceSlot.ORDER.onResultOf(TASK_TO_RESOURCES).reverse();
-
- /**
- * Optional.absent indicates that this slave does not have enough resources to satisfy the task.
- * The empty set indicates the offers (slack) are enough.
- * A set with elements indicates those tasks and the offers are enough.
- */
- private Optional<Set<IAssignedTask>> getTasksToPreempt(
- Iterable<IAssignedTask> possibleVictims,
- Iterable<Offer> offers,
- IAssignedTask pendingTask) {
-
- // This enforces the precondition that all of the resources are from the same host. We need to
- // get the host for the schedulingFilter.
- Set<String> hosts = ImmutableSet.<String>builder()
- .addAll(Iterables.transform(possibleVictims, TASK_TO_HOST))
- .addAll(Iterables.transform(offers, OFFER_TO_HOST)).build();
-
- String host = Iterables.getOnlyElement(hosts);
-
- ResourceSlot slackResources =
- ResourceSlot.sum(Iterables.transform(offers, OFFER_TO_RESOURCE_SLOT));
-
- if (!Iterables.isEmpty(offers)) {
- Set<SchedulingFilter.Veto> vetos = schedulingFilter.filter(
- slackResources,
- host,
- pendingTask.getTask(),
- pendingTask.getTaskId());
-
- if (vetos.isEmpty()) {
- return Optional.<Set<IAssignedTask>>of(ImmutableSet.<IAssignedTask>of());
- }
- }
-
- FluentIterable<IAssignedTask> preemptableTasks =
- FluentIterable.from(possibleVictims).filter(canPreempt(pendingTask));
-
- if (preemptableTasks.isEmpty()) {
- return Optional.absent();
- }
-
- List<IAssignedTask> toPreemptTasks = Lists.newArrayList();
-
- Iterable<IAssignedTask> sortedVictims = RESOURCE_ORDER.immutableSortedCopy(preemptableTasks);
-
- for (IAssignedTask victim : sortedVictims) {
- toPreemptTasks.add(victim);
-
- ResourceSlot totalResource = ResourceSlot.sum(
- ResourceSlot.sum(Iterables.transform(toPreemptTasks, TASK_TO_RESOURCES)),
- slackResources);
-
- Set<SchedulingFilter.Veto> vetos = schedulingFilter.filter(
- totalResource,
- host,
- pendingTask.getTask(),
- pendingTask.getTaskId());
-
- if (vetos.isEmpty()) {
- return Optional.<Set<IAssignedTask>>of(ImmutableSet.copyOf(toPreemptTasks));
- }
- }
- return Optional.absent();
- }
-
- private static final Function<Offer, String> OFFER_TO_SLAVE_ID =
- new Function<Offer, String>() {
- @Override public String apply(Offer offer) {
- return offer.getSlaveId().getValue();
- }
- };
-
- private Multimap<String, IAssignedTask> getSlavesToActiveTasks() {
- // Only non-pending active tasks may be preempted.
- List<IAssignedTask> activeTasks = fetch(CANDIDATE_QUERY);
-
- // Walk through the preemption candidates in reverse scheduling order.
- Collections.sort(activeTasks, Tasks.SCHEDULING_ORDER.reverse());
-
- // Group the tasks by slave id so they can be paired with offers from the same slave.
- return Multimaps.index(activeTasks, TASK_TO_SLAVE_ID);
- }
-
- @Override
- public synchronized Optional<String> findPreemptionSlotFor(String taskId) {
- List<IAssignedTask> pendingTasks =
- fetch(Query.statusScoped(PENDING).byId(taskId), isIdleTask);
-
- // Task is no longer PENDING no need to preempt
- if (pendingTasks.isEmpty()) {
- return Optional.absent();
- }
-
- IAssignedTask pendingTask = Iterables.getOnlyElement(pendingTasks);
-
- Multimap<String, IAssignedTask> slavesToActiveTasks = getSlavesToActiveTasks();
-
- if (slavesToActiveTasks.isEmpty()) {
- return Optional.absent();
- }
-
- attemptedPreemptions.incrementAndGet();
-
- // Group the offers by slave id so they can be paired with active tasks from the same slave.
- Multimap<String, Offer> slavesToOffers =
- Multimaps.index(offerQueue.getOffers(), OFFER_TO_SLAVE_ID);
-
- Set<String> allSlaves = ImmutableSet.<String>builder()
- .addAll(slavesToOffers.keySet())
- .addAll(slavesToActiveTasks.keySet())
- .build();
-
- for (String slaveID : allSlaves) {
- Optional<Set<IAssignedTask>> toPreemptTasks = getTasksToPreempt(
- slavesToActiveTasks.get(slaveID),
- slavesToOffers.get(slaveID),
- pendingTask);
-
- if (toPreemptTasks.isPresent()) {
- try {
- for (IAssignedTask toPreempt : toPreemptTasks.get()) {
- scheduler.preemptTask(toPreempt, pendingTask);
- tasksPreempted.incrementAndGet();
- }
- return Optional.of(slaveID);
- } catch (ScheduleException e) {
- LOG.log(Level.SEVERE, "Preemption failed", e);
- failedPreemptions.incrementAndGet();
- }
- }
- }
-
- noSlotsFound.incrementAndGet();
- return Optional.absent();
- }
-
- private static final Predicate<IAssignedTask> IS_PRODUCTION =
- Predicates.compose(Tasks.IS_PRODUCTION, Tasks.ASSIGNED_TO_INFO);
-
- /**
- * Creates a static filter that will identify tasks that may preempt the provided task.
- * A task may preempt another task if the following conditions hold true:
- * - The resources reserved for {@code preemptableTask} are sufficient to satisfy the task.
- * - The tasks are owned by the same user and the priority of {@code preemptableTask} is lower
- * OR {@code preemptableTask} is non-production and the compared task is production.
- *
- * @param preemptableTask Task to possibly preempt.
- * @return A filter that will compare the priorities and resources required by other tasks
- * with {@code preemptableTask}.
- */
- private static Predicate<IAssignedTask> preemptionFilter(IAssignedTask preemptableTask) {
- Predicate<IAssignedTask> preemptableIsProduction = preemptableTask.getTask().isProduction()
- ? Predicates.<IAssignedTask>alwaysTrue()
- : Predicates.<IAssignedTask>alwaysFalse();
-
- Predicate<IAssignedTask> priorityFilter =
- greaterPriorityFilter(GET_PRIORITY.apply(preemptableTask));
- return Predicates.or(
- Predicates.and(Predicates.not(preemptableIsProduction), IS_PRODUCTION),
- Predicates.and(isOwnedBy(getRole(preemptableTask)), priorityFilter)
- );
- }
-
- private static Predicate<IAssignedTask> isOwnedBy(final String role) {
- return new Predicate<IAssignedTask>() {
- @Override public boolean apply(IAssignedTask task) {
- return getRole(task).equals(role);
- }
- };
- }
-
- private static String getRole(IAssignedTask task) {
- return task.getTask().getOwner().getRole();
- }
-
- private static Predicate<Integer> greaterThan(final int value) {
- return new Predicate<Integer>() {
- @Override public boolean apply(Integer input) {
- return input > value;
- }
- };
- }
-
- private static Predicate<IAssignedTask> greaterPriorityFilter(int priority) {
- return Predicates.compose(greaterThan(priority), GET_PRIORITY);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java b/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java
deleted file mode 100644
index eefc03a..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/RescheduleCalculator.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskEvent;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.util.BackoffStrategy;
-import com.twitter.common.util.Clock;
-import com.twitter.common.util.Random;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
-import static com.twitter.aurora.gen.ScheduleStatus.RESTARTING;
-
-/**
- * Calculates scheduling delays for tasks.
- */
-interface RescheduleCalculator {
- /**
- * Gets a timestamp for the task to become eligible for (re)scheduling at scheduler startup.
- *
- * @param task Task to calculate timestamp for.
- * @return Timestamp in msec.
- */
- long getStartupReadyTimeMs(IScheduledTask task);
-
- /**
- * Gets a timestamp for the task to become eligible for (re)scheduling.
- *
- * @param task Task to calculate timestamp for.
- * @return Timestamp in msec.
- */
- long getReadyTimeMs(IScheduledTask task);
-
- class RescheduleCalculatorImpl implements RescheduleCalculator {
-
- private static final Logger LOG = Logger.getLogger(TaskGroups.class.getName());
-
- private final Storage storage;
- private final RescheduleCalculatorSettings settings;
- private final Clock clock;
- private final Random random = new Random.SystemRandom(new java.util.Random());
-
- private static final Predicate<ScheduleStatus> IS_ACTIVE_STATUS =
- Predicates.in(Tasks.ACTIVE_STATES);
-
- private static final Function<ITaskEvent, ScheduleStatus> TO_STATUS =
- new Function<ITaskEvent, ScheduleStatus>() {
- @Override public ScheduleStatus apply(ITaskEvent input) {
- return input.getStatus();
- }
- };
-
- private static final Set<ScheduleStatus> INTERRUPTED_TASK_STATES =
- EnumSet.of(RESTARTING, KILLING);
-
- private final Predicate<IScheduledTask> flapped = new Predicate<IScheduledTask>() {
- @Override public boolean apply(IScheduledTask task) {
- if (!task.isSetTaskEvents()) {
- return false;
- }
-
- List<ITaskEvent> events = Lists.reverse(task.getTaskEvents());
-
- // Avoid penalizing tasks that were interrupted by outside action, such as a user
- // restarting them.
- if (Iterables.any(Iterables.transform(events, TO_STATUS),
- Predicates.in(INTERRUPTED_TASK_STATES))) {
- return false;
- }
-
- ITaskEvent terminalEvent = Iterables.get(events, 0);
- ScheduleStatus terminalState = terminalEvent.getStatus();
- Preconditions.checkState(Tasks.isTerminated(terminalState));
-
- ITaskEvent activeEvent =
- Iterables.find(events, Predicates.compose(IS_ACTIVE_STATUS, TO_STATUS));
-
- long thresholdMs = settings.flappingTaskThreashold.as(Time.MILLISECONDS);
-
- return (terminalEvent.getTimestamp() - activeEvent.getTimestamp()) < thresholdMs;
- }
- };
-
- static class RescheduleCalculatorSettings {
- private final BackoffStrategy flappingTaskBackoff;
- private final Amount<Long, Time> flappingTaskThreashold;
- private final Amount<Integer, Time> maxStartupRescheduleDelay;
-
- RescheduleCalculatorSettings(
- BackoffStrategy flappingTaskBackoff,
- Amount<Long, Time> flappingTaskThreashold,
- Amount<Integer, Time> maxStartupRescheduleDelay) {
-
- this.flappingTaskBackoff = checkNotNull(flappingTaskBackoff);
- this.flappingTaskThreashold = checkNotNull(flappingTaskThreashold);
- this.maxStartupRescheduleDelay = checkNotNull(maxStartupRescheduleDelay);
- }
- }
-
- @Inject
- RescheduleCalculatorImpl(
- Storage storage,
- RescheduleCalculatorSettings settings,
- Clock clock) {
-
- this.storage = checkNotNull(storage);
- this.settings = checkNotNull(settings);
- this.clock = checkNotNull(clock);
- }
-
- @Override
- public long getStartupReadyTimeMs(IScheduledTask task) {
- return random.nextInt(settings.maxStartupRescheduleDelay.as(Time.MILLISECONDS))
- + getTaskReadyTimestamp(task);
- }
-
- @Override
- public long getReadyTimeMs(IScheduledTask task) {
- return getTaskReadyTimestamp(task);
- }
-
- private Optional<IScheduledTask> getTaskAncestor(IScheduledTask task) {
- if (!task.isSetAncestorId()) {
- return Optional.absent();
- }
-
- ImmutableSet<IScheduledTask> res =
- Storage.Util.weaklyConsistentFetchTasks(storage, Query.taskScoped(task.getAncestorId()));
-
- return Optional.fromNullable(Iterables.getOnlyElement(res, null));
- }
-
- private long getTaskReadyTimestamp(IScheduledTask task) {
- Optional<IScheduledTask> curTask = getTaskAncestor(task);
- long penaltyMs = 0;
- while (curTask.isPresent() && flapped.apply(curTask.get())) {
- LOG.info(
- String.format("Ancestor of %s flapped: %s", Tasks.id(task), Tasks.id(curTask.get())));
- long newPenalty = settings.flappingTaskBackoff.calculateBackoffMs(penaltyMs);
- // If the backoff strategy is truncated then there is no need for us to continue.
- if (newPenalty == penaltyMs) {
- break;
- }
- penaltyMs = newPenalty;
- curTask = getTaskAncestor(curTask.get());
- }
-
- return penaltyMs + clock.nowMillis();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/TaskGroup.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroup.java b/src/main/java/com/twitter/aurora/scheduler/async/TaskGroup.java
deleted file mode 100644
index 018022b..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroup.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-
-import com.twitter.aurora.scheduler.async.TaskGroups.GroupKey;
-import com.twitter.common.base.Function;
-import com.twitter.common.util.BackoffStrategy;
-
-/**
- * A group of task IDs that are eligible for scheduling, but may be waiting for a backoff to expire.
- */
-class TaskGroup {
- private final GroupKey key;
- private final BackoffStrategy backoffStrategy;
-
- private static final Function<Task, Long> TO_TIMESTAMP = new Function<Task, Long>() {
- @Override public Long apply(Task item) {
- return item.readyTimestampMs;
- }
- };
-
- // Order the tasks by the time they are ready to be scheduled
- private static final Ordering<Task> TASK_ORDERING = Ordering.natural().onResultOf(TO_TIMESTAMP);
- // 11 is the magic number used by PriorityBlockingQueue as the initial size.
- private final Queue<Task> tasks = new PriorityBlockingQueue<>(11, TASK_ORDERING);
- // Penalty for the task group for failing to schedule.
- private final AtomicLong penaltyMs;
-
- TaskGroup(GroupKey key, BackoffStrategy backoffStrategy) {
- this.key = key;
- this.backoffStrategy = backoffStrategy;
- penaltyMs = new AtomicLong();
- resetPenaltyAndGet();
- }
-
- GroupKey getKey() {
- return key;
- }
-
- private static final Function<Task, String> TO_TASK_ID =
- new Function<Task, String>() {
- @Override public String apply(Task item) {
- return item.taskId;
- }
- };
-
- /**
- * Removes the task at the head of the queue.
- *
- * @return String the id of the head task.
- * @throws IllegalStateException if the queue is empty.
- */
- String pop() throws IllegalStateException {
- Task head = tasks.poll();
- Preconditions.checkState(head != null);
- return head.taskId;
- }
-
- void remove(String taskId) {
- Iterables.removeIf(tasks, Predicates.compose(Predicates.equalTo(taskId), TO_TASK_ID));
- }
-
- void push(final String taskId, long readyTimestamp) {
- tasks.offer(new Task(taskId, readyTimestamp));
- }
-
- synchronized long resetPenaltyAndGet() {
- penaltyMs.set(backoffStrategy.calculateBackoffMs(0));
- return getPenaltyMs();
- }
-
- synchronized long penalizeAndGet() {
- penaltyMs.set(backoffStrategy.calculateBackoffMs(getPenaltyMs()));
- return getPenaltyMs();
- }
-
- GroupState isReady(long nowMs) {
- Task task = tasks.peek();
- if (task == null) {
- return GroupState.EMPTY;
- }
-
- if (task.readyTimestampMs > nowMs) {
- return GroupState.NOT_READY;
- }
- return GroupState.READY;
- }
- // Begin methods used for debug interfaces.
-
- public String getName() {
- return key.toString();
- }
-
- // TODO(zmanji): Return Task instances here. Can use them to display flapping penalty on web UI.
- public Set<String> getTaskIds() {
- return ImmutableSet.copyOf(Iterables.transform(tasks, TO_TASK_ID));
- }
-
- public long getPenaltyMs() {
- return penaltyMs.get();
- }
-
- private static class Task {
- private final String taskId;
- private final long readyTimestampMs;
-
- Task(String taskId, long readyTimestampMs) {
- this.taskId = Preconditions.checkNotNull(taskId);
- this.readyTimestampMs = readyTimestampMs;
- }
- }
-
- enum GroupState {
- EMPTY, // The group is empty.
- NOT_READY, // Every task in the group is not ready yet.
- READY // The task at the head of the queue is ready.
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java b/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
deleted file mode 100644
index a59e5c8..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/async/TaskGroups.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.async;
-
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Objects;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.eventbus.Subscribe;
-import com.google.common.util.concurrent.RateLimiter;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
-import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.application.ShutdownRegistry;
-import com.twitter.common.base.Command;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.BackoffStrategy;
-import com.twitter.common.util.Clock;
-import com.twitter.common.util.concurrent.ExecutorServiceShutdown;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
-import static com.twitter.aurora.scheduler.async.TaskGroup.GroupState;
-
-/**
- * A collection of task groups, where a task group is a collection of tasks that are known to be
- * equal in the way they schedule. This is expected to be tasks associated with the same job key,
- * who also have {@code equal()} {@link ITaskConfig} values.
- * <p>
- * This is used to prevent redundant work in trying to schedule tasks as well as to provide
- * nearly-equal responsiveness when scheduling across jobs. In other words, a 1000 instance job
- * cannot starve a 1 instance job.
- */
-public class TaskGroups implements EventSubscriber {
-
- private static final Logger LOG = Logger.getLogger(TaskGroups.class.getName());
-
- private final Storage storage;
- private final LoadingCache<GroupKey, TaskGroup> groups;
- private final Clock clock;
- private final RescheduleCalculator rescheduleCalculator;
-
- static class TaskGroupsSettings {
- private final BackoffStrategy taskGroupBackoff;
- private final RateLimiter rateLimiter;
-
- TaskGroupsSettings(BackoffStrategy taskGroupBackoff, RateLimiter rateLimiter) {
- this.taskGroupBackoff = checkNotNull(taskGroupBackoff);
- this.rateLimiter = checkNotNull(rateLimiter);
- }
- }
-
- @Inject
- TaskGroups(
- ShutdownRegistry shutdownRegistry,
- Storage storage,
- TaskGroupsSettings settings,
- TaskScheduler taskScheduler,
- Clock clock,
- RescheduleCalculator rescheduleCalculator) {
-
- this(
- createThreadPool(shutdownRegistry),
- storage,
- settings.taskGroupBackoff,
- settings.rateLimiter,
- taskScheduler,
- clock,
- rescheduleCalculator);
- }
-
- TaskGroups(
- final ScheduledExecutorService executor,
- final Storage storage,
- final BackoffStrategy taskGroupBackoffStrategy,
- final RateLimiter rateLimiter,
- final TaskScheduler taskScheduler,
- final Clock clock,
- final RescheduleCalculator rescheduleCalculator) {
-
- this.storage = checkNotNull(storage);
- checkNotNull(executor);
- checkNotNull(taskGroupBackoffStrategy);
- checkNotNull(rateLimiter);
- checkNotNull(taskScheduler);
- this.clock = checkNotNull(clock);
- this.rescheduleCalculator = checkNotNull(rescheduleCalculator);
-
- final TaskScheduler ratelLimitedScheduler = new TaskScheduler() {
- @Override public TaskSchedulerResult schedule(String taskId) {
- rateLimiter.acquire();
- return taskScheduler.schedule(taskId);
- }
- };
-
- groups = CacheBuilder.newBuilder().build(new CacheLoader<GroupKey, TaskGroup>() {
- @Override public TaskGroup load(GroupKey key) {
- TaskGroup group = new TaskGroup(key, taskGroupBackoffStrategy);
- LOG.info("Evaluating group " + key + " in " + group.getPenaltyMs() + " ms");
- startGroup(group, executor, ratelLimitedScheduler);
- return group;
- }
- });
- }
-
- private synchronized boolean maybeInvalidate(TaskGroup group) {
- if (group.getTaskIds().isEmpty()) {
- groups.invalidate(group.getKey());
- return true;
- }
- return false;
- }
-
- private void startGroup(
- final TaskGroup group,
- final ScheduledExecutorService executor,
- final TaskScheduler taskScheduler) {
-
- Runnable monitor = new Runnable() {
- @Override public void run() {
- GroupState state = group.isReady(clock.nowMillis());
-
- switch (state) {
- case EMPTY:
- maybeInvalidate(group);
- break;
-
- case READY:
- String id = group.pop();
- TaskScheduler.TaskSchedulerResult result = taskScheduler.schedule(id);
- switch (result) {
- case SUCCESS:
- if (!maybeInvalidate(group)) {
- executor.schedule(this, group.resetPenaltyAndGet(), TimeUnit.MILLISECONDS);
- }
- break;
-
- case TRY_AGAIN:
- group.push(id, clock.nowMillis());
- executor.schedule(this, group.penalizeAndGet(), TimeUnit.MILLISECONDS);
- break;
-
- default:
- throw new IllegalStateException("Unknown TaskSchedulerResult " + result);
- }
- break;
-
- case NOT_READY:
- executor.schedule(this, group.getPenaltyMs(), TimeUnit.MILLISECONDS);
- break;
-
- default:
- throw new IllegalStateException("Unknown GroupState " + state);
- }
- }
- };
- executor.schedule(monitor, group.getPenaltyMs(), TimeUnit.MILLISECONDS);
- }
-
- private static ScheduledExecutorService createThreadPool(ShutdownRegistry shutdownRegistry) {
- // TODO(William Farner): Leverage ExceptionHandlingScheduledExecutorService:
- // com.twitter.common.util.concurrent.ExceptionHandlingScheduledExecutorService
- final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
- 1,
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TaskScheduler-%d").build());
- Stats.exportSize("schedule_queue_size", executor.getQueue());
- shutdownRegistry.addAction(new Command() {
- @Override public void execute() {
- new ExecutorServiceShutdown(executor, Amount.of(1L, Time.SECONDS)).execute();
- }
- });
- return executor;
- }
-
- private synchronized void add(IAssignedTask task, long readyTimestamp) {
- groups.getUnchecked(new GroupKey(task.getTask())).push(task.getTaskId(), readyTimestamp);
- }
-
- /**
- * Informs the task groups of a task state change.
- * <p>
- * This is used to observe {@link com.twitter.aurora.gen.ScheduleStatus#PENDING} tasks and begin
- * attempting to schedule them.
- *
- * @param stateChange State change notification.
- */
- @Subscribe
- public synchronized void taskChangedState(TaskStateChange stateChange) {
- if (stateChange.getNewState() == PENDING) {
- add(
- stateChange.getTask().getAssignedTask(),
- rescheduleCalculator.getReadyTimeMs(stateChange.getTask()));
- }
- }
-
- /**
- * Signals that storage has started and is consistent.
- * <p>
- * Upon this signal, all {@link com.twitter.aurora.gen.ScheduleStatus#PENDING} tasks in the stoage
- * will become eligible for scheduling.
- *
- * @param event Storage started notification.
- */
- @Subscribe
- public void storageStarted(StorageStarted event) {
- for (IScheduledTask task
- : Storage.Util.consistentFetchTasks(storage, Query.unscoped().byStatus(PENDING))) {
-
- add(task.getAssignedTask(), rescheduleCalculator.getStartupReadyTimeMs(task));
- }
- }
-
- /**
- * Signals the scheduler that tasks have been deleted.
- *
- * @param deleted Tasks deleted event.
- */
- @Subscribe
- public synchronized void tasksDeleted(TasksDeleted deleted) {
- for (IAssignedTask task
- : Iterables.transform(deleted.getTasks(), Tasks.SCHEDULED_TO_ASSIGNED)) {
- TaskGroup group = groups.getIfPresent(new GroupKey(task.getTask()));
- if (group != null) {
- group.remove(task.getTaskId());
- }
- }
- }
-
- public Iterable<TaskGroup> getGroups() {
- return ImmutableSet.copyOf(groups.asMap().values());
- }
-
- static class GroupKey {
- private final ITaskConfig canonicalTask;
-
- GroupKey(ITaskConfig task) {
- this.canonicalTask = task;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(canonicalTask);
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof GroupKey)) {
- return false;
- }
- GroupKey other = (GroupKey) o;
- return Objects.equal(canonicalTask, other.canonicalTask);
- }
-
- @Override
- public String toString() {
- return JobKeys.toPath(Tasks.INFO_TO_JOB_KEY.apply(canonicalTask));
- }
- }
-}
[21/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/core/runner.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/core/runner.py b/src/main/python/apache/thermos/core/runner.py
new file mode 100644
index 0000000..b10cf16
--- /dev/null
+++ b/src/main/python/apache/thermos/core/runner.py
@@ -0,0 +1,905 @@
+""" Thermos runner.
+
+This module contains the TaskRunner, the core component of Thermos responsible for actually running
+tasks. It also contains several Handlers which define the behaviour on state transitions within the
+TaskRunner.
+
+There are three "active" states in a running Thermos task:
+ ACTIVE
+ CLEANING
+ FINALIZING
+
+A task in ACTIVE state is running regular processes. The moment this task succeeds or goes over its
+failure limit, it then goes into CLEANING state, where it begins the staged termination of leftover
+processes (with SIGTERMs). Once all processes have terminated, the task goes into FINALIZING state,
+where the processes marked with the 'final' bit run. Once the task has gone into CLEANING state, it
+has a deadline for going into terminal state. If it doesn't make it in time (at any point, whether
+in CLEANING or FINALIZING state), it is forced into terminal state through SIGKILLs of all live
+processes (coordinators, shells and the full process trees rooted at the shells.)
+
+TaskRunner.kill is implemented by forcing the task into CLEANING state and setting its finalization
+deadline manually. So in practice, we implement Task preemption by calling kill with the
+finalization deadline = now + preemption wait, which gives the Task an opportunity to do graceful
+shutdown. If preemption_wait=0, it will result in immediate SIGKILLs and then transition to the
+terminal state.
+
+"""
+
+from contextlib import contextmanager
+import errno
+from functools import partial
+import os
+import socket
+import sys
+import time
+import traceback
+
+from twitter.common import log
+from twitter.common.dirutil import safe_mkdir
+from twitter.common.quantity import Amount, Time
+from twitter.common.recordio import ThriftRecordReader
+
+from twitter.thermos.common.ckpt import (
+ CheckpointDispatcher,
+ UniversalStateHandler,
+ ProcessStateHandler,
+ TaskStateHandler)
+from twitter.thermos.common.path import TaskPath
+from twitter.thermos.common.planner import TaskPlanner
+from twitter.thermos.config.loader import (
+ ThermosConfigLoader,
+ ThermosProcessWrapper,
+ ThermosTaskWrapper,
+ ThermosTaskValidator)
+from twitter.thermos.config.schema import ThermosContext
+
+from gen.twitter.thermos.ttypes import (
+ ProcessState,
+ ProcessStatus,
+ RunnerCkpt,
+ RunnerHeader,
+ RunnerState,
+ TaskState,
+ TaskStatus,
+)
+
+from .helper import TaskRunnerHelper
+from .muxer import ProcessMuxer
+from .process import Process
+
+from pystachio import Environment
+
+
+# TODO(wickman) Currently this is messy because of all the private access into ._runner.
+# Clean this up by giving the TaskRunnerProcessHandler the components it should own, and
+# create a legitimate API contract into the Runner.
+class TaskRunnerProcessHandler(ProcessStateHandler):
+ """
+ Accesses these parts of the runner:
+
+ | _task_processes [array set, pop]
+ | _task_process_from_process_name [process name / sequence number => Process]
+ | _watcher [ProcessMuxer.register, unregister]
+ | _plan [add_success, add_failure, set_running]
+ """
+
+ def __init__(self, runner):
+ self._runner = runner
+
+ def on_waiting(self, process_update):
+ log.debug('Process on_waiting %s' % process_update)
+ self._runner._task_processes[process_update.process] = (
+ self._runner._task_process_from_process_name(
+ process_update.process, process_update.seq + 1))
+ self._runner._watcher.register(process_update.process, process_update.seq - 1)
+
+ def on_forked(self, process_update):
+ log.debug('Process on_forked %s' % process_update)
+ task_process = self._runner._task_processes[process_update.process]
+ task_process.rebind(process_update.coordinator_pid, process_update.fork_time)
+ self._runner._plan.set_running(process_update.process)
+
+ def on_running(self, process_update):
+ log.debug('Process on_running %s' % process_update)
+ self._runner._plan.set_running(process_update.process)
+
+ def _cleanup(self, process_update):
+ if not self._runner._recovery:
+ TaskRunnerHelper.kill_process(self._runner.state, process_update.process)
+
+ def on_success(self, process_update):
+ log.debug('Process on_success %s' % process_update)
+ log.info('Process(%s) finished successfully [rc=%s]' % (
+ process_update.process, process_update.return_code))
+ self._cleanup(process_update)
+ self._runner._task_processes.pop(process_update.process)
+ self._runner._watcher.unregister(process_update.process)
+ self._runner._plan.add_success(process_update.process)
+
+ def _on_abnormal(self, process_update):
+ log.info('Process %s had an abnormal termination' % process_update.process)
+ self._runner._task_processes.pop(process_update.process)
+ self._runner._watcher.unregister(process_update.process)
+
+ def on_failed(self, process_update):
+ log.debug('Process on_failed %s' % process_update)
+ log.info('Process(%s) failed [rc=%s]' % (process_update.process, process_update.return_code))
+ self._cleanup(process_update)
+ self._on_abnormal(process_update)
+ self._runner._plan.add_failure(process_update.process)
+ if process_update.process in self._runner._plan.failed:
+ log.info('Process %s reached maximum failures, marking process run failed.' %
+ process_update.process)
+ else:
+ log.info('Process %s under maximum failure limit, restarting.' % process_update.process)
+
+ def on_lost(self, process_update):
+ log.debug('Process on_lost %s' % process_update)
+ self._cleanup(process_update)
+ self._on_abnormal(process_update)
+ self._runner._plan.lost(process_update.process)
+
+ def on_killed(self, process_update):
+ log.debug('Process on_killed %s' % process_update)
+ self._cleanup(process_update)
+ self._runner._task_processes.pop(process_update.process)
+ self._runner._watcher.unregister(process_update.process)
+ log.debug('Process killed, marking it as a loss.')
+ self._runner._plan.lost(process_update.process)
+
+
+class TaskRunnerTaskHandler(TaskStateHandler):
+ """
+ Accesses these parts of the runner:
+ _plan [set to regular_plan or finalizing_plan]
+ _recovery [boolean, whether or not to side-effect]
+ _pathspec [path creation]
+ _task [ThermosTask]
+ _set_finalization_start
+ _kill
+ """
+
+ def __init__(self, runner):
+ self._runner = runner
+ self._pathspec = self._runner._pathspec
+
+ def on_active(self, task_update):
+ log.debug('Task on_active(%s)' % task_update)
+ self._runner._plan = self._runner._regular_plan
+ if self._runner._recovery:
+ return
+ TaskRunnerHelper.initialize_task(self._pathspec,
+ ThermosTaskWrapper(self._runner._task).to_json())
+
+ def on_cleaning(self, task_update):
+ log.debug('Task on_cleaning(%s)' % task_update)
+ self._runner._finalization_start = task_update.timestamp_ms / 1000.0
+ self._runner._terminate_plan(self._runner._regular_plan)
+
+ def on_finalizing(self, task_update):
+ log.debug('Task on_finalizing(%s)' % task_update)
+ if not self._runner._recovery:
+ self._runner._kill()
+ self._runner._plan = self._runner._finalizing_plan
+ if self._runner._finalization_start is None:
+ self._runner._finalization_start = task_update.timestamp_ms / 1000.0
+
+ def on_killed(self, task_update):
+ log.debug('Task on_killed(%s)' % task_update)
+ self._cleanup()
+
+ def on_success(self, task_update):
+ log.debug('Task on_success(%s)' % task_update)
+ self._cleanup()
+ log.info('Task succeeded.')
+
+ def on_failed(self, task_update):
+ log.debug('Task on_failed(%s)' % task_update)
+ self._cleanup()
+
+ def on_lost(self, task_update):
+ log.debug('Task on_lost(%s)' % task_update)
+ self._cleanup()
+
+ def _cleanup(self):
+ if not self._runner._recovery:
+ self._runner._kill()
+ TaskRunnerHelper.finalize_task(self._pathspec)
+
+
+class TaskRunnerUniversalHandler(UniversalStateHandler):
+ """
+ Universal handler to checkpoint every process and task transition of the runner.
+
+ Accesses these parts of the runner:
+ _ckpt_write
+ """
+
+ def __init__(self, runner):
+ self._runner = runner
+
+ def _checkpoint(self, record):
+ self._runner._ckpt_write(record)
+
+ def on_process_transition(self, state, process_update):
+ log.debug('_on_process_transition: %s' % process_update)
+ self._checkpoint(RunnerCkpt(process_status=process_update))
+
+ def on_task_transition(self, state, task_update):
+ log.debug('_on_task_transition: %s' % task_update)
+ self._checkpoint(RunnerCkpt(task_status=task_update))
+
+ def on_initialization(self, header):
+ log.debug('_on_initialization: %s' % header)
+ ThermosTaskValidator.assert_valid_task(self._runner.task)
+ ThermosTaskValidator.assert_valid_ports(self._runner.task, header.ports)
+ self._checkpoint(RunnerCkpt(runner_header=header))
+
+
+class TaskRunnerStage(object):
+ """
+ A stage of the task runner pipeline.
+ """
+ MAX_ITERATION_WAIT = Amount(1, Time.SECONDS)
+
+ def __init__(self, runner):
+ self.runner = runner
+ self.clock = runner._clock
+
+ def run(self):
+ """
+ Perform any work necessary at this stage of the task.
+
+ If there is no more work to be done, return None. [This will invoke a state transition.]
+
+ If there is still work to be done, return the number of seconds from now in which you'd like
+ to be called to re-run the plan.
+ """
+ return None
+
+ def transition_to(self):
+ """
+ The stage to which we should transition.
+ """
+ raise NotImplementedError
+
+
+class TaskRunnerStage_ACTIVE(TaskRunnerStage):
+ """
+ Run the regular plan (i.e. normal, non-finalizing processes.)
+ """
+ MAX_ITERATION_WAIT = Amount(15, Time.SECONDS)
+ MIN_ITERATION_WAIT = Amount(1, Time.SECONDS)
+
+ def __init__(self, runner):
+ super(TaskRunnerStage_ACTIVE, self).__init__(runner)
+
+ def run(self):
+ launched = self.runner._run_plan(self.runner._regular_plan)
+
+ # Have we terminated?
+ terminal_state = None
+ if self.runner._regular_plan.is_complete():
+ log.info('Regular plan complete.')
+ terminal_state = TaskState.SUCCESS if self.runner.is_healthy() else TaskState.FAILED
+ elif not self.runner.is_healthy():
+ log.error('Regular plan unhealthy!')
+ terminal_state = TaskState.FAILED
+
+ if terminal_state:
+ # No more work to do
+ return None
+ elif launched > 0:
+ # We want to run ASAP after updates have been collected
+ return max(self.MIN_ITERATION_WAIT.as_(Time.SECONDS), self.runner._regular_plan.min_wait())
+ else:
+ # We want to run as soon as something is available to run or after a prescribed timeout.
+ return min(self.MAX_ITERATION_WAIT.as_(Time.SECONDS), self.runner._regular_plan.min_wait())
+
+ def transition_to(self):
+ return TaskState.CLEANING
+
+
+class TaskRunnerStage_CLEANING(TaskRunnerStage):
+ """
+ Start the cleanup of the regular plan (e.g. if it failed.) On ACTIVE -> CLEANING,
+ we send SIGTERMs to all still-running processes. We wait at most finalization_wait
+ for all processes to complete before SIGKILLs are sent. If everything exits cleanly
+ prior to that point in time, we transition to FINALIZING, which kicks into gear
+ the finalization schedule (if any.)
+ """
+ def run(self):
+ log.debug('TaskRunnerStage[CLEANING]: Finalization remaining: %s' %
+ self.runner._finalization_remaining())
+ if self.runner._finalization_remaining() > 0 and self.runner.has_running_processes():
+ return min(self.runner._finalization_remaining(), self.MAX_ITERATION_WAIT.as_(Time.SECONDS))
+
+ def transition_to(self):
+ if self.runner._finalization_remaining() <= 0:
+ log.info('Exceeded finalization wait, skipping finalization.')
+ return self.runner.terminal_state()
+ return TaskState.FINALIZING
+
+
+class TaskRunnerStage_FINALIZING(TaskRunnerStage):
+ """
+ Run the finalizing plan, specifically the plan of tasks with the 'final'
+ bit marked (e.g. log savers, checkpointers and the like.) Anything in this
+ plan will be SIGKILLed if we go over the finalization_wait.
+ """
+
+ def run(self):
+ self.runner._run_plan(self.runner._finalizing_plan)
+ log.debug('TaskRunnerStage[FINALIZING]: Finalization remaining: %s' %
+ self.runner._finalization_remaining())
+ if self.runner.deadlocked(self.runner._finalizing_plan):
+ log.warning('Finalizing plan deadlocked.')
+ return None
+ if self.runner._finalization_remaining() > 0 and not self.runner._finalizing_plan.is_complete():
+ return min(self.runner._finalization_remaining(), self.MAX_ITERATION_WAIT.as_(Time.SECONDS))
+
+ def transition_to(self):
+ if self.runner._finalization_remaining() <= 0:
+ log.info('Exceeded finalization wait, terminating finalization.')
+ return self.runner.terminal_state()
+
+
+class TaskRunner(object):
+ """
+ Run a ThermosTask.
+
+ This class encapsulates the core logic to run and control the state of a Thermos task.
+ Typically, it will be instantiated directly to control a new task, but a TaskRunner can also be
+ synthesised from an existing task's checkpoint root
+ """
+ class Error(Exception): pass
+ class InvalidTask(Error): pass
+ class InternalError(Error): pass
+ class PermissionError(Error): pass
+ class StateError(Error): pass
+
+ # Maximum amount of time we spend waiting for new updates from the checkpoint streams
+ # before doing housecleaning (checking for LOST tasks, dead PIDs.)
+ MAX_ITERATION_TIME = Amount(10, Time.SECONDS)
+
+ # Minimum amount of time we wait between polls for updates on coordinator checkpoints.
+ COORDINATOR_INTERVAL_SLEEP = Amount(1, Time.SECONDS)
+
+ # Amount of time we're willing to wait after forking before we expect the runner to have
+ # exec'ed the child process.
+ LOST_TIMEOUT = Amount(60, Time.SECONDS)
+
+ # Active task stages
+ STAGES = {
+ TaskState.ACTIVE: TaskRunnerStage_ACTIVE,
+ TaskState.CLEANING: TaskRunnerStage_CLEANING,
+ TaskState.FINALIZING: TaskRunnerStage_FINALIZING
+ }
+
+ @classmethod
+ def get(cls, task_id, checkpoint_root):
+ """
+ Get a TaskRunner bound to the task_id in checkpoint_root.
+ """
+ path = TaskPath(root=checkpoint_root, task_id=task_id, state='active')
+ task_json = path.getpath('task_path')
+ task_checkpoint = path.getpath('runner_checkpoint')
+ if not os.path.exists(task_json):
+ return None
+ task = ThermosConfigLoader.load_json(task_json)
+ if task is None:
+ return None
+ if len(task.tasks()) == 0:
+ return None
+ try:
+ checkpoint = CheckpointDispatcher.from_file(task_checkpoint)
+ if checkpoint is None or checkpoint.header is None:
+ return None
+ return cls(task.tasks()[0].task(), checkpoint_root, checkpoint.header.sandbox,
+ log_dir=checkpoint.header.log_dir, task_id=task_id,
+ portmap=checkpoint.header.ports)
+ except Exception as e:
+ log.error('Failed to reconstitute checkpoint in TaskRunner.get: %s' % e, exc_info=True)
+ return None
+
+ def __init__(self, task, checkpoint_root, sandbox, log_dir=None,
+ task_id=None, portmap=None, user=None, chroot=False, clock=time,
+ universal_handler=None, planner_class=TaskPlanner):
+ """
+ required:
+ task (config.Task) = the task to run
+ checkpoint_root (path) = the checkpoint root
+ sandbox (path) = the sandbox in which the path will be run
+ [if None, cwd will be assumed, but garbage collection will be
+ disabled for this task.]
+
+ optional:
+ log_dir (string) = directory to house stdout/stderr logs. If not specified, logs will be
+ written into the sandbox directory under .logs/
+ task_id (string) = bind to this task id. if not specified, will synthesize an id based
+ upon task.name()
+ portmap (dict) = a map (string => integer) from name to port, e.g. { 'http': 80 }
+ user (string) = the user to run the task as. if not current user, requires setuid
+ privileges.
+ chroot (boolean) = whether or not to chroot into the sandbox prior to exec.
+ clock (time interface) = the clock to use throughout
+ universal_handler = checkpoint record handler (only used for testing)
+ planner_class (TaskPlanner class) = TaskPlanner class to use for constructing the task
+ planning policy.
+ """
+ if not issubclass(planner_class, TaskPlanner):
+ raise TypeError('planner_class must be a TaskPlanner.')
+ self._clock = clock
+ launch_time = self._clock.time()
+ launch_time_ms = '%06d' % int((launch_time - int(launch_time)) * 10**6)
+ if not task_id:
+ self._task_id = '%s-%s.%s' % (task.name(),
+ time.strftime('%Y%m%d-%H%M%S', time.localtime(launch_time)),
+ launch_time_ms)
+ else:
+ self._task_id = task_id
+ current_user = TaskRunnerHelper.get_actual_user()
+ self._user = user or current_user
+ # TODO(wickman) This should be delegated to the ProcessPlatform / Helper
+ if self._user != current_user:
+ if os.geteuid() != 0:
+ raise ValueError('task specifies user as %s, but %s does not have setuid permission!' % (
+ self._user, current_user))
+ self._portmap = portmap or {}
+ self._launch_time = launch_time
+ self._log_dir = log_dir or os.path.join(sandbox, '.logs')
+ self._pathspec = TaskPath(root=checkpoint_root, task_id=self._task_id, log_dir=self._log_dir)
+ try:
+ ThermosTaskValidator.assert_valid_task(task)
+ ThermosTaskValidator.assert_valid_ports(task, self._portmap)
+ except ThermosTaskValidator.InvalidTaskError as e:
+ raise self.InvalidTask('Invalid task: %s' % e)
+ context = ThermosContext(
+ task_id=self._task_id,
+ ports=self._portmap,
+ user=self._user)
+ self._task, uninterp = (task % Environment(thermos=context)).interpolate()
+ if len(uninterp) > 0:
+ raise self.InvalidTask('Failed to interpolate task, missing: %s' %
+ ', '.join(str(ref) for ref in uninterp))
+ try:
+ ThermosTaskValidator.assert_same_task(self._pathspec, self._task)
+ except ThermosTaskValidator.InvalidTaskError as e:
+ raise self.InvalidTask('Invalid task: %s' % e)
+ self._plan = None # plan currently being executed (updated by Handlers)
+ self._regular_plan = planner_class(self._task, clock=clock,
+ process_filter=lambda proc: proc.final().get() == False)
+ self._finalizing_plan = planner_class(self._task, clock=clock,
+ process_filter=lambda proc: proc.final().get() == True)
+ self._chroot = chroot
+ self._sandbox = sandbox
+ self._terminal_state = None
+ self._ckpt = None
+ self._process_map = dict((p.name().get(), p) for p in self._task.processes())
+ self._task_processes = {}
+ self._stages = dict((state, stage(self)) for state, stage in self.STAGES.items())
+ self._finalization_start = None
+ self._preemption_deadline = None
+ self._watcher = ProcessMuxer(self._pathspec)
+ self._state = RunnerState(processes = {})
+
+ # create runner state
+ universal_handler = universal_handler or TaskRunnerUniversalHandler
+ self._dispatcher = CheckpointDispatcher()
+ self._dispatcher.register_handler(universal_handler(self))
+ self._dispatcher.register_handler(TaskRunnerProcessHandler(self))
+ self._dispatcher.register_handler(TaskRunnerTaskHandler(self))
+
+ # recover checkpointed runner state and update plan
+ self._recovery = True
+ self._replay_runner_ckpt()
+
+ @property
+ def task(self):
+ return self._task
+
+ @property
+ def task_id(self):
+ return self._task_id
+
+ @property
+ def state(self):
+ return self._state
+
+ @property
+ def processes(self):
+ return self._task_processes
+
+ def task_state(self):
+ return self._state.statuses[-1].state if self._state.statuses else TaskState.ACTIVE
+
+ def close_ckpt(self):
+ """Force close the checkpoint stream. This is necessary for runners terminated through
+ exception propagation."""
+ log.debug('Closing the checkpoint stream.')
+ self._ckpt.close()
+
+ @contextmanager
+ def control(self, force=False):
+ """
+ Bind to the checkpoint associated with this task, position to the end of the log if
+ it exists, or create it if it doesn't. Fails if we cannot get "leadership" i.e. a
+ file lock on the checkpoint stream.
+ """
+ if self.is_terminal():
+ raise TaskRunner.StateError('Cannot take control of a task in terminal state.')
+ if self._sandbox:
+ safe_mkdir(self._sandbox)
+ ckpt_file = self._pathspec.getpath('runner_checkpoint')
+ try:
+ self._ckpt = TaskRunnerHelper.open_checkpoint(ckpt_file, force=force, state=self._state)
+ except TaskRunnerHelper.PermissionError:
+ raise TaskRunner.PermissionError('Unable to open checkpoint %s' % ckpt_file)
+ log.debug('Flipping recovery mode off.')
+ self._recovery = False
+ self._set_task_status(self.task_state())
+ self._resume_task()
+ try:
+ yield
+ except Exception as e:
+ log.error('Caught exception in self.control(): %s' % e)
+ log.error(' %s' % traceback.format_exc())
+ self._ckpt.close()
+
+ def _resume_task(self):
+ assert self._ckpt is not None
+ unapplied_updates = self._replay_process_ckpts()
+ if self.is_terminal():
+ raise self.StateError('Cannot resume terminal task.')
+ self._initialize_ckpt_header()
+ self._replay(unapplied_updates)
+
+ def _ckpt_write(self, record):
+ """
+ Write to the checkpoint stream if we're not in recovery mode.
+ """
+ if not self._recovery:
+ self._ckpt.write(record)
+
+ def _replay(self, checkpoints):
+ """
+ Replay a sequence of RunnerCkpts.
+ """
+ for checkpoint in checkpoints:
+ self._dispatcher.dispatch(self._state, checkpoint)
+
+ def _replay_runner_ckpt(self):
+ """
+ Replay the checkpoint stream associated with this task.
+ """
+ ckpt_file = self._pathspec.getpath('runner_checkpoint')
+ if os.path.exists(ckpt_file):
+ fp = open(ckpt_file, "r")
+ ckpt_recover = ThriftRecordReader(fp, RunnerCkpt)
+ for record in ckpt_recover:
+ log.debug('Replaying runner checkpoint record: %s' % record)
+ self._dispatcher.dispatch(self._state, record, recovery=True)
+ ckpt_recover.close()
+
+ def _replay_process_ckpts(self):
+ """
+ Replay the unmutating process checkpoints. Return the unapplied process updates that
+ would mutate the runner checkpoint stream.
+ """
+ process_updates = self._watcher.select()
+ unapplied_process_updates = []
+ for process_update in process_updates:
+ if self._dispatcher.would_update(self._state, process_update):
+ unapplied_process_updates.append(process_update)
+ else:
+ self._dispatcher.dispatch(self._state, process_update, recovery=True)
+ return unapplied_process_updates
+
+ def _initialize_ckpt_header(self):
+ """
+ Initializes the RunnerHeader for this checkpoint stream if it has not already
+ been constructed.
+ """
+ if self._state.header is None:
+ header = RunnerHeader(
+ task_id=self._task_id,
+ launch_time_ms=int(self._launch_time*1000),
+ sandbox=self._sandbox,
+ log_dir=self._log_dir,
+ hostname=socket.gethostname(),
+ user=self._user,
+ ports=self._portmap)
+ runner_ckpt = RunnerCkpt(runner_header=header)
+ self._dispatcher.dispatch(self._state, runner_ckpt)
+
+ def _set_task_status(self, state):
+ update = TaskStatus(state=state, timestamp_ms=int(self._clock.time() * 1000),
+ runner_pid=os.getpid(), runner_uid=os.getuid())
+ runner_ckpt = RunnerCkpt(task_status=update)
+ self._dispatcher.dispatch(self._state, runner_ckpt, self._recovery)
+
+ def _finalization_remaining(self):
+ # If a preemption deadline has been set, use that.
+ if self._preemption_deadline:
+ return max(0, self._preemption_deadline - self._clock.time())
+
+ # Otherwise, use the finalization wait provided in the configuration.
+ finalization_allocation = self.task.finalization_wait().get()
+ if self._finalization_start is None:
+ return sys.float_info.max
+ else:
+ waited = max(0, self._clock.time() - self._finalization_start)
+ return max(0, finalization_allocation - waited)
+
+ def _set_process_status(self, process_name, process_state, **kw):
+ if 'sequence_number' in kw:
+ sequence_number = kw.pop('sequence_number')
+ log.debug('_set_process_status(%s <= %s, seq=%s[force])' % (process_name,
+ ProcessState._VALUES_TO_NAMES.get(process_state), sequence_number))
+ else:
+ current_run = self._current_process_run(process_name)
+ if not current_run:
+ assert process_state == ProcessState.WAITING
+ sequence_number = 0
+ else:
+ sequence_number = current_run.seq + 1
+ log.debug('_set_process_status(%s <= %s, seq=%s[auto])' % (process_name,
+ ProcessState._VALUES_TO_NAMES.get(process_state), sequence_number))
+ runner_ckpt = RunnerCkpt(process_status=ProcessStatus(
+ process=process_name, state=process_state, seq=sequence_number, **kw))
+ self._dispatcher.dispatch(self._state, runner_ckpt, self._recovery)
+
+ def _task_process_from_process_name(self, process_name, sequence_number):
+ """
+ Construct a Process() object from a process_name, populated with its
+ correct run number and fully interpolated commandline.
+ """
+ run_number = len(self.state.processes[process_name]) - 1
+ pathspec = self._pathspec.given(process=process_name, run=run_number)
+ process = self._process_map.get(process_name)
+ if process is None:
+ raise self.InternalError('FATAL: Could not find process: %s' % process_name)
+ def close_ckpt_and_fork():
+ pid = os.fork()
+ if pid == 0 and self._ckpt is not None:
+ self._ckpt.close()
+ return pid
+ return Process(
+ process.name().get(),
+ process.cmdline().get(),
+ sequence_number,
+ pathspec,
+ self._sandbox,
+ self._user,
+ chroot=self._chroot,
+ fork=close_ckpt_and_fork)
+
+ def deadlocked(self, plan=None):
+ """Check whether a plan is deadlocked, i.e. there are no running/runnable processes, and the
+ plan is not complete."""
+ plan = plan or self._regular_plan
+ now = self._clock.time()
+ running = list(plan.running)
+ runnable = list(plan.runnable_at(now))
+ waiting = list(plan.waiting_at(now))
+ log.debug('running:%d runnable:%d waiting:%d complete:%s' % (
+ len(running), len(runnable), len(waiting), plan.is_complete()))
+ return len(running + runnable + waiting) == 0 and not plan.is_complete()
+
+ def is_healthy(self):
+ """Check whether the TaskRunner is healthy. A healthy TaskRunner is not deadlocked and has not
+ reached its max_failures count."""
+ max_failures = self._task.max_failures().get()
+ deadlocked = self.deadlocked()
+ under_failure_limit = max_failures == 0 or len(self._regular_plan.failed) < max_failures
+ log.debug('max_failures:%d failed:%d under_failure_limit:%s deadlocked:%s ==> health:%s' % (
+ max_failures, len(self._regular_plan.failed), under_failure_limit, deadlocked,
+ not deadlocked and under_failure_limit))
+ return not deadlocked and under_failure_limit
+
+ def _current_process_run(self, process_name):
+ if process_name not in self._state.processes or len(self._state.processes[process_name]) == 0:
+ return None
+ return self._state.processes[process_name][-1]
+
+ def is_process_lost(self, process_name):
+ """Determine whether or not we should mark a task as LOST and do so if necessary."""
+ current_run = self._current_process_run(process_name)
+ if not current_run:
+ raise self.InternalError('No current_run for process %s!' % process_name)
+
+ def forked_but_never_came_up():
+ return current_run.state == ProcessState.FORKED and (
+ self._clock.time() - current_run.fork_time > TaskRunner.LOST_TIMEOUT.as_(Time.SECONDS))
+
+ def running_but_coordinator_died():
+ if current_run.state != ProcessState.RUNNING:
+ return False
+ coordinator_pid, _, _ = TaskRunnerHelper.scan_process(self.state, process_name)
+ if coordinator_pid is not None:
+ return False
+ elif self._watcher.has_data(process_name):
+ return False
+ return True
+
+ if forked_but_never_came_up() or running_but_coordinator_died():
+ log.info('Detected a LOST task: %s' % current_run)
+ log.debug(' forked_but_never_came_up: %s' % forked_but_never_came_up())
+ log.debug(' running_but_coordinator_died: %s' % running_but_coordinator_died())
+ return True
+
+ return False
+
+ def _run_plan(self, plan):
+ log.debug('Schedule pass:')
+
+ running = list(plan.running)
+ log.debug('running: %s' % ' '.join(plan.running))
+ log.debug('finished: %s' % ' '.join(plan.finished))
+
+ launched = []
+ for process_name in plan.running:
+ if self.is_process_lost(process_name):
+ self._set_process_status(process_name, ProcessState.LOST)
+
+ now = self._clock.time()
+ runnable = list(plan.runnable_at(now))
+ waiting = list(plan.waiting_at(now))
+ log.debug('runnable: %s' % ' '.join(runnable))
+ log.debug('waiting: %s' % ' '.join(
+ '%s[T-%.1fs]' % (process, plan.get_wait(process)) for process in waiting))
+
+ def pick_processes(process_list):
+ if self._task.max_concurrency().get() == 0:
+ return process_list
+ num_to_pick = max(self._task.max_concurrency().get() - len(running), 0)
+ return process_list[:num_to_pick]
+
+ for process_name in pick_processes(runnable):
+ tp = self._task_processes.get(process_name)
+ if tp:
+ current_run = self._current_process_run(process_name)
+ assert current_run.state == ProcessState.WAITING
+ else:
+ self._set_process_status(process_name, ProcessState.WAITING)
+ tp = self._task_processes[process_name]
+ log.info('Forking Process(%s)' % process_name)
+ tp.start()
+ launched.append(tp)
+
+ return len(launched) > 0
+
+ def _terminate_plan(self, plan):
+ for process in plan.running:
+ last_run = self._current_process_run(process)
+ if last_run and last_run.state in (ProcessState.FORKED, ProcessState.RUNNING):
+ TaskRunnerHelper.terminate_process(self.state, process)
+
+ def has_running_processes(self):
+ """
+ Returns True if any processes associated with this task have active pids.
+ """
+ process_tree = TaskRunnerHelper.scantree(self.state)
+ return any(any(process_set) for process_set in process_tree.values())
+
+ def has_active_processes(self):
+ """
+ Returns True if any processes are in non-terminal states.
+ """
+ return any(not TaskRunnerHelper.is_process_terminal(run.state) for run in
+ filter(None, (self._current_process_run(process) for process in self.state.processes)))
+
+ def collect_updates(self, timeout=None):
+ """
+ Collects and applies updates from process checkpoint streams. Returns the number
+ of applied process checkpoints.
+ """
+ if self.has_active_processes():
+ sleep_interval = self.COORDINATOR_INTERVAL_SLEEP.as_(Time.SECONDS)
+ total_time = 0.0
+ while True:
+ process_updates = self._watcher.select()
+ for process_update in process_updates:
+ self._dispatcher.dispatch(self._state, process_update, self._recovery)
+ if process_updates:
+ return len(process_updates)
+ if timeout and total_time >= timeout:
+ break
+ total_time += sleep_interval
+ self._clock.sleep(sleep_interval)
+ return 0
+
+ def is_terminal(self):
+ return TaskRunnerHelper.is_task_terminal(self.task_state())
+
+ def terminal_state(self):
+ if self._terminal_state:
+ log.debug('Forced terminal state: %s' %
+ TaskState._VALUES_TO_NAMES.get(self._terminal_state, 'UNKNOWN'))
+ return self._terminal_state
+ else:
+ return TaskState.SUCCESS if self.is_healthy() else TaskState.FAILED
+
+ def run(self, force=False):
+ """
+ Entrypoint to runner. Assume control of checkpoint stream, and execute TaskRunnerStages
+ until runner is terminal.
+ """
+ if self.is_terminal():
+ return
+ with self.control(force):
+ self._run()
+
+ def _run(self):
+ iteration_time = self.MAX_ITERATION_TIME.as_(Time.SECONDS)
+ while not self.is_terminal():
+ start = self._clock.time()
+ # step 1: execute stage corresponding to the state we're currently in
+ runner = self._stages[self.task_state()]
+ iteration_wait = runner.run()
+ if iteration_wait is None:
+ log.debug('Run loop: No more work to be done in state %s' %
+ TaskState._VALUES_TO_NAMES.get(self.task_state(), 'UNKNOWN'))
+ self._set_task_status(runner.transition_to())
+ continue
+ log.debug('Run loop: Work to be done within %.1fs' % iteration_wait)
+ # step 2: check child process checkpoint streams for updates
+ if not self.collect_updates(iteration_wait):
+ # If we don't collect any updates, at least 'touch' the checkpoint stream
+ # so as to prevent garbage collection.
+ elapsed = self._clock.time() - start
+ if elapsed < iteration_wait:
+ log.debug('Update collection only took %.1fs, idling %.1fs' % (
+ elapsed, iteration_wait - elapsed))
+ self._clock.sleep(iteration_wait - elapsed)
+ log.debug('Run loop: No updates collected, touching checkpoint.')
+ os.utime(self._pathspec.getpath('runner_checkpoint'), None)
+ # step 3: reap any zombie child processes
+ TaskRunnerHelper.reap_children()
+
+ def kill(self, force=False, terminal_status=TaskState.KILLED,
+ preemption_wait=Amount(1, Time.MINUTES)):
+ """
+ Kill all processes associated with this task and set task/process states as terminal_status
+ (defaults to KILLED)
+ """
+ log.debug('Runner issued kill: force:%s, preemption_wait:%s' % (
+ force, preemption_wait))
+ assert terminal_status in (TaskState.KILLED, TaskState.LOST)
+ self._preemption_deadline = self._clock.time() + preemption_wait.as_(Time.SECONDS)
+ with self.control(force):
+ if self.is_terminal():
+ log.warning('Task is not in ACTIVE state, cannot issue kill.')
+ return
+ self._terminal_state = terminal_status
+ if self.task_state() == TaskState.ACTIVE:
+ self._set_task_status(TaskState.CLEANING)
+ self._run()
+
+ def lose(self, force=False):
+ """
+ Mark a task as LOST and kill any straggling processes.
+ """
+ self.kill(force, preemption_wait=Amount(0, Time.SECONDS), terminal_status=TaskState.LOST)
+
+ def _kill(self):
+ processes = TaskRunnerHelper.scantree(self._state)
+ for process, pid_tuple in processes.items():
+ current_run = self._current_process_run(process)
+ coordinator_pid, pid, tree = pid_tuple
+ if TaskRunnerHelper.is_process_terminal(current_run.state):
+ if coordinator_pid or pid or tree:
+ log.warning('Terminal process (%s) still has running pids:' % process)
+ log.warning(' coordinator_pid: %s' % coordinator_pid)
+ log.warning(' pid: %s' % pid)
+ log.warning(' tree: %s' % tree)
+ TaskRunnerHelper.kill_process(self.state, process)
+ else:
+ if coordinator_pid or pid or tree:
+ log.info('Transitioning %s to KILLED' % process)
+ self._set_process_status(process, ProcessState.KILLED,
+ stop_time=self._clock.time(), return_code=-1)
+ else:
+ log.info('Transitioning %s to LOST' % process)
+ if current_run.state != ProcessState.WAITING:
+ self._set_process_status(process, ProcessState.LOST)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/BUILD b/src/main/python/apache/thermos/monitoring/BUILD
new file mode 100644
index 0000000..a977878
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/BUILD
@@ -0,0 +1,97 @@
+import os
+
+python_library(
+ name = 'detector',
+ sources = ['detector.py'],
+ dependencies = [
+ pants('src/main/python/twitter/thermos/common:path')
+ ]
+)
+
+python_library(
+ name = 'garbage',
+ sources = ['garbage.py'],
+ dependencies = [
+ pants(':detector'),
+ pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ ]
+)
+
+python_library(
+ name = 'monitor',
+ sources = ['monitor.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'disk',
+ sources = ['disk.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ python_requirement('watchdog'),
+ ]
+)
+
+python_library(
+ name = 'process',
+ sources = ['process.py'],
+)
+
+python_library(
+ name = 'process_collector_psutil',
+ sources = ['process_collector_psutil.py'],
+ dependencies = [
+ pants(':process'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/thermos:psutil'),
+ ]
+)
+
+python_library(
+ name = 'resource',
+ sources = ['resource.py'],
+ dependencies = [
+ pants(':disk'),
+ pants(':monitor'),
+ pants(':process'),
+ pants(':process_collector_psutil'),
+ pants('aurora/twitterdeps/src/python/twitter/common/collections'),
+ pants('aurora/twitterdeps/src/python/twitter/common/concurrent'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ ]
+)
+
+python_library(
+ name = 'monitoring',
+ dependencies = [
+ pants(':detector'),
+ pants(':disk'),
+ pants(':garbage'),
+ pants(':monitor'),
+ pants(':process'),
+ pants(':resource'),
+
+ # covering dependency for common
+ pants('src/main/python/twitter/thermos/common'),
+ ],
+ provides = setup_py(
+ name = 'twitter.thermos.monitoring',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ description = 'Thermos monitoring library.',
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/__init__.py b/src/main/python/apache/thermos/monitoring/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/detector.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/detector.py b/src/main/python/apache/thermos/monitoring/detector.py
new file mode 100644
index 0000000..ed48c93
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/detector.py
@@ -0,0 +1,91 @@
+"""Detect Thermos tasks on disk
+
+This module contains the TaskDetector, used to detect Thermos tasks within a given checkpoint root.
+
+"""
+
+import glob
+import os
+import re
+
+from twitter.thermos.common.path import TaskPath
+
+
+class TaskDetector(object):
+ """
+ Helper class in front of TaskPath to detect active/finished/running tasks. Performs no
+ introspection on the state of a task; merely detects based on file paths on disk.
+ """
+ class MatchingError(Exception): pass
+
+ def __init__(self, root):
+ self._root_dir = root
+ self._pathspec = TaskPath()
+
+ def get_task_ids(self, state=None):
+ paths = glob.glob(self._pathspec.given(root=self._root_dir,
+ task_id="*",
+ state=state or '*')
+ .getpath('task_path'))
+ path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
+ task_id="(\S+)",
+ state='(\S+)')
+ .getpath('task_path'))
+ for path in paths:
+ try:
+ task_state, task_id = path_re.match(path).groups()
+ except:
+ continue
+ if state is None or task_state == state:
+ yield (task_state, task_id)
+
+ def get_process_runs(self, task_id, log_dir):
+ paths = glob.glob(self._pathspec.given(root=self._root_dir,
+ task_id=task_id,
+ log_dir=log_dir,
+ process='*',
+ run='*')
+ .getpath('process_logdir'))
+ path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
+ task_id=re.escape(task_id),
+ log_dir=log_dir,
+ process='(\S+)',
+ run='(\d+)')
+ .getpath('process_logdir'))
+ for path in paths:
+ try:
+ process, run = path_re.match(path).groups()
+ except:
+ continue
+ yield process, int(run)
+
+ def get_process_logs(self, task_id, log_dir):
+ for process, run in self.get_process_runs(task_id, log_dir):
+ for logtype in ('stdout', 'stderr'):
+ path = (self._pathspec.with_filename(logtype).given(root=self._root_dir,
+ task_id=task_id,
+ log_dir=log_dir,
+ process=process,
+ run=run)
+ .getpath('process_logdir'))
+ if os.path.exists(path):
+ yield path
+
+ def get_checkpoint(self, task_id):
+ return self._pathspec.given(root=self._root_dir, task_id=task_id).getpath('runner_checkpoint')
+
+ def get_process_checkpoints(self, task_id):
+ matching_paths = glob.glob(self._pathspec.given(root=self._root_dir,
+ task_id=task_id,
+ process='*')
+ .getpath('process_checkpoint'))
+ path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
+ task_id=re.escape(task_id),
+ process='(\S+)')
+ .getpath('process_checkpoint'))
+ for path in matching_paths:
+ try:
+ process, = path_re.match(path).groups()
+ except:
+ continue
+ yield path
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/disk.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/disk.py b/src/main/python/apache/thermos/monitoring/disk.py
new file mode 100644
index 0000000..9116746
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/disk.py
@@ -0,0 +1,181 @@
+"""Sample disk usage under a particular path
+
+This module provides threads which can be used to gather information on the disk utilisation
+under a particular path.
+
+Currently, there are two threads available:
+ - DiskCollectorThread, which periodically uses a basic brute-force approach (os.stat()ing every
+ file within the path)
+ - InotifyDiskCollectorThread, which updates disk utilisation dynamically by using inotify to
+ monitor disk changes within the path
+
+"""
+
+import os
+import threading
+import time
+from Queue import Empty, Queue
+
+from twitter.common import log
+from twitter.common.dirutil import du, safe_bsize
+from twitter.common.exceptions import ExceptionalThread
+from twitter.common.lang import Lockable
+from twitter.common.quantity import Amount, Time
+
+from watchdog.observers import Observer as WatchdogObserver
+from watchdog.events import (
+ FileSystemEventHandler,
+ FileCreatedEvent,
+ FileDeletedEvent,
+ FileModifiedEvent,
+ FileMovedEvent,
+)
+
+
+class DiskCollectorThread(ExceptionalThread):
+ """ Thread to calculate aggregate disk usage under a given path using a simple algorithm """
+ def __init__(self, path):
+ self.path = path
+ self.value = None
+ self.event = threading.Event()
+ super(DiskCollectorThread, self).__init__()
+ self.daemon = True
+
+ def run(self):
+ log.debug("DiskCollectorThread: starting collection of %s" % self.path)
+ self.value = du(self.path)
+ log.debug("DiskCollectorThread: finished collection of %s" % self.path)
+ self.event.set()
+
+ def finished(self):
+ return self.event.is_set()
+
+
+class DiskCollector(Lockable):
+ """ Spawn a background thread to sample disk usage """
+ def __init__(self, root):
+ self._root = root
+ self._thread = None
+ self._value = 0
+ super(DiskCollector, self).__init__()
+
+ @Lockable.sync
+ def sample(self):
+ """ Trigger collection of sample, if not already begun """
+ if self._thread is None:
+ self._thread = DiskCollectorThread(self._root)
+ self._thread.start()
+
+ @property
+ @Lockable.sync
+ def value(self):
+ """ Retrieve value of disk usage """
+ if self._thread is not None and self._thread.finished():
+ self._value = self._thread.value
+ self._thread = None
+ return self._value
+
+ @property
+ @Lockable.sync
+ def completed_event(self):
+ """ Return a threading.Event that will block until an in-progress disk collection is complete,
+ or block indefinitely otherwise. Use with caution! (i.e.: set a timeout) """
+ if self._thread is not None:
+ return self._thread.event
+ else:
+ return threading.Event()
+
+
+class InotifyDiskCollectorThread(ExceptionalThread, FileSystemEventHandler):
+ """ Thread to calculate aggregate disk usage under a given path
+
+ Note that while this thread uses inotify (through the watchdog module) to monitor disk events in
+ "real time", the actual processing of events is only performed periodically (configured via
+ COLLECTION_INTERVAL)
+
+ """
+ INTERESTING_EVENTS = (FileCreatedEvent, FileDeletedEvent, FileModifiedEvent, FileMovedEvent)
+ COLLECTION_INTERVAL = Amount(5, Time.SECONDS)
+
+ def __init__(self, path):
+ self._path = path
+ self._files = {} # file path => size (bytes)
+ self._queue = Queue()
+ self._observer = WatchdogObserver()
+ super(InotifyDiskCollectorThread, self).__init__()
+ self.daemon = True
+
+ def dispatch(self, event):
+ """ Dispatch all interesting events to the internal queue """
+ if isinstance(event, self.INTERESTING_EVENTS):
+ self._queue.put(event)
+
+ def _initialize(self):
+ """ Collect an initial snapshot of the disk usage in the path """
+ log.debug("Starting watchdog observer to collect events...")
+ self._observer.schedule(self, path=self._path, recursive=True)
+ self._observer.start()
+ log.debug("Collecting initial disk usage sample...")
+ for root, _, files in os.walk(self._path):
+ for filename in files:
+ f = os.path.join(root, filename)
+ self._files[f] = safe_bsize(f)
+
+ def _process_events(self):
+ """ Deduplicate and process watchdog events, updating the internal file store appropriately """
+ file_ops = {}
+
+ def remove_file(path):
+ self._files.pop(path, None)
+ def stat_file(path):
+ self._files[path] = safe_bsize(path)
+
+ while not self._to_process.empty():
+ event = self._to_process.get()
+ if isinstance(event, (FileCreatedEvent, FileModifiedEvent)):
+ file_ops[event.src_path] = lambda: stat_file(event.src_path)
+ elif isinstance(event, FileDeletedEvent):
+ file_ops[event.src_path] = lambda: remove_file(event.src_path)
+ elif isinstance(event, FileMovedEvent):
+ file_ops[event.src_path] = lambda: remove_file(event.src_path)
+ file_ops[event.dest_path] = lambda: stat_file(event.dest_path)
+
+ for op in file_ops.values():
+ op()
+
+ def run(self):
+ """ Loop indefinitely, periodically processing watchdog/inotify events. """
+ self._initialize()
+ log.debug("Initialization complete. Moving to handling events.")
+ while True:
+ next = time.time() + self.COLLECTION_INTERVAL.as_(Time.SECONDS)
+ if not self._queue.empty():
+ self._to_process, self._queue = self._queue, Queue()
+ self._process_events()
+ time.sleep(max(0, next - time.time()))
+
+ @property
+ def value(self):
+ return sum(self._files.itervalues())
+
+
+class InotifyDiskCollector(object):
+ """ Spawn a background thread to sample disk usage """
+ def __init__(self, root):
+ self._root = root
+ self._thread = InotifyDiskCollectorThread(self._root)
+
+ def sample(self):
+ """ Trigger disk collection loop. """
+ if not os.path.exists(self._root):
+ log.error('Cannot start monitoring path until it exists')
+ elif not self._thread.is_alive():
+ self._thread.start()
+
+ @property
+ def value(self):
+ return self._thread.value
+
+ @property
+ def completed_event(self):
+ return threading.Event()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/garbage.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/garbage.py b/src/main/python/apache/thermos/monitoring/garbage.py
new file mode 100644
index 0000000..4ac0d3a
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/garbage.py
@@ -0,0 +1,183 @@
+from abc import abstractmethod
+from collections import namedtuple
+import os
+import sys
+import time
+
+from twitter.common.dirutil import safe_delete, safe_rmtree, safe_bsize
+from twitter.common.lang import Interface
+from twitter.common.quantity import Amount, Data, Time
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+from twitter.thermos.common.path import TaskPath
+
+from .detector import TaskDetector
+
+
+class TaskGarbageCollector(object):
+ def __init__(self, root):
+ self._root = root
+ self._detector = TaskDetector(root=self._root)
+ self._states = {}
+
+ def state(self, task_id):
+ if task_id not in self._states:
+ self._states[task_id] = CheckpointDispatcher.from_file(self._detector.get_checkpoint(task_id))
+ return self._states[task_id]
+
+ def get_age(self, task_id):
+ return os.path.getmtime(self._detector.get_checkpoint(task_id))
+
+ def get_finished_tasks(self):
+ return [task_id for _, task_id in self._detector.get_task_ids(state='finished')]
+
+ def get_metadata(self, task_id, with_size=True):
+ runner_ckpt = self._detector.get_checkpoint(task_id)
+ process_ckpts = [ckpt for ckpt in self._detector.get_process_checkpoints(task_id)]
+ json_spec = TaskPath(root=self._root, task_id=task_id, state='finished').getpath('task_path')
+ for path in [json_spec, runner_ckpt] + process_ckpts:
+ if with_size:
+ yield path, safe_bsize(path)
+ else:
+ yield path
+
+ def get_logs(self, task_id, with_size=True):
+ state = self.state(task_id)
+ if state and state.header:
+ for path in self._detector.get_process_logs(task_id, state.header.log_dir):
+ if with_size:
+ yield path, safe_bsize(path)
+ else:
+ yield path
+
+ def get_data(self, task_id, with_size=True):
+ state = self.state(task_id)
+ if state and state.header and state.header.sandbox:
+ for root, dirs, files in os.walk(state.header.sandbox):
+ for file in files:
+ filename = os.path.join(root, file)
+ if with_size:
+ yield filename, safe_bsize(filename)
+ else:
+ yield filename
+
+ def erase_task(self, task_id):
+ self.erase_data(task_id)
+ self.erase_logs(task_id)
+ self.erase_metadata(task_id)
+
+ def erase_metadata(self, task_id):
+ for fn in self.get_metadata(task_id, with_size=False):
+ safe_delete(fn)
+ safe_rmtree(TaskPath(root=self._root, task_id=task_id).getpath('checkpoint_path'))
+
+ def erase_logs(self, task_id):
+ for fn in self.get_logs(task_id, with_size=False):
+ safe_delete(fn)
+ state = self.state(task_id)
+ if state and state.header:
+ safe_rmtree(TaskPath(root=self._root, task_id=task_id, log_dir=state.header.log_dir)
+ .getpath('process_logbase'))
+
+ def erase_data(self, task_id):
+ # TODO(wickman)
+ # This could be potentially dangerous if somebody naively runs their sandboxes in e.g.
+ # $HOME or / or similar. Perhaps put a guard somewhere?
+ for fn in self.get_data(task_id, with_size=False):
+ os.unlink(fn)
+ state = self.state(task_id)
+ if state and state.header and state.header.sandbox:
+ safe_rmtree(state.header.sandbox)
+
+
+class TaskGarbageCollectionPolicy(Interface):
+ def __init__(self, collector):
+ if not isinstance(collector, TaskGarbageCollector):
+ raise ValueError(
+ "Expected collector to be a TaskGarbageCollector, got %s" % collector.__class__.__name__)
+ self._collector = collector
+
+ @property
+ def collector(self):
+ return self._collector
+
+ @abstractmethod
+ def run(self):
+ """Returns a list of task_ids that should be garbage collected given the specified policy."""
+
+
+class DefaultCollector(TaskGarbageCollectionPolicy):
+ def __init__(self, collector, **kw):
+ """
+ Default garbage collection policy.
+
+ Arguments that may be specified:
+ max_age: Amount(Time) (max age of a retained task) [default: infinity]
+ max_space: Amount(Data) (max space to keep) [default: infinity]
+ max_tasks: int (max number of tasks to keep) [default: infinity]
+ include_metadata: boolean (Whether or not to include metadata in the
+ space calculations.) [default: True]
+ include_logs: boolean (Whether or not to include logs in the
+ space calculations.) [default: True]
+ verbose: boolean (whether or not to log) [default: False]
+ logger: callable (function to call with log messages) [default: sys.stdout.write]
+ """
+ self._max_age = kw.get('max_age', Amount(10**10, Time.DAYS))
+ self._max_space = kw.get('max_space', Amount(10**10, Data.TB))
+ self._max_tasks = kw.get('max_tasks', 10**10)
+ self._include_metadata = kw.get('include_metadata', True)
+ self._include_logs = kw.get('include_logs', True)
+ self._verbose = kw.get('verbose', False)
+ self._logger = kw.get('logger', sys.stdout.write)
+ TaskGarbageCollectionPolicy.__init__(self, collector)
+
+ def log(self, msg):
+ if self._verbose:
+ self._logger(msg)
+
+ def run(self):
+ tasks = []
+ now = time.time()
+
+ TaskTuple = namedtuple('TaskTuple', 'task_id age metadata_size log_size data_size')
+ for task_id in self.collector.get_finished_tasks():
+ age = Amount(int(now - self.collector.get_age(task_id)), Time.SECONDS)
+ self.log('Analyzing task %s (age: %s)... ' % (task_id, age))
+ metadata_size = Amount(sum(sz for _, sz in self.collector.get_metadata(task_id)), Data.BYTES)
+ self.log(' metadata %.1fKB ' % metadata_size.as_(Data.KB))
+ log_size = Amount(sum(sz for _, sz in self.collector.get_logs(task_id)), Data.BYTES)
+ self.log(' logs %.1fKB ' % log_size.as_(Data.KB))
+ data_size = Amount(sum(sz for _, sz in self.collector.get_data(task_id)), Data.BYTES)
+ self.log(' data %.1fMB ' % data_size.as_(Data.MB))
+ tasks.append(TaskTuple(task_id, age, metadata_size, log_size, data_size))
+
+ gc_tasks = set()
+ gc_tasks.update(task for task in tasks if task.age > self._max_age)
+ self.log('After age filter: %s tasks' % len(gc_tasks))
+
+ def total_gc_size(task):
+ return sum([task.data_size,
+ task.metadata_size if self._include_metadata else Amount(0, Data.BYTES),
+ task.log_size if self._include_logs else Amount(0, Data.BYTES)],
+ Amount(0, Data.BYTES))
+
+ total_used = Amount(0, Data.BYTES)
+ for task in sorted(tasks, key=lambda tsk: tsk.age, reverse=True):
+ if task not in gc_tasks:
+ total_used += total_gc_size(task)
+ if total_used > self._max_space:
+ gc_tasks.add(task)
+ self.log('After size filter: %s tasks' % len(gc_tasks))
+
+ for task in sorted(tasks, key=lambda tsk: tsk.age, reverse=True):
+ if task not in gc_tasks and len(tasks) - len(gc_tasks) > self._max_tasks:
+ gc_tasks.add(task)
+ self.log('After total task filter: %s tasks' % len(gc_tasks))
+
+ self.log('Deciding to garbage collect the following tasks:')
+ if gc_tasks:
+ for task in gc_tasks:
+ self.log(' %s' % repr(task))
+ else:
+ self.log(' None.')
+
+ return gc_tasks
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/monitor.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/monitor.py b/src/main/python/apache/thermos/monitoring/monitor.py
new file mode 100644
index 0000000..5af67fe
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/monitor.py
@@ -0,0 +1,125 @@
+"""Monitor the state of Thermos tasks on a system
+
+This module contains the TaskMonitor, used to reconstruct the state of active or finished Thermos
+tasks based on their checkpoint streams. It exposes two key pieces of information about a Task, both
+as their corresponding Thrift structs:
+ - a RunnerState, representing the latest state of the Task
+ - a list of ProcessStates, representing the processes currently running within the Task
+
+"""
+
+import os
+import copy
+import errno
+import threading
+
+from twitter.common import log
+from twitter.common.recordio import ThriftRecordReader
+from twitter.thermos.common.ckpt import CheckpointDispatcher
+
+from gen.twitter.thermos.ttypes import (
+ ProcessState,
+ RunnerCkpt,
+ RunnerState,
+ TaskState,
+)
+
+
+class TaskMonitor(object):
+ """
+ Class responsible for reconstructing and monitoring the state of an individual Thermos task via
+ its runner checkpoint. Also exports information on active processes in the task.
+ """
+
+ def __init__(self, pathspec, task_id):
+ self._task_id = task_id
+ self._dispatcher = CheckpointDispatcher()
+ self._runnerstate = RunnerState(processes={})
+ self._runner_ckpt = pathspec.given(task_id=task_id).getpath('runner_checkpoint')
+ self._active_file, self._finished_file = (
+ pathspec.given(task_id=task_id, state=state).getpath('task_path')
+ for state in ('active', 'finished'))
+ self._ckpt_head = 0
+ self._apply_states()
+ self._lock = threading.Lock()
+
+ def _apply_states(self):
+ """
+ os.stat() the corresponding checkpoint stream of this task and determine if there are new ckpt
+ records. Attempt to read those records and update the high watermark for that stream.
+ Returns True if new states were applied, False otherwise.
+ """
+ ckpt_offset = None
+ try:
+ ckpt_offset = os.stat(self._runner_ckpt).st_size
+
+ updated = False
+ if self._ckpt_head < ckpt_offset:
+ with open(self._runner_ckpt, 'r') as fp:
+ fp.seek(self._ckpt_head)
+ rr = ThriftRecordReader(fp, RunnerCkpt)
+ while True:
+ runner_update = rr.try_read()
+ if not runner_update:
+ break
+ try:
+ self._dispatcher.dispatch(self._runnerstate, runner_update)
+ except CheckpointDispatcher.InvalidSequenceNumber as e:
+ log.error('Checkpoint stream is corrupt: %s' % e)
+ break
+ new_ckpt_head = fp.tell()
+ updated = self._ckpt_head != new_ckpt_head
+ self._ckpt_head = new_ckpt_head
+ return updated
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ # The log doesn't yet exist, will retry later.
+ log.warning('Could not read from discovered task %s.' % self._task_id)
+ return False
+ else:
+ raise
+
+ def refresh(self):
+ """
+ Check to see if there are new updates and apply them. Return true if
+ updates were applied, false otherwise.
+ """
+ with self._lock:
+ return self._apply_states()
+
+ def get_state(self):
+ """
+ Get the latest state of this Task.
+ """
+ with self._lock:
+ self._apply_states()
+ return copy.deepcopy(self._runnerstate)
+
+ def task_state(self):
+ state = self.get_state()
+ return state.statuses[-1].state if state.statuses else TaskState.ACTIVE
+
+ @property
+ def active(self):
+ return os.path.exists(self._active_file)
+
+ @property
+ def finished(self):
+ return os.path.exists(self._finished_file)
+
+ def get_active_processes(self):
+ """
+ Get active processes. Returned is a list of tuples of the form:
+ (ProcessStatus object of running object, its run number)
+ """
+ active_processes = []
+ with self._lock:
+ self._apply_states()
+ state = self._runnerstate
+ for process, runs in state.processes.items():
+ if len(runs) == 0:
+ continue
+ last_run = runs[-1]
+ if last_run.state == ProcessState.RUNNING:
+ active_processes.append((last_run, len(runs) - 1))
+ return active_processes
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/process.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/process.py b/src/main/python/apache/thermos/monitoring/process.py
new file mode 100644
index 0000000..294682f
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/process.py
@@ -0,0 +1,49 @@
+"""Represent resource consumption statistics for processes
+
+This module exposes one class: the ProcessSample, used to represent resource consumption. A single
+ProcessSample might correspond to one individual process, or to an aggregate of multiple processes.
+
+"""
+
+from collections import namedtuple
+
+
+class ProcessSample(namedtuple('ProcessSample', 'rate user system rss vms nice status threads')):
+ """ Sample of statistics about a process's resource consumption (either a single process or an
+ aggregate of processes) """
+
+ @staticmethod
+ def empty():
+ return ProcessSample(rate=0, user=0, system=0, rss=0, vms=0, nice=None, status=None, threads=0)
+
+ def __add__(self, other):
+ if self.nice is not None and other.nice is None:
+ nice = self.nice
+ else:
+ nice = other.nice
+ if self.status is not None and other.status is None:
+ status = self.status
+ else:
+ status = other.status
+ return ProcessSample(
+ rate = self.rate + other.rate,
+ user = self.user + other.user,
+ system = self.system + other.system,
+ rss = self.rss + other.rss,
+ vms = self.vms + other.vms,
+ nice = nice,
+ status = status,
+ threads = self.threads + other.threads)
+
+ def to_dict(self):
+ return dict(
+ cpu = self.rate,
+ ram = self.rss,
+ user = self.user,
+ system = self.system,
+ rss = self.rss,
+ vms = self.vms,
+ nice = self.nice,
+ status = str(self.status),
+ threads = self.threads
+ )
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/process_collector_psutil.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/process_collector_psutil.py b/src/main/python/apache/thermos/monitoring/process_collector_psutil.py
new file mode 100644
index 0000000..7b68173
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/process_collector_psutil.py
@@ -0,0 +1,92 @@
+""" Sample resource consumption statistics for processes using psutil """
+
+from operator import attrgetter
+from time import time
+
+from twitter.common import log
+
+from .process import ProcessSample
+
+from psutil import Process
+from psutil import NoSuchProcess, AccessDenied, Error as PsutilError
+
+
+def process_to_sample(process):
+ """ Given a psutil.Process, return a current ProcessSample """
+ try:
+ # the nonblocking get_cpu_percent call is stateful on a particular Process object, and hence
+ # >2 consecutive calls are required before it will return a non-zero value
+ rate = process.get_cpu_percent(0.0) / 100.0
+ user, system = process.get_cpu_times()
+ rss, vms = process.get_memory_info()
+ nice = process.nice
+ status = process.status
+ threads = process.get_num_threads()
+ return ProcessSample(rate, user, system, rss, vms, nice, status, threads)
+ except (AccessDenied, NoSuchProcess) as e:
+ log.warning('Error during process sampling [pid=%s]: %s' % (process.pid, e))
+ return ProcessSample.empty()
+
+
+class ProcessTreeCollector(object):
+ """ Collect resource consumption statistics for a process and its children """
+ def __init__(self, pid):
+ """ Given a pid """
+ self._pid = pid
+ self._process = None # psutil.Process
+ self._sampled_tree = {} # pid => ProcessSample
+ self._sample = ProcessSample.empty()
+ self._stamp = None
+ self._rate = 0.0
+ self._procs = 1
+
+ def sample(self):
+ """ Collate and aggregate ProcessSamples for process and children
+ Returns None: result is stored in self.value
+ """
+ try:
+ last_sample, last_stamp = self._sample, self._stamp
+ if self._process is None:
+ self._process = Process(self._pid)
+ parent = self._process
+ parent_sample = process_to_sample(parent)
+ new_samples = dict(
+ (proc.pid, process_to_sample(proc))
+ for proc in parent.get_children(recursive=True)
+ )
+ new_samples[self._pid] = parent_sample
+
+ except PsutilError as e:
+ log.warning('Error during process sampling: %s' % e)
+ self._sample = ProcessSample.empty()
+ self._rate = 0.0
+
+ else:
+ last_stamp = self._stamp
+ self._stamp = time()
+ # for most stats, calculate simple sum to aggregate
+ self._sample = sum(new_samples.values(), ProcessSample.empty())
+ # cpu consumption is more complicated
+ # We require at least 2 generations of a process before we can calculate rate, so for all
+ # current processes that were not running in the previous sample, compare to an empty sample
+ if self._sampled_tree and last_stamp:
+ new = new_samples.values()
+ old = [self._sampled_tree.get(pid, ProcessSample.empty()) for pid in new_samples.keys()]
+ new_user_sys = sum(map(attrgetter('user'), new)) + sum(map(attrgetter('system'), new))
+ old_user_sys = sum(map(attrgetter('user'), old)) + sum(map(attrgetter('system'), old))
+ self._rate = (new_user_sys - old_user_sys) / (self._stamp - last_stamp)
+ log.debug("Calculated rate for pid=%s and children: %s" % (self._process.pid, self._rate))
+ self._sampled_tree = new_samples
+
+ @property
+ def value(self):
+ """ Aggregated ProcessSample representing resource consumption of the tree """
+ # Since we don't trust the CPU consumption returned by psutil, replace it with our own in the
+ # exported ProcessSample
+ return self._sample._replace(rate=self._rate)
+
+ @property
+ def procs(self):
+ """ Number of active processes in the tree """
+ return len(self._sampled_tree)
+
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/monitoring/resource.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/monitoring/resource.py b/src/main/python/apache/thermos/monitoring/resource.py
new file mode 100644
index 0000000..468f1b1
--- /dev/null
+++ b/src/main/python/apache/thermos/monitoring/resource.py
@@ -0,0 +1,222 @@
+"""Monitor the resource consumption of Thermos tasks
+
+This module contains classes used to monitor the resource consumption (e.g. CPU, RAM, disk) of
+Thermos tasks. Resource monitoring of a Thermos task typically treats the task as an aggregate of
+all the processes within it. Importantly, this excludes the process(es) of Thermos itself (i.e. the
+TaskRunner and any other wrappers involved in launching a task).
+
+The ResourceMonitorBase defines the interface for other components (for example, the Thermos
+TaskObserver) to interact with and retrieve information about a Task's resource consumption. The
+canonical/reference implementation of a ResourceMonitor is the TaskResourceMonitor, a thread which
+actively monitors resources for a particular task by periodically polling process information and
+disk consumption and retaining a limited (FIFO) in-memory history of this data.
+
+"""
+
+from abc import abstractmethod
+from bisect import bisect_left
+from collections import namedtuple
+from operator import attrgetter
+import platform
+import threading
+import time
+
+from twitter.common import log
+from twitter.common.collections import RingBuffer
+from twitter.common.concurrent import EventMuxer
+from twitter.common.lang import Interface
+from twitter.common.quantity import Amount, Time
+
+from .disk import DiskCollector
+from .monitor import TaskMonitor
+from .process import ProcessSample
+from .process_collector_psutil import ProcessTreeCollector
+
+
+class ResourceMonitorBase(Interface):
+ """ Defines the interface for interacting with a ResourceMonitor """
+
+ class Error(Exception): pass
+
+ class ResourceResult(namedtuple('ResourceResult', 'num_procs process_sample disk_usage')):
+ pass
+
+ @abstractmethod
+ def sample(self):
+ """ Return a sample of the resource consumption of the task right now
+
+ Returns a tuple of (timestamp, ResourceResult)
+ """
+
+ @abstractmethod
+ def sample_at(self, time):
+ """ Return a sample of the resource consumption as close as possible to the specified time
+
+ Returns a tuple of (timestamp, ResourceResult)
+ """
+
+ @abstractmethod
+ def sample_by_process(self, process_name):
+ """ Return a sample of the resource consumption of a specific process in the task right now
+
+ Returns a ProcessSample
+ """
+
+
+class ResourceHistory(object):
+ """Simple class to contain a RingBuffer (fixed-length FIFO) history of resource samples, with the
+ mapping: timestamp => (number_of_procs, ProcessSample, disk_usage_in_bytes)
+ """
+ def __init__(self, maxlen, initialize=True):
+ if not maxlen >= 1:
+ raise ValueError("maxlen must be greater than 0")
+ self._maxlen = maxlen
+ self._values = RingBuffer(maxlen, None)
+ if initialize:
+ self.add(time.time(), ResourceMonitorBase.ResourceResult(0, ProcessSample.empty(), 0))
+
+ def add(self, timestamp, value):
+ """Store a new resource sample corresponding to the given timestamp"""
+ if self._values and not timestamp >= self._values[-1][0]:
+ raise ValueError("Refusing to add timestamp in the past!")
+ self._values.append((timestamp, value))
+
+ def get(self, timestamp):
+ """Get the resource sample nearest to the given timestamp"""
+ closest = min(bisect_left(self._values, (timestamp, None)), len(self) - 1)
+ return self._values[closest]
+
+ def __iter__(self):
+ return iter(self._values)
+
+ def __len__(self):
+ return len(self._values)
+
+ def __repr__(self):
+ return 'ResourceHistory(%s)' % ', '.join([str(r) for r in self._values])
+
+
+class TaskResourceMonitor(ResourceMonitorBase, threading.Thread):
+ """ Lightweight thread to aggregate resource consumption for a task's constituent processes.
+ Actual resource calculation is delegated to collectors; this class periodically polls the
+ collectors and aggregates into a representation for the entire task. Also maintains a limited
+ history of previous sample results.
+ """
+
+ MAX_HISTORY = 10000 # magic number
+
+ def __init__(self, task_monitor, sandbox,
+ process_collector=ProcessTreeCollector, disk_collector=DiskCollector,
+ process_collection_interval=Amount(20, Time.SECONDS),
+ disk_collection_interval=Amount(1, Time.MINUTES),
+ history_time=Amount(1, Time.HOURS)):
+ """
+ task_monitor: TaskMonitor object specifying the task whose resources should be monitored
+ sandbox: Directory for which to monitor disk utilisation
+ """
+ self._task_monitor = task_monitor # exposes PIDs, sandbox
+ self._task_id = task_monitor._task_id
+ log.debug('Initialising resource collection for task %s' % self._task_id)
+ self._process_collectors = dict() # ProcessStatus => ProcessTreeCollector
+ # TODO(jon): sandbox is also available through task_monitor, but typically the first checkpoint
+ # isn't written (and hence the header is not available) by the time we initialise here
+ self._sandbox = sandbox
+ self._process_collector_factory = process_collector
+ self._disk_collector = disk_collector(self._sandbox)
+ self._process_collection_interval = process_collection_interval.as_(Time.SECONDS)
+ self._disk_collection_interval = disk_collection_interval.as_(Time.SECONDS)
+ min_collection_interval = min(self._process_collection_interval, self._disk_collection_interval)
+ history_length = int(history_time.as_(Time.SECONDS) / min_collection_interval)
+ if history_length > self.MAX_HISTORY:
+ raise ValueError("Requested history length too large")
+ log.debug("Initialising ResourceHistory of length %s" % history_length)
+ self._history = ResourceHistory(history_length)
+ self._kill_signal = threading.Event()
+ threading.Thread.__init__(self)
+ self.daemon = True
+
+ def sample(self):
+ if not self.is_alive():
+ log.warning("TaskResourceMonitor not running - sample may be inaccurate")
+ return self.sample_at(time.time())
+
+ def sample_at(self, timestamp):
+ return self._history.get(timestamp)
+
+ def sample_by_process(self, process_name):
+ try:
+ process = [process for process in self._get_active_processes()
+ if process.process == process_name].pop()
+ except IndexError:
+ raise ValueError('No active process found with name "%s" in this task' % process_name)
+ else:
+ # Since this might be called out of band (before the main loop is aware of the process)
+ if process not in self._process_collectors:
+ self._process_collectors[process] = self._process_collector_factory(process.pid)
+
+ self._process_collectors[process].sample()
+ return self._process_collectors[process].value
+
+ def _get_active_processes(self):
+ """Get a list of ProcessStatus objects representing currently-running processes in the task"""
+ return [process for process, _ in self._task_monitor.get_active_processes()]
+
+ def kill(self):
+ """Signal that the thread should cease collecting resources and terminate"""
+ self._kill_signal.set()
+
+ def run(self):
+ """Thread entrypoint. Loop indefinitely, polling collectors at self._collection_interval and
+ collating samples."""
+
+ log.debug('Commencing resource monitoring for task "%s"' % self._task_id)
+ next_process_collection = 0
+ next_disk_collection = 0
+
+ while not self._kill_signal.is_set():
+
+ now = time.time()
+
+ if now > next_process_collection:
+ next_process_collection = now + self._process_collection_interval
+ actives = set(self._get_active_processes())
+ current = set(self._process_collectors)
+ for process in current - actives:
+ log.debug('Process "%s" (pid %s) no longer active, removing from monitored processes' %
+ (process.process, process.pid))
+ self._process_collectors.pop(process)
+ for process in actives - current:
+ log.debug('Adding process "%s" (pid %s) to resource monitoring' %
+ (process.process, process.pid))
+ self._process_collectors[process] = self._process_collector_factory(process.pid)
+ for process, collector in self._process_collectors.iteritems():
+ log.debug('Collecting sample for process "%s" (pid %s) and children' %
+ (process.process, process.pid))
+ collector.sample()
+
+ if now > next_disk_collection:
+ next_disk_collection = now + self._disk_collection_interval
+ log.debug('Collecting disk sample for %s' % self._sandbox)
+ self._disk_collector.sample()
+
+ try:
+ aggregated_procs = sum(map(attrgetter('procs'), self._process_collectors.values()))
+ aggregated_sample = sum(map(attrgetter('value'), self._process_collectors.values()),
+ ProcessSample.empty())
+ self._history.add(now, self.ResourceResult(aggregated_procs, aggregated_sample,
+ self._disk_collector.value))
+ log.debug("Recorded resource sample at %s" % now)
+ except ValueError as err:
+ log.warning("Error recording resource sample: %s" % err)
+
+ # Sleep until any of the following conditions are met:
+ # - it's time for the next disk collection
+ # - it's time for the next process collection
+ # - the result from the last disk collection is available via the DiskCollector
+ # - the TaskResourceMonitor has been killed via self._kill_signal
+ now = time.time()
+ next_collection = min(next_process_collection - now, next_disk_collection - now)
+ EventMuxer(self._kill_signal, self._disk_collector.completed_event
+ ).wait(timeout=max(0, next_collection))
+
+ log.debug('Stopping resource monitoring for task "%s"' % self._task_id)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/BUILD b/src/main/python/apache/thermos/observer/BUILD
new file mode 100644
index 0000000..00ea9cb
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/BUILD
@@ -0,0 +1,51 @@
+import os
+
+python_library(
+ name = 'observed_task',
+ sources = ['observed_task.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('src/main/python/twitter/thermos:pystachio'),
+ pants('src/main/python/twitter/thermos/common:ckpt'),
+ pants('src/main/python/twitter/thermos/config'),
+ ]
+)
+
+python_library(
+ name = 'task_observer',
+ sources = ['task_observer.py'],
+ dependencies = [
+ pants(':observed_task'),
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('aurora/twitterdeps/src/python/twitter/common/lang'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/python/twitter/thermos/monitoring:detector'),
+ pants('src/main/python/twitter/thermos/monitoring:monitor'),
+ pants('src/main/python/twitter/thermos/monitoring:process'),
+ pants('src/main/python/twitter/thermos/monitoring:resource'),
+ pants('src/main/thrift/com/twitter/thermos:py-thrift'),
+ ]
+)
+
+python_library(
+ name = 'observer',
+ dependencies = [
+ pants(':task_observer'),
+ pants('src/main/python/twitter/thermos/observer/http:http_observer'),
+
+ # covering libraries
+ pants('src/main/python/twitter/thermos/common'),
+ pants('src/main/python/twitter/thermos/config'),
+ pants('src/main/python/twitter/thermos/monitoring'),
+ ],
+ provides = setup_py(
+ name = 'twitter.thermos.observer',
+ version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
+ description = 'The Thermos observer web interface.',
+ ).with_binaries(
+ thermos_observer = pants('src/main/python/twitter/thermos/observer/bin:thermos_observer'),
+ )
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/__init__.py b/src/main/python/apache/thermos/observer/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/bin/BUILD b/src/main/python/apache/thermos/observer/bin/BUILD
new file mode 100644
index 0000000..26eb148
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/bin/BUILD
@@ -0,0 +1,14 @@
+python_binary(
+ name = 'thermos_observer',
+ source = 'thermos_observer.py',
+ entry_point = 'twitter.thermos.observer.bin.thermos_observer:proxy_main',
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/app'),
+ pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
+ pants('aurora/twitterdeps/src/python/twitter/common/http'),
+ pants('src/main/python/twitter/thermos:cherrypy'),
+ pants('src/main/python/twitter/thermos/common:path'),
+ pants('src/main/python/twitter/thermos/observer/http:http_observer'),
+ pants('src/main/python/twitter/thermos/observer:task_observer'),
+ ],
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/bin/__init__.py b/src/main/python/apache/thermos/observer/bin/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/bin/thermos_observer.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/bin/thermos_observer.py b/src/main/python/apache/thermos/observer/bin/thermos_observer.py
new file mode 100644
index 0000000..9e7f7a0
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/bin/thermos_observer.py
@@ -0,0 +1,55 @@
+from __future__ import print_function
+
+import socket
+import sys
+import time
+
+from twitter.common import app
+from twitter.common.exceptions import ExceptionalThread
+from twitter.common.http import HttpServer
+from twitter.common.http.diagnostics import DiagnosticsEndpoints
+from twitter.thermos.common.path import TaskPath
+from twitter.thermos.observer.task_observer import TaskObserver
+from twitter.thermos.observer.http.http_observer import BottleObserver
+
+
+app.add_option("--root",
+ dest="root",
+ metavar="DIR",
+ default=TaskPath.DEFAULT_CHECKPOINT_ROOT,
+ help="root checkpoint directory for thermos task runners")
+
+
+app.add_option("--port",
+ dest="port",
+ metavar="INT",
+ default=1338,
+ help="port number to listen on.")
+
+
+def proxy_main():
+ def main(args, opts):
+ if args:
+ print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr)
+ app.help()
+ sys.exit(1)
+
+ root_server = HttpServer()
+ root_server.mount_routes(DiagnosticsEndpoints())
+
+ task_observer = TaskObserver(opts.root)
+ task_observer.start()
+
+ bottle_wrapper = BottleObserver(task_observer)
+
+ root_server.mount_routes(bottle_wrapper)
+
+ def run():
+ root_server.run('0.0.0.0', opts.port, 'cherrypy')
+
+ et = ExceptionalThread(target=run)
+ et.daemon = True
+ et.start()
+ et.join()
+
+ app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/BUILD b/src/main/python/apache/thermos/observer/http/BUILD
new file mode 100644
index 0000000..9f3d587
--- /dev/null
+++ b/src/main/python/apache/thermos/observer/http/BUILD
@@ -0,0 +1,47 @@
+python_library(
+ name = 'json',
+ sources = ['json.py'],
+ dependencies = [
+ pants('aurora/twitterdeps/src/python/twitter/common/http'),
+ ]
+)
+
+python_library(
+ name = 'static_assets',
+ sources = ['static_assets.py'],
+ resources = rglobs('assets/*'),
+ dependencies = [
+ pants('src/main/python/twitter/thermos:bottle'),
+ ]
+)
+
+python_library(
+ name = 'templating',
+ sources = ['templating.py'],
+ resources = globs('templates/*.tpl'),
+)
+
+python_library(
+ name = 'file_browser',
+ sources = ['file_browser.py'],
+ dependencies = [
+ pants(':templating'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/http'),
+ pants('src/main/python/twitter/thermos:bottle'),
+ pants('src/main/python/twitter/thermos:mako'),
+ ]
+)
+
+python_library(
+ name = 'http_observer',
+ sources = ['__init__.py', 'http_observer.py'],
+ dependencies = [
+ pants(':file_browser'),
+ pants(':json'),
+ pants(':static_assets'),
+ pants(':templating'),
+ pants('aurora/twitterdeps/src/python/twitter/common/log'),
+ pants('aurora/twitterdeps/src/python/twitter/common/http'),
+ ]
+)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/apache/thermos/observer/http/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/thermos/observer/http/__init__.py b/src/main/python/apache/thermos/observer/http/__init__.py
new file mode 100644
index 0000000..e69de29
[11/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/tools/java/thrift_wrapper_codegen.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/tools/java/thrift_wrapper_codegen.py b/src/main/python/twitter/aurora/tools/java/thrift_wrapper_codegen.py
deleted file mode 100644
index 4376e85..0000000
--- a/src/main/python/twitter/aurora/tools/java/thrift_wrapper_codegen.py
+++ /dev/null
@@ -1,475 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-
-import os
-import re
-import sys
-
-
-class Type(object):
- '''A data type.'''
-
- def __init__(self, name, package=None, immutable=False):
- self.name = name
- self.package = package
- self.immutable = immutable
-
- def absolute_name(self):
- return '%s.%s' % (self.package, self.name) if self.package else self.name
-
- def __str__(self):
- return '%s (%smutable)' % (self.absolute_name(), 'im' if self.immutable else '')
-
-
-class PrimitiveType(Type):
- '''A primitive type, with its associated typeboxed name.'''
-
- def __init__(self, name, boxed_name):
- Type.__init__(self, name, package=None, immutable=True)
- self.boxed_name = boxed_name
-
-
-class ParameterizedType(Type):
- '''A parameterized type, usually a collection.'''
-
- def __init__(self, name, params):
- Type.__init__(self, name, None)
- self.params = params
-
- def param_names(self):
- def name(t):
- if isinstance(t, StructType):
- return t.codegen_name
- elif isinstance(t, PrimitiveType):
- return t.boxed_name
- else:
- return t.name
- return ', '.join([name(p) for p in self.params])
-
-
-class StructType(Type):
- '''A thrift-defined type, which composes other types as fields.'''
-
- def __init__(self, name, package, kind, fields):
- Type.__init__(self, name, package, kind == 'enum')
- self.codegen_name = 'I%s' % name
- self.kind = kind
- self.fields = fields
-
- def __str__(self):
- return '%s %s { %s }' % (self.kind, self.name, ', '.join(map(str, self.fields)))
-
-
-class Field(object):
- '''A field within a thrift structure.'''
-
- def __init__(self, ttype, name):
- self.ttype = ttype
- self.name = name
-
- def accessor_method(self):
- return '%s%s' % (
- 'is' if self.ttype.name == 'boolean' else 'get',
- self.name[:1].capitalize() + self.name[1:])
-
- def isset_method(self):
- return 'isSet%s' % (self.name[0].upper() + self.name[1:])
-
- def __str__(self):
- return '%s: %s' % (self.name, self.ttype)
-
-
-FIELD_TEMPLATE = ''' public %(type)s %(fn_name)s() {
- return %(field)s;
- }'''
-
-
-# Template string for a method to access an immutable field.
-IMMUTABLE_FIELD_TEMPLATE = ''' public %(type)s %(fn_name)s() {
- return wrapped.%(fn_name)s();
- }'''
-
-
-STRUCT_DECLARATION = '''private final %(type)s %(field)s;'''
-STRUCT_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
- ? null
- : %(type)s.buildNoCopy(wrapped.%(fn_name)s());'''
-
-
-IMMUTABLE_COLLECTION_DECLARATION = '''private final Immutable%(collection)s<%(params)s> %(field)s;'''
-IMMUTABLE_COLLECTION_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
- ? Immutable%(collection)s.<%(params)s>of()
- : Immutable%(collection)s.copyOf(wrapped.%(fn_name)s());'''
-
-
-# Tempalte string for assignment for a collection field containing a struct.
-STRUCT_COLLECTION_FIELD_ASSIGNMENT = '''this.%(field)s = !wrapped.%(isset)s()
- ? Immutable%(collection)s.<%(params)s>of()
- : FluentIterable.from(wrapped.%(fn_name)s())
- .transform(%(params)s.FROM_BUILDER)
- .to%(collection)s();'''
-
-PACKAGE_NAME = 'com.twitter.aurora.scheduler.storage.entities'
-
-CLASS_TEMPLATE = '''/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package %(package)s;
-
-%(imports)s
-
-/**
- * An immutable wrapper class.
- * <p>
- * This code is auto-generated, and should not be directly modified.
- */
-public final class %(name)s {
- private final %(wrapped)s wrapped;
-%(fields)s
- private %(name)s(%(wrapped)s wrapped) {
- this.wrapped = Preconditions.checkNotNull(wrapped);%(assignments)s
- }
-
- static %(name)s buildNoCopy(%(wrapped)s wrapped) {
- return new %(name)s(wrapped);
- }
-
- public static %(name)s build(%(wrapped)s wrapped) {
- return buildNoCopy(wrapped.deepCopy());
- }
-
- public static final Function<%(name)s, %(wrapped)s> TO_BUILDER =
- new Function<%(name)s, %(wrapped)s>() {
- @Override
- public %(wrapped)s apply(%(name)s input) {
- return input.newBuilder();
- }
- };
-
- public static final Function<%(wrapped)s, %(name)s> FROM_BUILDER =
- new Function<%(wrapped)s, %(name)s>() {
- @Override
- public %(name)s apply(%(wrapped)s input) {
- return new %(name)s(input);
- }
- };
-
- public static ImmutableList<%(wrapped)s> toBuildersList(Iterable<%(name)s> w) {
- return FluentIterable.from(w).transform(TO_BUILDER).toList();
- }
-
- public static ImmutableList<%(name)s> listFromBuilders(Iterable<%(wrapped)s> b) {
- return FluentIterable.from(b).transform(FROM_BUILDER).toList();
- }
-
- public static ImmutableSet<%(wrapped)s> toBuildersSet(Iterable<%(name)s> w) {
- return FluentIterable.from(w).transform(TO_BUILDER).toSet();
- }
-
- public static ImmutableSet<%(name)s> setFromBuilders(Iterable<%(wrapped)s> b) {
- return FluentIterable.from(b).transform(FROM_BUILDER).toSet();
- }
-
- public %(wrapped)s newBuilder() {
- return wrapped.deepCopy();
- }
-
-%(accessors)s
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof %(name)s)) {
- return false;
- }
- %(name)s other = (%(name)s) o;
- return wrapped.equals(other.wrapped);
- }
-
- @Override
- public int hashCode() {
- return wrapped.hashCode();
- }
-
- @Override
- public String toString() {
- return wrapped.toString();
- }
-}'''
-
-
-class GeneratedCode(object):
- def __init__(self, class_name, wrapped_type):
- self._class_name = class_name
- self._wrapped_type = wrapped_type
- self._imports = set()
- self._accessors = []
- self._fields = []
- self._assignments = []
-
- def add(self, s, end='\n'):
- print('This no longer does anything.')
-
- def add_import(self, import_class):
- self._imports.add(import_class)
-
- def add_assignment(self, field, assignment):
- self._fields.append(field)
- self._assignments.append(assignment)
-
- def add_accessor(self, accessor_method):
- self._accessors.append(accessor_method)
-
- def dump(self, f):
- remaining_imports = list(self._imports)
- import_groups = []
- def remove_by_prefix(prefix):
- group = [i for i in remaining_imports if i.startswith(prefix)]
- remaining_imports[:] = [i for i in remaining_imports if not i.startswith(prefix)]
- return group
-
- def add_import_group(group):
- if group:
- import_groups.append('\n'.join(['import %s;' % i for i in sorted(group)]))
-
- twitter_imports = remove_by_prefix('com.twitter')
- add_import_group(remove_by_prefix('java'))
- add_import_group(remove_by_prefix('com'))
- add_import_group(remove_by_prefix('net'))
- add_import_group(remove_by_prefix('org'))
- add_import_group(twitter_imports)
-
- print(CLASS_TEMPLATE % {
- 'package': PACKAGE_NAME,
- 'name': self._class_name,
- 'wrapped': self._wrapped_type,
- 'imports': '\n\n'.join(import_groups),
- 'accessors': '\n\n'.join(self._accessors),
- 'fields': (' ' + '\n '.join(self._fields) + '\n') if self._fields else '',
- 'assignments': ('\n ' + '\n '.join(self._assignments)) if self._assignments else '',
- }, file=f)
-
-
-# A namespace declaration, e.g.:
-# namespace java com.twitter.aurora.gen
-NAMESPACE_RE = 'namespace\s+(?P<lang>\w+)\s+(?P<namespace>[^\s]+)'
-
-# A possibly-parameterized type name, e.g.:
-# int
-# TaskConfig
-# Set<String>
-# Map<String, TaskConfig>
-TYPE_PATTERN = '(?P<type>\w+)(?:<(?P<params>[^>]+)>)?'
-
-
-# Matches a complete struct definnition, capturing the type and body.
-STRUCT_RE = '(?P<kind>enum|struct|union)\s+(?P<name>\w+)\s+{(?P<body>[^}]+)}'
-
-
-# A field definition within a struct, e.g.:
-# 1: string name
-# 15: Map<String, TaskConfig> configs # Configs mapped by name.
-FIELD_RE = '\s*\d+:\s+(?:(?:required|optional)\s+)?(%s)\s+(?P<name>\w+).*' % TYPE_PATTERN
-
-
-def parse_structs(thrift_defs):
- '''Read all thrift structures found in a file.
-
- This returns a list of Type objects representing the structs found
- and the fields they contain.
- '''
- # Capture all namespace definitions.
- namespaces = dict(re.findall(NAMESPACE_RE, thrift_defs))
-
- def parse_field(field):
- type_name = field.group('type')
- type_params = field.group('params')
- if type_params:
- params = [Type(p) for p in type_params.replace(' ', '').split(',')]
- ttype = ParameterizedType(type_name, params)
- else:
- ttype = Type(type_name)
- return Field(ttype, field.group('name'))
-
- def parse_fields(field_str):
- return map(parse_field, re.finditer(FIELD_RE, field_str))
-
- return [StructType(s.group('name'),
- namespaces['java'],
- s.group('kind'),
- parse_fields(s.group('body')))
- for s in re.finditer(STRUCT_RE, thrift_defs, flags=re.MULTILINE)]
-
-
-def generate_java(struct):
- code = GeneratedCode(struct.codegen_name, struct.name)
- code.add_import('com.google.common.base.Preconditions')
- code.add_import('com.google.common.base.Function')
- code.add_import('com.google.common.collect.ImmutableList')
- code.add_import('com.google.common.collect.ImmutableSet')
- code.add_import('com.google.common.collect.FluentIterable')
- code.add_import(struct.absolute_name())
-
- if struct.kind == 'union':
- code.add_accessor(IMMUTABLE_FIELD_TEMPLATE
- % {'type': '%s._Fields' % struct.name, 'fn_name': 'getSetField'})
-
- # Accessor for each field.
- for field in struct.fields:
- if not (isinstance(field.ttype, StructType) and (field.ttype.kind == 'enum' or struct.kind == 'union')):
- code.add_accessor(IMMUTABLE_FIELD_TEMPLATE
- % {'type': 'boolean',
- 'fn_name': field.isset_method()})
- if field.ttype.immutable:
- code.add_accessor(IMMUTABLE_FIELD_TEMPLATE % {'type': field.ttype.name,
- 'fn_name': field.accessor_method()})
- elif not struct.kind == 'union':
- if isinstance(field.ttype, StructType):
- return_type = field.ttype.codegen_name
- elif isinstance(field.ttype, ParameterizedType):
- return_type = '%s<%s>' % (field.ttype.name, field.ttype.param_names())
- else:
- return_type = field.ttype.name
- code.add_accessor(FIELD_TEMPLATE % {'type': return_type,
- 'fn_name': field.accessor_method(),
- 'field': field.name})
-
- if isinstance(field.ttype, StructType):
- if field.ttype.kind == 'enum':
- code.add_import(field.ttype.absolute_name())
-
- if field.ttype.immutable:
- # Direct accessor was already added.
- pass
- elif struct.kind == 'union':
- copy_field = '%s.build(wrapped.%s())' % (field.ttype.codegen_name,
- field.accessor_method())
- code.add_accessor(FIELD_TEMPLATE % {'type': field.ttype.codegen_name,
- 'fn_name': field.accessor_method(),
- 'field': copy_field})
- else:
- args = {
- 'field': field.name,
- 'fn_name': field.accessor_method(),
- 'isset': field.isset_method(),
- 'type': field.ttype.codegen_name,
- }
- code.add_assignment(STRUCT_DECLARATION % args, STRUCT_ASSIGNMENT % args)
- elif isinstance(field.ttype, ParameterizedType):
- # Add necessary imports, supporting only List, Map, Set.
- assert field.ttype.name in ['List', 'Map', 'Set'], 'Unrecognized type %s' % field.ttype.name
- code.add_import('com.google.common.collect.Immutable%s' % field.ttype.name)
- code.add_import('java.util.%s' % field.ttype.name)
-
- params = field.ttype.params
- if all([p.immutable for p in params]):
- # All parameter types are immutable.
- assignment = IMMUTABLE_COLLECTION_ASSIGNMENT
- elif len(params) == 1:
- # Only one non-immutable parameter.
- # Assumes the parameter type is a struct and our code generator
- # will make a compatible wrapper class and constructor.
- assignment = STRUCT_COLLECTION_FIELD_ASSIGNMENT
- else:
- assert False, 'Unable to codegen accessor field for %s' % field.name
- args = {'collection': field.ttype.name,
- 'field': field.name,
- 'fn_name': field.accessor_method(),
- 'isset': field.isset_method(),
- 'params': field.ttype.param_names()}
- code.add_assignment(IMMUTABLE_COLLECTION_DECLARATION % args, assignment % args)
- elif not field.ttype.immutable:
- assert False, 'Making type %s immutable is not supported.' % field.ttype.name
- return code
-
-
-THRIFT_ALIASES = {
- 'bool': 'boolean',
- 'i32': 'int',
- 'i64': 'long',
- 'double': 'double',
- 'string': 'String',
- 'list': 'List',
- 'set': 'Set',
- 'map': 'Map',
- 'binary': 'byte[]',
-}
-
-
-def main(args):
- if len(args) != 4:
- print('usage: %s thrift_file struct_name output_directory' % sys.argv[0])
- sys.exit(1)
-
- thrift_file, struct_name, output_directory = sys.argv[1:]
- print('Searching for %s in %s' % (sys.argv[2], sys.argv[1]))
- with open(sys.argv[1]) as f:
- # Load all structs found in the thrift file.
- structs = parse_structs(f.read())
-
- # The symbol table stores information about types we recognize.
- # As new symbols are parsed, they are accumulated here.
- # This is also seeded with JDK types.
- # Note: byte[] is not immutable, but we'd rather accept it than copy.
- primitives = dict((t, PrimitiveType(t, b)) for (t, b) in [('boolean', 'Boolean'),
- ('int', 'Integer'),
- ('long', 'Long'),
- ('double', 'Double')])
- lang_symbols = dict((t, Type(t, 'java.lang', immutable=True)) for t in ['String', 'byte[]'])
- util_symbols = dict((t, Type(t, 'java.util')) for t in ['List', 'Map', 'Set'])
- symbol_table = dict(primitives.items() + lang_symbols.items() + util_symbols.items())
-
- def load_dependencies(struct):
- # Fill in type information for fields by searching for dependencies.
- for field in struct.fields:
- if isinstance(field.ttype, ParameterizedType):
- field.ttype.name = find_symbol(field.ttype.name).name
- field.ttype.params = [find_symbol(p.name) for p in field.ttype.params]
- else:
- field.ttype = find_symbol(field.ttype.name)
-
- def find_symbol(name):
- name = THRIFT_ALIASES.get(name, name)
- if name in symbol_table:
- return symbol_table[name]
-
- symbol = next((s for s in structs if s.name == name), None)
- assert symbol, 'Failed to find required struct %s' % name
- load_dependencies(symbol)
- symbol_table[name] = symbol
- return symbol
-
- find_symbol(sys.argv[2])
- print('Symbol table:')
- for _, symbol in symbol_table.items():
- print(' %s' % symbol)
-
- for _, symbol in symbol_table.items():
- if isinstance(symbol, StructType):
- if symbol.kind == 'enum':
- print('Skipping code generation for %s, since it is immutable' % symbol.name)
- else:
- package_dir = os.path.join(sys.argv[3], PACKAGE_NAME.replace('.', os.path.sep))
- if not os.path.isdir(package_dir):
- os.makedirs(package_dir)
- gen_file = os.path.join(package_dir, '%s.java' % symbol.codegen_name)
- print('Generating %s' % gen_file)
- with open(gen_file, 'w') as f:
- code = generate_java(symbol)
- code.dump(f)
-
-
-if __name__ == '__main__':
- main(sys.argv)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/BUILD b/src/main/python/twitter/thermos/BUILD
deleted file mode 100644
index 937bcf3..0000000
--- a/src/main/python/twitter/thermos/BUILD
+++ /dev/null
@@ -1,18 +0,0 @@
-import os
-
-page(name = 'readme', source = 'README.md')
-
-python_library(
- name = 'thermos',
- dependencies = [
- pants('src/main/python/twitter/thermos/core'),
- pants('src/main/python/twitter/thermos/monitoring'),
- ],
- provides = setup_py(
- name = 'twitter.thermos',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- descripton = 'The Twitter Thermos runtime.',
- ).with_binaries(
- thermos = pants('src/main/python/twitter/thermos/bin:thermos'),
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/BUILD.thirdparty
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/BUILD.thirdparty b/src/main/python/twitter/thermos/BUILD.thirdparty
deleted file mode 100644
index 5a54f4a..0000000
--- a/src/main/python/twitter/thermos/BUILD.thirdparty
+++ /dev/null
@@ -1,34 +0,0 @@
-python_requirement(
- name = 'psutil',
- requirement = 'psutil==1.1.2',
-)
-
-python_requirement(
- name = 'pystachio',
- requirement = 'pystachio==0.7.2',
-)
-
-python_requirement(
- name = 'mako',
- requirement = 'mako==0.4.0',
-)
-
-python_requirement(
- name = 'cherrypy',
- requirement = 'cherrypy==3.2.2',
-)
-
-python_requirement(
- name = 'bottle',
- requirement = 'bottle==0.11.6',
-)
-
-python_requirement(
- name = 'thrift',
- requirement = 'thrift==0.9.1',
-)
-
-python_requirement(
- name = 'mock',
- requirement = 'mock==1.0.1',
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/README.md
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/README.md b/src/main/python/twitter/thermos/README.md
deleted file mode 100644
index e456e93..0000000
--- a/src/main/python/twitter/thermos/README.md
+++ /dev/null
@@ -1,461 +0,0 @@
-** This document is deprecated and no longer updated. **
-
-# Thermos manual #
-
-[TOC]
-
-
-## tl;dr ##
-
-** You noticed the deprecation notice, right? **
-
-### What is Thermos? ###
-
-Thermos a simple process management framework used for orchestrating
-dependent processes within a single chroot. At Twitter, it is used as a
-process manager for Mesos tasks. In practice, there is a one-to-one
-correspondence between a Mesos task and Thermos task. This document
-describes how to use Thermos in a local development environment and does not
-describe how to run Thermos tasks on Mesos, though once you have a valid
-Thermos configuration doing so is a small step.
-
-### Simplest invocation ###
-
-Thermos lives in `science` at Twitter and publically on GitHub in `twitter-commons` (TBD).
-
-#### build ####
-
-Build the Thermos CLI and the Thermos observer.
-
-```shell
-$ ./pants src/python/twitter/thermos
-$ ./pants src/python/twitter/thermos/bin:thermos_observe
-```
-
-You can copy `dist/thermos.pex` to `thermos` somewhere in your `$PATH` or use a la carte.
-
-#### simplerun ####
-
-You can run Thermos tasks without first writing a configuration file using `simplerun`:
-
-```shell
-$ thermos simplerun 'echo hello world'
-Running command: 'echo hello world'
- INFO] Forking Process(simple)
- INFO] Process(simple) finished successfully [rc=0]
- INFO] Killing task id: simple-20120529-162532.018646
- INFO] => Current user: wickman
- INFO] Kill complete.
- INFO] Task succeeded.
-```
-
-#### introspection ####
-
-```shell
-$ thermos status --verbosity=3 simple-20120529-162532.018646
-Found task simple-20120529-162532.018646
- simple-20120529-162532.018646 [owner: wickman] state: SUCCESS start: Tue May 29 16:25:32 2012
- user: wickman ports: None
- sandbox: None
- process table:
- - simple runs: 1 last: pid=57471, rc=0, finish:Tue May 29 16:25:32 2012, state:SUCCESS
-
-$ thermos tail simple-20120529-162532.018646
-Tail of terminal log /Users/wickman/.thermos/logs/simple-20120529-162532.018646/simple/0/stdout
-hello world
-```
-
-#### thermos observer ####
-
-```shell
-$ dist/thermos_observe.pex --root=$HOME/.thermos
-```
-
-This will fire up a webserver on `localhost:1338` that you can use as a web interface to interact with
-locally running Thermos tasks.
-
-
-## Building blocks ##
-
-Thermos is made of Processes and Tasks.
-
-### Thermos processes ###
-
-A Thermos process is simply a command-line that is invoked in a subshell. A single
-run of a process may succeed (have a zero exit status) or fail (have a
-non-zero exit status.) A process is considered permanently failed after a
-maximum number of individual run failures, by default one.
-
-### Thermos tasks ###
-
-A Thermos task is a collection of processes and constraints that dictate how
-and when to run them. By default there are no constraints between processes
-bound to a task and they run in parallel. The simplest (and currently only)
-Task level constraint is the `order` dependency: process B should not be run
-until process A has completed successfully. For example, process A could be
-`git clone` and process B could be `rake test`. It doesn't make sense to
-run process B until process A is fully completed and successful.
-
-
-### Thermos configuration ###
-
-The easiest way to invoke Thermos is using `thermos simplerun`. Under the
-covers, this synthesizes a Thermos task with a single Thermos process named
-`simple`. To do the same thing via Thermos configuration, add
-the following to `simple.thermos`:
-
-```python
-process = Process(name = 'simple', cmdline = 'echo hello world')
-task = Task(name = 'simple', processes = [process])
-export(task)
-```
-
-Then invoke by `thermos run simple.thermos`.
-
-Configuring Thermos is done through
-[pystachio](http://github.com/wickman/pystachio) templates. These templates
-behave like structurally typed collections of key-value pairs. You can
-either construct configuration through plain old Python objects (as above)
-or as Python dictionaries or JSON that is coerced into Python dictionaries.
-
-For example, the above configuration is equivalent to:
-
-```python
-export({
- 'name': 'simple',
- 'processes': [{
- 'name': 'simple',
- 'cmdline': 'echo hello world'
- }]
-})
-```
-
-The full Thermos pystachio schema can be found at
-`src/python/twitter/thermos/config/schema.py` and is mostly described below.
-
-
-## Configuration reference ##
-
-### Process objects ###
-
-Processes fundamentally consist of a `name`, `cmdline`. You will rarely
-need to specify anything more.
-
-<table>
- <tr> <td colspan=2> <b>Process schema</b> </td> </tr>
- <tr> <td> <em>name</em> (required) </td> <td> Process name (String) </td> </tr>
- <tr> <td> <em>cmdline</em> (required) </td> <td> Command line (String) </td> </tr>
- <tr> <td> <em>max_failures</em> </td> <td> Max failures (Integer, default 1) </td> </tr>
- <tr> <td> <em>daemon</em> </td> <td> Daemon process? (Boolean, default False) </td> </tr>
- <tr> <td> <em>ephemeral</em> </td> <td> Ephemeral process? (Boolean, default False) </td> </tr>
- <tr> <td> <em>min_duration</em> </td> <td> Min duration between runs in seconds (Integer, default 15) </td> </tr>
- <tr> <td> <em>final</em> </td> <td> This is a finalizing process that should run last (Boolean, default False) </td> </tr>
-</table>
-
-
-#### name ####
-
-The name is any string that is a valid UNIX filename (specifically no
-slashes, NULLs or leading periods.) Each process name within a task must be
-unique.
-
-#### cmdline ####
-
-The command line is invoked in a bash subshell, so can be full-blown bash
-scripts, though no command-line arguments are supplied.
-
-#### max_failures ####
-
-The maximum number of failures (non-zero exit statuses) this process can
-sustain before being marked permanently failed and not retried. If a
-process becomes permanently failed, Thermos looks to the task failure limit
-(usually 1) to determine whether or not the Thermos task should be failed.
-
-Setting max_failures to 0 means that this process will be retried
-indefinitely until a successful (zero) exit status is achieved. It will be retried
-at most once every `min_duration` seconds in order to prevent DoSing the
-coordinating thermos scheduler.
-
-#### daemon ####
-
-By default Thermos processes are non-daemon. If `daemon` is set to True, a
-successful (zero) exit status will not prevent future process runs.
-Instead, the process will be reinvoked after min_duration seconds. However,
-the maximum failure limit will still apply. A combination of `daemon=True`
-and `max_failures=0` will cause a process to be retried indefinitely
-regardless of exit status. This should generally be avoided for very
-short-lived processes because of the accumulation of checkpointed state for
-each process run.
-
-#### ephemeral ####
-
-By default Thermos processes are non-ephemeral. If `ephemeral` is set to
-True, the status of the process is not used to determine whether the task in
-which it is bound has completed. Take for example a Thermos task with a
-non-ephemeral webserver process and an ephemeral logsaver process that
-periodically checkpoints its log files to a centralized data store. The
-task is considered finished once the webserver process has completed,
-regardless of the current status of the logsaver.
-
-#### min_duration ####
-
-Processes may succeed or fail multiple times throughout the duration of a
-single task. Each of these is called a "process run." The `min_duration` is the minimum
-number of seconds the scheduler waits between running the same process.
-
-#### final ####
-
-Processes can be grouped into two classes: ordinary processes and finalizing
-processes. By default, Thermos processes are ordinary. They run as long as
-the Thermos Task is considered healthy (i.e., no failure limits have been
-reached.) But once all regular Thermos processes have either finished or the
-Task has reached a certain failure threshold, it moves into a "finalization"
-stage and then runs all finalizing processes. These are typically processes
-necessary for cleaning up the task, such as log checkpointers, or perhaps
-e-mail notifications that the task has completed.
-
-Finalizing processes may not depend upon ordinary processes or vice-versa, however
-finalizing processes may depend upon other finalizing processes and will otherwise run as
-a typical process schedule.
-
-
-### Task objects ###
-
-Tasks fundamentally consist of a `name` and a list of processes `processes`.
-Processes can be further constrained with `constraints`
-
-<table>
- <tr> <td colspan=2> <b>Task schema</b> </td> </tr>
- <tr> <td> <em>name</em> (required) </td> <td> Task name (String) </td> </tr>
- <tr> <td> <em>processes</em> (required) </td> <td> List of processes (List of Process objects) </td> </tr>
- <tr> <td> <em>constraints</em> </td> <td> Constraints (List of Constraint objects, default []) </td> </tr>
- <tr> <td> <em>resources</em> </td> <td> Resource footprint (Resource, optional) </td> </tr>
- <tr> <td> <em>max_failures</em> </td> <td> Max failures (Integer, default 1) </td> </tr>
- <tr> <td> <em>max_concurrency</em> </td> <td> Max concurrency (Integer, default 0 = unlimited concurrency) </td> </tr>
- <tr> <td> <em>finalization_wait</em> </td> <td> Amount of time allocated to run finalizing processes (Integer in seconds, default 30) </td> </tr>
-</table>
-
-<table>
- <tr> <td colspan=2> <b>Constraint schema</b> </td> </tr>
- <tr> <td> <em>order</em> </td> <td> List of process names that should run in order (List of Strings)</td> </tr>
-</table>
-
-<table>
- <tr> <td colspan=2> <b>Resource schema</b> </td> </tr>
- <tr> <td> <em>cpu</em> (required) </td> <td> Number of cores (Float) </td> </tr>
- <tr> <td> <em>ram</em> (required) </td> <td> Bytes of RAM (Integer) </td> </tr>
- <tr> <td> <em>disk</em> (required) </td> <td> Bytes of disk (Integer) </td> </tr>
-</table>
-
-#### name ####
-
-The name is used to label the task and is used for reporting in the observer UI and for
-management in the thermos CLI.
-
-#### processes ####
-
-Processes is an unordered list of `Process` objects. In order to place temporal constraints upon
-them, you must use `constraints`.
-
-#### constraints ####
-
-A list of `Constraint` objects. Currently only one type of constraint is supported, the `order` constraint.
-`order` is a list of process names that should run in order. For example,
-
-```python
-process = Process(cmdline = "echo hello world")
-task = Task(name = "echoes", processes = [process(name = "first"), process(name = "second")],
- constraints = [Constraint(order = ["first", "second"]))
-```
-
-Constraints can be supplied ad-hoc and in duplicate and not all processes need be constrained, however
-tasks with cycles will be rejected by the Thermos scheduler.
-
-#### resources ####
-
-`resources` is a `Resource` object described by `cpu`, `ram`, and `disk`. It is currently unused by
-Thermos but reserved for future use in constraining the resource consumption of a task.
-
-#### max_failures ####
-
-`max_failures` is the number of failed processes in order for this task to be marked as failed. A single
-process run does not consistute a failure. For example:
-
-```python
-template = Process(max_failures=10)
-task = Task(name = "fail", processes = [template(name = "failing", cmdline = "exit 1"),
- template(name = "succeeding", cmdline = "exit 0")],
- max_failures=2)
-```
-
-The `failing` process would fail 10 times before being marked as permanently
-failed, and the `succeeding` process would succeed on the first run. The
-task would succeed despite only allowing for two failed processes. To be
-more specific, there would be 10 failed process runs yet 1 failed process.
-
-#### max_concurrency ####
-
-For tasks with a number of expensive but otherwise independent processes, it
-may be desirable to limit the amount of concurrency provided by the Thermos
-scheduler rather than artificially constraining them through `order`
-constraints. For example, a test framework may generate a task with 100
-test run processes, but would like to run it on a machine with only 4 cores.
-You can limit the amount of parallelism to 4 by setting `max_concurrency=4`
-in your task configuration.
-
-For example, the following Thermos task spawns 180 processes ("mappers") to compute
-individual elements of a 180 degree sine table, all dependent upon one final process ("reducer")
-to tabulate the results:
-
-```python
-def make_mapper(id):
- return Process(
- name = "mapper%03d" % id,
- cmdline = "echo 'scale=50;s(%d*4*a(1)/180)' | bc -l > temp.sine_table.%03d" % (id, id))
-
-def make_reducer():
- return Process(name = "reducer",
- cmdline = "cat temp.* | nl > sine_table.txt && rm -f temp.*")
-
-processes = map(make_mapper, range(180))
-
-task = Task(
- name = "mapreduce",
- processes = processes + [make_reducer()],
- constraints = [Constraint(order = [mapper.name(), 'reducer']) for mapper in processes],
- max_concurrency = 8)
-
-export(task)
-```
-
-#### finalization_wait ####
-
-Tasks have three active stages: ACTIVE, CLEANING and FINALIZING. The ACTIVE stage is when
-ordinary processes run. This stage will last as long as processes are running and the
-task is healthy. The moment either all processes have finished successfully or the task
-has reached a maximum process failure limit, it will go into CLEANING stage and send SIGTERMs
-to all currently running processes and their process trees. Once all processes have
-terminated, the task goes into FINALIZING stage and invokes the schedule of all processes
-with the "final" bit set.
-
-This whole process from the end of ACTIVE stage to the end of FINALIZING must take place within
-"finalization_wait" seconds. If it does not complete within that time, all remaining
-processes will be sent SIGKILLs (or if they depend upon processes that have not yet completed,
-will never be invoked.)
-
-Client applications with higher priority may be able to force a shorter
-finalization wait (e.g. through parameters to `thermos kill`), so this is
-mostly a best-effort signal.
-
-
-## REPL ##
-
-You can interactively experiment with the Thermos configuration REPL via the
-`src/python/twitter/thermos/config:repl` target:
-
-```python
-$ ./pants py src/python/twitter/thermos/config:repl
-Build operating on target: PythonBinary(src/python/twitter/thermos/config/BUILD:repl)
-Thermos Config REPL
->>> boilerplate = Process(cmdline = "echo hello world")
->>> boilerplate
-Process(cmdline=echo hello world, daemon=0, max_failures=1, ephemeral=0, min_duration=5)
->>> boilerplate.check()
-TypeCheck(FAILED): Process[name] is required.
->>> boilerplate(name = "hello world").check()
-TypeCheck(OK)
-```
-
-## Thermos templating ##
-
-The Thermos DSL is implemented in [pystachio](http://github.com/wickman/pystachio) which means that
-a simple Mustache-like templating layer is available for use when configuring tasks.
-
-### Ordinary templates ###
-
-By using Mustache style templates in your job, you can do allow some amount of runtime configuration
-of your tasks:
-
-```
->>> process = Process(name = "hello", cmdline = "echo hello {{first}}")
->>> process
-Process(cmdline=echo hello {{first}}, daemon=0, name=hello, max_failures=1, ephemeral=0, min_duration=5)
->>> process.check()
-TypeCheck(FAILED): Process[cmdline] failed: Uninterpolated variables: {{first}}
-```
-
-This process leaves `{{first}}` as a free variable. It can be filled elsewhere in the configuration, e.g. via
-`%` or `bind`:
-```
->>> process % {'first': 'brian'}
-Process(cmdline=echo hello brian, daemon=0, name=hello, max_failures=1, ephemeral=0, min_duration=5)
->>> process.bind(first = 'brian')
-Process(cmdline=echo hello brian, daemon=0, name=hello, max_failures=1, ephemeral=0, min_duration=5)
-```
-
-If this is left unbound, the thermos CLI will complain:
-
-```
-$ thermos run thermos/examples/tutorial/unbound.thermos
-twitter.thermos.config.loader.InvalidTask: Task[processes] failed: Element in ProcessList failed check: Process[cmdline] failed: Uninterpolated variables: {{first}}
-```
-
-But free variables can be populated at runtime using the `-E` parameter:
-
-```
-$ thermos run -E first=brian thermos/examples/tutorial/unbound.thermos
-Writing log files to disk in /var/tmp
- INFO] Forking Process(hello)
- INFO] Process(hello) finished successfully [rc=0]
- INFO] Killing task id: unbound-20120530-124903.934384
- INFO] => Current user: wickman
- INFO] Kill complete.
- INFO] Task succeeded.
-```
-
-### Special templates ###
-
-Each Thermos task when run has a special `ThermosContext` template bound to the `thermos` variable.
-Currently this provides three things: `thermos.task_id`, `thermos.user` and `thermos.ports`. The
-`task_id` is the id generated (or supplied at runtime, in the case of the thermos CLI) for the task
-and the `thermos.user` is the real user the task runs as. `thermos.ports` is a mapping of named ports
-supplied by the user and exposed through the user interface. For example, to run (and background)
-the observer on port 1338:
-
-```
-$ thermos simplerun --daemon -P http:1338 'dist/thermos_observe.pex --http_port={{thermos.ports[http]}} --root=$HOME/.thermos'
-```
-
-If you go to http://localhost:1338, this bound port `http` will be exposed
-via the UI on both the main and task pages.
-
-To kill the background daemon:
-
-```
-$ thermos kill simple.*
-```
-
-### Includes ###
-
-It is possible to include other Thermos configurations via the `include` parameter. For example,
-`thermos/examples/tutorial/lib/math.thermos`:
-
-```python
-bc = Process(cmdline = "echo 'scale={{precision}};{{command}}' | bc -l")
-pi = bc(name = "pi").bind(command = "4*a(1)")
-e = bc(name = "e").bind(command = "e(1)")
-```
-
-and `thermos/examples/tutorial/pi.thermos`:
-
-```python
-include('lib/math.thermos')
-
-export(Task(name = "compute_pi", processes = [pi]))
-```
-
-can then be executed with the free `precision` variable:
-
-```shell
-$ thermos run -E precision=500 thermos/examples/tutorial/pi.thermos
-```
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/__init__.py b/src/main/python/twitter/thermos/__init__.py
deleted file mode 100644
index de40ea7..0000000
--- a/src/main/python/twitter/thermos/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__import__('pkg_resources').declare_namespace(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/bin/BUILD b/src/main/python/twitter/thermos/bin/BUILD
deleted file mode 100644
index 5aa41a2..0000000
--- a/src/main/python/twitter/thermos/bin/BUILD
+++ /dev/null
@@ -1,24 +0,0 @@
-python_binary(
- name = 'thermos_ckpt',
- source = 'thermos_ckpt.py',
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/python/twitter/thermos/common'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift')
- ]
-)
-
-python_binary(
- name = 'thermos',
- source = 'thermos.py',
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/thermos/common'),
- pants('src/main/python/twitter/thermos/config:schema'),
- pants('src/main/python/twitter/thermos/core'),
- pants('src/main/python/twitter/thermos/monitoring'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ],
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/bin/__init__.py b/src/main/python/twitter/thermos/bin/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/bin/thermos.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/bin/thermos.py b/src/main/python/twitter/thermos/bin/thermos.py
deleted file mode 100644
index 5e695b4..0000000
--- a/src/main/python/twitter/thermos/bin/thermos.py
+++ /dev/null
@@ -1,653 +0,0 @@
-# TODO(wickman) This needs some usage/help refactoring.
-
-from __future__ import print_function
-
-from collections import namedtuple
-import getpass
-import os
-import pprint
-import pwd
-import re
-import sys
-import time
-
-from twitter.common import app, log
-from twitter.common.log.options import LogOptions
-from twitter.common.dirutil import du, tail_f
-from twitter.common.dirutil.tail import tail as tail_closed
-from twitter.common.quantity import Amount, Time, Data
-from twitter.common.quantity.parse_simple import parse_time, parse_data
-from twitter.common.recordio import RecordIO, ThriftRecordReader
-from twitter.thermos.common.path import TkPath
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-from twitter.thermos.common.options import add_port_to, add_binding_to
-from twitter.thermos.config.loader import ThermosConfigLoader, ThermosTaskWrapper
-from twitter.thermos.config.schema import (
- Process,
- Resources,
- Task)
-from twitter.thermos.core.helper import TaskRunnerHelper
-from twitter.thermos.core.runner import TaskRunner
-from twitter.thermos.monitoring.detector import TaskDetector
-from twitter.thermos.monitoring.garbage import TaskGarbageCollector, DefaultCollector
-from twitter.thermos.monitoring.monitor import TaskMonitor
-
-from gen.twitter.thermos.ttypes import (
- ProcessState,
- RunnerCkpt,
- RunnerState,
- TaskState)
-
-from pystachio import Ref
-from pystachio.naming import frozendict
-
-app.add_option("--root", dest="root", metavar="PATH",
- default=TaskPath.DEFAULT_CHECKPOINT_ROOT,
- help="the thermos config root")
-
-
-def set_keep(option, opt_str, value, parser):
- setattr(parser.values, option.dest, opt_str.startswith('--keep'))
-
-
-def get_task_from_options(args, opts, **kw):
- loader = ThermosConfigLoader.load_json if opts.json else ThermosConfigLoader.load
-
- if len(args) != 1:
- app.error('Should specify precisely one config, instead got: %s' % args)
-
- tasks = loader(args[0], bindings=opts.bindings, **kw)
-
- task_list = list(tasks.tasks())
- if len(task_list) == 0:
- app.error("No tasks specified!")
-
- if opts.task is None and len(task_list) > 1:
- app.error("Multiple tasks in config but no task name specified!")
-
- task = None
- if opts.task is not None:
- for t in task_list:
- if t.task().name().get() == opts.task:
- task = t
- break
- if task is None:
- app.error("Could not find task %s!" % opts.task)
- else:
- task = task_list[0]
-
- if kw.get('strict', False):
- if not task.task.check().ok():
- app.error(task.task.check().message())
-
- return task
-
-
-def daemonize():
- def daemon_fork():
- try:
- if os.fork() > 0:
- os._exit(0)
- except OSError as e:
- sys.stderr.write('[pid:%s] Failed to fork: %s\n' % (os.getpid(), e))
- sys.exit(1)
- daemon_fork()
- os.setsid()
- daemon_fork()
- sys.stdin, sys.stdout, sys.stderr = (open('/dev/null', 'r'),
- open('/dev/null', 'a+'),
- open('/dev/null', 'a+', 0))
-
-
-def tasks_from_re(expressions, root, state=None):
- task_ids = [t_id for _, t_id in TaskDetector(root=root).get_task_ids(state=state)]
- matched_tasks = set()
- for task_expr in map(re.compile, expressions):
- for task_id in task_ids:
- if task_expr.match(task_id):
- matched_tasks.add(task_id)
- return matched_tasks
-
-
-def _really_run(task, root, sandbox, task_id=None, user=None, prebound_ports=None, chroot=None,
- daemon=False):
- prebound_ports = prebound_ports or {}
- missing_ports = set(task.ports()) - set(prebound_ports.keys())
- if missing_ports:
- app.error('ERROR! Unbound ports: %s' % ' '.join(port for port in missing_ports))
- task_runner = TaskRunner(task.task, root, sandbox, task_id=task_id,
- user=user, portmap=prebound_ports, chroot=chroot)
- if daemon:
- print('Daemonizing and starting runner.')
- try:
- log.teardown_stderr_logging()
- daemonize()
- except Exception as e:
- print("Failed to daemonize: %s" % e)
- sys.exit(1)
- try:
- task_runner.run()
- except KeyboardInterrupt:
- print('Got keyboard interrupt, killing job!')
- task_runner.close_ckpt()
- task_runner.kill()
-
-
-@app.command
-@app.command_option("--user", metavar="USER", default=getpass.getuser(), dest='user',
- help="run as this user. if not $USER, must have setuid privilege.")
-@app.command_option("--enable_chroot", dest="chroot", default=False, action='store_true',
- help="chroot tasks to the sandbox before executing them, requires "
- "root privileges.")
-@app.command_option("--task", metavar="TASKNAME", default=None, dest='task',
- help="The thermos task within the config that should be run. Only required if "
- "there are multiple tasks exported from the thermos configuration.")
-@app.command_option("--task_id", metavar="STRING", default=None, dest='task_id',
- help="The id to which this task should be bound, synthesized from the task "
- "name if none provided.")
-@app.command_option("--json", default=False, action='store_true', dest='json',
- help="Read the source file in json format.")
-@app.command_option("--sandbox", metavar="PATH", default="/var/lib/thermos/sandbox", dest='sandbox',
- help="The sandbox in which to run the task.")
-@app.command_option("-P", "--port", type="string", nargs=1, action="callback",
- callback=add_port_to('prebound_ports'), dest="prebound_ports", default=[],
- metavar="NAME:PORT", help="bind named PORT to NAME.")
-@app.command_option("-E", "--environment", type="string", nargs=1, action="callback",
- callback=add_binding_to('bindings'), default=[], dest="bindings",
- metavar="NAME=VALUE",
- help="bind the configuration environment variable NAME to VALUE.")
-@app.command_option("--daemon", default=False, action='store_true', dest='daemon',
- help="fork and daemonize the thermos runner.")
-def run(args, options):
-
- """Run a thermos task.
-
- Usage: thermos run [options] config
- Options:
- --user=USER run as this user. if not $USER, must have setuid privilege.
- --enable_chroot chroot into the sandbox for this task, requires superuser
- privilege
- --task=TASKNAME the thermos task within the config that should be run. only
- required if there are multiple tasks exported from the thermos
- configuration.
- --task_id=STRING the id to which this task should be bound, synthesized from the
- task name if none provided.
- --json specify that the config is in json format instead of pystachio
- --sandbox=PATH the sandbox in which to run the task
- [default: /var/lib/thermos/sandbox]
- -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
- multiple times to bind multiple names.)
- -E/--environment=NAME=VALUE bind the configuration environment variable NAME to
- VALUE.
- --daemon Fork and daemonize the task.
- """
- thermos_task = get_task_from_options(args, options)
- _really_run(thermos_task,
- options.root,
- options.sandbox,
- task_id=options.task_id,
- user=options.user,
- prebound_ports=options.prebound_ports,
- chroot=options.chroot,
- daemon=options.daemon)
-
-
-def inspect_unwrap(obj):
- if isinstance(obj, frozendict):
- return dict((key, inspect_unwrap(val)) for (key, val) in obj.items())
- if isinstance(obj, (list, tuple, set)):
- return tuple(inspect_unwrap(val) for val in obj)
- return obj
-
-
-@app.command
-@app.command_option("--task", metavar="TASKNAME", default=None, dest='task',
- help="The thermos task within the config that should be inspected. Only "
- "required if there are multiple tasks exported from the thermos "
- "configuration.")
-@app.command_option("--json", default=False, action='store_true', dest='json',
- help="Read the source file in json format instead of pystachio.")
-@app.command_option("-P", "--port", type="string", nargs=1, action="callback",
- callback=add_port_to('prebound_ports'), dest="prebound_ports", default=[],
- metavar="NAME:PORT", help="bind named PORT to NAME.")
-@app.command_option("-E", "--environment", type="string", nargs=1, action="callback",
- callback=add_binding_to('bindings'), default=[], dest="bindings",
- metavar="NAME=VALUE",
- help="bind the configuration environment variable NAME to VALUE.")
-def inspect(args, options):
- """Inspect a thermos config and display the evaluated task
-
- Usage: thermos inspect [options] config
- Options:
- --task=TASKNAME the thermos task within the config that should be inspected. Only
- required if there are multiple tasks exported from the thermos
- configuration.
- --json specify that the config is in json format instead of pystachio
- -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
- multiple times to bind multiple names.)
- -E/--environment=NAME=VALUE bind the configuration environment variable NAME to
- VALUE.
- """
- thermos_task = get_task_from_options(args, options)
- ti, _ = thermos_task.task().interpolate()
- pprint.pprint(inspect_unwrap(ti.get()), indent=4)
-
-
-@app.command
-@app.command_option("--user", metavar="USER", default=getpass.getuser(), dest='user',
- help="run as this user. if not $USER, must have setuid privilege.")
-@app.command_option("--name", metavar="STRING", default='simple', dest='name',
- help="The name to give this task.")
-@app.command_option("--task_id", metavar="STRING", default=None, dest='task_id',
- help="The id to which this task should be bound, synthesized from the task "
- "name if none provided.")
-@app.command_option("-P", "--port", type="string", nargs=1, action="callback",
- callback=add_port_to('prebound_ports'), dest="prebound_ports", default=[],
- metavar="NAME:PORT", help="bind named PORT to NAME.")
-@app.command_option("-E", "--environment", type="string", nargs=1, action="callback",
- callback=add_binding_to('bindings'), default=[], dest="bindings",
- metavar="NAME=VALUE",
- help="bind the configuration environment variable NAME to VALUE.")
-@app.command_option("--daemon", default=False, action='store_true', dest='daemon',
- help="fork and daemonize the thermos runner.")
-def simplerun(args, options):
- """Run a simple command line as a thermos task.
-
- Usage: thermos simplerun [options] [--] commandline
- Options:
- --user=USER run as this user. if not $USER, must have setuid privilege.
- --name=STRING the name to give this task. ('simple' by default)
- --task_id=STRING the id to which this task should be bound, synthesized from the
- task name if none provided.
- -P/--port=NAME:PORT bind the named port NAME to port number PORT (may be specified
- multiple times to bind multiple names.)
- -E/--environment=NAME=VALUE bind the configuration environment variable NAME to
- VALUE.
- --daemon Fork and daemonize the task.
- """
- try:
- cutoff = args.index('--')
- cmdline = ' '.join(args[cutoff+1:])
- except ValueError:
- cmdline = ' '.join(args)
-
- print("Running command: '%s'" % cmdline)
-
- thermos_task = ThermosTaskWrapper(Task(
- name = options.name,
- resources = Resources(cpu = 1.0, ram = 256 * 1024 * 1024, disk = 0),
- processes = [Process(name = options.name, cmdline = cmdline)]))
-
- _really_run(thermos_task,
- options.root,
- None,
- task_id=options.task_id,
- user=options.user,
- prebound_ports=options.prebound_ports,
- chroot=False,
- daemon=options.daemon)
-
-
-@app.command
-@app.command_option("--simple", default=False, dest='simple', action='store_true',
- help="Only print the checkpoint records, do not replay them.")
-def read(args, options):
- """Replay a thermos checkpoint.
-
- Usage: thermos read [options] checkpoint_filename
- Options:
- --simple Do not replay the full task state machine. Only print out the contents of
- each checkpoint log message.
- """
- if len(args) != 1:
- app.error('Expected one checkpoint file, got %s' % len(args))
- if not os.path.exists(args[0]):
- app.error('Could not find %s' % args[0])
-
- dispatcher = CheckpointDispatcher()
- state = RunnerState(processes={})
- with open(args[0], 'r') as fp:
- try:
- for record in ThriftRecordReader(fp, RunnerCkpt):
- if not options.simple:
- dispatcher.dispatch(state, record)
- else:
- print('CKPT: %s' % record)
- except RecordIO.Error as err:
- print("Failed to recover from %s: %s" % (fp.name, err))
- return
-
- if not options.simple:
- if state is None or state.header is None:
- print('Checkpoint stream CORRUPT or outdated format')
- return
- print('Recovered Task Header:')
- print(' id: %s' % state.header.task_id)
- print(' user: %s' % state.header.user)
- print(' host: %s' % state.header.hostname)
- print(' sandbox: %s' % state.header.sandbox)
- if state.header.ports:
- print(' ports: %s' % ' '.join(
- '%s->%s' % (name, port) for (name, port) in state.header.ports.items()))
- print('Recovered Task States:')
- for task_status in state.statuses:
- print(' %s [pid: %d] => %s' % (
- time.asctime(time.localtime(task_status.timestamp_ms/1000.0)),
- task_status.runner_pid,
- TaskState._VALUES_TO_NAMES[task_status.state]))
- print('Recovered Processes:')
- for process, process_history in state.processes.items():
- print(' %s runs: %s' % (process, len(process_history)))
- for k in reversed(range(len(process_history))):
- run = process_history[k]
- print(' %2d: pid=%d, rc=%s, finish:%s, state:%s' % (
- k,
- run.pid,
- run.return_code if run.return_code is not None else '',
- time.asctime(time.localtime(run.stop_time)) if run.stop_time else 'None',
- ProcessState._VALUES_TO_NAMES.get(run.state, 'Unknown')))
-
-
-@app.command
-def kill(args, options):
- """Kill task(s)
-
- Usage: thermos kill task_id1 [task_id2 ...]
-
- Regular expressions may be used to match multiple tasks.
- """
- if not args:
- print('Must specify tasks!', file=sys.stderr)
- return
-
- matched_tasks = tasks_from_re(args, options.root, state='active')
-
- if not matched_tasks:
- print('No active tasks matched.')
- return
-
- for task_id in matched_tasks:
- print('Killing %s...' % task_id, end='')
- TaskRunnerHelper.kill(task_id, options.root, force=True)
- print('done.')
-
-
-@app.command
-@app.command_option("--max_age", metavar="AGE", default=None, dest='max_age',
- help="Max age in human readable form, e.g. 2d5h or 7200s")
-@app.command_option("--max_tasks", metavar="NUM", default=None, dest='max_tasks',
- help="Max number of tasks to keep.")
-@app.command_option("--max_space", metavar="SPACE", default=None, dest='max_space',
- help="Max space to allow for tasks, e.g. 20G.")
-@app.command_option("--keep-data", "--delete-data",
- metavar="PATH", default=True,
- action='callback', callback=set_keep, dest='keep_data',
- help="Keep data.")
-@app.command_option("--keep-logs", "--delete-logs",
- metavar="PATH", default=True,
- action='callback', callback=set_keep, dest='keep_logs',
- help="Keep logs.")
-@app.command_option("--keep-metadata", "--delete-metadata",
- metavar="PATH", default=True,
- action='callback', callback=set_keep, dest='keep_metadata',
- help="Keep metadata.")
-@app.command_option("--force", default=False, action='store_true', dest='force',
- help="Perform garbage collection without confirmation")
-@app.command_option("--dryrun", default=False, action='store_true', dest='dryrun',
- help="Don't actually run garbage collection.")
-def gc(args, options):
- """Garbage collect task(s) and task metadata.
-
- Usage: thermos gc [options] [task_id1 task_id2 ...]
-
- If tasks specified, restrict garbage collection to only those tasks,
- otherwise all tasks are considered. The optional constraints are still
- honored.
-
- Options:
- --max_age=AGE Max age in quasi-human readable form, e.g. --max_age=2d5h,
- format *d*h*m*s [default: skip]
- --max_tasks=NUM Max number of tasks to keep [default: skip]
- --max_space=SPACE Max space to allow for tasks [default: skip]
- --[keep/delete-]metadata Garbage collect metadata [default: keep]
- --[keep/delete-]logs Garbage collect logs [default: keep]
- --[keep/delete-]data Garbage collect data [default: keep]
- WARNING: Do NOT do this if your sandbox is $HOME.
- --force Perform garbage collection without confirmation [default: false]
- --dryrun Don't actually run garbage collection [default: false]
- """
- print('Analyzing root at %s' % options.root)
- gc_options = {}
- if options.max_age is not None:
- gc_options['max_age'] = parse_time(options.max_age)
- if options.max_space is not None:
- gc_options['max_space'] = parse_data(options.max_space)
- if options.max_tasks is not None:
- gc_options['max_tasks'] = int(options.max_tasks)
- gc_options.update(include_data = not options.keep_data,
- include_metadata = not options.keep_metadata,
- include_logs = not options.keep_logs,
- verbose = True, logger = print)
- tgc = TaskGarbageCollector(root=options.root)
-
- if args:
- gc_tasks = tasks_from_re(args, options.root, state='finished')
- else:
- print('No task ids specified, using default collector.')
- gc_tasks = [task.task_id for task in DefaultCollector(tgc, **gc_options).run()]
-
- if not gc_tasks:
- print('No tasks to garbage collect. Exiting')
- return
-
- def maybe(function, *args):
- if options.dryrun:
- print(' would run %s%r' % (function.__name__, args))
- else:
- function(*args)
-
- value = 'y'
- if not options.force:
- value = raw_input("Continue [y/N]? ") or 'N'
- if value.lower() == 'y':
- print('Running gc...')
- tgc = TaskGarbageCollector(root=options.root)
- for task in gc_tasks:
- print(' Task %s ' % task, end='')
- print('data (%s) ' % ('keeping' if options.keep_data else 'deleting'), end='')
- print('logs (%s) ' % ('keeping' if options.keep_logs else 'deleting'), end='')
- print('metadata (%s) ' % ('keeping' if options.keep_metadata else 'deleting'))
- if not options.keep_data:
- maybe(tgc.erase_data, task)
- if not options.keep_logs:
- maybe(tgc.erase_logs, task)
- if not options.keep_metadata:
- maybe(tgc.erase_metadata, task)
- print('done.')
- else:
- print('Cancelling gc.')
-
-
-@app.command
-@app.command_option("--verbosity", default=0, dest='verbose', type='int',
- help="Display more verbosity")
-@app.command_option("--only", default=None, dest='only', type='choice',
- choices=('active', 'finished'), help="Display only tasks of this type.")
-def status(args, options):
- """Get the status of task(s).
-
- Usage: thermos status [options] [task_name(s) or task_regexp(s)]
-
- Options:
- --verbosity=LEVEL Verbosity level for logging. [default: 0]
- --only=TYPE Only print tasks of TYPE (options: active finished)
- """
- detector = TaskDetector(root=options.root)
-
- def format_task(task_id):
- checkpoint_filename = detector.get_checkpoint(task_id)
- checkpoint_stat = os.stat(checkpoint_filename)
- try:
- checkpoint_owner = pwd.getpwuid(checkpoint_stat.st_uid).pw_name
- except:
- checkpoint_owner = 'uid:%s' % checkpoint_stat.st_uid
- print(' %-20s [owner: %8s]' % (task_id, checkpoint_owner), end='')
- if options.verbose == 0:
- print()
- if options.verbose > 0:
- state = CheckpointDispatcher.from_file(checkpoint_filename)
- if state is None or state.header is None:
- print(' - checkpoint stream CORRUPT or outdated format')
- return
- print(' state: %8s' % TaskState._VALUES_TO_NAMES.get(state.statuses[-1].state, 'Unknown'),
- end='')
- print(' start: %25s' % time.asctime(time.localtime(state.header.launch_time_ms/1000.0)))
- if options.verbose > 1:
- print(' user: %s' % state.header.user, end='')
- if state.header.ports:
- print(' ports: %s' % ' '.join('%s -> %s' % (key, val)
- for key, val in state.header.ports.items()))
- else:
- print(' ports: None')
- print(' sandbox: %s' % state.header.sandbox)
- if options.verbose > 2:
- print(' process table:')
- for process, process_history in state.processes.items():
- print(' - %s runs: %s' % (process, len(process_history)), end='')
- last_run = process_history[-1]
- print(' last: pid=%s, rc=%s, finish:%s, state:%s' % (
- last_run.pid or 'None',
- last_run.return_code if last_run.return_code is not None else '',
- time.asctime(time.localtime(last_run.stop_time)) if last_run.stop_time else 'None',
- ProcessState._VALUES_TO_NAMES.get(last_run.state, 'Unknown')))
- print()
-
- matchers = map(re.compile, args or ['.*'])
- active = [t_id for _, t_id in detector.get_task_ids(state='active')
- if any(pattern.match(t_id) for pattern in matchers)]
- finished = [t_id for _, t_id in detector.get_task_ids(state='finished')
- if any(pattern.match(t_id) for pattern in matchers)]
-
- found = False
- if options.only is None or options.only == 'active':
- if active:
- print('Active tasks:')
- found = True
- for task_id in active:
- format_task(task_id)
- print()
-
- if options.only is None or options.only == 'finished':
- if finished:
- print('Finished tasks:')
- found = True
- for task_id in finished:
- format_task(task_id)
- print()
-
- if not found:
- print('No tasks found in root [%s]' % options.root)
- sys.exit(1)
-
-
-
-@app.command
-@app.command_option("--stderr", default=False, dest='use_stderr', action='store_true',
- help="Tail stderr instead of stdout")
-def tail(args, options):
- """Tail the logs of a task process.
-
- Usage: thermos tail task_name [process_name]
- """
- if len(args) == 0:
- app.error('Expected a task to tail, got nothing!')
- if len(args) not in (1, 2):
- app.error('Expected at most two arguments (task and optional process), got %d' % len(args))
-
- task_id = args[0]
- detector = TaskDetector(root=options.root)
- checkpoint = CheckpointDispatcher.from_file(detector.get_checkpoint(task_id))
- log_dir = checkpoint.header.log_dir
- process_runs = [(process, run) for (process, run) in detector.get_process_runs(task_id, log_dir)]
- if len(args) == 2:
- process_runs = [(process, run) for (process, run) in process_runs if process == args[1]]
-
- if len(process_runs) == 0:
- print('ERROR: No processes found.', file=sys.stderr)
- sys.exit(1)
-
- processes = set([process for process, _ in process_runs])
- if len(processes) != 1:
- print('ERROR: More than one process matches query.', file=sys.stderr)
- sys.exit(1)
-
- process = processes.pop()
- run = max([run for _, run in process_runs])
-
- logdir = TaskPath(root=options.root, task_id=args[0], process=process,
- run=run, log_dir=log_dir).getpath('process_logdir')
- logfile = os.path.join(logdir, 'stderr' if options.use_stderr else 'stdout')
-
- monitor = TaskMonitor(TaskPath(root = options.root), args[0])
- def log_is_active():
- active_processes = monitor.get_active_processes()
- for process_status, process_run in active_processes:
- if process_status.process == process and process_run == run:
- return True
- return False
-
- if not log_is_active():
- print('Tail of terminal log %s' % logfile)
- for line in tail_closed(logfile):
- print(line.rstrip())
- return
-
- now = time.time()
- next_check = now + 5.0
- print('Tail of active log %s' % logfile)
- for line in tail_f(logfile, include_last=True, forever=False):
- print(line.rstrip())
- if time.time() > next_check:
- if not log_is_active():
- break
- else:
- next_check = time.time() + 5.0
-
-
-@app.command
-def help(args, options):
- """Get help about a specific command.
- """
- if len(args) == 0:
- app.help()
- for (command, doc) in app.get_commands_and_docstrings():
- if args[0] == command:
- print('command %s:' % command)
- print(doc)
- app.quit(0)
- print('unknown command: %s' % args[0], file=sys.stderr)
-
-
-
-
-def generate_usage():
- usage = """
-thermos
-
-commands:
-"""
-
- for (command, doc) in app.get_commands_and_docstrings():
- usage += ' ' + '%-10s' % command + '\t' + doc.split('\n')[0].strip() + '\n'
- app.set_usage(usage)
-
-
-LogOptions.set_disk_log_level('NONE')
-LogOptions.set_stdout_log_level('INFO')
-generate_usage()
-
-
-proxy_main = app.main
-
-
-proxy_main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/bin/thermos_ckpt.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/bin/thermos_ckpt.py b/src/main/python/twitter/thermos/bin/thermos_ckpt.py
deleted file mode 100755
index 36fd419..0000000
--- a/src/main/python/twitter/thermos/bin/thermos_ckpt.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import os
-import sys
-import pprint
-import time
-
-from gen.twitter.thermos.ttypes import RunnerState, RunnerCkpt, TaskState
-
-from twitter.common import app
-from twitter.common.recordio import RecordIO, ThriftRecordReader
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-
-app.add_option("--checkpoint", dest = "ckpt", metavar = "CKPT",
- help = "read checkpoint from CKPT")
-app.add_option("--assemble", dest = "assemble", metavar = "CKPT", default=True,
- help = "read checkpoint from CKPT")
-
-def main(args):
- values = app.get_options()
-
- if len(args) > 0:
- print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % (" ".join(args))
- app.help()
- sys.exit(1)
-
- if not values.ckpt:
- print >> sys.stderr, "ERROR: must supply --checkpoint"
- app.help()
- sys.exit(1)
-
- fp = file(values.ckpt, "r")
- rr = ThriftRecordReader(fp, RunnerCkpt)
- wrs = RunnerState(processes = {})
- dispatcher = CheckpointDispatcher()
- try:
- for wts in rr:
- print 'Recovering: ', wts
- if values.assemble is True:
- dispatcher.dispatch(wrs, wts)
- except RecordIO.Error as err:
- print 'Error recovering checkpoint stream: %s' % err
- return
- print '\n\n\n'
- if values.assemble:
- print 'Recovered Task Header'
- pprint.pprint(wrs.header, indent=4)
-
- print '\nRecovered Task States'
- for task_status in wrs.statuses:
- print ' %s [pid: %d] => %s' % (time.asctime(time.localtime(task_status.timestamp_ms/1000.0)),
- task_status.runner_pid, TaskState._VALUES_TO_NAMES[task_status.state])
-
- print '\nRecovered Processes'
- pprint.pprint(wrs.processes, indent=4)
-
-app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/common/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/common/BUILD b/src/main/python/twitter/thermos/common/BUILD
deleted file mode 100644
index d97c771..0000000
--- a/src/main/python/twitter/thermos/common/BUILD
+++ /dev/null
@@ -1,45 +0,0 @@
-import os
-
-python_library(
- name = 'ckpt',
- sources = ['ckpt.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift')
- ]
-)
-
-python_library(
- name = 'path',
- sources = ['path.py'],
-)
-
-python_library(
- name = 'planner',
- sources = ['planner.py'],
-)
-
-python_library(
- name = 'options',
- sources = ['options.py'],
- dependencies = [
- pants('src/main/python/twitter/thermos:pystachio'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'common',
- dependencies = [
- pants(':ckpt'),
- pants(':options'),
- pants(':path'),
- pants(':planner'),
- ],
- provides = setup_py(
- name = 'twitter.thermos.common',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- description = 'Thermos common libraries.',
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/common/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/common/__init__.py b/src/main/python/twitter/thermos/common/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/common/ckpt.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/common/ckpt.py b/src/main/python/twitter/thermos/common/ckpt.py
deleted file mode 100644
index 9c5a949..0000000
--- a/src/main/python/twitter/thermos/common/ckpt.py
+++ /dev/null
@@ -1,375 +0,0 @@
-"""Read checkpoint streams for the Thermos runner, and dispatch events on state transitions
-
-This module contains the CheckpointDispatcher, which reconstructs checkpoint streams containing the
-state of the Thermos runner and its constituent processes.
-
-It also defines several Handler interfaces to define behaviour on transitions in the Process and
-Task state machines.
-
-"""
-
-import os
-
-from twitter.common import log
-from twitter.common.recordio import RecordIO, ThriftRecordReader
-from gen.twitter.thermos.ttypes import (
- ProcessState,
- ProcessStatus,
- RunnerCkpt,
- RunnerState,
- TaskState,
-)
-
-
-class UniversalStateHandler(object):
- """
- Generic interface for a handler to be called on any process/state transition, and at task
- initialization
- """
- def on_process_transition(self, state, process_update):
- pass
-
- def on_task_transition(self, state, task_update):
- pass
-
- def on_initialization(self, header):
- pass
-
-
-class ProcessStateHandler(object):
- """
- Interface for handlers for the Process state machine, called on process transitions
-
- () - starting state, [] - terminal state
-
- [FAILED]
- ^
- |
- (WAITING) ----> FORKED ----> RUNNING -----> [KILLED]
- | | |
- v | `---> [SUCCESS]
- [LOST] <------'
- """
- def on_waiting(self, process_update):
- pass
-
- def on_forked(self, process_update):
- pass
-
- def on_running(self, process_update):
- pass
-
- def on_success(self, process_update):
- pass
-
- def on_failed(self, process_update):
- pass
-
- def on_lost(self, process_update):
- pass
-
- def on_killed(self, process_update):
- pass
-
-
-class TaskStateHandler(object):
- """
- Interface for handlers for the Task state machine, called on task transitions
-
- () - starting state, [] - terminal state
-
- .--------------------------------------------+----.
- | | |
- | .----------> [SUCCESS] | |
- | | | |
- | | .--------> [FAILED] | |
- | | | | |
- (ACTIVE) FINALIZING ---> [KILLED] <---' |
- | ^ | .------^ |
- | | | | |
- `---> CLEANING ---' `----)--> [LOST] <---------'
- | | | ^
- | `----------------' |
- `-------------------------'
-
- ACTIVE -> KILLED/LOST only happens under garbage collection situations.
- Ordinary task preemption/kill still goes through CLEANING/FINALIZING before
- reaching a terminal state.
- """
-
- def on_active(self, task_update):
- pass
-
- def on_cleaning(self, task_update):
- pass
-
- def on_finalizing(self, task_update):
- pass
-
- def on_success(self, task_update):
- pass
-
- def on_failed(self, task_update):
- pass
-
- def on_killed(self, task_update):
- pass
-
- def on_lost(self, task_update):
- pass
-
-
-def assert_nonempty(state, fields):
- for field in fields:
- assert getattr(state, field, None) is not None, "Missing field %s from %s!" % (field, state)
-
-
-def copy_fields(state, state_update, fields):
- assert_nonempty(state_update, fields)
- for field in fields:
- setattr(state, field, getattr(state_update, field))
-
-
-class CheckpointDispatcher(object):
- """
- The reconstruction/dispatching mechanism for logic triggered on task/process state transitions.
-
- Most applications should build an event-loop around the CheckpointDispatcher.
- """
-
- class Error(Exception): pass
- class ErrorRecoveringState(Error): pass
- class InvalidSequenceNumber(Error): pass
- class InvalidHandler(Error): pass
-
- @classmethod
- def iter_updates(cls, filename):
- try:
- with open(filename) as fp:
- rr = ThriftRecordReader(fp, RunnerCkpt)
- for update in rr:
- yield update
- except (IOError, OSError, RecordIO.Error) as err:
- raise cls.ErrorRecoveringState(err)
-
- @classmethod
- def iter_statuses(cls, filename):
- for update in cls.iter_updates(filename):
- if update.task_status:
- yield update.task_status
-
- @classmethod
- def from_file(cls, filename, truncate=False):
- """Reconstruct a RunnerState from a checkpoint stream contained in a file
-
- Returns a hydrated RunnerState, or None on any failures.
- """
- state = RunnerState(processes={})
- builder = cls()
- try:
- for update in cls.iter_updates(filename):
- builder.dispatch(state, update, truncate=truncate)
- return state
- except cls.Error as e:
- log.error('Failed to recover from %s: %s' % (filename, e))
-
- def __init__(self):
- self._task_handlers = []
- self._process_handlers = []
- self._universal_handlers = []
-
- def register_handler(self, handler):
- HANDLER_MAP = {
- TaskStateHandler: self._task_handlers,
- ProcessStateHandler: self._process_handlers,
- UniversalStateHandler: self._universal_handlers
- }
-
- for handler_type, handler_list in HANDLER_MAP.items():
- if isinstance(handler, handler_type):
- handler_list.append(handler)
- break
- else:
- raise self.InvalidHandler("Unknown handler type %s" % type(handler))
-
- def _run_process_dispatch(self, state, process_update):
- for handler in self._universal_handlers:
- handler.on_process_transition(state, process_update)
- for handler in self._process_handlers:
- handler_function = 'on_' + ProcessState._VALUES_TO_NAMES[state].lower()
- getattr(handler, handler_function)(process_update)
-
- def _run_task_dispatch(self, state, task_update):
- for handler in self._universal_handlers:
- handler.on_task_transition(state, task_update)
- for handler in self._task_handlers:
- handler_function = 'on_' + TaskState._VALUES_TO_NAMES[state].lower()
- getattr(handler, handler_function)(task_update)
-
- def _run_header_dispatch(self, header):
- for handler in self._universal_handlers:
- handler.on_initialization(header)
-
- @staticmethod
- def is_terminal(process_state_update):
- TERMINAL_STATES = [
- ProcessState.SUCCESS,
- ProcessState.FAILED,
- ProcessState.KILLED,
- ProcessState.LOST]
- return process_state_update.state in TERMINAL_STATES
-
- @classmethod
- def _update_process_state(cls, process_state, process_state_update):
- """
- Apply process_state_update against process_state.
- Raises ErrorRecoveringState on failure.
- """
- def assert_process_state_in(*expected_states):
- if process_state.state not in expected_states:
- raise cls.ErrorRecoveringState(
- 'Detected invalid state transition %s => %s' % (
- ProcessState._VALUES_TO_NAMES.get(process_state.state),
- ProcessState._VALUES_TO_NAMES.get(process_state_update.state)))
-
- # CREATION => WAITING
- if process_state_update.state == ProcessState.WAITING:
- assert_process_state_in(None)
- required_fields = ['seq', 'state', 'process']
- copy_fields(process_state, process_state_update, required_fields)
-
- # WAITING => FORKED
- elif process_state_update.state == ProcessState.FORKED:
- assert_process_state_in(ProcessState.WAITING)
- required_fields = ['seq', 'state', 'fork_time', 'coordinator_pid']
- copy_fields(process_state, process_state_update, required_fields)
-
- # FORKED => RUNNING
- elif process_state_update.state == ProcessState.RUNNING:
- assert_process_state_in(ProcessState.FORKED)
- required_fields = ['seq', 'state', 'start_time', 'pid']
- copy_fields(process_state, process_state_update, required_fields)
-
- # RUNNING => SUCCESS
- elif process_state_update.state == ProcessState.SUCCESS:
- assert_process_state_in(ProcessState.RUNNING)
- required_fields = ['seq', 'state', 'stop_time', 'return_code']
- copy_fields(process_state, process_state_update, required_fields)
-
- # RUNNING => FAILED
- elif process_state_update.state == ProcessState.FAILED:
- assert_process_state_in(ProcessState.RUNNING)
- required_fields = ['seq', 'state', 'stop_time', 'return_code']
- copy_fields(process_state, process_state_update, required_fields)
-
- # {FORKED, RUNNING} => KILLED
- elif process_state_update.state == ProcessState.KILLED:
- assert_process_state_in(ProcessState.FORKED, ProcessState.RUNNING)
- required_fields = ['seq', 'state', 'stop_time', 'return_code']
- copy_fields(process_state, process_state_update, required_fields)
-
- # {FORKED, RUNNING} => LOST
- elif process_state_update.state == ProcessState.LOST:
- assert_process_state_in(ProcessState.FORKED, ProcessState.RUNNING)
- required_fields = ['seq', 'state']
- copy_fields(process_state, process_state_update, required_fields)
-
- else:
- raise cls.ErrorRecoveringState(
- "Unknown state = %s" % process_state_update.state)
-
- def would_update(self, state, runner_ckpt):
- """
- Provided a ProcessStatus, would this perform a transition and update state?
- """
- process_update = runner_ckpt.process_status
- if process_update is None:
- return False
- process = process_update.process
- if process not in state.processes: # never seen before
- return True
- else:
- # if this sequence number is ahead of the current high water mark, it would
- # produce a transition
- return state.processes[process][-1].seq < process_update.seq
-
- def dispatch(self, state, runner_ckpt, recovery=False, truncate=False):
- """
- Given a RunnerState and a RunnerCkpt to apply to it, determine the appropriate action and
- dispatch to the appropriate handlers.
-
- state = RunnerState to be updated
- runner_ckpt = RunnerCkpt update to apply
- recovery = if true, enable recovery mode (accept out-of-order sequence updates)
- truncate = if true, store only the latest task/process states, instead of
- history for all runs.
-
- Raises ErrorRecoveringState on failure.
- """
- # case 1: runner_header
- # -> Initialization of the task stream.
- if runner_ckpt.runner_header is not None:
- if state.header is not None:
- raise self.ErrorRecoveringState(
- "Attempting to rebind task with different parameters!")
- else:
- log.debug('Initializing TaskRunner header to %s' % runner_ckpt.runner_header)
- state.header = runner_ckpt.runner_header
- self._run_header_dispatch(runner_ckpt.runner_header)
- return
-
- # case 2: task_status
- # -> State transition on the task (ACTIVE, FAILED, SUCCESS, LOST)
- if runner_ckpt.task_status is not None:
- if state.statuses is None:
- state.statuses = []
- old_state = None
- else:
- old_state = state.statuses[-1].state
- if not truncate:
- state.statuses.append(runner_ckpt.task_status)
- else:
- state.statuses = [runner_ckpt.task_status]
- new_state = runner_ckpt.task_status.state
- log.debug('Flipping task state from %s to %s' % (
- TaskState._VALUES_TO_NAMES.get(old_state, '(undefined)'),
- TaskState._VALUES_TO_NAMES.get(new_state, '(undefined)')))
- self._run_task_dispatch(new_state, runner_ckpt.task_status)
- return
-
- # case 3: process_status
- # -> State transition on a process itself
- # (WAITING, FORKED, RUNNING, SUCCESS, KILLED, FAILED, LOST)
- if runner_ckpt.process_status is not None:
- process_update = runner_ckpt.process_status
- name = process_update.process
- current_run = state.processes[name][-1] if name in state.processes else None
- if current_run and process_update.seq != current_run.seq + 1:
- if recovery:
- log.debug('Skipping replayed out-of-order update: %s' % process_update)
- return
- else:
- raise self.InvalidSequenceNumber(
- "Out of order sequence number! %s => %s" % (current_run, process_update))
-
- # One special case for WAITING: Initialize a new target ProcessState.
- if process_update.state == ProcessState.WAITING:
- assert current_run is None or self.is_terminal(current_run)
- if name not in state.processes:
- state.processes[name] = [ProcessStatus(seq=-1)]
- else:
- if not truncate:
- state.processes[name].append(ProcessStatus(seq=current_run.seq))
- else:
- state.processes[name] = [ProcessStatus(seq=current_run.seq)]
-
- # Run the process state machine.
- log.debug('Running state machine for process=%s/seq=%s' % (name, process_update.seq))
- if not state.processes or name not in state.processes:
- raise self.ErrorRecoveringState("Encountered potentially out of order "
- "process update. Are you sure this is a full checkpoint stream?")
- self._update_process_state(state.processes[name][-1], process_update)
- self._run_process_dispatch(process_update.state, process_update)
- return
-
- raise self.ErrorRecoveringState("Empty RunnerCkpt encountered!")
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/common/options.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/common/options.py b/src/main/python/twitter/thermos/common/options.py
deleted file mode 100644
index 9948d03..0000000
--- a/src/main/python/twitter/thermos/common/options.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from pystachio import Ref
-
-class ParseError(Exception):
- pass
-
-def add_port_to(option_name):
- def add_port_callback(option, opt, value, parser):
- if not getattr(parser.values, option_name, None):
- setattr(parser.values, option_name, {})
- try:
- name, port = value.split(':')
- except (ValueError, TypeError):
- raise ParseError('Invalid value for %s: %s should be of form NAME:PORT' % (
- opt, value))
- try:
- port = int(port)
- except ValueError:
- raise ParseError('Port does not appear to be an integer: %s' % port)
- getattr(parser.values, option_name)[name] = port
- return add_port_callback
-
-def add_binding_to(option_name):
- def add_binding_callback(option, opt, value, parser):
- if not getattr(parser.values, option_name, None):
- setattr(parser.values, option_name, [])
- if len(value.split('=')) != 2:
- raise ParseError('Binding must be of the form NAME=VALUE')
- name, value = value.split('=')
- try:
- ref = Ref.from_address(name)
- except Ref.InvalidRefError as e:
- raise ParseError('Could not parse ref %s: %s' % (name, e))
- getattr(parser.values, option_name).append({ref: value})
- return add_binding_callback
[03/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled.png
deleted file mode 100644
index a4e6b53..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png
deleted file mode 100644
index fc46c5e..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/forward_enabled_hover.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc.png
deleted file mode 100644
index a88d797..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png
deleted file mode 100644
index 4e144cf..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_asc_disabled.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_both.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_both.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_both.png
deleted file mode 100644
index 1867040..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_both.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc.png
deleted file mode 100644
index def071e..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png
deleted file mode 100644
index 7824973..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/images/sort_desc_disabled.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js
deleted file mode 100644
index 420e507..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.bootstrap.js
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * File: dataTables.bootstrap.js
- * Version: 1.9.4
- * Author: Allan Jardine (www.sprymedia.co.uk)
- * Info: www.datatables.net
- *
- * Copyright 2008-2012 Allan Jardine, all rights reserved.
- *
- * This source file is free software, under either the GPL v2 license or a
- * BSD style license, available at:
- * http://datatables.net/license_gpl2
- * http://datatables.net/license_bsd
- *
- * This source file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
- */
-
-/* Set the defaults for DataTables initialisation */
-$.extend( true, $.fn.dataTable.defaults, {
- "sDom": "<'row-fluid'<'span4'l><'span2'r><'span6'f>>t<'row-fluid'<'span6'i><'span6'p>>",
- "sPaginationType": "bootstrap",
- "oLanguage": {
- "sLengthMenu": "_MENU_ records per page"
- }
-} );
-
-
-/* Default class modification */
-$.extend( $.fn.dataTableExt.oStdClasses, {
- "sWrapper": "dataTables_wrapper form-inline"
-} );
-
-
-/* API method to get paging information */
-$.fn.dataTableExt.oApi.fnPagingInfo = function ( oSettings )
-{
- return {
- "iStart": oSettings._iDisplayStart,
- "iEnd": oSettings.fnDisplayEnd(),
- "iLength": oSettings._iDisplayLength,
- "iTotal": oSettings.fnRecordsTotal(),
- "iFilteredTotal": oSettings.fnRecordsDisplay(),
- "iPage": oSettings._iDisplayLength === -1 ?
- 0 : Math.ceil( oSettings._iDisplayStart / oSettings._iDisplayLength ),
- "iTotalPages": oSettings._iDisplayLength === -1 ?
- 0 : Math.ceil( oSettings.fnRecordsDisplay() / oSettings._iDisplayLength )
- };
-};
-
-
-/* Bootstrap style pagination control */
-$.extend( $.fn.dataTableExt.oPagination, {
- "bootstrap": {
- "fnInit": function( oSettings, nPaging, fnDraw ) {
- var oLang = oSettings.oLanguage.oPaginate;
- var fnClickHandler = function ( e ) {
- e.preventDefault();
- if ( oSettings.oApi._fnPageChange(oSettings, e.data.action) ) {
- fnDraw( oSettings );
- }
- };
-
- $(nPaging).addClass('pagination').append(
- '<ul>'+
- '<li class="prev disabled"><a href="#">← '+oLang.sPrevious+'</a></li>'+
- '<li class="next disabled"><a href="#">'+oLang.sNext+' → </a></li>'+
- '</ul>'
- );
- var els = $('a', nPaging);
- $(els[0]).bind( 'click.DT', { action: "previous" }, fnClickHandler );
- $(els[1]).bind( 'click.DT', { action: "next" }, fnClickHandler );
- },
-
- "fnUpdate": function ( oSettings, fnDraw ) {
- var iListLength = 5;
- var oPaging = oSettings.oInstance.fnPagingInfo();
- var an = oSettings.aanFeatures.p;
- var i, ien, j, sClass, iStart, iEnd, iHalf=Math.floor(iListLength/2);
-
- if ( oPaging.iTotalPages < iListLength) {
- iStart = 1;
- iEnd = oPaging.iTotalPages;
- }
- else if ( oPaging.iPage <= iHalf ) {
- iStart = 1;
- iEnd = iListLength;
- } else if ( oPaging.iPage >= (oPaging.iTotalPages-iHalf) ) {
- iStart = oPaging.iTotalPages - iListLength + 1;
- iEnd = oPaging.iTotalPages;
- } else {
- iStart = oPaging.iPage - iHalf + 1;
- iEnd = iStart + iListLength - 1;
- }
-
- for ( i=0, ien=an.length ; i<ien ; i++ ) {
- // Remove the middle elements
- $('li:gt(0)', an[i]).filter(':not(:last)').remove();
-
- // Add the new list items and their event handlers
- for ( j=iStart ; j<=iEnd ; j++ ) {
- sClass = (j==oPaging.iPage+1) ? 'class="active"' : '';
- $('<li '+sClass+'><a href="#">'+j+'</a></li>')
- .insertBefore( $('li:last', an[i])[0] )
- .bind('click', function (e) {
- e.preventDefault();
- oSettings._iDisplayStart = (parseInt($('a', this).text(),10)-1) * oPaging.iLength;
- fnDraw( oSettings );
- } );
- }
-
- // Add / remove disabled classes from the static elements
- if ( oPaging.iPage === 0 ) {
- $('li:first', an[i]).addClass('disabled');
- } else {
- $('li:first', an[i]).removeClass('disabled');
- }
-
- if ( oPaging.iPage === oPaging.iTotalPages-1 || oPaging.iTotalPages === 0 ) {
- $('li:last', an[i]).addClass('disabled');
- } else {
- $('li:last', an[i]).removeClass('disabled');
- }
- }
- }
- }
-} );
-
-
-/*
- * TableTools Bootstrap compatibility
- * Required TableTools 2.1+
- */
-if ( $.fn.DataTable.TableTools ) {
- // Set the classes that TableTools uses to something suitable for Bootstrap
- $.extend( true, $.fn.DataTable.TableTools.classes, {
- "container": "DTTT btn-group",
- "buttons": {
- "normal": "btn",
- "disabled": "disabled"
- },
- "collection": {
- "container": "DTTT_dropdown dropdown-menu",
- "buttons": {
- "normal": "",
- "disabled": "disabled"
- }
- },
- "print": {
- "info": "DTTT_print_info modal"
- },
- "select": {
- "row": "active"
- }
- } );
-
- // Have the collection use a bootstrap compatible dropdown
- $.extend( true, $.fn.DataTable.TableTools.DEFAULTS.oTags, {
- "collection": {
- "container": "ul",
- "button": "li",
- "liner": "a"
- }
- } );
-};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js
deleted file mode 100644
index 885ad34..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.htmlNumberType.js
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * DataTables custom plugin to extract numbers from HTML tags
- * for sorting rows in numeric order.
- *
- * Ref: http://www.datatables.net/plug-ins/sorting
- *
- * Compatible with DataTables v1.9.4
- *
- */
-
-jQuery.extend(jQuery.fn.dataTableExt.oSort, {
- "num-html-pre": function (a) {
- // Extract the numeric value from the html by removing all tags.
- var x = String(a).replace(/<[\s\S]*?>/g, "");
- return parseFloat(x);
- },
-
- "num-html-asc": function (a, b) {
- // Compare the extracted numeric values for asc sort.
- return ((a < b) ? -1 : ((a > b) ? 1 : 0));
- },
-
- "num-html-desc": function (a, b) {
- // Compare the extracted numeric values for desc sort.
- return ((a < b) ? 1 : ((a > b) ? -1 : 0));
- }
-});
-
-/*
- * Type detection function for custom type "num-html".
- *
- * Ref: http://www.datatables.net/plug-ins/type-detection#how_to
- */
-jQuery.fn.dataTableExt.aTypes.unshift(function (sData) {
- sData = typeof sData.replace == "function" ?
- sData.replace( /<[\s\S]*?>/g, "" ) : sData;
- sData = $.trim(sData);
-
- var sValidFirstChars = "0123456789-";
- var sValidChars = "0123456789.";
- var Char;
- var bDecimal = false;
-
- // Check for a valid first char (no period and allow negatives).
- Char = sData.charAt(0);
- if (sValidFirstChars.indexOf(Char) == -1) {
- return null;
- }
-
- // Check all the other characters are valid.
- for ( var i=1 ; i<sData.length ; i++ ) {
- Char = sData.charAt(i);
- if (sValidChars.indexOf(Char) == -1) {
- return null;
- }
-
- // Only one decimal place allowed.
- if (Char == ".") {
- if (bDecimal) {
- return null;
- }
- bDecimal = true;
- }
- }
-
- return "num-html";
-});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js
deleted file mode 100644
index 3c0b39a..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/dataTables.localstorage.js
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Make browser's local storage as the default storage for Datatables plugin.
- * Compatible with Datatables v1.9.4
- *
- * http://www.datatables.net/beta/1.9/examples/advanced_init/localstorage.html
- *
- * Always link this file after linking jquery.dataTables.min.js
- */
-$.extend(true, $.fn.dataTable.defaults, {
- "fnStateSave": function (oSettings, oData) {
- localStorage.setItem("DataTables_" + window.location.pathname + "_" + oSettings.sTableId,
- JSON.stringify(oData));
- },
- "fnStateLoad": function (oSettings) {
- return JSON.parse(localStorage
- .getItem("DataTables_" + window.location.pathname + "_" + oSettings.sTableId));
- }
-});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js b/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js
deleted file mode 100644
index 02694a4..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/assets/datatables/js/jquery.dataTables.min.js
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * File: jquery.dataTables.min.js
- * Version: 1.9.4
- * Author: Allan Jardine (www.sprymedia.co.uk)
- * Info: www.datatables.net
- *
- * Copyright 2008-2012 Allan Jardine, all rights reserved.
- *
- * This source file is free software, under either the GPL v2 license or a
- * BSD style license, available at:
- * http://datatables.net/license_gpl2
- * http://datatables.net/license_bsd
- *
- * This source file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE. See the license files for details.
- */
-(function(X,l,n){var L=function(h){var j=function(e){function o(a,b){var c=j.defaults.columns,d=a.aoColumns.length,c=h.extend({},j.models.oColumn,c,{sSortingClass:a.oClasses.sSortable,sSortingClassJUI:a.oClasses.sSortJUI,nTh:b?b:l.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.oDefaults:d});a.aoColumns.push(c);if(a.aoPreSearchCols[d]===n||null===a.aoPreSearchCols[d])a.aoPreSearchCols[d]=h.extend({},j.models.oSearch);else if(c=a.aoPreSearchCols[d],
-c.bRegex===n&&(c.bRegex=!0),c.bSmart===n&&(c.bSmart=!0),c.bCaseInsensitive===n)c.bCaseInsensitive=!0;m(a,d,null)}function m(a,b,c){var d=a.aoColumns[b];c!==n&&null!==c&&(c.mDataProp&&!c.mData&&(c.mData=c.mDataProp),c.sType!==n&&(d.sType=c.sType,d._bAutoType=!1),h.extend(d,c),p(d,c,"sWidth","sWidthOrig"),c.iDataSort!==n&&(d.aDataSort=[c.iDataSort]),p(d,c,"aDataSort"));var i=d.mRender?Q(d.mRender):null,f=Q(d.mData);d.fnGetData=function(a,b){var c=f(a,b);return d.mRender&&b&&""!==b?i(c,b,a):c};d.fnSetData=
-L(d.mData);a.oFeatures.bSort||(d.bSortable=!1);!d.bSortable||-1==h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortableNone,d.sSortingClassJUI=""):-1==h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortable,d.sSortingClassJUI=a.oClasses.sSortJUI):-1!=h.inArray("asc",d.asSorting)&&-1==h.inArray("desc",d.asSorting)?(d.sSortingClass=a.oClasses.sSortableAsc,d.sSortingClassJUI=a.oClasses.sSortJUIAscAllowed):-1==
-h.inArray("asc",d.asSorting)&&-1!=h.inArray("desc",d.asSorting)&&(d.sSortingClass=a.oClasses.sSortableDesc,d.sSortingClassJUI=a.oClasses.sSortJUIDescAllowed)}function k(a){if(!1===a.oFeatures.bAutoWidth)return!1;da(a);for(var b=0,c=a.aoColumns.length;b<c;b++)a.aoColumns[b].nTh.style.width=a.aoColumns[b].sWidth}function G(a,b){var c=r(a,"bVisible");return"number"===typeof c[b]?c[b]:null}function R(a,b){var c=r(a,"bVisible"),c=h.inArray(b,c);return-1!==c?c:null}function t(a){return r(a,"bVisible").length}
-function r(a,b){var c=[];h.map(a.aoColumns,function(a,i){a[b]&&c.push(i)});return c}function B(a){for(var b=j.ext.aTypes,c=b.length,d=0;d<c;d++){var i=b[d](a);if(null!==i)return i}return"string"}function u(a,b){for(var c=b.split(","),d=[],i=0,f=a.aoColumns.length;i<f;i++)for(var g=0;g<f;g++)if(a.aoColumns[i].sName==c[g]){d.push(g);break}return d}function M(a){for(var b="",c=0,d=a.aoColumns.length;c<d;c++)b+=a.aoColumns[c].sName+",";return b.length==d?"":b.slice(0,-1)}function ta(a,b,c,d){var i,f,
-g,e,w;if(b)for(i=b.length-1;0<=i;i--){var j=b[i].aTargets;h.isArray(j)||D(a,1,"aTargets must be an array of targets, not a "+typeof j);f=0;for(g=j.length;f<g;f++)if("number"===typeof j[f]&&0<=j[f]){for(;a.aoColumns.length<=j[f];)o(a);d(j[f],b[i])}else if("number"===typeof j[f]&&0>j[f])d(a.aoColumns.length+j[f],b[i]);else if("string"===typeof j[f]){e=0;for(w=a.aoColumns.length;e<w;e++)("_all"==j[f]||h(a.aoColumns[e].nTh).hasClass(j[f]))&&d(e,b[i])}}if(c){i=0;for(a=c.length;i<a;i++)d(i,c[i])}}function H(a,
-b){var c;c=h.isArray(b)?b.slice():h.extend(!0,{},b);var d=a.aoData.length,i=h.extend(!0,{},j.models.oRow);i._aData=c;a.aoData.push(i);for(var f,i=0,g=a.aoColumns.length;i<g;i++)c=a.aoColumns[i],"function"===typeof c.fnRender&&c.bUseRendered&&null!==c.mData?F(a,d,i,S(a,d,i)):F(a,d,i,v(a,d,i)),c._bAutoType&&"string"!=c.sType&&(f=v(a,d,i,"type"),null!==f&&""!==f&&(f=B(f),null===c.sType?c.sType=f:c.sType!=f&&"html"!=c.sType&&(c.sType="string")));a.aiDisplayMaster.push(d);a.oFeatures.bDeferRender||ea(a,
-d);return d}function ua(a){var b,c,d,i,f,g,e;if(a.bDeferLoading||null===a.sAjaxSource)for(b=a.nTBody.firstChild;b;){if("TR"==b.nodeName.toUpperCase()){c=a.aoData.length;b._DT_RowIndex=c;a.aoData.push(h.extend(!0,{},j.models.oRow,{nTr:b}));a.aiDisplayMaster.push(c);f=b.firstChild;for(d=0;f;){g=f.nodeName.toUpperCase();if("TD"==g||"TH"==g)F(a,c,d,h.trim(f.innerHTML)),d++;f=f.nextSibling}}b=b.nextSibling}i=T(a);d=[];b=0;for(c=i.length;b<c;b++)for(f=i[b].firstChild;f;)g=f.nodeName.toUpperCase(),("TD"==
-g||"TH"==g)&&d.push(f),f=f.nextSibling;c=0;for(i=a.aoColumns.length;c<i;c++){e=a.aoColumns[c];null===e.sTitle&&(e.sTitle=e.nTh.innerHTML);var w=e._bAutoType,o="function"===typeof e.fnRender,k=null!==e.sClass,n=e.bVisible,m,p;if(w||o||k||!n){g=0;for(b=a.aoData.length;g<b;g++)f=a.aoData[g],m=d[g*i+c],w&&"string"!=e.sType&&(p=v(a,g,c,"type"),""!==p&&(p=B(p),null===e.sType?e.sType=p:e.sType!=p&&"html"!=e.sType&&(e.sType="string"))),e.mRender?m.innerHTML=v(a,g,c,"display"):e.mData!==c&&(m.innerHTML=v(a,
-g,c,"display")),o&&(p=S(a,g,c),m.innerHTML=p,e.bUseRendered&&F(a,g,c,p)),k&&(m.className+=" "+e.sClass),n?f._anHidden[c]=null:(f._anHidden[c]=m,m.parentNode.removeChild(m)),e.fnCreatedCell&&e.fnCreatedCell.call(a.oInstance,m,v(a,g,c,"display"),f._aData,g,c)}}if(0!==a.aoRowCreatedCallback.length){b=0;for(c=a.aoData.length;b<c;b++)f=a.aoData[b],A(a,"aoRowCreatedCallback",null,[f.nTr,f._aData,b])}}function I(a,b){return b._DT_RowIndex!==n?b._DT_RowIndex:null}function fa(a,b,c){for(var b=J(a,b),d=0,a=
-a.aoColumns.length;d<a;d++)if(b[d]===c)return d;return-1}function Y(a,b,c,d){for(var i=[],f=0,g=d.length;f<g;f++)i.push(v(a,b,d[f],c));return i}function v(a,b,c,d){var i=a.aoColumns[c];if((c=i.fnGetData(a.aoData[b]._aData,d))===n)return a.iDrawError!=a.iDraw&&null===i.sDefaultContent&&(D(a,0,"Requested unknown parameter "+("function"==typeof i.mData?"{mData function}":"'"+i.mData+"'")+" from the data source for row "+b),a.iDrawError=a.iDraw),i.sDefaultContent;if(null===c&&null!==i.sDefaultContent)c=
-i.sDefaultContent;else if("function"===typeof c)return c();return"display"==d&&null===c?"":c}function F(a,b,c,d){a.aoColumns[c].fnSetData(a.aoData[b]._aData,d)}function Q(a){if(null===a)return function(){return null};if("function"===typeof a)return function(b,d,i){return a(b,d,i)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("["))){var b=function(a,d,i){var f=i.split("."),g;if(""!==i){var e=0;for(g=f.length;e<g;e++){if(i=f[e].match(U)){f[e]=f[e].replace(U,"");""!==f[e]&&(a=a[f[e]]);
-g=[];f.splice(0,e+1);for(var f=f.join("."),e=0,h=a.length;e<h;e++)g.push(b(a[e],d,f));a=i[0].substring(1,i[0].length-1);a=""===a?g:g.join(a);break}if(null===a||a[f[e]]===n)return n;a=a[f[e]]}}return a};return function(c,d){return b(c,d,a)}}return function(b){return b[a]}}function L(a){if(null===a)return function(){};if("function"===typeof a)return function(b,d){a(b,"set",d)};if("string"===typeof a&&(-1!==a.indexOf(".")||-1!==a.indexOf("["))){var b=function(a,d,i){var i=i.split("."),f,g,e=0;for(g=
-i.length-1;e<g;e++){if(f=i[e].match(U)){i[e]=i[e].replace(U,"");a[i[e]]=[];f=i.slice();f.splice(0,e+1);g=f.join(".");for(var h=0,j=d.length;h<j;h++)f={},b(f,d[h],g),a[i[e]].push(f);return}if(null===a[i[e]]||a[i[e]]===n)a[i[e]]={};a=a[i[e]]}a[i[i.length-1].replace(U,"")]=d};return function(c,d){return b(c,d,a)}}return function(b,d){b[a]=d}}function Z(a){for(var b=[],c=a.aoData.length,d=0;d<c;d++)b.push(a.aoData[d]._aData);return b}function ga(a){a.aoData.splice(0,a.aoData.length);a.aiDisplayMaster.splice(0,
-a.aiDisplayMaster.length);a.aiDisplay.splice(0,a.aiDisplay.length);y(a)}function ha(a,b){for(var c=-1,d=0,i=a.length;d<i;d++)a[d]==b?c=d:a[d]>b&&a[d]--; -1!=c&&a.splice(c,1)}function S(a,b,c){var d=a.aoColumns[c];return d.fnRender({iDataRow:b,iDataColumn:c,oSettings:a,aData:a.aoData[b]._aData,mDataProp:d.mData},v(a,b,c,"display"))}function ea(a,b){var c=a.aoData[b],d;if(null===c.nTr){c.nTr=l.createElement("tr");c.nTr._DT_RowIndex=b;c._aData.DT_RowId&&(c.nTr.id=c._aData.DT_RowId);c._aData.DT_RowClass&&
-(c.nTr.className=c._aData.DT_RowClass);for(var i=0,f=a.aoColumns.length;i<f;i++){var g=a.aoColumns[i];d=l.createElement(g.sCellType);d.innerHTML="function"===typeof g.fnRender&&(!g.bUseRendered||null===g.mData)?S(a,b,i):v(a,b,i,"display");null!==g.sClass&&(d.className=g.sClass);g.bVisible?(c.nTr.appendChild(d),c._anHidden[i]=null):c._anHidden[i]=d;g.fnCreatedCell&&g.fnCreatedCell.call(a.oInstance,d,v(a,b,i,"display"),c._aData,b,i)}A(a,"aoRowCreatedCallback",null,[c.nTr,c._aData,b])}}function va(a){var b,
-c,d;if(0!==h("th, td",a.nTHead).length){b=0;for(d=a.aoColumns.length;b<d;b++)if(c=a.aoColumns[b].nTh,c.setAttribute("role","columnheader"),a.aoColumns[b].bSortable&&(c.setAttribute("tabindex",a.iTabIndex),c.setAttribute("aria-controls",a.sTableId)),null!==a.aoColumns[b].sClass&&h(c).addClass(a.aoColumns[b].sClass),a.aoColumns[b].sTitle!=c.innerHTML)c.innerHTML=a.aoColumns[b].sTitle}else{var i=l.createElement("tr");b=0;for(d=a.aoColumns.length;b<d;b++)c=a.aoColumns[b].nTh,c.innerHTML=a.aoColumns[b].sTitle,
-c.setAttribute("tabindex","0"),null!==a.aoColumns[b].sClass&&h(c).addClass(a.aoColumns[b].sClass),i.appendChild(c);h(a.nTHead).html("")[0].appendChild(i);V(a.aoHeader,a.nTHead)}h(a.nTHead).children("tr").attr("role","row");if(a.bJUI){b=0;for(d=a.aoColumns.length;b<d;b++){c=a.aoColumns[b].nTh;i=l.createElement("div");i.className=a.oClasses.sSortJUIWrapper;h(c).contents().appendTo(i);var f=l.createElement("span");f.className=a.oClasses.sSortIcon;i.appendChild(f);c.appendChild(i)}}if(a.oFeatures.bSort)for(b=
-0;b<a.aoColumns.length;b++)!1!==a.aoColumns[b].bSortable?ia(a,a.aoColumns[b].nTh,b):h(a.aoColumns[b].nTh).addClass(a.oClasses.sSortableNone);""!==a.oClasses.sFooterTH&&h(a.nTFoot).children("tr").children("th").addClass(a.oClasses.sFooterTH);if(null!==a.nTFoot){c=N(a,null,a.aoFooter);b=0;for(d=a.aoColumns.length;b<d;b++)c[b]&&(a.aoColumns[b].nTf=c[b],a.aoColumns[b].sClass&&h(c[b]).addClass(a.aoColumns[b].sClass))}}function W(a,b,c){var d,i,f,g=[],e=[],h=a.aoColumns.length,j;c===n&&(c=!1);d=0;for(i=
-b.length;d<i;d++){g[d]=b[d].slice();g[d].nTr=b[d].nTr;for(f=h-1;0<=f;f--)!a.aoColumns[f].bVisible&&!c&&g[d].splice(f,1);e.push([])}d=0;for(i=g.length;d<i;d++){if(a=g[d].nTr)for(;f=a.firstChild;)a.removeChild(f);f=0;for(b=g[d].length;f<b;f++)if(j=h=1,e[d][f]===n){a.appendChild(g[d][f].cell);for(e[d][f]=1;g[d+h]!==n&&g[d][f].cell==g[d+h][f].cell;)e[d+h][f]=1,h++;for(;g[d][f+j]!==n&&g[d][f].cell==g[d][f+j].cell;){for(c=0;c<h;c++)e[d+c][f+j]=1;j++}g[d][f].cell.rowSpan=h;g[d][f].cell.colSpan=j}}}function x(a){var b=
-A(a,"aoPreDrawCallback","preDraw",[a]);if(-1!==h.inArray(!1,b))E(a,!1);else{var c,d,b=[],i=0,f=a.asStripeClasses.length;c=a.aoOpenRows.length;a.bDrawing=!0;a.iInitDisplayStart!==n&&-1!=a.iInitDisplayStart&&(a._iDisplayStart=a.oFeatures.bServerSide?a.iInitDisplayStart:a.iInitDisplayStart>=a.fnRecordsDisplay()?0:a.iInitDisplayStart,a.iInitDisplayStart=-1,y(a));if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++;else if(a.oFeatures.bServerSide){if(!a.bDestroying&&!wa(a))return}else a.iDraw++;if(0!==a.aiDisplay.length){var g=
-a._iDisplayStart;d=a._iDisplayEnd;a.oFeatures.bServerSide&&(g=0,d=a.aoData.length);for(;g<d;g++){var e=a.aoData[a.aiDisplay[g]];null===e.nTr&&ea(a,a.aiDisplay[g]);var j=e.nTr;if(0!==f){var o=a.asStripeClasses[i%f];e._sRowStripe!=o&&(h(j).removeClass(e._sRowStripe).addClass(o),e._sRowStripe=o)}A(a,"aoRowCallback",null,[j,a.aoData[a.aiDisplay[g]]._aData,i,g]);b.push(j);i++;if(0!==c)for(e=0;e<c;e++)if(j==a.aoOpenRows[e].nParent){b.push(a.aoOpenRows[e].nTr);break}}}else b[0]=l.createElement("tr"),a.asStripeClasses[0]&&
-(b[0].className=a.asStripeClasses[0]),c=a.oLanguage,f=c.sZeroRecords,1==a.iDraw&&null!==a.sAjaxSource&&!a.oFeatures.bServerSide?f=c.sLoadingRecords:c.sEmptyTable&&0===a.fnRecordsTotal()&&(f=c.sEmptyTable),c=l.createElement("td"),c.setAttribute("valign","top"),c.colSpan=t(a),c.className=a.oClasses.sRowEmpty,c.innerHTML=ja(a,f),b[i].appendChild(c);A(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Z(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);A(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],
-Z(a),a._iDisplayStart,a.fnDisplayEnd(),a.aiDisplay]);i=l.createDocumentFragment();c=l.createDocumentFragment();if(a.nTBody){f=a.nTBody.parentNode;c.appendChild(a.nTBody);if(!a.oScroll.bInfinite||!a._bInitComplete||a.bSorted||a.bFiltered)for(;c=a.nTBody.firstChild;)a.nTBody.removeChild(c);c=0;for(d=b.length;c<d;c++)i.appendChild(b[c]);a.nTBody.appendChild(i);null!==f&&f.appendChild(a.nTBody)}A(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1;a.oFeatures.bServerSide&&(E(a,!1),
-a._bInitComplete||$(a))}}function aa(a){a.oFeatures.bSort?O(a,a.oPreviousSearch):a.oFeatures.bFilter?K(a,a.oPreviousSearch):(y(a),x(a))}function xa(a){var b=h("<div></div>")[0];a.nTable.parentNode.insertBefore(b,a.nTable);a.nTableWrapper=h('<div id="'+a.sTableId+'_wrapper" class="'+a.oClasses.sWrapper+'" role="grid"></div>')[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var c=a.nTableWrapper,d=a.sDom.split(""),i,f,g,e,w,o,k,m=0;m<d.length;m++){f=0;g=d[m];if("<"==g){e=h("<div></div>")[0];w=d[m+
-1];if("'"==w||'"'==w){o="";for(k=2;d[m+k]!=w;)o+=d[m+k],k++;"H"==o?o=a.oClasses.sJUIHeader:"F"==o&&(o=a.oClasses.sJUIFooter);-1!=o.indexOf(".")?(w=o.split("."),e.id=w[0].substr(1,w[0].length-1),e.className=w[1]):"#"==o.charAt(0)?e.id=o.substr(1,o.length-1):e.className=o;m+=k}c.appendChild(e);c=e}else if(">"==g)c=c.parentNode;else if("l"==g&&a.oFeatures.bPaginate&&a.oFeatures.bLengthChange)i=ya(a),f=1;else if("f"==g&&a.oFeatures.bFilter)i=za(a),f=1;else if("r"==g&&a.oFeatures.bProcessing)i=Aa(a),f=
-1;else if("t"==g)i=Ba(a),f=1;else if("i"==g&&a.oFeatures.bInfo)i=Ca(a),f=1;else if("p"==g&&a.oFeatures.bPaginate)i=Da(a),f=1;else if(0!==j.ext.aoFeatures.length){e=j.ext.aoFeatures;k=0;for(w=e.length;k<w;k++)if(g==e[k].cFeature){(i=e[k].fnInit(a))&&(f=1);break}}1==f&&null!==i&&("object"!==typeof a.aanFeatures[g]&&(a.aanFeatures[g]=[]),a.aanFeatures[g].push(i),c.appendChild(i))}b.parentNode.replaceChild(a.nTableWrapper,b)}function V(a,b){var c=h(b).children("tr"),d,i,f,g,e,j,o,k,m,p;a.splice(0,a.length);
-f=0;for(j=c.length;f<j;f++)a.push([]);f=0;for(j=c.length;f<j;f++){d=c[f];for(i=d.firstChild;i;){if("TD"==i.nodeName.toUpperCase()||"TH"==i.nodeName.toUpperCase()){k=1*i.getAttribute("colspan");m=1*i.getAttribute("rowspan");k=!k||0===k||1===k?1:k;m=!m||0===m||1===m?1:m;g=0;for(e=a[f];e[g];)g++;o=g;p=1===k?!0:!1;for(e=0;e<k;e++)for(g=0;g<m;g++)a[f+g][o+e]={cell:i,unique:p},a[f+g].nTr=d}i=i.nextSibling}}}function N(a,b,c){var d=[];c||(c=a.aoHeader,b&&(c=[],V(c,b)));for(var b=0,i=c.length;b<i;b++)for(var f=
-0,g=c[b].length;f<g;f++)if(c[b][f].unique&&(!d[f]||!a.bSortCellsTop))d[f]=c[b][f].cell;return d}function wa(a){if(a.bAjaxDataGet){a.iDraw++;E(a,!0);var b=Ea(a);ka(a,b);a.fnServerData.call(a.oInstance,a.sAjaxSource,b,function(b){Fa(a,b)},a);return!1}return!0}function Ea(a){var b=a.aoColumns.length,c=[],d,i,f,g;c.push({name:"sEcho",value:a.iDraw});c.push({name:"iColumns",value:b});c.push({name:"sColumns",value:M(a)});c.push({name:"iDisplayStart",value:a._iDisplayStart});c.push({name:"iDisplayLength",
-value:!1!==a.oFeatures.bPaginate?a._iDisplayLength:-1});for(f=0;f<b;f++)d=a.aoColumns[f].mData,c.push({name:"mDataProp_"+f,value:"function"===typeof d?"function":d});if(!1!==a.oFeatures.bFilter){c.push({name:"sSearch",value:a.oPreviousSearch.sSearch});c.push({name:"bRegex",value:a.oPreviousSearch.bRegex});for(f=0;f<b;f++)c.push({name:"sSearch_"+f,value:a.aoPreSearchCols[f].sSearch}),c.push({name:"bRegex_"+f,value:a.aoPreSearchCols[f].bRegex}),c.push({name:"bSearchable_"+f,value:a.aoColumns[f].bSearchable})}if(!1!==
-a.oFeatures.bSort){var e=0;d=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(f=0;f<d.length;f++){i=a.aoColumns[d[f][0]].aDataSort;for(g=0;g<i.length;g++)c.push({name:"iSortCol_"+e,value:i[g]}),c.push({name:"sSortDir_"+e,value:d[f][1]}),e++}c.push({name:"iSortingCols",value:e});for(f=0;f<b;f++)c.push({name:"bSortable_"+f,value:a.aoColumns[f].bSortable})}return c}function ka(a,b){A(a,"aoServerParams","serverParams",[b])}function Fa(a,b){if(b.sEcho!==n){if(1*b.sEcho<
-a.iDraw)return;a.iDraw=1*b.sEcho}(!a.oScroll.bInfinite||a.oScroll.bInfinite&&(a.bSorted||a.bFiltered))&&ga(a);a._iRecordsTotal=parseInt(b.iTotalRecords,10);a._iRecordsDisplay=parseInt(b.iTotalDisplayRecords,10);var c=M(a),c=b.sColumns!==n&&""!==c&&b.sColumns!=c,d;c&&(d=u(a,b.sColumns));for(var i=Q(a.sAjaxDataProp)(b),f=0,g=i.length;f<g;f++)if(c){for(var e=[],h=0,j=a.aoColumns.length;h<j;h++)e.push(i[f][d[h]]);H(a,e)}else H(a,i[f]);a.aiDisplay=a.aiDisplayMaster.slice();a.bAjaxDataGet=!1;x(a);a.bAjaxDataGet=
-!0;E(a,!1)}function za(a){var b=a.oPreviousSearch,c=a.oLanguage.sSearch,c=-1!==c.indexOf("_INPUT_")?c.replace("_INPUT_",'<input type="text" />'):""===c?'<input type="text" />':c+' <input type="text" />',d=l.createElement("div");d.className=a.oClasses.sFilter;d.innerHTML="<label>"+c+"</label>";a.aanFeatures.f||(d.id=a.sTableId+"_filter");c=h('input[type="text"]',d);d._DT_Input=c[0];c.val(b.sSearch.replace('"',"""));c.bind("keyup.DT",function(){for(var c=a.aanFeatures.f,d=this.value===""?"":this.value,
-g=0,e=c.length;g<e;g++)c[g]!=h(this).parents("div.dataTables_filter")[0]&&h(c[g]._DT_Input).val(d);d!=b.sSearch&&K(a,{sSearch:d,bRegex:b.bRegex,bSmart:b.bSmart,bCaseInsensitive:b.bCaseInsensitive})});c.attr("aria-controls",a.sTableId).bind("keypress.DT",function(a){if(a.keyCode==13)return false});return d}function K(a,b,c){var d=a.oPreviousSearch,i=a.aoPreSearchCols,f=function(a){d.sSearch=a.sSearch;d.bRegex=a.bRegex;d.bSmart=a.bSmart;d.bCaseInsensitive=a.bCaseInsensitive};if(a.oFeatures.bServerSide)f(b);
-else{Ga(a,b.sSearch,c,b.bRegex,b.bSmart,b.bCaseInsensitive);f(b);for(b=0;b<a.aoPreSearchCols.length;b++)Ha(a,i[b].sSearch,b,i[b].bRegex,i[b].bSmart,i[b].bCaseInsensitive);Ia(a)}a.bFiltered=!0;h(a.oInstance).trigger("filter",a);a._iDisplayStart=0;y(a);x(a);la(a,0)}function Ia(a){for(var b=j.ext.afnFiltering,c=r(a,"bSearchable"),d=0,i=b.length;d<i;d++)for(var f=0,g=0,e=a.aiDisplay.length;g<e;g++){var h=a.aiDisplay[g-f];b[d](a,Y(a,h,"filter",c),h)||(a.aiDisplay.splice(g-f,1),f++)}}function Ha(a,b,c,
-d,i,f){if(""!==b)for(var g=0,b=ma(b,d,i,f),d=a.aiDisplay.length-1;0<=d;d--)i=Ja(v(a,a.aiDisplay[d],c,"filter"),a.aoColumns[c].sType),b.test(i)||(a.aiDisplay.splice(d,1),g++)}function Ga(a,b,c,d,i,f){d=ma(b,d,i,f);i=a.oPreviousSearch;c||(c=0);0!==j.ext.afnFiltering.length&&(c=1);if(0>=b.length)a.aiDisplay.splice(0,a.aiDisplay.length),a.aiDisplay=a.aiDisplayMaster.slice();else if(a.aiDisplay.length==a.aiDisplayMaster.length||i.sSearch.length>b.length||1==c||0!==b.indexOf(i.sSearch)){a.aiDisplay.splice(0,
-a.aiDisplay.length);la(a,1);for(b=0;b<a.aiDisplayMaster.length;b++)d.test(a.asDataSearch[b])&&a.aiDisplay.push(a.aiDisplayMaster[b])}else for(b=c=0;b<a.asDataSearch.length;b++)d.test(a.asDataSearch[b])||(a.aiDisplay.splice(b-c,1),c++)}function la(a,b){if(!a.oFeatures.bServerSide){a.asDataSearch=[];for(var c=r(a,"bSearchable"),d=1===b?a.aiDisplayMaster:a.aiDisplay,i=0,f=d.length;i<f;i++)a.asDataSearch[i]=na(a,Y(a,d[i],"filter",c))}}function na(a,b){var c=b.join(" ");-1!==c.indexOf("&")&&(c=h("<div>").html(c).text());
-return c.replace(/[\n\r]/g," ")}function ma(a,b,c,d){if(c)return a=b?a.split(" "):oa(a).split(" "),a="^(?=.*?"+a.join(")(?=.*?")+").*$",RegExp(a,d?"i":"");a=b?a:oa(a);return RegExp(a,d?"i":"")}function Ja(a,b){return"function"===typeof j.ext.ofnSearch[b]?j.ext.ofnSearch[b](a):null===a?"":"html"==b?a.replace(/[\r\n]/g," ").replace(/<.*?>/g,""):"string"===typeof a?a.replace(/[\r\n]/g," "):a}function oa(a){return a.replace(RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),
-"\\$1")}function Ca(a){var b=l.createElement("div");b.className=a.oClasses.sInfo;a.aanFeatures.i||(a.aoDrawCallback.push({fn:Ka,sName:"information"}),b.id=a.sTableId+"_info");a.nTable.setAttribute("aria-describedby",a.sTableId+"_info");return b}function Ka(a){if(a.oFeatures.bInfo&&0!==a.aanFeatures.i.length){var b=a.oLanguage,c=a._iDisplayStart+1,d=a.fnDisplayEnd(),i=a.fnRecordsTotal(),f=a.fnRecordsDisplay(),g;g=0===f?b.sInfoEmpty:b.sInfo;f!=i&&(g+=" "+b.sInfoFiltered);g+=b.sInfoPostFix;g=ja(a,g);
-null!==b.fnInfoCallback&&(g=b.fnInfoCallback.call(a.oInstance,a,c,d,i,f,g));a=a.aanFeatures.i;b=0;for(c=a.length;b<c;b++)h(a[b]).html(g)}}function ja(a,b){var c=a.fnFormatNumber(a._iDisplayStart+1),d=a.fnDisplayEnd(),d=a.fnFormatNumber(d),i=a.fnRecordsDisplay(),i=a.fnFormatNumber(i),f=a.fnRecordsTotal(),f=a.fnFormatNumber(f);a.oScroll.bInfinite&&(c=a.fnFormatNumber(1));return b.replace(/_START_/g,c).replace(/_END_/g,d).replace(/_TOTAL_/g,i).replace(/_MAX_/g,f)}function ba(a){var b,c,d=a.iInitDisplayStart;
-if(!1===a.bInitialised)setTimeout(function(){ba(a)},200);else{xa(a);va(a);W(a,a.aoHeader);a.nTFoot&&W(a,a.aoFooter);E(a,!0);a.oFeatures.bAutoWidth&&da(a);b=0;for(c=a.aoColumns.length;b<c;b++)null!==a.aoColumns[b].sWidth&&(a.aoColumns[b].nTh.style.width=q(a.aoColumns[b].sWidth));a.oFeatures.bSort?O(a):a.oFeatures.bFilter?K(a,a.oPreviousSearch):(a.aiDisplay=a.aiDisplayMaster.slice(),y(a),x(a));null!==a.sAjaxSource&&!a.oFeatures.bServerSide?(c=[],ka(a,c),a.fnServerData.call(a.oInstance,a.sAjaxSource,
-c,function(c){var f=a.sAjaxDataProp!==""?Q(a.sAjaxDataProp)(c):c;for(b=0;b<f.length;b++)H(a,f[b]);a.iInitDisplayStart=d;if(a.oFeatures.bSort)O(a);else{a.aiDisplay=a.aiDisplayMaster.slice();y(a);x(a)}E(a,false);$(a,c)},a)):a.oFeatures.bServerSide||(E(a,!1),$(a))}}function $(a,b){a._bInitComplete=!0;A(a,"aoInitComplete","init",[a,b])}function pa(a){var b=j.defaults.oLanguage;!a.sEmptyTable&&(a.sZeroRecords&&"No data available in table"===b.sEmptyTable)&&p(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&
-(a.sZeroRecords&&"Loading..."===b.sLoadingRecords)&&p(a,a,"sZeroRecords","sLoadingRecords")}function ya(a){if(a.oScroll.bInfinite)return null;var b='<select size="1" '+('name="'+a.sTableId+'_length"')+">",c,d,i=a.aLengthMenu;if(2==i.length&&"object"===typeof i[0]&&"object"===typeof i[1]){c=0;for(d=i[0].length;c<d;c++)b+='<option value="'+i[0][c]+'">'+i[1][c]+"</option>"}else{c=0;for(d=i.length;c<d;c++)b+='<option value="'+i[c]+'">'+i[c]+"</option>"}b+="</select>";i=l.createElement("div");a.aanFeatures.l||
-(i.id=a.sTableId+"_length");i.className=a.oClasses.sLength;i.innerHTML="<label>"+a.oLanguage.sLengthMenu.replace("_MENU_",b)+"</label>";h('select option[value="'+a._iDisplayLength+'"]',i).attr("selected",!0);h("select",i).bind("change.DT",function(){var b=h(this).val(),i=a.aanFeatures.l;c=0;for(d=i.length;c<d;c++)i[c]!=this.parentNode&&h("select",i[c]).val(b);a._iDisplayLength=parseInt(b,10);y(a);if(a.fnDisplayEnd()==a.fnRecordsDisplay()){a._iDisplayStart=a.fnDisplayEnd()-a._iDisplayLength;if(a._iDisplayStart<
-0)a._iDisplayStart=0}if(a._iDisplayLength==-1)a._iDisplayStart=0;x(a)});h("select",i).attr("aria-controls",a.sTableId);return i}function y(a){a._iDisplayEnd=!1===a.oFeatures.bPaginate?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength>a.aiDisplay.length||-1==a._iDisplayLength?a.aiDisplay.length:a._iDisplayStart+a._iDisplayLength}function Da(a){if(a.oScroll.bInfinite)return null;var b=l.createElement("div");b.className=a.oClasses.sPaging+a.sPaginationType;j.ext.oPagination[a.sPaginationType].fnInit(a,
-b,function(a){y(a);x(a)});a.aanFeatures.p||a.aoDrawCallback.push({fn:function(a){j.ext.oPagination[a.sPaginationType].fnUpdate(a,function(a){y(a);x(a)})},sName:"pagination"});return b}function qa(a,b){var c=a._iDisplayStart;if("number"===typeof b)a._iDisplayStart=b*a._iDisplayLength,a._iDisplayStart>a.fnRecordsDisplay()&&(a._iDisplayStart=0);else if("first"==b)a._iDisplayStart=0;else if("previous"==b)a._iDisplayStart=0<=a._iDisplayLength?a._iDisplayStart-a._iDisplayLength:0,0>a._iDisplayStart&&(a._iDisplayStart=
-0);else if("next"==b)0<=a._iDisplayLength?a._iDisplayStart+a._iDisplayLength<a.fnRecordsDisplay()&&(a._iDisplayStart+=a._iDisplayLength):a._iDisplayStart=0;else if("last"==b)if(0<=a._iDisplayLength){var d=parseInt((a.fnRecordsDisplay()-1)/a._iDisplayLength,10)+1;a._iDisplayStart=(d-1)*a._iDisplayLength}else a._iDisplayStart=0;else D(a,0,"Unknown paging action: "+b);h(a.oInstance).trigger("page",a);return c!=a._iDisplayStart}function Aa(a){var b=l.createElement("div");a.aanFeatures.r||(b.id=a.sTableId+
-"_processing");b.innerHTML=a.oLanguage.sProcessing;b.className=a.oClasses.sProcessing;a.nTable.parentNode.insertBefore(b,a.nTable);return b}function E(a,b){if(a.oFeatures.bProcessing)for(var c=a.aanFeatures.r,d=0,i=c.length;d<i;d++)c[d].style.visibility=b?"visible":"hidden";h(a.oInstance).trigger("processing",[a,b])}function Ba(a){if(""===a.oScroll.sX&&""===a.oScroll.sY)return a.nTable;var b=l.createElement("div"),c=l.createElement("div"),d=l.createElement("div"),i=l.createElement("div"),f=l.createElement("div"),
-g=l.createElement("div"),e=a.nTable.cloneNode(!1),j=a.nTable.cloneNode(!1),o=a.nTable.getElementsByTagName("thead")[0],k=0===a.nTable.getElementsByTagName("tfoot").length?null:a.nTable.getElementsByTagName("tfoot")[0],m=a.oClasses;c.appendChild(d);f.appendChild(g);i.appendChild(a.nTable);b.appendChild(c);b.appendChild(i);d.appendChild(e);e.appendChild(o);null!==k&&(b.appendChild(f),g.appendChild(j),j.appendChild(k));b.className=m.sScrollWrapper;c.className=m.sScrollHead;d.className=m.sScrollHeadInner;
-i.className=m.sScrollBody;f.className=m.sScrollFoot;g.className=m.sScrollFootInner;a.oScroll.bAutoCss&&(c.style.overflow="hidden",c.style.position="relative",f.style.overflow="hidden",i.style.overflow="auto");c.style.border="0";c.style.width="100%";f.style.border="0";d.style.width=""!==a.oScroll.sXInner?a.oScroll.sXInner:"100%";e.removeAttribute("id");e.style.marginLeft="0";a.nTable.style.marginLeft="0";null!==k&&(j.removeAttribute("id"),j.style.marginLeft="0");d=h(a.nTable).children("caption");0<
-d.length&&(d=d[0],"top"===d._captionSide?e.appendChild(d):"bottom"===d._captionSide&&k&&j.appendChild(d));""!==a.oScroll.sX&&(c.style.width=q(a.oScroll.sX),i.style.width=q(a.oScroll.sX),null!==k&&(f.style.width=q(a.oScroll.sX)),h(i).scroll(function(){c.scrollLeft=this.scrollLeft;if(k!==null)f.scrollLeft=this.scrollLeft}));""!==a.oScroll.sY&&(i.style.height=q(a.oScroll.sY));a.aoDrawCallback.push({fn:La,sName:"scrolling"});a.oScroll.bInfinite&&h(i).scroll(function(){if(!a.bDrawing&&h(this).scrollTop()!==
-0&&h(this).scrollTop()+h(this).height()>h(a.nTable).height()-a.oScroll.iLoadGap&&a.fnDisplayEnd()<a.fnRecordsDisplay()){qa(a,"next");y(a);x(a)}});a.nScrollHead=c;a.nScrollFoot=f;return b}function La(a){var b=a.nScrollHead.getElementsByTagName("div")[0],c=b.getElementsByTagName("table")[0],d=a.nTable.parentNode,i,f,g,e,j,o,k,m,p=[],n=[],l=null!==a.nTFoot?a.nScrollFoot.getElementsByTagName("div")[0]:null,R=null!==a.nTFoot?l.getElementsByTagName("table")[0]:null,r=a.oBrowser.bScrollOversize,s=function(a){k=
-a.style;k.paddingTop="0";k.paddingBottom="0";k.borderTopWidth="0";k.borderBottomWidth="0";k.height=0};h(a.nTable).children("thead, tfoot").remove();i=h(a.nTHead).clone()[0];a.nTable.insertBefore(i,a.nTable.childNodes[0]);g=a.nTHead.getElementsByTagName("tr");e=i.getElementsByTagName("tr");null!==a.nTFoot&&(j=h(a.nTFoot).clone()[0],a.nTable.insertBefore(j,a.nTable.childNodes[1]),o=a.nTFoot.getElementsByTagName("tr"),j=j.getElementsByTagName("tr"));""===a.oScroll.sX&&(d.style.width="100%",b.parentNode.style.width=
-"100%");var t=N(a,i);i=0;for(f=t.length;i<f;i++)m=G(a,i),t[i].style.width=a.aoColumns[m].sWidth;null!==a.nTFoot&&C(function(a){a.style.width=""},j);a.oScroll.bCollapse&&""!==a.oScroll.sY&&(d.style.height=d.offsetHeight+a.nTHead.offsetHeight+"px");i=h(a.nTable).outerWidth();if(""===a.oScroll.sX){if(a.nTable.style.width="100%",r&&(h("tbody",d).height()>d.offsetHeight||"scroll"==h(d).css("overflow-y")))a.nTable.style.width=q(h(a.nTable).outerWidth()-a.oScroll.iBarWidth)}else""!==a.oScroll.sXInner?a.nTable.style.width=
-q(a.oScroll.sXInner):i==h(d).width()&&h(d).height()<h(a.nTable).height()?(a.nTable.style.width=q(i-a.oScroll.iBarWidth),h(a.nTable).outerWidth()>i-a.oScroll.iBarWidth&&(a.nTable.style.width=q(i))):a.nTable.style.width=q(i);i=h(a.nTable).outerWidth();C(s,e);C(function(a){p.push(q(h(a).width()))},e);C(function(a,b){a.style.width=p[b]},g);h(e).height(0);null!==a.nTFoot&&(C(s,j),C(function(a){n.push(q(h(a).width()))},j),C(function(a,b){a.style.width=n[b]},o),h(j).height(0));C(function(a,b){a.innerHTML=
-"";a.style.width=p[b]},e);null!==a.nTFoot&&C(function(a,b){a.innerHTML="";a.style.width=n[b]},j);if(h(a.nTable).outerWidth()<i){g=d.scrollHeight>d.offsetHeight||"scroll"==h(d).css("overflow-y")?i+a.oScroll.iBarWidth:i;if(r&&(d.scrollHeight>d.offsetHeight||"scroll"==h(d).css("overflow-y")))a.nTable.style.width=q(g-a.oScroll.iBarWidth);d.style.width=q(g);a.nScrollHead.style.width=q(g);null!==a.nTFoot&&(a.nScrollFoot.style.width=q(g));""===a.oScroll.sX?D(a,1,"The table cannot fit into the current element which will cause column misalignment. The table has been drawn at its minimum possible width."):
-""!==a.oScroll.sXInner&&D(a,1,"The table cannot fit into the current element which will cause column misalignment. Increase the sScrollXInner value or remove it to allow automatic calculation")}else d.style.width=q("100%"),a.nScrollHead.style.width=q("100%"),null!==a.nTFoot&&(a.nScrollFoot.style.width=q("100%"));""===a.oScroll.sY&&r&&(d.style.height=q(a.nTable.offsetHeight+a.oScroll.iBarWidth));""!==a.oScroll.sY&&a.oScroll.bCollapse&&(d.style.height=q(a.oScroll.sY),r=""!==a.oScroll.sX&&a.nTable.offsetWidth>
-d.offsetWidth?a.oScroll.iBarWidth:0,a.nTable.offsetHeight<d.offsetHeight&&(d.style.height=q(a.nTable.offsetHeight+r)));r=h(a.nTable).outerWidth();c.style.width=q(r);b.style.width=q(r);c=h(a.nTable).height()>d.clientHeight||"scroll"==h(d).css("overflow-y");b.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px";null!==a.nTFoot&&(R.style.width=q(r),l.style.width=q(r),l.style.paddingRight=c?a.oScroll.iBarWidth+"px":"0px");h(d).scroll();if(a.bSorted||a.bFiltered)d.scrollTop=0}function C(a,b,c){for(var d=
-0,i=0,f=b.length,g,e;i<f;){g=b[i].firstChild;for(e=c?c[i].firstChild:null;g;)1===g.nodeType&&(c?a(g,e,d):a(g,d),d++),g=g.nextSibling,e=c?e.nextSibling:null;i++}}function Ma(a,b){if(!a||null===a||""===a)return 0;b||(b=l.body);var c,d=l.createElement("div");d.style.width=q(a);b.appendChild(d);c=d.offsetWidth;b.removeChild(d);return c}function da(a){var b=0,c,d=0,i=a.aoColumns.length,f,e,j=h("th",a.nTHead),o=a.nTable.getAttribute("width");e=a.nTable.parentNode;for(f=0;f<i;f++)a.aoColumns[f].bVisible&&
-(d++,null!==a.aoColumns[f].sWidth&&(c=Ma(a.aoColumns[f].sWidthOrig,e),null!==c&&(a.aoColumns[f].sWidth=q(c)),b++));if(i==j.length&&0===b&&d==i&&""===a.oScroll.sX&&""===a.oScroll.sY)for(f=0;f<a.aoColumns.length;f++)c=h(j[f]).width(),null!==c&&(a.aoColumns[f].sWidth=q(c));else{b=a.nTable.cloneNode(!1);f=a.nTHead.cloneNode(!0);d=l.createElement("tbody");c=l.createElement("tr");b.removeAttribute("id");b.appendChild(f);null!==a.nTFoot&&(b.appendChild(a.nTFoot.cloneNode(!0)),C(function(a){a.style.width=
-""},b.getElementsByTagName("tr")));b.appendChild(d);d.appendChild(c);d=h("thead th",b);0===d.length&&(d=h("tbody tr:eq(0)>td",b));j=N(a,f);for(f=d=0;f<i;f++){var k=a.aoColumns[f];k.bVisible&&null!==k.sWidthOrig&&""!==k.sWidthOrig?j[f-d].style.width=q(k.sWidthOrig):k.bVisible?j[f-d].style.width="":d++}for(f=0;f<i;f++)a.aoColumns[f].bVisible&&(d=Na(a,f),null!==d&&(d=d.cloneNode(!0),""!==a.aoColumns[f].sContentPadding&&(d.innerHTML+=a.aoColumns[f].sContentPadding),c.appendChild(d)));e.appendChild(b);
-""!==a.oScroll.sX&&""!==a.oScroll.sXInner?b.style.width=q(a.oScroll.sXInner):""!==a.oScroll.sX?(b.style.width="",h(b).width()<e.offsetWidth&&(b.style.width=q(e.offsetWidth))):""!==a.oScroll.sY?b.style.width=q(e.offsetWidth):o&&(b.style.width=q(o));b.style.visibility="hidden";Oa(a,b);i=h("tbody tr:eq(0)",b).children();0===i.length&&(i=N(a,h("thead",b)[0]));if(""!==a.oScroll.sX){for(f=d=e=0;f<a.aoColumns.length;f++)a.aoColumns[f].bVisible&&(e=null===a.aoColumns[f].sWidthOrig?e+h(i[d]).outerWidth():
-e+(parseInt(a.aoColumns[f].sWidth.replace("px",""),10)+(h(i[d]).outerWidth()-h(i[d]).width())),d++);b.style.width=q(e);a.nTable.style.width=q(e)}for(f=d=0;f<a.aoColumns.length;f++)a.aoColumns[f].bVisible&&(e=h(i[d]).width(),null!==e&&0<e&&(a.aoColumns[f].sWidth=q(e)),d++);i=h(b).css("width");a.nTable.style.width=-1!==i.indexOf("%")?i:q(h(b).outerWidth());b.parentNode.removeChild(b)}o&&(a.nTable.style.width=q(o))}function Oa(a,b){""===a.oScroll.sX&&""!==a.oScroll.sY?(h(b).width(),b.style.width=q(h(b).outerWidth()-
-a.oScroll.iBarWidth)):""!==a.oScroll.sX&&(b.style.width=q(h(b).outerWidth()))}function Na(a,b){var c=Pa(a,b);if(0>c)return null;if(null===a.aoData[c].nTr){var d=l.createElement("td");d.innerHTML=v(a,c,b,"");return d}return J(a,c)[b]}function Pa(a,b){for(var c=-1,d=-1,i=0;i<a.aoData.length;i++){var e=v(a,i,b,"display")+"",e=e.replace(/<.*?>/g,"");e.length>c&&(c=e.length,d=i)}return d}function q(a){if(null===a)return"0px";if("number"==typeof a)return 0>a?"0px":a+"px";var b=a.charCodeAt(a.length-1);
-return 48>b||57<b?a:a+"px"}function Qa(){var a=l.createElement("p"),b=a.style;b.width="100%";b.height="200px";b.padding="0px";var c=l.createElement("div"),b=c.style;b.position="absolute";b.top="0px";b.left="0px";b.visibility="hidden";b.width="200px";b.height="150px";b.padding="0px";b.overflow="hidden";c.appendChild(a);l.body.appendChild(c);b=a.offsetWidth;c.style.overflow="scroll";a=a.offsetWidth;b==a&&(a=c.clientWidth);l.body.removeChild(c);return b-a}function O(a,b){var c,d,i,e,g,k,o=[],m=[],p=
-j.ext.oSort,l=a.aoData,q=a.aoColumns,G=a.oLanguage.oAria;if(!a.oFeatures.bServerSide&&(0!==a.aaSorting.length||null!==a.aaSortingFixed)){o=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(c=0;c<o.length;c++)if(d=o[c][0],i=R(a,d),e=a.aoColumns[d].sSortDataType,j.ext.afnSortData[e])if(g=j.ext.afnSortData[e].call(a.oInstance,a,d,i),g.length===l.length){i=0;for(e=l.length;i<e;i++)F(a,i,d,g[i])}else D(a,0,"Returned data sort array (col "+d+") is the wrong length");c=
-0;for(d=a.aiDisplayMaster.length;c<d;c++)m[a.aiDisplayMaster[c]]=c;var r=o.length,s;c=0;for(d=l.length;c<d;c++)for(i=0;i<r;i++){s=q[o[i][0]].aDataSort;g=0;for(k=s.length;g<k;g++)e=q[s[g]].sType,e=p[(e?e:"string")+"-pre"],l[c]._aSortData[s[g]]=e?e(v(a,c,s[g],"sort")):v(a,c,s[g],"sort")}a.aiDisplayMaster.sort(function(a,b){var c,d,e,i,f;for(c=0;c<r;c++){f=q[o[c][0]].aDataSort;d=0;for(e=f.length;d<e;d++)if(i=q[f[d]].sType,i=p[(i?i:"string")+"-"+o[c][1]](l[a]._aSortData[f[d]],l[b]._aSortData[f[d]]),0!==
-i)return i}return p["numeric-asc"](m[a],m[b])})}(b===n||b)&&!a.oFeatures.bDeferRender&&P(a);c=0;for(d=a.aoColumns.length;c<d;c++)e=q[c].sTitle.replace(/<.*?>/g,""),i=q[c].nTh,i.removeAttribute("aria-sort"),i.removeAttribute("aria-label"),q[c].bSortable?0<o.length&&o[0][0]==c?(i.setAttribute("aria-sort","asc"==o[0][1]?"ascending":"descending"),i.setAttribute("aria-label",e+("asc"==(q[c].asSorting[o[0][2]+1]?q[c].asSorting[o[0][2]+1]:q[c].asSorting[0])?G.sSortAscending:G.sSortDescending))):i.setAttribute("aria-label",
-e+("asc"==q[c].asSorting[0]?G.sSortAscending:G.sSortDescending)):i.setAttribute("aria-label",e);a.bSorted=!0;h(a.oInstance).trigger("sort",a);a.oFeatures.bFilter?K(a,a.oPreviousSearch,1):(a.aiDisplay=a.aiDisplayMaster.slice(),a._iDisplayStart=0,y(a),x(a))}function ia(a,b,c,d){Ra(b,{},function(b){if(!1!==a.aoColumns[c].bSortable){var e=function(){var d,e;if(b.shiftKey){for(var f=!1,h=0;h<a.aaSorting.length;h++)if(a.aaSorting[h][0]==c){f=!0;d=a.aaSorting[h][0];e=a.aaSorting[h][2]+1;a.aoColumns[d].asSorting[e]?
-(a.aaSorting[h][1]=a.aoColumns[d].asSorting[e],a.aaSorting[h][2]=e):a.aaSorting.splice(h,1);break}!1===f&&a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0])}else 1==a.aaSorting.length&&a.aaSorting[0][0]==c?(d=a.aaSorting[0][0],e=a.aaSorting[0][2]+1,a.aoColumns[d].asSorting[e]||(e=0),a.aaSorting[0][1]=a.aoColumns[d].asSorting[e],a.aaSorting[0][2]=e):(a.aaSorting.splice(0,a.aaSorting.length),a.aaSorting.push([c,a.aoColumns[c].asSorting[0],0]));O(a)};a.oFeatures.bProcessing?(E(a,!0),setTimeout(function(){e();
-a.oFeatures.bServerSide||E(a,!1)},0)):e();"function"==typeof d&&d(a)}})}function P(a){var b,c,d,e,f,g=a.aoColumns.length,j=a.oClasses;for(b=0;b<g;b++)a.aoColumns[b].bSortable&&h(a.aoColumns[b].nTh).removeClass(j.sSortAsc+" "+j.sSortDesc+" "+a.aoColumns[b].sSortingClass);c=null!==a.aaSortingFixed?a.aaSortingFixed.concat(a.aaSorting):a.aaSorting.slice();for(b=0;b<a.aoColumns.length;b++)if(a.aoColumns[b].bSortable){f=a.aoColumns[b].sSortingClass;e=-1;for(d=0;d<c.length;d++)if(c[d][0]==b){f="asc"==c[d][1]?
-j.sSortAsc:j.sSortDesc;e=d;break}h(a.aoColumns[b].nTh).addClass(f);a.bJUI&&(f=h("span."+j.sSortIcon,a.aoColumns[b].nTh),f.removeClass(j.sSortJUIAsc+" "+j.sSortJUIDesc+" "+j.sSortJUI+" "+j.sSortJUIAscAllowed+" "+j.sSortJUIDescAllowed),f.addClass(-1==e?a.aoColumns[b].sSortingClassJUI:"asc"==c[e][1]?j.sSortJUIAsc:j.sSortJUIDesc))}else h(a.aoColumns[b].nTh).addClass(a.aoColumns[b].sSortingClass);f=j.sSortColumn;if(a.oFeatures.bSort&&a.oFeatures.bSortClasses){a=J(a);e=[];for(b=0;b<g;b++)e.push("");b=0;
-for(d=1;b<c.length;b++)j=parseInt(c[b][0],10),e[j]=f+d,3>d&&d++;f=RegExp(f+"[123]");var o;b=0;for(c=a.length;b<c;b++)j=b%g,d=a[b].className,o=e[j],j=d.replace(f,o),j!=d?a[b].className=h.trim(j):0<o.length&&-1==d.indexOf(o)&&(a[b].className=d+" "+o)}}function ra(a){if(a.oFeatures.bStateSave&&!a.bDestroying){var b,c;b=a.oScroll.bInfinite;var d={iCreate:(new Date).getTime(),iStart:b?0:a._iDisplayStart,iEnd:b?a._iDisplayLength:a._iDisplayEnd,iLength:a._iDisplayLength,aaSorting:h.extend(!0,[],a.aaSorting),
-oSearch:h.extend(!0,{},a.oPreviousSearch),aoSearchCols:h.extend(!0,[],a.aoPreSearchCols),abVisCols:[]};b=0;for(c=a.aoColumns.length;b<c;b++)d.abVisCols.push(a.aoColumns[b].bVisible);A(a,"aoStateSaveParams","stateSaveParams",[a,d]);a.fnStateSave.call(a.oInstance,a,d)}}function Sa(a,b){if(a.oFeatures.bStateSave){var c=a.fnStateLoad.call(a.oInstance,a);if(c){var d=A(a,"aoStateLoadParams","stateLoadParams",[a,c]);if(-1===h.inArray(!1,d)){a.oLoadedState=h.extend(!0,{},c);a._iDisplayStart=c.iStart;a.iInitDisplayStart=
-c.iStart;a._iDisplayEnd=c.iEnd;a._iDisplayLength=c.iLength;a.aaSorting=c.aaSorting.slice();a.saved_aaSorting=c.aaSorting.slice();h.extend(a.oPreviousSearch,c.oSearch);h.extend(!0,a.aoPreSearchCols,c.aoSearchCols);b.saved_aoColumns=[];for(d=0;d<c.abVisCols.length;d++)b.saved_aoColumns[d]={},b.saved_aoColumns[d].bVisible=c.abVisCols[d];A(a,"aoStateLoaded","stateLoaded",[a,c])}}}}function s(a){for(var b=0;b<j.settings.length;b++)if(j.settings[b].nTable===a)return j.settings[b];return null}function T(a){for(var b=
-[],a=a.aoData,c=0,d=a.length;c<d;c++)null!==a[c].nTr&&b.push(a[c].nTr);return b}function J(a,b){var c=[],d,e,f,g,h,j;e=0;var o=a.aoData.length;b!==n&&(e=b,o=b+1);for(f=e;f<o;f++)if(j=a.aoData[f],null!==j.nTr){e=[];for(d=j.nTr.firstChild;d;)g=d.nodeName.toLowerCase(),("td"==g||"th"==g)&&e.push(d),d=d.nextSibling;g=d=0;for(h=a.aoColumns.length;g<h;g++)a.aoColumns[g].bVisible?c.push(e[g-d]):(c.push(j._anHidden[g]),d++)}return c}function D(a,b,c){a=null===a?"DataTables warning: "+c:"DataTables warning (table id = '"+
-a.sTableId+"'): "+c;if(0===b)if("alert"==j.ext.sErrMode)alert(a);else throw Error(a);else X.console&&console.log&&console.log(a)}function p(a,b,c,d){d===n&&(d=c);b[c]!==n&&(a[d]=b[c])}function Ta(a,b){var c,d;for(d in b)b.hasOwnProperty(d)&&(c=b[d],"object"===typeof e[d]&&null!==c&&!1===h.isArray(c)?h.extend(!0,a[d],c):a[d]=c);return a}function Ra(a,b,c){h(a).bind("click.DT",b,function(b){a.blur();c(b)}).bind("keypress.DT",b,function(a){13===a.which&&c(a)}).bind("selectstart.DT",function(){return!1})}
-function z(a,b,c,d){c&&a[b].push({fn:c,sName:d})}function A(a,b,c,d){for(var b=a[b],e=[],f=b.length-1;0<=f;f--)e.push(b[f].fn.apply(a.oInstance,d));null!==c&&h(a.oInstance).trigger(c,d);return e}function Ua(a){var b=h('<div style="position:absolute; top:0; left:0; height:1px; width:1px; overflow:hidden"><div style="position:absolute; top:1px; left:1px; width:100px; overflow:scroll;"><div id="DT_BrowserTest" style="width:100%; height:10px;"></div></div></div>')[0];l.body.appendChild(b);a.oBrowser.bScrollOversize=
-100===h("#DT_BrowserTest",b)[0].offsetWidth?!0:!1;l.body.removeChild(b)}function Va(a){return function(){var b=[s(this[j.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return j.ext.oApi[a].apply(this,b)}}var U=/\[.*?\]$/,Wa=X.JSON?JSON.stringify:function(a){var b=typeof a;if("object"!==b||null===a)return"string"===b&&(a='"'+a+'"'),a+"";var c,d,e=[],f=h.isArray(a);for(c in a)d=a[c],b=typeof d,"string"===b?d='"'+d+'"':"object"===b&&null!==d&&(d=Wa(d)),e.push((f?"":'"'+c+'":')+d);return(f?
-"[":"{")+e+(f?"]":"}")};this.$=function(a,b){var c,d,e=[],f;d=s(this[j.ext.iApiIndex]);var g=d.aoData,o=d.aiDisplay,k=d.aiDisplayMaster;b||(b={});b=h.extend({},{filter:"none",order:"current",page:"all"},b);if("current"==b.page){c=d._iDisplayStart;for(d=d.fnDisplayEnd();c<d;c++)(f=g[o[c]].nTr)&&e.push(f)}else if("current"==b.order&&"none"==b.filter){c=0;for(d=k.length;c<d;c++)(f=g[k[c]].nTr)&&e.push(f)}else if("current"==b.order&&"applied"==b.filter){c=0;for(d=o.length;c<d;c++)(f=g[o[c]].nTr)&&e.push(f)}else if("original"==
-b.order&&"none"==b.filter){c=0;for(d=g.length;c<d;c++)(f=g[c].nTr)&&e.push(f)}else if("original"==b.order&&"applied"==b.filter){c=0;for(d=g.length;c<d;c++)f=g[c].nTr,-1!==h.inArray(c,o)&&f&&e.push(f)}else D(d,1,"Unknown selection options");e=h(e);c=e.filter(a);e=e.find(a);return h([].concat(h.makeArray(c),h.makeArray(e)))};this._=function(a,b){var c=[],d,e,f=this.$(a,b);d=0;for(e=f.length;d<e;d++)c.push(this.fnGetData(f[d]));return c};this.fnAddData=function(a,b){if(0===a.length)return[];var c=[],
-d,e=s(this[j.ext.iApiIndex]);if("object"===typeof a[0]&&null!==a[0])for(var f=0;f<a.length;f++){d=H(e,a[f]);if(-1==d)return c;c.push(d)}else{d=H(e,a);if(-1==d)return c;c.push(d)}e.aiDisplay=e.aiDisplayMaster.slice();(b===n||b)&&aa(e);return c};this.fnAdjustColumnSizing=function(a){var b=s(this[j.ext.iApiIndex]);k(b);a===n||a?this.fnDraw(!1):(""!==b.oScroll.sX||""!==b.oScroll.sY)&&this.oApi._fnScrollDraw(b)};this.fnClearTable=function(a){var b=s(this[j.ext.iApiIndex]);ga(b);(a===n||a)&&x(b)};this.fnClose=
-function(a){for(var b=s(this[j.ext.iApiIndex]),c=0;c<b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a)return(a=b.aoOpenRows[c].nTr.parentNode)&&a.removeChild(b.aoOpenRows[c].nTr),b.aoOpenRows.splice(c,1),0;return 1};this.fnDeleteRow=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e,f,a="object"===typeof a?I(d,a):a,g=d.aoData.splice(a,1);e=0;for(f=d.aoData.length;e<f;e++)null!==d.aoData[e].nTr&&(d.aoData[e].nTr._DT_RowIndex=e);e=h.inArray(a,d.aiDisplay);d.asDataSearch.splice(e,1);ha(d.aiDisplayMaster,
-a);ha(d.aiDisplay,a);"function"===typeof b&&b.call(this,d,g);d._iDisplayStart>=d.fnRecordsDisplay()&&(d._iDisplayStart-=d._iDisplayLength,0>d._iDisplayStart&&(d._iDisplayStart=0));if(c===n||c)y(d),x(d);return g};this.fnDestroy=function(a){var b=s(this[j.ext.iApiIndex]),c=b.nTableWrapper.parentNode,d=b.nTBody,i,f,a=a===n?!1:a;b.bDestroying=!0;A(b,"aoDestroyCallback","destroy",[b]);if(!a){i=0;for(f=b.aoColumns.length;i<f;i++)!1===b.aoColumns[i].bVisible&&this.fnSetColumnVis(i,!0)}h(b.nTableWrapper).find("*").andSelf().unbind(".DT");
-h("tbody>tr>td."+b.oClasses.sRowEmpty,b.nTable).parent().remove();b.nTable!=b.nTHead.parentNode&&(h(b.nTable).children("thead").remove(),b.nTable.appendChild(b.nTHead));b.nTFoot&&b.nTable!=b.nTFoot.parentNode&&(h(b.nTable).children("tfoot").remove(),b.nTable.appendChild(b.nTFoot));b.nTable.parentNode.removeChild(b.nTable);h(b.nTableWrapper).remove();b.aaSorting=[];b.aaSortingFixed=[];P(b);h(T(b)).removeClass(b.asStripeClasses.join(" "));h("th, td",b.nTHead).removeClass([b.oClasses.sSortable,b.oClasses.sSortableAsc,
-b.oClasses.sSortableDesc,b.oClasses.sSortableNone].join(" "));b.bJUI&&(h("th span."+b.oClasses.sSortIcon+", td span."+b.oClasses.sSortIcon,b.nTHead).remove(),h("th, td",b.nTHead).each(function(){var a=h("div."+b.oClasses.sSortJUIWrapper,this),c=a.contents();h(this).append(c);a.remove()}));!a&&b.nTableReinsertBefore?c.insertBefore(b.nTable,b.nTableReinsertBefore):a||c.appendChild(b.nTable);i=0;for(f=b.aoData.length;i<f;i++)null!==b.aoData[i].nTr&&d.appendChild(b.aoData[i].nTr);!0===b.oFeatures.bAutoWidth&&
-(b.nTable.style.width=q(b.sDestroyWidth));if(f=b.asDestroyStripes.length){a=h(d).children("tr");for(i=0;i<f;i++)a.filter(":nth-child("+f+"n + "+i+")").addClass(b.asDestroyStripes[i])}i=0;for(f=j.settings.length;i<f;i++)j.settings[i]==b&&j.settings.splice(i,1);e=b=null};this.fnDraw=function(a){var b=s(this[j.ext.iApiIndex]);!1===a?(y(b),x(b)):aa(b)};this.fnFilter=function(a,b,c,d,e,f){var g=s(this[j.ext.iApiIndex]);if(g.oFeatures.bFilter){if(c===n||null===c)c=!1;if(d===n||null===d)d=!0;if(e===n||null===
-e)e=!0;if(f===n||null===f)f=!0;if(b===n||null===b){if(K(g,{sSearch:a+"",bRegex:c,bSmart:d,bCaseInsensitive:f},1),e&&g.aanFeatures.f){b=g.aanFeatures.f;c=0;for(d=b.length;c<d;c++)try{b[c]._DT_Input!=l.activeElement&&h(b[c]._DT_Input).val(a)}catch(o){h(b[c]._DT_Input).val(a)}}}else h.extend(g.aoPreSearchCols[b],{sSearch:a+"",bRegex:c,bSmart:d,bCaseInsensitive:f}),K(g,g.oPreviousSearch,1)}};this.fnGetData=function(a,b){var c=s(this[j.ext.iApiIndex]);if(a!==n){var d=a;if("object"===typeof a){var e=a.nodeName.toLowerCase();
-"tr"===e?d=I(c,a):"td"===e&&(d=I(c,a.parentNode),b=fa(c,d,a))}return b!==n?v(c,d,b,""):c.aoData[d]!==n?c.aoData[d]._aData:null}return Z(c)};this.fnGetNodes=function(a){var b=s(this[j.ext.iApiIndex]);return a!==n?b.aoData[a]!==n?b.aoData[a].nTr:null:T(b)};this.fnGetPosition=function(a){var b=s(this[j.ext.iApiIndex]),c=a.nodeName.toUpperCase();return"TR"==c?I(b,a):"TD"==c||"TH"==c?(c=I(b,a.parentNode),a=fa(b,c,a),[c,R(b,a),a]):null};this.fnIsOpen=function(a){for(var b=s(this[j.ext.iApiIndex]),c=0;c<
-b.aoOpenRows.length;c++)if(b.aoOpenRows[c].nParent==a)return!0;return!1};this.fnOpen=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e=T(d);if(-1!==h.inArray(a,e)){this.fnClose(a);var e=l.createElement("tr"),f=l.createElement("td");e.appendChild(f);f.className=c;f.colSpan=t(d);"string"===typeof b?f.innerHTML=b:h(f).html(b);b=h("tr",d.nTBody);-1!=h.inArray(a,b)&&h(e).insertAfter(a);d.aoOpenRows.push({nTr:e,nParent:a});return e}};this.fnPageChange=function(a,b){var c=s(this[j.ext.iApiIndex]);qa(c,a);
-y(c);(b===n||b)&&x(c)};this.fnSetColumnVis=function(a,b,c){var d=s(this[j.ext.iApiIndex]),e,f,g=d.aoColumns,h=d.aoData,o,m;if(g[a].bVisible!=b){if(b){for(e=f=0;e<a;e++)g[e].bVisible&&f++;m=f>=t(d);if(!m)for(e=a;e<g.length;e++)if(g[e].bVisible){o=e;break}e=0;for(f=h.length;e<f;e++)null!==h[e].nTr&&(m?h[e].nTr.appendChild(h[e]._anHidden[a]):h[e].nTr.insertBefore(h[e]._anHidden[a],J(d,e)[o]))}else{e=0;for(f=h.length;e<f;e++)null!==h[e].nTr&&(o=J(d,e)[a],h[e]._anHidden[a]=o,o.parentNode.removeChild(o))}g[a].bVisible=
-b;W(d,d.aoHeader);d.nTFoot&&W(d,d.aoFooter);e=0;for(f=d.aoOpenRows.length;e<f;e++)d.aoOpenRows[e].nTr.colSpan=t(d);if(c===n||c)k(d),x(d);ra(d)}};this.fnSettings=function(){return s(this[j.ext.iApiIndex])};this.fnSort=function(a){var b=s(this[j.ext.iApiIndex]);b.aaSorting=a;O(b)};this.fnSortListener=function(a,b,c){ia(s(this[j.ext.iApiIndex]),a,b,c)};this.fnUpdate=function(a,b,c,d,e){var f=s(this[j.ext.iApiIndex]),b="object"===typeof b?I(f,b):b;if(h.isArray(a)&&c===n){f.aoData[b]._aData=a.slice();
-for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(v(f,b,c),b,c,!1,!1)}else if(h.isPlainObject(a)&&c===n){f.aoData[b]._aData=h.extend(!0,{},a);for(c=0;c<f.aoColumns.length;c++)this.fnUpdate(v(f,b,c),b,c,!1,!1)}else{F(f,b,c,a);var a=v(f,b,c,"display"),g=f.aoColumns[c];null!==g.fnRender&&(a=S(f,b,c),g.bUseRendered&&F(f,b,c,a));null!==f.aoData[b].nTr&&(J(f,b)[c].innerHTML=a)}c=h.inArray(b,f.aiDisplay);f.asDataSearch[c]=na(f,Y(f,b,"filter",r(f,"bSearchable")));(e===n||e)&&k(f);(d===n||d)&&aa(f);return 0};
-this.fnVersionCheck=j.ext.fnVersionCheck;this.oApi={_fnExternApiFunc:Va,_fnInitialise:ba,_fnInitComplete:$,_fnLanguageCompat:pa,_fnAddColumn:o,_fnColumnOptions:m,_fnAddData:H,_fnCreateTr:ea,_fnGatherData:ua,_fnBuildHead:va,_fnDrawHead:W,_fnDraw:x,_fnReDraw:aa,_fnAjaxUpdate:wa,_fnAjaxParameters:Ea,_fnAjaxUpdateDraw:Fa,_fnServerParams:ka,_fnAddOptionsHtml:xa,_fnFeatureHtmlTable:Ba,_fnScrollDraw:La,_fnAdjustColumnSizing:k,_fnFeatureHtmlFilter:za,_fnFilterComplete:K,_fnFilterCustom:Ia,_fnFilterColumn:Ha,
-_fnFilter:Ga,_fnBuildSearchArray:la,_fnBuildSearchRow:na,_fnFilterCreateSearch:ma,_fnDataToSearch:Ja,_fnSort:O,_fnSortAttachListener:ia,_fnSortingClasses:P,_fnFeatureHtmlPaginate:Da,_fnPageChange:qa,_fnFeatureHtmlInfo:Ca,_fnUpdateInfo:Ka,_fnFeatureHtmlLength:ya,_fnFeatureHtmlProcessing:Aa,_fnProcessingDisplay:E,_fnVisibleToColumnIndex:G,_fnColumnIndexToVisible:R,_fnNodeToDataIndex:I,_fnVisbleColumns:t,_fnCalculateEnd:y,_fnConvertToWidth:Ma,_fnCalculateColumnWidths:da,_fnScrollingWidthAdjust:Oa,_fnGetWidestNode:Na,
-_fnGetMaxLenString:Pa,_fnStringToCss:q,_fnDetectType:B,_fnSettingsFromNode:s,_fnGetDataMaster:Z,_fnGetTrNodes:T,_fnGetTdNodes:J,_fnEscapeRegex:oa,_fnDeleteIndex:ha,_fnReOrderIndex:u,_fnColumnOrdering:M,_fnLog:D,_fnClearTable:ga,_fnSaveState:ra,_fnLoadState:Sa,_fnCreateCookie:function(a,b,c,d,e){var f=new Date;f.setTime(f.getTime()+1E3*c);var c=X.location.pathname.split("/"),a=a+"_"+c.pop().replace(/[\/:]/g,"").toLowerCase(),g;null!==e?(g="function"===typeof h.parseJSON?h.parseJSON(b):eval("("+b+")"),
-b=e(a,g,f.toGMTString(),c.join("/")+"/")):b=a+"="+encodeURIComponent(b)+"; expires="+f.toGMTString()+"; path="+c.join("/")+"/";a=l.cookie.split(";");e=b.split(";")[0].length;f=[];if(4096<e+l.cookie.length+10){for(var j=0,o=a.length;j<o;j++)if(-1!=a[j].indexOf(d)){var k=a[j].split("=");try{(g=eval("("+decodeURIComponent(k[1])+")"))&&g.iCreate&&f.push({name:k[0],time:g.iCreate})}catch(m){}}for(f.sort(function(a,b){return b.time-a.time});4096<e+l.cookie.length+10;){if(0===f.length)return;d=f.pop();l.cookie=
-d.name+"=; expires=Thu, 01-Jan-1970 00:00:01 GMT; path="+c.join("/")+"/"}}l.cookie=b},_fnReadCookie:function(a){for(var b=X.location.pathname.split("/"),a=a+"_"+b[b.length-1].replace(/[\/:]/g,"").toLowerCase()+"=",b=l.cookie.split(";"),c=0;c<b.length;c++){for(var d=b[c];" "==d.charAt(0);)d=d.substring(1,d.length);if(0===d.indexOf(a))return decodeURIComponent(d.substring(a.length,d.length))}return null},_fnDetectHeader:V,_fnGetUniqueThs:N,_fnScrollBarWidth:Qa,_fnApplyToChildren:C,_fnMap:p,_fnGetRowData:Y,
-_fnGetCellData:v,_fnSetCellData:F,_fnGetObjectDataFn:Q,_fnSetObjectDataFn:L,_fnApplyColumnDefs:ta,_fnBindAction:Ra,_fnExtend:Ta,_fnCallbackReg:z,_fnCallbackFire:A,_fnJsonString:Wa,_fnRender:S,_fnNodeToColumnIndex:fa,_fnInfoMacros:ja,_fnBrowserDetect:Ua,_fnGetColumns:r};h.extend(j.ext.oApi,this.oApi);for(var sa in j.ext.oApi)sa&&(this[sa]=Va(sa));var ca=this;this.each(function(){var a=0,b,c,d;c=this.getAttribute("id");var i=!1,f=!1;if("table"!=this.nodeName.toLowerCase())D(null,0,"Attempted to initialise DataTables on a node which is not a table: "+
-this.nodeName);else{a=0;for(b=j.settings.length;a<b;a++){if(j.settings[a].nTable==this){if(e===n||e.bRetrieve)return j.settings[a].oInstance;if(e.bDestroy){j.settings[a].oInstance.fnDestroy();break}else{D(j.settings[a],0,"Cannot reinitialise DataTable.\n\nTo retrieve the DataTables object for this table, pass no arguments or see the docs for bRetrieve and bDestroy");return}}if(j.settings[a].sTableId==this.id){j.settings.splice(a,1);break}}if(null===c||""===c)this.id=c="DataTables_Table_"+j.ext._oExternConfig.iNextUnique++;
-var g=h.extend(!0,{},j.models.oSettings,{nTable:this,oApi:ca.oApi,oInit:e,sDestroyWidth:h(this).width(),sInstance:c,sTableId:c});j.settings.push(g);g.oInstance=1===ca.length?ca:h(this).dataTable();e||(e={});e.oLanguage&&pa(e.oLanguage);e=Ta(h.extend(!0,{},j.defaults),e);p(g.oFeatures,e,"bPaginate");p(g.oFeatures,e,"bLengthChange");p(g.oFeatures,e,"bFilter");p(g.oFeatures,e,"bSort");p(g.oFeatures,e,"bInfo");p(g.oFeatures,e,"bProcessing");p(g.oFeatures,e,"bAutoWidth");p(g.oFeatures,e,"bSortClasses");
-p(g.oFeatures,e,"bServerSide");p(g.oFeatures,e,"bDeferRender");p(g.oScroll,e,"sScrollX","sX");p(g.oScroll,e,"sScrollXInner","sXInner");p(g.oScroll,e,"sScrollY","sY");p(g.oScroll,e,"bScrollCollapse","bCollapse");p(g.oScroll,e,"bScrollInfinite","bInfinite");p(g.oScroll,e,"iScrollLoadGap","iLoadGap");p(g.oScroll,e,"bScrollAutoCss","bAutoCss");p(g,e,"asStripeClasses");p(g,e,"asStripClasses","asStripeClasses");p(g,e,"fnServerData");p(g,e,"fnFormatNumber");p(g,e,"sServerMethod");p(g,e,"aaSorting");p(g,
-e,"aaSortingFixed");p(g,e,"aLengthMenu");p(g,e,"sPaginationType");p(g,e,"sAjaxSource");p(g,e,"sAjaxDataProp");p(g,e,"iCookieDuration");p(g,e,"sCookiePrefix");p(g,e,"sDom");p(g,e,"bSortCellsTop");p(g,e,"iTabIndex");p(g,e,"oSearch","oPreviousSearch");p(g,e,"aoSearchCols","aoPreSearchCols");p(g,e,"iDisplayLength","_iDisplayLength");p(g,e,"bJQueryUI","bJUI");p(g,e,"fnCookieCallback");p(g,e,"fnStateLoad");p(g,e,"fnStateSave");p(g.oLanguage,e,"fnInfoCallback");z(g,"aoDrawCallback",e.fnDrawCallback,"user");
-z(g,"aoServerParams",e.fnServerParams,"user");z(g,"aoStateSaveParams",e.fnStateSaveParams,"user");z(g,"aoStateLoadParams",e.fnStateLoadParams,"user");z(g,"aoStateLoaded",e.fnStateLoaded,"user");z(g,"aoRowCallback",e.fnRowCallback,"user");z(g,"aoRowCreatedCallback",e.fnCreatedRow,"user");z(g,"aoHeaderCallback",e.fnHeaderCallback,"user");z(g,"aoFooterCallback",e.fnFooterCallback,"user");z(g,"aoInitComplete",e.fnInitComplete,"user");z(g,"aoPreDrawCallback",e.fnPreDrawCallback,"user");g.oFeatures.bServerSide&&
-g.oFeatures.bSort&&g.oFeatures.bSortClasses?z(g,"aoDrawCallback",P,"server_side_sort_classes"):g.oFeatures.bDeferRender&&z(g,"aoDrawCallback",P,"defer_sort_classes");e.bJQueryUI?(h.extend(g.oClasses,j.ext.oJUIClasses),e.sDom===j.defaults.sDom&&"lfrtip"===j.defaults.sDom&&(g.sDom='<"H"lfr>t<"F"ip>')):h.extend(g.oClasses,j.ext.oStdClasses);h(this).addClass(g.oClasses.sTable);if(""!==g.oScroll.sX||""!==g.oScroll.sY)g.oScroll.iBarWidth=Qa();g.iInitDisplayStart===n&&(g.iInitDisplayStart=e.iDisplayStart,
-g._iDisplayStart=e.iDisplayStart);e.bStateSave&&(g.oFeatures.bStateSave=!0,Sa(g,e),z(g,"aoDrawCallback",ra,"state_save"));null!==e.iDeferLoading&&(g.bDeferLoading=!0,a=h.isArray(e.iDeferLoading),g._iRecordsDisplay=a?e.iDeferLoading[0]:e.iDeferLoading,g._iRecordsTotal=a?e.iDeferLoading[1]:e.iDeferLoading);null!==e.aaData&&(f=!0);""!==e.oLanguage.sUrl?(g.oLanguage.sUrl=e.oLanguage.sUrl,h.getJSON(g.oLanguage.sUrl,null,function(a){pa(a);h.extend(true,g.oLanguage,e.oLanguage,a);ba(g)}),i=!0):h.extend(!0,
-g.oLanguage,e.oLanguage);null===e.asStripeClasses&&(g.asStripeClasses=[g.oClasses.sStripeOdd,g.oClasses.sStripeEven]);b=g.asStripeClasses.length;g.asDestroyStripes=[];if(b){c=!1;d=h(this).children("tbody").children("tr:lt("+b+")");for(a=0;a<b;a++)d.hasClass(g.asStripeClasses[a])&&(c=!0,g.asDestroyStripes.push(g.asStripeClasses[a]));c&&d.removeClass(g.asStripeClasses.join(" "))}c=[];a=this.getElementsByTagName("thead");0!==a.length&&(V(g.aoHeader,a[0]),c=N(g));if(null===e.aoColumns){d=[];a=0;for(b=
-c.length;a<b;a++)d.push(null)}else d=e.aoColumns;a=0;for(b=d.length;a<b;a++)e.saved_aoColumns!==n&&e.saved_aoColumns.length==b&&(null===d[a]&&(d[a]={}),d[a].bVisible=e.saved_aoColumns[a].bVisible),o(g,c?c[a]:null);ta(g,e.aoColumnDefs,d,function(a,b){m(g,a,b)});a=0;for(b=g.aaSorting.length;a<b;a++){g.aaSorting[a][0]>=g.aoColumns.length&&(g.aaSorting[a][0]=0);var k=g.aoColumns[g.aaSorting[a][0]];g.aaSorting[a][2]===n&&(g.aaSorting[a][2]=0);e.aaSorting===n&&g.saved_aaSorting===n&&(g.aaSorting[a][1]=
-k.asSorting[0]);c=0;for(d=k.asSorting.length;c<d;c++)if(g.aaSorting[a][1]==k.asSorting[c]){g.aaSorting[a][2]=c;break}}P(g);Ua(g);a=h(this).children("caption").each(function(){this._captionSide=h(this).css("caption-side")});b=h(this).children("thead");0===b.length&&(b=[l.createElement("thead")],this.appendChild(b[0]));g.nTHead=b[0];b=h(this).children("tbody");0===b.length&&(b=[l.createElement("tbody")],this.appendChild(b[0]));g.nTBody=b[0];g.nTBody.setAttribute("role","alert");g.nTBody.setAttribute("aria-live",
-"polite");g.nTBody.setAttribute("aria-relevant","all");b=h(this).children("tfoot");if(0===b.length&&0<a.length&&(""!==g.oScroll.sX||""!==g.oScroll.sY))b=[l.createElement("tfoot")],this.appendChild(b[0]);0<b.length&&(g.nTFoot=b[0],V(g.aoFooter,g.nTFoot));if(f)for(a=0;a<e.aaData.length;a++)H(g,e.aaData[a]);else ua(g);g.aiDisplay=g.aiDisplayMaster.slice();g.bInitialised=!0;!1===i&&ba(g)}});ca=null;return this};j.fnVersionCheck=function(e){for(var h=function(e,h){for(;e.length<h;)e+="0";return e},m=j.ext.sVersion.split("."),
-e=e.split("."),k="",n="",l=0,t=e.length;l<t;l++)k+=h(m[l],3),n+=h(e[l],3);return parseInt(k,10)>=parseInt(n,10)};j.fnIsDataTable=function(e){for(var h=j.settings,m=0;m<h.length;m++)if(h[m].nTable===e||h[m].nScrollHead===e||h[m].nScrollFoot===e)return!0;return!1};j.fnTables=function(e){var o=[];jQuery.each(j.settings,function(j,k){(!e||!0===e&&h(k.nTable).is(":visible"))&&o.push(k.nTable)});return o};j.version="1.9.4";j.settings=[];j.models={};j.models.ext={afnFiltering:[],afnSortData:[],aoFeatures:[],
-aTypes:[],fnVersionCheck:j.fnVersionCheck,iApiIndex:0,ofnSearch:{},oApi:{},oStdClasses:{},oJUIClasses:{},oPagination:{},oSort:{},sVersion:j.version,sErrMode:"alert",_oExternConfig:{iNextUnique:0}};j.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};j.models.oRow={nTr:null,_aData:[],_aSortData:[],_anHidden:[],_sRowStripe:""};j.models.oColumn={aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bUseRendered:null,bVisible:null,_bAutoType:!0,fnCreatedCell:null,fnGetData:null,
-fnRender:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};j.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,
-bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollAutoCss:!0,bScrollCollapse:!1,bScrollInfinite:!1,bServerSide:!1,bSort:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCookieCallback:null,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(e){if(1E3>e)return e;for(var h=e+"",e=h.split(""),j="",h=h.length,k=0;k<h;k++)0===k%3&&0!==k&&(j=this.oLanguage.sInfoThousands+j),j=e[h-k-1]+j;return j},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,
-fnRowCallback:null,fnServerData:function(e,j,m,k){k.jqXHR=h.ajax({url:e,data:j,success:function(e){e.sError&&k.oApi._fnLog(k,0,e.sError);h(k.oInstance).trigger("xhr",[k,e]);m(e)},dataType:"json",cache:!1,type:k.sServerMethod,error:function(e,h){"parsererror"==h&&k.oApi._fnLog(k,0,"DataTables warning: JSON data from server could not be parsed. This is caused by a JSON formatting error.")}})},fnServerParams:null,fnStateLoad:function(e){var e=this.oApi._fnReadCookie(e.sCookiePrefix+e.sInstance),j;try{j=
-"function"===typeof h.parseJSON?h.parseJSON(e):eval("("+e+")")}catch(m){j=null}return j},fnStateLoadParams:null,fnStateLoaded:null,fnStateSave:function(e,h){this.oApi._fnCreateCookie(e.sCookiePrefix+e.sInstance,this.oApi._fnJsonString(h),e.iCookieDuration,e.sCookiePrefix,e.fnCookieCallback)},fnStateSaveParams:null,iCookieDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iScrollLoadGap:100,iTabIndex:0,oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},
-oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sInfoThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},j.models.oSearch),sAjaxDataProp:"aaData",
-sAjaxSource:null,sCookiePrefix:"SpryMedia_DataTables_",sDom:"lfrtip",sPaginationType:"two_button",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET"};j.defaults.columns={aDataSort:null,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bUseRendered:!0,bVisible:!0,fnCreatedCell:null,fnRender:null,iDataSort:-1,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};j.models.oSettings={oFeatures:{bAutoWidth:null,
-bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortClasses:null,bStateSave:null},oScroll:{bAutoCss:null,bCollapse:null,bInfinite:null,iBarWidth:0,iLoadGap:null,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1},aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aoColumns:[],aoHeader:[],aoFooter:[],asDataSearch:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:null,
-asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,sPaginationType:"two_button",iCookieDuration:0,sCookiePrefix:"",fnCookieCallback:null,aoStateSave:[],aoStateLoad:[],
-oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iDisplayEnd:10,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsTotal,10):this.aiDisplayMaster.length},
-fnRecordsDisplay:function(){return this.oFeatures.bServerSide?parseInt(this._iRecordsDisplay,10):this.aiDisplay.length},fnDisplayEnd:function(){return this.oFeatures.bServerSide?!1===this.oFeatures.bPaginate||-1==this._iDisplayLength?this._iDisplayStart+this.aiDisplay.length:Math.min(this._iDisplayStart+this._iDisplayLength,this._iRecordsDisplay):this._iDisplayEnd},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null};j.ext=h.extend(!0,{},j.models.ext);h.extend(j.ext.oStdClasses,
-{sTable:"dataTable",sPagePrevEnabled:"paginate_enabled_previous",sPagePrevDisabled:"paginate_disabled_previous",sPageNextEnabled:"paginate_enabled_next",sPageNextDisabled:"paginate_disabled_next",sPageJUINext:"",sPageJUIPrev:"",sPageButton:"paginate_button",sPageButtonActive:"paginate_active",sPageButtonStaticDisabled:"paginate_button paginate_button_disabled",sPageFirst:"first",sPagePrevious:"previous",sPageNext:"next",sPageLast:"last",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",
-sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",
-sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sFooterTH:"",sJUIHeader:"",sJUIFooter:""});h.extend(j.ext.oJUIClasses,j.ext.oStdClasses,{sPagePrevEnabled:"fg-button ui-button ui-state-default ui-corner-left",sPagePrevDisabled:"fg-button ui-button ui-state-default ui-corner-left ui-state-disabled",sPageNextEnabled:"fg-button ui-button ui-state-default ui-corner-right",
-sPageNextDisabled:"fg-button ui-button ui-state-default ui-corner-right ui-state-disabled",sPageJUINext:"ui-icon ui-icon-circle-arrow-e",sPageJUIPrev:"ui-icon ui-icon-circle-arrow-w",sPageButton:"fg-button ui-button ui-state-default",sPageButtonActive:"fg-button ui-button ui-state-default ui-state-disabled",sPageButtonStaticDisabled:"fg-button ui-button ui-state-default ui-state-disabled",sPageFirst:"first ui-corner-tl ui-corner-bl",sPageLast:"last ui-corner-tr ui-corner-br",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",
-sSortAsc:"ui-state-default",sSortDesc:"ui-state-default",sSortable:"ui-state-default",sSortableAsc:"ui-state-default",sSortableDesc:"ui-state-default",sSortableNone:"ui-state-default",sSortJUIAsc:"css_right ui-icon ui-icon-triangle-1-n",sSortJUIDesc:"css_right ui-icon ui-icon-triangle-1-s",sSortJUI:"css_right ui-icon ui-icon-carat-2-n-s",sSortJUIAscAllowed:"css_right ui-icon ui-icon-carat-1-n",sSortJUIDescAllowed:"css_right ui-icon ui-icon-carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",
-sScrollHead:"dataTables_scrollHead ui-state-default",sScrollFoot:"dataTables_scrollFoot ui-state-default",sFooterTH:"ui-state-default",sJUIHeader:"fg-toolbar ui-toolbar ui-widget-header ui-corner-tl ui-corner-tr ui-helper-clearfix",sJUIFooter:"fg-toolbar ui-toolbar ui-widget-header ui-corner-bl ui-corner-br ui-helper-clearfix"});h.extend(j.ext.oPagination,{two_button:{fnInit:function(e,j,m){var k=e.oLanguage.oPaginate,n=function(h){e.oApi._fnPageChange(e,h.data.action)&&m(e)},k=!e.bJUI?'<a class="'+
-e.oClasses.sPagePrevDisabled+'" tabindex="'+e.iTabIndex+'" role="button">'+k.sPrevious+'</a><a class="'+e.oClasses.sPageNextDisabled+'" tabindex="'+e.iTabIndex+'" role="button">'+k.sNext+"</a>":'<a class="'+e.oClasses.sPagePrevDisabled+'" tabindex="'+e.iTabIndex+'" role="button"><span class="'+e.oClasses.sPageJUIPrev+'"></span></a><a class="'+e.oClasses.sPageNextDisabled+'" tabindex="'+e.iTabIndex+'" role="button"><span class="'+e.oClasses.sPageJUINext+'"></span></a>';h(j).append(k);var l=h("a",j),
-k=l[0],l=l[1];e.oApi._fnBindAction(k,{action:"previous"},n);e.oApi._fnBindAction(l,{action:"next"},n);e.aanFeatures.p||(j.id=e.sTableId+"_paginate",k.id=e.sTableId+"_previous",l.id=e.sTableId+"_next",k.setAttribute("aria-controls",e.sTableId),l.setAttribute("aria-controls",e.sTableId))},fnUpdate:function(e){if(e.aanFeatures.p)for(var h=e.oClasses,j=e.aanFeatures.p,k,l=0,n=j.length;l<n;l++)if(k=j[l].firstChild)k.className=0===e._iDisplayStart?h.sPagePrevDisabled:h.sPagePrevEnabled,k=k.nextSibling,
-k.className=e.fnDisplayEnd()==e.fnRecordsDisplay()?h.sPageNextDisabled:h.sPageNextEnabled}},iFullNumbersShowPages:5,full_numbers:{fnInit:function(e,j,m){var k=e.oLanguage.oPaginate,l=e.oClasses,n=function(h){e.oApi._fnPageChange(e,h.data.action)&&m(e)};h(j).append('<a tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPageFirst+'">'+k.sFirst+'</a><a tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPagePrevious+'">'+k.sPrevious+'</a><span></span><a tabindex="'+e.iTabIndex+'" class="'+
-l.sPageButton+" "+l.sPageNext+'">'+k.sNext+'</a><a tabindex="'+e.iTabIndex+'" class="'+l.sPageButton+" "+l.sPageLast+'">'+k.sLast+"</a>");var t=h("a",j),k=t[0],l=t[1],r=t[2],t=t[3];e.oApi._fnBindAction(k,{action:"first"},n);e.oApi._fnBindAction(l,{action:"previous"},n);e.oApi._fnBindAction(r,{action:"next"},n);e.oApi._fnBindAction(t,{action:"last"},n);e.aanFeatures.p||(j.id=e.sTableId+"_paginate",k.id=e.sTableId+"_first",l.id=e.sTableId+"_previous",r.id=e.sTableId+"_next",t.id=e.sTableId+"_last")},
-fnUpdate:function(e,o){if(e.aanFeatures.p){var m=j.ext.oPagination.iFullNumbersShowPages,k=Math.floor(m/2),l=Math.ceil(e.fnRecordsDisplay()/e._iDisplayLength),n=Math.ceil(e._iDisplayStart/e._iDisplayLength)+1,t="",r,B=e.oClasses,u,M=e.aanFeatures.p,L=function(h){e.oApi._fnBindAction(this,{page:h+r-1},function(h){e.oApi._fnPageChange(e,h.data.page);o(e);h.preventDefault()})};-1===e._iDisplayLength?n=k=r=1:l<m?(r=1,k=l):n<=k?(r=1,k=m):n>=l-k?(r=l-m+1,k=l):(r=n-Math.ceil(m/2)+1,k=r+m-1);for(m=r;m<=k;m++)t+=
-n!==m?'<a tabindex="'+e.iTabIndex+'" class="'+B.sPageButton+'">'+e.fnFormatNumber(m)+"</a>":'<a tabindex="'+e.iTabIndex+'" class="'+B.sPageButtonActive+'">'+e.fnFormatNumber(m)+"</a>";m=0;for(k=M.length;m<k;m++)u=M[m],u.hasChildNodes()&&(h("span:eq(0)",u).html(t).children("a").each(L),u=u.getElementsByTagName("a"),u=[u[0],u[1],u[u.length-2],u[u.length-1]],h(u).removeClass(B.sPageButton+" "+B.sPageButtonActive+" "+B.sPageButtonStaticDisabled),h([u[0],u[1]]).addClass(1==n?B.sPageButtonStaticDisabled:
-B.sPageButton),h([u[2],u[3]]).addClass(0===l||n===l||-1===e._iDisplayLength?B.sPageButtonStaticDisabled:B.sPageButton))}}}});h.extend(j.ext.oSort,{"string-pre":function(e){"string"!=typeof e&&(e=null!==e&&e.toString?e.toString():"");return e.toLowerCase()},"string-asc":function(e,h){return e<h?-1:e>h?1:0},"string-desc":function(e,h){return e<h?1:e>h?-1:0},"html-pre":function(e){return e.replace(/<.*?>/g,"").toLowerCase()},"html-asc":function(e,h){return e<h?-1:e>h?1:0},"html-desc":function(e,h){return e<
-h?1:e>h?-1:0},"date-pre":function(e){e=Date.parse(e);if(isNaN(e)||""===e)e=Date.parse("01/01/1970 00:00:00");return e},"date-asc":function(e,h){return e-h},"date-desc":function(e,h){return h-e},"numeric-pre":function(e){return"-"==e||""===e?0:1*e},"numeric-asc":function(e,h){return e-h},"numeric-desc":function(e,h){return h-e}});h.extend(j.ext.aTypes,[function(e){if("number"===typeof e)return"numeric";if("string"!==typeof e)return null;var h,j=!1;h=e.charAt(0);if(-1=="0123456789-".indexOf(h))return null;
-for(var k=1;k<e.length;k++){h=e.charAt(k);if(-1=="0123456789.".indexOf(h))return null;if("."==h){if(j)return null;j=!0}}return"numeric"},function(e){var h=Date.parse(e);return null!==h&&!isNaN(h)||"string"===typeof e&&0===e.length?"date":null},function(e){return"string"===typeof e&&-1!=e.indexOf("<")&&-1!=e.indexOf(">")?"html":null}]);h.fn.DataTable=j;h.fn.dataTable=j;h.fn.dataTableSettings=j.settings;h.fn.dataTableExt=j.ext};"function"===typeof define&&define.amd?define(["jquery"],L):jQuery&&!jQuery.fn.dataTable&&
-L(jQuery)})(window,document);
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/dictionary.js
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/dictionary.js b/src/main/resources/com/twitter/aurora/scheduler/http/assets/dictionary.js
deleted file mode 100644
index 92045c4..0000000
--- a/src/main/resources/com/twitter/aurora/scheduler/http/assets/dictionary.js
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Dictionary to use for attaching tool-tips to various terminologies in Aurora.
- */
-var dictionary = {
- states: {
- PENDING: 'The scheduler is searching for a machine that satisfies the resources and '
- + 'constraints for this task.',
-
- ASSIGNED: 'The scheduler has selected a machine to run the task and is instructing the '
- + 'slave to launch it.',
-
- STARTING: 'The executor is preparing to launch the task.',
- RUNNING: 'The user process(es) are running.',
- FAILED: 'The task ran, but did not exit indicating success.',
- FINISHED: 'The task ran and exited successfully.',
- KILLED: 'A user or cron invocation terminated the task.',
- PREEMPTING: 'This task is being killed to make resources available for a production task.',
- KILLING: 'A user request or cron invocation has requested the task be killed.',
- LOST: 'The task cannot be accounted for, usually a result of slave process or machine failure.'
- }
-};
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/aurora.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/aurora.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/aurora.png
deleted file mode 100644
index 2ddeb24..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/aurora.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/viz.png
----------------------------------------------------------------------
diff --git a/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/viz.png b/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/viz.png
deleted file mode 100644
index 68ee1fd..0000000
Binary files a/src/main/resources/com/twitter/aurora/scheduler/http/assets/images/viz.png and /dev/null differ
[13/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/BUILD b/src/main/python/twitter/aurora/common/BUILD
deleted file mode 100644
index 4e839f7..0000000
--- a/src/main/python/twitter/aurora/common/BUILD
+++ /dev/null
@@ -1,63 +0,0 @@
-import os
-
-python_library(
- name = 'aurora_job_key',
- sources = ['aurora_job_key.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'cluster',
- sources = ['cluster.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- ]
-)
-
-python_library(
- name = 'clusters',
- sources = ['clusters.py'],
- dependencies = [
- pants(':cluster'),
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- pants('aurora/twitterdeps/src/python/twitter/common/collections'),
- ]
-)
-
-python_library(
- name = 'cluster_option',
- sources = ['cluster_option.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- ]
-)
-
-python_library(
- name = 'http_signaler',
- sources = ['http_signaler.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- ]
-)
-
-python_library(
- name = 'common',
- dependencies = [
- pants(':aurora_job_key'),
- pants(':cluster'),
- pants(':cluster_option'),
- pants(':clusters'),
- pants(':http_signaler'),
- pants('src/main/python/twitter/aurora/common/auth'),
- ],
- provides = setup_py(
- name = 'twitter.aurora.common',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- description = 'Aurora common libraries.',
- license = 'Apache License, Version 2.0',
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/__init__.py b/src/main/python/twitter/aurora/common/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/aurora_job_key.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/aurora_job_key.py b/src/main/python/twitter/aurora/common/aurora_job_key.py
deleted file mode 100644
index 81a8687..0000000
--- a/src/main/python/twitter/aurora/common/aurora_job_key.py
+++ /dev/null
@@ -1,97 +0,0 @@
-import re
-
-from twitter.common.lang import Compatibility, total_ordering
-
-from gen.twitter.aurora.constants import GOOD_IDENTIFIER_PATTERN_PYTHON
-from gen.twitter.aurora.ttypes import Identity, JobKey, TaskQuery
-
-# TODO(ksweeney): This can just probably just extend namedtuple.
-@total_ordering
-class AuroraJobKey(object):
- """A canonical representation of a key that can identify a job in any of the clusters the client
- is aware of."""
- class Error(Exception): pass
- class TypeError(TypeError, Error): pass
- class InvalidIdentifier(ValueError, Error): pass
- class ParseError(ValueError, Error): pass
-
- VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
-
- def __init__(self, cluster, role, env, name):
- if not isinstance(cluster, Compatibility.string):
- raise self.TypeError("cluster should be a string, got %s" % (cluster.__class__.__name__))
- self._cluster = cluster
- self._role = self._assert_valid_identifier("role", role)
- self._env = self._assert_valid_identifier("env", env)
- self._name = self._assert_valid_identifier("name", name)
-
- @classmethod
- def from_path(cls, path):
- try:
- cluster, role, env, name = path.split('/', 4)
- except ValueError:
- raise cls.ParseError(
- "Invalid path '%s'. path should be a string in the form CLUSTER/ROLE/ENV/NAME" % path)
- return cls(cluster, role, env, name)
-
- @classmethod
- def from_thrift(cls, cluster, job_key):
- if not isinstance(job_key, JobKey):
- raise cls.TypeError("job_key must be a Thrift JobKey struct")
- return cls(cluster, job_key.role, job_key.environment, job_key.name)
-
- @classmethod
- def _assert_valid_identifier(cls, field, identifier):
- if not isinstance(identifier, Compatibility.string):
- raise cls.TypeError("%s must be a string" % field)
- if not cls.VALID_IDENTIFIER.match(identifier):
- raise cls.InvalidIdentifier("Invalid %s '%s'" % (field, identifier))
- return identifier
-
- @property
- def cluster(self):
- return self._cluster
-
- @property
- def role(self):
- return self._role
-
- @property
- def env(self):
- return self._env
-
- @property
- def name(self):
- return self._name
-
- def to_path(self):
- return "%s/%s/%s/%s" % (self.cluster, self.role, self.env, self.name)
-
- def to_thrift(self):
- return JobKey(role=self.role, environment=self.env, name=self.name)
-
- def to_thrift_query(self):
- return TaskQuery(owner=Identity(role=self.role), environment=self.env, jobName=self.name)
-
- def __iter__(self):
- """Support 'cluster, role, env, name = job_key' assignment."""
- return iter((self._cluster, self._role, self._env, self._name))
-
- def __repr__(self):
- return "%s(%r, %r, %r, %r)" % (self.__class__, self._cluster, self._role, self._env, self._name)
-
- def __str__(self):
- return self.to_path()
-
- def __hash__(self):
- return hash(AuroraJobKey) + hash(self.to_path())
-
- def __eq__(self, other):
- if not isinstance(other, AuroraJobKey):
- return NotImplemented
- return self.to_path() == other.to_path()
-
- def __lt__(self, other):
- if not isinstance(other, AuroraJobKey):
- return NotImplemented
- return self.to_path() < other.to_path()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/auth/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/auth/BUILD b/src/main/python/twitter/aurora/common/auth/BUILD
deleted file mode 100644
index fd5b024..0000000
--- a/src/main/python/twitter/aurora/common/auth/BUILD
+++ /dev/null
@@ -1,9 +0,0 @@
-python_library(
- name = 'auth',
- sources = globs('*.py'),
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/auth/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/auth/__init__.py b/src/main/python/twitter/aurora/common/auth/__init__.py
deleted file mode 100644
index 5418228..0000000
--- a/src/main/python/twitter/aurora/common/auth/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .auth_module_manager import make_session_key, register_auth_module, SessionKeyError
-from .auth_module import AuthModule, InsecureAuthModule
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/auth/auth_module.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/auth/auth_module.py b/src/main/python/twitter/aurora/common/auth/auth_module.py
deleted file mode 100644
index 7b146a9..0000000
--- a/src/main/python/twitter/aurora/common/auth/auth_module.py
+++ /dev/null
@@ -1,30 +0,0 @@
-from abc import abstractmethod, abstractproperty
-
-import getpass
-import time
-
-from twitter.common.lang import Interface
-
-from gen.twitter.aurora.ttypes import SessionKey
-
-
-class AuthModule(Interface):
- @abstractproperty
- def mechanism(self):
- """Return the mechanism provided by this AuthModule."""
-
- @abstractmethod
- def payload(self):
- """Return the payload generated by the AuthModule."""
-
- def __call__(self):
- return SessionKey(mechanism=self.mechanism, data=self.payload())
-
-
-class InsecureAuthModule(AuthModule):
- @property
- def mechanism(self):
- return 'UNAUTHENTICATED'
-
- def payload(self):
- return 'UNAUTHENTICATED'
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/auth/auth_module_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/auth/auth_module_manager.py b/src/main/python/twitter/aurora/common/auth/auth_module_manager.py
deleted file mode 100644
index 03d51b3..0000000
--- a/src/main/python/twitter/aurora/common/auth/auth_module_manager.py
+++ /dev/null
@@ -1,47 +0,0 @@
-from twitter.common import log
-
-from gen.twitter.aurora.ttypes import SessionKey
-
-from .auth_module import AuthModule, InsecureAuthModule
-
-
-_INSECURE_AUTH_MODULE = InsecureAuthModule()
-_AUTH_MODULES = {
- _INSECURE_AUTH_MODULE.mechanism: _INSECURE_AUTH_MODULE
-}
-
-
-class SessionKeyError(Exception): pass
-
-
-def register_auth_module(auth_module):
- """
- Add an auth module into the registry used by make_session_key. An auth module is discovered
- via its auth mechanism.
-
- args:
- auth_module: A 0-arg callable that should return a SessionKey or raises a SessionKeyError
- and extend AuthModule.
- """
- if not isinstance(auth_module, AuthModule):
- raise TypeError('Given auth module must be a AuthModule subclass, got %s' % type(auth_module))
- if not callable(auth_module):
- raise TypeError('auth_module should be callable.')
- _AUTH_MODULES[auth_module.mechanism] = auth_module
-
-
-def make_session_key(auth_mechanism='UNAUTHENTICATED'):
- """
- Attempts to create a session key by calling the auth module registered to the auth mechanism.
- If an auth module does not exist for an auth mechanism, an InsecureAuthModule will be used.
- """
- if not _AUTH_MODULES:
- raise SessionKeyError('No auth modules have been registered. Please call register_auth_module.')
-
- auth_module = _AUTH_MODULES.get(auth_mechanism) or _INSECURE_AUTH_MODULE
- log.debug('Using auth module: %r' % auth_module)
- session_key = auth_module()
- if not isinstance(session_key, SessionKey):
- raise SessionKeyError('Expected %r but got %r from auth module %r' % (
- SessionKey, session_key.__class__, auth_module))
- return session_key
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/cluster.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/cluster.py b/src/main/python/twitter/aurora/common/cluster.py
deleted file mode 100644
index 21d8ac3..0000000
--- a/src/main/python/twitter/aurora/common/cluster.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from pystachio import Empty, Struct
-from pystachio.composite import Structural
-
-__all__ = ('Cluster',)
-
-
-# TODO(wickman) It seems like some of this Trait/Mixin stuff should be a
-# first-class construct in Pystachio. It could be a solution for extensible
-# Job/Task definitions.
-class Cluster(dict):
- """Cluster encapsulates a set of K/V attributes describing cluster configurations.
-
- Given a cluster, attributes may be accessed directly on them, e.g.
- cluster.name
- cluster.scheduler_zk_path
-
- In order to enforce particular "traits" of Cluster, use Cluster.Trait to construct
- enforceable schemas, e.g.
-
- class ResolverTrait(Cluster.Trait):
- scheduler_zk_ensemble = Required(String)
- scheduler_zk_path = Default(String, '/twitter/service/mesos/prod/scheduler')
-
- cluster = Cluster(name = 'west', scheduler_zk_ensemble = 'zookeeper.west.twttr.net')
-
- # Ensures that scheduler_zk_ensemble is defined in the cluster or it will raise a TypeError
- cluster.with_trait(ResolverTrait).scheduler_zk_ensemble
-
- # Will use the default if none is provided on Cluster.
- cluster.with_trait(ResolverTrait).scheduler_zk_path
- """
- Trait = Struct
-
- def __init__(self, **kwargs):
- self._traits = ()
- super(Cluster, self).__init__(**kwargs)
-
- def get_trait(self, trait):
- """Given a Cluster.Trait, extract that trait."""
- if not issubclass(trait, Structural):
- raise TypeError('provided trait must be a Cluster.Trait subclass, got %s' % type(trait))
- # TODO(wickman) Expose this in pystachio as a non-private or add a load method with strict=
- return trait(trait._filter_against_schema(self))
-
- def check_trait(self, trait):
- """Given a Cluster.Trait, typecheck that trait."""
- trait_check = self.get_trait(trait).check()
- if not trait_check.ok():
- raise TypeError(trait_check.message())
-
- def with_traits(self, *traits):
- """Return a cluster annotated with a set of traits."""
- new_cluster = self.__class__(**self)
- for trait in traits:
- new_cluster.check_trait(trait)
- new_cluster._traits = traits
- return new_cluster
-
- def with_trait(self, trait):
- """Return a cluster annotated with a single trait (helper for self.with_traits)."""
- return self.with_traits(trait)
-
- def __setitem__(self, key, value):
- raise TypeError('Clusters are immutable.')
-
- def __getattr__(self, attribute):
- for trait in self._traits:
- expressed_trait = self.get_trait(trait)
- if hasattr(expressed_trait, attribute):
- value = getattr(expressed_trait, attribute)()
- return None if value is Empty else value.get()
- try:
- return self[attribute]
- except KeyError:
- return self.__getattribute__(attribute)
-
- def __copy__(self):
- return self
-
- def __deepcopy__(self, memo):
- return self
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/cluster_option.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/cluster_option.py b/src/main/python/twitter/aurora/common/cluster_option.py
deleted file mode 100644
index 3e07476..0000000
--- a/src/main/python/twitter/aurora/common/cluster_option.py
+++ /dev/null
@@ -1,82 +0,0 @@
-from copy import copy
-from optparse import (
- NO_DEFAULT,
- OptionValueError,
- Option)
-
-
-def _check_mesos_cluster(option, opt, value):
- cluster_name = value
- if option.clusters and cluster_name in option.clusters:
- return option.clusters[cluster_name]
- elif option.cluster_provider:
- return option.cluster_provider(cluster_name)
-
- cluster_list = ""
- if option.clusters:
- cluster_list = 'Valid options for clusters are %s' % ' '.join(option.clusters)
-
- raise OptionValueError(
- '%s is not a valid cluster for the %s option. %s' % (value, opt, cluster_list))
-
-
-class ClusterOption(Option):
- """A command-line Option that requires a valid cluster name and returns a Cluster object.
-
- Use in an @app.command_option decorator to avoid boilerplate. For example:
-
- CLUSTER_PATH = os.path.expanduser('~/.clusters')
- CLUSTERS = Clusters.from_json(CLUSTER_PATH)
-
- @app.command
- @app.command_option(ClusterOption('--cluster', default='smf1-test', clusters=CLUSTERS))
- def get_health(args, options):
- if options.cluster.zk_server:
- do_something(options.cluster)
-
- @app.command
- @app.command_option(ClusterOption('-s',
- '--source_cluster',
- default='smf1-test',
- clusters=CLUSTERS,
- help='Source cluster to pull metadata from.'))
- @app.command_option(ClusterOption('-d',
- '--dest_cluster',
- clusters=CLUSTERS,
- default='smf1-test'))
- def copy_metadata(args, options):
- if not options.source_cluster:
- print('required option source_cluster missing!')
- metadata_copy(options.source_cluster, options.dest_cluster)
- """
-
- # Needed since we're creating a new type for validation - see optparse docs.
- TYPES = copy(Option.TYPES) + ('mesos_cluster',)
- TYPE_CHECKER = copy(Option.TYPE_CHECKER)
- TYPE_CHECKER['mesos_cluster'] = _check_mesos_cluster
-
- def __init__(self, *opt_str, **attrs):
- """
- *opt_str: Same meaning as in twitter.common.options.Option, at least one is required.
- **attrs: See twitter.common.options.Option, with the following caveats:
-
- Exactly one of the following must be provided:
-
- clusters: A static Clusters object from which to pick clusters.
- cluster_provider: A function that takes a cluster name and returns a Cluster object.
- """
- self.clusters = attrs.pop('clusters', None)
- self.cluster_provider = attrs.pop('cluster_provider', None)
- if not (self.clusters is not None) ^ (self.cluster_provider is not None):
- raise ValueError('Must specify exactly one of clusters and cluster_provider.')
-
- default_attrs = dict(
- default=None,
- action='store',
- type='mesos_cluster',
- help='Mesos cluster to use (Default: %%default)'
- )
-
- combined_attrs = default_attrs
- combined_attrs.update(attrs) # Defensive copy
- Option.__init__(self, *opt_str, **combined_attrs) # old-style superclass
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/clusters.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/clusters.py b/src/main/python/twitter/aurora/common/clusters.py
deleted file mode 100644
index 2777569..0000000
--- a/src/main/python/twitter/aurora/common/clusters.py
+++ /dev/null
@@ -1,135 +0,0 @@
-from __future__ import print_function
-
-from collections import Mapping, namedtuple
-from contextlib import contextmanager
-import itertools
-import json
-import os
-import sys
-
-from twitter.common.collections import maybe_list
-
-from .cluster import Cluster
-
-from pystachio import Required, String
-
-try:
- import yaml
- HAS_YAML = True
-except ImportError:
- HAS_YAML = False
-
-
-__all__ = (
- 'CLUSTERS',
- 'Clusters',
-)
-
-
-class NameTrait(Cluster.Trait):
- name = Required(String)
-
-
-Parser = namedtuple('Parser', 'loader exception')
-
-
-class Clusters(Mapping):
- class Error(Exception): pass
- class ClusterExists(Error): pass
- class ClusterNotFound(KeyError, Error): pass
- class UnknownFormatError(Error): pass
- class ParseError(Error): pass
-
- LOADERS = {'.json': Parser(json.load, ValueError)}
- if HAS_YAML:
- LOADERS['.yml'] = Parser(yaml.load, yaml.parser.ParserError)
-
- @classmethod
- def from_file(cls, filename):
- return cls(list(cls.iter_clusters(filename)))
-
- @classmethod
- def iter_clusters(cls, filename):
- _, ext = os.path.splitext(filename)
- if ext not in cls.LOADERS:
- raise cls.UnknownFormatError('Unknown clusters file extension: %r' % ext)
- with open(filename) as fp:
- loader, exc_type = cls.LOADERS[ext]
- try:
- document = loader(fp)
- except exc_type as e:
- raise cls.ParseError('Unable to parse %s: %s' % (filename, e))
- if isinstance(document, list):
- iterator = document
- elif isinstance(document, dict):
- iterator = document.values()
- else:
- raise cls.ParseError('Unknown layout in %s' % filename)
- for document in iterator:
- if not isinstance(document, dict):
- raise cls.ParseError('Clusters must be maps of key/value pairs, got %s' % type(document))
- # documents not adhering to NameTrait are ignored.
- if 'name' not in document:
- continue
- yield Cluster(**document)
-
- def __init__(self, cluster_list):
- self.replace(cluster_list)
-
- def replace(self, cluster_list):
- self._clusters = {}
- self.update(cluster_list)
-
- def update(self, cluster_list):
- cluster_list = maybe_list(cluster_list, expected_type=Cluster, raise_type=TypeError)
- for cluster in cluster_list:
- self.add(cluster)
-
- def add(self, cluster):
- """Add a cluster to this Clusters map."""
- cluster = Cluster(**cluster)
- cluster.check_trait(NameTrait)
- self._clusters[cluster.name] = cluster
-
- @contextmanager
- def patch(self, cluster_list):
- """Patch this Clusters instance with a new list of clusters in a
- contextmanager. Intended for testing purposes."""
- old_clusters = self._clusters.copy()
- self.replace(cluster_list)
- yield self
- self._clusters = old_clusters
-
- def __iter__(self):
- return iter(self._clusters)
-
- def __len__(self):
- return len(self._clusters)
-
- def __getitem__(self, name):
- try:
- return self._clusters[name]
- except KeyError:
- raise self.ClusterNotFound('Unknown cluster %s, valid clusters: %s' % (
- name, ', '.join(self._clusters.keys())))
-
-
-
-DEFAULT_SEARCH_PATHS = (
- os.environ.get('AURORA_CONFIG_ROOT') or '/etc/aurora',
- os.path.expanduser('~/.aurora')
-)
-
-
-CLUSTERS = Clusters(())
-
-
-def load():
- """(re-)load all clusters from the search path."""
- for search_path, ext in itertools.product(DEFAULT_SEARCH_PATHS, Clusters.LOADERS):
- filename = os.path.join(search_path, 'clusters' + ext)
- if os.path.exists(filename):
- CLUSTERS.update(Clusters.from_file(filename).values())
-
-
-load()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/common/http_signaler.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/common/http_signaler.py b/src/main/python/twitter/aurora/common/http_signaler.py
deleted file mode 100644
index 778ddc4..0000000
--- a/src/main/python/twitter/aurora/common/http_signaler.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import contextlib
-from socket import timeout as SocketTimeout
-import sys
-
-from twitter.common import log
-from twitter.common.lang import Compatibility
-
-if Compatibility.PY3:
- from http.client import HTTPException
- import urllib.request as urllib_request
- from urllib.error import URLError, HTTPError
-else:
- from httplib import HTTPException
- import urllib2 as urllib_request
- from urllib2 import URLError, HTTPError
-
-
-class HttpSignaler(object):
- """Simple HTTP endpoint wrapper to check health or trigger quitquitquit/abortabortabort"""
- TIMEOUT_SECS = 1.0
- FAILURE_REASON_LENGTH = 10
-
- class Error(Exception): pass
- class QueryError(Error): pass
-
- def __init__(self, port, host='localhost', timeout_secs=TIMEOUT_SECS):
- self._host = host
- self._url_base = 'http://%s:%d/' % (host, port)
- self._timeout_secs = timeout_secs
-
- def url(self, endpoint):
- return self._url_base + endpoint
-
- @property
- def opener(self):
- return urllib_request.urlopen
-
- def query(self, endpoint, data=None):
- """Request an HTTP endpoint with a GET request (or POST if data is not None)"""
- url = self.url(endpoint)
- log.debug("%s: %s %s" % (self.__class__.__name__, 'GET' if data is None else 'POST', url))
-
- def raise_error(reason):
- raise self.QueryError('Failed to signal %s: %s' % (self.url(endpoint), reason))
-
- try:
- with contextlib.closing(
- self.opener(url, data, timeout=self._timeout_secs)) as fp:
- return fp.read()
- except (HTTPException, SocketTimeout) as e:
- # the type of an HTTPException is typically more useful than its contents (since for example
- # BadStatusLines are often empty). likewise with socket.timeout.
- raise_error('Error within %s' % e.__class__.__name__)
- except (URLError, HTTPError) as e:
- raise_error(e)
- except Exception as e:
- raise_error('Unexpected error: %s' % e)
-
- def __call__(self, endpoint, use_post_method=False, expected_response=None):
- """Returns a (boolean, string|None) tuple of (call success, failure reason)"""
- try:
- response = self.query(endpoint, '' if use_post_method else None).strip().lower()
- if expected_response is not None and response != expected_response:
- def shorten(string):
- return (string if len(string) < self.FAILURE_REASON_LENGTH
- else "%s..." % string[:self.FAILURE_REASON_LENGTH - 3])
- reason = 'Response differs from expected response (expected "%s", got "%s")'
- log.warning(reason % (expected_response, response))
- return (False, reason % (shorten(str(expected_response)), shorten(str(response))))
- else:
- return (True, None)
- except self.QueryError as e:
- return (False, str(e))
-
- def health(self):
- return self('health', use_post_method=False, expected_response='ok')
-
- def quitquitquit(self):
- return self('quitquitquit', use_post_method=True)
-
- def abortabortabort(self):
- return self('abortabortabort', use_post_method=True)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/BUILD b/src/main/python/twitter/aurora/config/BUILD
deleted file mode 100644
index 91cb3e4..0000000
--- a/src/main/python/twitter/aurora/config/BUILD
+++ /dev/null
@@ -1,43 +0,0 @@
-import os
-
-# Alias for src/main/python/twitter/aurora/config/schema
-python_library(
- name = 'schema',
- dependencies = [
- pants('src/main/python/twitter/aurora/config/schema'),
- ]
-)
-
-python_library(
- name = 'config',
- sources = (
- '__init__.py',
- 'loader.py',
- 'port_resolver.py',
- 'thrift.py',
- ),
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('src/main/python/twitter/aurora/common'),
- pants('src/main/python/twitter/aurora/config/schema'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ],
-
-)
-
-python_library(
- name = 'config-packaged',
- dependencies = [
- pants(':config'),
-
- # covering dependencies
- pants('src/main/python/twitter/thermos/config'),
- ],
- provides = setup_py(
- name = 'twitter.aurora.config',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- description = 'Aurora/Thermos Pystachio schemas for describing job configurations.',
- license = 'Apache License, Version 2.0',
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/__init__.py b/src/main/python/twitter/aurora/config/__init__.py
deleted file mode 100644
index a3ba981..0000000
--- a/src/main/python/twitter/aurora/config/__init__.py
+++ /dev/null
@@ -1,271 +0,0 @@
-from __future__ import absolute_import
-
-from collections import defaultdict
-
-from twitter.aurora.common.aurora_job_key import AuroraJobKey
-from twitter.aurora.config.schema.base import MesosContext
-from twitter.thermos.config.loader import PortExtractor, ThermosTaskWrapper
-from twitter.thermos.config.schema import ThermosContext
-
-from .loader import AuroraConfigLoader
-from .port_resolver import PortResolver
-from .thrift import convert as convert_thrift, InvalidConfig as InvalidThriftConfig
-
-from pystachio import Empty, Environment, Ref
-
-__all__ = ('AuroraConfig', 'PortResolver')
-
-
-class AuroraConfig(object):
- class Error(Exception): pass
-
- class InvalidConfig(Error):
- def __str__(self):
- return 'The configuration was invalid: %s' % self.args[0]
-
- @classmethod
- def plugins(cls):
- """A stack of callables to apply to the config on load."""
- return []
-
- @classmethod
- def pick(cls, env, name, bindings, select_cluster=None, select_role=None, select_env=None):
- # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
-
- job_list = env.get('jobs', [])
- if not job_list:
- raise ValueError('No job defined in this config!')
-
- def maybe_bind(j):
- return j.bind(*bindings) if bindings else j
-
- if name is None:
- if len(job_list) > 1:
- raise ValueError('Configuration has multiple jobs but no job name specified!')
- return maybe_bind(job_list[0])
-
- # TODO(wfarner): Rework this and calling code to make name optional as well.
- def match_name(job):
- return str(job.name()) == name
- def match_cluster(job):
- return select_cluster is None or str(job.cluster()) == select_cluster
- def match_env(job):
- return select_env is None or str(job.environment()) == select_env
- def match_role(job):
- return select_role is None or str(job.role()) == select_role
-
- bound_jobs = map(maybe_bind, job_list)
- matches = [j for j in bound_jobs if
- all([match_cluster(j), match_role(j), match_env(j), match_name(j)])]
-
- if len(matches) == 0:
- msg = "Could not find job %s/%s/%s/%s\n" % (
- select_cluster or '*', select_role or '*', select_env or '*', name)
- for j in bound_jobs:
- if j.environment() is Empty:
- msg += "Job %s/%s/%s/%s in configuration file doesn't specify an environment\n" % (
- j.cluster(), j.role(), '{MISSING}', j.name()
- )
- msg += cls._candidate_jobs_str(bound_jobs)
- raise ValueError(msg)
-
- elif len(matches) > 1:
- msg = 'Multiple jobs match, please disambiguate by specifying a job key.\n'
- msg += cls._candidate_jobs_str(bound_jobs)
- raise ValueError(msg)
- else:
- return matches[0]
-
- @staticmethod
- def _candidate_jobs_str(job_list):
- assert(job_list)
- job_list = [" %s/%s/%s/%s" % (
- j.cluster(), j.role(),
- j.environment() if j.environment() is not Empty else "{MISSING}",
- j.name())
- for j in job_list]
- return 'Candidates are:\n' + '\n'.join(job_list)
-
- @classmethod
- def apply_plugins(cls, config, env=None):
- for plugin in cls.plugins():
- if not callable(plugin):
- raise cls.Error('Invalid configuration plugin %r, should be callable!' % plugin)
- plugin(config, env)
- return config
-
- @classmethod
- def load(
- cls, filename, name=None, bindings=None,
- select_cluster=None, select_role=None, select_env=None):
- # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
- env = AuroraConfigLoader.load(filename)
- return cls.apply_plugins(
- cls(cls.pick(env, name, bindings, select_cluster, select_role, select_env)), env)
-
- @classmethod
- def load_json(
- cls, filename, name=None, bindings=None,
- select_cluster=None, select_role=None, select_env=None):
- # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
- job = AuroraConfigLoader.load_json(filename)
- return cls.apply_plugins(cls(job.bind(*bindings) if bindings else job))
-
- @classmethod
- def loads_json(cls, string, name=None, bindings=None, select_cluster=None, select_env=None):
- # TODO(atollenaere): should take a JobKey when non-jobkey interface is deprecated
- job = AuroraConfigLoader.loads_json(string)
- return cls.apply_plugins(cls(job.bind(*bindings) if bindings else job))
-
- @classmethod
- def validate_job(cls, job):
- """
- Validate and sanitize the input job
-
- Currently, the validation stage simply ensures that the job has all required fields.
- self.InvalidConfig is raised if any required fields are not present.
- """
- def has(pystachio_type, thing):
- return getattr(pystachio_type, 'has_%s' % thing)()
- for required in ("cluster", "task", "role"):
- if not has(job, required):
- raise cls.InvalidConfig(
- '%s required for job "%s"' % (required.capitalize(), job.name()))
- if not has(job.task(), 'processes'):
- raise cls.InvalidConfig('Processes required for task on job "%s"' % job.name())
-
- @classmethod
- def standard_bindings(cls, job):
- # Rewrite now-deprecated bindings into their proper form.
- return job.bind({
- Ref.from_address('mesos.role'): '{{role}}',
- Ref.from_address('mesos.cluster'): '{{cluster}}',
- Ref.from_address('thermos.user'): '{{role}}',
- })
-
- def __init__(self, job):
- self.validate_job(job)
- self._job = self.standard_bindings(job)
- self._packages = []
- self.binding_dicts = defaultdict(dict)
- self.hooks = []
-
- def context(self, instance=None):
- context = dict(instance=instance)
- # Filter unspecified values
- return Environment(mesos=MesosContext(dict((k, v) for k, v in context.items() if v)))
-
- def job(self):
- interpolated_job = self._job % self.context()
-
- # TODO(wickman) Once thermos is onto thrift instead of pystachio, use
- # %%replacements%% instead.
- #
- # Typecheck against the Job, with the following free variables unwrapped at the Task level:
- # - a dummy {{mesos.instance}}
- # - dummy values for the {{thermos.ports}} context, to allow for their use in task_links
- env = dict(mesos=Environment(instance=0))
- if interpolated_job.task_links() is not Empty:
- try:
- dummy_ports = dict(
- (port, 31337) for port in PortExtractor.extract(interpolated_job.task_links()))
- except PortExtractor.InvalidPorts as err:
- raise self.InvalidConfig('Invalid port references in task_links! %s' % err)
- env.update(thermos=ThermosContext(ports=dummy_ports))
- typecheck = interpolated_job.bind(Environment(env)).check()
- if not typecheck.ok():
- raise self.InvalidConfig(typecheck.message())
- interpolated_job = interpolated_job(task_links=self.task_links())
- try:
- return convert_thrift(interpolated_job, self._packages, self.ports())
- except InvalidThriftConfig as e:
- raise self.InvalidConfig(str(e))
-
- def bind(self, binding):
- self._job = self._job.bind(binding)
-
- def raw(self):
- return self._job
-
- # This stinks to high heaven
- def update_job(self, new_job):
- self._job = new_job
-
- def instances(self):
- return self._job.instances().get()
-
- def task(self, instance):
- return (self._job % self.context(instance)).task()
-
- def name(self):
- return self._job.name().get()
-
- def role(self):
- return self._job.role().get()
-
- def cluster(self):
- return self._job.cluster().get()
-
- def environment(self):
- return self._job.environment().get()
-
- def job_key(self):
- return AuroraJobKey(self.cluster(), self.role(), self.environment(), self.name())
-
- def ports(self):
- """Return the list of ports that need to be allocated by the scheduler."""
-
- # Strictly speaking this is wrong -- it is possible to do things like
- # {{thermos.ports[instance_{{mesos.instance}}]}}
- # which can only be extracted post-unwrapping. This means that validating
- # the state of the announce configuration could be problematic if people
- # try to do complicated things.
- referenced_ports = ThermosTaskWrapper(self._job.task(), strict=False).ports()
- resolved_portmap = PortResolver.resolve(self._job.announce().portmap().get()
- if self._job.has_announce() else {})
-
- # values of the portmap that are not integers => unallocated
- unallocated = set(port for port in resolved_portmap.values() if not isinstance(port, int))
-
- # find referenced {{thermos.portmap[ports]}} that are not resolved by the portmap
- unresolved_references = set(
- port for port in (resolved_portmap.get(port_ref, port_ref) for port_ref in referenced_ports)
- if not isinstance(port, int))
-
- return unallocated | unresolved_references
-
- def has_health_port(self):
- return "health" in ThermosTaskWrapper(self._job.task(), strict=False).ports()
-
- def task_links(self):
- # {{mesos.instance}} --> %shard_id%
- # {{thermos.ports[foo]}} --> %port:foo%
- task_links = self._job.task_links()
- if task_links is Empty:
- return task_links
- _, uninterp = task_links.interpolate()
- substitutions = {
- Ref.from_address('mesos.instance'): '%shard_id%'
- }
- port_scope = Ref.from_address('thermos.ports')
- for ref in uninterp:
- subscope = port_scope.scoped_to(ref)
- if subscope:
- substitutions[ref] = '%%port:%s%%' % subscope.action().value
- return task_links.bind(substitutions)
-
- def update_config(self):
- return self._job.update_config()
-
- def add_package(self, package):
- self._packages.append(package)
-
- # TODO(wickman) Kill package() once MESOS-3191 is in.
- def package(self):
- pass
-
- def is_dedicated(self):
- return self._job.has_constraints() and 'dedicated' in self._job.constraints()
-
- def __repr__(self):
- return '%s(%r)' % (self.__class__.__name__, self._job)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/loader.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/loader.py b/src/main/python/twitter/aurora/config/loader.py
deleted file mode 100644
index e3e5559..0000000
--- a/src/main/python/twitter/aurora/config/loader.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import json
-import pkgutil
-import textwrap
-
-from twitter.aurora.config.schema import base as base_schema
-
-from pystachio.config import Config as PystachioConfig
-
-
-class AuroraConfigLoader(PystachioConfig):
- SCHEMA_MODULES = []
-
- @classmethod
- def assembled_schema(cls, schema_modules):
- default_schema = [super(AuroraConfigLoader, cls).DEFAULT_SCHEMA]
- default_schema.extend('from %s import *' % module.__name__ for module in schema_modules)
- return '\n'.join(default_schema)
-
- @classmethod
- def register_schema(cls, schema_module):
- """Register the schema defined in schema_module, equivalent to doing
-
- from schema_module.__name__ import *
-
- before all pystachio configurations are evaluated.
- """
- cls.SCHEMA_MODULES.append(schema_module)
- cls.DEFAULT_SCHEMA = cls.assembled_schema(cls.SCHEMA_MODULES)
-
- @classmethod
- def register_schemas_from(cls, package):
- """Register schemas from all modules in a particular package."""
- for _, submodule, is_package in pkgutil.iter_modules(package.__path__):
- if is_package:
- continue
- cls.register_schema(
- __import__('%s.%s' % (package.__name__, submodule), fromlist=[package.__name__]))
-
- @classmethod
- def flush_schemas(cls):
- """Flush all schemas from AuroraConfigLoader. Intended for test use only."""
- cls.SCHEMA_MODULES = []
- cls.register_schema(base_schema)
-
- @classmethod
- def load(cls, loadable):
- return cls.load_raw(loadable).environment
-
- @classmethod
- def load_raw(cls, loadable):
- return cls(loadable)
-
- @classmethod
- def load_json(cls, filename):
- with open(filename) as fp:
- return base_schema.Job.json_load(fp)
-
- @classmethod
- def loads_json(cls, string):
- return base_schema.Job(json.loads(string))
-
-
-AuroraConfigLoader.flush_schemas()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/port_resolver.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/port_resolver.py b/src/main/python/twitter/aurora/config/port_resolver.py
deleted file mode 100644
index 486095f..0000000
--- a/src/main/python/twitter/aurora/config/port_resolver.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from twitter.common.lang import Compatibility
-
-
-class PortResolver(object):
- class CycleException(Exception): pass
-
- @classmethod
- def resolve(cls, portmap):
- """
- Given an announce-style portmap, return a fully dereferenced portmap.
-
- For example, given the portmap:
- {
- 'http': 80,
- 'aurora: 'http',
- 'https': 'aurora',
- 'thrift': 'service'
- }
-
- Returns {'http': 80, 'aurora': 80, 'https': 80, 'thrift': 'service'}
- """
- for (name, port) in portmap.items():
- if not isinstance(name, Compatibility.string):
- raise ValueError('All portmap keys must be strings!')
- if not isinstance(port, (int, Compatibility.string)):
- raise ValueError('All portmap values must be strings or integers!')
-
- portmap = portmap.copy()
- for port in list(portmap):
- try:
- portmap[port] = int(portmap[port])
- except ValueError:
- continue
-
- def resolve_one(static_port):
- visited = set()
- root = portmap[static_port]
- while root in portmap:
- visited.add(root)
- if portmap[root] in visited:
- raise cls.CycleException('Found cycle in portmap!')
- root = portmap[root]
- return root
-
- return dict((name, resolve_one(name)) for name in portmap)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/recipes.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/recipes.py b/src/main/python/twitter/aurora/config/recipes.py
deleted file mode 100644
index b2dc23c..0000000
--- a/src/main/python/twitter/aurora/config/recipes.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import os
-
-from .loader import AuroraConfigLoader
-
-import pkg_resources
-
-
-class Recipes(object):
- """
- Encapsulate a registry of Recipes (i.e. tasks to mutate the behavior of other tasks.)
- """
- REGISTRY = {}
- RECIPE_EXTENSION = '.aurora_recipe'
-
- class Error(Exception): pass
- class UnknownRecipe(Error): pass
-
- @classmethod
- def get(cls, name):
- if name not in cls.REGISTRY:
- raise cls.UnknownRecipe('Could not find recipe %s!' % name)
- return cls.REGISTRY[name]
-
- @classmethod
- def include_one(cls, filename):
- recipe_env = AuroraConfigLoader.load(filename)
- cls.REGISTRY.update(recipe_env.get('recipes', {}))
-
- @classmethod
- def include_module(cls, module):
- for filename in pkg_resources.resource_listdir(module, ''):
- if filename.endswith(cls.RECIPE_EXTENSION):
- cls.include_one(os.path.join(module.replace('.', os.sep), filename))
-
- @classmethod
- def include(cls, path):
- if os.path.isfile(path):
- cls.include_one(path)
- elif os.path.isdir(path):
- for filename in os.listdir(path):
- if filename.endswith(cls.RECIPE_EXTENSION):
- cls.include_one(os.path.join(path, filename))
- else:
- raise ValueError('Could not find %s' % path)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/repl.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/repl.py b/src/main/python/twitter/aurora/config/repl.py
deleted file mode 100644
index d26f94d..0000000
--- a/src/main/python/twitter/aurora/config/repl.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from twitter.aurora.config.loader import AuroraConfigLoader
-from twitter.common.lang import Compatibility
-
-
-
-import code
-code.interact('Mesos Config REPL',
- local=Compatibility.exec_function(AuroraConfigLoader.DEFAULT_SCHEMA, globals()))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/schema/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/schema/BUILD b/src/main/python/twitter/aurora/config/schema/BUILD
deleted file mode 100644
index fd08ec7..0000000
--- a/src/main/python/twitter/aurora/config/schema/BUILD
+++ /dev/null
@@ -1,9 +0,0 @@
-python_library(
- name = 'schema',
- sources = ['base.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- pants('src/main/python/twitter/thermos/config:schema'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/schema/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/schema/__init__.py b/src/main/python/twitter/aurora/config/schema/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/schema/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/schema/base.py b/src/main/python/twitter/aurora/config/schema/base.py
deleted file mode 100644
index b3d437f..0000000
--- a/src/main/python/twitter/aurora/config/schema/base.py
+++ /dev/null
@@ -1,122 +0,0 @@
-from twitter.thermos.config.schema import *
-
-from gen.twitter.aurora.constants import DEFAULT_ENVIRONMENT
-
-
-# TODO(wickman) Bind {{mesos.instance}} to %shard_id%
-class MesosContext(Struct):
- # The instance id (i.e. replica id, shard id) in the context of a task
- instance = Required(Integer)
-
-
-# AppApp layout setup
-class AppPackage(Struct):
- name = Required(String)
- version = Default(String, 'latest')
-
-class AppLayout(Struct):
- packages = Default(List(AppPackage), [])
-
-
-# The object bound into the {{packer}} namespace.
-# Referenced by
-# {{packer[role][name][version]}}
-#
-# Where version =
-# number (integer)
-# 'live' (live package)
-# 'latest' (highest version number)
-#
-# For example if you'd like to create a copy process for a particular
-# package,
-# copy_latest = Process(
-# name = 'copy-{{package_name}}',
-# cmdline = '{{packer[{{role}}][{{package_name}}][latest].copy_command}}')
-# processes = [
-# copy_latest.bind(package_name = 'labrat'),
-# copy_latest.bind(package_name = 'packer')
-# ]
-class PackerObject(Struct):
- package = String
- package_uri = String
- copy_command = String
-
-
-class UpdateConfig(Struct):
- batch_size = Default(Integer, 1)
- restart_threshold = Default(Integer, 60)
- watch_secs = Default(Integer, 30)
- max_per_shard_failures = Default(Integer, 0)
- max_total_failures = Default(Integer, 0)
-
-
-class HealthCheckConfig(Struct):
- initial_interval_secs = Default(Float, 60.0)
- interval_secs = Default(Float, 30.0)
- timeout_secs = Default(Float, 1.0)
- max_consecutive_failures = Default(Integer, 0)
-
-
-class Announcer(Struct):
- primary_port = Default(String, 'http')
-
- # Portmap can either alias two ports together, e.g.
- # aurora <= http
- # Or it can be used to alias static ports to endpoints, e.g.
- # http <= 80
- # https <= 443
- # aurora <= https
- portmap = Default(Map(String, String), {
- 'aurora': '{{primary_port}}'
- })
-
-
-# The executorConfig populated inside of TaskConfig.
-class MesosTaskInstance(Struct):
- task = Required(Task)
- layout = AppLayout
- instance = Required(Integer)
- role = Required(String)
- announce = Announcer
- environment = Default(String, DEFAULT_ENVIRONMENT)
- health_check_interval_secs = Default(Integer, 30) # DEPRECATED (MESOS-2649)
- health_check_config = Default(HealthCheckConfig, HealthCheckConfig())
-
-
-class MesosJob(Struct):
- name = Default(String, '{{task.name}}')
- role = Required(String)
- contact = String
- cluster = Required(String)
- environment = Required(String)
- instances = Default(Integer, 1)
- task = Required(Task)
- recipes = List(String)
- announce = Announcer
-
- cron_schedule = String
- cron_policy = String # these two are aliases of each other. default is KILL_EXISTING
- cron_collision_policy = String # if unspecified.
- # cron_policy is DEPRECATED (MESOS-2491) in favor of
- # cron_collision_policy.
-
- update_config = Default(UpdateConfig, UpdateConfig())
-
- constraints = Map(String, String)
- daemon = Boolean # daemon and service are aliased together.
- service = Boolean # daemon is DEPRECATED (MESOS-2492) in favor of
- # service. by default, service is False.
- max_task_failures = Default(Integer, 1)
- production = Default(Boolean, False)
- priority = Default(Integer, 0)
- health_check_interval_secs = Integer # DEPRECATED in favor of health_check_config (MESOS-2649).
- health_check_config = HealthCheckConfig
- task_links = Map(String, String)
-
- layout = AppLayout # DEPRECATED in favor of directory sandboxes
-
- enable_hooks = Default(Boolean, False) # enable client API hooks; from env python-list 'hooks'
-
-
-Job = MesosJob
-Service = Job(service = True)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/config/thrift.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/config/thrift.py b/src/main/python/twitter/aurora/config/thrift.py
deleted file mode 100644
index 8f6a5ce..0000000
--- a/src/main/python/twitter/aurora/config/thrift.py
+++ /dev/null
@@ -1,259 +0,0 @@
-import getpass
-import re
-
-from twitter.aurora.config.schema.base import (
- HealthCheckConfig,
- MesosContext,
- MesosTaskInstance,
-)
-from twitter.common.lang import Compatibility
-from twitter.thermos.config.loader import ThermosTaskValidator
-
-from gen.twitter.aurora.constants import GOOD_IDENTIFIER_PATTERN_PYTHON, AURORA_EXECUTOR_NAME
-from gen.twitter.aurora.ttypes import (
- Constraint,
- CronCollisionPolicy,
- ExecutorConfig,
- Identity,
- JobConfiguration,
- JobKey,
- LimitConstraint,
- Package,
- TaskConfig,
- TaskConstraint,
- ValueConstraint,
-)
-
-from pystachio import Empty, Ref
-
-__all__ = (
- 'InvalidConfig',
- 'convert'
-)
-
-
-class InvalidConfig(ValueError):
- pass
-
-
-def constraints_to_thrift(constraints):
- """Convert a python dictionary to a set of Constraint thrift objects."""
- result = set()
- for attribute, constraint_value in constraints.items():
- assert isinstance(attribute, Compatibility.string) and (
- isinstance(constraint_value, Compatibility.string)), (
- "Both attribute name and value in constraints must be string")
- constraint = Constraint()
- constraint.name = attribute
- task_constraint = TaskConstraint()
- if constraint_value.startswith('limit:'):
- task_constraint.limit = LimitConstraint()
- try:
- task_constraint.limit.limit = int(constraint_value.replace('limit:', '', 1))
- except ValueError:
- print('%s is not a valid limit value, must be integer' % constraint_value)
- raise
- else:
- # Strip off the leading negation if present.
- negated = constraint_value.startswith('!')
- if negated:
- constraint_value = constraint_value[1:]
- task_constraint.value = ValueConstraint(negated, set(constraint_value.split(',')))
- constraint.constraint = task_constraint
- result.add(constraint)
- return result
-
-
-def task_instance_from_job(job, instance):
- instance_context = MesosContext(instance=instance)
- # TODO(Sathya): Remove health_check_interval_secs references after deprecation cycle is complete.
- health_check_config = HealthCheckConfig()
- if job.has_health_check_interval_secs():
- health_check_config = HealthCheckConfig(interval_secs=job.health_check_interval_secs().get())
- elif job.has_health_check_config():
- health_check_config = job.health_check_config()
- ti = MesosTaskInstance(task=job.task(),
- layout=job.layout(),
- role=job.role(),
- health_check_interval_secs=health_check_config.interval_secs().get(),
- health_check_config=health_check_config,
- instance=instance)
- if job.has_announce():
- ti = ti(announce=job.announce())
- if job.has_environment():
- ti = ti(environment=job.environment())
- return ti.bind(mesos=instance_context).interpolate()
-
-
-def translate_cron_policy(policy):
- cron_policy = CronCollisionPolicy._NAMES_TO_VALUES.get(policy.get())
- if cron_policy is None:
- raise InvalidConfig('Invalid cron policy: %s' % policy.get())
- return cron_policy
-
-
-def fully_interpolated(pystachio_object, coerce_fn=lambda i: i):
- # Extract a fully-interpolated unwrapped object from pystachio_object or raise InvalidConfig.
- #
- # TODO(ksweeney): Remove this once Pystachio 1.0 changes the behavior of interpolate() to return
- # unwrapped objects and fail when there are any unbound refs.
- if not pystachio_object.check().ok():
- raise InvalidConfig(pystachio_object.check().message())
-
- # If an object type-checks it's okay to use the raw value from the wrapped object returned by
- # interpolate. Without the previous check value.get() could return a string with mustaches
- # instead of an object of the expected type.
- value, _ = pystachio_object.interpolate()
- return coerce_fn(value.get())
-
-
-def select_cron_policy(cron_policy, cron_collision_policy):
- if cron_policy is Empty and cron_collision_policy is Empty:
- return CronCollisionPolicy.KILL_EXISTING
- elif cron_policy is not Empty and cron_collision_policy is Empty:
- return translate_cron_policy(cron_policy)
- elif cron_policy is Empty and cron_collision_policy is not Empty:
- return translate_cron_policy(cron_collision_policy)
- else:
- raise InvalidConfig('Specified both cron_policy and cron_collision_policy!')
-
-
-def select_service_bit(job):
- if not job.has_daemon() and not job.has_service():
- return False
- elif job.has_daemon() and not job.has_service():
- return fully_interpolated(job.daemon(), bool)
- elif not job.has_daemon() and job.has_service():
- return fully_interpolated(job.service(), bool)
- else:
- raise InvalidConfig('Specified both daemon and service bits!')
-
-
-# TODO(wickman) Due to MESOS-2718 we should revert to using the MesosTaskInstance.
-#
-# Using the MesosJob instead of the MesosTaskInstance was to allow for
-# planned future use of fields such as 'cluster' and to allow for conversion
-# from Job=>Task to be done entirely on the executor, but instead this had
-# made it impossible to run idempotent updates.
-#
-# In the meantime, we are erasing fields of the Job that are controversial.
-# This achieves roughly the same effect as using the MesosTaskInstance.
-# The future work is tracked at MESOS-2727.
-ALIASED_FIELDS = (
- 'cron_policy',
- 'cron_collision_policy',
- 'update_config',
- 'daemon',
- 'service',
- 'instances'
-)
-
-
-def filter_aliased_fields(job):
- return job(**dict((key, Empty) for key in ALIASED_FIELDS))
-
-
-def assert_valid_field(field, identifier):
- VALID_IDENTIFIER = re.compile(GOOD_IDENTIFIER_PATTERN_PYTHON)
- if not isinstance(identifier, Compatibility.string):
- raise InvalidConfig("%s must be a string" % field)
- if not VALID_IDENTIFIER.match(identifier):
- raise InvalidConfig("Invalid %s '%s'" % (field, identifier))
- return identifier
-
-
-MESOS_INSTANCE_REF = Ref.from_address('mesos.instance')
-THERMOS_PORT_SCOPE_REF = Ref.from_address('thermos.ports')
-THERMOS_TASK_ID_REF = Ref.from_address('thermos.task_id')
-
-
-# TODO(wickman) Make this a method directly on an AuroraConfig so that we don't
-# need the packages/ports shenanigans.
-def convert(job, packages=frozenset(), ports=frozenset()):
- """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration."""
-
- owner = Identity(role=fully_interpolated(job.role()), user=getpass.getuser())
- key = JobKey(
- role=assert_valid_field('role', fully_interpolated(job.role())),
- environment=assert_valid_field('environment', fully_interpolated(job.environment())),
- name=assert_valid_field('name', fully_interpolated(job.name())))
-
- task_raw = job.task()
-
- MB = 1024 * 1024
- task = TaskConfig()
-
- def not_empty_or(item, default):
- return default if item is Empty else fully_interpolated(item)
-
- # job components
- task.jobName = fully_interpolated(job.name())
- task.environment = fully_interpolated(job.environment())
- task.production = fully_interpolated(job.production(), bool)
- task.isService = select_service_bit(job)
- task.maxTaskFailures = fully_interpolated(job.max_task_failures())
- task.priority = fully_interpolated(job.priority())
- task.contactEmail = not_empty_or(job.contact(), None)
-
- # Add package tuples to a task, to display in the scheduler UI.
- task.packages = frozenset(
- Package(role=str(role), name=str(package_name), version=int(version))
- for role, package_name, version in packages)
-
- # task components
- if not task_raw.has_resources():
- raise InvalidConfig('Task must specify resources!')
-
- if (fully_interpolated(task_raw.resources().ram()) == 0
- or fully_interpolated(task_raw.resources().disk()) == 0):
- raise InvalidConfig('Must specify ram and disk resources, got ram:%r disk:%r' % (
- fully_interpolated(task_raw.resources().ram()),
- fully_interpolated(task_raw.resources().disk())))
-
- task.numCpus = fully_interpolated(task_raw.resources().cpu())
- task.ramMb = fully_interpolated(task_raw.resources().ram()) / MB
- task.diskMb = fully_interpolated(task_raw.resources().disk()) / MB
- if task.numCpus <= 0 or task.ramMb <= 0 or task.diskMb <= 0:
- raise InvalidConfig('Task has invalid resources. cpu/ramMb/diskMb must all be positive: '
- 'cpu:%r ramMb:%r diskMb:%r' % (task.numCpus, task.ramMb, task.diskMb))
-
- task.owner = owner
- task.requestedPorts = ports
- task.taskLinks = not_empty_or(job.task_links(), {})
- task.constraints = constraints_to_thrift(not_empty_or(job.constraints(), {}))
-
- underlying, refs = job.interpolate()
-
- # need to fake an instance id for the sake of schema checking
- underlying_checked = underlying.bind(mesos = {'instance': 31337})
- try:
- ThermosTaskValidator.assert_valid_task(underlying_checked.task())
- except ThermosTaskValidator.InvalidTaskError as e:
- raise InvalidConfig('Task is invalid: %s' % e)
- if not underlying_checked.check().ok():
- raise InvalidConfig('Job not fully specified: %s' % underlying.check().message())
-
- unbound = []
- for ref in refs:
- if ref == THERMOS_TASK_ID_REF or ref == MESOS_INSTANCE_REF or (
- Ref.subscope(THERMOS_PORT_SCOPE_REF, ref)):
- continue
- unbound.append(ref)
-
- if unbound:
- raise InvalidConfig('Config contains unbound variables: %s' % ' '.join(map(str, unbound)))
-
- cron_schedule = not_empty_or(job.cron_schedule(), '')
- cron_policy = select_cron_policy(job.cron_policy(), job.cron_collision_policy())
-
- task.executorConfig = ExecutorConfig(
- name=AURORA_EXECUTOR_NAME,
- data=filter_aliased_fields(underlying).json_dumps())
-
- return JobConfiguration(
- key=key,
- owner=owner,
- cronSchedule=cron_schedule,
- cronCollisionPolicy=cron_policy,
- taskConfig=task,
- instanceCount=fully_interpolated(job.instances()))
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/BUILD b/src/main/python/twitter/aurora/executor/BUILD
deleted file mode 100644
index 01701e9..0000000
--- a/src/main/python/twitter/aurora/executor/BUILD
+++ /dev/null
@@ -1,139 +0,0 @@
-import os
-
-python_library(
- name = 'thermos_task_runner',
- sources = ['thermos_task_runner.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/thermos/common'),
- pants('src/main/python/twitter/thermos/config:schema'),
- pants('src/main/python/twitter/thermos/core'),
- pants('src/main/python/twitter/thermos/monitoring:monitor'),
- pants('src/main/python/twitter/aurora/common:http_signaler'),
- pants('src/main/python/twitter/aurora/executor/common:status_checker'),
- pants('src/main/python/twitter/aurora/executor/common:task_info'),
- pants('src/main/python/twitter/aurora/executor/common:task_runner'),
- ]
-)
-
-python_library(
- name = 'executor_detector',
- sources = ['executor_detector.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/string'),
- ]
-)
-
-python_library(
- name = 'executor_vars',
- sources = ['executor_vars.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:psutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
- pants('aurora/twitterdeps/src/python/twitter/common/python'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('aurora/twitterdeps/src/python/twitter/common/string'),
- ]
-)
-
-python_library(
- name = 'status_manager',
- sources = ['status_manager.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/aurora/executor/common:status_checker'),
- ]
-)
-
-python_library(
- name = 'thermos_executor_base',
- sources = ['executor_base.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- pants('src/main/python/twitter/aurora:mesos-core'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'thermos_executor',
- sources = ['thermos_executor.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- pants(':status_manager'),
- pants(':thermos_executor_base'),
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/concurrent'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/aurora/executor/common:kill_manager'),
- pants('src/main/python/twitter/aurora/executor/common:sandbox'),
- pants('src/main/python/twitter/aurora/executor/common:task_info'),
- pants('src/main/python/twitter/aurora/executor/common:task_runner'),
- pants('src/main/python/twitter/aurora:mesos-core'),
- ]
-)
-
-python_library(
- name = 'thermos_runner',
- sources = ['thermos_runner.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/thermos/common:planner'),
- pants('src/main/python/twitter/thermos/config:schema'),
- pants('src/main/python/twitter/thermos/core'),
- ],
-)
-
-python_library(
- name = 'gc_executor',
- sources = ['gc_executor.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:psutil'),
- pants(':executor_detector'),
- pants(':thermos_executor_base'),
- pants('aurora/twitterdeps/src/python/twitter/common/collections'),
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/python/twitter/thermos/core:helper'),
- pants('src/main/python/twitter/thermos/core:inspector'),
- pants('src/main/python/twitter/thermos/monitoring:detector'),
- pants('src/main/python/twitter/thermos/monitoring:garbage'),
- pants('src/main/python/twitter/aurora/config:schema'),
- pants('src/main/python/twitter/aurora/executor/common:sandbox'),
- pants('src/main/python/twitter/aurora:mesos-core'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
-
-python_library(
- name = 'executor-packaged',
- dependencies = [
- # Covering dependencies
- pants('src/main/python/twitter/aurora/common'),
- pants('src/main/python/twitter/aurora/config'),
- pants('src/main/python/twitter/thermos/common'),
- pants('src/main/python/twitter/thermos/config'),
- pants('src/main/python/twitter/thermos/core'),
- pants('src/main/python/twitter/thermos/monitoring'),
- ],
- provides = setup_py(
- name = 'twitter.aurora.executor',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- ).with_binaries(
- gc_executor = pants('src/main/python/twitter/aurora/executor/bin:gc_executor'),
- thermos_executor = pants('src/main/python/twitter/aurora/executor/bin:thermos_executor'),
- thermos_runner = pants('src/main/python/twitter/aurora/executor/bin:thermos_runner'),
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/__init__.py b/src/main/python/twitter/aurora/executor/__init__.py
deleted file mode 100644
index b0d6433..0000000
--- a/src/main/python/twitter/aurora/executor/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__import__('pkg_resources').declare_namespace(__name__)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/bin/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/bin/BUILD b/src/main/python/twitter/aurora/executor/bin/BUILD
deleted file mode 100644
index 5e9cab1..0000000
--- a/src/main/python/twitter/aurora/executor/bin/BUILD
+++ /dev/null
@@ -1,47 +0,0 @@
-python_binary(
- name = 'thermos_executor',
- source = 'thermos_executor_main.py',
- entry_point = 'twitter.aurora.executor.bin.thermos_executor_main:proxy_main',
- ignore_errors = True,
- always_write_cache = True,
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
- pants('src/main/python/twitter/aurora/executor/common:executor_timeout'),
- pants('src/main/python/twitter/aurora/executor/common:health_checker'),
- pants('src/main/python/twitter/aurora/executor/common:sandbox'),
- pants('src/main/python/twitter/aurora/executor:executor_detector'),
- pants('src/main/python/twitter/aurora/executor:executor_vars'),
- pants('src/main/python/twitter/aurora/executor:thermos_executor'),
- pants('src/main/python/twitter/aurora/executor:thermos_task_runner'),
- ]
-)
-
-python_binary(
- name = 'gc_executor',
- source = 'gc_executor_main.py',
- entry_point = 'twitter.aurora.executor.bin.gc_executor_main:proxy_main',
- ignore_errors = True,
- always_write_cache = True,
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/python/twitter/aurora/executor:executor_detector'),
- pants('src/main/python/twitter/aurora/executor:executor_vars'),
- pants('src/main/python/twitter/aurora/executor:gc_executor'),
- ]
-)
-
-python_binary(
- name = 'thermos_runner',
- source = 'thermos_runner_main.py',
- entry_point = 'twitter.aurora.executor.bin.thermos_runner_main:proxy_main',
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/app'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/executor:thermos_runner'),
- ],
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/bin/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/bin/__init__.py b/src/main/python/twitter/aurora/executor/bin/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/bin/gc_executor_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/bin/gc_executor_main.py b/src/main/python/twitter/aurora/executor/bin/gc_executor_main.py
deleted file mode 100644
index fad20e6..0000000
--- a/src/main/python/twitter/aurora/executor/bin/gc_executor_main.py
+++ /dev/null
@@ -1,45 +0,0 @@
-"""Command-line entry point to the Thermos GC executor
-
-This module wraps the Thermos GC executor into an executable suitable for launching by a Mesos
-slave.
-
-"""
-
-from twitter.aurora.executor.executor_detector import ExecutorDetector
-from twitter.aurora.executor.gc_executor import ThermosGCExecutor
-from twitter.common import app, log
-from twitter.common.log.options import LogOptions
-from twitter.common.metrics.sampler import DiskMetricWriter
-from twitter.thermos.common.path import TaskPath
-
-import mesos
-
-
-app.configure(debug=True)
-
-
-# locate logs locally in executor sandbox
-LogOptions.set_simple(True)
-LogOptions.set_disk_log_level('DEBUG')
-LogOptions.set_log_dir(ExecutorDetector.LOG_PATH)
-
-
-def proxy_main():
- def main():
- # Create executor stub
- thermos_gc_executor = ThermosGCExecutor(checkpoint_root=TaskPath.DEFAULT_CHECKPOINT_ROOT)
- thermos_gc_executor.start()
-
- # Start metrics collection
- metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH)
- metric_writer.start()
-
- # Create driver stub
- driver = mesos.MesosExecutorDriver(thermos_gc_executor)
-
- # Start GC executor
- driver.run()
-
- log.info('MesosExecutorDriver.run() has finished.')
-
- app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/bin/thermos_executor_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/bin/thermos_executor_main.py b/src/main/python/twitter/aurora/executor/bin/thermos_executor_main.py
deleted file mode 100644
index 31e718e..0000000
--- a/src/main/python/twitter/aurora/executor/bin/thermos_executor_main.py
+++ /dev/null
@@ -1,67 +0,0 @@
-"""Command-line entry point to the Thermos Executor
-
-This module wraps the Thermos Executor into an executable suitable for launching by a Mesos
-slave.
-
-"""
-
-import os
-
-from twitter.common import app, log
-from twitter.common.log.options import LogOptions
-
-from twitter.aurora.executor.common.executor_timeout import ExecutorTimeout
-from twitter.aurora.executor.common.health_checker import HealthCheckerProvider
-from twitter.aurora.executor.thermos_executor import ThermosExecutor
-from twitter.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider
-
-import mesos
-
-
-app.configure(debug=True)
-LogOptions.set_simple(True)
-LogOptions.set_disk_log_level('DEBUG')
-LogOptions.set_log_dir('.')
-
-
-# TODO(wickman) Consider just having the OSS version require pip installed
-# thermos_runner binaries on every machine and instead of embedding the pex
-# as a resource, shell out to one on the PATH.
-def dump_runner_pex():
- import pkg_resources
- import twitter.aurora.executor.resources
- pex_name = 'thermos_runner.pex'
- runner_pex = os.path.join(os.path.realpath('.'), pex_name)
- with open(runner_pex, 'w') as fp:
- # TODO(wickman) Use shutil.copyfileobj to reduce memory footprint here.
- fp.write(pkg_resources.resource_stream(
- twitter.aurora.executor.resources.__name__, pex_name).read())
- return runner_pex
-
-
-def proxy_main():
- def main():
- runner_provider = DefaultThermosTaskRunnerProvider(
- dump_runner_pex(),
- artifact_dir=os.path.realpath('.'),
- )
-
- # Create executor stub
- thermos_executor = ThermosExecutor(
- runner_provider=runner_provider,
- status_providers=(HealthCheckerProvider(),),
- )
-
- # Create driver stub
- driver = mesos.MesosExecutorDriver(thermos_executor)
-
- # This is an ephemeral executor -- shutdown if we receive no tasks within a certain
- # time period
- ExecutorTimeout(thermos_executor.launched, driver).start()
-
- # Start executor
- driver.run()
-
- log.info('MesosExecutorDriver.run() has finished.')
-
- app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/bin/thermos_runner_main.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/bin/thermos_runner_main.py b/src/main/python/twitter/aurora/executor/bin/thermos_runner_main.py
deleted file mode 100644
index 602111e..0000000
--- a/src/main/python/twitter/aurora/executor/bin/thermos_runner_main.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from twitter.common import app
-from twitter.common.log.options import LogOptions
-from twitter.aurora.executor.thermos_runner import proxy_main as runner_proxy_main
-
-
-LogOptions.set_simple(True)
-
-
-def proxy_main():
- main = runner_proxy_main
-
- app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/BUILD b/src/main/python/twitter/aurora/executor/common/BUILD
deleted file mode 100644
index 9c932d1..0000000
--- a/src/main/python/twitter/aurora/executor/common/BUILD
+++ /dev/null
@@ -1,69 +0,0 @@
-python_library(
- name = 'status_checker',
- sources = ['status_checker.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/metrics'),
- ]
-)
-
-python_library(
- name = 'task_runner',
- sources = ['task_runner.py'],
- dependencies = [
- pants(':status_checker'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- ]
-)
-
-python_library(
- name = 'health_checker',
- sources = ['health_checker.py'],
- dependencies = [
- pants(':status_checker'),
- pants(':task_info'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('src/main/python/twitter/aurora/common:http_signaler'),
- ]
-)
-
-python_library(
- name = 'executor_timeout',
- sources = ['executor_timeout.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/exceptions'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- ]
-)
-
-python_library(
- name = 'kill_manager',
- sources = ['kill_manager.py'],
- dependencies = [
- pants(':status_checker'),
- ]
-)
-
-python_library(
- name = 'sandbox',
- sources = ['sandbox.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- ]
-)
-
-python_library(
- name = 'task_info',
- sources = ['task_info.py'],
- dependencies = [
- pants('src/main/python/twitter/aurora/BUILD.thirdparty:pystachio'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('src/main/python/twitter/aurora/config'),
- pants('src/main/python/twitter/aurora/config/schema'),
- pants('src/main/thrift/com/twitter/aurora/gen:py-thrift'),
- ]
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/__init__.py b/src/main/python/twitter/aurora/executor/common/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/executor_timeout.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/executor_timeout.py b/src/main/python/twitter/aurora/executor/common/executor_timeout.py
deleted file mode 100644
index 2828973..0000000
--- a/src/main/python/twitter/aurora/executor/common/executor_timeout.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from twitter.common import log
-from twitter.common.quantity import Amount, Time
-from twitter.common.exceptions import ExceptionalThread
-
-
-class ExecutorTimeout(ExceptionalThread):
- DEFAULT_TIMEOUT = Amount(10, Time.SECONDS)
-
- def __init__(self, event, driver, logger=log.error, timeout=DEFAULT_TIMEOUT):
- self._event = event
- self._logger = logger
- self._driver = driver
- self._timeout = timeout
- super(ExecutorTimeout, self).__init__()
- self.daemon = True
-
- def run(self):
- self._event.wait(self._timeout.as_(Time.SECONDS))
- if not self._event.is_set():
- self._logger('Executor timing out.')
- self._driver.stop()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/health_checker.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/health_checker.py b/src/main/python/twitter/aurora/executor/common/health_checker.py
deleted file mode 100644
index 3b25f0e..0000000
--- a/src/main/python/twitter/aurora/executor/common/health_checker.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import threading
-import time
-
-from twitter.aurora.common.http_signaler import HttpSignaler
-from twitter.common import log
-from twitter.common.exceptions import ExceptionalThread
-
-from .status_checker import (
- ExitState,
- StatusChecker,
- StatusCheckerProvider,
- StatusResult,
-)
-from .task_info import mesos_task_instance_from_assigned_task, resolve_ports
-
-
-class HealthCheckerThread(StatusChecker, ExceptionalThread):
- """Generic, StatusChecker-conforming thread for arbitrary periodic health checks
-
- health_checker should be a callable returning a tuple of (boolean, reason), indicating
- respectively the health of the service and the reason for its failure (or None if the service is
- still healthy).
- """
- def __init__(self,
- health_checker,
- interval_secs=30,
- initial_interval_secs=None,
- max_consecutive_failures=0,
- clock=time):
- self._checker = health_checker
- self._interval = interval_secs
- if initial_interval_secs is not None:
- self._initial_interval = initial_interval_secs
- else:
- self._initial_interval = interval_secs * 2
- self._current_consecutive_failures = 0
- self._max_consecutive_failures = max_consecutive_failures
- self._dead = threading.Event()
- if self._initial_interval > 0:
- self._healthy, self._reason = True, None
- else:
- self._healthy, self._reason = self._checker()
- self._clock = clock
- super(HealthCheckerThread, self).__init__()
- self.daemon = True
-
- @property
- def status(self):
- if not self._healthy:
- return StatusResult('Failed health check! %s' % self._reason, ExitState.FAILED)
-
- def run(self):
- log.debug('Health checker thread started.')
- self._clock.sleep(self._initial_interval)
- log.debug('Initial interval expired.')
- while not self._dead.is_set():
- self._maybe_update_failure_count(*self._checker())
- self._clock.sleep(self._interval)
-
- def _maybe_update_failure_count(self, is_healthy, reason):
- if not is_healthy:
- log.warning('Health check failure: %s' % reason)
- self._current_consecutive_failures += 1
- if self._current_consecutive_failures > self._max_consecutive_failures:
- log.warning('Reached consecutive failure limit.')
- self._healthy = False
- self._reason = reason
- else:
- if self._current_consecutive_failures > 0:
- log.debug('Reset consecutive failures counter.')
- self._current_consecutive_failures = 0
-
- def start(self):
- StatusChecker.start(self)
- ExceptionalThread.start(self)
-
- def stop(self):
- log.debug('Health checker thread stopped.')
- self._dead.set()
-
-
-class HealthCheckerProvider(StatusCheckerProvider):
- def from_assigned_task(self, assigned_task, _):
- mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
- portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
-
- if 'health' not in portmap:
- return None
-
- health_check_config = mesos_task.health_check_config().get()
- http_signaler = HttpSignaler(
- portmap['health'],
- timeout_secs=health_check_config.get('timeout_secs'))
- health_checker = HealthCheckerThread(
- http_signaler.health,
- interval_secs=health_check_config.get('interval_secs'),
- initial_interval_secs=health_check_config.get('initial_interval_secs'),
- max_consecutive_failures=health_check_config.get('max_consecutive_failures'))
- return health_checker
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/kill_manager.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/kill_manager.py b/src/main/python/twitter/aurora/executor/common/kill_manager.py
deleted file mode 100644
index 70ab733..0000000
--- a/src/main/python/twitter/aurora/executor/common/kill_manager.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from .status_checker import ExitState, StatusChecker, StatusResult
-
-
-class KillManager(StatusChecker):
- """
- A health interface that provides a kill-switch for a task monitored by the status manager.
- """
- def __init__(self):
- self._killed = False
- self._reason = None
-
- @property
- def status(self):
- if self._killed:
- return StatusResult(self._reason, ExitState.KILLED)
-
- def kill(self, reason):
- self._reason = reason
- self._killed = True
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/aurora/executor/common/sandbox.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/aurora/executor/common/sandbox.py b/src/main/python/twitter/aurora/executor/common/sandbox.py
deleted file mode 100644
index 551d360..0000000
--- a/src/main/python/twitter/aurora/executor/common/sandbox.py
+++ /dev/null
@@ -1,72 +0,0 @@
-from abc import abstractmethod, abstractproperty
-import getpass
-import grp
-import os
-import pwd
-
-from twitter.common import log
-from twitter.common.dirutil import safe_mkdir, safe_rmtree
-from twitter.common.lang import Interface
-
-
-class SandboxInterface(Interface):
- class Error(Exception): pass
- class CreationError(Error): pass
- class DeletionError(Error): pass
-
- @abstractproperty
- def root(self):
- """Return the root path of the sandbox."""
-
- @abstractproperty
- def chrooted(self):
- """Returns whether or not the sandbox is a chroot."""
-
- @abstractmethod
- def exists(self):
- """Returns true if the sandbox appears to exist."""
-
- @abstractmethod
- def create(self, *args, **kw):
- """Create the sandbox."""
-
- @abstractmethod
- def destroy(self, *args, **kw):
- """Destroy the sandbox."""
-
-
-class SandboxProvider(Interface):
- @abstractmethod
- def from_assigned_task(self, assigned_task):
- """Return the appropriate Sandbox implementation from an AssignedTask."""
-
-
-class DirectorySandbox(SandboxInterface):
- """ Basic sandbox implementation using a directory on the filesystem """
- def __init__(self, root, user=getpass.getuser()):
- self._root = root
- self._user = user
-
- @property
- def root(self):
- return self._root
-
- @property
- def chrooted(self):
- return False
-
- def exists(self):
- return os.path.exists(self.root)
-
- def create(self):
- log.debug('DirectorySandbox: mkdir %s' % self.root)
- safe_mkdir(self.root)
- pwent = pwd.getpwnam(self._user)
- grent = grp.getgrgid(pwent.pw_gid)
- log.debug('DirectorySandbox: chown %s:%s %s' % (self._user, grent.gr_name, self.root))
- os.chown(self.root, pwent.pw_uid, pwent.pw_gid)
- log.debug('DirectorySandbox: chmod 700 %s' % self.root)
- os.chmod(self.root, 0700)
-
- def destroy(self):
- safe_rmtree(self.root)
[28/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java b/src/main/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java
new file mode 100644
index 0000000..d735828
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.storage.testing;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.easymock.Capture;
+import org.easymock.IAnswer;
+import org.easymock.IExpectationSetters;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.storage.AttributeStore;
+import com.twitter.aurora.scheduler.storage.JobStore;
+import com.twitter.aurora.scheduler.storage.LockStore;
+import com.twitter.aurora.scheduler.storage.QuotaStore;
+import com.twitter.aurora.scheduler.storage.SchedulerStore;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.NonVolatileStorage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.TaskStore;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.testing.easymock.EasyMockTest;
+
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.expect;
+
+/**
+ * Auxiliary class to simplify testing against a mocked storage. This allows callers to directly
+ * set up call expectations on individual stores rather than writing plumbing code to deal with
+ * operations and {@link StoreProvider}.
+ */
+public class StorageTestUtil {
+
+ public final StoreProvider storeProvider;
+ public final MutableStoreProvider mutableStoreProvider;
+ public final TaskStore.Mutable taskStore;
+ public final QuotaStore.Mutable quotaStore;
+ public final AttributeStore.Mutable attributeStore;
+ public final JobStore.Mutable jobStore;
+ public final LockStore.Mutable lockStore;
+ public final SchedulerStore.Mutable schedulerStore;
+ public final NonVolatileStorage storage;
+
+ /**
+ * Creates a new storage test utility.
+ *
+ * @param easyMock Mocking framework to use for setting up mocks and expectations.
+ */
+ public StorageTestUtil(EasyMockTest easyMock) {
+ this.storeProvider = easyMock.createMock(StoreProvider.class);
+ this.mutableStoreProvider = easyMock.createMock(MutableStoreProvider.class);
+ this.taskStore = easyMock.createMock(TaskStore.Mutable.class);
+ this.quotaStore = easyMock.createMock(QuotaStore.Mutable.class);
+ this.attributeStore = easyMock.createMock(AttributeStore.Mutable.class);
+ this.jobStore = easyMock.createMock(JobStore.Mutable.class);
+ this.lockStore = easyMock.createMock(LockStore.Mutable.class);
+ this.schedulerStore = easyMock.createMock(SchedulerStore.Mutable.class);
+ this.storage = easyMock.createMock(NonVolatileStorage.class);
+ }
+
+ private <T> IExpectationSetters<T> expectConsistentRead() {
+ final Capture<Work<T, RuntimeException>> work = EasyMockTest.createCapture();
+ return expect(storage.consistentRead(capture(work))).andAnswer(new IAnswer<T>() {
+ @Override public T answer() {
+ return work.getValue().apply(storeProvider);
+ }
+ });
+ }
+
+ private <T> IExpectationSetters<T> expectWeaklyConsistentRead() {
+ final Capture<Work<T, RuntimeException>> work = EasyMockTest.createCapture();
+ return expect(storage.weaklyConsistentRead(capture(work))).andAnswer(new IAnswer<T>() {
+ @Override public T answer() {
+ return work.getValue().apply(storeProvider);
+ }
+ });
+ }
+
+ private <T> IExpectationSetters<T> expectWriteOperation() {
+ final Capture<MutateWork<T, RuntimeException>> work = EasyMockTest.createCapture();
+ return expect(storage.write(capture(work))).andAnswer(new IAnswer<T>() {
+ @Override public T answer() {
+ return work.getValue().apply(mutableStoreProvider);
+ }
+ });
+ }
+
+ /**
+ * Expects any number of read or write operations.
+ */
+ public void expectOperations() {
+ expect(storeProvider.getTaskStore()).andReturn(taskStore).anyTimes();
+ expect(storeProvider.getQuotaStore()).andReturn(quotaStore).anyTimes();
+ expect(storeProvider.getAttributeStore()).andReturn(attributeStore).anyTimes();
+ expect(storeProvider.getJobStore()).andReturn(jobStore).anyTimes();
+ expect(storeProvider.getLockStore()).andReturn(lockStore).anyTimes();
+ expect(storeProvider.getSchedulerStore()).andReturn(schedulerStore).anyTimes();
+ expect(mutableStoreProvider.getTaskStore()).andReturn(taskStore).anyTimes();
+ expect(mutableStoreProvider.getUnsafeTaskStore()).andReturn(taskStore).anyTimes();
+ expect(mutableStoreProvider.getQuotaStore()).andReturn(quotaStore).anyTimes();
+ expect(mutableStoreProvider.getAttributeStore()).andReturn(attributeStore).anyTimes();
+ expect(mutableStoreProvider.getJobStore()).andReturn(jobStore).anyTimes();
+ expect(mutableStoreProvider.getLockStore()).andReturn(lockStore).anyTimes();
+ expect(mutableStoreProvider.getSchedulerStore()).andReturn(schedulerStore).anyTimes();
+ expectConsistentRead().anyTimes();
+ expectWeaklyConsistentRead().anyTimes();
+ expectWriteOperation().anyTimes();
+ }
+
+ public IExpectationSetters<?> expectTaskFetch(
+ Query.Builder query,
+ ImmutableSet<IScheduledTask> result) {
+
+ return expect(taskStore.fetchTasks(query)).andReturn(result);
+ }
+
+ public IExpectationSetters<?> expectTaskFetch(Query.Builder query, IScheduledTask... result) {
+ return expectTaskFetch(query, ImmutableSet.<IScheduledTask>builder().add(result).build());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerAPIServlet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerAPIServlet.java b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerAPIServlet.java
new file mode 100644
index 0000000..2acf5c8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerAPIServlet.java
@@ -0,0 +1,19 @@
+package com.twitter.aurora.scheduler.thrift;
+
+import javax.inject.Inject;
+
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.apache.thrift.server.TServlet;
+
+import com.twitter.aurora.gen.AuroraAdmin;
+
+/**
+ * A servlet that exposes the scheduler Thrift API over HTTP/JSON.
+ */
+class SchedulerAPIServlet extends TServlet {
+
+ @Inject
+ SchedulerAPIServlet(AuroraAdmin.Iface schedulerThriftInterface) {
+ super(new AuroraAdmin.Processor<>(schedulerThriftInterface), new TJSONProtocol.Factory());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
new file mode 100644
index 0000000..3ff0f1c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
@@ -0,0 +1,1000 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Throwables;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.twitter.aurora.auth.CapabilityValidator;
+import com.twitter.aurora.auth.CapabilityValidator.AuditCheck;
+import com.twitter.aurora.auth.CapabilityValidator.Capability;
+import com.twitter.aurora.auth.SessionValidator.AuthFailedException;
+import com.twitter.aurora.gen.AcquireLockResult;
+import com.twitter.aurora.gen.AddInstancesConfig;
+import com.twitter.aurora.gen.AuroraAdmin;
+import com.twitter.aurora.gen.ConfigRewrite;
+import com.twitter.aurora.gen.DrainHostsResult;
+import com.twitter.aurora.gen.EndMaintenanceResult;
+import com.twitter.aurora.gen.GetJobsResult;
+import com.twitter.aurora.gen.GetQuotaResult;
+import com.twitter.aurora.gen.Hosts;
+import com.twitter.aurora.gen.InstanceConfigRewrite;
+import com.twitter.aurora.gen.InstanceKey;
+import com.twitter.aurora.gen.JobConfigRewrite;
+import com.twitter.aurora.gen.JobConfigValidation;
+import com.twitter.aurora.gen.JobConfiguration;
+import com.twitter.aurora.gen.JobKey;
+import com.twitter.aurora.gen.JobSummary;
+import com.twitter.aurora.gen.JobSummaryResult;
+import com.twitter.aurora.gen.ListBackupsResult;
+import com.twitter.aurora.gen.Lock;
+import com.twitter.aurora.gen.LockKey;
+import com.twitter.aurora.gen.LockValidation;
+import com.twitter.aurora.gen.MaintenanceStatusResult;
+import com.twitter.aurora.gen.PopulateJobResult;
+import com.twitter.aurora.gen.QueryRecoveryResult;
+import com.twitter.aurora.gen.Quota;
+import com.twitter.aurora.gen.Response;
+import com.twitter.aurora.gen.ResponseCode;
+import com.twitter.aurora.gen.Result;
+import com.twitter.aurora.gen.RewriteConfigsRequest;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.ScheduleStatusResult;
+import com.twitter.aurora.gen.SessionKey;
+import com.twitter.aurora.gen.StartMaintenanceResult;
+import com.twitter.aurora.gen.TaskConfig;
+import com.twitter.aurora.gen.TaskQuery;
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.ScheduleException;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
+import com.twitter.aurora.scheduler.configuration.SanitizedConfiguration;
+import com.twitter.aurora.scheduler.quota.Quotas;
+import com.twitter.aurora.scheduler.state.CronJobManager;
+import com.twitter.aurora.scheduler.state.LockManager;
+import com.twitter.aurora.scheduler.state.LockManager.LockException;
+import com.twitter.aurora.scheduler.state.MaintenanceController;
+import com.twitter.aurora.scheduler.state.SchedulerCore;
+import com.twitter.aurora.scheduler.storage.JobStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.backup.Recovery;
+import com.twitter.aurora.scheduler.storage.backup.Recovery.RecoveryException;
+import com.twitter.aurora.scheduler.storage.backup.StorageBackup;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.ILock;
+import com.twitter.aurora.scheduler.storage.entities.ILockKey;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.aurora.scheduler.thrift.auth.DecoratedThrift;
+import com.twitter.aurora.scheduler.thrift.auth.Requires;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+import com.twitter.common.base.MorePreconditions;
+import com.twitter.common.base.Supplier;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.util.BackoffHelper;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.auth.SessionValidator.SessionContext;
+import static com.twitter.aurora.gen.ResponseCode.AUTH_FAILED;
+import static com.twitter.aurora.gen.ResponseCode.ERROR;
+import static com.twitter.aurora.gen.ResponseCode.INVALID_REQUEST;
+import static com.twitter.aurora.gen.ResponseCode.LOCK_ERROR;
+import static com.twitter.aurora.gen.ResponseCode.OK;
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
+import static com.twitter.common.base.MorePreconditions.checkNotBlank;
+
+/**
+ * Aurora scheduler thrift server implementation.
+ * <p>
+ * Interfaces between users and the scheduler to access/modify jobs and perform cluster
+ * administration tasks.
+ */
+@DecoratedThrift
+class SchedulerThriftInterface implements AuroraAdmin.Iface {
+ private static final Logger LOG = Logger.getLogger(SchedulerThriftInterface.class.getName());
+
+ @CmdLine(name = "kill_task_initial_backoff",
+ help = "Initial backoff delay while waiting for the tasks to transition to KILLED.")
+ private static final Arg<Amount<Long, Time>> KILL_TASK_INITIAL_BACKOFF =
+ Arg.create(Amount.of(1L, Time.SECONDS));
+
+ @CmdLine(name = "kill_task_max_backoff",
+ help = "Max backoff delay while waiting for the tasks to transition to KILLED.")
+ private static final Arg<Amount<Long, Time>> KILL_TASK_MAX_BACKOFF =
+ Arg.create(Amount.of(30L, Time.SECONDS));
+
+ private static final Function<IScheduledTask, String> GET_ROLE = Functions.compose(
+ new Function<ITaskConfig, String>() {
+ @Override public String apply(ITaskConfig task) {
+ return task.getOwner().getRole();
+ }
+ },
+ Tasks.SCHEDULED_TO_INFO);
+
+ private final Storage storage;
+ private final SchedulerCore schedulerCore;
+ private final LockManager lockManager;
+ private final CapabilityValidator sessionValidator;
+ private final StorageBackup backup;
+ private final Recovery recovery;
+ private final MaintenanceController maintenance;
+ private final CronJobManager cronJobManager;
+ private final Amount<Long, Time> killTaskInitialBackoff;
+ private final Amount<Long, Time> killTaskMaxBackoff;
+
+ @Inject
+ SchedulerThriftInterface(
+ Storage storage,
+ SchedulerCore schedulerCore,
+ LockManager lockManager,
+ CapabilityValidator sessionValidator,
+ StorageBackup backup,
+ Recovery recovery,
+ CronJobManager cronJobManager,
+ MaintenanceController maintenance) {
+
+ this(storage,
+ schedulerCore,
+ lockManager,
+ sessionValidator,
+ backup,
+ recovery,
+ maintenance,
+ cronJobManager,
+ KILL_TASK_INITIAL_BACKOFF.get(),
+ KILL_TASK_MAX_BACKOFF.get());
+ }
+
+ @VisibleForTesting
+ SchedulerThriftInterface(
+ Storage storage,
+ SchedulerCore schedulerCore,
+ LockManager lockManager,
+ CapabilityValidator sessionValidator,
+ StorageBackup backup,
+ Recovery recovery,
+ MaintenanceController maintenance,
+ CronJobManager cronJobManager,
+ Amount<Long, Time> initialBackoff,
+ Amount<Long, Time> maxBackoff) {
+
+ this.storage = checkNotNull(storage);
+ this.schedulerCore = checkNotNull(schedulerCore);
+ this.lockManager = checkNotNull(lockManager);
+ this.sessionValidator = checkNotNull(sessionValidator);
+ this.backup = checkNotNull(backup);
+ this.recovery = checkNotNull(recovery);
+ this.maintenance = checkNotNull(maintenance);
+ this.cronJobManager = checkNotNull(cronJobManager);
+ this.killTaskInitialBackoff = checkNotNull(initialBackoff);
+ this.killTaskMaxBackoff = checkNotNull(maxBackoff);
+ }
+
+ @Override
+ public Response createJob(
+ JobConfiguration mutableJob,
+ @Nullable Lock mutableLock,
+ SessionKey session) {
+
+ IJobConfiguration job = IJobConfiguration.build(mutableJob);
+ IJobKey jobKey = JobKeys.assertValid(job.getKey());
+ checkNotNull(session);
+
+ Response response = new Response();
+
+ try {
+ sessionValidator.checkAuthenticated(session, ImmutableSet.of(job.getOwner().getRole()));
+ } catch (AuthFailedException e) {
+ return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ }
+
+ try {
+ SanitizedConfiguration sanitized = SanitizedConfiguration.fromUnsanitized(job);
+
+ lockManager.validateIfLocked(
+ ILockKey.build(LockKey.job(jobKey.newBuilder())),
+ Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
+
+ schedulerCore.createJob(sanitized);
+ response.setResponseCode(OK)
+ .setMessage(String.format("%d new tasks pending for job %s",
+ sanitized.getJobConfig().getInstanceCount(), JobKeys.toPath(job)));
+ } catch (LockException e) {
+ response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
+ } catch (TaskDescriptionException | ScheduleException e) {
+ response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
+ }
+
+ return response;
+ }
+
+ @Override
+ public Response replaceCronTemplate(
+ JobConfiguration mutableConfig,
+ @Nullable Lock mutableLock,
+ SessionKey session) {
+
+ checkNotNull(mutableConfig);
+ IJobConfiguration job = IJobConfiguration.build(mutableConfig);
+ IJobKey jobKey = JobKeys.assertValid(job.getKey());
+ checkNotNull(session);
+
+ Response response = new Response();
+ try {
+ sessionValidator.checkAuthenticated(session, ImmutableSet.of(job.getOwner().getRole()));
+ } catch (AuthFailedException e) {
+ return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ }
+
+ try {
+ lockManager.validateIfLocked(
+ ILockKey.build(LockKey.job(jobKey.newBuilder())),
+ Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
+
+ SanitizedConfiguration sanitized = SanitizedConfiguration.fromUnsanitized(job);
+
+ if (!cronJobManager.hasJob(jobKey)) {
+ return response.setResponseCode(INVALID_REQUEST).setMessage(
+ "No cron template found for the given key: " + jobKey);
+ }
+ cronJobManager.updateJob(sanitized);
+ return response.setResponseCode(OK).setMessage("Replaced template for: " + jobKey);
+
+ } catch (LockException e) {
+ return response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
+ } catch (TaskDescriptionException | ScheduleException e) {
+ return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
+ }
+ }
+
+ @Override
+ public Response populateJobConfig(JobConfiguration description, JobConfigValidation validation) {
+
+ checkNotNull(description);
+
+ Response response = new Response();
+ try {
+ SanitizedConfiguration sanitized =
+ SanitizedConfiguration.fromUnsanitized(IJobConfiguration.build(description));
+
+ // TODO(maximk): Consider moving job validation logic into a dedicated RPC. MESOS-4476.
+ if (validation != null && validation == JobConfigValidation.RUN_FILTERS) {
+ schedulerCore.validateJobResources(sanitized);
+ }
+
+ PopulateJobResult result = new PopulateJobResult()
+ .setPopulated(ITaskConfig.toBuildersSet(sanitized.getTaskConfigs().values()));
+ response.setResult(Result.populateJobResult(result))
+ .setResponseCode(OK)
+ .setMessage("Tasks populated");
+ } catch (TaskDescriptionException | ScheduleException e) {
+ response.setResponseCode(INVALID_REQUEST)
+ .setMessage("Invalid configuration: " + e.getMessage());
+ }
+ return response;
+ }
+
+ @Override
+ public Response startCronJob(JobKey mutableJobKey, SessionKey session) {
+ checkNotNull(session);
+ IJobKey jobKey = JobKeys.assertValid(IJobKey.build(mutableJobKey));
+
+ Response response = new Response();
+ try {
+ sessionValidator.checkAuthenticated(session, ImmutableSet.of(jobKey.getRole()));
+ } catch (AuthFailedException e) {
+ response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ return response;
+ }
+
+ try {
+ schedulerCore.startCronJob(jobKey);
+ response.setResponseCode(OK).setMessage("Cron run started.");
+ } catch (ScheduleException e) {
+ response.setResponseCode(INVALID_REQUEST)
+ .setMessage("Failed to start cron job - " + e.getMessage());
+ } catch (TaskDescriptionException e) {
+ response.setResponseCode(ERROR).setMessage("Invalid task description: " + e.getMessage());
+ }
+
+ return response;
+ }
+
+ // TODO(William Farner): Provide status information about cron jobs here.
+ @Override
+ public Response getTasksStatus(TaskQuery query) {
+ checkNotNull(query);
+
+ Set<IScheduledTask> tasks =
+ Storage.Util.weaklyConsistentFetchTasks(storage, Query.arbitrary(query));
+
+ Response response = new Response();
+
+ if (tasks.isEmpty()) {
+ response.setResponseCode(INVALID_REQUEST)
+ .setMessage("No tasks found for query: " + query);
+ } else {
+ response.setResponseCode(OK)
+ .setResult(Result.scheduleStatusResult(
+ new ScheduleStatusResult().setTasks(IScheduledTask.toBuildersList(tasks))));
+ }
+
+ return response;
+ }
+
+ @Override
+ public Response getJobSummary() {
+ Set<IScheduledTask> tasks = Storage.Util.weaklyConsistentFetchTasks(storage, Query.unscoped());
+ Multimap<String, IJobKey> jobsByRole = Multimaps.index(
+ FluentIterable.from(tasks).transform(Tasks.SCHEDULED_TO_JOB_KEY),
+ JobKeys.TO_ROLE);
+
+ Multimap<String, IJobKey> cronJobsByRole = Multimaps.index(
+ FluentIterable.from(cronJobManager.getJobs()).transform(JobKeys.FROM_CONFIG),
+ JobKeys.TO_ROLE);
+
+ List<JobSummary> jobSummaries = Lists.newLinkedList();
+ for (String role : Sets.union(jobsByRole.keySet(), cronJobsByRole.keySet())) {
+ JobSummary summary = new JobSummary();
+ summary.setRole(role);
+ summary.setJobCount(jobsByRole.get(role).size());
+ summary.setCronJobCount(cronJobsByRole.get(role).size());
+ jobSummaries.add(summary);
+ }
+
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.jobSummaryResult(new JobSummaryResult(jobSummaries)));
+ }
+
+ @Override
+ public Response getJobs(@Nullable String maybeNullRole) {
+ Optional<String> ownerRole = Optional.fromNullable(maybeNullRole);
+
+
+ // Ensure we only return one JobConfiguration for each JobKey.
+ Map<IJobKey, IJobConfiguration> jobs = Maps.newHashMap();
+
+ // Query the task store, find immediate jobs, and synthesize a JobConfiguration for them.
+ // This is necessary because the ImmediateJobManager doesn't store jobs directly and
+ // ImmediateJobManager#getJobs always returns an empty Collection.
+ Query.Builder scope = ownerRole.isPresent()
+ ? Query.roleScoped(ownerRole.get())
+ : Query.unscoped();
+ Multimap<IJobKey, IScheduledTask> tasks =
+ Tasks.byJobKey(Storage.Util.weaklyConsistentFetchTasks(storage, scope.active()));
+
+ jobs.putAll(Maps.transformEntries(tasks.asMap(),
+ new Maps.EntryTransformer<IJobKey, Collection<IScheduledTask>, IJobConfiguration>() {
+ @Override
+ public IJobConfiguration transformEntry(
+ IJobKey jobKey,
+ Collection<IScheduledTask> tasks) {
+
+ // Pick an arbitrary task for each immediate job. The chosen task might not be the most
+ // recent if the job is in the middle of an update or some shards have been selectively
+ // created.
+ TaskConfig firstTask = tasks.iterator().next().getAssignedTask().getTask().newBuilder();
+ return IJobConfiguration.build(new JobConfiguration()
+ .setKey(jobKey.newBuilder())
+ .setOwner(firstTask.getOwner())
+ .setTaskConfig(firstTask)
+ .setInstanceCount(tasks.size()));
+ }
+ }));
+
+ // Get cron jobs directly from the manager. Do this after querying the task store so the real
+ // template JobConfiguration for a cron job will overwrite the synthesized one that could have
+ // been created above.
+ Predicate<IJobConfiguration> configFilter = ownerRole.isPresent()
+ ? Predicates.compose(Predicates.equalTo(ownerRole.get()), JobKeys.CONFIG_TO_ROLE)
+ : Predicates.<IJobConfiguration>alwaysTrue();
+ jobs.putAll(Maps.uniqueIndex(
+ FluentIterable.from(cronJobManager.getJobs()).filter(configFilter),
+ JobKeys.FROM_CONFIG));
+
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.getJobsResult(new GetJobsResult()
+ .setConfigs(IJobConfiguration.toBuildersSet(jobs.values()))));
+ }
+
+ private void validateLockForTasks(Optional<ILock> lock, Iterable<IScheduledTask> tasks)
+ throws LockException {
+
+ ImmutableSet<IJobKey> uniqueKeys = FluentIterable.from(tasks)
+ .transform(Tasks.SCHEDULED_TO_JOB_KEY)
+ .toSet();
+
+ // Validate lock against every unique job key derived from the tasks.
+ for (IJobKey key : uniqueKeys) {
+ lockManager.validateIfLocked(ILockKey.build(LockKey.job(key.newBuilder())), lock);
+ }
+ }
+
+ private SessionContext validateSessionKeyForTasks(
+ SessionKey session,
+ TaskQuery taskQuery,
+ Iterable<IScheduledTask> tasks) throws AuthFailedException {
+
+ // Authenticate the session against any affected roles, always including the role for a
+ // role-scoped query. This papers over the implementation detail that dormant cron jobs are
+ // authenticated this way.
+ ImmutableSet.Builder<String> targetRoles = ImmutableSet.<String>builder()
+ .addAll(FluentIterable.from(tasks).transform(GET_ROLE));
+ if (taskQuery.isSetOwner()) {
+ targetRoles.add(taskQuery.getOwner().getRole());
+ }
+ return sessionValidator.checkAuthenticated(session, targetRoles.build());
+ }
+
+ private Optional<SessionContext> isAdmin(SessionKey session) {
+ try {
+ return Optional.of(
+ sessionValidator.checkAuthorized(session, Capability.ROOT, AuditCheck.REQUIRED));
+ } catch (AuthFailedException e) {
+ return Optional.absent();
+ }
+ }
+
+ @Override
+ public Response killTasks(final TaskQuery query, Lock mutablelock, SessionKey session) {
+ checkNotNull(query);
+ checkNotNull(session);
+
+ Response response = new Response();
+
+ if (query.getJobName() != null && StringUtils.isBlank(query.getJobName())) {
+ response.setResponseCode(INVALID_REQUEST).setMessage(
+ String.format("Invalid job name: '%s'", query.getJobName()));
+ return response;
+ }
+
+ Set<IScheduledTask> tasks = Storage.Util.consistentFetchTasks(storage, Query.arbitrary(query));
+
+ Optional<SessionContext> context = isAdmin(session);
+ if (context.isPresent()) {
+ LOG.info("Granting kill query to admin user: " + query);
+ } else {
+ try {
+ context = Optional.of(validateSessionKeyForTasks(session, query, tasks));
+ } catch (AuthFailedException e) {
+ response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ return response;
+ }
+ }
+
+ try {
+ validateLockForTasks(Optional.fromNullable(mutablelock).transform(ILock.FROM_BUILDER), tasks);
+ schedulerCore.killTasks(Query.arbitrary(query), context.get().getIdentity());
+ } catch (LockException e) {
+ return response.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
+ } catch (ScheduleException e) {
+ return response.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
+ }
+
+ // TODO(William Farner): Move this into the client.
+ BackoffHelper backoff = new BackoffHelper(killTaskInitialBackoff, killTaskMaxBackoff, true);
+ final Query.Builder activeQuery = Query.arbitrary(query.setStatuses(Tasks.ACTIVE_STATES));
+ try {
+ backoff.doUntilSuccess(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ Set<IScheduledTask> tasks = Storage.Util.consistentFetchTasks(storage, activeQuery);
+ if (tasks.isEmpty()) {
+ LOG.info("Tasks all killed, done waiting.");
+ return true;
+ } else {
+ LOG.info("Jobs not yet killed, waiting...");
+ return false;
+ }
+ }
+ });
+ response.setResponseCode(OK).setMessage("Tasks killed.");
+ } catch (InterruptedException e) {
+ LOG.warning("Interrupted while trying to kill tasks: " + e);
+ Thread.currentThread().interrupt();
+ response.setResponseCode(ERROR).setMessage("killTasks thread was interrupted.");
+ } catch (BackoffHelper.BackoffStoppedException e) {
+ response.setResponseCode(ERROR).setMessage("Tasks were not killed in time.");
+ }
+ return response;
+ }
+
+ @Override
+ public Response restartShards(
+ JobKey mutableJobKey,
+ Set<Integer> shardIds,
+ Lock mutableLock,
+ SessionKey session) {
+
+ IJobKey jobKey = JobKeys.assertValid(IJobKey.build(mutableJobKey));
+ MorePreconditions.checkNotBlank(shardIds);
+ checkNotNull(session);
+
+ Response response = new Response();
+ SessionContext context;
+ try {
+ context = sessionValidator.checkAuthenticated(session, ImmutableSet.of(jobKey.getRole()));
+ } catch (AuthFailedException e) {
+ response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ return response;
+ }
+
+ try {
+ lockManager.validateIfLocked(
+ ILockKey.build(LockKey.job(jobKey.newBuilder())),
+ Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
+ schedulerCore.restartShards(jobKey, shardIds, context.getIdentity());
+ response.setResponseCode(OK).setMessage("Shards are restarting.");
+ } catch (LockException e) {
+ response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
+ } catch (ScheduleException e) {
+ response.setResponseCode(ResponseCode.INVALID_REQUEST).setMessage(e.getMessage());
+ }
+
+ return response;
+ }
+
+ @Override
+ public Response getQuota(final String ownerRole) {
+ checkNotBlank(ownerRole);
+
+ IQuota quota = storage.consistentRead(new Work.Quiet<IQuota>() {
+ @Override public IQuota apply(StoreProvider storeProvider) {
+ return storeProvider.getQuotaStore().fetchQuota(ownerRole).or(Quotas.noQuota());
+ }
+ });
+
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.getQuotaResult(new GetQuotaResult()
+ .setQuota(quota.newBuilder())));
+ }
+
+ @Override
+ public Response startMaintenance(Hosts hosts, SessionKey session) {
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.startMaintenanceResult(new StartMaintenanceResult()
+ .setStatuses(maintenance.startMaintenance(hosts.getHostNames()))));
+ }
+
+ @Override
+ public Response drainHosts(Hosts hosts, SessionKey session) {
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.drainHostsResult(new DrainHostsResult()
+ .setStatuses(maintenance.drain(hosts.getHostNames()))));
+ }
+
+ @Override
+ public Response maintenanceStatus(Hosts hosts, SessionKey session) {
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.maintenanceStatusResult(new MaintenanceStatusResult()
+ .setStatuses(maintenance.getStatus(hosts.getHostNames()))));
+ }
+
+ @Override
+ public Response endMaintenance(Hosts hosts, SessionKey session) {
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.endMaintenanceResult(new EndMaintenanceResult()
+ .setStatuses(maintenance.endMaintenance(hosts.getHostNames()))));
+ }
+
+ @Requires(whitelist = Capability.PROVISIONER)
+ @Override
+ public Response setQuota(final String ownerRole, final Quota quota, SessionKey session) {
+ checkNotBlank(ownerRole);
+ checkNotNull(quota);
+ checkNotNull(session);
+
+ // TODO(Kevin Sweeney): Input validation for Quota.
+
+ storage.write(new MutateWork.NoResult.Quiet() {
+ @Override protected void execute(MutableStoreProvider storeProvider) {
+ storeProvider.getQuotaStore().saveQuota(ownerRole, IQuota.build(quota));
+ }
+ });
+
+ return new Response().setResponseCode(OK).setMessage("Quota applied.");
+ }
+
+ @Override
+ public Response forceTaskState(
+ String taskId,
+ ScheduleStatus status,
+ SessionKey session) {
+
+ checkNotBlank(taskId);
+ checkNotNull(status);
+ checkNotNull(session);
+
+ Response response = new Response();
+ SessionContext context;
+ try {
+ // TODO(Sathya): Remove this after AOP-style session validation passes in a SessionContext.
+ context = sessionValidator.checkAuthorized(session, Capability.ROOT, AuditCheck.REQUIRED);
+ } catch (AuthFailedException e) {
+ response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ return response;
+ }
+
+ schedulerCore.setTaskStatus(
+ Query.taskScoped(taskId), status, transitionMessage(context.getIdentity()));
+ return new Response().setResponseCode(OK).setMessage("Transition attempted.");
+ }
+
+ @Override
+ public Response performBackup(SessionKey session) {
+ backup.backupNow();
+ return new Response().setResponseCode(OK);
+ }
+
+ @Override
+ public Response listBackups(SessionKey session) {
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.listBackupsResult(new ListBackupsResult()
+ .setBackups(recovery.listBackups())));
+ }
+
+ @Override
+ public Response stageRecovery(String backupId, SessionKey session) {
+ Response response = new Response().setResponseCode(OK);
+ try {
+ recovery.stage(backupId);
+ } catch (RecoveryException e) {
+ response.setResponseCode(ERROR).setMessage(e.getMessage());
+ LOG.log(Level.WARNING, "Failed to stage recovery: " + e, e);
+ }
+
+ return response;
+ }
+
+ @Override
+ public Response queryRecovery(TaskQuery query, SessionKey session) {
+ Response response = new Response();
+ try {
+ response.setResponseCode(OK)
+ .setResult(Result.queryRecoveryResult(new QueryRecoveryResult()
+ .setTasks(IScheduledTask.toBuildersSet(recovery.query(Query.arbitrary(query))))));
+ } catch (RecoveryException e) {
+ response.setResponseCode(ERROR).setMessage(e.getMessage());
+ LOG.log(Level.WARNING, "Failed to query recovery: " + e, e);
+ }
+
+ return response;
+ }
+
+ @Override
+ public Response deleteRecoveryTasks(TaskQuery query, SessionKey session) {
+ Response response = new Response().setResponseCode(OK);
+ try {
+ recovery.deleteTasks(Query.arbitrary(query));
+ } catch (RecoveryException e) {
+ response.setResponseCode(ERROR).setMessage(e.getMessage());
+ LOG.log(Level.WARNING, "Failed to delete recovery tasks: " + e, e);
+ }
+
+ return response;
+ }
+
+ @Override
+ public Response commitRecovery(SessionKey session) {
+ Response response = new Response().setResponseCode(OK);
+ try {
+ recovery.commit();
+ } catch (RecoveryException e) {
+ response.setResponseCode(ERROR).setMessage(e.getMessage());
+ }
+
+ return response;
+ }
+
+ @Override
+ public Response unloadRecovery(SessionKey session) {
+ recovery.unload();
+ return new Response().setResponseCode(OK);
+ }
+
+ @Override
+ public Response snapshot(SessionKey session) {
+ Response response = new Response();
+ try {
+ storage.snapshot();
+ return response.setResponseCode(OK).setMessage("Compaction successful.");
+ } catch (Storage.StorageException e) {
+ LOG.log(Level.WARNING, "Requested snapshot failed.", e);
+ return response.setResponseCode(ERROR).setMessage(e.getMessage());
+ }
+ }
+
+ private static Multimap<String, IJobConfiguration> jobsByKey(JobStore jobStore, IJobKey jobKey) {
+ ImmutableMultimap.Builder<String, IJobConfiguration> matches = ImmutableMultimap.builder();
+ for (String managerId : jobStore.fetchManagerIds()) {
+ for (IJobConfiguration job : jobStore.fetchJobs(managerId)) {
+ if (job.getKey().equals(jobKey)) {
+ matches.put(managerId, job);
+ }
+ }
+ }
+ return matches.build();
+ }
+
+ @Override
+ public Response rewriteConfigs(
+ final RewriteConfigsRequest request,
+ SessionKey session) {
+
+ if (request.getRewriteCommandsSize() == 0) {
+ return new Response()
+ .setResponseCode(ResponseCode.ERROR)
+ .setMessage("No rewrite commands provided.");
+ }
+
+ return storage.write(new MutateWork.Quiet<Response>() {
+ @Override public Response apply(MutableStoreProvider storeProvider) {
+ List<String> errors = Lists.newArrayList();
+
+ for (ConfigRewrite command : request.getRewriteCommands()) {
+ Optional<String> error = rewriteConfig(command, storeProvider);
+ if (error.isPresent()) {
+ errors.add(error.get());
+ }
+ }
+
+ Response resp = new Response();
+ if (!errors.isEmpty()) {
+ resp.setResponseCode(ResponseCode.WARNING).setMessage(Joiner.on(", ").join(errors));
+ } else {
+ resp.setResponseCode(OK).setMessage("All rewrites completed successfully.");
+ }
+ return resp;
+ }
+ });
+ }
+
+ private Optional<String> rewriteConfig(
+ ConfigRewrite command,
+ MutableStoreProvider storeProvider) {
+
+ Optional<String> error = Optional.absent();
+ switch (command.getSetField()) {
+ case JOB_REWRITE:
+ JobConfigRewrite jobRewrite = command.getJobRewrite();
+ IJobConfiguration existingJob = IJobConfiguration.build(jobRewrite.getOldJob());
+ IJobConfiguration rewrittenJob;
+ try {
+ rewrittenJob = ConfigurationManager.validateAndPopulate(
+ IJobConfiguration.build(jobRewrite.getRewrittenJob()));
+ } catch (TaskDescriptionException e) {
+ // We could add an error here, but this is probably a hint of something wrong in
+ // the client that's causing a bad configuration to be applied.
+ throw Throwables.propagate(e);
+ }
+ if (!existingJob.getKey().equals(rewrittenJob.getKey())) {
+ error = Optional.of("Disallowing rewrite attempting to change job key.");
+ } else if (!existingJob.getOwner().equals(rewrittenJob.getOwner())) {
+ error = Optional.of("Disallowing rewrite attempting to change job owner.");
+ } else {
+ JobStore.Mutable jobStore = storeProvider.getJobStore();
+ Multimap<String, IJobConfiguration> matches =
+ jobsByKey(jobStore, existingJob.getKey());
+ switch (matches.size()) {
+ case 0:
+ error = Optional.of("No jobs found for key " + JobKeys.toPath(existingJob));
+ break;
+
+ case 1:
+ Map.Entry<String, IJobConfiguration> match =
+ Iterables.getOnlyElement(matches.entries());
+ IJobConfiguration storedJob = match.getValue();
+ if (!storedJob.equals(existingJob)) {
+ error = Optional.of("CAS compare failed for " + JobKeys.toPath(storedJob));
+ } else {
+ jobStore.saveAcceptedJob(match.getKey(), rewrittenJob);
+ }
+ break;
+
+ default:
+ error = Optional.of("Multiple jobs found for key " + JobKeys.toPath(existingJob));
+ }
+ }
+ break;
+
+ case INSTANCE_REWRITE:
+ InstanceConfigRewrite instanceRewrite = command.getInstanceRewrite();
+ InstanceKey instanceKey = instanceRewrite.getInstanceKey();
+ Iterable<IScheduledTask> tasks = storeProvider.getTaskStore().fetchTasks(
+ Query.instanceScoped(IJobKey.build(instanceKey.getJobKey()),
+ instanceKey.getInstanceId())
+ .active());
+ Optional<IAssignedTask> task =
+ Optional.fromNullable(Iterables.getOnlyElement(tasks, null))
+ .transform(Tasks.SCHEDULED_TO_ASSIGNED);
+ if (!task.isPresent()) {
+ error = Optional.of("No active task found for " + instanceKey);
+ } else if (!task.get().getTask().newBuilder().equals(instanceRewrite.getOldTask())) {
+ error = Optional.of("CAS compare failed for " + instanceKey);
+ } else {
+ ITaskConfig newConfiguration = ITaskConfig.build(
+ ConfigurationManager.applyDefaultsIfUnset(instanceRewrite.getRewrittenTask()));
+ boolean changed = storeProvider.getUnsafeTaskStore().unsafeModifyInPlace(
+ task.get().getTaskId(), newConfiguration);
+ if (!changed) {
+ error = Optional.of("Did not change " + task.get().getTaskId());
+ }
+ }
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unhandled command type " + command.getSetField());
+ }
+
+ return error;
+ }
+
+ @Override
+ public Response getVersion() {
+ return new Response()
+ .setResponseCode(OK)
+ .setResult(Result.getVersionResult(CURRENT_API_VERSION));
+ }
+
+ @Override
+ public Response addInstances(
+ AddInstancesConfig config,
+ @Nullable Lock mutableLock,
+ SessionKey session) {
+
+ checkNotNull(config);
+ checkNotNull(session);
+ checkNotBlank(config.getInstanceIds());
+ IJobKey jobKey = JobKeys.assertValid(IJobKey.build(config.getKey()));
+
+ Response resp = new Response();
+ try {
+ sessionValidator.checkAuthenticated(session, ImmutableSet.of(jobKey.getRole()));
+ ITaskConfig task = ConfigurationManager.validateAndPopulate(
+ ITaskConfig.build(config.getTaskConfig()));
+
+ if (cronJobManager.hasJob(jobKey)) {
+ return resp.setResponseCode(INVALID_REQUEST)
+ .setMessage("Cron jobs are not supported here.");
+ }
+
+ lockManager.validateIfLocked(
+ ILockKey.build(LockKey.job(jobKey.newBuilder())),
+ Optional.fromNullable(mutableLock).transform(ILock.FROM_BUILDER));
+
+ schedulerCore.addInstances(jobKey, ImmutableSet.copyOf(config.getInstanceIds()), task);
+ return resp.setResponseCode(OK).setMessage("Successfully added instances.");
+ } catch (AuthFailedException e) {
+ return resp.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ } catch (LockException e) {
+ return resp.setResponseCode(LOCK_ERROR).setMessage(e.getMessage());
+ } catch (TaskDescriptionException | ScheduleException e) {
+ return resp.setResponseCode(INVALID_REQUEST).setMessage(e.getMessage());
+ }
+ }
+
+ private String getRoleFromLockKey(ILockKey lockKey) {
+ switch (lockKey.getSetField()) {
+ case JOB:
+ JobKeys.assertValid(lockKey.getJob());
+ return lockKey.getJob().getRole();
+ default:
+ throw new IllegalArgumentException("Unhandled LockKey: " + lockKey.getSetField());
+ }
+ }
+
+ @Override
+ public Response acquireLock(LockKey mutableLockKey, SessionKey session) {
+ checkNotNull(mutableLockKey);
+ checkNotNull(session);
+
+ ILockKey lockKey = ILockKey.build(mutableLockKey);
+ Response response = new Response();
+
+ try {
+ SessionContext context = sessionValidator.checkAuthenticated(
+ session,
+ ImmutableSet.of(getRoleFromLockKey(lockKey)));
+
+ ILock lock = lockManager.acquireLock(lockKey, context.getIdentity());
+ response.setResult(Result.acquireLockResult(
+ new AcquireLockResult().setLock(lock.newBuilder())));
+
+ return response.setResponseCode(OK).setMessage("Lock has been acquired.");
+ } catch (AuthFailedException e) {
+ return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ } catch (LockException e) {
+ return response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
+ }
+ }
+
+ @Override
+ public Response releaseLock(Lock mutableLock, LockValidation validation, SessionKey session) {
+ checkNotNull(mutableLock);
+ checkNotNull(validation);
+ checkNotNull(session);
+
+ Response response = new Response();
+ ILock lock = ILock.build(mutableLock);
+
+ try {
+ sessionValidator.checkAuthenticated(
+ session,
+ ImmutableSet.of(getRoleFromLockKey(lock.getKey())));
+
+ if (validation == LockValidation.CHECKED) {
+ lockManager.validateIfLocked(lock.getKey(), Optional.of(lock));
+ }
+ lockManager.releaseLock(lock);
+ return response.setResponseCode(OK).setMessage("Lock has been released.");
+ } catch (AuthFailedException e) {
+ return response.setResponseCode(AUTH_FAILED).setMessage(e.getMessage());
+ } catch (LockException e) {
+ return response.setResponseCode(ResponseCode.LOCK_ERROR).setMessage(e.getMessage());
+ }
+ }
+
+ @VisibleForTesting
+ static Optional<String> transitionMessage(String user) {
+ return Optional.of("Transition forced by " + user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/ThriftConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/ThriftConfiguration.java b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftConfiguration.java
new file mode 100644
index 0000000..c6e9b18
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftConfiguration.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.google.common.base.Optional;
+
+/**
+ * Container for thrift server configuration options.
+ */
+public interface ThriftConfiguration {
+ /**
+ * Gets a stream for the thrift socket SSL key if this server is configured to use SSL.
+ *
+ * @return A stream that contains the SSL key data if SSL is enabled, absent otherwise.
+ * @throws IOException If the stream could not be opened.
+ */
+ Optional<? extends InputStream> getSslKeyStream() throws IOException;
+
+ /**
+ * Gets the port that the thrift server should listen on.
+ *
+ * @return Thrift server port.
+ */
+ int getServingPort();
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/ThriftModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/ThriftModule.java b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftModule.java
new file mode 100644
index 0000000..cca9053
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftModule.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift;
+
+import javax.inject.Singleton;
+
+import com.google.inject.AbstractModule;
+
+import com.twitter.aurora.gen.AuroraAdmin;
+import com.twitter.aurora.scheduler.thrift.aop.AopModule;
+import com.twitter.common.application.http.Registration;
+import com.twitter.common.application.modules.LifecycleModule;
+
+/**
+ * Binding module to configure a thrift server.
+ */
+public class ThriftModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(AuroraAdmin.Iface.class).to(SchedulerThriftInterface.class);
+ bind(ThriftServer.class).in(Singleton.class);
+ LifecycleModule.bindServiceRunner(binder(), ThriftServerLauncher.class);
+
+ Registration.registerServlet(binder(), "/api", SchedulerAPIServlet.class, true);
+
+ install(new AopModule());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServer.java b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServer.java
new file mode 100644
index 0000000..7b9abd1
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServer.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift;
+
+import java.net.ServerSocket;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.thrift.TProcessor;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+
+import com.twitter.thrift.Status;
+
+class ThriftServer {
+ private static final Logger LOG = Logger.getLogger(ThriftServer.class.getName());
+
+ private TServer server = null;
+
+ // Current health status of the server.
+ private Status status = Status.STARTING;
+
+ /**
+ * Starts the server.
+ * This may be called at any point except when the server is already alive. That is, it's
+ * allowable to start, stop, and re-start the server.
+ *
+ * @param socket The socket to use.
+ * @param processor The processor to handle requests.
+ */
+ public synchronized void start(ServerSocket socket, TProcessor processor) {
+ Preconditions.checkNotNull(socket);
+ Preconditions.checkNotNull(processor);
+ Preconditions.checkState(status != Status.ALIVE, "Server must only be started once.");
+ setStatus(Status.ALIVE);
+ TThreadPoolServer.Args args = new TThreadPoolServer.Args(new TServerSocket(socket))
+ .processor(processor)
+ .protocolFactory(new TBinaryProtocol.Factory(false, true));
+
+ final TServer starting = new TThreadPoolServer(args);
+ server = starting;
+ LOG.info("Starting thrift server on port " + socket.getLocalPort());
+
+ Thread listeningThread = new ThreadFactoryBuilder().setDaemon(false).build()
+ .newThread(new Runnable() {
+ @Override public void run() {
+ try {
+ starting.serve();
+ } catch (Throwable t) {
+ LOG.log(Level.WARNING,
+ "Uncaught exception while attempting to handle service requests: " + t, t);
+ setStatus(Status.DEAD);
+ }
+ }
+ });
+
+ listeningThread.start();
+ }
+
+ private synchronized void setStatus(Status status) {
+ LOG.info("Moving from status " + this.status + " to " + status);
+ this.status = status;
+ }
+
+ /**
+ * Attempts to shut down the server.
+ * The server may be shut down at any time, though the request will be ignored if the server is
+ * already stopped.
+ */
+ public synchronized void shutdown() {
+ if (status == Status.STOPPED) {
+ LOG.info("Server already stopped, shutdown request ignored.");
+ return;
+ }
+
+ LOG.info("Received shutdown request, stopping server.");
+ setStatus(Status.STOPPING);
+
+ // TODO(William Farner): Figure out what happens to queued / in-process requests when the server
+ // is stopped. Might want to allow a sleep period for the active requests to be completed.
+
+ if (server != null) {
+ server.stop();
+ }
+
+ server = null;
+ setStatus(Status.STOPPED);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServerLauncher.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServerLauncher.java b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServerLauncher.java
new file mode 100644
index 0000000..6743060
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/ThriftServerLauncher.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.ServerSocket;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLServerSocketFactory;
+
+import com.google.common.base.Optional;
+
+import com.twitter.aurora.gen.AuroraAdmin;
+import com.twitter.aurora.gen.AuroraAdmin.Iface;
+import com.twitter.common.application.modules.LifecycleModule.ServiceRunner;
+import com.twitter.common.application.modules.LocalServiceRegistry.LocalService;
+import com.twitter.common.base.Command;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Service launcher that starts up and registers the scheduler thrift server as a primary service
+ * for the application.
+ */
+class ThriftServerLauncher implements ServiceRunner {
+
+ private static final Logger LOG = Logger.getLogger(ThriftServerLauncher.class.getName());
+
+ private final ThriftConfiguration configuration;
+
+ // Security is enforced via file permissions, not via this password, for what it's worth.
+ private static final String SSL_KEYFILE_PASSWORD = "MesosKeyStorePassword";
+
+ private final Iface schedulerThriftInterface;
+ private final ThriftServer schedulerThriftServer;
+
+ @Inject
+ ThriftServerLauncher(
+ Iface schedulerThriftInterface,
+ ThriftServer schedulerThriftServer,
+ ThriftConfiguration configuration) {
+
+ this.schedulerThriftInterface = checkNotNull(schedulerThriftInterface);
+ this.schedulerThriftServer = checkNotNull(schedulerThriftServer);
+ this.configuration = checkNotNull(configuration);
+ }
+
+ @Override
+ public LocalService launch() {
+ ServerSocket socket = getServerSocket();
+ schedulerThriftServer.start(
+ socket,
+ new AuroraAdmin.Processor<>(schedulerThriftInterface));
+
+ Command shutdown = new Command() {
+ @Override public void execute() {
+ LOG.info("Stopping thrift server.");
+ schedulerThriftServer.shutdown();
+ }
+ };
+
+ return LocalService.primaryService(socket.getLocalPort(), shutdown);
+ }
+
+ private ServerSocket getServerSocket() {
+ try {
+ Optional<? extends InputStream> sslKeyStream = configuration.getSslKeyStream();
+ if (!sslKeyStream.isPresent()) {
+ LOG.warning("Running Thrift Server without SSL.");
+ return new ServerSocket(configuration.getServingPort());
+ } else {
+ // TODO(Kevin Sweeney): Add helper to perform this keyfile import.
+ KeyStore ks = KeyStore.getInstance("JKS");
+ ks.load(sslKeyStream.get(), SSL_KEYFILE_PASSWORD.toCharArray());
+
+ KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+ kmf.init(ks, SSL_KEYFILE_PASSWORD.toCharArray());
+
+ SSLContext ctx = SSLContext.getInstance("TLS");
+ ctx.init(kmf.getKeyManagers(), null, null);
+
+ SSLServerSocketFactory ssf = ctx.getServerSocketFactory();
+ SSLServerSocket serverSocket = (SSLServerSocket) ssf.createServerSocket(
+ configuration.getServingPort());
+ serverSocket.setEnabledCipherSuites(serverSocket.getSupportedCipherSuites());
+ serverSocket.setNeedClientAuth(false);
+ return serverSocket;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read key file.", e);
+ } catch (GeneralSecurityException e) {
+ throw new RuntimeException("SSL setup failed.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/APIVersionInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
new file mode 100644
index 0000000..d66a2b2
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/APIVersionInterceptor.java
@@ -0,0 +1,20 @@
+package com.twitter.aurora.scheduler.thrift.aop;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+
+import com.twitter.aurora.gen.Response;
+
+import static com.twitter.aurora.gen.apiConstants.CURRENT_API_VERSION;
+
+class APIVersionInterceptor implements MethodInterceptor {
+
+ @Override
+ public Object invoke(MethodInvocation invocation) throws Throwable {
+ Response resp = (Response) invocation.proceed();
+ if (resp.version == null) {
+ resp.setVersion(CURRENT_API_VERSION);
+ }
+ return resp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
new file mode 100644
index 0000000..4afc263
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.aop;
+
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.inject.AbstractModule;
+import com.google.inject.Binder;
+import com.google.inject.Key;
+import com.google.inject.PrivateModule;
+import com.google.inject.TypeLiteral;
+import com.google.inject.matcher.Matcher;
+import com.google.inject.matcher.Matchers;
+
+import org.aopalliance.intercept.MethodInterceptor;
+
+import com.twitter.aurora.GuiceUtils;
+import com.twitter.aurora.auth.CapabilityValidator;
+import com.twitter.aurora.gen.AuroraAdmin;
+import com.twitter.aurora.gen.AuroraSchedulerManager;
+import com.twitter.aurora.scheduler.thrift.auth.DecoratedThrift;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+
+/**
+ * Binding module for AOP-style decorations of the thrift API.
+ */
+public class AopModule extends AbstractModule {
+
+ @CmdLine(name = "enable_job_updates", help = "Whether new job updates should be accepted.")
+ private static final Arg<Boolean> ENABLE_UPDATES = Arg.create(true);
+
+ @CmdLine(name = "enable_job_creation",
+ help = "Allow new jobs to be created, if false all job creation requests will be denied.")
+ private static final Arg<Boolean> ENABLE_JOB_CREATION = Arg.create(true);
+
+ private static final Matcher<? super Class<?>> THRIFT_IFACE_MATCHER =
+ Matchers.subclassesOf(AuroraAdmin.Iface.class)
+ .and(Matchers.annotatedWith(DecoratedThrift.class));
+
+ private final Map<String, Boolean> toggledMethods;
+
+ public AopModule() {
+ this(ImmutableMap.of(
+ "createJob", ENABLE_JOB_CREATION.get(),
+ "acquireLock", ENABLE_UPDATES.get()));
+ }
+
+ @VisibleForTesting
+ AopModule(Map<String, Boolean> toggledMethods) {
+ this.toggledMethods = ImmutableMap.copyOf(toggledMethods);
+ }
+
+ private static final Function<Method, String> GET_NAME = new Function<Method, String>() {
+ @Override public String apply(Method method) {
+ return method.getName();
+ }
+ };
+
+ @Override
+ protected void configure() {
+ requireBinding(CapabilityValidator.class);
+
+ // Layer ordering:
+ // Log -> CapabilityValidator -> FeatureToggle -> StatsExporter -> APIVersion ->
+ // SchedulerThriftInterface
+
+ // TODO(Sathya): Consider using provider pattern for constructing interceptors to facilitate
+ // unit testing without the creation of Guice injectors.
+ bindThriftDecorator(new LoggingInterceptor());
+
+ // Note: it's important that the capability interceptor is only applied to AuroraAdmin.Iface
+ // methods, and does not pick up methods on AuroraSchedulerManager.Iface.
+ MethodInterceptor authInterceptor = new UserCapabilityInterceptor();
+ requestInjection(authInterceptor);
+ bindInterceptor(
+ THRIFT_IFACE_MATCHER,
+ GuiceUtils.interfaceMatcher(AuroraAdmin.Iface.class, true),
+ authInterceptor);
+
+ install(new PrivateModule() {
+ @Override protected void configure() {
+ // Ensure that the provided methods exist on the decorated interface.
+ List<Method> methods =
+ ImmutableList.copyOf(AuroraSchedulerManager.Iface.class.getMethods());
+ for (String toggledMethod : toggledMethods.keySet()) {
+ Preconditions.checkArgument(
+ Iterables.any(methods,
+ Predicates.compose(Predicates.equalTo(toggledMethod), GET_NAME)),
+ String.format("Method %s was not found in class %s",
+ toggledMethod,
+ AuroraSchedulerManager.Iface.class));
+ }
+
+ bind(new TypeLiteral<Map<String, Boolean>>() { }).toInstance(toggledMethods);
+ bind(IsFeatureEnabled.class).in(Singleton.class);
+ Key<Predicate<Method>> predicateKey = Key.get(new TypeLiteral<Predicate<Method>>() { });
+ bind(predicateKey).to(IsFeatureEnabled.class);
+ expose(predicateKey);
+ }
+ });
+ bindThriftDecorator(new FeatureToggleInterceptor());
+ bindThriftDecorator(new ThriftStatsExporterInterceptor());
+ bindThriftDecorator(new APIVersionInterceptor());
+ }
+
+ private void bindThriftDecorator(MethodInterceptor interceptor) {
+ bindThriftDecorator(binder(), THRIFT_IFACE_MATCHER, interceptor);
+ }
+
+ @VisibleForTesting
+ static void bindThriftDecorator(
+ Binder binder,
+ Matcher<? super Class<?>> classMatcher,
+ MethodInterceptor interceptor) {
+
+ binder.bindInterceptor(classMatcher, Matchers.any(), interceptor);
+ binder.requestInjection(interceptor);
+ }
+
+ private static class IsFeatureEnabled implements Predicate<Method> {
+ private final Predicate<String> methodEnabled;
+
+ @Inject
+ IsFeatureEnabled(Map<String, Boolean> toggleMethods) {
+ Predicate<String> builder = Predicates.alwaysTrue();
+ for (Map.Entry<String, Boolean> toggleMethod : toggleMethods.entrySet()) {
+ Predicate<String> enableMethod = Predicates.or(
+ toggleMethod.getValue()
+ ? Predicates.<String>alwaysTrue()
+ : Predicates.<String>alwaysFalse(),
+ Predicates.not(Predicates.equalTo(toggleMethod.getKey())));
+ builder = Predicates.and(builder, enableMethod);
+ }
+ methodEnabled = builder;
+ }
+
+ @Override
+ public boolean apply(Method method) {
+ return methodEnabled.apply(method.getName());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
new file mode 100644
index 0000000..03c3d99
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.aop;
+
+import java.lang.reflect.Method;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Predicate;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+
+import com.twitter.aurora.gen.ResponseCode;
+
+/**
+ * A method interceptor that blocks access to features based on a supplied predicate.
+ */
+public class FeatureToggleInterceptor implements MethodInterceptor {
+
+ @Inject private Predicate<Method> allowMethod;
+
+ @Override
+ public Object invoke(MethodInvocation invocation) throws Throwable {
+ Method method = invocation.getMethod();
+ if (!allowMethod.apply(method)) {
+ return Interceptors.properlyTypedResponse(
+ method,
+ ResponseCode.ERROR,
+ "The " + method.getName() + " feature is currently disabled on this scheduler.");
+ } else {
+ return invocation.proceed();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/aop/Interceptors.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/Interceptors.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/Interceptors.java
new file mode 100644
index 0000000..d0cb9c1
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/Interceptors.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.aop;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.logging.Logger;
+
+import com.google.common.base.Throwables;
+
+import com.twitter.aurora.gen.ResponseCode;
+
+/**
+ * Utility class for functions useful when implementing an interceptor on the thrift interface.
+ */
+final class Interceptors {
+
+ private Interceptors() {
+ // Utility class.
+ }
+
+ private static final Logger LOG = Logger.getLogger(Interceptors.class.getName());
+
+ static Object properlyTypedResponse(Method method, ResponseCode responseCode, String message)
+ throws IllegalAccessException, InstantiationException {
+
+ Class<?> returnType = method.getReturnType();
+ Object response = returnType.newInstance();
+ invoke(returnType, response, "setResponseCode", ResponseCode.class, responseCode);
+ invoke(returnType, response, "setMessage", String.class, message);
+ return response;
+ }
+
+ private static <T> void invoke(
+ Class<?> type,
+ Object obj,
+ String name,
+ Class<T> parameterType,
+ T argument) {
+
+ Method method;
+ try {
+ method = type.getMethod(name, parameterType);
+ } catch (NoSuchMethodException e) {
+ LOG.severe(type + " does not support " + name);
+ throw Throwables.propagate(e);
+ }
+ try {
+ method.invoke(obj, argument);
+ } catch (IllegalAccessException e) {
+ LOG.severe("Method " + name + " is not accessible in " + type);
+ throw Throwables.propagate(e);
+ } catch (InvocationTargetException e) {
+ LOG.severe("Failed to invoke " + name + " on " + type);
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/aop/LoggingInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/LoggingInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/LoggingInterceptor.java
new file mode 100644
index 0000000..5f773cc
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/LoggingInterceptor.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.aop;
+
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+
+import com.twitter.aurora.auth.CapabilityValidator;
+import com.twitter.aurora.gen.ExecutorConfig;
+import com.twitter.aurora.gen.JobConfiguration;
+import com.twitter.aurora.gen.ResponseCode;
+import com.twitter.aurora.gen.SessionKey;
+
+import static com.twitter.aurora.scheduler.thrift.aop.Interceptors.properlyTypedResponse;
+
+/**
+ * A method interceptor that logs all invocations as well as any unchecked exceptions thrown from
+ * the underlying call.
+ */
+class LoggingInterceptor implements MethodInterceptor {
+
+ private static final Logger LOG = Logger.getLogger(LoggingInterceptor.class.getName());
+
+ @Inject private CapabilityValidator validator;
+
+ // TODO(wfarner): Scrub updateToken when it is identifiable by type.
+ private final Map<Class<?>, Function<Object, String>> printFunctions =
+ ImmutableMap.<Class<?>, Function<Object, String>>of(
+ JobConfiguration.class,
+ new Function<Object, String>() {
+ @Override public String apply(Object input) {
+ JobConfiguration configuration = ((JobConfiguration) input).deepCopy();
+ if (configuration.isSetTaskConfig()) {
+ configuration.getTaskConfig().setExecutorConfig(
+ new ExecutorConfig("BLANKED", "BLANKED"));
+ }
+ return configuration.toString();
+ }
+ },
+ SessionKey.class,
+ new Function<Object, String>() {
+ @Override public String apply(Object input) {
+ SessionKey key = (SessionKey) input;
+ return validator.toString(key);
+ }
+ }
+ );
+
+ @Override
+ public Object invoke(MethodInvocation invocation) throws Throwable {
+ List<String> argStrings = Lists.newArrayList();
+ for (Object arg : invocation.getArguments()) {
+ if (arg == null) {
+ argStrings.add("null");
+ } else {
+ Function<Object, String> printFunction = printFunctions.get(arg.getClass());
+ argStrings.add((printFunction == null) ? arg.toString() : printFunction.apply(arg));
+ }
+ }
+ String methodName = invocation.getMethod().getName();
+ String message = String.format("%s(%s)", methodName, Joiner.on(", ").join(argStrings));
+ LOG.info(message);
+ try {
+ return invocation.proceed();
+ } catch (RuntimeException e) {
+ LOG.log(Level.WARNING, "Uncaught exception while handling " + message, e);
+ return properlyTypedResponse(invocation.getMethod(), ResponseCode.ERROR, e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
new file mode 100644
index 0000000..d700ab5
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/ThriftStatsExporterInterceptor.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.aop;
+
+import java.lang.reflect.Method;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+
+import com.twitter.common.stats.SlidingStats;
+import com.twitter.common.stats.Stats;
+
+/**
+ * A method interceptor that exports counterStats about thrift calls.
+ */
+class ThriftStatsExporterInterceptor implements MethodInterceptor {
+
+ private final LoadingCache<Method, SlidingStats> stats =
+ CacheBuilder.newBuilder().build(new CacheLoader<Method, SlidingStats>() {
+ @Override public SlidingStats load(Method method) {
+ return new SlidingStats(
+ Stats.normalizeName(String.format("scheduler_thrift_%s", method.getName())),
+ "nanos");
+ }
+ });
+
+ @Override
+ public Object invoke(MethodInvocation invocation) throws Throwable {
+ SlidingStats stat = stats.get(invocation.getMethod());
+ long start = System.nanoTime();
+ try {
+ return invocation.proceed();
+ } finally {
+ stat.accumulate(System.nanoTime() - start);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java
new file mode 100644
index 0000000..d9240bc
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/UserCapabilityInterceptor.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.aop;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+
+import org.aopalliance.intercept.MethodInterceptor;
+import org.aopalliance.intercept.MethodInvocation;
+
+import com.twitter.aurora.auth.CapabilityValidator;
+import com.twitter.aurora.auth.CapabilityValidator.AuditCheck;
+import com.twitter.aurora.auth.CapabilityValidator.Capability;
+import com.twitter.aurora.auth.SessionValidator.AuthFailedException;
+import com.twitter.aurora.gen.ResponseCode;
+import com.twitter.aurora.gen.SessionKey;
+import com.twitter.aurora.scheduler.thrift.auth.Requires;
+
+/**
+ * A method interceptor that will authenticate users identified by a {@link SessionKey} argument
+ * to invoked methods.
+ * <p>
+ * Intercepted methods will require {@link Capability#ROOT}, but additional capabilities
+ * may be specified by annotating methods with {@link Requires} and supplying a whitelist.
+ */
+class UserCapabilityInterceptor implements MethodInterceptor {
+ private static final Logger LOG = Logger.getLogger(UserCapabilityInterceptor.class.getName());
+
+ @Inject private CapabilityValidator capabilityValidator;
+
+ private static final Function<Object, SessionKey> CAST = new Function<Object, SessionKey>() {
+ @Override public SessionKey apply(Object o) {
+ return (SessionKey) o;
+ }
+ };
+
+ @Override
+ public Object invoke(MethodInvocation invocation) throws Throwable {
+ Preconditions.checkNotNull(capabilityValidator, "Session validator has not yet been set.");
+
+ // Ensure ROOT is always permitted.
+ ImmutableList.Builder<Capability> whitelistBuilder =
+ ImmutableList.<Capability>builder().add(Capability.ROOT);
+
+ Method method = invocation.getMethod();
+ Requires requires = method.getAnnotation(Requires.class);
+ if (requires != null) {
+ whitelistBuilder.add(requires.whitelist());
+ }
+
+ List<Capability> whitelist = whitelistBuilder.build();
+ LOG.fine("Operation " + method.getName() + " may be performed by: " + whitelist);
+ Optional<SessionKey> sessionKey = FluentIterable.from(Arrays.asList(invocation.getArguments()))
+ .firstMatch(Predicates.instanceOf(SessionKey.class)).transform(CAST);
+ if (!sessionKey.isPresent()) {
+ LOG.severe("Interceptor should only be applied to methods accepting a SessionKey, but "
+ + method + " does not.");
+ return invocation.proceed();
+ }
+
+ String key = capabilityValidator.toString(sessionKey.get());
+ for (Capability user : whitelist) {
+ LOG.fine("Attempting to validate " + key + " against " + user);
+ try {
+ capabilityValidator.checkAuthorized(sessionKey.get(), user, AuditCheck.NONE);
+
+ LOG.info("Permitting " + key + " to act as "
+ + user + " and perform action " + method.getName());
+ return invocation.proceed();
+ } catch (AuthFailedException e) {
+ LOG.fine("Auth failed: " + e);
+ }
+ }
+
+ // User is not permitted to perform this operation.
+ return Interceptors.properlyTypedResponse(
+ method,
+ ResponseCode.AUTH_FAILED,
+ "Session identified by '" + key
+ + "' does not have the required capability to perform this action: " + whitelist);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/auth/DecoratedThrift.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/auth/DecoratedThrift.java b/src/main/java/org/apache/aurora/scheduler/thrift/auth/DecoratedThrift.java
new file mode 100644
index 0000000..4a667c5
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/auth/DecoratedThrift.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.auth;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.ElementType.TYPE;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Type annotation to apply to a thrift interface implementation that should be decorated with
+ * additional functionality.
+ */
+@Target({PARAMETER, TYPE}) @Retention(RUNTIME)
+public @interface DecoratedThrift {
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/thrift/auth/Requires.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/auth/Requires.java b/src/main/java/org/apache/aurora/scheduler/thrift/auth/Requires.java
new file mode 100644
index 0000000..0fff3f6
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/auth/Requires.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.thrift.auth;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import com.twitter.aurora.auth.CapabilityValidator.Capability;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Annotation applied to a method that may allow users with non-ROOT capabilities to perform
+ * an action.
+ */
+@Target(METHOD) @Retention(RUNTIME)
+public @interface Requires {
+ /**
+ * The list of capabilities required to perform an action.
+ */
+ Capability[] whitelist() default { Capability.ROOT };
+}
[45/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/Slaves.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Slaves.java b/src/main/java/com/twitter/aurora/scheduler/http/Slaves.java
deleted file mode 100644
index 401ee30..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/Slaves.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Ordering;
-
-import org.antlr.stringtemplate.StringTemplate;
-
-import com.twitter.aurora.gen.Attribute;
-import com.twitter.aurora.gen.HostAttributes;
-import com.twitter.aurora.gen.MaintenanceMode;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.common.base.Closure;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import static com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import static com.twitter.aurora.scheduler.storage.Storage.Work;
-import static com.twitter.common.base.MorePreconditions.checkNotBlank;
-
-/**
- * HTTP interface to serve as a HUD for the mesos slaves tracked in the scheduler.
- */
-@Path("/slaves")
-public class Slaves extends JerseyTemplateServlet {
- private final String clusterName;
- private Storage storage;
-
- /**
- * Injected constructor.
- *
- * @param clusterName cluster name
- * @param storage store to fetch the host attributes from
- */
- @Inject
- public Slaves(@ClusterName String clusterName, Storage storage) {
- super("slaves");
- this.clusterName = checkNotBlank(clusterName);
- this.storage = checkNotNull(storage);
- }
-
- private Iterable<HostAttributes> getHostAttributes() {
- return storage.weaklyConsistentRead(new Work.Quiet<Iterable<HostAttributes>>() {
- @Override public Iterable<HostAttributes> apply(StoreProvider storeProvider) {
- return storeProvider.getAttributeStore().getHostAttributes();
- }
- });
- }
-
- private static final Function<HostAttributes, Slave> TO_SLAVE =
- new Function<HostAttributes, Slave>() {
- @Override public Slave apply(HostAttributes attributes) {
- return new Slave(attributes);
- }
- };
-
- /**
- * Fetches the listing of known slaves.
- *
- * @return HTTP response.
- */
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response get() {
- return fillTemplate(new Closure<StringTemplate>() {
- @Override public void execute(StringTemplate template) {
- template.setAttribute("cluster_name", clusterName);
-
- template.setAttribute("slaves",
- FluentIterable.from(getHostAttributes()).transform(TO_SLAVE).toList());
- }
- });
- }
-
- private static final Ordering<Attribute> ATTR_ORDER = Ordering.natural().onResultOf(
- new Function<Attribute, String>() {
- @Override public String apply(Attribute attr) {
- return attr .getName();
- }
- });
-
- /**
- * Template object to represent a slave.
- */
- private static class Slave {
- private final HostAttributes attributes;
-
- Slave(HostAttributes attributes) {
- this.attributes = attributes;
- }
-
- public String getHost() {
- return attributes.getHost();
- }
-
- public String getId() {
- return attributes.getSlaveId();
- }
-
- public MaintenanceMode getMode() {
- return attributes.getMode();
- }
-
- private static final Function<Attribute, String> ATTR_TO_STRING =
- new Function<Attribute, String>() {
- @Override public String apply(Attribute attr) {
- return attr.getName() + "=[" + Joiner.on(",").join(attr.getValues()) + "]";
- }
- };
-
- public String getAttributes() {
- return Joiner.on(", ").join(
- Iterables.transform(ATTR_ORDER.sortedCopy(attributes.getAttributes()), ATTR_TO_STRING));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/StructDump.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/StructDump.java b/src/main/java/com/twitter/aurora/scheduler/http/StructDump.java
deleted file mode 100644
index 8da8b80..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/StructDump.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-
-import org.antlr.stringtemplate.StringTemplate;
-import org.apache.thrift.TBase;
-
-import com.twitter.aurora.gen.JobConfiguration;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.state.CronJobManager;
-import com.twitter.aurora.scheduler.storage.Storage;
-import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
-import com.twitter.aurora.scheduler.storage.Storage.Work;
-import com.twitter.aurora.scheduler.storage.Storage.Work.Quiet;
-import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
-import com.twitter.aurora.scheduler.storage.entities.IJobKey;
-import com.twitter.common.base.Closure;
-import com.twitter.common.thrift.Util;
-
-/**
- * Servlet that prints out the raw configuration for a specified struct.
- */
-@Path("/structdump")
-public class StructDump extends JerseyTemplateServlet {
-
- private final Storage storage;
-
- @Inject
- public StructDump(Storage storage) {
- super("structdump");
- this.storage = Preconditions.checkNotNull(storage);
- }
-
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response getUsage() {
- return Response
- .status(Status.BAD_REQUEST)
- .entity("<html>Usage: /structdump/{task_id} or /structdump/job/{role}/{env}/{job}</html>")
- .build();
- }
-
- /**
- * Dumps a task struct.
- *
- * @return HTTP response.
- */
- @GET
- @Path("/task/{task}")
- @Produces(MediaType.TEXT_HTML)
- public Response dumpJob(
- @PathParam("task") final String taskId) {
-
- return dumpEntity("Task " + taskId, new Work.Quiet<Optional<? extends TBase<?, ?>>>() {
- @Override public Optional<? extends TBase<?, ?>> apply(StoreProvider storeProvider) {
- // Deep copy the struct to sidestep any subclass trickery inside the storage system.
- return Optional.fromNullable(Iterables.getOnlyElement(
- storeProvider.getTaskStore().fetchTasks(Query.taskScoped(taskId)),
- null)
- .newBuilder());
- }
- });
- }
-
- /**
- * Dumps a cron job struct.
- *
- * @return HTTP response.
- */
- @GET
- @Path("/cron/{role}/{environment}/{job}")
- @Produces(MediaType.TEXT_HTML)
- public Response dump(
- @PathParam("role") final String role,
- @PathParam("environment") final String environment,
- @PathParam("job") final String job) {
-
- final IJobKey jobKey = JobKeys.from(role, environment, job);
- return dumpEntity("Cron job " + JobKeys.toPath(jobKey),
- new Work.Quiet<Optional<? extends TBase<?, ?>>>() {
- @Override public Optional<JobConfiguration> apply(StoreProvider storeProvider) {
- return storeProvider.getJobStore().fetchJob(CronJobManager.MANAGER_KEY, jobKey)
- .transform(IJobConfiguration.TO_BUILDER);
- }
- });
- }
-
- private Response dumpEntity(final String id, final Quiet<Optional<? extends TBase<?, ?>>> work) {
- return fillTemplate(new Closure<StringTemplate>() {
- @Override public void execute(StringTemplate template) {
- template.setAttribute("id", id);
- Optional<? extends TBase<?, ?>> struct = storage.weaklyConsistentRead(work);
- if (!struct.isPresent()) {
- template.setAttribute("exception", "Entity not found");
- } else {
- template.setAttribute("structPretty", Util.prettyPrint(struct.get()));
- }
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/TransformationUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/TransformationUtils.java b/src/main/java/com/twitter/aurora/scheduler/http/TransformationUtils.java
deleted file mode 100644
index 2f531a7..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/TransformationUtils.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.util.Collection;
-
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Range;
-
-import com.twitter.aurora.scheduler.base.Numbers;
-import com.twitter.aurora.scheduler.storage.entities.IPackage;
-
-/**
- * Utility class to hold common object to string transformation helper functions.
- */
-final class TransformationUtils {
- public static final Function<IPackage, String> PACKAGE_TOSTRING =
- new Function<IPackage, String>() {
- @Override public String apply(IPackage pkg) {
- return pkg.getRole() + "/" + pkg.getName() + " v" + pkg.getVersion();
- }
- };
-
- public static final Function<Range<Integer>, String> RANGE_TOSTRING =
- new Function<Range<Integer>, String>() {
- @Override public String apply(Range<Integer> range) {
- int lower = range.lowerEndpoint();
- int upper = range.upperEndpoint();
- return (lower == upper) ? String.valueOf(lower) : (lower + " - " + upper);
- }
- };
-
- public static final Function<Collection<Integer>, String> INSTANCES_TOSTRING =
- new Function<Collection<Integer>, String>() {
- @Override public String apply(Collection<Integer> instances) {
- return Joiner.on(", ")
- .join(Iterables.transform(Numbers.toRanges(instances), RANGE_TOSTRING));
- }
- };
-
- private TransformationUtils() {
- // Utility class
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/http/Utilization.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Utilization.java b/src/main/java/com/twitter/aurora/scheduler/http/Utilization.java
deleted file mode 100644
index 60a7d95..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/http/Utilization.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.http;
-
-import java.io.StringWriter;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.FluentIterable;
-
-import org.antlr.stringtemplate.StringTemplate;
-
-import com.twitter.aurora.scheduler.base.Query;
-import com.twitter.aurora.scheduler.stats.ResourceCounter;
-import com.twitter.aurora.scheduler.stats.ResourceCounter.GlobalMetric;
-import com.twitter.aurora.scheduler.stats.ResourceCounter.Metric;
-import com.twitter.aurora.scheduler.stats.ResourceCounter.MetricType;
-import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
-import com.twitter.common.base.Closure;
-import com.twitter.common.base.MorePreconditions;
-import com.twitter.common.util.templating.StringTemplateHelper;
-import com.twitter.common.util.templating.StringTemplateHelper.TemplateException;
-
-/**
- * A servlet to give an aggregate view of cluster resources consumed, grouped by category.
- */
-@Path("/utilization")
-public class Utilization {
-
- private final String clusterName;
- private final ResourceCounter counter;
- private final StringTemplateHelper templateHelper;
-
- @Inject
- Utilization(ResourceCounter counter, @ClusterName String clusterName) {
- templateHelper = new StringTemplateHelper(getClass(), "utilization", true);
- this.counter = Preconditions.checkNotNull(counter);
- this.clusterName = MorePreconditions.checkNotBlank(clusterName);
- }
-
- private String fillTemplate(Map<Display, Metric> metrics) {
- Function<Entry<Display, Metric>, DisplayMetric> transform =
- new Function<Entry<Display, Metric>, DisplayMetric>() {
- @Override public DisplayMetric apply(Entry<Display, Metric> entry) {
- return new DisplayMetric(entry.getKey(), entry.getValue());
- }
- };
- return fillTemplate(FluentIterable.from(metrics.entrySet()).transform(transform).toList());
- }
-
- private String fillTemplate(final Iterable<DisplayMetric> metrics) {
- StringWriter output = new StringWriter();
- try {
- templateHelper.writeTemplate(output, new Closure<StringTemplate>() {
- @Override public void execute(StringTemplate template) {
- template.setAttribute("cluster_name", clusterName);
- template.setAttribute("metrics", metrics);
- }
- });
- } catch (TemplateException e) {
- throw new WebApplicationException(e);
- }
- return output.toString();
- }
-
- private static class Display {
- private final String title;
- @Nullable
- private final String link;
-
- Display(String title, @Nullable String link) {
- this.title = title;
- this.link = link;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(title, link);
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof Display)) {
- return false;
- }
-
- Display other = (Display) o;
- return Objects.equal(title, other.title) && Objects.equal(link, other.link);
- }
- }
-
- private static class DisplayMetric extends Metric {
- private final Display display;
-
- DisplayMetric(Display display, Metric wrapped) {
- super(wrapped);
- this.display = display;
- }
-
- public String getTitle() {
- return display.title;
- }
-
- @Nullable
- public String getLink() {
- return display.link;
- }
- }
-
- private static final Function<GlobalMetric, DisplayMetric> TO_DISPLAY =
- new Function<GlobalMetric, DisplayMetric>() {
- @Override public DisplayMetric apply(GlobalMetric count) {
- return new DisplayMetric(
- new Display(
- count.type.name().replace('_', ' ').toLowerCase(),
- count.type.name().toLowerCase()),
- count);
- }
- };
-
- /**
- * Displays the aggregate utilization for the entire cluster.
- *
- * @return HTML-formatted cluster utilization.
- */
- @GET
- @Produces(MediaType.TEXT_HTML)
- public Response aggregateCluster() {
- Iterable<DisplayMetric> metrics =
- FluentIterable.from(counter.computeConsumptionTotals()).transform(TO_DISPLAY).toList();
- return Response.ok(fillTemplate(metrics)).build();
- }
-
- private MetricType getTypeByName(String name) throws WebApplicationException {
- MetricType type = MetricType.valueOf(name.toUpperCase());
- if (type == null) {
- throw new WebApplicationException(
- Response.status(Status.BAD_REQUEST).entity("Invalid metric type.").build());
- }
- return type;
- }
-
- /**
- * Displays the aggregate utilization for roles within a metric type.
- *
- * @param metric Metric id.
- * @return HTML-formatted utilization within the metric type.
- */
- @GET
- @Path("/{metric}")
- @Produces(MediaType.TEXT_HTML)
- public Response aggregateRoles(@PathParam("metric") final String metric) {
- final MetricType type = getTypeByName(metric);
-
- Function<ITaskConfig, Display> toKey = new Function<ITaskConfig, Display>() {
- @Override public Display apply(ITaskConfig task) {
- String role = task.getOwner().getRole();
- return new Display(role, metric + "/" + role);
- }
- };
- Map<Display, Metric> byRole =
- counter.computeAggregates(Query.unscoped().active(), type.filter, toKey);
- return Response.ok(fillTemplate(byRole)).build();
- }
-
- /**
- * Displays the aggregate utilization for jobs within a role.
- *
- * @param metric Metric id.
- * @param role Role for jobs to aggregate.
- * @return HTML-formatted utilization within the metric/role.
- */
- @GET
- @Path("/{metric}/{role}")
- @Produces(MediaType.TEXT_HTML)
- public Response aggregateJobs(
- @PathParam("metric") String metric,
- @PathParam("role") String role) {
-
- MetricType type = getTypeByName(metric);
- Function<ITaskConfig, Display> toKey = new Function<ITaskConfig, Display>() {
- @Override public Display apply(ITaskConfig task) {
- return new Display(task.getJobName(), null);
- }
- };
- Map<Display, Metric> byJob =
- counter.computeAggregates(Query.roleScoped(role).active(), type.filter, toKey);
- return Response.ok(fillTemplate(byJob)).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/local/FakeDriverFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/local/FakeDriverFactory.java b/src/main/java/com/twitter/aurora/scheduler/local/FakeDriverFactory.java
deleted file mode 100644
index ae5062e..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/local/FakeDriverFactory.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.local;
-
-import java.util.Collection;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.inject.Provider;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-
-import org.apache.mesos.Protos.ExecutorID;
-import org.apache.mesos.Protos.Filters;
-import org.apache.mesos.Protos.FrameworkID;
-import org.apache.mesos.Protos.MasterInfo;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.Request;
-import org.apache.mesos.Protos.SlaveID;
-import org.apache.mesos.Protos.Status;
-import org.apache.mesos.Protos.TaskID;
-import org.apache.mesos.Protos.TaskInfo;
-import org.apache.mesos.Protos.TaskStatus;
-import org.apache.mesos.Scheduler;
-import org.apache.mesos.SchedulerDriver;
-
-import com.twitter.aurora.scheduler.DriverFactory;
-import com.twitter.common.application.Lifecycle;
-
-/**
- * A factory for fake scheduler driver instances.
- */
-class FakeDriverFactory implements DriverFactory {
-
- private final Provider<Scheduler> scheduler;
- private final Lifecycle lifecycle;
-
- @Inject
- FakeDriverFactory(Provider<Scheduler> scheduler, Lifecycle lifecycle) {
- this.scheduler = Preconditions.checkNotNull(scheduler);
- this.lifecycle = Preconditions.checkNotNull(lifecycle);
- }
-
- @Override
- public SchedulerDriver apply(@Nullable final String frameworkId) {
- return new FakeSchedulerDriver() {
- @Override public Status run() {
- scheduler.get().registered(
- this,
- FrameworkID.newBuilder().setValue(
- Optional.fromNullable(frameworkId).or("new-framework-id")).build(),
- MasterInfo.newBuilder().setId("master-id").setIp(100).setPort(200).build());
- lifecycle.awaitShutdown();
- return null;
- }
- };
- }
-
- static class FakeSchedulerDriver implements SchedulerDriver {
- @Override public Status start() {
- return null;
- }
-
- @Override public Status stop(boolean b) {
- return null;
- }
-
- @Override public Status stop() {
- return null;
- }
-
- @Override public Status abort() {
- return null;
- }
-
- @Override public Status join() {
- return run();
- }
-
- @Override public Status run() {
- return null;
- }
-
- @Override public Status requestResources(Collection<Request> requests) {
- return null;
- }
-
- @Override public Status launchTasks(OfferID offerID, Collection<TaskInfo> taskInfos,
- Filters filters) {
- return null;
- }
-
- @Override public Status launchTasks(OfferID offerID, Collection<TaskInfo> taskInfos) {
- return null;
- }
-
- @Override public Status killTask(TaskID taskID) {
- return null;
- }
-
- @Override public Status declineOffer(OfferID offerID, Filters filters) {
- return null;
- }
-
- @Override public Status declineOffer(OfferID offerID) {
- return null;
- }
-
- @Override public Status reviveOffers() {
- return null;
- }
-
- @Override public Status sendFrameworkMessage(ExecutorID executorID, SlaveID slaveID,
- byte[] bytes) {
- return null;
- }
-
- @Override
- public Status reconcileTasks(Collection<TaskStatus> statuses) {
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/local/IsolatedSchedulerModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/local/IsolatedSchedulerModule.java b/src/main/java/com/twitter/aurora/scheduler/local/IsolatedSchedulerModule.java
deleted file mode 100644
index c07359e..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/local/IsolatedSchedulerModule.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.local;
-
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.inject.Provider;
-import javax.inject.Singleton;
-
-import com.google.common.base.Function;
-import com.google.common.base.Throwables;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.eventbus.Subscribe;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.inject.AbstractModule;
-
-import org.apache.mesos.Protos.Attribute;
-import org.apache.mesos.Protos.FrameworkID;
-import org.apache.mesos.Protos.Offer;
-import org.apache.mesos.Protos.OfferID;
-import org.apache.mesos.Protos.Resource;
-import org.apache.mesos.Protos.SlaveID;
-import org.apache.mesos.Protos.TaskID;
-import org.apache.mesos.Protos.TaskState;
-import org.apache.mesos.Protos.TaskStatus;
-import org.apache.mesos.Protos.Value.Scalar;
-import org.apache.mesos.Protos.Value.Text;
-import org.apache.mesos.Protos.Value.Type;
-import org.apache.mesos.Scheduler;
-import org.apache.mesos.SchedulerDriver;
-import org.apache.thrift.TException;
-
-import com.twitter.aurora.gen.AuroraAdmin;
-import com.twitter.aurora.gen.ExecutorConfig;
-import com.twitter.aurora.gen.Identity;
-import com.twitter.aurora.gen.JobConfiguration;
-import com.twitter.aurora.gen.Package;
-import com.twitter.aurora.gen.Quota;
-import com.twitter.aurora.gen.Response;
-import com.twitter.aurora.gen.SessionKey;
-import com.twitter.aurora.gen.TaskConfig;
-import com.twitter.aurora.scheduler.DriverFactory;
-import com.twitter.aurora.scheduler.base.JobKeys;
-import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
-import com.twitter.aurora.scheduler.configuration.Resources;
-import com.twitter.aurora.scheduler.events.PubsubEvent.DriverRegistered;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import com.twitter.aurora.scheduler.events.PubsubEventModule;
-import com.twitter.aurora.scheduler.local.FakeDriverFactory.FakeSchedulerDriver;
-import com.twitter.aurora.scheduler.log.testing.FileLogStreamModule;
-import com.twitter.common.application.ShutdownRegistry;
-import com.twitter.common.base.Command;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.Stats;
-import com.twitter.common.util.concurrent.ExecutorServiceShutdown;
-
-/**
- * A module that binds a fake mesos driver factory and a local (non-replicated) storage system.
- * <p>
- * The easiest way to run the scheduler in local/isolated mode is by executing:
- * <pre>
- * $ ./pants goal bundle aurora:scheduler-local && ./aurora/scripts/scheduler.sh -c local
- * </pre>
- */
-public class IsolatedSchedulerModule extends AbstractModule {
-
- private static final Logger LOG = Logger.getLogger(IsolatedSchedulerModule.class.getName());
-
- @Override
- protected void configure() {
- bind(DriverFactory.class).to(FakeDriverFactory.class);
- bind(FakeDriverFactory.class).in(Singleton.class);
- PubsubEventModule.bindSubscriber(binder(), FakeClusterRunner.class);
- install(new FileLogStreamModule());
- }
-
- static class FakeClusterRunner implements EventSubscriber {
- private final FrameworkID frameworkId =
- FrameworkID.newBuilder().setValue("framework-id").build();
- private final List<FakeSlave> cluster = ImmutableList.of(
- new FakeSlave(frameworkId, "fake-host1", "rack1", "slave-id1"),
- new FakeSlave(frameworkId, "fake-host2", "rack2", "slave-id2")
- );
-
- private final AtomicLong offerId = new AtomicLong();
- private final Function<FakeSlave, Offer> slaveToOffer = new Function<FakeSlave, Offer>() {
- @Override public Offer apply(FakeSlave slave) {
- return slave.makeOffer(offerId.incrementAndGet());
- }
- };
-
- private final Provider<Scheduler> scheduler;
- private final AuroraAdmin.Iface thrift;
- private final ScheduledExecutorService executor;
- private final SchedulerDriver driver;
-
- @Inject
- FakeClusterRunner(
- Provider<Scheduler> scheduler,
- AuroraAdmin.Iface thrift,
- ShutdownRegistry shutdownRegistry) {
-
- this.scheduler = scheduler;
- this.thrift = thrift;
- this.executor = createThreadPool(shutdownRegistry);
- this.driver = new FakeSchedulerDriver();
- }
-
- private static ScheduledExecutorService createThreadPool(ShutdownRegistry shutdownRegistry) {
- final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
- 1,
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TaskScheduler-%d").build()) {
-
- @Override protected void afterExecute(Runnable runnable, @Nullable Throwable throwable) {
- if (throwable != null) {
- LOG.log(Level.WARNING, "Error: " + throwable, throwable);
- } else if (runnable instanceof Future) {
- Future<?> future = (Future<?>) runnable;
- try {
- future.get();
- } catch (InterruptedException | ExecutionException e) {
- e.printStackTrace();
- }
- }
- }
- };
- Stats.exportSize("schedule_queue_size", executor.getQueue());
- shutdownRegistry.addAction(new Command() {
- @Override public void execute() {
- new ExecutorServiceShutdown(executor, Amount.of(1L, Time.SECONDS)).execute();
- }
- });
- return executor;
- }
-
- private void offerClusterResources() {
- executor.submit(new Runnable() {
- @Override public void run() {
- scheduler.get().resourceOffers(
- driver,
- FluentIterable.from(cluster).transform(slaveToOffer).toList());
- }
- });
- }
-
- private void setQuotas() {
- executor.submit(new Runnable() {
- @Override public void run() {
- try {
- thrift.setQuota(
- "mesos",
- new Quota(2.0 * 1000000, 100000000, 100000000),
- new SessionKey());
- } catch (TException e) {
- throw Throwables.propagate(e);
- }
- }
- });
- }
-
- @Subscribe
- public void registered(DriverRegistered event) {
- executor.submit(new Runnable() {
- @Override public void run() {
- Identity mesosUser = new Identity("mesos", "mesos");
- for (int i = 0; i < 20; i++) {
- JobConfiguration service = createJob("serviceJob" + i, mesosUser);
- service.getTaskConfig().setProduction((i % 2) == 0);
- service.getTaskConfig().setIsService(true);
- submitJob(service);
- }
-
- for (int i = 0; i < 20; i++) {
- JobConfiguration adhocJob = createJob("adhocJob" + i, mesosUser);
- adhocJob.getTaskConfig().setProduction((i % 2) == 0);
- adhocJob.getTaskConfig();
- submitJob(adhocJob);
- }
-
- for (int i = 0; i < 20; i++) {
- JobConfiguration cron = createJob("cronJob" + i, mesosUser);
- cron.getTaskConfig().setProduction((i % 2) == 0);
- cron.setCronSchedule("* * * * *");
- submitJob(cron);
- }
- }
- });
-
- setQuotas();
- offerClusterResources();
- // Send the offers again, since the first batch of offers will be consumed by GC executors.
- offerClusterResources();
- }
-
- private void moveTaskToState(final String taskId, final TaskState state, long delaySeconds) {
- Runnable changeState = new Runnable() {
- @Override public void run() {
- scheduler.get().statusUpdate(
- driver,
- TaskStatus.newBuilder()
- .setTaskId(TaskID.newBuilder().setValue(taskId))
- .setState(state)
- .build());
- }
- };
- executor.schedule(changeState, delaySeconds, TimeUnit.SECONDS);
- }
-
- @Subscribe
- public void stateChanged(TaskStateChange stateChange) {
- String taskId = stateChange.getTaskId();
- switch (stateChange.getNewState()) {
- case ASSIGNED:
- moveTaskToState(taskId, TaskState.TASK_STARTING, 1);
- break;
-
- case STARTING:
- moveTaskToState(taskId, TaskState.TASK_RUNNING, 1);
- break;
-
- case RUNNING:
- // Let the task finish some time randomly in the next 5 minutes.
- moveTaskToState(taskId, TaskState.TASK_FINISHED, (long) (Math.random() * 300));
- break;
-
- case FINISHED:
- offerClusterResources();
- break;
-
- default:
- break;
- }
- }
-
- private JobConfiguration createJob(String jobName, Identity owner) {
- return new JobConfiguration()
- .setKey(JobKeys.from(owner.getRole(), "test", jobName).newBuilder())
- .setOwner(owner)
- .setInstanceCount(5)
- .setTaskConfig(new TaskConfig()
- .setOwner(owner)
- .setJobName(jobName)
- .setEnvironment("test")
- .setNumCpus(1.0)
- .setDiskMb(1024)
- .setRamMb(1024)
- .setPackages(ImmutableSet.of(new Package(owner.getRole(), "package", 15)))
- .setExecutorConfig(new ExecutorConfig("aurora", "opaque")));
- }
-
- private void submitJob(JobConfiguration job) {
- Response response;
- try {
- response = thrift.createJob(job, null, new SessionKey());
- } catch (TException e) {
- throw Throwables.propagate(e);
- }
- LOG.info("Create job response: " + response);
- }
- }
-
- private static class FakeSlave {
- private final FrameworkID framework;
- private final String host;
- private final String rack;
- private final String slaveId;
-
- FakeSlave(FrameworkID framework, String host, String rack, String slaveId) {
- this.framework = framework;
- this.host = host;
- this.rack = rack;
- this.slaveId = slaveId;
- }
-
- private static Resource.Builder scalar(String name, double value) {
- return Resource.newBuilder()
- .setName(name)
- .setType(Type.SCALAR)
- .setScalar(Scalar.newBuilder().setValue(value));
- }
-
- private static Attribute.Builder attribute(String name, String value) {
- return Attribute.newBuilder()
- .setName(name)
- .setType(Type.TEXT)
- .setText(Text.newBuilder().setValue(value));
- }
-
- Offer makeOffer(long offerId) {
- return Offer.newBuilder()
- .setId(OfferID.newBuilder().setValue("offer" + offerId))
- .setFrameworkId(framework)
- .setSlaveId(SlaveID.newBuilder().setValue(slaveId))
- .setHostname(host)
- .addResources(scalar(Resources.CPUS, 16))
- .addResources(scalar(Resources.RAM_MB, 24576))
- .addResources(scalar(Resources.DISK_MB, 102400))
- .addAttributes(attribute(ConfigurationManager.RACK_CONSTRAINT, rack))
- .addAttributes(attribute(ConfigurationManager.HOST_CONSTRAINT, host))
- .build();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/log/Log.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/log/Log.java b/src/main/java/com/twitter/aurora/scheduler/log/Log.java
deleted file mode 100644
index c1036e0..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/log/Log.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.log;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Iterator;
-
-/**
- * Represents an append only log that can be read after and truncated before a known
- * {@link Position}.
- *
- * <p>Logs are accessed by {@link #open() opening} a {@link Stream}. All stream
- * access occurs with references to log entry {@link Position positions} in the stream.
- */
-public interface Log {
-
- /**
- * An opaque ordered handle to a log entry's position in the log stream.
- */
- interface Position extends Comparable<Position> {
- }
-
- /**
- * Represents a single entry in the log stream.
- */
- interface Entry {
-
- /**
- * Gets the contents of the log entry.
- *
- * @return the data stored in this log entry
- */
- byte[] contents();
- }
-
- /**
- * An interface to the live {@link Log} stream that allows for appending, reading and writing
- * entries.
- */
- interface Stream extends Closeable {
-
- /**
- * Indicates a {@link Position} that is not (currently) contained in this log stream.
- * This might indicate the position id from a different log or that the position was from this
- * log but has been truncated.
- */
- class InvalidPositionException extends RuntimeException {
- public InvalidPositionException(String message) {
- super(message);
- }
- public InvalidPositionException(String message, Throwable cause) {
- super(message, cause);
- }
- }
-
- /**
- * Indicates a {@link Stream} could not be read from, written to or truncated due to some
- * underlying IO error.
- */
- class StreamAccessException extends RuntimeException {
- public StreamAccessException(String message, Throwable cause) {
- super(message, cause);
- }
- }
-
- /**
- * Appends an {@link Entry} to the end of the log stream.
- *
- * @param contents the data to store in the appended entry
- * @return the posiiton of the appended entry
- * @throws StreamAccessException if contents could not be appended to the stream
- */
- Position append(byte[] contents) throws StreamAccessException;
-
- /**
- * Allows reading all entries from the log stream. Implementations may
- * materialize all entries at once or the may provide some form of streaming to back the
- * returned entry iterator. If the implementation does use some form of streaming or batching,
- * it may throw a
- * {@code StreamAccessException} on any call to {@link Iterator#hasNext()} or
- * {@link Iterator#next()}.
- *
- * @return an iterator that ranges from the entry from the given {@code position} to the last
- * entry in the log.
- * @throws InvalidPositionException if the specified position does not exist in this log
- * @throws StreamAccessException if the stream could not be read from
- */
- Iterator<Entry> readAll() throws InvalidPositionException, StreamAccessException;
-
- /**
- * Removes all log entries preceding the log entry at the given {@code position}.
- *
- * @param position the position of the latest entry to remove
- * @throws InvalidPositionException if the specified position does not exist in this log
- * @throws StreamAccessException if the stream could not be truncated
- */
- void truncateBefore(Position position) throws InvalidPositionException, StreamAccessException;
- }
-
- /**
- * Opens the log stream for reading writing and truncation. Clients should ensure the stream is
- * closed when they are done using it.
- *
- * @return the log stream
- * @throws IOException if there was a problem opening the log stream
- */
- Stream open() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/log/mesos/LogInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/log/mesos/LogInterface.java b/src/main/java/com/twitter/aurora/scheduler/log/mesos/LogInterface.java
deleted file mode 100644
index 9ae4685..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/log/mesos/LogInterface.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package com.twitter.aurora.scheduler.log.mesos;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.mesos.Log;
-import org.apache.mesos.Log.Entry;
-import org.apache.mesos.Log.Position;
-
-/**
- * An interface for {@link Log}, since the mesos Java API doesn't provide one.
- * <p>
- * This is needed because a static initializer that loads a native library, prevents us from
- * mocking the Mesos Log API in tests. These wrapper interfaces and their corresponding
- * implementations that delegate the calls to the underlying Mesos Log objects will help
- * us mock the Log API.
- * <p>
- * TODO(Suman Karumuri): Remove this interface after https://issues.apache.org/jira/browse/MESOS-796
- * is resolved.
- */
-interface LogInterface {
-
- Position position(byte[] identity);
-
- interface ReaderInterface {
- List<Entry> read(Position from, Position to, long timeout, TimeUnit unit)
- throws TimeoutException, Log.OperationFailedException;
-
- Position beginning();
-
- Position ending();
- }
-
- interface WriterInterface {
- Position append(byte[] data, long timeout, TimeUnit unit)
- throws TimeoutException, Log.WriterFailedException;
-
- Position truncate(Position to, long timeout, TimeUnit unit)
- throws TimeoutException, Log.WriterFailedException;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLog.java b/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLog.java
deleted file mode 100644
index d625734..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLog.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.log.mesos;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.Target;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.inject.Inject;
-import javax.inject.Provider;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.UnmodifiableIterator;
-import com.google.common.primitives.Longs;
-import com.google.inject.BindingAnnotation;
-
-import org.apache.mesos.Log;
-
-import com.twitter.aurora.scheduler.log.mesos.LogInterface.ReaderInterface;
-import com.twitter.aurora.scheduler.log.mesos.LogInterface.WriterInterface;
-import com.twitter.common.base.Function;
-import com.twitter.common.base.MorePreconditions;
-import com.twitter.common.inject.TimedInterceptor.Timed;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.stats.SlidingStats;
-import com.twitter.common.stats.Stats;
-
-import static java.lang.annotation.ElementType.METHOD;
-import static java.lang.annotation.ElementType.PARAMETER;
-import static java.lang.annotation.RetentionPolicy.RUNTIME;
-
-/**
- * A {@code Log} implementation backed by a true distributed log in mesos core.
- */
-public class MesosLog implements com.twitter.aurora.scheduler.log.Log {
-
- private static final Logger LOG = Logger.getLogger(MesosLog.class.getName());
-
- /**
- * Binding annotation for the opaque value of a log noop entry.
- */
- @BindingAnnotation
- @Retention(RUNTIME)
- @Target({ PARAMETER, METHOD })
- public @interface NoopEntry { }
-
- /**
- * Binding annotation for log read timeouts.
- */
- @BindingAnnotation
- @Retention(RUNTIME)
- @Target({ PARAMETER, METHOD })
- public @interface ReadTimeout { }
-
- /**
- * Binding annotation for log write timeouts - used for truncates and appends.
- */
- @BindingAnnotation
- @Retention(RUNTIME)
- @Target({ PARAMETER, METHOD })
- public @interface WriteTimeout { }
-
- private final Provider<LogInterface> logFactory;
-
- private final Provider<ReaderInterface> readerFactory;
- private final Amount<Long, Time> readTimeout;
-
- private final Provider<WriterInterface> writerFactory;
- private final Amount<Long, Time> writeTimeout;
-
- private final byte[] noopEntry;
-
- /**
- * Creates a new mesos log.
- *
- * @param logFactory Factory to provide access to log.
- * @param readerFactory Factory to provide access to log readers.
- * @param readTimeout Log read timeout.
- * @param writerFactory Factory to provide access to log writers.
- * @param writeTimeout Log write timeout.
- * @param noopEntry A no-op log entry blob.
- */
- @Inject
- public MesosLog(
- Provider<LogInterface> logFactory,
- Provider<ReaderInterface> readerFactory,
- @ReadTimeout Amount<Long, Time> readTimeout,
- Provider<WriterInterface> writerFactory,
- @WriteTimeout Amount<Long, Time> writeTimeout,
- @NoopEntry byte[] noopEntry) {
-
- this.logFactory = Preconditions.checkNotNull(logFactory);
-
- this.readerFactory = Preconditions.checkNotNull(readerFactory);
- this.readTimeout = readTimeout;
-
- this.writerFactory = Preconditions.checkNotNull(writerFactory);
- this.writeTimeout = writeTimeout;
-
- this.noopEntry = Preconditions.checkNotNull(noopEntry);
- }
-
- @Override
- public Stream open() {
- return new LogStream(
- logFactory.get(), readerFactory.get(), readTimeout, writerFactory, writeTimeout, noopEntry);
- }
-
- @VisibleForTesting
- static class LogStream implements com.twitter.aurora.scheduler.log.Log.Stream {
- @VisibleForTesting
- static final class OpStats {
- private final String opName;
- private final SlidingStats timing;
- private final AtomicLong timeouts;
- private final AtomicLong failures;
-
- OpStats(String opName) {
- this.opName = MorePreconditions.checkNotBlank(opName);
- timing = new SlidingStats("scheduler_log_native_" + opName, "nanos");
- timeouts = exportLongStat("scheduler_log_native_%s_timeouts", opName);
- failures = exportLongStat("scheduler_log_native_%s_failures", opName);
- }
-
- private static AtomicLong exportLongStat(String template, Object... args) {
- return Stats.exportLong(String.format(template, args));
- }
- }
-
- private static final Function<Log.Entry, LogEntry> MESOS_ENTRY_TO_ENTRY =
- new Function<Log.Entry, LogEntry>() {
- @Override public LogEntry apply(Log.Entry entry) {
- return new LogEntry(entry);
- }
- };
-
- private final OpStats read = new OpStats("read");
- private final OpStats append = new OpStats("append");
- private final OpStats truncate = new OpStats("truncate");
- private final AtomicLong entriesSkipped =
- Stats.exportLong("scheduler_log_native_native_entries_skipped");
-
- private final LogInterface log;
-
- private final ReaderInterface reader;
- private final long readTimeout;
- private final TimeUnit readTimeUnit;
-
- private final Provider<WriterInterface> writerFactory;
- private final long writeTimeout;
- private final TimeUnit writeTimeUnit;
-
- private final byte[] noopEntry;
-
- private WriterInterface writer;
-
- LogStream(LogInterface log, ReaderInterface reader, Amount<Long, Time> readTimeout,
- Provider<WriterInterface> writerFactory, Amount<Long, Time> writeTimeout,
- byte[] noopEntry) {
-
- this.log = log;
-
- this.reader = reader;
- this.readTimeout = readTimeout.getValue();
- this.readTimeUnit = readTimeout.getUnit().getTimeUnit();
-
- this.writerFactory = writerFactory;
- this.writeTimeout = writeTimeout.getValue();
- this.writeTimeUnit = writeTimeout.getUnit().getTimeUnit();
-
- this.noopEntry = noopEntry;
- }
-
- @Override
- public Iterator<Entry> readAll() throws StreamAccessException {
- // TODO(John Sirois): Currently we must be the coordinator to ensure we get the 'full read'
- // of log entries expected by the users of the com.twitter.aurora.scheduler.log.Log interface.
- // Switch to another method of ensuring this when it becomes available in mesos' log
- // interface.
- try {
- append(noopEntry);
- } catch (StreamAccessException e) {
- throw new StreamAccessException("Error writing noop prior to a read", e);
- }
-
- final Log.Position from = reader.beginning();
- final Log.Position to = end().unwrap();
-
- // Reading all the entries at once may cause large garbage collections. Instead, we
- // lazily read the entries one by one as they are requested.
- // TODO(Benjamin Hindman): Eventually replace this functionality with functionality
- // from the Mesos Log.
- return new UnmodifiableIterator<Entry>() {
- private long position = Longs.fromByteArray(from.identity());
- private final long endPosition = Longs.fromByteArray(to.identity());
- private Entry entry = null;
-
- @Override
- public boolean hasNext() {
- if (entry != null) {
- return true;
- }
-
- while (position <= endPosition) {
- long start = System.nanoTime();
- try {
- Log.Position p = log.position(Longs.toByteArray(position));
- if (LOG.isLoggable(Level.FINE)) {
- LOG.fine("Reading position " + position + " from the log");
- }
- List<Log.Entry> entries = reader.read(p, p, readTimeout, readTimeUnit);
-
- // N.B. HACK! There is currently no way to "increment" a position. Until the Mesos
- // Log actually provides a way to "stream" the log, we approximate as much by
- // using longs via Log.Position.identity and Log.position.
- position++;
-
- // Reading positions in this way means it's possible that we get an "invalid" entry
- // (e.g., in the underlying log terminology this would be anything but an append)
- // which will be removed from the returned entries resulting in an empty list.
- // We skip these.
- if (entries.isEmpty()) {
- entriesSkipped.getAndIncrement();
- } else {
- entry = MESOS_ENTRY_TO_ENTRY.apply(Iterables.getOnlyElement(entries));
- return true;
- }
- } catch (TimeoutException e) {
- read.timeouts.getAndIncrement();
- throw new StreamAccessException("Timeout reading from log.", e);
- } catch (Log.OperationFailedException e) {
- read.failures.getAndIncrement();
- throw new StreamAccessException("Problem reading from log", e);
- } finally {
- read.timing.accumulate(System.nanoTime() - start);
- }
- }
- return false;
- }
-
- @Override
- public Entry next() {
- if (entry == null && !hasNext()) {
- throw new NoSuchElementException();
- }
-
- Entry result = Preconditions.checkNotNull(entry);
- entry = null;
- return result;
- }
- };
- }
-
- @Override
- public LogPosition append(final byte[] contents) throws StreamAccessException {
- Preconditions.checkNotNull(contents);
-
- Log.Position position = mutate(append, new Mutation<Log.Position>() {
- @Override public Log.Position apply(WriterInterface logWriter)
- throws TimeoutException, Log.WriterFailedException {
- return logWriter.append(contents, writeTimeout, writeTimeUnit);
- }
- });
- return LogPosition.wrap(position);
- }
-
- @Timed("scheduler_log_native_truncate_before")
- @Override
- public void truncateBefore(com.twitter.aurora.scheduler.log.Log.Position position)
- throws StreamAccessException {
-
- Preconditions.checkArgument(position instanceof LogPosition);
-
- final Log.Position before = ((LogPosition) position).unwrap();
- mutate(truncate, new Mutation<Void>() {
- @Override public Void apply(WriterInterface logWriter)
- throws TimeoutException, Log.WriterFailedException {
- logWriter.truncate(before, writeTimeout, writeTimeUnit);
- return null;
- }
- });
- }
-
- @VisibleForTesting
- interface Mutation<T> {
- T apply(WriterInterface writer) throws TimeoutException, Log.WriterFailedException;
- }
-
- @VisibleForTesting
- synchronized <T> T mutate(OpStats stats, Mutation<T> mutation) {
- long start = System.nanoTime();
- if (writer == null) {
- writer = writerFactory.get();
- }
- try {
- return mutation.apply(writer);
- } catch (TimeoutException e) {
- stats.timeouts.getAndIncrement();
- throw new StreamAccessException("Timeout performing log " + stats.opName, e);
- } catch (Log.WriterFailedException e) {
- stats.failures.getAndIncrement();
-
- // We must throw away a writer on any write failure - this could be because of a coordinator
- // election in which case we must trigger a new election.
- writer = null;
-
- throw new StreamAccessException("Problem performing log" + stats.opName, e);
- } finally {
- stats.timing.accumulate(System.nanoTime() - start);
- }
- }
-
- private LogPosition end() {
- return LogPosition.wrap(reader.ending());
- }
-
- @Override
- public void close() {
- // noop
- }
-
- private static class LogPosition implements com.twitter.aurora.scheduler.log.Log.Position {
- private final Log.Position underlying;
-
- LogPosition(Log.Position underlying) {
- this.underlying = underlying;
- }
-
- static LogPosition wrap(Log.Position position) {
- return new LogPosition(position);
- }
-
- Log.Position unwrap() {
- return underlying;
- }
-
- @Override public int compareTo(Position o) {
- Preconditions.checkArgument(o instanceof LogPosition);
- return underlying.compareTo(((LogPosition) o).underlying);
- }
- }
-
- private static class LogEntry implements com.twitter.aurora.scheduler.log.Log.Entry {
- private final Log.Entry underlying;
-
- public LogEntry(Log.Entry entry) {
- this.underlying = entry;
- }
-
- @Override
- public byte[] contents() {
- return underlying.data;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLogStreamModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLogStreamModule.java b/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLogStreamModule.java
deleted file mode 100644
index 7063cb7..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/log/mesos/MesosLogStreamModule.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.log.mesos;
-
-import java.io.File;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import javax.inject.Singleton;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.inject.PrivateModule;
-import com.google.inject.Provides;
-import com.google.inject.TypeLiteral;
-
-import org.apache.mesos.Log;
-import org.apache.zookeeper.common.PathUtils;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.gen.storage.LogEntry;
-import com.twitter.aurora.scheduler.log.mesos.LogInterface.ReaderInterface;
-import com.twitter.aurora.scheduler.log.mesos.LogInterface.WriterInterface;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-import com.twitter.common.net.InetSocketAddressHelper;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-import com.twitter.common.zookeeper.guice.client.ZooKeeperClientModule.ClientConfig;
-
-/**
- * Binds a native mesos Log implementation.
- *
- * <p>Exports the following bindings:
- * <ul>
- * <li>{@link Log} - a log backed by the mesos native distributed log</li>
- * </ul>
- */
-public class MesosLogStreamModule extends PrivateModule {
- @CmdLine(name = "native_log_quorum_size",
- help = "The size of the quorum required for all log mutations.")
- private static final Arg<Integer> QUORUM_SIZE = Arg.create(1);
-
- @CmdLine(name = "native_log_file_path",
- help = "Path to a file to store the native log data in. If the parent directory does"
- + "not exist it will be created.")
- private static final Arg<File> LOG_PATH = Arg.create(null);
-
- @CmdLine(name = "native_log_zk_group_path",
- help = "A zookeeper node for use by the native log to track the master coordinator.")
- private static final Arg<String> ZK_LOG_GROUP_PATH = Arg.create(null);
-
- /*
- * This timeout includes the time to get a quorum to promise leadership to the coordinator and
- * the time to fill any holes in the coordinator's log.
- */
- @CmdLine(name = "native_log_election_timeout",
- help = "The timeout for a single attempt to obtain a new log writer.")
- private static final Arg<Amount<Long, Time>> COORDINATOR_ELECTION_TIMEOUT =
- Arg.create(Amount.of(15L, Time.SECONDS));
-
- /*
- * Normally retries would not be expected to help much - however in the small replica set where
- * a few down replicas doom a coordinator election attempt, retrying effectively gives us a wider
- * window in which to await a live quorum before giving up and thrashing the global election
- * process. Observed log replica recovery times as of 4/6/2012 can be ~45 seconds so giving a
- * window >= 2x this should support 1-round election events (that possibly use several retries in
- * the single round).
- */
- @CmdLine(name = "native_log_election_retries",
- help = "The maximum number of attempts to obtain a new log writer.")
- private static final Arg<Integer> COORDINATOR_ELECTION_RETRIES = Arg.create(20);
-
- @CmdLine(name = "native_log_read_timeout",
- help = "The timeout for doing log reads.")
- private static final Arg<Amount<Long, Time>> READ_TIMEOUT =
- Arg.create(Amount.of(5L, Time.SECONDS));
-
- @CmdLine(name = "native_log_write_timeout",
- help = "The timeout for doing log appends and truncations.")
- private static final Arg<Amount<Long, Time>> WRITE_TIMEOUT =
- Arg.create(Amount.of(3L, Time.SECONDS));
-
- private final ClientConfig zkClientConfig;
-
- public MesosLogStreamModule(ClientConfig zkClientConfig) {
- this.zkClientConfig = Preconditions.checkNotNull(zkClientConfig);
- }
-
- @Override
- protected void configure() {
- bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.ReadTimeout.class)
- .toInstance(READ_TIMEOUT.get());
- bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.WriteTimeout.class)
- .toInstance(WRITE_TIMEOUT.get());
-
- bind(com.twitter.aurora.scheduler.log.Log.class).to(MesosLog.class);
- bind(MesosLog.class).in(Singleton.class);
- expose(com.twitter.aurora.scheduler.log.Log.class);
- }
-
- @Provides
- @Singleton
- Log provideLog() {
- File logPath = LOG_PATH.get();
- File parentDir = logPath.getParentFile();
- if (!parentDir.exists() && !parentDir.mkdirs()) {
- addError("Failed to create parent directory to store native log at: %s", parentDir);
- }
-
- String zkConnectString = Joiner.on(',').join(
- Iterables.transform(zkClientConfig.servers, InetSocketAddressHelper.INET_TO_STR));
-
- PathUtils.validatePath(ZK_LOG_GROUP_PATH.get());
- return new Log(
- QUORUM_SIZE.get(),
- logPath.getAbsolutePath(),
- zkConnectString,
- zkClientConfig.sessionTimeout.getValue(),
- zkClientConfig.sessionTimeout.getUnit().getTimeUnit(),
- ZK_LOG_GROUP_PATH.get(),
- zkClientConfig.credentials.scheme(),
- zkClientConfig.credentials.authToken());
- }
-
- @Provides
- Log.Reader provideReader(Log log) {
- return new Log.Reader(log);
- }
-
- @Provides
- Log.Writer provideWriter(Log log) {
- Amount<Long, Time> electionTimeout = COORDINATOR_ELECTION_TIMEOUT.get();
- return new Log.Writer(log, electionTimeout.getValue(), electionTimeout.getUnit().getTimeUnit(),
- COORDINATOR_ELECTION_RETRIES.get());
- }
-
- @Provides
- LogInterface provideLogInterface(final Log log) {
- return new LogInterface() {
- @Override public Log.Position position(byte[] identity) {
- return log.position(identity);
- }
- };
- }
-
- @Provides
- ReaderInterface provideReaderInterface(final Log.Reader reader) {
- return new ReaderInterface() {
- @Override
- public List<Log.Entry> read(Log.Position from, Log.Position to, long timeout, TimeUnit unit)
- throws TimeoutException, Log.OperationFailedException {
-
- return reader.read(from, to, timeout, unit);
- }
-
- @Override public Log.Position beginning() {
- return reader.beginning();
- }
-
- @Override public Log.Position ending() {
- return reader.ending();
- }
- };
- }
-
- @Provides
- WriterInterface provideWriterInterface(final Log.Writer writer) {
- return new WriterInterface() {
- @Override public Log.Position append(byte[] data, long timeout, TimeUnit unit)
- throws TimeoutException, Log.WriterFailedException {
- return writer.append(data, timeout, unit);
- }
-
- @Override public Log.Position truncate(Log.Position to, long timeout, TimeUnit unit)
- throws TimeoutException, Log.WriterFailedException {
- return writer.truncate(to, timeout, unit);
- }
- };
- }
-
- @Provides
- @Singleton
- @MesosLog.NoopEntry
- byte[] provideNoopEntry() throws ThriftBinaryCodec.CodingException {
- return ThriftBinaryCodec.encodeNonNull(LogEntry.noop(true));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLog.java b/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLog.java
deleted file mode 100644
index 3dff4cf..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLog.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.log.testing;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Iterator;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Function;
-import com.google.common.base.Functions;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Throwables;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Ordering;
-import com.google.common.io.Files;
-import com.google.common.primitives.Longs;
-
-import com.twitter.aurora.codec.ThriftBinaryCodec;
-import com.twitter.aurora.codec.ThriftBinaryCodec.CodingException;
-import com.twitter.aurora.gen.test.FileLogContents;
-import com.twitter.aurora.gen.test.LogRecord;
-import com.twitter.aurora.scheduler.log.Log;
-import com.twitter.common.base.Closure;
-
-/**
- * A log implementation that reads from and writes to a local file.
- * <p>
- * This should never be used in a production setting, it is only intended for local testing.
- * TODO(wfarner): Bind/inject a settable flag that indicates we are running with test settings.
- * Surface this in a banner on the web UI.
- */
-class FileLog implements Log {
-
- private final File logFile;
-
- @Inject
- FileLog(File logFile) {
- this.logFile = Preconditions.checkNotNull(logFile);
- }
-
- @Override
- public Stream open() throws IOException {
- try {
- FileLogContents logContents;
- // Treat an empty file as a new file.
- // NOTE: we can't use logFile.length == 0 to test for an empty file, since empty unicode files
- // contain a byte order mark (BOM) header that is 255 bytes in length.
- if (logFile.createNewFile()
- || Files.readFirstLine(logFile, Charset.defaultCharset()) == null) {
- logContents = new FileLogContents(Maps.<Long, LogRecord>newHashMap());
- } else {
- logContents = ThriftBinaryCodec.decode(FileLogContents.class, Files.toByteArray(logFile));
- }
- Closure<FileLogContents> logWriter = new Closure<FileLogContents>() {
- @Override public void execute(FileLogContents logContents) {
- try {
- Files.write(ThriftBinaryCodec.encode(logContents), logFile);
- } catch (IOException | CodingException e) {
- throw Throwables.propagate(e);
- }
- }
- };
- return new FileStream(logContents, logWriter);
- } catch (CodingException e) {
- throw new IOException("Failed to interpret log contents: " + e, e);
- }
- }
-
- private static class FileStream implements Stream {
- private final FileLogContents logContents;
- private final Closure<FileLogContents> logWriter;
- private long nextPosition;
-
- FileStream(FileLogContents logContents, Closure<FileLogContents> logWriter) {
- this.logContents = logContents;
- this.logWriter = logWriter;
- nextPosition = logContents.getRecords().isEmpty()
- ? 1
- : Ordering.natural().max(logContents.getRecords().keySet()) + 1;
- }
-
- @Override
- public Position append(byte[] contents) throws StreamAccessException {
- logContents.getRecords().put(nextPosition, new LogRecord(ByteBuffer.wrap(contents)));
- Position position = new CounterPosition(nextPosition);
- logWriter.execute(logContents);
- nextPosition++;
- return position;
- }
-
- private static final Function<LogRecord, Entry> TO_ENTRY = new Function<LogRecord, Entry>() {
- @Override public Entry apply(final LogRecord record) {
- return new Entry() {
- @Override public byte[] contents() {
- return record.getContents();
- }
- };
- }
- };
-
- @Override
- public Iterator<Entry> readAll() throws InvalidPositionException, StreamAccessException {
- return FluentIterable.from(Ordering.natural().sortedCopy(logContents.getRecords().keySet()))
- .transform(Functions.forMap(logContents.getRecords()))
- .transform(TO_ENTRY)
- .iterator();
- }
-
- @Override
- public void truncateBefore(Position position)
- throws InvalidPositionException, StreamAccessException {
-
- if (!(position instanceof CounterPosition)) {
- throw new InvalidPositionException("Unrecognized position " + position);
- }
-
- final long truncateBefore = ((CounterPosition) position).value;
- Iterables.removeIf(logContents.getRecords().keySet(), new Predicate<Long>() {
- @Override public boolean apply(Long recordPosition) {
- return recordPosition < truncateBefore;
- }
- });
- logWriter.execute(logContents);
- }
-
- @Override
- public void close() throws IOException {
- // No-op.
- }
-
- private static class CounterPosition implements Position {
- private final long value;
-
- CounterPosition(long value) {
- this.value = value;
- }
-
- @Override
- public int compareTo(Position position) {
- return Longs.compare(value, ((CounterPosition) position).value);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLogStreamModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLogStreamModule.java b/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLogStreamModule.java
deleted file mode 100644
index 5ef3e8e..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/log/testing/FileLogStreamModule.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.log.testing;
-
-import java.io.File;
-
-import javax.inject.Singleton;
-
-import com.google.common.base.Preconditions;
-import com.google.inject.PrivateModule;
-
-import com.twitter.aurora.scheduler.log.Log;
-import com.twitter.common.args.Arg;
-import com.twitter.common.args.CmdLine;
-
-/**
- * Binding module that uses a local log file, intended for testing.
- */
-public class FileLogStreamModule extends PrivateModule {
-
- // TODO(William Farner): Make this a required argument and ensure it is not included in production
- // builds (MESOS-471).
- //@NotNull
- @CmdLine(name = "testing_log_file_path", help = "Path to a file to store local log file data in.")
- private static final Arg<File> LOG_PATH = Arg.create(null);
-
- @Override
- protected void configure() {
- Preconditions.checkNotNull(LOG_PATH.get());
- bind(File.class).toInstance(LOG_PATH.get());
- bind(Log.class).to(FileLog.class);
- bind(FileLog.class).in(Singleton.class);
- expose(Log.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/metadata/MetadataModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/metadata/MetadataModule.java b/src/main/java/com/twitter/aurora/scheduler/metadata/MetadataModule.java
deleted file mode 100644
index 08377ae..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/metadata/MetadataModule.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.metadata;
-
-import javax.inject.Singleton;
-
-import com.google.inject.AbstractModule;
-
-import com.twitter.aurora.scheduler.events.PubsubEventModule;
-
-/**
- * Binding module for scheduler metadata management.
- */
-public class MetadataModule extends AbstractModule {
-
- @Override
- protected void configure() {
- bind(NearestFit.class).in(Singleton.class);
- PubsubEventModule.bindSubscriber(binder(), NearestFit.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/com/twitter/aurora/scheduler/metadata/NearestFit.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/metadata/NearestFit.java b/src/main/java/com/twitter/aurora/scheduler/metadata/NearestFit.java
deleted file mode 100644
index 47c1f5c..0000000
--- a/src/main/java/com/twitter/aurora/scheduler/metadata/NearestFit.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Copyright 2013 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.twitter.aurora.scheduler.metadata;
-
-import java.util.Set;
-
-import javax.inject.Inject;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Ticker;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.eventbus.Subscribe;
-
-import com.twitter.aurora.gen.ScheduleStatus;
-import com.twitter.aurora.scheduler.base.Tasks;
-import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
-import com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
-import com.twitter.aurora.scheduler.events.PubsubEvent.Vetoed;
-import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
-import com.twitter.common.quantity.Amount;
-import com.twitter.common.quantity.Time;
-
-/**
- * Tracks vetoes against scheduling decisions and maintains the closest fit among all the vetoes
- * for a task.
- */
-public class NearestFit implements EventSubscriber {
- @VisibleForTesting
- static final Amount<Long, Time> EXPIRATION = Amount.of(10L, Time.MINUTES);
-
- @VisibleForTesting
- static final ImmutableSet<Veto> NO_VETO = ImmutableSet.of();
-
- private final LoadingCache<String, Fit> fitByTask;
-
- @VisibleForTesting
- NearestFit(Ticker ticker) {
- fitByTask = CacheBuilder.newBuilder()
- .expireAfterWrite(EXPIRATION.getValue(), EXPIRATION.getUnit().getTimeUnit())
- .ticker(ticker)
- .build(new CacheLoader<String, Fit>() {
- @Override public Fit load(String taskId) {
- return new Fit();
- }
- });
- }
-
- @Inject
- NearestFit() {
- this(Ticker.systemTicker());
- }
-
- /**
- * Gets the vetoes that represent the nearest fit for the given task.
- *
- * @param taskId The task to look up.
- * @return The nearest fit vetoes for the given task. This will return an empty set if
- * no vetoes have been recorded for the task.
- */
- public synchronized ImmutableSet<Veto> getNearestFit(String taskId) {
- Fit fit = fitByTask.getIfPresent(taskId);
- return (fit == null) ? NO_VETO : fit.vetoes;
- }
-
- /**
- * Records a task deletion event.
- *
- * @param deletedEvent Task deleted event.
- */
- @Subscribe
- public synchronized void remove(TasksDeleted deletedEvent) {
- fitByTask.invalidateAll(Tasks.ids(deletedEvent.getTasks()));
- }
-
- /**
- * Records a task state change event.
- * This will ignore any events where the previous state is not {@link ScheduleStatus#PENDING}.
- *
- * @param stateChangeEvent Task state change.
- */
- @Subscribe
- public synchronized void stateChanged(TaskStateChange stateChangeEvent) {
- if (stateChangeEvent.getOldState() == ScheduleStatus.PENDING) {
- fitByTask.invalidate(stateChangeEvent.getTaskId());
- }
- }
-
- private static final Predicate<Veto> IS_CONSTRAINT_MISMATCH = new Predicate<Veto>() {
- @Override public boolean apply(Veto veto) {
- return veto.isConstraintMismatch();
- }
- };
-
- /**
- * Records a task veto event.
- * This will ignore any veto events where any veto returns {@code true} from
- * {@link Veto#isConstraintMismatch()}.
- *
- * @param vetoEvent Veto event.
- */
- @Subscribe
- public synchronized void vetoed(Vetoed vetoEvent) {
- Preconditions.checkNotNull(vetoEvent);
- fitByTask.getUnchecked(vetoEvent.getTaskId()).maybeUpdate(vetoEvent.getVetoes());
- }
-
- private static class Fit {
- private ImmutableSet<Veto> vetoes;
-
- private static int score(Iterable<Veto> vetoes) {
- int total = 0;
- for (Veto veto : vetoes) {
- total += veto.getScore();
- }
- return total;
- }
-
- private void update(Iterable<Veto> newVetoes) {
- vetoes = ImmutableSet.copyOf(newVetoes);
- }
-
- /**
- * Updates the nearest fit if the provided vetoes represents a closer fit than the current
- * best fit.
- * <p>
- * There are two classes of vetoes: those with and without constraint mismatches. A set of
- * vetoes without a constraint mismatch is always a better fit than a set with constraint
- * mismatches.
- * <p>
- * If two sets are equivalent in that they do or do not have constraint mismatches, they are
- * compared by the following criteria:
- * <ul>
- * <li> the one with fewer vetoes is a better fit, irrespective of scores
- * <li> if the veto count is equal, the one with the smaller aggregate score is a better fit
- * </ul>
- *
- * @param newVetoes The vetoes for the scheduling assignment with {@code newHost}.
- */
- void maybeUpdate(Set<Veto> newVetoes) {
- if (vetoes == null) {
- update(newVetoes);
- return;
- }
-
- boolean valueMismatchOld = Iterables.any(vetoes, IS_CONSTRAINT_MISMATCH);
- boolean valueMismatchNew = Iterables.any(newVetoes, IS_CONSTRAINT_MISMATCH);
- if (valueMismatchOld == valueMismatchNew) {
- if (newVetoes.size() < vetoes.size()) {
- update(newVetoes);
- } else if ((newVetoes.size() == vetoes.size()) && (score(newVetoes) < score(vetoes))) {
- update(newVetoes);
- }
- } else if (valueMismatchOld) {
- update(newVetoes);
- }
- }
- }
-}
[37/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/OfferQueue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/OfferQueue.java b/src/main/java/org/apache/aurora/scheduler/async/OfferQueue.java
new file mode 100644
index 0000000..95334ff
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/OfferQueue.java
@@ -0,0 +1,341 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+import com.google.common.eventbus.Subscribe;
+
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.OfferID;
+import org.apache.mesos.Protos.TaskInfo;
+
+import com.twitter.aurora.gen.HostStatus;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.scheduler.Driver;
+import com.twitter.aurora.scheduler.events.PubsubEvent.DriverDisconnected;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.HostMaintenanceStateChange;
+import com.twitter.aurora.scheduler.state.MaintenanceController;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
+import static com.twitter.aurora.gen.MaintenanceMode.NONE;
+import static com.twitter.aurora.gen.MaintenanceMode.SCHEDULED;
+
+/**
+ * Tracks the Offers currently known by the scheduler
+ */
+public interface OfferQueue extends EventSubscriber {
+
+ /**
+ * Notifies the scheduler of a new resource offer.
+ *
+ * @param offer Newly-available resource offer.
+ */
+ void addOffer(Offer offer);
+
+ /**
+ * Invalidates an offer. This indicates that the scheduler should not attempt to match any
+ * tasks against the offer.
+ *
+ * @param offer Canceled offer.
+ */
+ void cancelOffer(OfferID offer);
+
+ /**
+ * Launches the first task that satisfies the {@code acceptor} by returning a {@link TaskInfo}.
+ *
+ * @param acceptor Function that determines if an offer is accepted.
+ * @return {@code true} if the task was launched, {@code false} if no offers satisfied the
+ * {@code acceptor}.
+ * @throws LaunchException If the acceptor accepted an offer, but there was an error launching the
+ * task.
+ */
+ boolean launchFirst(Function<Offer, Optional<TaskInfo>> acceptor) throws LaunchException;
+
+ /**
+ * Notifies the offer queue that a host has changed state.
+ *
+ * @param change State change notification.
+ */
+ void hostChangedState(HostMaintenanceStateChange change);
+
+ /**
+ * Gets the offers that the scheduler is holding.
+ *
+ * @return A snapshot of the offers that the scheduler is currently holding.
+ */
+ Iterable<Offer> getOffers();
+
+ /**
+ * Calculates the amount of time before an offer should be 'returned' by declining it.
+ * The delay is calculated for each offer that is received, so the return delay may be
+ * fixed or variable.
+ */
+ public interface OfferReturnDelay extends Supplier<Amount<Integer, Time>> {
+ }
+
+ /**
+ * Thrown when there was an unexpected failure trying to launch a task.
+ */
+ static class LaunchException extends Exception {
+ LaunchException(String msg) {
+ super(msg);
+ }
+
+ LaunchException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+ }
+
+ class OfferQueueImpl implements OfferQueue {
+ private static final Logger LOG = Logger.getLogger(OfferQueueImpl.class.getName());
+
+ static final Comparator<HostOffer> PREFERENCE_COMPARATOR =
+ // Currently, the only preference is based on host maintenance status.
+ Ordering.explicit(NONE, SCHEDULED, DRAINING, DRAINED)
+ .onResultOf(new Function<HostOffer, MaintenanceMode>() {
+ @Override public MaintenanceMode apply(HostOffer offer) {
+ return offer.mode;
+ }
+ })
+ .compound(Ordering.arbitrary());
+
+ private final Set<HostOffer> hostOffers = new ConcurrentSkipListSet<>(PREFERENCE_COMPARATOR);
+ private final AtomicLong offerRaces = Stats.exportLong("offer_accept_races");
+
+ private final Driver driver;
+ private final OfferReturnDelay returnDelay;
+ private final ScheduledExecutorService executor;
+ private final MaintenanceController maintenance;
+
+ @Inject
+ OfferQueueImpl(Driver driver,
+ OfferReturnDelay returnDelay,
+ ScheduledExecutorService executor,
+ MaintenanceController maintenance) {
+
+ this.driver = driver;
+ this.returnDelay = returnDelay;
+ this.executor = executor;
+ this.maintenance = maintenance;
+ // Potential gotcha - since this is now a ConcurrentSkipListSet, size() is more expensive.
+ // Could track this separately if it turns out to pose problems.
+ Stats.exportSize("outstanding_offers", hostOffers);
+ }
+
+ @Override
+ public void addOffer(final Offer offer) {
+ // We run a slight risk of a race here, which is acceptable. The worst case is that we
+ // temporarily hold two offers for the same host, which should be corrected when we return
+ // them after the return delay.
+ // There's also a chance that we return an offer for compaction ~simultaneously with the
+ // same-host offer being canceled/returned. This is also fine.
+ List<HostOffer> sameSlave = FluentIterable.from(hostOffers)
+ .filter(new Predicate<HostOffer>() {
+ @Override public boolean apply(HostOffer hostOffer) {
+ return hostOffer.offer.getSlaveId().equals(offer.getSlaveId());
+ }
+ })
+ .toList();
+ if (sameSlave.isEmpty()) {
+ hostOffers.add(new HostOffer(offer, maintenance.getMode(offer.getHostname())));
+ executor.schedule(
+ new Runnable() {
+ @Override public void run() {
+ removeAndDecline(offer.getId());
+ }
+ },
+ returnDelay.get().as(Time.MILLISECONDS),
+ TimeUnit.MILLISECONDS);
+ } else {
+ // If there are existing offers for the slave, decline all of them so the master can
+ // compact all of those offers into a single offer and send them back.
+ LOG.info("Returning " + (sameSlave.size() + 1)
+ + " offers for " + offer.getSlaveId().getValue() + " for compaction.");
+ decline(offer.getId());
+ for (HostOffer sameSlaveOffer : sameSlave) {
+ removeAndDecline(sameSlaveOffer.offer.getId());
+ }
+ }
+ }
+
+ void removeAndDecline(OfferID id) {
+ if (removeFromHostOffers(id)) {
+ decline(id);
+ }
+ }
+
+ void decline(OfferID id) {
+ LOG.fine("Declining offer " + id);
+ driver.declineOffer(id);
+ }
+
+ @Override
+ public void cancelOffer(final OfferID offerId) {
+ removeFromHostOffers(offerId);
+ }
+
+ private boolean removeFromHostOffers(final OfferID offerId) {
+ Preconditions.checkNotNull(offerId);
+
+ // The small risk of inconsistency is acceptable here - if we have an accept/remove race
+ // on an offer, the master will mark the task as LOST and it will be retried.
+ return Iterables.removeIf(hostOffers,
+ new Predicate<HostOffer>() {
+ @Override public boolean apply(HostOffer input) {
+ return input.offer.getId().equals(offerId);
+ }
+ });
+ }
+
+ @Override
+ public Iterable<Offer> getOffers() {
+ return Iterables.unmodifiableIterable(
+ FluentIterable.from(hostOffers)
+ .transform(new Function<HostOffer, Offer>() {
+ @Override public Offer apply(HostOffer offer) {
+ return offer.offer;
+ }
+ }));
+ }
+
+ /**
+ * Updates the preference of a host's offers.
+ *
+ * @param change Host change notification.
+ */
+ @Subscribe
+ public void hostChangedState(HostMaintenanceStateChange change) {
+ final HostStatus hostStatus = change.getStatus();
+
+ // Remove and re-add a host's offers to re-sort based on its new hostStatus
+ Set<HostOffer> changedOffers = FluentIterable.from(hostOffers)
+ .filter(new Predicate<HostOffer>() {
+ @Override public boolean apply(HostOffer hostOffer) {
+ return hostOffer.offer.getHostname().equals(hostStatus.getHost());
+ }
+ })
+ .toSet();
+ hostOffers.removeAll(changedOffers);
+ hostOffers.addAll(
+ FluentIterable.from(changedOffers)
+ .transform(new Function<HostOffer, HostOffer>() {
+ @Override public HostOffer apply(HostOffer hostOffer) {
+ return new HostOffer(hostOffer.offer, hostStatus.getMode());
+ }
+ })
+ .toSet());
+ }
+
+ /**
+ * Notifies the queue that the driver is disconnected, and all the stored offers are now
+ * invalid.
+ * <p>
+ * The queue takes this as a signal to flush its queue.
+ *
+ * @param event Disconnected event.
+ */
+ @Subscribe
+ public void driverDisconnected(DriverDisconnected event) {
+ LOG.info("Clearing stale offers since the driver is disconnected.");
+ hostOffers.clear();
+ }
+
+ /**
+ * Encapsulate an offer from a host, and the host's maintenance mode.
+ */
+ private static class HostOffer {
+ private final Offer offer;
+ private final MaintenanceMode mode;
+
+ HostOffer(Offer offer, MaintenanceMode mode) {
+ this.offer = offer;
+ this.mode = mode;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof HostOffer)) {
+ return false;
+ }
+ HostOffer other = (HostOffer) o;
+ return Objects.equal(offer, other.offer) && (mode == other.mode);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(offer, mode);
+ }
+ }
+
+ @Override
+ public boolean launchFirst(Function<Offer, Optional<TaskInfo>> acceptor)
+ throws LaunchException {
+
+ // It's important that this method is not called concurrently - doing so would open up the
+ // possibility of a race between the same offers being accepted by different threads.
+
+ for (HostOffer hostOffer : hostOffers) {
+ Optional<TaskInfo> assignment = acceptor.apply(hostOffer.offer);
+ if (assignment.isPresent()) {
+ // Guard against an offer being removed after we grabbed it from the iterator.
+ // If that happens, the offer will not exist in hostOffers, and we can immediately
+ // send it back to LOST for quick reschedule.
+ if (hostOffers.remove(hostOffer)) {
+ try {
+ driver.launchTask(hostOffer.offer.getId(), assignment.get());
+ return true;
+ } catch (IllegalStateException e) {
+ // TODO(William Farner): Catch only the checked exception produced by Driver
+ // once it changes from throwing IllegalStateException when the driver is not yet
+ // registered.
+ throw new LaunchException("Failed to launch task.", e);
+ }
+ } else {
+ offerRaces.incrementAndGet();
+ throw new LaunchException(
+ "Accepted offer no longer exists in offer queue, likely data race.");
+ }
+ }
+ }
+
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/Preemptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/Preemptor.java b/src/main/java/org/apache/aurora/scheduler/async/Preemptor.java
new file mode 100644
index 0000000..a01790c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/Preemptor.java
@@ -0,0 +1,411 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
+import com.google.inject.BindingAnnotation;
+
+import com.twitter.aurora.scheduler.ResourceSlot;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.ScheduleException;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter;
+import com.twitter.aurora.scheduler.state.SchedulerCore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import org.apache.mesos.Protos.Offer;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
+import static com.twitter.aurora.gen.ScheduleStatus.PREEMPTING;
+import static com.twitter.aurora.scheduler.base.Tasks.SCHEDULED_TO_ASSIGNED;
+
+/**
+ * Preempts active tasks in favor of higher priority tasks.
+ */
+public interface Preemptor {
+
+ /**
+ * Preempts active tasks in favor of the input task.
+ *
+ * @param taskId ID of the preempting task.
+ * @return ID of the slave where preemption occured.
+ */
+ Optional<String> findPreemptionSlotFor(String taskId);
+
+ /**
+ * A task preemptor that tries to find tasks that are waiting to be scheduled, which are of higher
+ * priority than tasks that are currently running.
+ *
+ * To avoid excessive churn, the preemptor requires that a task is PENDING for a duration
+ * (dictated by {@link #preemptionCandidacyDelay}) before it becomes eligible to preempt other
+ * tasks.
+ */
+ class PreemptorImpl implements Preemptor {
+
+ /**
+ * Binding annotation for the time interval after which a pending task becomes eligible to
+ * preempt other tasks.
+ */
+ @BindingAnnotation
+ @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+ @interface PreemptionDelay { }
+
+ @VisibleForTesting
+ static final Query.Builder CANDIDATE_QUERY = Query.statusScoped(
+ EnumSet.copyOf(Sets.difference(Tasks.ACTIVE_STATES, EnumSet.of(PENDING, PREEMPTING))));
+
+ private static final Logger LOG = Logger.getLogger(PreemptorImpl.class.getName());
+
+ private static final Function<IAssignedTask, Integer> GET_PRIORITY =
+ new Function<IAssignedTask, Integer>() {
+ @Override public Integer apply(IAssignedTask task) {
+ return task.getTask().getPriority();
+ }
+ };
+
+ private final AtomicLong tasksPreempted = Stats.exportLong("preemptor_tasks_preempted");
+ private final AtomicLong failedPreemptions = Stats.exportLong("preemptor_failed_preemptions");
+ // Incremented every time the preemptor is invoked and finds tasks pending and preemptable tasks
+ private final AtomicLong attemptedPreemptions = Stats.exportLong("preemptor_attempts");
+ // Incremented every time we fail to find tasks to preempt for a pending task.
+ private final AtomicLong noSlotsFound = Stats.exportLong("preemptor_no_slots_found");
+
+ private final Predicate<IScheduledTask> isIdleTask = new Predicate<IScheduledTask>() {
+ @Override public boolean apply(IScheduledTask task) {
+ return (clock.nowMillis() - Iterables.getLast(task.getTaskEvents()).getTimestamp())
+ >= preemptionCandidacyDelay.as(Time.MILLISECONDS);
+ }
+ };
+
+ private final Storage storage;
+ private final SchedulerCore scheduler;
+ private final OfferQueue offerQueue;
+ private final SchedulingFilter schedulingFilter;
+ private final Amount<Long, Time> preemptionCandidacyDelay;
+ private final Clock clock;
+
+ /**
+ * Creates a new preemptor.
+ *
+ * @param storage Backing store for tasks.
+ * @param scheduler Scheduler to fetch task information from, and instruct when preempting
+ * tasks.
+ * @param offerQueue Queue that contains available Mesos resource offers.
+ * @param schedulingFilter Filter to identify whether tasks may reside on given slaves.
+ * @param preemptionCandidacyDelay Time a task must be PENDING before it may preempt other
+ * tasks.
+ * @param clock Clock to check current time.
+ */
+ @Inject
+ PreemptorImpl(
+ Storage storage,
+ SchedulerCore scheduler,
+ OfferQueue offerQueue,
+ SchedulingFilter schedulingFilter,
+ @PreemptionDelay Amount<Long, Time> preemptionCandidacyDelay,
+ Clock clock) {
+
+ this.storage = checkNotNull(storage);
+ this.scheduler = checkNotNull(scheduler);
+ this.offerQueue = checkNotNull(offerQueue);
+ this.schedulingFilter = checkNotNull(schedulingFilter);
+ this.preemptionCandidacyDelay = checkNotNull(preemptionCandidacyDelay);
+ this.clock = checkNotNull(clock);
+ }
+
+ private List<IAssignedTask> fetch(Query.Builder query, Predicate<IScheduledTask> filter) {
+ return Lists.newArrayList(Iterables.transform(Iterables.filter(
+ Storage.Util.consistentFetchTasks(storage, query), filter),
+ SCHEDULED_TO_ASSIGNED));
+ }
+
+ private List<IAssignedTask> fetch(Query.Builder query) {
+ return fetch(query, Predicates.<IScheduledTask>alwaysTrue());
+ }
+
+ private static final Function<IAssignedTask, String> TASK_TO_SLAVE_ID =
+ new Function<IAssignedTask, String>() {
+ @Override public String apply(IAssignedTask input) {
+ return input.getSlaveId();
+ }
+ };
+
+ private static final Function<IAssignedTask, String> TASK_TO_HOST =
+ new Function<IAssignedTask, String>() {
+ @Override public String apply(IAssignedTask input) {
+ return input.getSlaveHost();
+ }
+ };
+
+ private static Predicate<IAssignedTask> canPreempt(final IAssignedTask pending) {
+ return new Predicate<IAssignedTask>() {
+ @Override public boolean apply(IAssignedTask possibleVictim) {
+ return preemptionFilter(possibleVictim).apply(pending);
+ }
+ };
+ }
+
+ private static final Function<IAssignedTask, ResourceSlot> TASK_TO_RESOURCES =
+ new Function<IAssignedTask, ResourceSlot>() {
+ @Override public ResourceSlot apply(IAssignedTask input) {
+ return ResourceSlot.from(input.getTask());
+ }
+ };
+
+ private static final Function<Offer, ResourceSlot> OFFER_TO_RESOURCE_SLOT =
+ new Function<Offer, ResourceSlot>() {
+ @Override public ResourceSlot apply(Offer offer) {
+ return ResourceSlot.from(offer);
+ }
+ };
+
+ private static final Function<Offer, String> OFFER_TO_HOST =
+ new Function<Offer, String>() {
+ @Override public String apply(Offer offer) {
+ return offer.getHostname();
+ }
+ };
+
+ // TODO(zmanji) Consider using Dominant Resource Fairness for ordering instead of the vector
+ // ordering
+ private static final Ordering<IAssignedTask> RESOURCE_ORDER =
+ ResourceSlot.ORDER.onResultOf(TASK_TO_RESOURCES).reverse();
+
+ /**
+ * Optional.absent indicates that this slave does not have enough resources to satisfy the task.
+ * The empty set indicates the offers (slack) are enough.
+ * A set with elements indicates those tasks and the offers are enough.
+ */
+ private Optional<Set<IAssignedTask>> getTasksToPreempt(
+ Iterable<IAssignedTask> possibleVictims,
+ Iterable<Offer> offers,
+ IAssignedTask pendingTask) {
+
+ // This enforces the precondition that all of the resources are from the same host. We need to
+ // get the host for the schedulingFilter.
+ Set<String> hosts = ImmutableSet.<String>builder()
+ .addAll(Iterables.transform(possibleVictims, TASK_TO_HOST))
+ .addAll(Iterables.transform(offers, OFFER_TO_HOST)).build();
+
+ String host = Iterables.getOnlyElement(hosts);
+
+ ResourceSlot slackResources =
+ ResourceSlot.sum(Iterables.transform(offers, OFFER_TO_RESOURCE_SLOT));
+
+ if (!Iterables.isEmpty(offers)) {
+ Set<SchedulingFilter.Veto> vetos = schedulingFilter.filter(
+ slackResources,
+ host,
+ pendingTask.getTask(),
+ pendingTask.getTaskId());
+
+ if (vetos.isEmpty()) {
+ return Optional.<Set<IAssignedTask>>of(ImmutableSet.<IAssignedTask>of());
+ }
+ }
+
+ FluentIterable<IAssignedTask> preemptableTasks =
+ FluentIterable.from(possibleVictims).filter(canPreempt(pendingTask));
+
+ if (preemptableTasks.isEmpty()) {
+ return Optional.absent();
+ }
+
+ List<IAssignedTask> toPreemptTasks = Lists.newArrayList();
+
+ Iterable<IAssignedTask> sortedVictims = RESOURCE_ORDER.immutableSortedCopy(preemptableTasks);
+
+ for (IAssignedTask victim : sortedVictims) {
+ toPreemptTasks.add(victim);
+
+ ResourceSlot totalResource = ResourceSlot.sum(
+ ResourceSlot.sum(Iterables.transform(toPreemptTasks, TASK_TO_RESOURCES)),
+ slackResources);
+
+ Set<SchedulingFilter.Veto> vetos = schedulingFilter.filter(
+ totalResource,
+ host,
+ pendingTask.getTask(),
+ pendingTask.getTaskId());
+
+ if (vetos.isEmpty()) {
+ return Optional.<Set<IAssignedTask>>of(ImmutableSet.copyOf(toPreemptTasks));
+ }
+ }
+ return Optional.absent();
+ }
+
+ private static final Function<Offer, String> OFFER_TO_SLAVE_ID =
+ new Function<Offer, String>() {
+ @Override public String apply(Offer offer) {
+ return offer.getSlaveId().getValue();
+ }
+ };
+
+ private Multimap<String, IAssignedTask> getSlavesToActiveTasks() {
+ // Only non-pending active tasks may be preempted.
+ List<IAssignedTask> activeTasks = fetch(CANDIDATE_QUERY);
+
+ // Walk through the preemption candidates in reverse scheduling order.
+ Collections.sort(activeTasks, Tasks.SCHEDULING_ORDER.reverse());
+
+ // Group the tasks by slave id so they can be paired with offers from the same slave.
+ return Multimaps.index(activeTasks, TASK_TO_SLAVE_ID);
+ }
+
+ @Override
+ public synchronized Optional<String> findPreemptionSlotFor(String taskId) {
+ List<IAssignedTask> pendingTasks =
+ fetch(Query.statusScoped(PENDING).byId(taskId), isIdleTask);
+
+ // Task is no longer PENDING no need to preempt
+ if (pendingTasks.isEmpty()) {
+ return Optional.absent();
+ }
+
+ IAssignedTask pendingTask = Iterables.getOnlyElement(pendingTasks);
+
+ Multimap<String, IAssignedTask> slavesToActiveTasks = getSlavesToActiveTasks();
+
+ if (slavesToActiveTasks.isEmpty()) {
+ return Optional.absent();
+ }
+
+ attemptedPreemptions.incrementAndGet();
+
+ // Group the offers by slave id so they can be paired with active tasks from the same slave.
+ Multimap<String, Offer> slavesToOffers =
+ Multimaps.index(offerQueue.getOffers(), OFFER_TO_SLAVE_ID);
+
+ Set<String> allSlaves = ImmutableSet.<String>builder()
+ .addAll(slavesToOffers.keySet())
+ .addAll(slavesToActiveTasks.keySet())
+ .build();
+
+ for (String slaveID : allSlaves) {
+ Optional<Set<IAssignedTask>> toPreemptTasks = getTasksToPreempt(
+ slavesToActiveTasks.get(slaveID),
+ slavesToOffers.get(slaveID),
+ pendingTask);
+
+ if (toPreemptTasks.isPresent()) {
+ try {
+ for (IAssignedTask toPreempt : toPreemptTasks.get()) {
+ scheduler.preemptTask(toPreempt, pendingTask);
+ tasksPreempted.incrementAndGet();
+ }
+ return Optional.of(slaveID);
+ } catch (ScheduleException e) {
+ LOG.log(Level.SEVERE, "Preemption failed", e);
+ failedPreemptions.incrementAndGet();
+ }
+ }
+ }
+
+ noSlotsFound.incrementAndGet();
+ return Optional.absent();
+ }
+
+ private static final Predicate<IAssignedTask> IS_PRODUCTION =
+ Predicates.compose(Tasks.IS_PRODUCTION, Tasks.ASSIGNED_TO_INFO);
+
+ /**
+ * Creates a static filter that will identify tasks that may preempt the provided task.
+ * A task may preempt another task if the following conditions hold true:
+ * - The resources reserved for {@code preemptableTask} are sufficient to satisfy the task.
+ * - The tasks are owned by the same user and the priority of {@code preemptableTask} is lower
+ * OR {@code preemptableTask} is non-production and the compared task is production.
+ *
+ * @param preemptableTask Task to possibly preempt.
+ * @return A filter that will compare the priorities and resources required by other tasks
+ * with {@code preemptableTask}.
+ */
+ private static Predicate<IAssignedTask> preemptionFilter(IAssignedTask preemptableTask) {
+ Predicate<IAssignedTask> preemptableIsProduction = preemptableTask.getTask().isProduction()
+ ? Predicates.<IAssignedTask>alwaysTrue()
+ : Predicates.<IAssignedTask>alwaysFalse();
+
+ Predicate<IAssignedTask> priorityFilter =
+ greaterPriorityFilter(GET_PRIORITY.apply(preemptableTask));
+ return Predicates.or(
+ Predicates.and(Predicates.not(preemptableIsProduction), IS_PRODUCTION),
+ Predicates.and(isOwnedBy(getRole(preemptableTask)), priorityFilter)
+ );
+ }
+
+ private static Predicate<IAssignedTask> isOwnedBy(final String role) {
+ return new Predicate<IAssignedTask>() {
+ @Override public boolean apply(IAssignedTask task) {
+ return getRole(task).equals(role);
+ }
+ };
+ }
+
+ private static String getRole(IAssignedTask task) {
+ return task.getTask().getOwner().getRole();
+ }
+
+ private static Predicate<Integer> greaterThan(final int value) {
+ return new Predicate<Integer>() {
+ @Override public boolean apply(Integer input) {
+ return input > value;
+ }
+ };
+ }
+
+ private static Predicate<IAssignedTask> greaterPriorityFilter(int priority) {
+ return Predicates.compose(greaterThan(priority), GET_PRIORITY);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/RescheduleCalculator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/RescheduleCalculator.java b/src/main/java/org/apache/aurora/scheduler/async/RescheduleCalculator.java
new file mode 100644
index 0000000..eefc03a
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/RescheduleCalculator.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskEvent;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.util.BackoffStrategy;
+import com.twitter.common.util.Clock;
+import com.twitter.common.util.Random;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.KILLING;
+import static com.twitter.aurora.gen.ScheduleStatus.RESTARTING;
+
+/**
+ * Calculates scheduling delays for tasks.
+ */
+interface RescheduleCalculator {
+ /**
+ * Gets a timestamp for the task to become eligible for (re)scheduling at scheduler startup.
+ *
+ * @param task Task to calculate timestamp for.
+ * @return Timestamp in msec.
+ */
+ long getStartupReadyTimeMs(IScheduledTask task);
+
+ /**
+ * Gets a timestamp for the task to become eligible for (re)scheduling.
+ *
+ * @param task Task to calculate timestamp for.
+ * @return Timestamp in msec.
+ */
+ long getReadyTimeMs(IScheduledTask task);
+
+ class RescheduleCalculatorImpl implements RescheduleCalculator {
+
+ private static final Logger LOG = Logger.getLogger(TaskGroups.class.getName());
+
+ private final Storage storage;
+ private final RescheduleCalculatorSettings settings;
+ private final Clock clock;
+ private final Random random = new Random.SystemRandom(new java.util.Random());
+
+ private static final Predicate<ScheduleStatus> IS_ACTIVE_STATUS =
+ Predicates.in(Tasks.ACTIVE_STATES);
+
+ private static final Function<ITaskEvent, ScheduleStatus> TO_STATUS =
+ new Function<ITaskEvent, ScheduleStatus>() {
+ @Override public ScheduleStatus apply(ITaskEvent input) {
+ return input.getStatus();
+ }
+ };
+
+ private static final Set<ScheduleStatus> INTERRUPTED_TASK_STATES =
+ EnumSet.of(RESTARTING, KILLING);
+
+ private final Predicate<IScheduledTask> flapped = new Predicate<IScheduledTask>() {
+ @Override public boolean apply(IScheduledTask task) {
+ if (!task.isSetTaskEvents()) {
+ return false;
+ }
+
+ List<ITaskEvent> events = Lists.reverse(task.getTaskEvents());
+
+ // Avoid penalizing tasks that were interrupted by outside action, such as a user
+ // restarting them.
+ if (Iterables.any(Iterables.transform(events, TO_STATUS),
+ Predicates.in(INTERRUPTED_TASK_STATES))) {
+ return false;
+ }
+
+ ITaskEvent terminalEvent = Iterables.get(events, 0);
+ ScheduleStatus terminalState = terminalEvent.getStatus();
+ Preconditions.checkState(Tasks.isTerminated(terminalState));
+
+ ITaskEvent activeEvent =
+ Iterables.find(events, Predicates.compose(IS_ACTIVE_STATUS, TO_STATUS));
+
+ long thresholdMs = settings.flappingTaskThreashold.as(Time.MILLISECONDS);
+
+ return (terminalEvent.getTimestamp() - activeEvent.getTimestamp()) < thresholdMs;
+ }
+ };
+
+ static class RescheduleCalculatorSettings {
+ private final BackoffStrategy flappingTaskBackoff;
+ private final Amount<Long, Time> flappingTaskThreashold;
+ private final Amount<Integer, Time> maxStartupRescheduleDelay;
+
+ RescheduleCalculatorSettings(
+ BackoffStrategy flappingTaskBackoff,
+ Amount<Long, Time> flappingTaskThreashold,
+ Amount<Integer, Time> maxStartupRescheduleDelay) {
+
+ this.flappingTaskBackoff = checkNotNull(flappingTaskBackoff);
+ this.flappingTaskThreashold = checkNotNull(flappingTaskThreashold);
+ this.maxStartupRescheduleDelay = checkNotNull(maxStartupRescheduleDelay);
+ }
+ }
+
+ @Inject
+ RescheduleCalculatorImpl(
+ Storage storage,
+ RescheduleCalculatorSettings settings,
+ Clock clock) {
+
+ this.storage = checkNotNull(storage);
+ this.settings = checkNotNull(settings);
+ this.clock = checkNotNull(clock);
+ }
+
+ @Override
+ public long getStartupReadyTimeMs(IScheduledTask task) {
+ return random.nextInt(settings.maxStartupRescheduleDelay.as(Time.MILLISECONDS))
+ + getTaskReadyTimestamp(task);
+ }
+
+ @Override
+ public long getReadyTimeMs(IScheduledTask task) {
+ return getTaskReadyTimestamp(task);
+ }
+
+ private Optional<IScheduledTask> getTaskAncestor(IScheduledTask task) {
+ if (!task.isSetAncestorId()) {
+ return Optional.absent();
+ }
+
+ ImmutableSet<IScheduledTask> res =
+ Storage.Util.weaklyConsistentFetchTasks(storage, Query.taskScoped(task.getAncestorId()));
+
+ return Optional.fromNullable(Iterables.getOnlyElement(res, null));
+ }
+
+ private long getTaskReadyTimestamp(IScheduledTask task) {
+ Optional<IScheduledTask> curTask = getTaskAncestor(task);
+ long penaltyMs = 0;
+ while (curTask.isPresent() && flapped.apply(curTask.get())) {
+ LOG.info(
+ String.format("Ancestor of %s flapped: %s", Tasks.id(task), Tasks.id(curTask.get())));
+ long newPenalty = settings.flappingTaskBackoff.calculateBackoffMs(penaltyMs);
+ // If the backoff strategy is truncated then there is no need for us to continue.
+ if (newPenalty == penaltyMs) {
+ break;
+ }
+ penaltyMs = newPenalty;
+ curTask = getTaskAncestor(curTask.get());
+ }
+
+ return penaltyMs + clock.nowMillis();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/TaskGroup.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/TaskGroup.java b/src/main/java/org/apache/aurora/scheduler/async/TaskGroup.java
new file mode 100644
index 0000000..018022b
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/TaskGroup.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+
+import com.twitter.aurora.scheduler.async.TaskGroups.GroupKey;
+import com.twitter.common.base.Function;
+import com.twitter.common.util.BackoffStrategy;
+
+/**
+ * A group of task IDs that are eligible for scheduling, but may be waiting for a backoff to expire.
+ */
+class TaskGroup {
+ private final GroupKey key;
+ private final BackoffStrategy backoffStrategy;
+
+ private static final Function<Task, Long> TO_TIMESTAMP = new Function<Task, Long>() {
+ @Override public Long apply(Task item) {
+ return item.readyTimestampMs;
+ }
+ };
+
+ // Order the tasks by the time they are ready to be scheduled
+ private static final Ordering<Task> TASK_ORDERING = Ordering.natural().onResultOf(TO_TIMESTAMP);
+ // 11 is the magic number used by PriorityBlockingQueue as the initial size.
+ private final Queue<Task> tasks = new PriorityBlockingQueue<>(11, TASK_ORDERING);
+ // Penalty for the task group for failing to schedule.
+ private final AtomicLong penaltyMs;
+
+ TaskGroup(GroupKey key, BackoffStrategy backoffStrategy) {
+ this.key = key;
+ this.backoffStrategy = backoffStrategy;
+ penaltyMs = new AtomicLong();
+ resetPenaltyAndGet();
+ }
+
+ GroupKey getKey() {
+ return key;
+ }
+
+ private static final Function<Task, String> TO_TASK_ID =
+ new Function<Task, String>() {
+ @Override public String apply(Task item) {
+ return item.taskId;
+ }
+ };
+
+ /**
+ * Removes the task at the head of the queue.
+ *
+ * @return String the id of the head task.
+ * @throws IllegalStateException if the queue is empty.
+ */
+ String pop() throws IllegalStateException {
+ Task head = tasks.poll();
+ Preconditions.checkState(head != null);
+ return head.taskId;
+ }
+
+ void remove(String taskId) {
+ Iterables.removeIf(tasks, Predicates.compose(Predicates.equalTo(taskId), TO_TASK_ID));
+ }
+
+ void push(final String taskId, long readyTimestamp) {
+ tasks.offer(new Task(taskId, readyTimestamp));
+ }
+
+ synchronized long resetPenaltyAndGet() {
+ penaltyMs.set(backoffStrategy.calculateBackoffMs(0));
+ return getPenaltyMs();
+ }
+
+ synchronized long penalizeAndGet() {
+ penaltyMs.set(backoffStrategy.calculateBackoffMs(getPenaltyMs()));
+ return getPenaltyMs();
+ }
+
+ GroupState isReady(long nowMs) {
+ Task task = tasks.peek();
+ if (task == null) {
+ return GroupState.EMPTY;
+ }
+
+ if (task.readyTimestampMs > nowMs) {
+ return GroupState.NOT_READY;
+ }
+ return GroupState.READY;
+ }
+ // Begin methods used for debug interfaces.
+
+ public String getName() {
+ return key.toString();
+ }
+
+ // TODO(zmanji): Return Task instances here. Can use them to display flapping penalty on web UI.
+ public Set<String> getTaskIds() {
+ return ImmutableSet.copyOf(Iterables.transform(tasks, TO_TASK_ID));
+ }
+
+ public long getPenaltyMs() {
+ return penaltyMs.get();
+ }
+
+ private static class Task {
+ private final String taskId;
+ private final long readyTimestampMs;
+
+ Task(String taskId, long readyTimestampMs) {
+ this.taskId = Preconditions.checkNotNull(taskId);
+ this.readyTimestampMs = readyTimestampMs;
+ }
+ }
+
+ enum GroupState {
+ EMPTY, // The group is empty.
+ NOT_READY, // Every task in the group is not ready yet.
+ READY // The task at the head of the queue is ready.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/TaskGroups.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/TaskGroups.java b/src/main/java/org/apache/aurora/scheduler/async/TaskGroups.java
new file mode 100644
index 0000000..a59e5c8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/TaskGroups.java
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.base.Objects;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.Subscribe;
+import com.google.common.util.concurrent.RateLimiter;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TasksDeleted;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.application.ShutdownRegistry;
+import com.twitter.common.base.Command;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.BackoffStrategy;
+import com.twitter.common.util.Clock;
+import com.twitter.common.util.concurrent.ExecutorServiceShutdown;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
+import static com.twitter.aurora.scheduler.async.TaskGroup.GroupState;
+
+/**
+ * A collection of task groups, where a task group is a collection of tasks that are known to be
+ * equal in the way they schedule. This is expected to be tasks associated with the same job key,
+ * who also have {@code equal()} {@link ITaskConfig} values.
+ * <p>
+ * This is used to prevent redundant work in trying to schedule tasks as well as to provide
+ * nearly-equal responsiveness when scheduling across jobs. In other words, a 1000 instance job
+ * cannot starve a 1 instance job.
+ */
+public class TaskGroups implements EventSubscriber {
+
+ private static final Logger LOG = Logger.getLogger(TaskGroups.class.getName());
+
+ private final Storage storage;
+ private final LoadingCache<GroupKey, TaskGroup> groups;
+ private final Clock clock;
+ private final RescheduleCalculator rescheduleCalculator;
+
+ static class TaskGroupsSettings {
+ private final BackoffStrategy taskGroupBackoff;
+ private final RateLimiter rateLimiter;
+
+ TaskGroupsSettings(BackoffStrategy taskGroupBackoff, RateLimiter rateLimiter) {
+ this.taskGroupBackoff = checkNotNull(taskGroupBackoff);
+ this.rateLimiter = checkNotNull(rateLimiter);
+ }
+ }
+
+ @Inject
+ TaskGroups(
+ ShutdownRegistry shutdownRegistry,
+ Storage storage,
+ TaskGroupsSettings settings,
+ TaskScheduler taskScheduler,
+ Clock clock,
+ RescheduleCalculator rescheduleCalculator) {
+
+ this(
+ createThreadPool(shutdownRegistry),
+ storage,
+ settings.taskGroupBackoff,
+ settings.rateLimiter,
+ taskScheduler,
+ clock,
+ rescheduleCalculator);
+ }
+
+ TaskGroups(
+ final ScheduledExecutorService executor,
+ final Storage storage,
+ final BackoffStrategy taskGroupBackoffStrategy,
+ final RateLimiter rateLimiter,
+ final TaskScheduler taskScheduler,
+ final Clock clock,
+ final RescheduleCalculator rescheduleCalculator) {
+
+ this.storage = checkNotNull(storage);
+ checkNotNull(executor);
+ checkNotNull(taskGroupBackoffStrategy);
+ checkNotNull(rateLimiter);
+ checkNotNull(taskScheduler);
+ this.clock = checkNotNull(clock);
+ this.rescheduleCalculator = checkNotNull(rescheduleCalculator);
+
+ final TaskScheduler ratelLimitedScheduler = new TaskScheduler() {
+ @Override public TaskSchedulerResult schedule(String taskId) {
+ rateLimiter.acquire();
+ return taskScheduler.schedule(taskId);
+ }
+ };
+
+ groups = CacheBuilder.newBuilder().build(new CacheLoader<GroupKey, TaskGroup>() {
+ @Override public TaskGroup load(GroupKey key) {
+ TaskGroup group = new TaskGroup(key, taskGroupBackoffStrategy);
+ LOG.info("Evaluating group " + key + " in " + group.getPenaltyMs() + " ms");
+ startGroup(group, executor, ratelLimitedScheduler);
+ return group;
+ }
+ });
+ }
+
+ private synchronized boolean maybeInvalidate(TaskGroup group) {
+ if (group.getTaskIds().isEmpty()) {
+ groups.invalidate(group.getKey());
+ return true;
+ }
+ return false;
+ }
+
+ private void startGroup(
+ final TaskGroup group,
+ final ScheduledExecutorService executor,
+ final TaskScheduler taskScheduler) {
+
+ Runnable monitor = new Runnable() {
+ @Override public void run() {
+ GroupState state = group.isReady(clock.nowMillis());
+
+ switch (state) {
+ case EMPTY:
+ maybeInvalidate(group);
+ break;
+
+ case READY:
+ String id = group.pop();
+ TaskScheduler.TaskSchedulerResult result = taskScheduler.schedule(id);
+ switch (result) {
+ case SUCCESS:
+ if (!maybeInvalidate(group)) {
+ executor.schedule(this, group.resetPenaltyAndGet(), TimeUnit.MILLISECONDS);
+ }
+ break;
+
+ case TRY_AGAIN:
+ group.push(id, clock.nowMillis());
+ executor.schedule(this, group.penalizeAndGet(), TimeUnit.MILLISECONDS);
+ break;
+
+ default:
+ throw new IllegalStateException("Unknown TaskSchedulerResult " + result);
+ }
+ break;
+
+ case NOT_READY:
+ executor.schedule(this, group.getPenaltyMs(), TimeUnit.MILLISECONDS);
+ break;
+
+ default:
+ throw new IllegalStateException("Unknown GroupState " + state);
+ }
+ }
+ };
+ executor.schedule(monitor, group.getPenaltyMs(), TimeUnit.MILLISECONDS);
+ }
+
+ private static ScheduledExecutorService createThreadPool(ShutdownRegistry shutdownRegistry) {
+ // TODO(William Farner): Leverage ExceptionHandlingScheduledExecutorService:
+ // com.twitter.common.util.concurrent.ExceptionHandlingScheduledExecutorService
+ final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(
+ 1,
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TaskScheduler-%d").build());
+ Stats.exportSize("schedule_queue_size", executor.getQueue());
+ shutdownRegistry.addAction(new Command() {
+ @Override public void execute() {
+ new ExecutorServiceShutdown(executor, Amount.of(1L, Time.SECONDS)).execute();
+ }
+ });
+ return executor;
+ }
+
+ private synchronized void add(IAssignedTask task, long readyTimestamp) {
+ groups.getUnchecked(new GroupKey(task.getTask())).push(task.getTaskId(), readyTimestamp);
+ }
+
+ /**
+ * Informs the task groups of a task state change.
+ * <p>
+ * This is used to observe {@link com.twitter.aurora.gen.ScheduleStatus#PENDING} tasks and begin
+ * attempting to schedule them.
+ *
+ * @param stateChange State change notification.
+ */
+ @Subscribe
+ public synchronized void taskChangedState(TaskStateChange stateChange) {
+ if (stateChange.getNewState() == PENDING) {
+ add(
+ stateChange.getTask().getAssignedTask(),
+ rescheduleCalculator.getReadyTimeMs(stateChange.getTask()));
+ }
+ }
+
+ /**
+ * Signals that storage has started and is consistent.
+ * <p>
+ * Upon this signal, all {@link com.twitter.aurora.gen.ScheduleStatus#PENDING} tasks in the stoage
+ * will become eligible for scheduling.
+ *
+ * @param event Storage started notification.
+ */
+ @Subscribe
+ public void storageStarted(StorageStarted event) {
+ for (IScheduledTask task
+ : Storage.Util.consistentFetchTasks(storage, Query.unscoped().byStatus(PENDING))) {
+
+ add(task.getAssignedTask(), rescheduleCalculator.getStartupReadyTimeMs(task));
+ }
+ }
+
+ /**
+ * Signals the scheduler that tasks have been deleted.
+ *
+ * @param deleted Tasks deleted event.
+ */
+ @Subscribe
+ public synchronized void tasksDeleted(TasksDeleted deleted) {
+ for (IAssignedTask task
+ : Iterables.transform(deleted.getTasks(), Tasks.SCHEDULED_TO_ASSIGNED)) {
+ TaskGroup group = groups.getIfPresent(new GroupKey(task.getTask()));
+ if (group != null) {
+ group.remove(task.getTaskId());
+ }
+ }
+ }
+
+ public Iterable<TaskGroup> getGroups() {
+ return ImmutableSet.copyOf(groups.asMap().values());
+ }
+
+ static class GroupKey {
+ private final ITaskConfig canonicalTask;
+
+ GroupKey(ITaskConfig task) {
+ this.canonicalTask = task;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(canonicalTask);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof GroupKey)) {
+ return false;
+ }
+ GroupKey other = (GroupKey) o;
+ return Objects.equal(canonicalTask, other.canonicalTask);
+ }
+
+ @Override
+ public String toString() {
+ return JobKeys.toPath(Tasks.INFO_TO_JOB_KEY.apply(canonicalTask));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/TaskScheduler.java b/src/main/java/org/apache/aurora/scheduler/async/TaskScheduler.java
new file mode 100644
index 0000000..0ad9e13
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/TaskScheduler.java
@@ -0,0 +1,257 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Ticker;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.Subscribe;
+import com.google.inject.BindingAnnotation;
+
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.SlaveID;
+import org.apache.mesos.Protos.TaskInfo;
+
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import com.twitter.aurora.scheduler.state.StateManager;
+import com.twitter.aurora.scheduler.state.TaskAssigner;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.MutateWork;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.inject.TimedInterceptor.Timed;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.StatImpl;
+import com.twitter.common.stats.Stats;
+import com.twitter.common.util.Clock;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.LOST;
+import static com.twitter.aurora.gen.ScheduleStatus.PENDING;
+
+/**
+ * Enables scheduling and preemption of tasks.
+ */
+interface TaskScheduler extends EventSubscriber {
+
+ /**
+ * Attempts to schedule a task, possibly performing irreversible actions.
+ *
+ * @param taskId The task to attempt to schedule.
+ * @return SUCCESS if the task was scheduled, TRY_AGAIN otherwise. The caller should call schedule
+ * again if TRY_AGAIN is returned.
+ */
+ TaskSchedulerResult schedule(String taskId);
+
+ enum TaskSchedulerResult {
+ SUCCESS,
+ TRY_AGAIN
+ }
+
+ /**
+ * An asynchronous task scheduler. Scheduling of tasks is performed on a delay, where each task
+ * backs off after a failed scheduling attempt.
+ * <p>
+ * Pending tasks are advertised to the scheduler via internal pubsub notifications.
+ */
+ class TaskSchedulerImpl implements TaskScheduler {
+ /**
+ * Binding annotation for the time duration of reservations
+ */
+ @BindingAnnotation
+ @Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+ @interface ReservationDuration { }
+
+ private static final Logger LOG = Logger.getLogger(TaskSchedulerImpl.class.getName());
+
+ private final Storage storage;
+ private final StateManager stateManager;
+ private final TaskAssigner assigner;
+ private final OfferQueue offerQueue;
+ private final Preemptor preemptor;
+ private final Reservations reservations;
+
+ private final AtomicLong scheduleAttemptsFired = Stats.exportLong("schedule_attempts_fired");
+ private final AtomicLong scheduleAttemptsFailed = Stats.exportLong("schedule_attempts_failed");
+
+ @Inject
+ TaskSchedulerImpl(
+ Storage storage,
+ StateManager stateManager,
+ TaskAssigner assigner,
+ OfferQueue offerQueue,
+ Preemptor preemptor,
+ @ReservationDuration Amount<Long, Time> reservationDuration,
+ final Clock clock) {
+
+ this.storage = checkNotNull(storage);
+ this.stateManager = checkNotNull(stateManager);
+ this.assigner = checkNotNull(assigner);
+ this.offerQueue = checkNotNull(offerQueue);
+ this.preemptor = checkNotNull(preemptor);
+ this.reservations = new Reservations(reservationDuration, clock);
+ }
+
+ private Function<Offer, Optional<TaskInfo>> getAssignerFunction(
+ final String taskId,
+ final IScheduledTask task) {
+
+ return new Function<Offer, Optional<TaskInfo>>() {
+ @Override public Optional<TaskInfo> apply(Offer offer) {
+ Optional<String> reservedTaskId = reservations.getSlaveReservation(offer.getSlaveId());
+ if (reservedTaskId.isPresent()) {
+ if (taskId.equals(reservedTaskId.get())) {
+ // Slave is reserved to satisfy this task.
+ return assigner.maybeAssign(offer, task);
+ } else {
+ // Slave is reserved for another task.
+ return Optional.absent();
+ }
+ } else {
+ // Slave is not reserved.
+ return assigner.maybeAssign(offer, task);
+ }
+ }
+ };
+ }
+
+ @VisibleForTesting
+ static final Optional<String> LAUNCH_FAILED_MSG =
+ Optional.of("Unknown exception attempting to schedule task.");
+
+ @Timed("task_schedule_attempt")
+ @Override
+ public TaskSchedulerResult schedule(final String taskId) {
+ scheduleAttemptsFired.incrementAndGet();
+ try {
+ return storage.write(new MutateWork.Quiet<TaskSchedulerResult>() {
+ @Override public TaskSchedulerResult apply(MutableStoreProvider store) {
+ LOG.fine("Attempting to schedule task " + taskId);
+ Query.Builder pendingTaskQuery = Query.taskScoped(taskId).byStatus(PENDING);
+ final IScheduledTask task =
+ Iterables.getOnlyElement(store.getTaskStore().fetchTasks(pendingTaskQuery), null);
+ if (task == null) {
+ LOG.warning("Failed to look up task " + taskId + ", it may have been deleted.");
+ } else {
+ try {
+ if (!offerQueue.launchFirst(getAssignerFunction(taskId, task))) {
+ // Task could not be scheduled.
+ maybePreemptFor(taskId);
+ return TaskSchedulerResult.TRY_AGAIN;
+ }
+ } catch (OfferQueue.LaunchException e) {
+ LOG.log(Level.WARNING, "Failed to launch task.", e);
+ scheduleAttemptsFailed.incrementAndGet();
+
+ // The attempt to schedule the task failed, so we need to backpedal on the
+ // assignment.
+ // It is in the LOST state and a new task will move to PENDING to replace it.
+ // Should the state change fail due to storage issues, that's okay. The task will
+ // time out in the ASSIGNED state and be moved to LOST.
+ stateManager.changeState(pendingTaskQuery, LOST, LAUNCH_FAILED_MSG);
+ }
+ }
+
+ return TaskSchedulerResult.SUCCESS;
+ }
+ });
+ } catch (RuntimeException e) {
+ // We catch the generic unchecked exception here to ensure tasks are not abandoned
+ // if there is a transient issue resulting in an unchecked exception.
+ LOG.log(Level.WARNING, "Task scheduling unexpectedly failed, will be retried", e);
+ scheduleAttemptsFailed.incrementAndGet();
+ return TaskSchedulerResult.TRY_AGAIN;
+ }
+ }
+
+ private void maybePreemptFor(String taskId) {
+ if (reservations.hasReservationForTask(taskId)) {
+ return;
+ }
+ Optional<String> slaveId = preemptor.findPreemptionSlotFor(taskId);
+ if (slaveId.isPresent()) {
+ this.reservations.add(SlaveID.newBuilder().setValue(slaveId.get()).build(), taskId);
+ }
+ }
+
+ @Subscribe
+ public void taskChanged(final TaskStateChange stateChangeEvent) {
+ if (stateChangeEvent.getOldState() == PENDING) {
+ reservations.invalidateTask(stateChangeEvent.getTaskId());
+ }
+ }
+
+ private static class Reservations {
+ private final Cache<SlaveID, String> reservations;
+
+ Reservations(final Amount<Long, Time> duration, final Clock clock) {
+ checkNotNull(duration);
+ checkNotNull(clock);
+ this.reservations = CacheBuilder.newBuilder()
+ .expireAfterWrite(duration.as(Time.MINUTES), TimeUnit.MINUTES)
+ .ticker(new Ticker() {
+ @Override public long read() {
+ return clock.nowNanos();
+ }
+ })
+ .build();
+ Stats.export(new StatImpl<Long>("reservation_cache_size") {
+ @Override public Long read() {
+ return reservations.size();
+ }
+ });
+ }
+
+ private synchronized void add(SlaveID slaveId, String taskId) {
+ reservations.put(slaveId, taskId);
+ }
+
+ private synchronized boolean hasReservationForTask(String taskId) {
+ return reservations.asMap().containsValue(taskId);
+ }
+
+ private synchronized Optional<String> getSlaveReservation(SlaveID slaveID) {
+ return Optional.fromNullable(reservations.getIfPresent(slaveID));
+ }
+
+ private synchronized void invalidateTask(String taskId) {
+ reservations.asMap().values().remove(taskId);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/async/TaskTimeout.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/async/TaskTimeout.java b/src/main/java/org/apache/aurora/scheduler/async/TaskTimeout.java
new file mode 100644
index 0000000..19848c7
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/async/TaskTimeout.java
@@ -0,0 +1,269 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.async;
+
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+import com.google.common.eventbus.Subscribe;
+
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.StorageStarted;
+import com.twitter.aurora.scheduler.events.PubsubEvent.TaskStateChange;
+import com.twitter.aurora.scheduler.state.StateManager;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.stats.StatsProvider;
+import com.twitter.common.util.Clock;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Observes task transitions and identifies tasks that are 'stuck' in a transient state. Stuck
+ * tasks will be transitioned to the LOST state.
+ */
+class TaskTimeout implements EventSubscriber {
+ private static final Logger LOG = Logger.getLogger(TaskTimeout.class.getName());
+
+ @VisibleForTesting
+ static final String TIMED_OUT_TASKS_COUNTER = "timed_out_tasks";
+
+ @VisibleForTesting
+ static final String TRANSIENT_COUNT_STAT_NAME = "transient_states";
+
+ @VisibleForTesting
+ static final Optional<String> TIMEOUT_MESSAGE = Optional.of("Task timed out");
+
+ @VisibleForTesting
+ static final Set<ScheduleStatus> TRANSIENT_STATES = EnumSet.of(
+ ScheduleStatus.ASSIGNED,
+ ScheduleStatus.PREEMPTING,
+ ScheduleStatus.RESTARTING,
+ ScheduleStatus.KILLING);
+
+ @VisibleForTesting
+ static final Query.Builder TRANSIENT_QUERY = Query.unscoped().byStatus(TRANSIENT_STATES);
+
+ private final Map<TimeoutKey, Context> futures = Maps.newConcurrentMap();
+
+ private static final class TimeoutKey {
+ private final String taskId;
+ private final ScheduleStatus status;
+
+ private TimeoutKey(String taskId, ScheduleStatus status) {
+ this.taskId = taskId;
+ this.status = status;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(taskId, status);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TimeoutKey)) {
+ return false;
+ }
+ TimeoutKey key = (TimeoutKey) o;
+ return Objects.equal(taskId, key.taskId)
+ && (status == key.status);
+ }
+
+ @Override
+ public String toString() {
+ return taskId + ":" + status;
+ }
+ }
+
+ private final Storage storage;
+ private final ScheduledExecutorService executor;
+ private final StateManager stateManager;
+ private final long timeoutMillis;
+ private final Clock clock;
+ private final AtomicLong timedOutTasks;
+
+ @Inject
+ TaskTimeout(
+ Storage storage,
+ ScheduledExecutorService executor,
+ StateManager stateManager,
+ final Clock clock,
+ Amount<Long, Time> timeout,
+ StatsProvider statsProvider) {
+
+ this.storage = checkNotNull(storage);
+ this.executor = checkNotNull(executor);
+ this.stateManager = checkNotNull(stateManager);
+ this.timeoutMillis = timeout.as(Time.MILLISECONDS);
+ this.clock = checkNotNull(clock);
+ this.timedOutTasks = statsProvider.makeCounter(TIMED_OUT_TASKS_COUNTER);
+
+ exportStats(statsProvider);
+ }
+
+ private void registerTimeout(TimeoutKey key) {
+ // This is an obvious check-then-act, but:
+ // - there isn't much of a better option, given that we have to get the Future before
+ // inserting into the map
+ // - a key collision only happens in practice if something is wrong externally to this class
+ // (double event for the same state)
+ // - the outcome is low-risk, we would wind up with a redundant Future that will eventually
+ // no-op
+ if (!futures.containsKey(key)) {
+ Future<?> timeoutHandler = executor.schedule(
+ new TimedOutTaskHandler(key),
+ timeoutMillis,
+ TimeUnit.MILLISECONDS);
+ futures.put(key, new Context(clock.nowMillis(), timeoutHandler));
+ }
+ }
+
+ private static boolean isTransient(ScheduleStatus status) {
+ return TRANSIENT_STATES.contains(status);
+ }
+
+ @Subscribe
+ public void recordStateChange(TaskStateChange change) {
+ String taskId = change.getTaskId();
+ ScheduleStatus newState = change.getNewState();
+ if (isTransient(change.getOldState())) {
+ TimeoutKey oldKey = new TimeoutKey(taskId, change.getOldState());
+ Context context = futures.remove(oldKey);
+ if (context != null) {
+ LOG.fine("Canceling state timeout for task " + oldKey);
+ context.future.cancel(false);
+ }
+ }
+
+ if (isTransient(newState)) {
+ registerTimeout(new TimeoutKey(taskId, change.getNewState()));
+ }
+ }
+
+ @Subscribe
+ public void storageStarted(StorageStarted event) {
+ for (IScheduledTask task : Storage.Util.consistentFetchTasks(storage, TRANSIENT_QUERY)) {
+ registerTimeout(new TimeoutKey(Tasks.id(task), task.getStatus()));
+ }
+ }
+
+ private class TimedOutTaskHandler implements Runnable {
+ private final TimeoutKey key;
+
+ TimedOutTaskHandler(TimeoutKey key) {
+ this.key = key;
+ }
+
+ @Override public void run() {
+ Context context = futures.get(key);
+ try {
+ if (context == null) {
+ LOG.warning("Timeout context not found for " + key);
+ return;
+ }
+
+ LOG.info("Timeout reached for task " + key);
+ // This query acts as a CAS by including the state that we expect the task to be in if the
+ // timeout is still valid. Ideally, the future would have already been canceled, but in the
+ // event of a state transition race, including transientState prevents an unintended
+ // task timeout.
+ Query.Builder query = Query.taskScoped(key.taskId).byStatus(key.status);
+ // Note: This requires LOST transitions trigger Driver.killTask.
+ if (stateManager.changeState(query, ScheduleStatus.LOST, TIMEOUT_MESSAGE) > 0) {
+ timedOutTasks.incrementAndGet();
+ } else {
+ LOG.warning("Task " + key + " does not exist, or was not in the expected state.");
+ }
+ } finally {
+ futures.remove(key);
+ }
+ }
+ }
+
+ private class Context {
+ private final long timestampMillis;
+ private final Future<?> future;
+
+ Context(long timestampMillis, Future<?> future) {
+ this.timestampMillis = timestampMillis;
+ this.future = future;
+ }
+ }
+
+ private static final Function<Context, Long> CONTEXT_TIMESTAMP = new Function<Context, Long>() {
+ @Override public Long apply(Context context) {
+ return context.timestampMillis;
+ }
+ };
+
+ private static final Ordering<Context> TIMESTAMP_ORDER =
+ Ordering.natural().onResultOf(CONTEXT_TIMESTAMP);
+
+ @VisibleForTesting
+ static String waitingTimeStatName(ScheduleStatus status) {
+ return "scheduler_max_" + status + "_waiting_ms";
+ }
+
+ private void exportStats(StatsProvider statsProvider) {
+ statsProvider.makeGauge(TRANSIENT_COUNT_STAT_NAME, new Supplier<Number>() {
+ @Override public Number get() {
+ return futures.size();
+ }
+ });
+
+ for (final ScheduleStatus status : TRANSIENT_STATES) {
+ statsProvider.makeGauge(waitingTimeStatName(status), new Supplier<Number>() {
+ private final Predicate<TimeoutKey> statusMatcher = new Predicate<TimeoutKey>() {
+ @Override public boolean apply(TimeoutKey key) {
+ return key.status == status;
+ }
+ };
+
+ @Override public Number get() {
+ Iterable<Context> matches = Maps.filterKeys(futures, statusMatcher).values();
+ if (Iterables.isEmpty(matches)) {
+ return 0L;
+ } else {
+ return clock.nowMillis() - TIMESTAMP_ORDER.min(matches).timestampMillis;
+ }
+ }
+ });
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/CommandUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/CommandUtil.java b/src/main/java/org/apache/aurora/scheduler/base/CommandUtil.java
new file mode 100644
index 0000000..b11c683
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/CommandUtil.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+import org.apache.mesos.Protos.CommandInfo;
+import org.apache.mesos.Protos.CommandInfo.URI;
+
+import com.twitter.common.base.MorePreconditions;
+
+/**
+ * Utility class for constructing {@link CommandInfo} objects given an executor URI.
+ */
+public final class CommandUtil {
+
+ private CommandUtil() {
+ // Utility class.
+ }
+
+ private static String uriBasename(String uri) {
+ int lastSlash = uri.lastIndexOf("/");
+ if (lastSlash == -1) {
+ return uri;
+ } else {
+ String basename = uri.substring(lastSlash + 1);
+ MorePreconditions.checkNotBlank(basename, "URI must not end with a slash.");
+
+ return basename;
+ }
+ }
+
+ /**
+ * Creates a description of a command that will fetch and execute the given URI to an executor
+ * binary.
+ *
+ * @param executorUri URI to the executor.
+ * @return A command that will fetch and execute the executor.
+ */
+ public static CommandInfo create(String executorUri) {
+ MorePreconditions.checkNotBlank(executorUri);
+
+ return CommandInfo.newBuilder()
+ .addUris(URI.newBuilder().setValue(executorUri).setExecutable(true))
+ .setValue("./" + uriBasename(executorUri))
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/Conversions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/Conversions.java b/src/main/java/org/apache/aurora/scheduler/base/Conversions.java
new file mode 100644
index 0000000..2f84b5c
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/Conversions.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.logging.Logger;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+
+import org.apache.mesos.Protos;
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.TaskState;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.ScheduleStatus;
+
+/**
+ * Collection of utility functions to convert mesos protobuf types to internal thrift types.
+ */
+public final class Conversions {
+
+ private static final Logger LOG = Logger.getLogger(Conversions.class.getName());
+
+ private Conversions() {
+ // Utility class.
+ }
+
+ // Maps from mesos state to scheduler interface state.
+ private static final Map<TaskState, ScheduleStatus> STATE_TRANSLATION =
+ new ImmutableMap.Builder<TaskState, ScheduleStatus>()
+ .put(TaskState.TASK_STARTING, ScheduleStatus.STARTING)
+ .put(TaskState.TASK_RUNNING, ScheduleStatus.RUNNING)
+ .put(TaskState.TASK_FINISHED, ScheduleStatus.FINISHED)
+ .put(TaskState.TASK_FAILED, ScheduleStatus.FAILED)
+ .put(TaskState.TASK_KILLED, ScheduleStatus.KILLED)
+ .put(TaskState.TASK_LOST, ScheduleStatus.LOST)
+ .build();
+
+ /**
+ * Converts a protobuf state to an internal schedule status.
+ *
+ * @param taskState Protobuf state.
+ * @return Equivalent thrift-generated state.
+ */
+ public static ScheduleStatus convertProtoState(TaskState taskState) {
+ ScheduleStatus status = STATE_TRANSLATION.get(taskState);
+ Preconditions.checkArgument(status != null, "Unrecognized task state " + taskState);
+ return status;
+ }
+
+ private static final Function<Protos.Attribute, String> ATTRIBUTE_NAME =
+ new Function<Protos.Attribute, String>() {
+ @Override public String apply(Protos.Attribute attr) {
+ return attr.getName();
+ }
+ };
+
+ /**
+ * Typedef to make anonymous implementation more concise.
+ */
+ private abstract static class AttributeConverter
+ implements Function<Entry<String, Collection<Protos.Attribute>>, Attribute> {
+ }
+
+ private static final Function<Protos.Attribute, String> VALUE_CONVERTER =
+ new Function<Protos.Attribute, String>() {
+ @Override public String apply(Protos.Attribute attribute) {
+ switch (attribute.getType()) {
+ case SCALAR:
+ return String.valueOf(attribute.getScalar().getValue());
+
+ case TEXT:
+ return attribute.getText().getValue();
+
+ default:
+ LOG.finest("Unrecognized attribute type:" + attribute.getType() + " , ignoring.");
+ return null;
+ }
+ }
+ };
+
+ private static final AttributeConverter ATTRIBUTE_CONVERTER = new AttributeConverter() {
+ @Override public Attribute apply(Entry<String, Collection<Protos.Attribute>> entry) {
+ // Convert values and filter any that were ignored.
+ return new Attribute(
+ entry.getKey(),
+ FluentIterable.from(entry.getValue())
+ .transform(VALUE_CONVERTER)
+ .filter(Predicates.notNull())
+ .toSet());
+ }
+ };
+
+ /**
+ * Converts protobuf attributes into thrift-generated attributes.
+ *
+ * @param offer Resource offer.
+ * @return Equivalent thrift host attributes.
+ */
+ public static HostAttributes getAttributes(Offer offer) {
+ // Group by attribute name.
+ Multimap<String, Protos.Attribute> valuesByName =
+ Multimaps.index(offer.getAttributesList(), ATTRIBUTE_NAME);
+
+ // TODO(William Farner): Include slave id.
+ return new HostAttributes(
+ offer.getHostname(),
+ FluentIterable.from(valuesByName.asMap().entrySet())
+ .transform(ATTRIBUTE_CONVERTER)
+ .toSet())
+ .setSlaveId(offer.getSlaveId().getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/base/JobKeys.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/JobKeys.java b/src/main/java/org/apache/aurora/scheduler/base/JobKeys.java
new file mode 100644
index 0000000..008e1cb
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/base/JobKeys.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.base;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Optional;
+import com.google.common.base.Strings;
+
+import com.twitter.aurora.gen.JobKey;
+import com.twitter.aurora.gen.TaskQuery;
+import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Utility class providing convenience functions relating to JobKeys.
+ */
+public final class JobKeys {
+ private JobKeys() {
+ // Utility class.
+ }
+
+ public static final Function<IJobConfiguration, IJobKey> FROM_CONFIG =
+ new Function<IJobConfiguration, IJobKey>() {
+ @Override public IJobKey apply(IJobConfiguration job) {
+ return job.getKey();
+ }
+ };
+
+ public static final Function<IJobKey, String> TO_ROLE =
+ new Function<IJobKey, String>() {
+ @Override public String apply(IJobKey jobKey) {
+ return jobKey.getRole();
+ }
+ };
+
+ public static final Function<IJobKey, String> TO_ENVIRONMENT =
+ new Function<IJobKey, String>() {
+ @Override public String apply(IJobKey jobKey) {
+ return jobKey.getEnvironment();
+ }
+ };
+
+ public static final Function<IJobKey, String> TO_JOB_NAME =
+ new Function<IJobKey, String>() {
+ @Override public String apply(IJobKey jobKey) {
+ return jobKey.getName();
+ }
+ };
+
+ public static final Function<IJobConfiguration, String> CONFIG_TO_ROLE =
+ Functions.compose(TO_ROLE, FROM_CONFIG);
+
+ /**
+ * Check that a jobKey struct is valid.
+ *
+ * @param jobKey The jobKey to validate.
+ * @return {@code true} if the jobKey validates.
+ */
+ public static boolean isValid(@Nullable IJobKey jobKey) {
+ return jobKey != null
+ && !Strings.isNullOrEmpty(jobKey.getRole())
+ && !Strings.isNullOrEmpty(jobKey.getEnvironment())
+ && !Strings.isNullOrEmpty(jobKey.getName());
+ }
+
+ /**
+ * Assert that a jobKey struct is valid.
+ *
+ * @param jobKey The key struct to validate.
+ * @return The validated jobKey argument.
+ * @throws IllegalArgumentException if the key struct fails to validate.
+ */
+ public static IJobKey assertValid(IJobKey jobKey) throws IllegalArgumentException {
+ checkArgument(isValid(jobKey));
+
+ return jobKey;
+ }
+
+ /**
+ * Attempt to create a valid JobKey from the given (role, environment, name) triple.
+ *
+ * @param role The job's role.
+ * @param environment The job's environment.
+ * @param name The job's name.
+ * @return A valid JobKey if it can be created.
+ * @throws IllegalArgumentException if the key fails to validate.
+ */
+ public static IJobKey from(String role, String environment, String name)
+ throws IllegalArgumentException {
+
+ IJobKey job = IJobKey.build(new JobKey()
+ .setRole(role)
+ .setEnvironment(environment)
+ .setName(name));
+ return assertValid(job);
+ }
+
+ /**
+ * Attempts to create a valid JobKey from the given task.
+ *
+ * @param task The task to create job key from.
+ * @return A valid JobKey if it can be created.
+ * @throws IllegalArgumentException if the key fails to validate.
+ */
+ public static IJobKey from(ITaskConfig task) throws IllegalArgumentException {
+ return from(task.getOwner().getRole(), task.getEnvironment(), task.getJobName());
+ }
+
+ /**
+ * Create a "/"-delimited String representation of a job key, suitable for logging but not
+ * necessarily suitable for use as a unique identifier.
+ *
+ * @param jobKey Key to represent.
+ * @return "/"-delimited representation of the key.
+ */
+ public static String toPath(IJobKey jobKey) {
+ return jobKey.getRole() + "/" + jobKey.getEnvironment() + "/" + jobKey.getName();
+ }
+
+ /**
+ * Create a "/"-delimited String representation of job key, suitable for logging but not
+ * necessarily suitable for use as a unique identifier.
+ *
+ * @param job Job to represent.
+ * @return "/"-delimited representation of the job's key.
+ */
+ public static String toPath(IJobConfiguration job) {
+ return toPath(job.getKey());
+ }
+
+ /**
+ * Attempt to extract a job key from the given query if it is scoped to a single job.
+ *
+ * @param query Query to extract the key from.
+ * @return A present if one can be extracted, absent otherwise.
+ */
+ public static Optional<IJobKey> from(Query.Builder query) {
+ if (Query.isJobScoped(query)) {
+ TaskQuery taskQuery = query.get();
+ return Optional.of(
+ from(taskQuery.getOwner().getRole(), taskQuery.getEnvironment(), taskQuery.getJobName()));
+
+ } else {
+ return Optional.absent();
+ }
+ }
+}
[06/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/observer/http/assets/jquery.pailer.js
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/observer/http/assets/jquery.pailer.js b/src/main/python/twitter/thermos/observer/http/assets/jquery.pailer.js
deleted file mode 100644
index 5962f9e..0000000
--- a/src/main/python/twitter/thermos/observer/http/assets/jquery.pailer.js
+++ /dev/null
@@ -1,317 +0,0 @@
-// A jQuery plugin for PAging and taILing data (i.e., a
-// 'PAILer'). Paging occurs when scrolling reaches the "top" and
-// tailing occurs when scrolling has reached the "bottom".
-
-// A 'read' function must be provided for reading the data (in
-// bytes). This function should expect an "options" object with the
-// fields 'offset' and 'length' set for reading the data. The result
-// from of the function should be a "promise" like value which has a
-// 'success' and 'error' callback which each take a function. An
-// object with at least two fields defined ('offset' and 'length') is
-// expected on success. A third field, 'data' may be provided if
-// 'length' is greater than 0. If the offset requested is greater than
-// the available offset, the result should be an object with the
-// 'offset' field set to the available offset (i.e., the length of the
-// data) and the 'length' field set to 0.
-
-// The plugin prepends, appends, and updates the "html" component of
-// the elements specified in the jQuery selector (e.g., doing
-// $('#data').pailer(...) means that data will be updated within
-// $('#data') via $('#data').prepend(...) and $('#data').append(...)
-// and $('#data').html(...) calls).
-
-// An indicator paragraph element (i.e., <p>) can be specified that
-// the plugin will write text to describing any status/errors that
-// have been encountered.
-
-// Data will automagically get truncated at some specified length,
-// configurable via the 'truncate-length' option. Likewise, the amount
-// of data paged in at a time can be configured via the 'page-size'
-// option.
-
-// Example:
-// HTML:
-// <div id="data" style="white-space:pre-wrap;"></div>
-//
-// <div style="position: absolute; left: 5px; top: 0px;">
-// <p id="indicator"></p>
-// </div>
-// Javascript:
-// $('#data').pailer({
-// 'read': function(options) {
-// var settings = $.extend({
-// 'offset': -1,
-// 'length': -1
-// }, options);
-// var url = '/url/for/data'
-// + '?offset=' + settings.offset
-// + '&length=' + settings.length;
-// return $.getJSON(url);
-// },
-// 'indicator': $('#indicator')
-// });
-
-(function($) {
- function Pailer(read, element, indicator, page_size, truncate_length) {
- var this_ = this;
-
- this_.read = read;
- this_.element = element;
- this_.indicator = indicator;
- this_.initialized = false;
- this_.start = -1;
- this_.end = -1;
- this_.paging = false;
- this_.tailing = true;
-
- page_size || $.error('Expecting page_size to be defined');
- truncate_length || $.error('Expecting truncate_length to be defined');
-
- this_.page_size = page_size;
- this_.truncate_length = truncate_length;
-
- this_.element.css('overflow', 'auto');
-
- this_.element.scroll(function () {
- var scrollTop = this_.element.scrollTop();
- var height = this_.element.height();
- var scrollHeight = this_.element[0].scrollHeight;
-
- if (scrollTop == 0) {
- this_.page();
- } else if (scrollTop + height >= scrollHeight) {
- if (!this_.tailing) {
- this_.tailing = true;
- this_.tail();
- }
- } else {
- this_.tailing = false;
- }
- });
- }
-
-
- Pailer.prototype.initialize = function() {
- var this_ = this;
-
- // Set an indicator while we load the data.
- this_.indicate('(LOADING)');
-
- this_.read({'offset': -1})
- .success(function(data) {
- this_.indicate('');
-
- // Get the last page of data.
- if (data.offset > this_.page_size) {
- this_.start = this_.end = data.offset - this_.page_size;
- } else {
- this_.start = this_.end = 0;
- }
-
- this_.initialized = true;
- this_.element.html('');
- setTimeout(function() { this_.tail(); }, 0);
- })
- .error(function() {
- this_.indicate('(FAILED TO INITIALIZE ... RETRYING)');
- setTimeout(function() {
- this_.indicate('');
- this_.initialize();
- }, 1000);
- });
- }
-
-
- Pailer.prototype.page = function() {
- var this_ = this;
-
- if (!this_.initialized) {
- return;
- }
-
- if (this_.paging) {
- return;
- }
-
- this_.paging = true;
- this_.indicate('(PAGING)');
-
- if (this_.start == 0) {
- this_.paging = false;
- this_.indicate('(AT BEGINNING OF FILE)');
- setTimeout(function() { this_.indicate(''); }, 1000);
- return;
- }
-
- var offset = this_.start - this_.page_size;
- var length = this_.page_size;
-
- if (offset < 0) {
- offset = 0;
- length = this_.start;
- }
-
- // Buffer the data in case what gets read is less than 'length'.
- var buffer = '';
-
- var read = function(offset, length) {
- this_.read({'offset': offset, 'length': length})
- .success(function(data) {
- if (data.length < length) {
- buffer += data.data;
- read(offset + data.length, length - data.length);
- } else if (data.length > 0) {
- this_.indicate('(PAGED)');
- setTimeout(function() { this_.indicate(''); }, 1000);
-
- // Prepend buffer onto data.
- data.offset -= buffer.length;
- data.data = buffer + data.data;
- data.length = data.data.length;
-
- // Truncate to the first newline (unless this is the beginning).
- if (data.offset != 0) {
- var index = data.data.indexOf('\n') + 1;
- data.offset += index;
- data.data = data.data.substring(index);
- data.length -= index;
- }
-
- this_.start = data.offset;
-
- var scrollTop = this_.element.scrollTop();
- var scrollHeight = this_.element[0].scrollHeight;
-
- this_.element.prepend(data.data);
-
- scrollTop += this_.element[0].scrollHeight - scrollHeight;
- this_.element.scrollTop(scrollTop);
-
- this_.paging = false;
- }
- })
- .error(function() {
- this_.indicate('(FAILED TO PAGE ... RETRYING)');
- setTimeout(function() {
- this_.indicate('');
- this_.page();
- }, 1000);
- });
- }
-
- read(offset, length);
- }
-
-
- Pailer.prototype.tail = function() {
- var this_ = this;
-
- if (!this_.initialized) {
- return;
- }
-
- this_.read({'offset': this_.end, 'length': this_.truncate_length})
- .success(function(data) {
- var scrollTop = this_.element.scrollTop();
- var height = this_.element.height();
- var scrollHeight = this_.element[0].scrollHeight;
-
- // Check if we are still at the bottom (since this event might
- // have fired before the scroll event has been dispatched).
- if (scrollTop + height < scrollHeight) {
- this_.tailing = false;
- return;
- }
-
- if (data.length > 0) {
- // Truncate to the first newline if this is the first time
- // (and we aren't reading from the beginning of the log).
- if (this_.start == this_.end && data.offset != 0) {
- var index = data.data.indexOf('\n') + 1;
- data.offset += index;
- data.data = data.data.substring(index);
- data.length -= index;
- this_.start = data.offset; // Adjust the actual start too!
- }
-
- this_.end = data.offset + data.length;
-
- this_.element.append(data.data);
-
- scrollTop += this_.element[0].scrollHeight - scrollHeight;
- this_.element.scrollTop(scrollTop);
-
- // Also, only if we're at the bottom, truncate data so that we
- // don't consume too much memory. TODO(benh): Only do
- // truncations if we've been at the bottom for a while.
- this_.truncate();
- }
-
- // Tail immediately if we got as much data as requested (since
- // this probably means we've waited around a while). The
- // alternative here would be to not get data in chunks, but the
- // potential issue here is that we might end up requesting GB of
- // log data at a time ... the right solution here might be to do
- // a request to determine the new ending offset and then request
- // the proper length.
- if (data.length == this_.truncate_length) {
- setTimeout(function() { this_.tail(); }, 0);
- } else {
- setTimeout(function() { this_.tail(); }, 1000);
- }
- })
- .error(function() {
- this_.indicate('(FAILED TO TAIL ... RETRYING)');
- this_.initialized = false;
- setTimeout(function() {
- this_.indicate('');
- this_.initialize();
- }, 1000);
- });
- }
-
-
- Pailer.prototype.indicate = function(text) {
- var this_ = this;
-
- if (this_.indicator) {
- this_.indicator.text(text);
- }
- }
-
-
- Pailer.prototype.truncate = function() {
- var this_ = this;
-
- var length = this_.element.html().length;
- if (length >= this_.truncate_length) {
- var index = length - this_.truncate_length;
- this_.start = this_.end - this_.truncate_length;
- this_.element.html(this_.element.html().substring(index));
- }
- }
-
- $.fn.pailer = function(options) {
- var settings = $.extend({
- 'read': function() { return {
- 'success': function() {},
- 'error': function(f) { f(); }
- }},
- 'page_size': 8 * 4096, // 8 "pages".
- 'truncate_length': 50000
- }, options);
-
- this.each(function() {
- var pailer = $.data(this, 'pailer');
- if (!pailer) {
- pailer = new Pailer(settings.read,
- $(this),
- settings.indicator,
- settings.page_size,
- settings.truncate_length);
- $.data(this, 'pailer', pailer);
- pailer.initialize();
- }
- });
- }
-})(jQuery);
[10/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/common/path.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/common/path.py b/src/main/python/twitter/thermos/common/path.py
deleted file mode 100644
index 28e07ac..0000000
--- a/src/main/python/twitter/thermos/common/path.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import os
-
-
-class TaskPath(object):
- """
- Handle the resolution / detection of the path structure for thermos tasks.
-
- This is used by the runner to determine where it should be dumping checkpoints and writing
- stderr/stdout, and by the observer to determine how to detect the running tasks on the system.
-
- Examples:
- pathspec = TaskPath(root = "/var/run/thermos")
- ^ substitution dictionary for DIR_TEMPLATE
-
-
- which template to acquire
- v
- pathspec.given(task_id = "12345-thermos-wickman-23", state='active').getpath("task_path")
- ^
- further substitutions DIR_TEMPLATE
-
-
- As a detection mechanism:
- path_glob = pathspec.given(task_id = "*").getpath(task_type)
- matching_paths = glob.glob(path_glob)
-
- path_re = pathspec.given(task_id = "(\S+)").getpath(task_type)
- path_re = re.compile(path_re)
-
- ids = []
- for path in matching_paths:
- matched_blobs = path_re.match(path).groups()
- ids.append(int(matched_blobs[0]))
- return ids
- """
-
- class UnknownPath(Exception): pass
- class UnderspecifiedPath(Exception): pass
-
- DEFAULT_CHECKPOINT_ROOT = "/var/run/thermos"
- KNOWN_KEYS = [ 'root', 'task_id', 'state', 'process', 'run', 'log_dir' ]
- LEGACY_KNOWN_KEYS = KNOWN_KEYS[:-1]
-
- DIR_TEMPLATE = {
- 'task_path': ['%(root)s', 'tasks', '%(state)s', '%(task_id)s'],
- 'checkpoint_path': ['%(root)s', 'checkpoints', '%(task_id)s'],
- 'runner_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'runner'],
- 'process_checkpoint': ['%(root)s', 'checkpoints', '%(task_id)s', 'coordinator.%(process)s'],
- 'process_logbase': ['%(log_dir)s'],
- 'process_logdir': ['%(log_dir)s', '%(process)s', '%(run)s']
- }
-
- LEGACY_DIR_TEMPLATE = DIR_TEMPLATE.copy()
- LEGACY_DIR_TEMPLATE.update(
- process_logbase = ['%(root)s', 'logs', '%(task_id)s'],
- process_logdir = ['%(root)s', 'logs', '%(task_id)s', '%(process)s', '%(run)s']
- )
-
- def __init__(self, **kw):
- self._filename = None
- # initialize with self-interpolating values
- if kw.get('root') is None:
- kw['root'] = self.DEFAULT_CHECKPOINT_ROOT
- # Before log_dir was added explicitly to RunnerHeader, it resolved to %(root)s/logs
- if kw.get('log_dir'):
- self._template, keys = self.DIR_TEMPLATE, self.KNOWN_KEYS
- else:
- self._template, keys = self.LEGACY_DIR_TEMPLATE, self.LEGACY_KNOWN_KEYS
- self._data = dict((key, '%%(%s)s' % key) for key in keys)
- self._data.update(kw)
-
- def given(self, **kw):
- """ Perform further interpolation of the templates given the kwargs """
- eval_dict = dict(self._data) # copy
- eval_dict.update(kw)
- tp = TaskPath(**eval_dict)
- tp._filename = self._filename
- return tp
-
- def with_filename(self, filename):
- """ Return a TaskPath with the specific filename appended to the end of the path """
- wp = TaskPath(**self._data)
- wp._filename = filename
- return wp
-
- def getpath(self, pathname):
- if pathname not in self._template:
- raise self.UnknownPath("Internal error, unknown id: %s" % pathname)
- path = self._template[pathname][:]
-
- if self._filename:
- path += [self._filename]
- path = os.path.join(*path)
- interpolated_path = path % self._data
- try:
- _ = interpolated_path % {}
- except KeyError:
- raise TaskPath.UnderspecifiedPath(
- "Tried to interpolate path with insufficient variables: %s as %s" % (
- pathname, interpolated_path))
- return interpolated_path
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/common/planner.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/common/planner.py b/src/main/python/twitter/thermos/common/planner.py
deleted file mode 100644
index a886dad..0000000
--- a/src/main/python/twitter/thermos/common/planner.py
+++ /dev/null
@@ -1,304 +0,0 @@
-"""Planners to schedule processes within Tasks.
-
-TaskPlanner:
- - a daemon process can depend upon a regular process
- - a regular process cannot depend upon a daemon process
- - a non-ephemeral process cannot depend upon an ephemeral process
-"""
-
-from collections import defaultdict, namedtuple
-import copy
-from functools import partial
-import sys
-import time
-
-class Planner(object):
- """
- Given a set of process names and a graph of dependencies between them, determine
- what can run predicated upon process completions.
- """
- class InvalidSchedule(Exception): pass
-
- @staticmethod
- def filter_runnable(processes, dependencies):
- return set(process for process in processes if not dependencies.get(process))
-
- @staticmethod
- def filter_dependencies(dependencies, given=frozenset()):
- """
- Provided a map of process => list of process :dependencies, and a set of satisfied
- prior processes in :given, return the new map of dependencies with priors removed.
- """
- dependencies = copy.deepcopy(dependencies)
- for process_set in dependencies.values():
- process_set -= given
- return dependencies
-
- @staticmethod
- def satisfiable(processes, dependencies):
- """
- Given a set of processes and a dependency map, determine if this is a consistent
- schedule without cycles.
- """
- processes = copy.copy(processes)
- dependencies = copy.deepcopy(dependencies)
-
- scheduling = True
- while scheduling:
- scheduling = False
- runnables = Planner.filter_runnable(processes, dependencies)
- if runnables:
- scheduling = True
- processes -= runnables
- dependencies = Planner.filter_dependencies(dependencies, given=runnables)
- return len(processes) == 0
-
- def __init__(self, processes, dependencies):
- self._processes = set(processes)
- self._dependencies = dict((process, set(dependencies.get(process, [])))
- for process in self._processes)
- if not Planner.satisfiable(self._processes, self._dependencies):
- raise Planner.InvalidSchedule("Cycles detected in the task schedule!")
- self._running = set()
- self._finished = set()
- self._failed = set()
-
- @property
- def runnable(self):
- return Planner.filter_runnable(self._processes - self._running - self._finished - self._failed,
- Planner.filter_dependencies(self._dependencies, given=self._finished))
-
- @property
- def processes(self):
- return set(self._processes)
-
- @property
- def running(self):
- return set(self._running)
-
- @property
- def finished(self):
- return set(self._finished)
-
- @property
- def failed(self):
- return set(self._failed)
-
- def reset(self, process):
- assert process in self._running
- assert process not in self._finished
- assert process not in self._failed
- self._running.discard(process)
-
- def set_running(self, process):
- assert process not in self._failed
- assert process not in self._finished
- assert process in self._running or process in self.runnable
- self._running.add(process)
-
- def set_finished(self, process):
- assert process in self._running
- assert process not in self._failed
- self._running.discard(process)
- self._finished.add(process)
-
- def set_failed(self, process):
- assert process in self._running
- assert process not in self._finished
- self._running.discard(process)
- self._failed.add(process)
-
- def is_complete(self):
- return self._finished.union(self._failed) == self._processes
-
-
-TaskAttributes = namedtuple('TaskAttributes', 'min_duration is_daemon max_failures is_ephemeral')
-
-class TaskPlanner(object):
- """
- A planner for the processes part of a Thermos task, taking into account ephemeral and daemon
- bits, in addition to duration restrictions [and failure limits?].
-
- is_daemon
- .----------------------------------------------------.
- | |
- | clock gate .----------------------. |
- | .---------------> | runnable && !waiting | |
- v | `----------------------' |
- .----------. | |
- | runnable | | set_running |
- `----------' v |
- ^ forget .---------. | !is_daemon .----------.
- `------------------------| running |--------------+------------> | finished |
- ^ `---------' add_success `----------'
- | |
- | under failure limit | add_failure
- `-----------------------------+
- | past failure limit
- v
- .--------.
- | failed |
- `--------'
- """
- InvalidSchedule = Planner.InvalidSchedule
- INFINITY = sys.float_info.max
- TOTAL_RUN_LIMIT = sys.maxsize
-
- @staticmethod
- def extract_dependencies(task, process_filter=None):
- """
- Construct a set of processes and the process dependencies from a Thermos Task.
- """
- process_map = dict((process.name().get(), process)
- for process in filter(process_filter, task.processes()))
- processes = set(process_map)
- dependencies = defaultdict(set)
- if task.has_constraints():
- for constraint in task.constraints():
- # handle process orders
- process_names = constraint.order().get()
- process_name_set = set(process_names)
- # either all process_names must be in processes or none should be
- if process_name_set.issubset(processes) == process_name_set.isdisjoint(processes):
- raise TaskPlanner.InvalidSchedule('Invalid process dependencies!')
- if not process_name_set.issubset(processes):
- continue
- for k in range(1, len(process_names)):
- pnk, pnk1 = process_names[k], process_names[k-1]
- if process_map[pnk1].daemon().get():
- raise TaskPlanner.InvalidSchedule(
- 'Process %s may not depend upon daemon process %s' % (pnk, pnk1))
- if not process_map[pnk].ephemeral().get() and process_map[pnk1].ephemeral().get():
- raise TaskPlanner.InvalidSchedule(
- 'Non-ephemeral process %s may not depend upon ephemeral process %s' % (pnk, pnk1))
- dependencies[pnk].add(pnk1)
- return (processes, dependencies)
-
- def __init__(self, task, clock=time, process_filter=None):
- self._filter = process_filter
- assert self._filter is None or callable(self._filter), (
- 'TaskPlanner must be given callable process filter.')
- self._planner = Planner(*self.extract_dependencies(task, self._filter))
- self._clock = clock
- self._last_terminal = {} # process => timestamp of last terminal state
- self._failures = defaultdict(int)
- self._successes = defaultdict(int)
- self._attributes = {}
- self._ephemerals = set(process.name().get() for process in task.processes()
- if (self._filter is None or self._filter(process)) and process.ephemeral().get())
-
- for process in filter(self._filter, task.processes()):
- self._attributes[process.name().get()] = TaskAttributes(
- is_daemon=bool(process.daemon().get()),
- is_ephemeral=bool(process.ephemeral().get()),
- max_failures=process.max_failures().get(),
- min_duration=process.min_duration().get())
-
- def get_wait(self, process, timestamp=None):
- now = timestamp if timestamp is not None else self._clock.time()
- if process not in self._last_terminal:
- return 0
- return self._attributes[process].min_duration - (now - self._last_terminal[process])
-
- def is_ready(self, process, timestamp=None):
- return self.get_wait(process, timestamp) <= 0
-
- def is_waiting(self, process, timestamp=None):
- return not self.is_ready(process, timestamp)
-
- @property
- def runnable(self):
- """A list of processes that are runnable w/o duration restrictions."""
- return self.runnable_at(self._clock.time())
-
- @property
- def waiting(self):
- """A list of processes that are waiting w/o duration restrictions."""
- return self.waiting_at(self._clock.time())
-
- def runnable_at(self, timestamp):
- return set(filter(partial(self.is_ready, timestamp=timestamp), self._planner.runnable))
-
- def waiting_at(self, timestamp):
- return set(filter(partial(self.is_waiting, timestamp=timestamp), self._planner.runnable))
-
- def min_wait(self, timestamp=None):
- """Return the current wait time for the next process to become runnable, 0 if something is ready
- immediately, or sys.float.max if there are no waiters."""
- if self.runnable_at(timestamp if timestamp is not None else self._clock.time()):
- return 0
- waits = [self.get_wait(waiter, timestamp) for waiter in self.waiting_at(timestamp)]
- return min(waits) if waits else self.INFINITY
-
- def set_running(self, process):
- self._planner.set_running(process)
-
- def add_failure(self, process, timestamp=None):
- """Increment the failure count of a process, and reset it to runnable if maximum number of
- failures has not been reached, or mark it as failed otherwise (ephemeral processes do not
- count towards the success of a task, and are hence marked finished instead)"""
- timestamp = timestamp if timestamp is not None else self._clock.time()
- self._last_terminal[process] = timestamp
- self._failures[process] += 1
- self.failure_transition(process)
-
- def has_reached_run_limit(self, process):
- return (self._successes[process] + self._failures[process]) >= self.TOTAL_RUN_LIMIT
-
- def failure_transition(self, process):
- if self.has_reached_run_limit(process):
- self._planner.set_failed(process)
- return
-
- if self._attributes[process].max_failures == 0 or (
- self._failures[process] < self._attributes[process].max_failures):
- self._planner.reset(process)
- elif self._attributes[process].is_ephemeral:
- self._planner.set_finished(process)
- else:
- self._planner.set_failed(process)
-
- def add_success(self, process, timestamp=None):
- """Reset a process to runnable if it is a daemon, or mark it as finished otherwise."""
- timestamp = timestamp if timestamp is not None else self._clock.time()
- self._last_terminal[process] = timestamp
- self._successes[process] += 1
- self.success_transition(process)
-
- def success_transition(self, process):
- if self.has_reached_run_limit(process):
- self._planner.set_failed(process)
- return
-
- if not self._attributes[process].is_daemon:
- self._planner.set_finished(process)
- else:
- self._planner.reset(process)
-
- def set_failed(self, process):
- """Force a process to be in failed state. E.g. kill -9 and you want it pinned failed."""
- self._planner.set_failed(process)
-
- def lost(self, process):
- """Mark a process as lost. This sets its runnable state back to the previous runnable
- state and does not increment its failure count."""
- self._planner.reset(process)
-
- def is_complete(self):
- """A task is complete if all ordinary processes are finished or failed (there may still be
- running ephemeral processes)"""
- terminals = self._planner.finished.union(self._planner.failed).union(self._ephemerals)
- return self._planner.processes == terminals
-
- # TODO(wickman) Should we consider subclassing again?
- @property
- def failed(self):
- return self._planner.failed
-
- @property
- def running(self):
- return self._planner.running
-
- @property
- def finished(self):
- return self._planner.finished
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/BUILD b/src/main/python/twitter/thermos/config/BUILD
deleted file mode 100644
index 57e7f0d..0000000
--- a/src/main/python/twitter/thermos/config/BUILD
+++ /dev/null
@@ -1,24 +0,0 @@
-import os
-
-python_library(
- name = 'schema',
- sources = globs('*.py'),
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('src/main/python/twitter/thermos:pystachio'),
- pants('src/main/python/twitter/thermos/common:planner')
- ],
-)
-
-python_library(
- name = 'config',
- dependencies = [
- pants(':schema'),
- pants('src/main/python/twitter/thermos/common'), # cover common:planner
- ],
- provides = setup_py(
- name = 'twitter.thermos.config',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- description = 'Thermos configuration schema and loader.',
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/__init__.py b/src/main/python/twitter/thermos/config/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/bin/config_load.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/bin/config_load.py b/src/main/python/twitter/thermos/config/bin/config_load.py
deleted file mode 100644
index c0680ec..0000000
--- a/src/main/python/twitter/thermos/config/bin/config_load.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import copy
-import sys
-import json
-import pprint
-
-from twitter.common import app
-from twitter.thermos.config.loader import ThermosConfigLoader
-
-def main(args):
- """
- Given .thermos configs, loads them and prints out information about them.
- """
-
- if len(args) == 0:
- app.help()
-
- for arg in args:
- print '\nparsing %s\n' % arg
- tc = ThermosConfigLoader.load(arg)
-
- for task_wrapper in tc.tasks():
- task = task_wrapper.task
- if not task.has_name():
- print 'Found unnamed task! Skipping...'
- continue
-
- print 'Task: %s [check: %s]' % (task.name(), task.check())
- if not task.processes():
- print ' No processes.'
- else:
- print ' Processes:'
- for proc in task.processes():
- print ' %s' % proc
-
- ports = task_wrapper.ports()
- if not ports:
- print ' No unbound ports.'
- else:
- print ' Ports:'
- for port in ports:
- print ' %s' % port
-
- print 'raw:'
- pprint.pprint(json.loads(task_wrapper.to_json()))
-
-app.set_usage("%s config1 config2 ..." % app.name())
-app.main()
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/bin/config_repl.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/bin/config_repl.py b/src/main/python/twitter/thermos/config/bin/config_repl.py
deleted file mode 100644
index 4a5c797..0000000
--- a/src/main/python/twitter/thermos/config/bin/config_repl.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from twitter.thermos.config.schema import *
-from code import interact
-interact('Thermos Config REPL', local=locals())
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/dsl.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/dsl.py b/src/main/python/twitter/thermos/config/dsl.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/loader.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/loader.py b/src/main/python/twitter/thermos/config/loader.py
deleted file mode 100644
index 54721c3..0000000
--- a/src/main/python/twitter/thermos/config/loader.py
+++ /dev/null
@@ -1,179 +0,0 @@
-import copy
-import json
-import os
-import re
-import textwrap
-
-from twitter.common.dirutil import safe_open
-from twitter.common.lang import Compatibility
-from twitter.thermos.common.planner import TaskPlanner
-from twitter.thermos.config.schema import Task
-
-from pystachio import Ref
-from pystachio.config import Config
-
-
-class PortExtractor(object):
- class InvalidPorts(Exception): pass
-
- @staticmethod
- def extract(obj):
- port_scope = Ref.from_address('thermos.ports')
- _, uninterp = obj.interpolate()
- ports = []
- for ref in uninterp:
- subscope = port_scope.scoped_to(ref)
- if subscope is not None:
- if not subscope.is_index():
- raise PortExtractor.InvalidPorts(
- 'Bad port specification "%s" (should be of form "thermos.ports[name]"' % ref.address())
- ports.append(subscope.action().value)
- return ports
-
-
-class ThermosProcessWrapper(object):
- # >=1 characters && anything but NULL and '/'
- VALID_PROCESS_NAME_RE = re.compile(r'^[^./][^/]*$')
- class InvalidProcess(Exception): pass
-
- def __init__(self, process):
- self._process = process
-
- def ports(self):
- try:
- return PortExtractor.extract(self._process)
- except PortExtractor.InvalidPorts:
- raise self.InvalidProcess('Process has invalid ports scoping!')
-
- @staticmethod
- def assert_valid_process_name(name):
- if not ThermosProcessWrapper.VALID_PROCESS_NAME_RE.match(name):
- raise ThermosProcessWrapper.InvalidProcess('Invalid process name: %s' % name)
-
-
-class ThermosTaskWrapper(object):
- class InvalidTask(Exception): pass
-
- def __init__(self, task, bindings=None, strict=True):
- if bindings:
- task = task.bind(*bindings)
- if not task.check().ok() and strict:
- raise ThermosTaskWrapper.InvalidTask(task.check().message())
- self._task = task
-
- @property
- def task(self):
- return self._task
-
- def ports(self):
- ti, _ = self._task.interpolate()
- ports = set()
- if ti.has_processes():
- for process in ti.processes():
- try:
- ports.update(ThermosProcessWrapper(process).ports())
- except ThermosProcessWrapper.InvalidProcess:
- raise self.InvalidTask('Task has invalid process: %s' % process)
- return ports
-
- def to_json(self):
- return json.dumps(self._task.get())
-
- def to_file(self, filename):
- ti, _ = self._task.interpolate()
- with safe_open(filename, 'w') as fp:
- json.dump(ti.get(), fp)
-
- @staticmethod
- def from_file(filename, **kw):
- try:
- with safe_open(filename) as fp:
- task = Task.json_load(fp)
- return ThermosTaskWrapper(task, **kw)
- except Exception as e:
- return None
-
-
-# TODO(wickman) These should be validators pushed onto ThermosConfigLoader.plugins
-class ThermosTaskValidator(object):
- class InvalidTaskError(Exception): pass
-
- @classmethod
- def assert_valid_task(cls, task):
- cls.assert_valid_names(task)
- cls.assert_typecheck(task)
- cls.assert_valid_plan(task)
-
- @classmethod
- def assert_valid_plan(cls, task):
- try:
- TaskPlanner(task, process_filter=lambda proc: proc.final().get() == False)
- TaskPlanner(task, process_filter=lambda proc: proc.final().get() == True)
- except TaskPlanner.InvalidSchedule as e:
- raise cls.InvalidTaskError('Task has invalid plan: %s' % e)
-
- @classmethod
- def assert_valid_names(cls, task):
- for process in task.processes():
- name = process.name().get()
- try:
- ThermosProcessWrapper.assert_valid_process_name(name)
- except ThermosProcessWrapper.InvalidProcess as e:
- raise cls.InvalidTaskError('Task has invalid process: %s' % e)
-
- @classmethod
- def assert_typecheck(cls, task):
- typecheck = task.check()
- if not typecheck.ok():
- raise cls.InvalidTaskError('Failed to fully evaluate task: %s' %
- typecheck.message())
-
- @classmethod
- def assert_valid_ports(cls, task, portmap):
- for port in ThermosTaskWrapper(task).ports():
- if port not in portmap:
- raise cls.InvalidTaskError('Task requires unbound port %s!' % port)
-
- @classmethod
- def assert_same_task(cls, spec, task):
- active_task = spec.given(state='active').getpath('task_path')
- if os.path.exists(active_task):
- task_on_disk = ThermosTaskWrapper.from_file(active_task)
- if not task_on_disk or task_on_disk.task != task:
- raise cls.InvalidTaskError('Task differs from on disk copy: %r vs %r' % (
- task_on_disk.task if task_on_disk else None, task))
-
-
-class ThermosConfigLoader(object):
- SCHEMA = textwrap.dedent("""
- from pystachio import *
- from twitter.thermos.config.schema import *
-
- __TASKS = []
-
- def export(task):
- __TASKS.append(Task(task) if isinstance(task, dict) else task)
- """)
-
- @classmethod
- def load(cls, loadable, **kw):
- config = Config(loadable, schema=cls.SCHEMA)
- return cls(ThermosTaskWrapper(task, **kw) for task in config.environment['__TASKS'])
-
- @classmethod
- def load_json(cls, filename, **kw):
- tc = cls()
- task = ThermosTaskWrapper.from_file(filename, **kw)
- if task:
- ThermosTaskValidator.assert_valid_task(task.task())
- tc.add_task(task)
- return tc
-
- def __init__(self, exported_tasks=None):
- self._exported_tasks = exported_tasks or []
-
- def add_task(self, task):
- self._exported_tasks.append(task)
-
- def tasks(self):
- return self._exported_tasks
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/schema.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/schema.py b/src/main/python/twitter/thermos/config/schema.py
deleted file mode 100644
index 2ee38a1..0000000
--- a/src/main/python/twitter/thermos/config/schema.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .schema_base import *
-from .schema_helpers import *
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/schema_base.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/schema_base.py b/src/main/python/twitter/thermos/config/schema_base.py
deleted file mode 100644
index a47cfdf..0000000
--- a/src/main/python/twitter/thermos/config/schema_base.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from pystachio import (
- Boolean,
- Default,
- Empty,
- Float,
- Integer,
- List,
- Map,
- Required,
- String,
- Struct
-)
-
-
-# Define constants for resources
-BYTES = 1
-KB = 1024 * BYTES
-MB = 1024 * KB
-GB = 1024 * MB
-TB = 1024 * GB
-
-
-class ThermosContext(Struct):
- # TODO(wickman) Move the underlying replacement mechanism to %port% replacements
- ports = Map(String, Integer)
-
- # TODO(wickman) Move the underlying replacement mechanism to %task_id%
- task_id = String
-
- # TODO(wickman) Move underlying mechanism to %user%
- user = String
-
-
-class Resources(Struct):
- cpu = Required(Float)
- ram = Required(Integer)
- disk = Required(Integer)
-
-
-class Constraint(Struct):
- order = List(String)
-
-
-class Process(Struct):
- cmdline = Required(String)
- name = Required(String)
-
- # This is currently unused but reserved for future use by Thermos.
- resources = Resources
-
- # optionals
- max_failures = Default(Integer, 1) # maximum number of failed process runs
- # before process is failed.
- daemon = Default(Boolean, False)
- ephemeral = Default(Boolean, False)
- min_duration = Default(Integer, 5) # integer seconds
- final = Default(Boolean, False) # if this process should be a finalizing process
- # that should always be run after regular processes
-
-
-class Task(Struct):
- name = Default(String, '{{processes[0].name}}')
- processes = List(Process)
-
- # optionals
- constraints = Default(List(Constraint), [])
- resources = Resources
- max_failures = Default(Integer, 1) # maximum number of failed processes before task is failed.
- max_concurrency = Default(Integer, 0) # 0 is infinite concurrency.
- # > 0 is max concurrent processes.
- finalization_wait = Default(Integer, 30) # the amount of time in seconds we allocate to run the
- # finalization schedule.
-
- # TODO(jon): remove/replace with proper solution to MESOS-3546
- user = String
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/config/schema_helpers.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/config/schema_helpers.py b/src/main/python/twitter/thermos/config/schema_helpers.py
deleted file mode 100644
index 502de5a..0000000
--- a/src/main/python/twitter/thermos/config/schema_helpers.py
+++ /dev/null
@@ -1,247 +0,0 @@
-"""Helpers for composing Thermos workflows."""
-import itertools
-
-from twitter.common.lang import Compatibility
-
-from .schema_base import (
- Constraint,
- GB,
- Process,
- Resources,
- Task,
-)
-
-from pystachio import Empty, List
-
-
-__all__ = (
- # shorthand for process ordering constraint
- 'order',
-
- # task combinators
- 'combine_tasks', # merge N tasks in parallel
- 'concat_tasks', # serialize N tasks
-
- # options helpers
- 'java_options',
- 'python_options',
-
- # the automatically-sequential version of a task
- 'SequentialTask',
-
- # create a simple task from a command line + name
- 'SimpleTask',
-
- # helper classes
- 'Options',
- 'Processes',
- 'Tasks',
- 'Units',
-)
-
-
-class Units(object):
- """Helpers for base units of Tasks and Processes."""
-
- @classmethod
- def optional_resources(cls, resources):
- return Resources() if resources is Empty else resources
-
- @classmethod
- def resources_sum(cls, *resources):
- """Add two Resources objects together."""
- def add_unit(f1, f2):
- return (0 if f1 is Empty else f1.get()) + (0 if f2 is Empty else f2.get())
-
- def add(r1, r2):
- return Resources(cpu=add_unit(r1.cpu(), r2.cpu()),
- ram=add_unit(r1.ram(), r2.ram()),
- disk=add_unit(r1.disk(), r2.disk()))
-
- return reduce(add, map(cls.optional_resources, resources), Resources(cpu=0, ram=0, disk=0))
-
- @classmethod
- def resources_max(cls, resources):
- """Return a Resource object that is the maximum of the inputs along each
- resource dimension."""
- def max_unit(f1, f2):
- return max(0 if f1 is Empty else f1.get(), 0 if f2 is Empty else f2.get())
-
- def resource_max(r1, r2):
- return Resources(cpu=max_unit(r1.cpu(), r2.cpu()),
- ram=max_unit(r1.ram(), r2.ram()),
- disk=max_unit(r1.disk(), r2.disk()))
-
- return reduce(resource_max,
- map(cls.optional_resources, resources), Resources(cpu=0, ram=0, disk=0))
-
- @classmethod
- def processes_merge(cls, tasks):
- """Return a deduped list of the processes from all tasks."""
- return list(set(itertools.chain.from_iterable(task.processes() for task in tasks)))
-
- @classmethod
- def constraints_merge(cls, tasks):
- """Return a deduped list of the constraints from all tasks."""
- return list(set(itertools.chain.from_iterable(task.constraints() for task in tasks)))
-
-
-class Processes(object):
- """Helper class for Process objects."""
-
- @classmethod
- def _process_name(cls, process):
- if isinstance(process, Process):
- return process.name()
- elif isinstance(process, Compatibility.string):
- return process
- raise ValueError("Unknown value for process order: %s" % repr(process))
-
- @classmethod
- def order(cls, *processes):
- """Given a list of processes, return the list of constraints that keeps them in order, e.g.
- order(p1, p2, p3) => [Constraint(order=[p1.name(), p2.name(), p3.name()])].
-
- Similarly, concatenation operations are valid, i.e.
- order(p1, p2) + order(p2, p3) <=> order(p1, p2, p3)
- """
- return [Constraint(order=[cls._process_name(p) for p in processes])]
-
-
-class Tasks(object):
- """Helper class for Task objects."""
-
- SIMPLE_CPU = 1.0
- SIMPLE_RAM = 1 * GB
- SIMPLE_DISK = 1 * GB
-
- @classmethod
- def _combine_processes(cls, *tasks):
- """Given multiple tasks, merge their processes together, retaining the identity of the first
- task."""
- if len(tasks) == 0:
- return Task()
- head_task = tasks[-1]
- return head_task(processes=Units.processes_merge(tasks))
-
- @classmethod
- def combine(cls, *tasks, **kw):
- """Given multiple tasks, return a Task that runs all processes in parallel."""
- if len(tasks) == 0:
- return Task()
- base = cls._combine_processes(*tasks)
- return base(
- resources=Units.resources_sum(*(task.resources() for task in tasks)),
- constraints=Units.constraints_merge(tasks),
- **kw
- )
-
- @classmethod
- def concat(cls, *tasks, **kw):
- """Given tasks T1...TN, return a single Task that runs all processes such that
- all processes in Tk run before any process in Tk+1."""
- if len(tasks) == 0:
- return Task()
- base = cls._combine_processes(*tasks)
- base = base(resources=Units.resources_max(task.resources() for task in tasks))
- base_constraints = Units.constraints_merge(tasks)
- # TODO(wickman) be smarter about this in light of existing constraints
- for (t1, t2) in zip(tasks[0:-1], tasks[1:]):
- for p1 in t1.processes():
- for p2 in t2.processes():
- if p1 != p2:
- base_constraints.extend(Processes.order(p1, p2))
- return base(constraints=base_constraints, **kw)
-
- @classmethod
- def simple(cls, name, command):
- """Create a usable Task from a provided name + command line and a default set of resources"""
- return Task(
- name=name,
- processes=[Process(name=name, cmdline=command)],
- resources=Resources(cpu=cls.SIMPLE_CPU,
- ram=cls.SIMPLE_RAM,
- disk=cls.SIMPLE_DISK))
-
- @classmethod
- def sequential(cls, task):
- """Add a constraint that makes all processes within a task run sequentially."""
- def maybe_constrain(task):
- return {'constraints': order(*task.processes())} if task.processes() is not Empty else {}
- if task.constraints() is Empty or task.constraints() == List(Constraint)([]):
- return task(**maybe_constrain(task))
- raise ValueError('Cannot turn a Task with existing constraints into a SequentialTask!')
-
-
-class Options(object):
- """Helper class for constructing command-line arguments."""
-
- @classmethod
- def render_option(cls, short_prefix, long_prefix, option, value=None):
- option = '%s%s' % (short_prefix if len(option) == 1 else long_prefix, option)
- return '%s %s' % (option, value) if value else option
-
- @classmethod
- def render_options(cls, short_prefix, long_prefix, *options, **kw_options):
- renders = []
-
- for option in options:
- if isinstance(option, Compatibility.string):
- renders.append(cls.render_option(short_prefix, long_prefix, option))
- elif isinstance(option, dict):
- # preserve order in case option is an OrderedDict, rather than recursing with **option
- for argument, value in option.items():
- renders.append(cls.render_option(short_prefix, long_prefix, argument, value))
- else:
- raise ValueError('Got an unexpected argument to render_options: %s' % repr(option))
-
- for argument, value in kw_options.items():
- renders.append(cls.render_option(short_prefix, long_prefix, argument, value))
-
- return renders
-
- @classmethod
- def java(cls, *options, **kw_options):
- """
- Given a set of arguments, keyword arguments or dictionaries, render
- command-line parameters accordingly. For example:
-
- java_options('a', 'b') == '-a -b'
- java_options({
- 'a': 23,
- 'b': 'foo'
- }) == '-a 23 -b foo'
- java_options(a=23, b='foo') == '-a 23 -b foo'
- """
- return ' '.join(cls.render_options('-', '-', *options, **kw_options))
-
- @classmethod
- def python(cls, *options, **kw_options):
- """
- Given a set of arguments, keyword arguments or dictionaries, render
- command-line parameters accordingly. Single letter parameters are
- rendered with single '-'. For example:
-
- python_options('a', 'boo') == '-a --boo'
- python_options({
- 'a': 23,
- 'boo': 'foo'
- }) == '-a 23 --boo foo'
- python_options(a=23, boo='foo') == '-a 23 --boo foo'
- """
- return ' '.join(cls.render_options('-', '--', *options, **kw_options))
-
-
-def SimpleTask(name, command):
- """A simple command-line Task with default resources"""
- return Tasks.simple(name, command)
-
-def SequentialTask(*args, **kw):
- """A Task whose processes are always sequential."""
- return Tasks.sequential(Task(*args, **kw))
-
-python_options = Options.python
-java_options = Options.java
-combine_tasks = Tasks.combine
-concat_tasks = Tasks.concat
-order = Processes.order
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/core/BUILD
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/core/BUILD b/src/main/python/twitter/thermos/core/BUILD
deleted file mode 100644
index d260112..0000000
--- a/src/main/python/twitter/thermos/core/BUILD
+++ /dev/null
@@ -1,88 +0,0 @@
-import os
-
-python_library(
- name = 'helper',
- sources = ['helper.py'],
- dependencies = [
- pants('src/main/python/twitter/thermos:psutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'inspector',
- sources = ['inspector.py'],
- dependencies = [
- pants(':muxer'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'muxer',
- sources = ['muxer.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'process',
- sources = ['process.py'],
- dependencies = [
- pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/lang'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'runner',
- sources = ['__init__.py', 'runner.py'],
- dependencies = [
- pants(':helper'),
- pants(':muxer'),
- pants(':process'),
- pants('aurora/twitterdeps/src/python/twitter/common/dirutil'),
- pants('aurora/twitterdeps/src/python/twitter/common/log'),
- pants('aurora/twitterdeps/src/python/twitter/common/quantity'),
- pants('aurora/twitterdeps/src/python/twitter/common/recordio:recordio-thrift'),
- pants('src/main/python/twitter/thermos:psutil'),
- pants('src/main/python/twitter/thermos/common:ckpt'),
- pants('src/main/python/twitter/thermos/common:path'),
- pants('src/main/python/twitter/thermos/common:planner'),
- pants('src/main/python/twitter/thermos/config:schema'),
- pants('src/main/thrift/com/twitter/thermos:py-thrift'),
- ]
-)
-
-python_library(
- name = 'core',
- dependencies = [
- pants(':inspector'),
- pants(':runner'),
-
- # covering libs
- pants('src/main/python/twitter/thermos/common'),
- pants('src/main/python/twitter/thermos/config'),
- ],
- provides = setup_py(
- name = 'twitter.thermos.core',
- version = open(os.path.join(get_buildroot(), '.auroraversion')).read().strip().lower(),
- description = 'The Thermos core state machine.',
- )
-)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/core/__init__.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/core/__init__.py b/src/main/python/twitter/thermos/core/__init__.py
deleted file mode 100644
index de40ea7..0000000
--- a/src/main/python/twitter/thermos/core/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__import__('pkg_resources').declare_namespace(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/core/helper.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/core/helper.py b/src/main/python/twitter/thermos/core/helper.py
deleted file mode 100644
index 3587a85..0000000
--- a/src/main/python/twitter/thermos/core/helper.py
+++ /dev/null
@@ -1,387 +0,0 @@
-from contextlib import closing
-import errno
-import os
-import signal
-import time
-
-from twitter.common import log
-from twitter.common.dirutil import lock_file, safe_mkdir
-from twitter.common.quantity import Amount, Time
-from twitter.common.recordio import ThriftRecordWriter
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-from twitter.thermos.common.path import TaskPath
-
-from gen.twitter.thermos.ttypes import (
- ProcessState,
- ProcessStatus,
- RunnerCkpt,
- TaskState,
- TaskStatus)
-
-import psutil
-
-
-class TaskKiller(object):
- """
- Task killing interface.
- """
-
- def __init__(self, task_id, checkpoint_root):
- self._task_id = task_id
- self._checkpoint_root = checkpoint_root
-
- def kill(self, force=True):
- TaskRunnerHelper.kill(self._task_id, self._checkpoint_root, force=force,
- terminal_status=TaskState.KILLED)
-
- def lose(self, force=True):
- TaskRunnerHelper.kill(self._task_id, self._checkpoint_root, force=force,
- terminal_status=TaskState.LOST)
-
-
-class TaskRunnerHelper(object):
- """
- TaskRunner helper methods that can be operated directly upon checkpoint
- state. These operations do not require knowledge of the underlying
- task.
-
- TaskRunnerHelper is sort of a mishmash of "checkpoint-only" operations and
- the "Process Platform" stuff that started to get pulled into process.py
-
- This really needs some hard design thought to see if it can be extracted out
- even further.
- """
- class Error(Exception): pass
- class PermissionError(Error): pass
-
- # Maximum drift between when the system says a task was forked and when we checkpointed
- # its fork_time (used as a heuristic to determine a forked task is really ours instead of
- # a task with coincidentally the same PID but just wrapped around.)
- MAX_START_TIME_DRIFT = Amount(10, Time.SECONDS)
-
- @staticmethod
- def get_actual_user():
- import getpass, pwd
- try:
- pwd_entry = pwd.getpwuid(os.getuid())
- except KeyError:
- return getpass.getuser()
- return pwd_entry[0]
-
- @staticmethod
- def process_from_name(task, process_name):
- if task.has_processes():
- for process in task.processes():
- if process.name().get() == process_name:
- return process
- return None
-
- @classmethod
- def this_is_really_our_pid(cls, process, current_user, start_time):
- """
- A heuristic to make sure that this is likely the pid that we own/forked. Necessary
- because of pid-space wrapping. We don't want to go and kill processes we don't own,
- especially if the killer is running as root.
-
- process: psutil.Process representing the process to check
- current_user: user expected to own the process
- start_time: time at which it's expected the process has started
-
- Raises:
- psutil.NoSuchProcess - if the Process supplied no longer exists
- """
- if process.username != current_user:
- log.info("Expected pid %s to be ours but the pid user is %s and we're %s" % (
- process.pid, process.username, current_user))
- return False
-
- if abs(start_time - process.create_time) >= cls.MAX_START_TIME_DRIFT.as_(Time.SECONDS):
- log.info("Expected pid %s start time to be %s but it's %s" % (
- process.pid, start_time, process.create_time))
- return False
-
- return True
-
- @classmethod
- def scan_process(cls, state, process_name):
- """
- Given a RunnerState and a process_name, return the following:
- (coordinator pid, process pid, process tree)
- (int or None, int or None, set)
-
- """
- process_run = state.processes[process_name][-1]
- process_owner = state.header.user
-
- coordinator_pid, pid, tree = None, None, set()
-
- if process_run.coordinator_pid:
- try:
- coordinator_process = psutil.Process(process_run.coordinator_pid)
- if cls.this_is_really_our_pid(coordinator_process, process_owner, process_run.fork_time):
- coordinator_pid = process_run.coordinator_pid
- except psutil.NoSuchProcess:
- log.info(' Coordinator %s [pid: %s] completed.' % (process_run.process,
- process_run.coordinator_pid))
- except psutil.Error as err:
- log.warning(' Error gathering information on pid %s: %s' % (process_run.coordinator_pid,
- err))
-
- if process_run.pid:
- try:
- process = psutil.Process(process_run.pid)
- if cls.this_is_really_our_pid(process, process_owner, process_run.start_time):
- pid = process.pid
- except psutil.NoSuchProcess:
- log.info(' Process %s [pid: %s] completed.' % (process_run.process, process_run.pid))
- except psutil.Error as err:
- log.warning(' Error gathering information on pid %s: %s' % (process_run.pid, err))
- else:
- if pid:
- try:
- tree = set(proc.pid for proc in process.get_children(recursive=True))
- except psutil.Error:
- log.warning(' Error gathering information on children of pid %s' % pid)
-
- return (coordinator_pid, pid, tree)
-
- @classmethod
- def scantree(cls, state):
- """
- Scan the process tree associated with the provided task state.
-
- Returns a dictionary of process name => (coordinator pid, pid, pid children)
- If the coordinator is no longer active, coordinator pid will be None. If the
- forked process is no longer active, pid will be None and its children will be
- an empty set.
- """
- return dict((process_name, cls.scan_process(state, process_name))
- for process_name in state.processes)
-
- @classmethod
- def safe_signal(cls, pid, sig=signal.SIGTERM):
- try:
- os.kill(pid, sig)
- except OSError as e:
- if e.errno not in (errno.ESRCH, errno.EPERM):
- log.error('Unexpected error in os.kill: %s' % e)
- except Exception as e:
- log.error('Unexpected error in os.kill: %s' % e)
-
- @classmethod
- def terminate_pid(cls, pid):
- cls.safe_signal(pid, signal.SIGTERM)
-
- @classmethod
- def kill_pid(cls, pid):
- cls.safe_signal(pid, signal.SIGKILL)
-
- @classmethod
- def kill_group(cls, pgrp):
- cls.safe_signal(-pgrp, signal.SIGKILL)
-
- @classmethod
- def _get_process_tuple(cls, state, process_name):
- assert process_name in state.processes and len(state.processes[process_name]) > 0
- return cls.scan_process(state, process_name)
-
- @classmethod
- def _get_coordinator_group(cls, state, process_name):
- assert process_name in state.processes and len(state.processes[process_name]) > 0
- return state.processes[process_name][-1].coordinator_pid
-
- @classmethod
- def terminate_process(cls, state, process_name):
- log.debug('TaskRunnerHelper.terminate_process(%s)' % process_name)
- _, pid, _ = cls._get_process_tuple(state, process_name)
- if pid:
- log.debug(' => SIGTERM pid %s' % pid)
- cls.terminate_pid(pid)
- return bool(pid)
-
- @classmethod
- def kill_process(cls, state, process_name):
- log.debug('TaskRunnerHelper.kill_process(%s)' % process_name)
- coordinator_pgid = cls._get_coordinator_group(state, process_name)
- coordinator_pid, pid, tree = cls._get_process_tuple(state, process_name)
- # This is super dangerous. TODO(wickman) Add a heuristic that determines
- # that 1) there are processes that currently belong to this process group
- # and 2) those processes have inherited the coordinator checkpoint filehandle
- # This way we validate that it is in fact the process group we expect.
- if coordinator_pgid:
- log.debug(' => SIGKILL coordinator group %s' % coordinator_pgid)
- cls.kill_group(coordinator_pgid)
- if coordinator_pid:
- log.debug(' => SIGKILL coordinator %s' % coordinator_pid)
- cls.kill_pid(coordinator_pid)
- if pid:
- log.debug(' => SIGKILL pid %s' % pid)
- cls.kill_pid(pid)
- for child in tree:
- log.debug(' => SIGKILL child %s' % child)
- cls.kill_pid(child)
- return bool(coordinator_pid or pid or tree)
-
- @classmethod
- def kill_runner(cls, state):
- log.debug('TaskRunnerHelper.kill_runner()')
- if not state or not state.statuses:
- raise cls.Error('Could not read state!')
- pid = state.statuses[-1].runner_pid
- if pid == os.getpid():
- raise cls.Error('Unwilling to commit seppuku.')
- try:
- os.kill(pid, signal.SIGKILL)
- return True
- except OSError as e:
- if e.errno == errno.EPERM:
- # Permission denied
- return False
- elif e.errno == errno.ESRCH:
- # pid no longer exists
- return True
- raise
-
- @classmethod
- def open_checkpoint(cls, filename, force=False, state=None):
- """
- Acquire a locked checkpoint stream.
- """
- safe_mkdir(os.path.dirname(filename))
- fp = lock_file(filename, "a+")
- if fp in (None, False):
- if force:
- log.info('Found existing runner, forcing leadership forfeit.')
- state = state or CheckpointDispatcher.from_file(filename)
- if cls.kill_runner(state):
- log.info('Successfully killed leader.')
- # TODO(wickman) Blocking may not be the best idea here. Perhaps block up to
- # a maximum timeout. But blocking is necessary because os.kill does not immediately
- # release the lock if we're in force mode.
- fp = lock_file(filename, "a+", blocking=True)
- else:
- log.error('Found existing runner, cannot take control.')
- if fp in (None, False):
- raise cls.PermissionError('Could not open locked checkpoint: %s, lock_file = %s' %
- (filename, fp))
- ckpt = ThriftRecordWriter(fp)
- ckpt.set_sync(True)
- return ckpt
-
- @classmethod
- def kill(cls, task_id, checkpoint_root, force=False,
- terminal_status=TaskState.KILLED, clock=time):
- """
- An implementation of Task killing that doesn't require a fully hydrated TaskRunner object.
- Terminal status must be either KILLED or LOST state.
- """
- if terminal_status not in (TaskState.KILLED, TaskState.LOST):
- raise cls.Error('terminal_status must be KILLED or LOST (got %s)' %
- TaskState._VALUES_TO_NAMES.get(terminal_status) or terminal_status)
- pathspec = TaskPath(root=checkpoint_root, task_id=task_id)
- checkpoint = pathspec.getpath('runner_checkpoint')
- state = CheckpointDispatcher.from_file(checkpoint)
-
- if state is None or state.header is None or state.statuses is None:
- if force:
- log.error('Task has uninitialized TaskState - forcibly finalizing')
- cls.finalize_task(pathspec)
- return
- else:
- log.error('Cannot update states in uninitialized TaskState!')
- return
-
- ckpt = cls.open_checkpoint(checkpoint, force=force, state=state)
-
- def write_task_state(state):
- update = TaskStatus(state=state, timestamp_ms=int(clock.time() * 1000),
- runner_pid=os.getpid(), runner_uid=os.getuid())
- ckpt.write(RunnerCkpt(task_status=update))
-
- def write_process_status(status):
- ckpt.write(RunnerCkpt(process_status=status))
-
- if cls.is_task_terminal(state.statuses[-1].state):
- log.info('Task is already in terminal state! Finalizing.')
- cls.finalize_task(pathspec)
- return
-
- with closing(ckpt):
- write_task_state(TaskState.ACTIVE)
- for process, history in state.processes.items():
- process_status = history[-1]
- if not cls.is_process_terminal(process_status.state):
- if cls.kill_process(state, process):
- write_process_status(ProcessStatus(process=process,
- state=ProcessState.KILLED, seq=process_status.seq + 1, return_code=-9,
- stop_time=clock.time()))
- else:
- if process_status.state is not ProcessState.WAITING:
- write_process_status(ProcessStatus(process=process,
- state=ProcessState.LOST, seq=process_status.seq + 1))
- write_task_state(terminal_status)
- cls.finalize_task(pathspec)
-
- @classmethod
- def reap_children(cls):
- pids = set()
-
- while True:
- try:
- pid, status, rusage = os.wait3(os.WNOHANG)
- if pid == 0:
- break
- pids.add(pid)
- log.debug('Detected terminated process: pid=%s, status=%s, rusage=%s' % (
- pid, status, rusage))
- except OSError as e:
- if e.errno != errno.ECHILD:
- log.warning('Unexpected error when calling waitpid: %s' % e)
- break
-
- return pids
-
- TERMINAL_PROCESS_STATES = frozenset([
- ProcessState.SUCCESS,
- ProcessState.KILLED,
- ProcessState.FAILED,
- ProcessState.LOST])
-
- TERMINAL_TASK_STATES = frozenset([
- TaskState.SUCCESS,
- TaskState.FAILED,
- TaskState.KILLED,
- TaskState.LOST])
-
- @classmethod
- def is_process_terminal(cls, process_status):
- return process_status in cls.TERMINAL_PROCESS_STATES
-
- @classmethod
- def is_task_terminal(cls, task_status):
- return task_status in cls.TERMINAL_TASK_STATES
-
- @classmethod
- def initialize_task(cls, spec, task):
- active_task = spec.given(state='active').getpath('task_path')
- finished_task = spec.given(state='finished').getpath('task_path')
- is_active, is_finished = os.path.exists(active_task), os.path.exists(finished_task)
- if is_finished:
- raise cls.Error('Cannot initialize task with "finished" record!')
- if not is_active:
- safe_mkdir(os.path.dirname(active_task))
- with open(active_task, 'w') as fp:
- fp.write(task)
-
- @classmethod
- def finalize_task(cls, spec):
- active_task = spec.given(state='active').getpath('task_path')
- finished_task = spec.given(state='finished').getpath('task_path')
- is_active, is_finished = os.path.exists(active_task), os.path.exists(finished_task)
- if not is_active:
- raise cls.Error('Cannot finalize task with no "active" record!')
- elif is_finished:
- raise cls.Error('Cannot finalize task with "finished" record!')
- safe_mkdir(os.path.dirname(finished_task))
- os.rename(active_task, finished_task)
- os.utime(finished_task, None)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/core/inspector.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/core/inspector.py b/src/main/python/twitter/thermos/core/inspector.py
deleted file mode 100644
index 8c646a1..0000000
--- a/src/main/python/twitter/thermos/core/inspector.py
+++ /dev/null
@@ -1,105 +0,0 @@
-from collections import namedtuple
-from contextlib import closing
-import pwd
-
-from twitter.common import log
-from twitter.common.recordio import RecordIO, ThriftRecordReader
-
-from twitter.thermos.common.ckpt import CheckpointDispatcher
-from twitter.thermos.common.path import TaskPath
-
-from gen.twitter.thermos.ttypes import (
- ProcessState,
- RunnerCkpt,
- RunnerState)
-
-from .muxer import ProcessMuxer
-
-
-CheckpointInspection = namedtuple('CheckpointInspection',
- ['runner_latest_update',
- 'process_latest_update',
- 'runner_processes',
- 'coordinator_processes',
- 'processes'])
-
-
-class CheckpointInspector(object):
- def __init__(self, checkpoint_root):
- self._path = TaskPath(root=checkpoint_root)
-
- @staticmethod
- def get_timestamp(process_record):
- if process_record :
- for timestamp in ('fork_time', 'start_time', 'stop_time'):
- stamp = getattr(process_record, timestamp, None)
- if stamp:
- return stamp
- return 0
-
- def inspect(self, task_id):
- """
- Reconstructs the checkpoint stream and returns a CheckpointInspection.
- """
- dispatcher = CheckpointDispatcher()
- state = RunnerState(processes = {})
- muxer = ProcessMuxer(self._path.given(task_id=task_id))
-
- runner_processes = []
- coordinator_processes = set()
- processes = set()
-
- def consume_process_record(record):
- if not record.process_status:
- return
- try:
- user_uid = pwd.getpwnam(state.header.user).pw_uid
- except KeyError:
- log.error('Could not find user: %s' % state.header.user)
- return
- if record.process_status.state == ProcessState.FORKED:
- coordinator_processes.add((record.process_status.coordinator_pid, user_uid,
- record.process_status.fork_time))
- elif record.process_status.state == ProcessState.RUNNING:
- processes.add((record.process_status.pid, user_uid,
- record.process_status.start_time))
-
- # replay runner checkpoint
- runner_pid = None
- runner_latest_update = 0
- try:
- with open(self._path.given(task_id=task_id).getpath('runner_checkpoint')) as fp:
- with closing(ThriftRecordReader(fp, RunnerCkpt)) as ckpt:
- for record in ckpt:
- dispatcher.dispatch(state, record)
- runner_latest_update = max(runner_latest_update,
- self.get_timestamp(record.process_status))
- # collect all bound runners
- if record.task_status:
- if record.task_status.runner_pid != runner_pid:
- runner_processes.append((record.task_status.runner_pid,
- record.task_status.runner_uid or 0,
- record.task_status.timestamp_ms))
- runner_pid = record.task_status.runner_pid
- elif record.process_status:
- consume_process_record(record)
- except (IOError, OSError, RecordIO.Error) as err:
- log.debug('Error inspecting task runner checkpoint: %s' % err)
- return
-
- # register existing processes in muxer
- for process_name in state.processes:
- muxer.register(process_name)
-
- # read process checkpoints
- process_latest_update = runner_latest_update
- for record in muxer.select():
- process_latest_update = max(process_latest_update, self.get_timestamp(record.process_status))
- consume_process_record(record)
-
- return CheckpointInspection(
- runner_latest_update=runner_latest_update,
- process_latest_update=process_latest_update,
- runner_processes=runner_processes,
- coordinator_processes=coordinator_processes,
- processes=processes)
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/core/muxer.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/core/muxer.py b/src/main/python/twitter/thermos/core/muxer.py
deleted file mode 100644
index 9c0b389..0000000
--- a/src/main/python/twitter/thermos/core/muxer.py
+++ /dev/null
@@ -1,142 +0,0 @@
-import os
-import errno
-
-from twitter.common import log
-from twitter.common.recordio import ThriftRecordReader
-from gen.twitter.thermos.ttypes import RunnerCkpt
-
-
-class ProcessMuxer(object):
- class ProcessExists(Exception): pass
- class ProcessNotFound(Exception): pass
- class CorruptCheckpoint(Exception): pass
-
- def __init__(self, pathspec):
- self._processes = {} # process_name => fp
- self._watermarks = {} # process_name => sequence high watermark
- self._pathspec = pathspec
-
- def __del__(self):
- for fp in filter(None, self._processes.values()):
- fp.close()
-
- def register(self, process_name, watermark=0):
- log.debug('registering %s' % process_name)
- if process_name in self._processes:
- raise ProcessMuxer.ProcessExists("Process %s is already registered" % process_name)
- self._processes[process_name] = None
- self._watermarks[process_name] = watermark
-
- def _bind_processes(self):
- for process_name, fp in self._processes.items():
- if fp is None:
- process_ckpt = self._pathspec.given(process=process_name).getpath('process_checkpoint')
- log.debug('ProcessMuxer binding %s => %s' % (process_name, process_ckpt))
- try:
- self._processes[process_name] = open(process_ckpt, 'r')
- except IOError as e:
- if e.errno == errno.ENOENT:
- log.debug(' => bind failed, checkpoint not available yet.')
- continue
- else:
- log.error("Unexpected inability to open %s! %s" % (process_ckpt, e))
- except Exception as e:
- log.error("Unexpected inability to open %s! %s" % (process_ckpt, e))
- self._fast_forward_stream(process_name)
-
- def _fast_forward_stream(self, process_name):
- log.debug('Fast forwarding %s stream to seq=%s' % (process_name,
- self._watermarks[process_name]))
- assert self._processes.get(process_name) is not None
- fp = self._processes[process_name]
- rr = ThriftRecordReader(fp, RunnerCkpt)
- current_watermark = -1
- records = 0
- while current_watermark < self._watermarks[process_name]:
- last_pos = fp.tell()
- record = rr.try_read()
- if record is None:
- break
- new_watermark = record.process_status.seq
- if new_watermark > self._watermarks[process_name]:
- log.debug('Over-seeked %s [watermark = %s, high watermark = %s], rewinding.' % (
- process_name, new_watermark, self._watermarks[process_name]))
- fp.seek(last_pos)
- break
- current_watermark = new_watermark
- records += 1
-
- if current_watermark < self._watermarks[process_name]:
- log.warning('Only able to fast forward to %s@sequence=%s, high watermark is %s' % (
- process_name, current_watermark, self._watermarks[process_name]))
-
- if records:
- log.debug('Fast forwarded %s %s record(s) to seq=%s.' % (process_name, records,
- current_watermark))
-
- def unregister(self, process_name):
- log.debug('unregistering %s' % process_name)
- if process_name not in self._processes:
- raise ProcessMuxer.ProcessNotFound("No trace of process: %s" % process_name)
- else:
- self._watermarks.pop(process_name)
- fp = self._processes.pop(process_name)
- if fp is not None:
- fp.close()
-
- def has_data(self, process):
- """
- Return true if we think that there are updates available from the supplied process.
- """
- self._bind_processes()
- # TODO(wickman) Should this raise ProcessNotFound?
- if process not in self._processes:
- return False
- fp = self._processes[process]
- rr = ThriftRecordReader(fp, RunnerCkpt)
- old_pos = fp.tell()
- try:
- expected_new_pos = os.fstat(fp.fileno()).st_size
- except OSError as e:
- log.debug('ProcessMuxer could not fstat for process %s' % process)
- return False
- update = rr.try_read()
- if update:
- fp.seek(old_pos)
- return True
- return False
-
- def select(self):
- """
- Read and multiplex checkpoint records from all the forked off process coordinators.
-
- Checkpoint records can come from one of two places:
- in-process: checkpoint records synthesized for FORKED and LOST events
- out-of-process: checkpoint records from from file descriptors of forked coordinators
-
- Returns a list of RunnerCkpt objects that were successfully read, or an empty
- list if none were read.
- """
- self._bind_processes()
- updates = []
- for handle in filter(None, self._processes.values()):
- try:
- fstat = os.fstat(handle.fileno())
- except OSError as e:
- log.error('Unable to fstat %s!' % handle.name)
- continue
- if handle.tell() > fstat.st_size:
- log.error('Truncated checkpoint record detected on %s!' % handle.name)
- elif handle.tell() < fstat.st_size:
- rr = ThriftRecordReader(handle, RunnerCkpt)
- while True:
- process_update = rr.try_read()
- if process_update:
- updates.append(process_update)
- else:
- break
- if len(updates) > 0:
- log.debug('select() returning %s updates:' % len(updates))
- for update in updates:
- log.debug(' = %s' % update)
- return updates
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/python/twitter/thermos/core/process.py
----------------------------------------------------------------------
diff --git a/src/main/python/twitter/thermos/core/process.py b/src/main/python/twitter/thermos/core/process.py
deleted file mode 100644
index dd1d62a..0000000
--- a/src/main/python/twitter/thermos/core/process.py
+++ /dev/null
@@ -1,371 +0,0 @@
-"""Run processes of a Thermos task.
-
-This module contains the Process class, used to manage the execution of the constituent processes of
-a Thermos task. Each process is represented by a "coordinator" process, which fires off the actual
-commandline in a subprocess of its own.
-
-"""
-
-from abc import abstractmethod
-import getpass
-import grp
-import os
-import pwd
-import signal
-import subprocess
-import sys
-import time
-
-from twitter.common import log
-from twitter.common.dirutil import (
- lock_file,
- safe_mkdir,
- safe_open,
-)
-from twitter.common.lang import Interface
-from twitter.common.quantity import Amount, Time
-from twitter.common.recordio import ThriftRecordReader, ThriftRecordWriter
-
-from gen.twitter.thermos.ttypes import (
- ProcessState,
- ProcessStatus,
- RunnerCkpt,
-)
-
-
-class Platform(Interface):
- """Abstract representation of a platform encapsulating system-level functions"""
- @abstractmethod
- def clock(self):
- pass
-
- @abstractmethod
- def fork(self):
- pass
-
- @abstractmethod
- def getpid(self):
- pass
-
-
-class ProcessBase(object):
- """
- Encapsulate a running process for a task.
- """
- class Error(Exception): pass
- class UnknownUserError(Error): pass
- class CheckpointError(Error): pass
- class UnspecifiedSandbox(Error): pass
- class PermissionError(Error): pass
-
- CONTROL_WAIT_CHECK_INTERVAL = Amount(100, Time.MILLISECONDS)
- MAXIMUM_CONTROL_WAIT = Amount(1, Time.MINUTES)
-
- def __init__(self, name, cmdline, sequence, pathspec, sandbox_dir, user=None, platform=None):
- """
- required:
- name = name of the process
- cmdline = cmdline of the process
- sequence = the next available sequence number for state updates
- pathspec = TaskPath object for synthesizing path names
- sandbox_dir = the sandbox in which to run the process
- platform = Platform providing fork, clock, getpid
-
- optional:
- user = the user to run as (if unspecified, will default to current user.)
- if specified to a user that is not the current user, you must have root access
- """
- self._name = name
- self._cmdline = cmdline
- self._pathspec = pathspec
- self._seq = sequence
- self._sandbox = sandbox_dir
- if self._sandbox:
- safe_mkdir(self._sandbox)
- self._pid = None
- self._fork_time = None
- self._stdout = None
- self._stderr = None
- self._user = user
- if self._user:
- user, current_user = self._getpwuid() # may raise self.UnknownUserError
- if user != current_user and os.geteuid() != 0:
- raise self.PermissionError('Must be root to run processes as other users!')
- self._ckpt = None
- self._ckpt_head = -1
- if platform is None:
- raise ValueError("Platform must be specified")
- self._platform = platform
-
- def _log(self, msg):
- log.debug('[process:%5s=%s]: %s' % (self._pid, self.name(), msg))
-
- def _ckpt_write(self, msg):
- self._init_ckpt_if_necessary()
- self._log("child state transition [%s] <= %s" % (self.ckpt_file(), msg))
- self._ckpt.write(msg)
-
- def _write_process_update(self, **kw):
- """Write a process update to the coordinator's checkpoint stream."""
- process_status = ProcessStatus(**kw)
- process_status.seq = self._seq
- process_status.process = self.name()
- self._ckpt_write(RunnerCkpt(process_status=process_status))
- self._seq += 1
-
- def _write_initial_update(self):
- self._write_process_update(state=ProcessState.FORKED,
- fork_time=self._fork_time,
- coordinator_pid=self._pid)
-
- def cmdline(self):
- return self._cmdline
-
- def name(self):
- return self._name
-
- def pid(self):
- """pid of the coordinator"""
- return self._pid
-
- def rebind(self, pid, fork_time):
- """rebind Process to an existing coordinator pid without forking"""
- self._pid = pid
- self._fork_time = fork_time
-
- def ckpt_file(self):
- return self._pathspec.getpath('process_checkpoint')
-
- def _setup_ckpt(self):
- """Set up the checkpoint: must be run on the parent."""
- self._log('initializing checkpoint file: %s' % self.ckpt_file())
- ckpt_fp = lock_file(self.ckpt_file(), "a+")
- if ckpt_fp in (None, False):
- raise self.CheckpointError('Could not acquire checkpoint permission or lock for %s!' %
- self.ckpt_file())
- self._ckpt_head = os.path.getsize(self.ckpt_file())
- ckpt_fp.seek(self._ckpt_head)
- self._ckpt = ThriftRecordWriter(ckpt_fp)
- self._ckpt.set_sync(True)
-
- def _init_ckpt_if_necessary(self):
- if self._ckpt is None:
- self._setup_ckpt()
-
- def _wait_for_control(self):
- """Wait for control of the checkpoint stream: must be run in the child."""
- total_wait_time = Amount(0, Time.SECONDS)
-
- with open(self.ckpt_file(), 'r') as fp:
- fp.seek(self._ckpt_head)
- rr = ThriftRecordReader(fp, RunnerCkpt)
- while total_wait_time < self.MAXIMUM_CONTROL_WAIT:
- ckpt_tail = os.path.getsize(self.ckpt_file())
- if ckpt_tail == self._ckpt_head:
- self._platform.clock().sleep(self.CONTROL_WAIT_CHECK_INTERVAL.as_(Time.SECONDS))
- total_wait_time += self.CONTROL_WAIT_CHECK_INTERVAL
- continue
- checkpoint = rr.try_read()
- if checkpoint:
- if not checkpoint.process_status:
- raise self.CheckpointError('No process status in checkpoint!')
- if (checkpoint.process_status.process != self.name() or
- checkpoint.process_status.state != ProcessState.FORKED or
- checkpoint.process_status.fork_time != self._fork_time or
- checkpoint.process_status.coordinator_pid != self._pid):
- self._log('Losing control of the checkpoint stream:')
- self._log(' fork_time [%s] vs self._fork_time [%s]' % (
- checkpoint.process_status.fork_time, self._fork_time))
- self._log(' coordinator_pid [%s] vs self._pid [%s]' % (
- checkpoint.process_status.coordinator_pid, self._pid))
- raise self.CheckpointError('Lost control of the checkpoint stream!')
- self._log('Taking control of the checkpoint stream at record: %s' %
- checkpoint.process_status)
- self._seq = checkpoint.process_status.seq + 1
- return True
- raise self.CheckpointError('Timed out waiting for checkpoint stream!')
-
- def _prepare_fork(self):
- user, current_user = self._getpwuid()
- uid, gid = user.pw_uid, user.pw_gid
- self._fork_time = self._platform.clock().time()
- self._setup_ckpt()
- self._stdout = safe_open(self._pathspec.with_filename('stdout').getpath('process_logdir'), "w")
- self._stderr = safe_open(self._pathspec.with_filename('stderr').getpath('process_logdir'), "w")
- os.chown(self._stdout.name, user.pw_uid, user.pw_gid)
- os.chown(self._stderr.name, user.pw_uid, user.pw_gid)
-
- def _finalize_fork(self):
- self._write_initial_update()
- self._ckpt.close()
- self._ckpt = None
-
- def _getpwuid(self):
- """Returns a tuple of the user (i.e. --user) and current user."""
- try:
- current_user = pwd.getpwuid(os.getuid())
- except KeyError:
- raise self.UnknownUserError('Unknown user %s!' % self._user)
- try:
- user = pwd.getpwnam(self._user) if self._user else current_user
- except KeyError:
- raise self.UnknownUserError('Unable to get pwent information!')
- return user, current_user
-
- def start(self):
- """
- This is the main call point from the runner, and forks a co-ordinator process to run the
- target process (i.e. self.cmdline())
-
- The parent returns immediately and populates information about the pid of the co-ordinator.
- The child (co-ordinator) will launch the target process in a subprocess.
- """
- self._prepare_fork()
- self._pid = self._platform.fork()
- if self._pid == 0:
- self._pid = self._platform.getpid()
- self._wait_for_control()
- try:
- self.execute()
- finally:
- self._ckpt.close()
- self.finish()
- else:
- self._finalize_fork()
-
- def execute(self):
- raise NotImplementedError
-
- def finish(self):
- pass
-
-
-class RealPlatform(Platform):
- IGNORE_SIGNALS = (signal.SIGINT,)
-
- def __init__(self, fork=os.fork):
- self._fork = fork
-
- def fork(self):
- pid = self._fork()
- if pid == 0:
- self._sanitize()
- return pid
-
- def _sanitize(self):
- for sig in self.IGNORE_SIGNALS:
- signal.signal(sig, signal.SIG_IGN)
-
- def getpid(self):
- return os.getpid()
-
- def clock(self):
- return time
-
-
-class Process(ProcessBase):
- """
- Encapsulate a running process for a task.
- """
- RCFILE = '.thermos_profile'
- FD_CLOEXEC = True
-
- def __init__(self, *args, **kw):
- """
- See ProcessBase.__init__
-
- Takes additional arguments:
- fork: the fork function to use [default: os.fork]
- chroot: whether or not to chroot into the sandbox [default: False]
- """
- fork = kw.pop('fork', os.fork)
- self._use_chroot = bool(kw.pop('chroot', False))
- self._rc = None
- kw['platform'] = RealPlatform(fork=fork)
- ProcessBase.__init__(self, *args, **kw)
- if self._use_chroot and self._sandbox is None:
- raise self.UnspecifiedSandbox('If using chroot, must specify sandbox!')
-
- def _chroot(self):
- """chdir and chroot to the sandbox directory."""
- os.chdir(self._sandbox)
- os.chroot(self._sandbox)
-
- def _setuid(self):
- """Drop privileges to the user supplied in Process creation (if necessary.)"""
- user, current_user = self._getpwuid()
- if user.pw_uid == current_user.pw_uid:
- return
-
- uid, gid = user.pw_uid, user.pw_gid
- username = user.pw_name
- group_ids = [group.gr_gid for group in grp.getgrall() if username in group.gr_mem]
- os.setgroups(group_ids)
- os.setgid(gid)
- os.setuid(uid)
-
- def execute(self):
- """Perform final initialization and launch target process commandline in a subprocess."""
- if not self._stderr:
- raise RuntimeError('self._stderr not set up!')
- if not self._stdout:
- raise RuntimeError('self._stdout not set up!')
-
- user, _ = self._getpwuid()
- username, homedir = user.pw_name, user.pw_dir
-
- # TODO(wickman) reconsider setsid now that we're invoking in a subshell
- os.setsid()
- if self._use_chroot:
- self._chroot()
- self._setuid()
-
- # start process
- start_time = self._platform.clock().time()
-
- if not self._sandbox:
- sandbox = os.getcwd()
- else:
- sandbox = self._sandbox if not self._use_chroot else '/'
-
- thermos_profile = os.path.join(sandbox, self.RCFILE)
- env = {
- 'HOME': homedir if self._use_chroot else sandbox,
- 'LOGNAME': username,
- 'USER': username,
- 'PATH': os.environ['PATH']
- }
-
- if os.path.exists(thermos_profile):
- env.update(BASH_ENV=thermos_profile)
-
- self._popen = subprocess.Popen(["/bin/bash", "-c", self.cmdline()],
- stderr=self._stderr,
- stdout=self._stdout,
- close_fds=self.FD_CLOEXEC,
- cwd=sandbox,
- env=env)
-
- self._write_process_update(state=ProcessState.RUNNING,
- pid=self._popen.pid,
- start_time=start_time)
-
- # wait for job to finish
- rc = self._popen.wait()
-
- # indicate that we have finished/failed
- if rc < 0:
- state = ProcessState.KILLED
- elif rc == 0:
- state = ProcessState.SUCCESS
- else:
- state = ProcessState.FAILED
-
- self._write_process_update(state=state,
- return_code=rc,
- stop_time=self._platform.clock().time())
- self._rc = rc
-
- def finish(self):
- self._log('Coordinator exiting.')
- sys.exit(0)
[35/51] [partial] Rename twitter* and com.twitter to apache and
org.apache directories to preserve all file history before the refactor.
Posted by wf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java b/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java
new file mode 100644
index 0000000..400d8b7
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/events/PubsubEvent.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.events;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.util.Set;
+
+import com.google.common.base.Objects;
+
+import com.twitter.aurora.gen.HostStatus;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Event notifications related to tasks.
+ */
+public interface PubsubEvent {
+
+ /**
+ * Interface with no functionality, but identifies a class as supporting task pubsub events.
+ */
+ public interface EventSubscriber {
+ }
+
+ /**
+ * Event sent when tasks were deleted.
+ */
+ public static class TasksDeleted implements PubsubEvent {
+ private final Set<IScheduledTask> tasks;
+
+ public TasksDeleted(Set<IScheduledTask> tasks) {
+ this.tasks = checkNotNull(tasks);
+ }
+
+ public Set<IScheduledTask> getTasks() {
+ return tasks;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TasksDeleted)) {
+ return false;
+ }
+
+ TasksDeleted other = (TasksDeleted) o;
+ return Objects.equal(tasks, other.tasks);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(tasks);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("tasks", Tasks.ids(tasks))
+ .toString();
+ }
+ }
+
+ /**
+ * Event sent when a task changed state.
+ */
+ public static class TaskStateChange implements PubsubEvent {
+ private final IScheduledTask task;
+ private final ScheduleStatus oldState;
+
+ public TaskStateChange(IScheduledTask task, ScheduleStatus oldState) {
+ this.task = checkNotNull(task);
+ this.oldState = checkNotNull(oldState);
+ }
+
+ public String getTaskId() {
+ return Tasks.id(task);
+ }
+
+ public ScheduleStatus getOldState() {
+ return oldState;
+ }
+
+ public IScheduledTask getTask() {
+ return task;
+ }
+
+ public ScheduleStatus getNewState() {
+ return task.getStatus();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TaskStateChange)) {
+ return false;
+ }
+
+ TaskStateChange other = (TaskStateChange) o;
+ return Objects.equal(task, other.task)
+ && Objects.equal(oldState, other.oldState);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(task, oldState);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("task", Tasks.id(task))
+ .add("oldState", getOldState())
+ .add("newState", getNewState())
+ .toString();
+ }
+ }
+
+ /**
+ * Event sent when a host changed maintenance state.
+ */
+ public static class HostMaintenanceStateChange implements PubsubEvent {
+ private final HostStatus status;
+
+ public HostMaintenanceStateChange(HostStatus status) {
+ this.status = checkNotNull(status);
+ }
+
+ public HostStatus getStatus() {
+ return status;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof HostMaintenanceStateChange)) {
+ return false;
+ }
+
+ HostMaintenanceStateChange other = (HostMaintenanceStateChange) o;
+ return Objects.equal(status, other.status);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(status);
+ }
+ }
+
+ /**
+ * Event sent when a scheduling assignment was vetoed.
+ */
+ public static class Vetoed implements PubsubEvent {
+ private final String taskId;
+ private final Set<Veto> vetoes;
+
+ public Vetoed(String taskId, Set<Veto> vetoes) {
+ this.taskId = checkNotNull(taskId);
+ this.vetoes = checkNotNull(vetoes);
+ }
+
+ public String getTaskId() {
+ return taskId;
+ }
+
+ public Set<Veto> getVetoes() {
+ return vetoes;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Vetoed)) {
+ return false;
+ }
+
+ Vetoed other = (Vetoed) o;
+ return Objects.equal(taskId, other.taskId)
+ && Objects.equal(vetoes, other.vetoes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(taskId, vetoes);
+ }
+ }
+
+ public static class TaskRescheduled implements PubsubEvent {
+ private final String role;
+ private final String job;
+ private final int instance;
+
+ public TaskRescheduled(String role, String job, int instance) {
+ this.role = role;
+ this.job = job;
+ this.instance = instance;
+ }
+
+ public String getRole() {
+ return role;
+ }
+
+ public String getJob() {
+ return job;
+ }
+
+ public int getInstance() {
+ return instance;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TaskRescheduled)) {
+ return false;
+ }
+
+ TaskRescheduled other = (TaskRescheduled) o;
+ return Objects.equal(role, other.role)
+ && Objects.equal(job, other.job)
+ && Objects.equal(instance, other.instance);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(role, job, instance);
+ }
+ }
+
+ public static class StorageStarted implements PubsubEvent {
+ @Override
+ public boolean equals(Object o) {
+ return (o != null) && getClass().equals(o.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
+ }
+
+ public static class DriverRegistered implements PubsubEvent {
+ @Override
+ public boolean equals(Object o) {
+ return (o != null) && getClass().equals(o.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
+ }
+
+ public static class DriverDisconnected implements PubsubEvent {
+ @Override
+ public boolean equals(Object o) {
+ return (o != null) && getClass().equals(o.getClass());
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
+ }
+
+ public static final class Interceptors {
+ private Interceptors() {
+ // Utility class.
+ }
+
+ public enum Event {
+ None(null),
+ StorageStarted(new StorageStarted()),
+ DriverRegistered(new DriverRegistered()),
+ DriverDisconnected(new DriverDisconnected());
+
+ private final PubsubEvent event;
+ private Event(PubsubEvent event) {
+ this.event = event;
+ }
+
+ public PubsubEvent getEvent() {
+ return event;
+ }
+ }
+
+ /**
+ * An annotation to place on methods of injected classes that which to fire events before
+ * and/or after their invocation.
+ */
+ @Target(METHOD) @Retention(RUNTIME)
+ public @interface SendNotification {
+ /**
+ * Event to fire prior to invocation.
+ */
+ Event before() default Event.None;
+
+ /**
+ * Event to fire after invocation.
+ */
+ Event after() default Event.None;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/events/PubsubEventModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/events/PubsubEventModule.java b/src/main/java/org/apache/aurora/scheduler/events/PubsubEventModule.java
new file mode 100644
index 0000000..a8b5633
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/events/PubsubEventModule.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.events;
+
+import java.util.Set;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.eventbus.DeadEvent;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.google.inject.AbstractModule;
+import com.google.inject.Binder;
+import com.google.inject.TypeLiteral;
+import com.google.inject.matcher.Matchers;
+import com.google.inject.multibindings.Multibinder;
+
+import org.aopalliance.intercept.MethodInterceptor;
+
+import com.twitter.aurora.scheduler.events.NotifyingSchedulingFilter.NotifyDelegate;
+import com.twitter.aurora.scheduler.events.PubsubEvent.EventSubscriber;
+import com.twitter.aurora.scheduler.events.PubsubEvent.Interceptors.SendNotification;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter;
+import com.twitter.common.application.modules.LifecycleModule;
+import com.twitter.common.base.Closure;
+import com.twitter.common.base.Command;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Binding module for plumbing event notifications.
+ */
+public final class PubsubEventModule extends AbstractModule {
+
+ private static final Logger LOG = Logger.getLogger(PubsubEventModule.class.getName());
+
+ private PubsubEventModule() {
+ // Must be constructed through factory.
+ }
+
+ @VisibleForTesting
+ public static void installForTest(Binder binder) {
+ binder.install(new PubsubEventModule());
+ }
+
+ @Override
+ protected void configure() {
+ final EventBus eventBus = new EventBus("TaskEvents");
+ eventBus.register(new Object() {
+ @Subscribe public void logDeadEvent(DeadEvent event) {
+ LOG.warning("Captured dead event " + event.getEvent());
+ }
+ });
+
+ bind(EventBus.class).toInstance(eventBus);
+
+ Closure<PubsubEvent> eventPoster = new Closure<PubsubEvent>() {
+ @Override public void execute(PubsubEvent event) {
+ eventBus.post(event);
+ }
+ };
+ bind(new TypeLiteral<Closure<PubsubEvent>>() { }).toInstance(eventPoster);
+
+ // Ensure at least an empty binding is present.
+ getSubscriberBinder(binder());
+ LifecycleModule.bindStartupAction(binder(), RegisterSubscribers.class);
+ bindNotifyingInterceptor(binder());
+ }
+
+ static class RegisterSubscribers implements Command {
+ private final EventBus eventBus;
+ private final Set<EventSubscriber> subscribers;
+
+ @Inject
+ RegisterSubscribers(EventBus eventBus, Set<EventSubscriber> subscribers) {
+ this.eventBus = checkNotNull(eventBus);
+ this.subscribers = checkNotNull(subscribers);
+ }
+
+ @Override
+ public void execute() {
+ for (EventSubscriber subscriber : subscribers) {
+ eventBus.register(subscriber);
+ }
+ }
+ }
+
+ /**
+ * Binds a task event module.
+ *
+ * @param binder Binder to bind against.
+ * @param filterClass Delegate scheduling filter implementation class.
+ */
+ public static void bind(Binder binder, final Class<? extends SchedulingFilter> filterClass) {
+ binder.bind(SchedulingFilter.class).annotatedWith(NotifyDelegate.class).to(filterClass);
+ binder.bind(SchedulingFilter.class).to(NotifyingSchedulingFilter.class);
+ binder.bind(NotifyingSchedulingFilter.class).in(Singleton.class);
+ binder.install(new PubsubEventModule());
+ }
+
+ private static Multibinder<EventSubscriber> getSubscriberBinder(Binder binder) {
+ return Multibinder.newSetBinder(binder, EventSubscriber.class);
+ }
+
+ /**
+ * Binds a subscriber to receive task events.
+ *
+ * @param binder Binder to bind the subscriber with.
+ * @param subscriber Subscriber implementation class to register for events.
+ */
+ public static void bindSubscriber(Binder binder, Class<? extends EventSubscriber> subscriber) {
+ getSubscriberBinder(binder).addBinding().to(subscriber);
+ }
+
+ /**
+ * Binds a method interceptor to all methods annotated with {@link SendNotification}.
+ * <p>
+ * The interceptor will send notifications before and/or after the wrapped method invocation.
+ *
+ * @param binder Guice binder.
+ */
+ @VisibleForTesting
+ public static void bindNotifyingInterceptor(Binder binder) {
+ MethodInterceptor interceptor = new NotifyingMethodInterceptor();
+ binder.requestInjection(interceptor);
+ binder.bindInterceptor(
+ Matchers.any(),
+ Matchers.annotatedWith(SendNotification.class),
+ interceptor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/filter/AttributeFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/filter/AttributeFilter.java b/src/main/java/org/apache/aurora/scheduler/filter/AttributeFilter.java
new file mode 100644
index 0000000..66327fa
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/filter/AttributeFilter.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.filter;
+
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.AttributeLoader;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.IValueConstraint;
+
+/**
+ * Utility class that matches attributes to constraints.
+ */
+final class AttributeFilter {
+
+ private static final Function<Attribute, Set<String>> GET_VALUES =
+ new Function<Attribute, Set<String>>() {
+ @Override public Set<String> apply(Attribute attribute) {
+ return attribute.getValues();
+ }
+ };
+
+ private AttributeFilter() {
+ // Utility class.
+ }
+
+ /**
+ * Tests whether a constraint is satisfied by attributes.
+ *
+ * @param attributes Host attributes.
+ * @param constraint Constraint to match.
+ * @return {@code true} if the attribute satisfies the constraint, {@code false} otherwise.
+ */
+ static boolean matches(Set<Attribute> attributes, IValueConstraint constraint) {
+ Set<String> allAttributes =
+ ImmutableSet.copyOf(Iterables.concat(Iterables.transform(attributes, GET_VALUES)));
+ boolean match = Iterables.any(constraint.getValues(), Predicates.in(allAttributes));
+ return constraint.isNegated() ^ match;
+ }
+
+ /**
+ * Tests whether an attribute matches a limit constraint.
+ *
+ * @param attributes Attributes to match against.
+ * @param jobKey Key of the job with the limited constraint.
+ * @param limit Limit value.
+ * @param activeTasks All active tasks in the system.
+ * @param attributeFetcher Interface for fetching attributes for hosts in the system.
+ * @return {@code true} if the limit constraint is satisfied, {@code false} otherwise.
+ */
+ static boolean matches(final Set<Attribute> attributes,
+ final IJobKey jobKey,
+ int limit,
+ Iterable<IScheduledTask> activeTasks,
+ final AttributeLoader attributeFetcher) {
+
+ Predicate<IScheduledTask> sameJob =
+ Predicates.compose(Predicates.equalTo(jobKey), Tasks.SCHEDULED_TO_JOB_KEY);
+
+ Predicate<IScheduledTask> hasAttribute = new Predicate<IScheduledTask>() {
+ @Override public boolean apply(IScheduledTask task) {
+ Iterable<Attribute> hostAttributes =
+ attributeFetcher.apply(task.getAssignedTask().getSlaveHost());
+ return Iterables.any(hostAttributes, Predicates.in(attributes));
+ }
+ };
+
+ return limit > Iterables.size(
+ Iterables.filter(activeTasks, Predicates.and(sameJob, hasAttribute)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/filter/ConstraintFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/filter/ConstraintFilter.java b/src/main/java/org/apache/aurora/scheduler/filter/ConstraintFilter.java
new file mode 100644
index 0000000..2612cb8
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/filter/ConstraintFilter.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.filter;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.scheduler.base.SchedulerException;
+import com.twitter.aurora.scheduler.filter.SchedulingFilter.Veto;
+import com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.AttributeLoader;
+import com.twitter.aurora.scheduler.storage.entities.IConstraint;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConstraint;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Filter that determines whether a task's constraints are satisfied.
+ */
+class ConstraintFilter implements Function<IConstraint, Optional<Veto>> {
+
+ private static final Logger LOG = Logger.getLogger(ConstraintFilter.class.getName());
+
+ private final IJobKey jobKey;
+ private final Supplier<Collection<IScheduledTask>> activeTasksSupplier;
+ private final AttributeLoader attributeLoader;
+ private final Iterable<Attribute> hostAttributes;
+
+ /**
+ * Creates a new constraint filer for a given job.
+ *
+ * @param jobKey Key for the job.
+ * @param activeTasksSupplier Supplier to fetch active tasks (if necessary).
+ * @param attributeLoader Interface to fetch host attributes (if necessary).
+ * @param hostAttributes The attributes of the host to test against.
+ */
+ ConstraintFilter(
+ IJobKey jobKey,
+ Supplier<Collection<IScheduledTask>> activeTasksSupplier,
+ AttributeLoader attributeLoader,
+ Iterable<Attribute> hostAttributes) {
+
+ this.jobKey = checkNotNull(jobKey);
+ this.activeTasksSupplier = checkNotNull(activeTasksSupplier);
+ this.attributeLoader = checkNotNull(attributeLoader);
+ this.hostAttributes = checkNotNull(hostAttributes);
+ }
+
+ @VisibleForTesting
+ static Veto limitVeto(String limit) {
+ return new Veto("Limit not satisfied: " + limit, Veto.MAX_SCORE);
+ }
+
+ @VisibleForTesting
+ static Veto mismatchVeto(String constraint) {
+ return Veto.constraintMismatch("Constraint not satisfied: " + constraint);
+ }
+
+ @VisibleForTesting
+ static Veto maintenanceVeto(String reason) {
+ return new Veto("Host " + reason + " for maintenance", Veto.MAX_SCORE);
+ }
+
+ @Override
+ public Optional<Veto> apply(IConstraint constraint) {
+ Set<Attribute> attributes =
+ ImmutableSet.copyOf(Iterables.filter(hostAttributes, new NameFilter(constraint.getName())));
+
+ ITaskConstraint taskConstraint = constraint.getConstraint();
+ switch (taskConstraint.getSetField()) {
+ case VALUE:
+ boolean matches =
+ AttributeFilter.matches(attributes, taskConstraint.getValue());
+ return matches
+ ? Optional.<Veto>absent()
+ : Optional.of(mismatchVeto(constraint.getName()));
+
+ case LIMIT:
+ if (attributes.isEmpty()) {
+ return Optional.of(mismatchVeto(constraint.getName()));
+ }
+
+ boolean satisfied = AttributeFilter.matches(
+ attributes,
+ jobKey,
+ taskConstraint.getLimit().getLimit(),
+ activeTasksSupplier.get(),
+ attributeLoader);
+ return satisfied
+ ? Optional.<Veto>absent()
+ : Optional.of(limitVeto(constraint.getName()));
+
+ default:
+ LOG.warning("Unrecognized constraint type: " + taskConstraint.getSetField());
+ throw new SchedulerException("Failed to recognize the constraint type: "
+ + taskConstraint.getSetField());
+ }
+ }
+
+ /**
+ * A filter to find attributes matching a name.
+ */
+ static class NameFilter implements Predicate<Attribute> {
+ private final String attributeName;
+
+ NameFilter(String attributeName) {
+ this.attributeName = attributeName;
+ }
+
+ @Override public boolean apply(Attribute attribute) {
+ return attributeName.equals(attribute.getName());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java
new file mode 100644
index 0000000..7e8fbdb
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilter.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.filter;
+
+import java.util.Set;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+
+import com.twitter.aurora.scheduler.ResourceSlot;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+
+/**
+ * Determines whether a proposed scheduling assignment should be allowed.
+ */
+public interface SchedulingFilter {
+
+ /**
+ * Reason for a proposed scheduling assignment to be filtered out.
+ * A veto also contains a score, which is an opaque indicator as to how strong a veto is. This
+ * is only intended to be used for relative ranking of vetoes for determining which veto against
+ * a scheduling assignment is 'weakest'.
+ */
+ public static class Veto {
+ public static final int MAX_SCORE = 1000;
+
+ private final String reason;
+ private final int score;
+ private final boolean valueMismatch;
+
+ private Veto(String reason, int score, boolean valueMismatch) {
+ this.reason = reason;
+ this.score = Math.min(MAX_SCORE, score);
+ this.valueMismatch = valueMismatch;
+ }
+
+ @VisibleForTesting
+ public Veto(String reason, int score) {
+ this(reason, score, false);
+ }
+
+ /**
+ * Creates a special veto that represents a mismatch between the server and task's configuration
+ * for an attribute.
+ *
+ * @param reason Information about the value mismatch.
+ * @return A constraint mismatch veto.
+ */
+ public static Veto constraintMismatch(String reason) {
+ return new Veto(reason, MAX_SCORE, true);
+ }
+
+ public String getReason() {
+ return reason;
+ }
+
+ public int getScore() {
+ return score;
+ }
+
+ public boolean isConstraintMismatch() {
+ return valueMismatch;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Veto)) {
+ return false;
+ }
+
+ Veto other = (Veto) o;
+ return Objects.equal(reason, other.reason)
+ && (score == other.score)
+ && (valueMismatch == other.valueMismatch);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(reason, score, valueMismatch);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("reason", reason)
+ .add("score", score)
+ .add("valueMismatch", valueMismatch)
+ .toString();
+ }
+ }
+
+ /**
+ * Applies a task against the filter with the given resources, and on the host.
+ *
+ * @param offer Resources offered.
+ * @param slaveHost Host that the resources are associated with.
+ * @param task Task.
+ * @param taskId Canonical ID of the task.
+ * @return A set of vetoes indicating reasons the task cannot be scheduled. If the task may be
+ * scheduled, the set will be empty.
+ */
+ Set<Veto> filter(ResourceSlot offer, String slaveHost, ITaskConfig task, String taskId);
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java
new file mode 100644
index 0000000..33272ce
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/filter/SchedulingFilterImpl.java
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.filter;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.Set;
+
+import javax.inject.Inject;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
+
+import com.twitter.aurora.gen.Attribute;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.gen.ScheduleStatus;
+import com.twitter.aurora.gen.TaskConstraint;
+import com.twitter.aurora.scheduler.ResourceSlot;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.configuration.ConfigurationManager;
+import com.twitter.aurora.scheduler.state.MaintenanceController;
+import com.twitter.aurora.scheduler.storage.AttributeStore;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work.Quiet;
+import com.twitter.aurora.scheduler.storage.entities.IConstraint;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+import com.twitter.aurora.scheduler.storage.entities.ITaskConfig;
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Data;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
+import static com.twitter.aurora.scheduler.configuration.ConfigurationManager.DEDICATED_ATTRIBUTE;
+import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.CPU;
+import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.DISK;
+import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.PORTS;
+import static com.twitter.aurora.scheduler.filter.SchedulingFilterImpl.ResourceVector.RAM;
+
+/**
+ * Implementation of the scheduling filter that ensures resource requirements of tasks are
+ * fulfilled, and that tasks are allowed to run on the given machine.
+ *
+ */
+public class SchedulingFilterImpl implements SchedulingFilter {
+
+ @VisibleForTesting static final Veto DEDICATED_HOST_VETO =
+ Veto.constraintMismatch("Host is dedicated");
+
+ private static final Optional<Veto> NO_VETO = Optional.absent();
+
+ private static final Set<MaintenanceMode> VETO_MODES = EnumSet.of(DRAINING, DRAINED);
+
+ private final Storage storage;
+ private final MaintenanceController maintenance;
+
+ /**
+ * Creates a new scheduling filter.
+ *
+ * @param storage Interface to accessing the task store.
+ * @param maintenance Interface to accessing the maintenance controller
+ */
+ @Inject
+ public SchedulingFilterImpl(Storage storage, MaintenanceController maintenance) {
+ this.storage = checkNotNull(storage);
+ this.maintenance = checkNotNull(maintenance);
+ }
+
+ /**
+ * A function that fetches attributes associated with a given host.
+ */
+ public interface AttributeLoader extends Function<String, Iterable<Attribute>> { }
+
+ /**
+ * A function that may veto a task.
+ */
+ private interface FilterRule extends Function<ITaskConfig, Iterable<Veto>> { }
+
+ /**
+ * Convenience class for a rule that will only ever have a single veto.
+ */
+ private abstract static class SingleVetoRule implements FilterRule {
+ @Override public final Iterable<Veto> apply(ITaskConfig task) {
+ return doApply(task).asSet();
+ }
+
+ abstract Optional<Veto> doApply(ITaskConfig task);
+ }
+
+ // Scaling ranges to use for comparison of vetos. This has no real bearing besides trying to
+ // determine if a veto along one resource vector is a 'stronger' veto than that of another vector.
+ // The values below represent the maximum resources on a typical slave machine.
+ @VisibleForTesting
+ enum ResourceVector {
+ CPU("CPU", 16),
+ RAM("RAM", Amount.of(24, Data.GB).as(Data.MB)),
+ DISK("disk", Amount.of(450, Data.GB).as(Data.MB)),
+ PORTS("ports", 1000);
+
+ private final String name;
+ private final int range;
+ @VisibleForTesting
+ int getRange() {
+ return range;
+ }
+
+ private ResourceVector(String name, int range) {
+ this.name = name;
+ this.range = range;
+ }
+
+ Optional<Veto> maybeVeto(double available, double requested) {
+ double tooLarge = requested - available;
+ if (tooLarge <= 0) {
+ return NO_VETO;
+ } else {
+ return Optional.of(veto(tooLarge));
+ }
+ }
+
+ private static int scale(double value, int range) {
+ return Math.min(Veto.MAX_SCORE, (int) ((Veto.MAX_SCORE * value)) / range);
+ }
+
+ @VisibleForTesting
+ Veto veto(double excess) {
+ return new Veto("Insufficient " + name, scale(excess, range));
+ }
+ }
+
+ private Iterable<FilterRule> rulesFromOffer(final ResourceSlot available) {
+ return ImmutableList.<FilterRule>of(
+ new SingleVetoRule() {
+ @Override public Optional<Veto> doApply(ITaskConfig task) {
+ return CPU.maybeVeto(
+ available.getNumCpus(),
+ ResourceSlot.from(task).getNumCpus());
+ }
+ },
+ new SingleVetoRule() {
+ @Override public Optional<Veto> doApply(ITaskConfig task) {
+ return RAM.maybeVeto(
+ available.getRam().as(Data.MB),
+ ResourceSlot.from(task).getRam().as(Data.MB));
+ }
+ },
+ new SingleVetoRule() {
+ @Override public Optional<Veto> doApply(ITaskConfig task) {
+ return DISK.maybeVeto(available.getDisk().as(Data.MB),
+ ResourceSlot.from(task).getDisk().as(Data.MB));
+ }
+ },
+ new SingleVetoRule() {
+ @Override public Optional<Veto> doApply(ITaskConfig task) {
+ return PORTS.maybeVeto(available.getNumPorts(),
+ ResourceSlot.from(task).getNumPorts());
+ }
+ }
+ );
+ }
+
+ private static boolean isValueConstraint(IConstraint constraint) {
+ return constraint.getConstraint().getSetField() == TaskConstraint._Fields.VALUE;
+ }
+
+ private static final Ordering<IConstraint> VALUES_FIRST = Ordering.from(
+ new Comparator<IConstraint>() {
+ @Override public int compare(IConstraint a, IConstraint b) {
+ if (a.getConstraint().getSetField() == b.getConstraint().getSetField()) {
+ return 0;
+ }
+ return isValueConstraint(a) ? -1 : 1;
+ }
+ });
+
+ private static final Iterable<ScheduleStatus> ACTIVE_NOT_PENDING_STATES =
+ EnumSet.copyOf(Sets.difference(Tasks.ACTIVE_STATES, EnumSet.of(ScheduleStatus.PENDING)));
+
+ private FilterRule getConstraintFilter(final String slaveHost) {
+ return new FilterRule() {
+ @Override public Iterable<Veto> apply(final ITaskConfig task) {
+ if (!task.isSetConstraints()) {
+ return ImmutableList.of();
+ }
+
+ // In the interest of performance, we perform a weakly consistent read here. The biggest
+ // risk of this is that we might schedule against stale host attributes, or we might fail
+ // to correctly satisfy a diversity constraint. Given that the likelihood is relatively low
+ // for both of these, and the impact is also low, the weak consistency is acceptable.
+ return storage.weaklyConsistentRead(new Quiet<Iterable<Veto>>() {
+ @Override public Iterable<Veto> apply(final StoreProvider storeProvider) {
+ AttributeLoader attributeLoader = new AttributeLoader() {
+ @Override public Iterable<Attribute> apply(String host) {
+ return AttributeStore.Util.attributesOrNone(storeProvider, host);
+ }
+ };
+
+ Supplier<Collection<IScheduledTask>> activeTasksSupplier =
+ Suppliers.memoize(new Supplier<Collection<IScheduledTask>>() {
+ @Override public Collection<IScheduledTask> get() {
+ return storeProvider.getTaskStore().fetchTasks(
+ Query.jobScoped(Tasks.INFO_TO_JOB_KEY.apply(task))
+ .byStatus(ACTIVE_NOT_PENDING_STATES));
+ }
+ });
+
+ ConstraintFilter constraintFilter = new ConstraintFilter(
+ Tasks.INFO_TO_JOB_KEY.apply(task),
+ activeTasksSupplier,
+ attributeLoader,
+ attributeLoader.apply(slaveHost));
+ ImmutableList.Builder<Veto> vetoes = ImmutableList.builder();
+ for (IConstraint constraint : VALUES_FIRST.sortedCopy(task.getConstraints())) {
+ Optional<Veto> veto = constraintFilter.apply(constraint);
+ if (veto.isPresent()) {
+ vetoes.add(veto.get());
+ if (isValueConstraint(constraint)) {
+ // Break when a value constraint mismatch is found to avoid other
+ // potentially-expensive operations to satisfy other constraints.
+ break;
+ }
+ }
+ }
+
+ return vetoes.build();
+ }
+ });
+ }
+ };
+ }
+
+ private Optional<Veto> getMaintenanceVeto(String slaveHost) {
+ MaintenanceMode mode = maintenance.getMode(slaveHost);
+ return VETO_MODES.contains(mode)
+ ? Optional.of(ConstraintFilter.maintenanceVeto(mode.toString().toLowerCase()))
+ : NO_VETO;
+ }
+
+ private Set<Veto> getResourceVetoes(ResourceSlot offer, ITaskConfig task) {
+ ImmutableSet.Builder<Veto> builder = ImmutableSet.builder();
+ for (FilterRule rule : rulesFromOffer(offer)) {
+ builder.addAll(rule.apply(task));
+ }
+ return builder.build();
+ }
+
+ private boolean isDedicated(final String slaveHost) {
+ Iterable<Attribute> slaveAttributes =
+ storage.weaklyConsistentRead(new Quiet<Iterable<Attribute>>() {
+ @Override public Iterable<Attribute> apply(final StoreProvider storeProvider) {
+ return AttributeStore.Util.attributesOrNone(storeProvider, slaveHost);
+ }
+ });
+
+ return Iterables.any(slaveAttributes, new ConstraintFilter.NameFilter(DEDICATED_ATTRIBUTE));
+ }
+
+ @Override
+ public Set<Veto> filter(ResourceSlot offer, String slaveHost, ITaskConfig task, String taskId) {
+ if (!ConfigurationManager.isDedicated(task) && isDedicated(slaveHost)) {
+ return ImmutableSet.of(DEDICATED_HOST_VETO);
+ }
+ return ImmutableSet.<Veto>builder()
+ .addAll(getConstraintFilter(slaveHost).apply(task))
+ .addAll(getResourceVetoes(offer, task))
+ .addAll(getMaintenanceVeto(slaveHost).asSet())
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/ClusterName.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/ClusterName.java b/src/main/java/org/apache/aurora/scheduler/http/ClusterName.java
new file mode 100644
index 0000000..90c87a7
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/ClusterName.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import com.google.inject.BindingAnnotation;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Binding annotation for the cluster name.
+ */
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface ClusterName { }
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/Cron.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Cron.java b/src/main/java/org/apache/aurora/scheduler/http/Cron.java
new file mode 100644
index 0000000..8c97b5e
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/Cron.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.collect.ImmutableMap;
+
+import com.twitter.aurora.scheduler.state.CronJobManager;
+
+/**
+ * HTTP interface to dump state of the internal cron scheduler.
+ */
+@Path("/cron")
+public class Cron {
+ private final CronJobManager cronManager;
+
+ @Inject
+ Cron(CronJobManager cronManager) {
+ this.cronManager = cronManager;
+ }
+
+ /**
+ * Dumps the state of the cron manager.
+ *
+ * @return An HTTP response containing the cron manager's state.
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response dumpContents() {
+ Map<String, Object> response = ImmutableMap.<String, Object>builder()
+ .put("scheduled", cronManager.getScheduledJobs())
+ .put("pending", cronManager.getPendingRuns())
+ .build();
+
+ return Response.ok(response).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/DisplayUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/DisplayUtils.java b/src/main/java/org/apache/aurora/scheduler/http/DisplayUtils.java
new file mode 100644
index 0000000..b4a2762
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/DisplayUtils.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Ordering;
+
+import com.twitter.aurora.scheduler.MesosTaskFactory.MesosTaskFactoryImpl;
+import com.twitter.aurora.scheduler.http.SchedulerzHome.Role;
+import com.twitter.aurora.scheduler.http.SchedulerzRole.Job;
+import com.twitter.aurora.scheduler.storage.entities.IJobKey;
+import com.twitter.common.args.Arg;
+import com.twitter.common.args.CmdLine;
+
+/**
+ * Utility class to hold common display helper functions.
+ */
+public final class DisplayUtils {
+
+ @CmdLine(name = "viz_job_url_prefix", help = "URL prefix for job container stats.")
+ private static final Arg<String> VIZ_JOB_URL_PREFIX = Arg.create("");
+
+ private DisplayUtils() {
+ // Utility class.
+ }
+
+ static final Ordering<Role> ROLE_ORDERING = Ordering.natural().onResultOf(
+ new Function<Role, String>() {
+ @Override public String apply(Role role) {
+ return role.getRole();
+ }
+ });
+
+ static final Ordering<Job> JOB_ORDERING = Ordering.natural().onResultOf(
+ new Function<Job, String>() {
+ @Override public String apply(Job job) {
+ return job.getName();
+ }
+ });
+
+ static String getJobDashboardUrl(IJobKey jobKey) {
+ return VIZ_JOB_URL_PREFIX.get() + MesosTaskFactoryImpl.getJobSourceName(jobKey);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/HttpStatsFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/HttpStatsFilter.java b/src/main/java/org/apache/aurora/scheduler/http/HttpStatsFilter.java
new file mode 100644
index 0000000..a846383
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/HttpStatsFilter.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.io.IOException;
+
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpServletResponseWrapper;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import com.twitter.common.net.http.filters.AbstractHttpFilter;
+import com.twitter.common.stats.SlidingStats;
+
+/**
+ * An HTTP filter that exports counts and timing for requests based on response code.
+ */
+public class HttpStatsFilter extends AbstractHttpFilter {
+
+ private final LoadingCache<Integer, SlidingStats> counters = CacheBuilder.newBuilder()
+ .build(new CacheLoader<Integer, SlidingStats>() {
+ @Override public SlidingStats load(Integer status) {
+ return new SlidingStats("http_" + status + "_responses", "nanos");
+ }
+ });
+
+ private static class ResponseWithStatus extends HttpServletResponseWrapper {
+ // 200 response code is the default if none is explicitly set.
+ private int wrappedStatus = 200;
+
+ ResponseWithStatus(HttpServletResponse resp) {
+ super(resp);
+ }
+
+ @Override public void setStatus(int sc) {
+ super.setStatus(sc);
+ wrappedStatus = sc;
+ }
+
+ @Override public void setStatus(int sc, String sm) {
+ super.setStatus(sc, sm);
+ wrappedStatus = sc;
+ }
+ }
+
+ @Override
+ public void doFilter(HttpServletRequest request, HttpServletResponse response, FilterChain chain)
+ throws IOException, ServletException {
+
+ long start = System.nanoTime();
+ ResponseWithStatus wrapper = new ResponseWithStatus(response);
+ chain.doFilter(request, wrapper);
+ counters.getUnchecked(wrapper.wrappedStatus).accumulate(System.nanoTime() - start);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/JerseyTemplateServlet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/JerseyTemplateServlet.java b/src/main/java/org/apache/aurora/scheduler/http/JerseyTemplateServlet.java
new file mode 100644
index 0000000..5c13de5
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/JerseyTemplateServlet.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.io.StringWriter;
+
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
+
+import org.antlr.stringtemplate.StringTemplate;
+
+import com.twitter.common.base.Closure;
+import com.twitter.common.util.templating.StringTemplateHelper;
+import com.twitter.common.util.templating.StringTemplateHelper.TemplateException;
+
+/**
+ * Base class for common functions needed in a jersey stringtemplate servlet.
+ */
+abstract class JerseyTemplateServlet {
+
+ private final StringTemplateHelper templateHelper;
+
+ JerseyTemplateServlet(String templatePath) {
+ templateHelper = new StringTemplateHelper(getClass(), templatePath, true);
+ }
+
+ protected final Response fillTemplate(Closure<StringTemplate> populator) {
+ StringWriter output = new StringWriter();
+ try {
+ templateHelper.writeTemplate(output, populator);
+ } catch (TemplateException e) {
+ throw new WebApplicationException(e);
+ }
+ return Response.ok(output.toString()).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirect.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirect.java b/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirect.java
new file mode 100644
index 0000000..20e4446
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirect.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.net.HostAndPort;
+import com.google.common.util.concurrent.Atomics;
+
+import com.twitter.common.application.modules.LocalServiceRegistry;
+import com.twitter.common.net.pool.DynamicHostSet;
+import com.twitter.common.net.pool.DynamicHostSet.HostChangeMonitor;
+import com.twitter.common.net.pool.DynamicHostSet.MonitorException;
+import com.twitter.thrift.Endpoint;
+import com.twitter.thrift.ServiceInstance;
+
+/**
+ * Redirect logic for finding the leading scheduler in the event that this process is not the
+ * leader.
+ */
+public class LeaderRedirect {
+
+ // TODO(wfarner): Should we tie this directly to the producer of the node (HttpModule? It seems
+ // like the right thing to do, but would introduce an otherwise unnecessary dependency.
+ @VisibleForTesting
+ static final String HTTP_PORT_NAME = "http";
+
+ private static final Logger LOG = Logger.getLogger(LeaderRedirect.class.getName());
+
+ private final LocalServiceRegistry serviceRegistry;
+ private final DynamicHostSet<ServiceInstance> schedulers;
+
+ private final AtomicReference<ServiceInstance> leader = Atomics.newReference();
+
+ @Inject
+ LeaderRedirect(LocalServiceRegistry serviceRegistry, DynamicHostSet<ServiceInstance> schedulers) {
+ this.serviceRegistry = Preconditions.checkNotNull(serviceRegistry);
+ this.schedulers = Preconditions.checkNotNull(schedulers);
+ }
+
+ /**
+ * Initiates the monitor that will watch the scheduler host set.
+ *
+ * @throws MonitorException If monitoring failed to initialize.
+ */
+ public void monitor() throws MonitorException {
+ schedulers.watch(new SchedulerMonitor());
+ }
+
+ private Optional<HostAndPort> getLeaderHttp() {
+ ServiceInstance leadingScheduler = leader.get();
+ if (leadingScheduler == null) {
+ return Optional.absent();
+ }
+
+ if (leadingScheduler.isSetAdditionalEndpoints()) {
+ Endpoint leaderHttp = leadingScheduler.getAdditionalEndpoints().get(HTTP_PORT_NAME);
+ if (leaderHttp != null && leaderHttp.isSetHost() && leaderHttp.isSetPort()) {
+ return Optional.of(HostAndPort.fromParts(leaderHttp.getHost(), leaderHttp.getPort()));
+ }
+ }
+
+ LOG.warning("Leader service instance seems to be incomplete: " + leadingScheduler);
+ return Optional.absent();
+ }
+
+ private Optional<HostAndPort> getLocalHttp() {
+ InetSocketAddress localHttp = serviceRegistry.getAuxiliarySockets().get(HTTP_PORT_NAME);
+ return (localHttp == null) ? Optional.<HostAndPort>absent()
+ : Optional.of(HostAndPort.fromParts(localHttp.getHostName(), localHttp.getPort()));
+ }
+
+ /**
+ * Gets the optional HTTP endpoint that should be redirected to in the event that this
+ * scheduler is not the leader.
+ *
+ * @return Optional redirect target.
+ */
+ @VisibleForTesting
+ Optional<HostAndPort> getRedirect() {
+ Optional<HostAndPort> leaderHttp = getLeaderHttp();
+ Optional<HostAndPort> localHttp = getLocalHttp();
+
+ if (leaderHttp.isPresent()) {
+ if (leaderHttp.equals(localHttp)) {
+ return Optional.absent();
+ } else {
+ return leaderHttp;
+ }
+ } else {
+ LOG.info("No leader found, not redirecting.");
+ return Optional.absent();
+ }
+ }
+
+ /**
+ * Gets the optional redirect URI target in the event that this process is not the leading
+ * scheduler.
+ *
+ * @param req HTTP request.
+ * @return An optional redirect destination to route the request to the leading scheduler.
+ */
+ public Optional<String> getRedirectTarget(HttpServletRequest req) {
+ Optional<HostAndPort> redirectTarget = getRedirect();
+ if (redirectTarget.isPresent()) {
+ HostAndPort target = redirectTarget.get();
+ StringBuilder redirect = new StringBuilder()
+ .append(req.getScheme())
+ .append("://")
+ .append(target.getHostText())
+ .append(":")
+ .append(target.getPort())
+ .append(req.getRequestURI());
+
+ String queryString = req.getQueryString();
+ if (queryString != null) {
+ redirect.append("?").append(queryString);
+ }
+
+ return Optional.of(redirect.toString());
+ } else {
+ return Optional.absent();
+ }
+ }
+
+ /**
+ * Monitor to track scheduler leader changes.
+ */
+ private class SchedulerMonitor implements HostChangeMonitor<ServiceInstance> {
+ @Override public void onChange(ImmutableSet<ServiceInstance> hostSet) {
+ switch (hostSet.size()) {
+ case 0:
+ LOG.warning("No schedulers in host set, will not redirect despite not being leader.");
+ leader.set(null);
+ break;
+
+ case 1:
+ LOG.info("Found leader scheduler at " + hostSet);
+ leader.set(Iterables.getOnlyElement(hostSet));
+ break;
+
+ default:
+ LOG.severe("Multiple schedulers detected, will not redirect: " + hostSet);
+ leader.set(null);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirectFilter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirectFilter.java b/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirectFilter.java
new file mode 100644
index 0000000..86f0567
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/LeaderRedirectFilter.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.io.IOException;
+
+import javax.inject.Inject;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+import com.twitter.common.net.http.filters.AbstractHttpFilter;
+
+/**
+ * An HTTP filter that will redirect the request to the leading scheduler.
+ */
+public class LeaderRedirectFilter extends AbstractHttpFilter {
+
+ private final LeaderRedirect redirector;
+
+ @Inject
+ LeaderRedirectFilter(LeaderRedirect redirector) {
+ this.redirector = Preconditions.checkNotNull(redirector);
+ }
+
+ @Override
+ public void doFilter(HttpServletRequest request, HttpServletResponse response, FilterChain chain)
+ throws IOException, ServletException {
+
+ Optional<String> leaderRedirect = redirector.getRedirectTarget(request);
+ if (leaderRedirect.isPresent()) {
+ response.sendRedirect(leaderRedirect.get());
+ } else {
+ chain.doFilter(request, response);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/Maintenance.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Maintenance.java b/src/main/java/org/apache/aurora/scheduler/http/Maintenance.java
new file mode 100644
index 0000000..fb71539
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/Maintenance.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
+import static com.twitter.aurora.gen.MaintenanceMode.SCHEDULED;
+
+/**
+ * Servlet that exposes the maintenance state of hosts.
+ */
+@Path("/maintenance")
+public class Maintenance {
+ private final Storage storage;
+
+ @Inject
+ Maintenance(Storage storage) {
+ this.storage = Preconditions.checkNotNull(storage);
+ }
+
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getHosts() {
+ return storage.weaklyConsistentRead(new Work.Quiet<Response>() {
+ @Override public Response apply(StoreProvider storeProvider) {
+ Multimap<MaintenanceMode, String> hostsByMode =
+ Multimaps.transformValues(
+ Multimaps.index(storeProvider.getAttributeStore().getHostAttributes(), GET_MODE),
+ HOST_NAME);
+
+ Map<MaintenanceMode, Object> hosts = Maps.newHashMap();
+ hosts.put(DRAINED, ImmutableSet.copyOf(hostsByMode.get(DRAINED)));
+ hosts.put(SCHEDULED, ImmutableSet.copyOf(hostsByMode.get(SCHEDULED)));
+ hosts.put(DRAINING, getTasksByHosts(storeProvider, hostsByMode.get(DRAINING)).asMap());
+ return Response.ok(hosts).build();
+ }
+ });
+ }
+
+ private Multimap<String, String> getTasksByHosts(StoreProvider provider, Iterable<String> hosts) {
+ ImmutableSet.Builder<IScheduledTask> drainingTasks = ImmutableSet.builder();
+ for (String host : hosts) {
+ drainingTasks.addAll(provider.getTaskStore().fetchTasks(Query.slaveScoped(host).active()));
+ }
+ return Multimaps.transformValues(
+ Multimaps.index(drainingTasks.build(), TASK_TO_HOST),
+ Tasks.SCHEDULED_TO_ID);
+ }
+
+ private static final Function<IScheduledTask, String> TASK_TO_HOST =
+ new Function<IScheduledTask, String>() {
+ @Override public String apply(IScheduledTask task) {
+ return task.getAssignedTask().getSlaveHost();
+ }
+ };
+
+ private static final Function<HostAttributes, String> HOST_NAME =
+ new Function<HostAttributes, String>() {
+ @Override public String apply(HostAttributes attributes) {
+ return attributes.getHost();
+ }
+ };
+
+ private static final Function<HostAttributes, MaintenanceMode> GET_MODE =
+ new Function<HostAttributes, MaintenanceMode>() {
+ @Override public MaintenanceMode apply(HostAttributes attrs) {
+ return attrs.getMode();
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/Mname.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Mname.java b/src/main/java/org/apache/aurora/scheduler/http/Mname.java
new file mode 100644
index 0000000..c97560f
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/Mname.java
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.core.UriInfo;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+import com.twitter.aurora.scheduler.base.JobKeys;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.entities.IAssignedTask;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
+
+import static javax.ws.rs.core.Response.Status.NOT_FOUND;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import static com.twitter.aurora.gen.ScheduleStatus.RUNNING;
+
+/**
+ * Simple redirector from the canonical name of a task to its configured HTTP port.
+ *
+ * <p>Forwards for GET, PUT, POST and DELETE requests using HTTP 307 allowing compliant clients to
+ * seamlessly perform re-directed mutations.
+ */
+@Path("/mname")
+public class Mname {
+
+ private static final Set<String> HTTP_PORT_NAMES = ImmutableSet.of(
+ "health", "http", "HTTP", "web");
+
+ private final Storage storage;
+
+ @Inject
+ public Mname(Storage storage) {
+ this.storage = checkNotNull(storage);
+ }
+
+ @GET
+ @Produces(MediaType.TEXT_HTML)
+ public Response getUsage() {
+ return Response
+ .status(Status.BAD_REQUEST)
+ .entity("<html>Usage: /mname/{role}/{env}/{job}/{instance}</html>")
+ .build();
+ }
+
+ @GET
+ @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response getWithForwardRequest(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @PathParam("forward") String forward,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
+ }
+
+ @PUT
+ @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response putWithForwardRequest(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @PathParam("forward") String forward,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
+ }
+
+ @POST
+ @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response postWithForwardRequest(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @PathParam("forward") String forward,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
+ }
+
+ @DELETE
+ @Path("/{role}/{env}/{job}/{instance}/{forward:.+}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response deleteWithForwardRequest(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @PathParam("forward") String forward,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.of(forward));
+ }
+
+ @GET
+ @Path("/{role}/{env}/{job}/{instance}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response get(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
+ }
+
+ @PUT
+ @Path("/{role}/{env}/{job}/{instance}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response put(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
+ }
+
+ @POST
+ @Path("/{role}/{env}/{job}/{instance}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response post(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
+ }
+
+ @DELETE
+ @Path("/{role}/{env}/{job}/{instance}")
+ @Produces(MediaType.TEXT_HTML)
+ public Response delete(
+ @PathParam("role") String role,
+ @PathParam("env") String env,
+ @PathParam("job") String job,
+ @PathParam("instance") int instanceId,
+ @Context UriInfo uriInfo) {
+
+ return get(role, env, job, instanceId, uriInfo, Optional.<String>absent());
+ }
+
+ private Response get(
+ String role,
+ String env,
+ String job,
+ int instanceId,
+ UriInfo uriInfo,
+ Optional<String> forwardRequest) {
+
+ IScheduledTask task = Iterables.getOnlyElement(
+ Storage.Util.consistentFetchTasks(storage,
+ Query.instanceScoped(JobKeys.from(role, env, job), instanceId).active()),
+ null);
+ if (task == null) {
+ return respond(NOT_FOUND, "No such live instance found.");
+ }
+
+ if (task.getStatus() != RUNNING) {
+ return respond(NOT_FOUND, "The selected instance is currently in state " + task.getStatus());
+ }
+
+ IAssignedTask assignedTask = task.getAssignedTask();
+ Optional<Integer> port = getRedirectPort(assignedTask);
+ if (!port.isPresent()) {
+ return respond(NOT_FOUND, "The task does not have a registered http port.");
+ }
+
+ UriBuilder redirect = UriBuilder
+ .fromPath(forwardRequest.or("/"))
+ .scheme("http")
+ .host(assignedTask.getSlaveHost())
+ .port(port.get());
+ for (Entry<String, List<String>> entry : uriInfo.getQueryParameters().entrySet()) {
+ for (String value : entry.getValue()) {
+ redirect.queryParam(entry.getKey(), value);
+ }
+ }
+
+ return Response.temporaryRedirect(redirect.build()).build();
+ }
+
+ @VisibleForTesting
+ static Optional<Integer> getRedirectPort(IAssignedTask task) {
+ Map<String, Integer> ports = task.isSetAssignedPorts()
+ ? task.getAssignedPorts() : ImmutableMap.<String, Integer>of();
+ for (String httpPortName : HTTP_PORT_NAMES) {
+ Integer port = ports.get(httpPortName);
+ if (port != null) {
+ return Optional.of(port);
+ }
+ }
+ return Optional.absent();
+ }
+
+ private Response respond(Status status, String message) {
+ return Response.status(status).entity(message).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/Offers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Offers.java b/src/main/java/org/apache/aurora/scheduler/http/Offers.java
new file mode 100644
index 0000000..e90f1cc
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/Offers.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.mesos.Protos.Attribute;
+import org.apache.mesos.Protos.ExecutorID;
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.Resource;
+import org.apache.mesos.Protos.Value.Range;
+
+import com.twitter.aurora.scheduler.async.OfferQueue;
+
+/**
+ * Servlet that exposes resource offers that the scheduler is currently retaining.
+ */
+@Path("/offers")
+public class Offers {
+
+ private final OfferQueue offerQueue;
+
+ @Inject
+ Offers(OfferQueue offerQueue) {
+ this.offerQueue = Preconditions.checkNotNull(offerQueue);
+ }
+
+ /**
+ * Dumps the offers queued in the scheduler.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getOffers() {
+ return Response.ok(
+ FluentIterable.from(offerQueue.getOffers()).transform(TO_BEAN).toList()).build();
+ }
+
+ private static final Function<ExecutorID, String> EXECUTOR_ID_TOSTRING =
+ new Function<ExecutorID, String>() {
+ @Override public String apply(ExecutorID id) {
+ return id.getValue();
+ }
+ };
+
+ private static final Function<Range, Object> RANGE_TO_BEAN = new Function<Range, Object>() {
+ @Override public Object apply(Range range) {
+ return range.getBegin() + "-" + range.getEnd();
+ }
+ };
+
+ private static final Function<Attribute, Object> ATTRIBUTE_TO_BEAN =
+ new Function<Attribute, Object>() {
+ @Override public Object apply(Attribute attr) {
+ ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+ builder.put("name", attr.getName());
+ if (attr.hasScalar()) {
+ builder.put("scalar", attr.getScalar().getValue());
+ }
+ if (attr.hasRanges()) {
+ builder.put("ranges", immutable(attr.getRanges().getRangeList(), RANGE_TO_BEAN));
+ }
+ if (attr.hasSet()) {
+ builder.put("set", attr.getSet().getItemList());
+ }
+ if (attr.hasText()) {
+ builder.put("text", attr.getText().getValue());
+ }
+ return builder.build();
+ }
+ };
+
+ private static final Function<Resource, Object> RESOURCE_TO_BEAN =
+ new Function<Resource, Object>() {
+ @Override public Object apply(Resource resource) {
+ ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+ builder.put("name", resource.getName());
+ if (resource.hasScalar()) {
+ builder.put("scalar", resource.getScalar().getValue());
+ }
+ if (resource.hasRanges()) {
+ builder.put("ranges", immutable(resource.getRanges().getRangeList(), RANGE_TO_BEAN));
+ }
+ if (resource.hasSet()) {
+ builder.put("set", resource.getSet().getItemList());
+ }
+ return builder.build();
+ }
+ };
+
+ private static <A, B> Iterable<B> immutable(Iterable<A> iterable, Function<A, B> transform) {
+ return FluentIterable.from(iterable).transform(transform).toList();
+ }
+
+ private static final Function<Offer, Map<String, ?>> TO_BEAN =
+ new Function<Offer, Map<String, ?>>() {
+ @Override public Map<String, ?> apply(Offer offer) {
+ return ImmutableMap.<String, Object>builder()
+ .put("id", offer.getId().getValue())
+ .put("framework_id", offer.getFrameworkId().getValue())
+ .put("slave_id", offer.getSlaveId().getValue())
+ .put("hostname", offer.getHostname())
+ .put("resources", immutable(offer.getResourcesList(), RESOURCE_TO_BEAN))
+ .put("attributes", immutable(offer.getAttributesList(), ATTRIBUTE_TO_BEAN))
+ .put("executor_ids", immutable(offer.getExecutorIdsList(), EXECUTOR_ID_TOSTRING))
+ .build();
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/PendingTasks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/PendingTasks.java b/src/main/java/org/apache/aurora/scheduler/http/PendingTasks.java
new file mode 100644
index 0000000..359fb97
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/PendingTasks.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.base.Preconditions;
+
+import com.twitter.aurora.scheduler.async.TaskGroups;
+
+/**
+ * Servlet that exposes detailed information about tasks that are pending.
+ */
+@Path("/pendingtasks")
+public class PendingTasks {
+
+ private final TaskGroups taskGroups;
+
+ @Inject
+ PendingTasks(TaskGroups taskGroups) {
+ this.taskGroups = Preconditions.checkNotNull(taskGroups);
+ }
+
+ /**
+ * Returns information about pending tasks.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getOffers() {
+ return Response.ok(taskGroups.getGroups()).build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/bc1635df/src/main/java/org/apache/aurora/scheduler/http/Quotas.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Quotas.java b/src/main/java/org/apache/aurora/scheduler/http/Quotas.java
new file mode 100644
index 0000000..1183275
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/Quotas.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.twitter.aurora.scheduler.http;
+
+import java.util.Map;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IQuota;
+
+/**
+ * Servlet that exposes allocated resource quotas.
+ */
+@Path("/quotas")
+public class Quotas {
+
+ private final Storage storage;
+
+ @Inject
+ Quotas(Storage storage) {
+ this.storage = Preconditions.checkNotNull(storage);
+ }
+
+ /**
+ * Dumps allocated resource quotas.
+ *
+ * @return HTTP response.
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getOffers(@QueryParam("role") final String role) {
+ return storage.weaklyConsistentRead(new Work.Quiet<Response>() {
+ @Override public Response apply(StoreProvider storeProvider) {
+ Map<String, IQuota> quotas;
+ if (role == null) {
+ quotas = storeProvider.getQuotaStore().fetchQuotas();
+ } else {
+ Optional<IQuota> quota = storeProvider.getQuotaStore().fetchQuota(role);
+ if (quota.isPresent()) {
+ quotas = ImmutableMap.of(role, quota.get());
+ } else {
+ quotas = ImmutableMap.of();
+ }
+ }
+
+ return Response.ok(Maps.transformValues(quotas, TO_BEAN)).build();
+ }
+ });
+ }
+
+ private static final Function<IQuota, QuotaBean> TO_BEAN = new Function<IQuota, QuotaBean>() {
+ @Override public QuotaBean apply(IQuota quota) {
+ return new QuotaBean(quota.getNumCpus(), quota.getRamMb(), quota.getDiskMb());
+ }
+ };
+
+ private static final class QuotaBean {
+ private final double cpu;
+ private final long ramMb;
+ private final long diskMb;
+
+ private QuotaBean(double cpu, long ramMb, long diskMb) {
+ this.cpu = cpu;
+ this.ramMb = ramMb;
+ this.diskMb = diskMb;
+ }
+
+ @JsonProperty("cpu_cores")
+ public double getCpu() {
+ return cpu;
+ }
+
+ @JsonProperty("ram_mb")
+ public long getRamMb() {
+ return ramMb;
+ }
+
+ @JsonProperty("disk_mb")
+ public long getDiskMb() {
+ return diskMb;
+ }
+ }
+}