You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/09/01 23:59:30 UTC
[46/69] [abbrv] [partial] Initial work to rename package to
org.apache.spark
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/resources/spark/ui/static/sorttable.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/spark/ui/static/sorttable.js b/core/src/main/resources/spark/ui/static/sorttable.js
deleted file mode 100644
index 7abb901..0000000
--- a/core/src/main/resources/spark/ui/static/sorttable.js
+++ /dev/null
@@ -1,495 +0,0 @@
-/*
- SortTable
- version 2
- 7th April 2007
- Stuart Langridge, http://www.kryogenix.org/code/browser/sorttable/
-
- Instructions:
- Download this file
- Add <script src="sorttable.js"></script> to your HTML
- Add class="sortable" to any table you'd like to make sortable
- Click on the headers to sort
-
- Thanks to many, many people for contributions and suggestions.
- Licenced as X11: http://www.kryogenix.org/code/browser/licence.html
- This basically means: do what you want with it.
-*/
-
-
-var stIsIE = /*@cc_on!@*/false;
-
-sorttable = {
- init: function() {
- // quit if this function has already been called
- if (arguments.callee.done) return;
- // flag this function so we don't do the same thing twice
- arguments.callee.done = true;
- // kill the timer
- if (_timer) clearInterval(_timer);
-
- if (!document.createElement || !document.getElementsByTagName) return;
-
- sorttable.DATE_RE = /^(\d\d?)[\/\.-](\d\d?)[\/\.-]((\d\d)?\d\d)$/;
-
- forEach(document.getElementsByTagName('table'), function(table) {
- if (table.className.search(/\bsortable\b/) != -1) {
- sorttable.makeSortable(table);
- }
- });
-
- },
-
- makeSortable: function(table) {
- if (table.getElementsByTagName('thead').length == 0) {
- // table doesn't have a tHead. Since it should have, create one and
- // put the first table row in it.
- the = document.createElement('thead');
- the.appendChild(table.rows[0]);
- table.insertBefore(the,table.firstChild);
- }
- // Safari doesn't support table.tHead, sigh
- if (table.tHead == null) table.tHead = table.getElementsByTagName('thead')[0];
-
- if (table.tHead.rows.length != 1) return; // can't cope with two header rows
-
- // Sorttable v1 put rows with a class of "sortbottom" at the bottom (as
- // "total" rows, for example). This is B&R, since what you're supposed
- // to do is put them in a tfoot. So, if there are sortbottom rows,
- // for backwards compatibility, move them to tfoot (creating it if needed).
- sortbottomrows = [];
- for (var i=0; i<table.rows.length; i++) {
- if (table.rows[i].className.search(/\bsortbottom\b/) != -1) {
- sortbottomrows[sortbottomrows.length] = table.rows[i];
- }
- }
- if (sortbottomrows) {
- if (table.tFoot == null) {
- // table doesn't have a tfoot. Create one.
- tfo = document.createElement('tfoot');
- table.appendChild(tfo);
- }
- for (var i=0; i<sortbottomrows.length; i++) {
- tfo.appendChild(sortbottomrows[i]);
- }
- delete sortbottomrows;
- }
-
- // work through each column and calculate its type
- headrow = table.tHead.rows[0].cells;
- for (var i=0; i<headrow.length; i++) {
- // manually override the type with a sorttable_type attribute
- if (!headrow[i].className.match(/\bsorttable_nosort\b/)) { // skip this col
- mtch = headrow[i].className.match(/\bsorttable_([a-z0-9]+)\b/);
- if (mtch) { override = mtch[1]; }
- if (mtch && typeof sorttable["sort_"+override] == 'function') {
- headrow[i].sorttable_sortfunction = sorttable["sort_"+override];
- } else {
- headrow[i].sorttable_sortfunction = sorttable.guessType(table,i);
- }
- // make it clickable to sort
- headrow[i].sorttable_columnindex = i;
- headrow[i].sorttable_tbody = table.tBodies[0];
- dean_addEvent(headrow[i],"click", function(e) {
-
- if (this.className.search(/\bsorttable_sorted\b/) != -1) {
- // if we're already sorted by this column, just
- // reverse the table, which is quicker
- sorttable.reverse(this.sorttable_tbody);
- this.className = this.className.replace('sorttable_sorted',
- 'sorttable_sorted_reverse');
- this.removeChild(document.getElementById('sorttable_sortfwdind'));
- sortrevind = document.createElement('span');
- sortrevind.id = "sorttable_sortrevind";
- sortrevind.innerHTML = stIsIE ? ' <font face="webdings">5</font>' : ' ▴';
- this.appendChild(sortrevind);
- return;
- }
- if (this.className.search(/\bsorttable_sorted_reverse\b/) != -1) {
- // if we're already sorted by this column in reverse, just
- // re-reverse the table, which is quicker
- sorttable.reverse(this.sorttable_tbody);
- this.className = this.className.replace('sorttable_sorted_reverse',
- 'sorttable_sorted');
- this.removeChild(document.getElementById('sorttable_sortrevind'));
- sortfwdind = document.createElement('span');
- sortfwdind.id = "sorttable_sortfwdind";
- sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾';
- this.appendChild(sortfwdind);
- return;
- }
-
- // remove sorttable_sorted classes
- theadrow = this.parentNode;
- forEach(theadrow.childNodes, function(cell) {
- if (cell.nodeType == 1) { // an element
- cell.className = cell.className.replace('sorttable_sorted_reverse','');
- cell.className = cell.className.replace('sorttable_sorted','');
- }
- });
- sortfwdind = document.getElementById('sorttable_sortfwdind');
- if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); }
- sortrevind = document.getElementById('sorttable_sortrevind');
- if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); }
-
- this.className += ' sorttable_sorted';
- sortfwdind = document.createElement('span');
- sortfwdind.id = "sorttable_sortfwdind";
- sortfwdind.innerHTML = stIsIE ? ' <font face="webdings">6</font>' : ' ▾';
- this.appendChild(sortfwdind);
-
- // build an array to sort. This is a Schwartzian transform thing,
- // i.e., we "decorate" each row with the actual sort key,
- // sort based on the sort keys, and then put the rows back in order
- // which is a lot faster because you only do getInnerText once per row
- row_array = [];
- col = this.sorttable_columnindex;
- rows = this.sorttable_tbody.rows;
- for (var j=0; j<rows.length; j++) {
- row_array[row_array.length] = [sorttable.getInnerText(rows[j].cells[col]), rows[j]];
- }
- /* If you want a stable sort, uncomment the following line */
- //sorttable.shaker_sort(row_array, this.sorttable_sortfunction);
- /* and comment out this one */
- row_array.sort(this.sorttable_sortfunction);
-
- tb = this.sorttable_tbody;
- for (var j=0; j<row_array.length; j++) {
- tb.appendChild(row_array[j][1]);
- }
-
- delete row_array;
- });
- }
- }
- },
-
- guessType: function(table, column) {
- // guess the type of a column based on its first non-blank row
- sortfn = sorttable.sort_alpha;
- for (var i=0; i<table.tBodies[0].rows.length; i++) {
- text = sorttable.getInnerText(table.tBodies[0].rows[i].cells[column]);
- if (text != '') {
- if (text.match(/^-?[�$�]?[\d,.]+%?$/)) {
- return sorttable.sort_numeric;
- }
- // check for a date: dd/mm/yyyy or dd/mm/yy
- // can have / or . or - as separator
- // can be mm/dd as well
- possdate = text.match(sorttable.DATE_RE)
- if (possdate) {
- // looks like a date
- first = parseInt(possdate[1]);
- second = parseInt(possdate[2]);
- if (first > 12) {
- // definitely dd/mm
- return sorttable.sort_ddmm;
- } else if (second > 12) {
- return sorttable.sort_mmdd;
- } else {
- // looks like a date, but we can't tell which, so assume
- // that it's dd/mm (English imperialism!) and keep looking
- sortfn = sorttable.sort_ddmm;
- }
- }
- }
- }
- return sortfn;
- },
-
- getInnerText: function(node) {
- // gets the text we want to use for sorting for a cell.
- // strips leading and trailing whitespace.
- // this is *not* a generic getInnerText function; it's special to sorttable.
- // for example, you can override the cell text with a customkey attribute.
- // it also gets .value for <input> fields.
-
- if (!node) return "";
-
- hasInputs = (typeof node.getElementsByTagName == 'function') &&
- node.getElementsByTagName('input').length;
-
- if (node.getAttribute("sorttable_customkey") != null) {
- return node.getAttribute("sorttable_customkey");
- }
- else if (typeof node.textContent != 'undefined' && !hasInputs) {
- return node.textContent.replace(/^\s+|\s+$/g, '');
- }
- else if (typeof node.innerText != 'undefined' && !hasInputs) {
- return node.innerText.replace(/^\s+|\s+$/g, '');
- }
- else if (typeof node.text != 'undefined' && !hasInputs) {
- return node.text.replace(/^\s+|\s+$/g, '');
- }
- else {
- switch (node.nodeType) {
- case 3:
- if (node.nodeName.toLowerCase() == 'input') {
- return node.value.replace(/^\s+|\s+$/g, '');
- }
- case 4:
- return node.nodeValue.replace(/^\s+|\s+$/g, '');
- break;
- case 1:
- case 11:
- var innerText = '';
- for (var i = 0; i < node.childNodes.length; i++) {
- innerText += sorttable.getInnerText(node.childNodes[i]);
- }
- return innerText.replace(/^\s+|\s+$/g, '');
- break;
- default:
- return '';
- }
- }
- },
-
- reverse: function(tbody) {
- // reverse the rows in a tbody
- newrows = [];
- for (var i=0; i<tbody.rows.length; i++) {
- newrows[newrows.length] = tbody.rows[i];
- }
- for (var i=newrows.length-1; i>=0; i--) {
- tbody.appendChild(newrows[i]);
- }
- delete newrows;
- },
-
- /* sort functions
- each sort function takes two parameters, a and b
- you are comparing a[0] and b[0] */
- sort_numeric: function(a,b) {
- aa = parseFloat(a[0].replace(/[^0-9.-]/g,''));
- if (isNaN(aa)) aa = 0;
- bb = parseFloat(b[0].replace(/[^0-9.-]/g,''));
- if (isNaN(bb)) bb = 0;
- return aa-bb;
- },
- sort_alpha: function(a,b) {
- if (a[0]==b[0]) return 0;
- if (a[0]<b[0]) return -1;
- return 1;
- },
- sort_ddmm: function(a,b) {
- mtch = a[0].match(sorttable.DATE_RE);
- y = mtch[3]; m = mtch[2]; d = mtch[1];
- if (m.length == 1) m = '0'+m;
- if (d.length == 1) d = '0'+d;
- dt1 = y+m+d;
- mtch = b[0].match(sorttable.DATE_RE);
- y = mtch[3]; m = mtch[2]; d = mtch[1];
- if (m.length == 1) m = '0'+m;
- if (d.length == 1) d = '0'+d;
- dt2 = y+m+d;
- if (dt1==dt2) return 0;
- if (dt1<dt2) return -1;
- return 1;
- },
- sort_mmdd: function(a,b) {
- mtch = a[0].match(sorttable.DATE_RE);
- y = mtch[3]; d = mtch[2]; m = mtch[1];
- if (m.length == 1) m = '0'+m;
- if (d.length == 1) d = '0'+d;
- dt1 = y+m+d;
- mtch = b[0].match(sorttable.DATE_RE);
- y = mtch[3]; d = mtch[2]; m = mtch[1];
- if (m.length == 1) m = '0'+m;
- if (d.length == 1) d = '0'+d;
- dt2 = y+m+d;
- if (dt1==dt2) return 0;
- if (dt1<dt2) return -1;
- return 1;
- },
-
- shaker_sort: function(list, comp_func) {
- // A stable sort function to allow multi-level sorting of data
- // see: http://en.wikipedia.org/wiki/Cocktail_sort
- // thanks to Joseph Nahmias
- var b = 0;
- var t = list.length - 1;
- var swap = true;
-
- while(swap) {
- swap = false;
- for(var i = b; i < t; ++i) {
- if ( comp_func(list[i], list[i+1]) > 0 ) {
- var q = list[i]; list[i] = list[i+1]; list[i+1] = q;
- swap = true;
- }
- } // for
- t--;
-
- if (!swap) break;
-
- for(var i = t; i > b; --i) {
- if ( comp_func(list[i], list[i-1]) < 0 ) {
- var q = list[i]; list[i] = list[i-1]; list[i-1] = q;
- swap = true;
- }
- } // for
- b++;
-
- } // while(swap)
- }
-}
-
-/* ******************************************************************
- Supporting functions: bundled here to avoid depending on a library
- ****************************************************************** */
-
-// Dean Edwards/Matthias Miller/John Resig
-
-/* for Mozilla/Opera9 */
-if (document.addEventListener) {
- document.addEventListener("DOMContentLoaded", sorttable.init, false);
-}
-
-/* for Internet Explorer */
-/*@cc_on @*/
-/*@if (@_win32)
- document.write("<script id=__ie_onload defer src=javascript:void(0)><\/script>");
- var script = document.getElementById("__ie_onload");
- script.onreadystatechange = function() {
- if (this.readyState == "complete") {
- sorttable.init(); // call the onload handler
- }
- };
-/*@end @*/
-
-/* for Safari */
-if (/WebKit/i.test(navigator.userAgent)) { // sniff
- var _timer = setInterval(function() {
- if (/loaded|complete/.test(document.readyState)) {
- sorttable.init(); // call the onload handler
- }
- }, 10);
-}
-
-/* for other browsers */
-window.onload = sorttable.init;
-
-// written by Dean Edwards, 2005
-// with input from Tino Zijdel, Matthias Miller, Diego Perini
-
-// http://dean.edwards.name/weblog/2005/10/add-event/
-
-function dean_addEvent(element, type, handler) {
- if (element.addEventListener) {
- element.addEventListener(type, handler, false);
- } else {
- // assign each event handler a unique ID
- if (!handler.$$guid) handler.$$guid = dean_addEvent.guid++;
- // create a hash table of event types for the element
- if (!element.events) element.events = {};
- // create a hash table of event handlers for each element/event pair
- var handlers = element.events[type];
- if (!handlers) {
- handlers = element.events[type] = {};
- // store the existing event handler (if there is one)
- if (element["on" + type]) {
- handlers[0] = element["on" + type];
- }
- }
- // store the event handler in the hash table
- handlers[handler.$$guid] = handler;
- // assign a global event handler to do all the work
- element["on" + type] = handleEvent;
- }
-};
-// a counter used to create unique IDs
-dean_addEvent.guid = 1;
-
-function removeEvent(element, type, handler) {
- if (element.removeEventListener) {
- element.removeEventListener(type, handler, false);
- } else {
- // delete the event handler from the hash table
- if (element.events && element.events[type]) {
- delete element.events[type][handler.$$guid];
- }
- }
-};
-
-function handleEvent(event) {
- var returnValue = true;
- // grab the event object (IE uses a global event object)
- event = event || fixEvent(((this.ownerDocument || this.document || this).parentWindow || window).event);
- // get a reference to the hash table of event handlers
- var handlers = this.events[event.type];
- // execute each event handler
- for (var i in handlers) {
- this.$$handleEvent = handlers[i];
- if (this.$$handleEvent(event) === false) {
- returnValue = false;
- }
- }
- return returnValue;
-};
-
-function fixEvent(event) {
- // add W3C standard event methods
- event.preventDefault = fixEvent.preventDefault;
- event.stopPropagation = fixEvent.stopPropagation;
- return event;
-};
-fixEvent.preventDefault = function() {
- this.returnValue = false;
-};
-fixEvent.stopPropagation = function() {
- this.cancelBubble = true;
-}
-
-// Dean's forEach: http://dean.edwards.name/base/forEach.js
-/*
- forEach, version 1.0
- Copyright 2006, Dean Edwards
- License: http://www.opensource.org/licenses/mit-license.php
-*/
-
-// array-like enumeration
-if (!Array.forEach) { // mozilla already supports this
- Array.forEach = function(array, block, context) {
- for (var i = 0; i < array.length; i++) {
- block.call(context, array[i], i, array);
- }
- };
-}
-
-// generic enumeration
-Function.prototype.forEach = function(object, block, context) {
- for (var key in object) {
- if (typeof this.prototype[key] == "undefined") {
- block.call(context, object[key], key, object);
- }
- }
-};
-
-// character enumeration
-String.forEach = function(string, block, context) {
- Array.forEach(string.split(""), function(chr, index) {
- block.call(context, chr, index, string);
- });
-};
-
-// globally resolve forEach enumeration
-var forEach = function(object, block, context) {
- if (object) {
- var resolve = Object; // default
- if (object instanceof Function) {
- // functions have a "length" property
- resolve = Function;
- } else if (object.forEach instanceof Function) {
- // the object implements a custom forEach method so use that
- object.forEach(block, context);
- return;
- } else if (typeof object == "string") {
- // the object is a string
- resolve = String;
- } else if (typeof object.length == "number") {
- // the object is array-like
- resolve = Array;
- }
- resolve.forEach(object, block, context);
- }
-};
-
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/resources/spark/ui/static/spark-logo-77x50px-hd.png
----------------------------------------------------------------------
diff --git a/core/src/main/resources/spark/ui/static/spark-logo-77x50px-hd.png b/core/src/main/resources/spark/ui/static/spark-logo-77x50px-hd.png
deleted file mode 100644
index 6c5f099..0000000
Binary files a/core/src/main/resources/spark/ui/static/spark-logo-77x50px-hd.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/resources/spark/ui/static/spark_logo.png
----------------------------------------------------------------------
diff --git a/core/src/main/resources/spark/ui/static/spark_logo.png b/core/src/main/resources/spark/ui/static/spark_logo.png
deleted file mode 100644
index 4b18734..0000000
Binary files a/core/src/main/resources/spark/ui/static/spark_logo.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/resources/spark/ui/static/webui.css
----------------------------------------------------------------------
diff --git a/core/src/main/resources/spark/ui/static/webui.css b/core/src/main/resources/spark/ui/static/webui.css
deleted file mode 100644
index a600c06..0000000
--- a/core/src/main/resources/spark/ui/static/webui.css
+++ /dev/null
@@ -1,63 +0,0 @@
-.navbar {
- height: 50px;
- font-size: 15px;
- margin-bottom: 15px;
-}
-
-.navbar .navbar-inner {
- height: 50px;
-}
-
-.navbar .brand {
- margin-right: 20px;
- margin-bottom: 0;
- margin-top: 0;
- margin-left: 10px;
- padding: 0;
-}
-
-.navbar .nav > li {
- height: 50px;
-}
-
-.navbar .nav > li a {
- height: 30px;
- line-height: 30px;
-}
-
-.navbar-text {
- height: 50px;
- line-height: 50px;
-}
-
-table.sortable thead {
- cursor: pointer;
-}
-
-.progress {
- margin-bottom: 0px; position: relative
-}
-
-.progress-completed .bar,
-.progress .bar-completed {
- background-color: #3EC0FF;
- background-image: -moz-linear-gradient(top, #44CBFF, #34B0EE);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#44CBFF), to(#34B0EE));
- background-image: -webkit-linear-gradient(top, #44CBFF, #34B0EE);
- background-image: -o-linear-gradient(top, #44CBFF, #34B0EE);
- background-image: linear-gradient(to bottom, #64CBFF, #54B0EE);
- background-repeat: repeat-x;
- filter: progid:dximagetransform.microsoft.gradient(startColorstr='#FF44CBFF', endColorstr='#FF34B0EE', GradientType=0);
-}
-
-.progress-running .bar,
-.progress .bar-running {
- background-color: #A0DFFF;
- background-image: -moz-linear-gradient(top, #A4EDFF, #94DDFF);
- background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#A4EDFF), to(#94DDFF));
- background-image: -webkit-linear-gradient(top, #A4EDFF, #94DDFF);
- background-image: -o-linear-gradient(top, #A4EDFF, #94DDFF);
- background-image: linear-gradient(to bottom, #A4EDFF, #94DDFF);
- background-repeat: repeat-x;
- filter: progid:dximagetransform.microsoft.gradient(startColorstr='#FFA4EDFF', endColorstr='#FF94DDFF', GradientType=0);
-}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/Accumulators.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Accumulators.scala b/core/src/main/scala/org/apache/spark/Accumulators.scala
new file mode 100644
index 0000000..5177ee5
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/Accumulators.scala
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.io._
+
+import scala.collection.mutable.Map
+import scala.collection.generic.Growable
+
+/**
+ * A datatype that can be accumulated, i.e. has an commutative and associative "add" operation,
+ * but where the result type, `R`, may be different from the element type being added, `T`.
+ *
+ * You must define how to add data, and how to merge two of these together. For some datatypes,
+ * such as a counter, these might be the same operation. In that case, you can use the simpler
+ * [[org.apache.spark.Accumulator]]. They won't always be the same, though -- e.g., imagine you are
+ * accumulating a set. You will add items to the set, and you will union two sets together.
+ *
+ * @param initialValue initial value of accumulator
+ * @param param helper object defining how to add elements of type `R` and `T`
+ * @tparam R the full accumulated data (result type)
+ * @tparam T partial data that can be added in
+ */
+class Accumulable[R, T] (
+ @transient initialValue: R,
+ param: AccumulableParam[R, T])
+ extends Serializable {
+
+ val id = Accumulators.newId
+ @transient private var value_ = initialValue // Current value on master
+ val zero = param.zero(initialValue) // Zero value to be passed to workers
+ var deserialized = false
+
+ Accumulators.register(this, true)
+
+ /**
+ * Add more data to this accumulator / accumulable
+ * @param term the data to add
+ */
+ def += (term: T) { value_ = param.addAccumulator(value_, term) }
+
+ /**
+ * Add more data to this accumulator / accumulable
+ * @param term the data to add
+ */
+ def add(term: T) { value_ = param.addAccumulator(value_, term) }
+
+ /**
+ * Merge two accumulable objects together
+ *
+ * Normally, a user will not want to use this version, but will instead call `+=`.
+ * @param term the other `R` that will get merged with this
+ */
+ def ++= (term: R) { value_ = param.addInPlace(value_, term)}
+
+ /**
+ * Merge two accumulable objects together
+ *
+ * Normally, a user will not want to use this version, but will instead call `add`.
+ * @param term the other `R` that will get merged with this
+ */
+ def merge(term: R) { value_ = param.addInPlace(value_, term)}
+
+ /**
+ * Access the accumulator's current value; only allowed on master.
+ */
+ def value: R = {
+ if (!deserialized) {
+ value_
+ } else {
+ throw new UnsupportedOperationException("Can't read accumulator value in task")
+ }
+ }
+
+ /**
+ * Get the current value of this accumulator from within a task.
+ *
+ * This is NOT the global value of the accumulator. To get the global value after a
+ * completed operation on the dataset, call `value`.
+ *
+ * The typical use of this method is to directly mutate the local value, eg., to add
+ * an element to a Set.
+ */
+ def localValue = value_
+
+ /**
+ * Set the accumulator's value; only allowed on master.
+ */
+ def value_= (newValue: R) {
+ if (!deserialized) value_ = newValue
+ else throw new UnsupportedOperationException("Can't assign accumulator value in task")
+ }
+
+ /**
+ * Set the accumulator's value; only allowed on master
+ */
+ def setValue(newValue: R) {
+ this.value = newValue
+ }
+
+ // Called by Java when deserializing an object
+ private def readObject(in: ObjectInputStream) {
+ in.defaultReadObject()
+ value_ = zero
+ deserialized = true
+ Accumulators.register(this, false)
+ }
+
+ override def toString = value_.toString
+}
+
+/**
+ * Helper object defining how to accumulate values of a particular type. An implicit
+ * AccumulableParam needs to be available when you create Accumulables of a specific type.
+ *
+ * @tparam R the full accumulated data (result type)
+ * @tparam T partial data that can be added in
+ */
+trait AccumulableParam[R, T] extends Serializable {
+ /**
+ * Add additional data to the accumulator value. Is allowed to modify and return `r`
+ * for efficiency (to avoid allocating objects).
+ *
+ * @param r the current value of the accumulator
+ * @param t the data to be added to the accumulator
+ * @return the new value of the accumulator
+ */
+ def addAccumulator(r: R, t: T): R
+
+ /**
+ * Merge two accumulated values together. Is allowed to modify and return the first value
+ * for efficiency (to avoid allocating objects).
+ *
+ * @param r1 one set of accumulated data
+ * @param r2 another set of accumulated data
+ * @return both data sets merged together
+ */
+ def addInPlace(r1: R, r2: R): R
+
+ /**
+ * Return the "zero" (identity) value for an accumulator type, given its initial value. For
+ * example, if R was a vector of N dimensions, this would return a vector of N zeroes.
+ */
+ def zero(initialValue: R): R
+}
+
+private[spark]
+class GrowableAccumulableParam[R <% Growable[T] with TraversableOnce[T] with Serializable, T]
+ extends AccumulableParam[R,T] {
+
+ def addAccumulator(growable: R, elem: T): R = {
+ growable += elem
+ growable
+ }
+
+ def addInPlace(t1: R, t2: R): R = {
+ t1 ++= t2
+ t1
+ }
+
+ def zero(initialValue: R): R = {
+ // We need to clone initialValue, but it's hard to specify that R should also be Cloneable.
+ // Instead we'll serialize it to a buffer and load it back.
+ val ser = new JavaSerializer().newInstance()
+ val copy = ser.deserialize[R](ser.serialize(initialValue))
+ copy.clear() // In case it contained stuff
+ copy
+ }
+}
+
+/**
+ * A simpler value of [[org.apache.spark.Accumulable]] where the result type being accumulated is the same
+ * as the types of elements being merged.
+ *
+ * @param initialValue initial value of accumulator
+ * @param param helper object defining how to add elements of type `T`
+ * @tparam T result type
+ */
+class Accumulator[T](@transient initialValue: T, param: AccumulatorParam[T])
+ extends Accumulable[T,T](initialValue, param)
+
+/**
+ * A simpler version of [[org.apache.spark.AccumulableParam]] where the only datatype you can add in is the same type
+ * as the accumulated value. An implicit AccumulatorParam object needs to be available when you create
+ * Accumulators of a specific type.
+ *
+ * @tparam T type of value to accumulate
+ */
+trait AccumulatorParam[T] extends AccumulableParam[T, T] {
+ def addAccumulator(t1: T, t2: T): T = {
+ addInPlace(t1, t2)
+ }
+}
+
+// TODO: The multi-thread support in accumulators is kind of lame; check
+// if there's a more intuitive way of doing it right
+private object Accumulators {
+ // TODO: Use soft references? => need to make readObject work properly then
+ val originals = Map[Long, Accumulable[_, _]]()
+ val localAccums = Map[Thread, Map[Long, Accumulable[_, _]]]()
+ var lastId: Long = 0
+
+ def newId: Long = synchronized {
+ lastId += 1
+ return lastId
+ }
+
+ def register(a: Accumulable[_, _], original: Boolean): Unit = synchronized {
+ if (original) {
+ originals(a.id) = a
+ } else {
+ val accums = localAccums.getOrElseUpdate(Thread.currentThread, Map())
+ accums(a.id) = a
+ }
+ }
+
+ // Clear the local (non-original) accumulators for the current thread
+ def clear() {
+ synchronized {
+ localAccums.remove(Thread.currentThread)
+ }
+ }
+
+ // Get the values of the local accumulators for the current thread (by ID)
+ def values: Map[Long, Any] = synchronized {
+ val ret = Map[Long, Any]()
+ for ((id, accum) <- localAccums.getOrElse(Thread.currentThread, Map())) {
+ ret(id) = accum.localValue
+ }
+ return ret
+ }
+
+ // Add values to the original accumulators with some given IDs
+ def add(values: Map[Long, Any]): Unit = synchronized {
+ for ((id, value) <- values) {
+ if (originals.contains(id)) {
+ originals(id).asInstanceOf[Accumulable[Any, Any]] ++= value
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/Aggregator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Aggregator.scala b/core/src/main/scala/org/apache/spark/Aggregator.scala
new file mode 100644
index 0000000..3ef4029
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/Aggregator.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.util.{HashMap => JHashMap}
+
+import scala.collection.JavaConversions._
+
+/** A set of functions used to aggregate data.
+ *
+ * @param createCombiner function to create the initial value of the aggregation.
+ * @param mergeValue function to merge a new value into the aggregation result.
+ * @param mergeCombiners function to merge outputs from multiple mergeValue function.
+ */
+case class Aggregator[K, V, C] (
+ createCombiner: V => C,
+ mergeValue: (C, V) => C,
+ mergeCombiners: (C, C) => C) {
+
+ def combineValuesByKey(iter: Iterator[_ <: Product2[K, V]]) : Iterator[(K, C)] = {
+ val combiners = new JHashMap[K, C]
+ for (kv <- iter) {
+ val oldC = combiners.get(kv._1)
+ if (oldC == null) {
+ combiners.put(kv._1, createCombiner(kv._2))
+ } else {
+ combiners.put(kv._1, mergeValue(oldC, kv._2))
+ }
+ }
+ combiners.iterator
+ }
+
+ def combineCombinersByKey(iter: Iterator[(K, C)]) : Iterator[(K, C)] = {
+ val combiners = new JHashMap[K, C]
+ iter.foreach { case(k, c) =>
+ val oldC = combiners.get(k)
+ if (oldC == null) {
+ combiners.put(k, c)
+ } else {
+ combiners.put(k, mergeCombiners(oldC, c))
+ }
+ }
+ combiners.iterator
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/BlockStoreShuffleFetcher.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/BlockStoreShuffleFetcher.scala b/core/src/main/scala/org/apache/spark/BlockStoreShuffleFetcher.scala
new file mode 100644
index 0000000..908ff56
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/BlockStoreShuffleFetcher.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.mutable.HashMap
+
+import org.apache.spark.executor.{ShuffleReadMetrics, TaskMetrics}
+import org.apache.spark.serializer.Serializer
+import org.apache.spark.storage.BlockManagerId
+import org.apache.spark.util.CompletionIterator
+
+
+private[spark] class BlockStoreShuffleFetcher extends ShuffleFetcher with Logging {
+
+ override def fetch[T](shuffleId: Int, reduceId: Int, metrics: TaskMetrics, serializer: Serializer)
+ : Iterator[T] =
+ {
+
+ logDebug("Fetching outputs for shuffle %d, reduce %d".format(shuffleId, reduceId))
+ val blockManager = SparkEnv.get.blockManager
+
+ val startTime = System.currentTimeMillis
+ val statuses = SparkEnv.get.mapOutputTracker.getServerStatuses(shuffleId, reduceId)
+ logDebug("Fetching map output location for shuffle %d, reduce %d took %d ms".format(
+ shuffleId, reduceId, System.currentTimeMillis - startTime))
+
+ val splitsByAddress = new HashMap[BlockManagerId, ArrayBuffer[(Int, Long)]]
+ for (((address, size), index) <- statuses.zipWithIndex) {
+ splitsByAddress.getOrElseUpdate(address, ArrayBuffer()) += ((index, size))
+ }
+
+ val blocksByAddress: Seq[(BlockManagerId, Seq[(String, Long)])] = splitsByAddress.toSeq.map {
+ case (address, splits) =>
+ (address, splits.map(s => ("shuffle_%d_%d_%d".format(shuffleId, s._1, reduceId), s._2)))
+ }
+
+ def unpackBlock(blockPair: (String, Option[Iterator[Any]])) : Iterator[T] = {
+ val blockId = blockPair._1
+ val blockOption = blockPair._2
+ blockOption match {
+ case Some(block) => {
+ block.asInstanceOf[Iterator[T]]
+ }
+ case None => {
+ val regex = "shuffle_([0-9]*)_([0-9]*)_([0-9]*)".r
+ blockId match {
+ case regex(shufId, mapId, _) =>
+ val address = statuses(mapId.toInt)._1
+ throw new FetchFailedException(address, shufId.toInt, mapId.toInt, reduceId, null)
+ case _ =>
+ throw new SparkException(
+ "Failed to get block " + blockId + ", which is not a shuffle block")
+ }
+ }
+ }
+ }
+
+ val blockFetcherItr = blockManager.getMultiple(blocksByAddress, serializer)
+ val itr = blockFetcherItr.flatMap(unpackBlock)
+
+ CompletionIterator[T, Iterator[T]](itr, {
+ val shuffleMetrics = new ShuffleReadMetrics
+ shuffleMetrics.shuffleFinishTime = System.currentTimeMillis
+ shuffleMetrics.remoteFetchTime = blockFetcherItr.remoteFetchTime
+ shuffleMetrics.fetchWaitTime = blockFetcherItr.fetchWaitTime
+ shuffleMetrics.remoteBytesRead = blockFetcherItr.remoteBytesRead
+ shuffleMetrics.totalBlocksFetched = blockFetcherItr.totalBlocks
+ shuffleMetrics.localBlocksFetched = blockFetcherItr.numLocalBlocks
+ shuffleMetrics.remoteBlocksFetched = blockFetcherItr.numRemoteBlocks
+ metrics.shuffleReadMetrics = Some(shuffleMetrics)
+ })
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/CacheManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/CacheManager.scala b/core/src/main/scala/org/apache/spark/CacheManager.scala
new file mode 100644
index 0000000..42e465b
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/CacheManager.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import scala.collection.mutable.{ArrayBuffer, HashSet}
+import org.apache.spark.storage.{BlockManager, StorageLevel}
+
+
+/** Spark class responsible for passing RDDs split contents to the BlockManager and making
+ sure a node doesn't load two copies of an RDD at once.
+ */
+private[spark] class CacheManager(blockManager: BlockManager) extends Logging {
+ private val loading = new HashSet[String]
+
+ /** Gets or computes an RDD split. Used by RDD.iterator() when an RDD is cached. */
+ def getOrCompute[T](rdd: RDD[T], split: Partition, context: TaskContext, storageLevel: StorageLevel)
+ : Iterator[T] = {
+ val key = "rdd_%d_%d".format(rdd.id, split.index)
+ logInfo("Cache key is " + key)
+ blockManager.get(key) match {
+ case Some(cachedValues) =>
+ // Partition is in cache, so just return its values
+ logInfo("Found partition in cache!")
+ return cachedValues.asInstanceOf[Iterator[T]]
+
+ case None =>
+ // Mark the split as loading (unless someone else marks it first)
+ loading.synchronized {
+ if (loading.contains(key)) {
+ logInfo("Loading contains " + key + ", waiting...")
+ while (loading.contains(key)) {
+ try {loading.wait()} catch {case _ : Throwable =>}
+ }
+ logInfo("Loading no longer contains " + key + ", so returning cached result")
+ // See whether someone else has successfully loaded it. The main way this would fail
+ // is for the RDD-level cache eviction policy if someone else has loaded the same RDD
+ // partition but we didn't want to make space for it. However, that case is unlikely
+ // because it's unlikely that two threads would work on the same RDD partition. One
+ // downside of the current code is that threads wait serially if this does happen.
+ blockManager.get(key) match {
+ case Some(values) =>
+ return values.asInstanceOf[Iterator[T]]
+ case None =>
+ logInfo("Whoever was loading " + key + " failed; we'll try it ourselves")
+ loading.add(key)
+ }
+ } else {
+ loading.add(key)
+ }
+ }
+ try {
+ // If we got here, we have to load the split
+ val elements = new ArrayBuffer[Any]
+ logInfo("Computing partition " + split)
+ elements ++= rdd.computeOrReadCheckpoint(split, context)
+ // Try to put this block in the blockManager
+ blockManager.put(key, elements, storageLevel, true)
+ return elements.iterator.asInstanceOf[Iterator[T]]
+ } finally {
+ loading.synchronized {
+ loading.remove(key)
+ loading.notifyAll()
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/ClosureCleaner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ClosureCleaner.scala b/core/src/main/scala/org/apache/spark/ClosureCleaner.scala
new file mode 100644
index 0000000..71d9e62
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/ClosureCleaner.scala
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.lang.reflect.Field
+
+import scala.collection.mutable.Map
+import scala.collection.mutable.Set
+
+import org.objectweb.asm.{ClassReader, ClassVisitor, MethodVisitor, Type}
+import org.objectweb.asm.Opcodes._
+import java.io.{InputStream, IOException, ByteArrayOutputStream, ByteArrayInputStream, BufferedInputStream}
+
+private[spark] object ClosureCleaner extends Logging {
+ // Get an ASM class reader for a given class from the JAR that loaded it
+ private def getClassReader(cls: Class[_]): ClassReader = {
+ // Copy data over, before delegating to ClassReader - else we can run out of open file handles.
+ val className = cls.getName.replaceFirst("^.*\\.", "") + ".class"
+ val resourceStream = cls.getResourceAsStream(className)
+ // todo: Fixme - continuing with earlier behavior ...
+ if (resourceStream == null) return new ClassReader(resourceStream)
+
+ val baos = new ByteArrayOutputStream(128)
+ Utils.copyStream(resourceStream, baos, true)
+ new ClassReader(new ByteArrayInputStream(baos.toByteArray))
+ }
+
+ // Check whether a class represents a Scala closure
+ private def isClosure(cls: Class[_]): Boolean = {
+ cls.getName.contains("$anonfun$")
+ }
+
+ // Get a list of the classes of the outer objects of a given closure object, obj;
+ // the outer objects are defined as any closures that obj is nested within, plus
+ // possibly the class that the outermost closure is in, if any. We stop searching
+ // for outer objects beyond that because cloning the user's object is probably
+ // not a good idea (whereas we can clone closure objects just fine since we
+ // understand how all their fields are used).
+ private def getOuterClasses(obj: AnyRef): List[Class[_]] = {
+ for (f <- obj.getClass.getDeclaredFields if f.getName == "$outer") {
+ f.setAccessible(true)
+ if (isClosure(f.getType)) {
+ return f.getType :: getOuterClasses(f.get(obj))
+ } else {
+ return f.getType :: Nil // Stop at the first $outer that is not a closure
+ }
+ }
+ return Nil
+ }
+
+ // Get a list of the outer objects for a given closure object.
+ private def getOuterObjects(obj: AnyRef): List[AnyRef] = {
+ for (f <- obj.getClass.getDeclaredFields if f.getName == "$outer") {
+ f.setAccessible(true)
+ if (isClosure(f.getType)) {
+ return f.get(obj) :: getOuterObjects(f.get(obj))
+ } else {
+ return f.get(obj) :: Nil // Stop at the first $outer that is not a closure
+ }
+ }
+ return Nil
+ }
+
+ private def getInnerClasses(obj: AnyRef): List[Class[_]] = {
+ val seen = Set[Class[_]](obj.getClass)
+ var stack = List[Class[_]](obj.getClass)
+ while (!stack.isEmpty) {
+ val cr = getClassReader(stack.head)
+ stack = stack.tail
+ val set = Set[Class[_]]()
+ cr.accept(new InnerClosureFinder(set), 0)
+ for (cls <- set -- seen) {
+ seen += cls
+ stack = cls :: stack
+ }
+ }
+ return (seen - obj.getClass).toList
+ }
+
+ private def createNullValue(cls: Class[_]): AnyRef = {
+ if (cls.isPrimitive) {
+ new java.lang.Byte(0: Byte) // Should be convertible to any primitive type
+ } else {
+ null
+ }
+ }
+
+ def clean(func: AnyRef) {
+ // TODO: cache outerClasses / innerClasses / accessedFields
+ val outerClasses = getOuterClasses(func)
+ val innerClasses = getInnerClasses(func)
+ val outerObjects = getOuterObjects(func)
+
+ val accessedFields = Map[Class[_], Set[String]]()
+ for (cls <- outerClasses)
+ accessedFields(cls) = Set[String]()
+ for (cls <- func.getClass :: innerClasses)
+ getClassReader(cls).accept(new FieldAccessFinder(accessedFields), 0)
+ //logInfo("accessedFields: " + accessedFields)
+
+ val inInterpreter = {
+ try {
+ val interpClass = Class.forName("spark.repl.Main")
+ interpClass.getMethod("interp").invoke(null) != null
+ } catch {
+ case _: ClassNotFoundException => true
+ }
+ }
+
+ var outerPairs: List[(Class[_], AnyRef)] = (outerClasses zip outerObjects).reverse
+ var outer: AnyRef = null
+ if (outerPairs.size > 0 && !isClosure(outerPairs.head._1)) {
+ // The closure is ultimately nested inside a class; keep the object of that
+ // class without cloning it since we don't want to clone the user's objects.
+ outer = outerPairs.head._2
+ outerPairs = outerPairs.tail
+ }
+ // Clone the closure objects themselves, nulling out any fields that are not
+ // used in the closure we're working on or any of its inner closures.
+ for ((cls, obj) <- outerPairs) {
+ outer = instantiateClass(cls, outer, inInterpreter)
+ for (fieldName <- accessedFields(cls)) {
+ val field = cls.getDeclaredField(fieldName)
+ field.setAccessible(true)
+ val value = field.get(obj)
+ //logInfo("1: Setting " + fieldName + " on " + cls + " to " + value);
+ field.set(outer, value)
+ }
+ }
+
+ if (outer != null) {
+ //logInfo("2: Setting $outer on " + func.getClass + " to " + outer);
+ val field = func.getClass.getDeclaredField("$outer")
+ field.setAccessible(true)
+ field.set(func, outer)
+ }
+ }
+
+ private def instantiateClass(cls: Class[_], outer: AnyRef, inInterpreter: Boolean): AnyRef = {
+ //logInfo("Creating a " + cls + " with outer = " + outer)
+ if (!inInterpreter) {
+ // This is a bona fide closure class, whose constructor has no effects
+ // other than to set its fields, so use its constructor
+ val cons = cls.getConstructors()(0)
+ val params = cons.getParameterTypes.map(createNullValue).toArray
+ if (outer != null)
+ params(0) = outer // First param is always outer object
+ return cons.newInstance(params: _*).asInstanceOf[AnyRef]
+ } else {
+ // Use reflection to instantiate object without calling constructor
+ val rf = sun.reflect.ReflectionFactory.getReflectionFactory()
+ val parentCtor = classOf[java.lang.Object].getDeclaredConstructor()
+ val newCtor = rf.newConstructorForSerialization(cls, parentCtor)
+ val obj = newCtor.newInstance().asInstanceOf[AnyRef]
+ if (outer != null) {
+ //logInfo("3: Setting $outer on " + cls + " to " + outer);
+ val field = cls.getDeclaredField("$outer")
+ field.setAccessible(true)
+ field.set(obj, outer)
+ }
+ return obj
+ }
+ }
+}
+
+private[spark] class FieldAccessFinder(output: Map[Class[_], Set[String]]) extends ClassVisitor(ASM4) {
+ override def visitMethod(access: Int, name: String, desc: String,
+ sig: String, exceptions: Array[String]): MethodVisitor = {
+ return new MethodVisitor(ASM4) {
+ override def visitFieldInsn(op: Int, owner: String, name: String, desc: String) {
+ if (op == GETFIELD) {
+ for (cl <- output.keys if cl.getName == owner.replace('/', '.')) {
+ output(cl) += name
+ }
+ }
+ }
+
+ override def visitMethodInsn(op: Int, owner: String, name: String,
+ desc: String) {
+ // Check for calls a getter method for a variable in an interpreter wrapper object.
+ // This means that the corresponding field will be accessed, so we should save it.
+ if (op == INVOKEVIRTUAL && owner.endsWith("$iwC") && !name.endsWith("$outer")) {
+ for (cl <- output.keys if cl.getName == owner.replace('/', '.')) {
+ output(cl) += name
+ }
+ }
+ }
+ }
+ }
+}
+
+private[spark] class InnerClosureFinder(output: Set[Class[_]]) extends ClassVisitor(ASM4) {
+ var myName: String = null
+
+ override def visit(version: Int, access: Int, name: String, sig: String,
+ superName: String, interfaces: Array[String]) {
+ myName = name
+ }
+
+ override def visitMethod(access: Int, name: String, desc: String,
+ sig: String, exceptions: Array[String]): MethodVisitor = {
+ return new MethodVisitor(ASM4) {
+ override def visitMethodInsn(op: Int, owner: String, name: String,
+ desc: String) {
+ val argTypes = Type.getArgumentTypes(desc)
+ if (op == INVOKESPECIAL && name == "<init>" && argTypes.length > 0
+ && argTypes(0).toString.startsWith("L") // is it an object?
+ && argTypes(0).getInternalName == myName)
+ output += Class.forName(
+ owner.replace('/', '.'),
+ false,
+ Thread.currentThread.getContextClassLoader)
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/Dependency.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Dependency.scala b/core/src/main/scala/org/apache/spark/Dependency.scala
new file mode 100644
index 0000000..cc3c247
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/Dependency.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+/**
+ * Base class for dependencies.
+ */
+abstract class Dependency[T](val rdd: RDD[T]) extends Serializable
+
+
+/**
+ * Base class for dependencies where each partition of the parent RDD is used by at most one
+ * partition of the child RDD. Narrow dependencies allow for pipelined execution.
+ */
+abstract class NarrowDependency[T](rdd: RDD[T]) extends Dependency(rdd) {
+ /**
+ * Get the parent partitions for a child partition.
+ * @param partitionId a partition of the child RDD
+ * @return the partitions of the parent RDD that the child partition depends upon
+ */
+ def getParents(partitionId: Int): Seq[Int]
+}
+
+
+/**
+ * Represents a dependency on the output of a shuffle stage.
+ * @param rdd the parent RDD
+ * @param partitioner partitioner used to partition the shuffle output
+ * @param serializerClass class name of the serializer to use
+ */
+class ShuffleDependency[K, V](
+ @transient rdd: RDD[_ <: Product2[K, V]],
+ val partitioner: Partitioner,
+ val serializerClass: String = null)
+ extends Dependency(rdd.asInstanceOf[RDD[Product2[K, V]]]) {
+
+ val shuffleId: Int = rdd.context.newShuffleId()
+}
+
+
+/**
+ * Represents a one-to-one dependency between partitions of the parent and child RDDs.
+ */
+class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
+ override def getParents(partitionId: Int) = List(partitionId)
+}
+
+
+/**
+ * Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs.
+ * @param rdd the parent RDD
+ * @param inStart the start of the range in the parent RDD
+ * @param outStart the start of the range in the child RDD
+ * @param length the length of the range
+ */
+class RangeDependency[T](rdd: RDD[T], inStart: Int, outStart: Int, length: Int)
+ extends NarrowDependency[T](rdd) {
+
+ override def getParents(partitionId: Int) = {
+ if (partitionId >= outStart && partitionId < outStart + length) {
+ List(partitionId - outStart + inStart)
+ } else {
+ Nil
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/DoubleRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/DoubleRDDFunctions.scala b/core/src/main/scala/org/apache/spark/DoubleRDDFunctions.scala
new file mode 100644
index 0000000..dd34449
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/DoubleRDDFunctions.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.partial.BoundedDouble
+import org.apache.spark.partial.MeanEvaluator
+import org.apache.spark.partial.PartialResult
+import org.apache.spark.partial.SumEvaluator
+import org.apache.spark.util.StatCounter
+
+/**
+ * Extra functions available on RDDs of Doubles through an implicit conversion.
+ * Import `spark.SparkContext._` at the top of your program to use these functions.
+ */
+class DoubleRDDFunctions(self: RDD[Double]) extends Logging with Serializable {
+ /** Add up the elements in this RDD. */
+ def sum(): Double = {
+ self.reduce(_ + _)
+ }
+
+ /**
+ * Return a [[org.apache.spark.util.StatCounter]] object that captures the mean, variance and count
+ * of the RDD's elements in one operation.
+ */
+ def stats(): StatCounter = {
+ self.mapPartitions(nums => Iterator(StatCounter(nums))).reduce((a, b) => a.merge(b))
+ }
+
+ /** Compute the mean of this RDD's elements. */
+ def mean(): Double = stats().mean
+
+ /** Compute the variance of this RDD's elements. */
+ def variance(): Double = stats().variance
+
+ /** Compute the standard deviation of this RDD's elements. */
+ def stdev(): Double = stats().stdev
+
+ /**
+ * Compute the sample standard deviation of this RDD's elements (which corrects for bias in
+ * estimating the standard deviation by dividing by N-1 instead of N).
+ */
+ def sampleStdev(): Double = stats().sampleStdev
+
+ /**
+ * Compute the sample variance of this RDD's elements (which corrects for bias in
+ * estimating the variance by dividing by N-1 instead of N).
+ */
+ def sampleVariance(): Double = stats().sampleVariance
+
+ /** (Experimental) Approximate operation to return the mean within a timeout. */
+ def meanApprox(timeout: Long, confidence: Double = 0.95): PartialResult[BoundedDouble] = {
+ val processPartition = (ctx: TaskContext, ns: Iterator[Double]) => StatCounter(ns)
+ val evaluator = new MeanEvaluator(self.partitions.size, confidence)
+ self.context.runApproximateJob(self, processPartition, evaluator, timeout)
+ }
+
+ /** (Experimental) Approximate operation to return the sum within a timeout. */
+ def sumApprox(timeout: Long, confidence: Double = 0.95): PartialResult[BoundedDouble] = {
+ val processPartition = (ctx: TaskContext, ns: Iterator[Double]) => StatCounter(ns)
+ val evaluator = new SumEvaluator(self.partitions.size, confidence)
+ self.context.runApproximateJob(self, processPartition, evaluator, timeout)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/FetchFailedException.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/FetchFailedException.scala b/core/src/main/scala/org/apache/spark/FetchFailedException.scala
new file mode 100644
index 0000000..d242047
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/FetchFailedException.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.storage.BlockManagerId
+
+private[spark] class FetchFailedException(
+ taskEndReason: TaskEndReason,
+ message: String,
+ cause: Throwable)
+ extends Exception {
+
+ def this (bmAddress: BlockManagerId, shuffleId: Int, mapId: Int, reduceId: Int, cause: Throwable) =
+ this(FetchFailed(bmAddress, shuffleId, mapId, reduceId),
+ "Fetch failed: %s %d %d %d".format(bmAddress, shuffleId, mapId, reduceId),
+ cause)
+
+ def this (shuffleId: Int, reduceId: Int, cause: Throwable) =
+ this(FetchFailed(null, shuffleId, -1, reduceId),
+ "Unable to fetch locations from master: %d %d".format(shuffleId, reduceId), cause)
+
+ override def getMessage(): String = message
+
+
+ override def getCause(): Throwable = cause
+
+ def toTaskEndReason: TaskEndReason = taskEndReason
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/HttpFileServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/HttpFileServer.scala b/core/src/main/scala/org/apache/spark/HttpFileServer.scala
new file mode 100644
index 0000000..9b3a896
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/HttpFileServer.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.io.{File}
+import com.google.common.io.Files
+
+private[spark] class HttpFileServer extends Logging {
+
+ var baseDir : File = null
+ var fileDir : File = null
+ var jarDir : File = null
+ var httpServer : HttpServer = null
+ var serverUri : String = null
+
+ def initialize() {
+ baseDir = Utils.createTempDir()
+ fileDir = new File(baseDir, "files")
+ jarDir = new File(baseDir, "jars")
+ fileDir.mkdir()
+ jarDir.mkdir()
+ logInfo("HTTP File server directory is " + baseDir)
+ httpServer = new HttpServer(baseDir)
+ httpServer.start()
+ serverUri = httpServer.uri
+ }
+
+ def stop() {
+ httpServer.stop()
+ }
+
+ def addFile(file: File) : String = {
+ addFileToDir(file, fileDir)
+ return serverUri + "/files/" + file.getName
+ }
+
+ def addJar(file: File) : String = {
+ addFileToDir(file, jarDir)
+ return serverUri + "/jars/" + file.getName
+ }
+
+ def addFileToDir(file: File, dir: File) : String = {
+ Files.copy(file, new File(dir, file.getName))
+ return dir + "/" + file.getName
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/HttpServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/HttpServer.scala b/core/src/main/scala/org/apache/spark/HttpServer.scala
new file mode 100644
index 0000000..db36c7c
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/HttpServer.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.io.File
+import java.net.InetAddress
+
+import org.eclipse.jetty.server.Server
+import org.eclipse.jetty.server.bio.SocketConnector
+import org.eclipse.jetty.server.handler.DefaultHandler
+import org.eclipse.jetty.server.handler.HandlerList
+import org.eclipse.jetty.server.handler.ResourceHandler
+import org.eclipse.jetty.util.thread.QueuedThreadPool
+
+/**
+ * Exception type thrown by HttpServer when it is in the wrong state for an operation.
+ */
+private[spark] class ServerStateException(message: String) extends Exception(message)
+
+/**
+ * An HTTP server for static content used to allow worker nodes to access JARs added to SparkContext
+ * as well as classes created by the interpreter when the user types in code. This is just a wrapper
+ * around a Jetty server.
+ */
+private[spark] class HttpServer(resourceBase: File) extends Logging {
+ private var server: Server = null
+ private var port: Int = -1
+
+ def start() {
+ if (server != null) {
+ throw new ServerStateException("Server is already started")
+ } else {
+ server = new Server()
+ val connector = new SocketConnector
+ connector.setMaxIdleTime(60*1000)
+ connector.setSoLingerTime(-1)
+ connector.setPort(0)
+ server.addConnector(connector)
+
+ val threadPool = new QueuedThreadPool
+ threadPool.setDaemon(true)
+ server.setThreadPool(threadPool)
+ val resHandler = new ResourceHandler
+ resHandler.setResourceBase(resourceBase.getAbsolutePath)
+ val handlerList = new HandlerList
+ handlerList.setHandlers(Array(resHandler, new DefaultHandler))
+ server.setHandler(handlerList)
+ server.start()
+ port = server.getConnectors()(0).getLocalPort()
+ }
+ }
+
+ def stop() {
+ if (server == null) {
+ throw new ServerStateException("Server is already stopped")
+ } else {
+ server.stop()
+ port = -1
+ server = null
+ }
+ }
+
+ /**
+ * Get the URI of this HTTP server (http://host:port)
+ */
+ def uri: String = {
+ if (server == null) {
+ throw new ServerStateException("Server is not started")
+ } else {
+ return "http://" + Utils.localIpAddress + ":" + port
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/JavaSerializer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/JavaSerializer.scala b/core/src/main/scala/org/apache/spark/JavaSerializer.scala
new file mode 100644
index 0000000..f43396c
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/JavaSerializer.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.io._
+import java.nio.ByteBuffer
+
+import serializer.{Serializer, SerializerInstance, DeserializationStream, SerializationStream}
+import org.apache.spark.util.ByteBufferInputStream
+
+private[spark] class JavaSerializationStream(out: OutputStream) extends SerializationStream {
+ val objOut = new ObjectOutputStream(out)
+ def writeObject[T](t: T): SerializationStream = { objOut.writeObject(t); this }
+ def flush() { objOut.flush() }
+ def close() { objOut.close() }
+}
+
+private[spark] class JavaDeserializationStream(in: InputStream, loader: ClassLoader)
+extends DeserializationStream {
+ val objIn = new ObjectInputStream(in) {
+ override def resolveClass(desc: ObjectStreamClass) =
+ Class.forName(desc.getName, false, loader)
+ }
+
+ def readObject[T](): T = objIn.readObject().asInstanceOf[T]
+ def close() { objIn.close() }
+}
+
+private[spark] class JavaSerializerInstance extends SerializerInstance {
+ def serialize[T](t: T): ByteBuffer = {
+ val bos = new ByteArrayOutputStream()
+ val out = serializeStream(bos)
+ out.writeObject(t)
+ out.close()
+ ByteBuffer.wrap(bos.toByteArray)
+ }
+
+ def deserialize[T](bytes: ByteBuffer): T = {
+ val bis = new ByteBufferInputStream(bytes)
+ val in = deserializeStream(bis)
+ in.readObject().asInstanceOf[T]
+ }
+
+ def deserialize[T](bytes: ByteBuffer, loader: ClassLoader): T = {
+ val bis = new ByteBufferInputStream(bytes)
+ val in = deserializeStream(bis, loader)
+ in.readObject().asInstanceOf[T]
+ }
+
+ def serializeStream(s: OutputStream): SerializationStream = {
+ new JavaSerializationStream(s)
+ }
+
+ def deserializeStream(s: InputStream): DeserializationStream = {
+ new JavaDeserializationStream(s, Thread.currentThread.getContextClassLoader)
+ }
+
+ def deserializeStream(s: InputStream, loader: ClassLoader): DeserializationStream = {
+ new JavaDeserializationStream(s, loader)
+ }
+}
+
+/**
+ * A Spark serializer that uses Java's built-in serialization.
+ */
+class JavaSerializer extends Serializer {
+ def newInstance(): SerializerInstance = new JavaSerializerInstance
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/KryoSerializer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/KryoSerializer.scala
new file mode 100644
index 0000000..db86e6d
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/KryoSerializer.scala
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.io._
+import java.nio.ByteBuffer
+import com.esotericsoftware.kryo.{Kryo, KryoException}
+import com.esotericsoftware.kryo.io.{Input => KryoInput, Output => KryoOutput}
+import com.esotericsoftware.kryo.serializers.{JavaSerializer => KryoJavaSerializer}
+import com.twitter.chill.ScalaKryoInstantiator
+import serializer.{SerializerInstance, DeserializationStream, SerializationStream}
+import org.apache.spark.broadcast._
+import org.apache.spark.storage._
+
+private[spark]
+class KryoSerializationStream(kryo: Kryo, outStream: OutputStream) extends SerializationStream {
+ val output = new KryoOutput(outStream)
+
+ def writeObject[T](t: T): SerializationStream = {
+ kryo.writeClassAndObject(output, t)
+ this
+ }
+
+ def flush() { output.flush() }
+ def close() { output.close() }
+}
+
+private[spark]
+class KryoDeserializationStream(kryo: Kryo, inStream: InputStream) extends DeserializationStream {
+ val input = new KryoInput(inStream)
+
+ def readObject[T](): T = {
+ try {
+ kryo.readClassAndObject(input).asInstanceOf[T]
+ } catch {
+ // DeserializationStream uses the EOF exception to indicate stopping condition.
+ case _: KryoException => throw new EOFException
+ }
+ }
+
+ def close() {
+ // Kryo's Input automatically closes the input stream it is using.
+ input.close()
+ }
+}
+
+private[spark] class KryoSerializerInstance(ks: KryoSerializer) extends SerializerInstance {
+ val kryo = ks.newKryo()
+ val output = ks.newKryoOutput()
+ val input = ks.newKryoInput()
+
+ def serialize[T](t: T): ByteBuffer = {
+ output.clear()
+ kryo.writeClassAndObject(output, t)
+ ByteBuffer.wrap(output.toBytes)
+ }
+
+ def deserialize[T](bytes: ByteBuffer): T = {
+ input.setBuffer(bytes.array)
+ kryo.readClassAndObject(input).asInstanceOf[T]
+ }
+
+ def deserialize[T](bytes: ByteBuffer, loader: ClassLoader): T = {
+ val oldClassLoader = kryo.getClassLoader
+ kryo.setClassLoader(loader)
+ input.setBuffer(bytes.array)
+ val obj = kryo.readClassAndObject(input).asInstanceOf[T]
+ kryo.setClassLoader(oldClassLoader)
+ obj
+ }
+
+ def serializeStream(s: OutputStream): SerializationStream = {
+ new KryoSerializationStream(kryo, s)
+ }
+
+ def deserializeStream(s: InputStream): DeserializationStream = {
+ new KryoDeserializationStream(kryo, s)
+ }
+}
+
+/**
+ * Interface implemented by clients to register their classes with Kryo when using Kryo
+ * serialization.
+ */
+trait KryoRegistrator {
+ def registerClasses(kryo: Kryo)
+}
+
+/**
+ * A Spark serializer that uses the [[http://code.google.com/p/kryo/wiki/V1Documentation Kryo 1.x library]].
+ */
+class KryoSerializer extends org.apache.spark.serializer.Serializer with Logging {
+ private val bufferSize = System.getProperty("spark.kryoserializer.buffer.mb", "2").toInt * 1024 * 1024
+
+ def newKryoOutput() = new KryoOutput(bufferSize)
+
+ def newKryoInput() = new KryoInput(bufferSize)
+
+ def newKryo(): Kryo = {
+ val instantiator = new ScalaKryoInstantiator
+ val kryo = instantiator.newKryo()
+ val classLoader = Thread.currentThread.getContextClassLoader
+
+ // Register some commonly used classes
+ val toRegister: Seq[AnyRef] = Seq(
+ ByteBuffer.allocate(1),
+ StorageLevel.MEMORY_ONLY,
+ PutBlock("1", ByteBuffer.allocate(1), StorageLevel.MEMORY_ONLY),
+ GotBlock("1", ByteBuffer.allocate(1)),
+ GetBlock("1")
+ )
+
+ for (obj <- toRegister) kryo.register(obj.getClass)
+
+ // Allow sending SerializableWritable
+ kryo.register(classOf[SerializableWritable[_]], new KryoJavaSerializer())
+ kryo.register(classOf[HttpBroadcast[_]], new KryoJavaSerializer())
+
+ // Allow the user to register their own classes by setting spark.kryo.registrator
+ try {
+ Option(System.getProperty("spark.kryo.registrator")).foreach { regCls =>
+ logDebug("Running user registrator: " + regCls)
+ val reg = Class.forName(regCls, true, classLoader).newInstance().asInstanceOf[KryoRegistrator]
+ reg.registerClasses(kryo)
+ }
+ } catch {
+ case _: Exception => println("Failed to register spark.kryo.registrator")
+ }
+
+ kryo.setClassLoader(classLoader)
+
+ // Allow disabling Kryo reference tracking if user knows their object graphs don't have loops
+ kryo.setReferences(System.getProperty("spark.kryo.referenceTracking", "true").toBoolean)
+
+ kryo
+ }
+
+ def newInstance(): SerializerInstance = {
+ new KryoSerializerInstance(this)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/46eecd11/core/src/main/scala/org/apache/spark/Logging.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Logging.scala b/core/src/main/scala/org/apache/spark/Logging.scala
new file mode 100644
index 0000000..6a973ea
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/Logging.scala
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+/**
+ * Utility trait for classes that want to log data. Creates a SLF4J logger for the class and allows
+ * logging messages at different levels using methods that only evaluate parameters lazily if the
+ * log level is enabled.
+ */
+trait Logging {
+ // Make the log field transient so that objects with Logging can
+ // be serialized and used on another machine
+ @transient private var log_ : Logger = null
+
+ // Method to get or create the logger for this object
+ protected def log: Logger = {
+ if (log_ == null) {
+ var className = this.getClass.getName
+ // Ignore trailing $'s in the class names for Scala objects
+ if (className.endsWith("$")) {
+ className = className.substring(0, className.length - 1)
+ }
+ log_ = LoggerFactory.getLogger(className)
+ }
+ return log_
+ }
+
+ // Log methods that take only a String
+ protected def logInfo(msg: => String) {
+ if (log.isInfoEnabled) log.info(msg)
+ }
+
+ protected def logDebug(msg: => String) {
+ if (log.isDebugEnabled) log.debug(msg)
+ }
+
+ protected def logTrace(msg: => String) {
+ if (log.isTraceEnabled) log.trace(msg)
+ }
+
+ protected def logWarning(msg: => String) {
+ if (log.isWarnEnabled) log.warn(msg)
+ }
+
+ protected def logError(msg: => String) {
+ if (log.isErrorEnabled) log.error(msg)
+ }
+
+ // Log methods that take Throwables (Exceptions/Errors) too
+ protected def logInfo(msg: => String, throwable: Throwable) {
+ if (log.isInfoEnabled) log.info(msg, throwable)
+ }
+
+ protected def logDebug(msg: => String, throwable: Throwable) {
+ if (log.isDebugEnabled) log.debug(msg, throwable)
+ }
+
+ protected def logTrace(msg: => String, throwable: Throwable) {
+ if (log.isTraceEnabled) log.trace(msg, throwable)
+ }
+
+ protected def logWarning(msg: => String, throwable: Throwable) {
+ if (log.isWarnEnabled) log.warn(msg, throwable)
+ }
+
+ protected def logError(msg: => String, throwable: Throwable) {
+ if (log.isErrorEnabled) log.error(msg, throwable)
+ }
+
+ protected def isTraceEnabled(): Boolean = {
+ log.isTraceEnabled
+ }
+
+ // Method for ensuring that logging is initialized, to avoid having multiple
+ // threads do it concurrently (as SLF4J initialization is not thread safe).
+ protected def initLogging() { log }
+}