You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/01/22 23:01:50 UTC
[07/50] git commit: initial large scale testing begin
initial large scale testing begin
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/642ab5c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/642ab5c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/642ab5c1
Branch: refs/heads/master
Commit: 642ab5c1e1ba98833265447447702c3c39fb2d40
Parents: 3369c2d
Author: Reza Zadeh <ri...@gmail.com>
Authored: Fri Dec 27 01:51:19 2013 -0500
Committer: Reza Zadeh <ri...@gmail.com>
Committed: Fri Dec 27 01:51:19 2013 -0500
----------------------------------------------------------------------
ec2/spark_ec2.py | 12 +-----------
.../scala/org/apache/spark/mllib/linalg/sparsesvd.scala | 8 ++++----
2 files changed, 5 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/642ab5c1/ec2/spark_ec2.py
----------------------------------------------------------------------
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py
index a2b0e7e..5e8b381 100755
--- a/ec2/spark_ec2.py
+++ b/ec2/spark_ec2.py
@@ -113,16 +113,6 @@ def parse_args():
# Boto config check
# http://boto.cloudhackers.com/en/latest/boto_config_tut.html
home_dir = os.getenv('HOME')
- if home_dir == None or not os.path.isfile(home_dir + '/.boto'):
- if not os.path.isfile('/etc/boto.cfg'):
- if os.getenv('AWS_ACCESS_KEY_ID') == None:
- print >> stderr, ("ERROR: The environment variable AWS_ACCESS_KEY_ID " +
- "must be set")
- sys.exit(1)
- if os.getenv('AWS_SECRET_ACCESS_KEY') == None:
- print >> stderr, ("ERROR: The environment variable AWS_SECRET_ACCESS_KEY " +
- "must be set")
- sys.exit(1)
return (opts, action, cluster_name)
@@ -646,7 +636,7 @@ def get_partition(total, num_partitions, current_partitions):
def real_main():
(opts, action, cluster_name) = parse_args()
try:
- conn = ec2.connect_to_region(opts.region)
+ conn = ec2.connect_to_region(opts.region,aws_access_key_id="AKIAI2EGAQ7GYNL4LRAA", aws_secret_access_key="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")
except Exception as e:
print >> stderr, (e)
sys.exit(1)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/642ab5c1/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala
index 1c9f67e..edf715d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/sparsesvd.scala
@@ -32,8 +32,8 @@ import org.jblas.{DoubleMatrix, Singular, MatrixFunctions}
* There is no restriction on m, but we require n^2 doubles to fit in memory.
* Further, n should be less than m.
*
- * This is computed by first computing A'A = V S^2 V',
- * computing locally on that (since n x n is small),
+ * The decomposition is computed by first computing A'A = V S^2 V',
+ * computing svd locally on that (since n x n is small),
* from which we recover S and V.
* Then we compute U via easy matrix multiplication
* as U = A * V * S^-1
@@ -43,8 +43,8 @@ import org.jblas.{DoubleMatrix, Singular, MatrixFunctions}
* such values, then the dimensions of the return will be:
*
* S is k x k and diagonal, holding the singular values on diagonal
- * U is m x k and satisfies U'U = eye(k,k)
- * V is n x k and satisfies V'V = eye(k,k)
+ * U is m x k and satisfies U'U = eye(k)
+ * V is n x k and satisfies V'V = eye(k)
*
* All input and output is expected in sparse matrix format, 1-indexed
* as tuples of the form ((i,j),value) all in RDDs