You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by va...@apache.org on 2017/06/27 16:14:55 UTC
[30/52] [partial] mahout git commit: removed all files except for
website directory
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/main/java/org/apache/mahout/math/MatrixWritable.java
----------------------------------------------------------------------
diff --git a/hdfs/src/main/java/org/apache/mahout/math/MatrixWritable.java b/hdfs/src/main/java/org/apache/mahout/math/MatrixWritable.java
deleted file mode 100644
index b8fc461..0000000
--- a/hdfs/src/main/java/org/apache/mahout/math/MatrixWritable.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.math;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.io.Writable;
-import org.apache.mahout.math.list.IntArrayList;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class MatrixWritable implements Writable {
-
- private static final int FLAG_DENSE = 0x01;
- private static final int FLAG_SEQUENTIAL = 0x02;
- private static final int FLAG_LABELS = 0x04;
- private static final int FLAG_SPARSE_ROW = 0x08;
- private static final int NUM_FLAGS = 4;
-
- private Matrix matrix;
-
- public MatrixWritable() {}
-
- public MatrixWritable(Matrix m) {
- this.matrix = m;
- }
-
- public Matrix get() {
- return matrix;
- }
-
- public void set(Matrix matrix) {
- this.matrix = matrix;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- writeMatrix(out, matrix);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- matrix = readMatrix(in);
- }
-
- public static void readLabels(DataInput in,
- Map<String, Integer> columnLabelBindings,
- Map<String, Integer> rowLabelBindings) throws IOException {
- int colSize = in.readInt();
- if (colSize > 0) {
- for (int i = 0; i < colSize; i++) {
- columnLabelBindings.put(in.readUTF(), in.readInt());
- }
- }
- int rowSize = in.readInt();
- if (rowSize > 0) {
- for (int i = 0; i < rowSize; i++) {
- rowLabelBindings.put(in.readUTF(), in.readInt());
- }
- }
- }
-
- public static void writeLabelBindings(DataOutput out,
- Map<String, Integer> columnLabelBindings,
- Map<String, Integer> rowLabelBindings) throws IOException {
- if (columnLabelBindings == null) {
- out.writeInt(0);
- } else {
- out.writeInt(columnLabelBindings.size());
- for (Map.Entry<String, Integer> stringIntegerEntry : columnLabelBindings.entrySet()) {
- out.writeUTF(stringIntegerEntry.getKey());
- out.writeInt(stringIntegerEntry.getValue());
- }
- }
- if (rowLabelBindings == null) {
- out.writeInt(0);
- } else {
- out.writeInt(rowLabelBindings.size());
- for (Map.Entry<String, Integer> stringIntegerEntry : rowLabelBindings.entrySet()) {
- out.writeUTF(stringIntegerEntry.getKey());
- out.writeInt(stringIntegerEntry.getValue());
- }
- }
- }
-
- /** Reads a typed Matrix instance from the input stream */
- public static Matrix readMatrix(DataInput in) throws IOException {
- int flags = in.readInt();
- Preconditions.checkArgument(flags >> NUM_FLAGS == 0, "Unknown flags set: %d", Integer.toString(flags, 2));
- boolean dense = (flags & FLAG_DENSE) != 0;
- boolean sequential = (flags & FLAG_SEQUENTIAL) != 0;
- boolean hasLabels = (flags & FLAG_LABELS) != 0;
- boolean isSparseRowMatrix = (flags & FLAG_SPARSE_ROW) != 0;
-
- int rows = in.readInt();
- int columns = in.readInt();
-
- byte vectorFlags = in.readByte();
-
- Matrix matrix;
-
- if (dense) {
- matrix = new DenseMatrix(rows, columns);
- for (int row = 0; row < rows; row++) {
- matrix.assignRow(row, VectorWritable.readVector(in, vectorFlags, columns));
- }
- } else if (isSparseRowMatrix) {
- Vector[] rowVectors = new Vector[rows];
- for (int row = 0; row < rows; row++) {
- rowVectors[row] = VectorWritable.readVector(in, vectorFlags, columns);
- }
- matrix = new SparseRowMatrix(rows, columns, rowVectors, true, !sequential);
- } else {
- matrix = new SparseMatrix(rows, columns);
- int numNonZeroRows = in.readInt();
- int rowsRead = 0;
- while (rowsRead++ < numNonZeroRows) {
- int rowIndex = in.readInt();
- matrix.assignRow(rowIndex, VectorWritable.readVector(in, vectorFlags, columns));
- }
- }
-
- if (hasLabels) {
- Map<String,Integer> columnLabelBindings = new HashMap<>();
- Map<String,Integer> rowLabelBindings = new HashMap<>();
- readLabels(in, columnLabelBindings, rowLabelBindings);
- if (!columnLabelBindings.isEmpty()) {
- matrix.setColumnLabelBindings(columnLabelBindings);
- }
- if (!rowLabelBindings.isEmpty()) {
- matrix.setRowLabelBindings(rowLabelBindings);
- }
- }
-
- return matrix;
- }
-
- /** Writes a typed Matrix instance to the output stream */
- public static void writeMatrix(final DataOutput out, Matrix matrix) throws IOException {
- int flags = 0;
- Vector row = matrix.viewRow(0);
- boolean isDense = row.isDense();
- if (isDense) {
- flags |= FLAG_DENSE;
- }
- if (row.isSequentialAccess()) {
- flags |= FLAG_SEQUENTIAL;
- }
- if (matrix.getRowLabelBindings() != null || matrix.getColumnLabelBindings() != null) {
- flags |= FLAG_LABELS;
- }
- boolean isSparseRowMatrix = matrix instanceof SparseRowMatrix;
- if (isSparseRowMatrix) {
- flags |= FLAG_SPARSE_ROW;
- }
-
- out.writeInt(flags);
- out.writeInt(matrix.rowSize());
- out.writeInt(matrix.columnSize());
-
- // We only use vectors of the same type, so we write out the type information only once!
- byte vectorFlags = VectorWritable.flags(matrix.viewRow(0), false);
- out.writeByte(vectorFlags);
-
- if (isDense || isSparseRowMatrix) {
- for (int i = 0; i < matrix.rowSize(); i++) {
- VectorWritable.writeVectorContents(out, matrix.viewRow(i), vectorFlags);
- }
- } else {
- IntArrayList rowIndices = ((SparseMatrix) matrix).nonZeroRowIndices();
- int numNonZeroRows = rowIndices.size();
- out.writeInt(numNonZeroRows);
- for (int i = 0; i < numNonZeroRows; i++) {
- int rowIndex = rowIndices.getQuick(i);
- out.writeInt(rowIndex);
- VectorWritable.writeVectorContents(out, matrix.viewRow(rowIndex), vectorFlags);
- }
- }
-
- if ((flags & FLAG_LABELS) != 0) {
- writeLabelBindings(out, matrix.getColumnLabelBindings(), matrix.getRowLabelBindings());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/main/java/org/apache/mahout/math/VarIntWritable.java
----------------------------------------------------------------------
diff --git a/hdfs/src/main/java/org/apache/mahout/math/VarIntWritable.java b/hdfs/src/main/java/org/apache/mahout/math/VarIntWritable.java
deleted file mode 100644
index e5cb173..0000000
--- a/hdfs/src/main/java/org/apache/mahout/math/VarIntWritable.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.math;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.WritableComparable;
-
-public class VarIntWritable implements WritableComparable<VarIntWritable>, Cloneable {
-
- private int value;
-
- public VarIntWritable() {
- }
-
- public VarIntWritable(int value) {
- this.value = value;
- }
-
- public int get() {
- return value;
- }
-
- public void set(int value) {
- this.value = value;
- }
-
- @Override
- public boolean equals(Object other) {
- return other instanceof VarIntWritable && ((VarIntWritable) other).value == value;
- }
-
- @Override
- public int hashCode() {
- return value;
- }
-
- @Override
- public String toString() {
- return String.valueOf(value);
- }
-
- @Override
- public VarIntWritable clone() {
- return new VarIntWritable(value);
- }
-
- @Override
- public int compareTo(VarIntWritable other) {
- if (value < other.value) {
- return -1;
- }
- if (value > other.value) {
- return 1;
- }
- return 0;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- Varint.writeSignedVarInt(value, out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- value = Varint.readSignedVarInt(in);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/main/java/org/apache/mahout/math/VarLongWritable.java
----------------------------------------------------------------------
diff --git a/hdfs/src/main/java/org/apache/mahout/math/VarLongWritable.java b/hdfs/src/main/java/org/apache/mahout/math/VarLongWritable.java
deleted file mode 100644
index 7b0d9c4..0000000
--- a/hdfs/src/main/java/org/apache/mahout/math/VarLongWritable.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.math;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.io.WritableComparable;
-
-public class VarLongWritable implements WritableComparable<VarLongWritable> {
-
- private long value;
-
- public VarLongWritable() {
- }
-
- public VarLongWritable(long value) {
- this.value = value;
- }
-
- public long get() {
- return value;
- }
-
- public void set(long value) {
- this.value = value;
- }
-
- @Override
- public boolean equals(Object other) {
- return other != null && getClass().equals(other.getClass()) && ((VarLongWritable) other).value == value;
- }
-
- @Override
- public int hashCode() {
- return Longs.hashCode(value);
- }
-
- @Override
- public String toString() {
- return String.valueOf(value);
- }
-
- @Override
- public int compareTo(VarLongWritable other) {
- if (value >= other.value) {
- if (value > other.value) {
- return 1;
- }
- } else {
- return -1;
- }
- return 0;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- Varint.writeSignedVarLong(value, out);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- value = Varint.readSignedVarLong(in);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/main/java/org/apache/mahout/math/Varint.java
----------------------------------------------------------------------
diff --git a/hdfs/src/main/java/org/apache/mahout/math/Varint.java b/hdfs/src/main/java/org/apache/mahout/math/Varint.java
deleted file mode 100644
index f380c6c..0000000
--- a/hdfs/src/main/java/org/apache/mahout/math/Varint.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.math;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import com.google.common.base.Preconditions;
-
-/**
- * <p>Encodes signed and unsigned values using a common variable-length
- * scheme, found for example in
- * <a href="http://code.google.com/apis/protocolbuffers/docs/encoding.html">
- * Google's Protocol Buffers</a>. It uses fewer bytes to encode smaller values,
- * but will use slightly more bytes to encode large values.</p>
- *
- * <p>Signed values are further encoded using so-called zig-zag encoding
- * in order to make them "compatible" with variable-length encoding.</p>
- */
-public final class Varint {
-
- private Varint() {
- }
-
- /**
- * Encodes a value using the variable-length encoding from
- * <a href="http://code.google.com/apis/protocolbuffers/docs/encoding.html">
- * Google Protocol Buffers</a>. It uses zig-zag encoding to efficiently
- * encode signed values. If values are known to be nonnegative,
- * {@link #writeUnsignedVarLong(long, java.io.DataOutput)} should be used.
- *
- * @param value value to encode
- * @param out to write bytes to
- * @throws java.io.IOException if {@link java.io.DataOutput} throws {@link java.io.IOException}
- */
- public static void writeSignedVarLong(long value, DataOutput out) throws IOException {
- // Great trick from http://code.google.com/apis/protocolbuffers/docs/encoding.html#types
- writeUnsignedVarLong((value << 1) ^ (value >> 63), out);
- }
-
- /**
- * Encodes a value using the variable-length encoding from
- * <a href="http://code.google.com/apis/protocolbuffers/docs/encoding.html">
- * Google Protocol Buffers</a>. Zig-zag is not used, so input must not be negative.
- * If values can be negative, use {@link #writeSignedVarLong(long, java.io.DataOutput)}
- * instead. This method treats negative input as like a large unsigned value.
- *
- * @param value value to encode
- * @param out to write bytes to
- * @throws java.io.IOException if {@link java.io.DataOutput} throws {@link java.io.IOException}
- */
- public static void writeUnsignedVarLong(long value, DataOutput out) throws IOException {
- while ((value & 0xFFFFFFFFFFFFFF80L) != 0L) {
- out.writeByte(((int) value & 0x7F) | 0x80);
- value >>>= 7;
- }
- out.writeByte((int) value & 0x7F);
- }
-
- /**
- * @see #writeSignedVarLong(long, java.io.DataOutput)
- */
- public static void writeSignedVarInt(int value, DataOutput out) throws IOException {
- // Great trick from http://code.google.com/apis/protocolbuffers/docs/encoding.html#types
- writeUnsignedVarInt((value << 1) ^ (value >> 31), out);
- }
-
- /**
- * @see #writeUnsignedVarLong(long, java.io.DataOutput)
- */
- public static void writeUnsignedVarInt(int value, DataOutput out) throws IOException {
- while ((value & 0xFFFFFF80) != 0L) {
- out.writeByte((value & 0x7F) | 0x80);
- value >>>= 7;
- }
- out.writeByte(value & 0x7F);
- }
-
- /**
- * @param in to read bytes from
- * @return decode value
- * @throws java.io.IOException if {@link java.io.DataInput} throws {@link java.io.IOException}
- * @throws IllegalArgumentException if variable-length value does not terminate
- * after 9 bytes have been read
- * @see #writeSignedVarLong(long, java.io.DataOutput)
- */
- public static long readSignedVarLong(DataInput in) throws IOException {
- long raw = readUnsignedVarLong(in);
- // This undoes the trick in writeSignedVarLong()
- long temp = (((raw << 63) >> 63) ^ raw) >> 1;
- // This extra step lets us deal with the largest signed values by treating
- // negative results from read unsigned methods as like unsigned values
- // Must re-flip the top bit if the original read value had it set.
- return temp ^ (raw & (1L << 63));
- }
-
- /**
- * @param in to read bytes from
- * @return decode value
- * @throws java.io.IOException if {@link java.io.DataInput} throws {@link java.io.IOException}
- * @throws IllegalArgumentException if variable-length value does not terminate
- * after 9 bytes have been read
- * @see #writeUnsignedVarLong(long, java.io.DataOutput)
- */
- public static long readUnsignedVarLong(DataInput in) throws IOException {
- long value = 0L;
- int i = 0;
- long b;
- while (((b = in.readByte()) & 0x80L) != 0) {
- value |= (b & 0x7F) << i;
- i += 7;
- Preconditions.checkArgument(i <= 63, "Variable length quantity is too long (must be <= 63)");
- }
- return value | (b << i);
- }
-
- /**
- * @throws IllegalArgumentException if variable-length value does not terminate
- * after 5 bytes have been read
- * @throws java.io.IOException if {@link java.io.DataInput} throws {@link java.io.IOException}
- * @see #readSignedVarLong(java.io.DataInput)
- */
- public static int readSignedVarInt(DataInput in) throws IOException {
- int raw = readUnsignedVarInt(in);
- // This undoes the trick in writeSignedVarInt()
- int temp = (((raw << 31) >> 31) ^ raw) >> 1;
- // This extra step lets us deal with the largest signed values by treating
- // negative results from read unsigned methods as like unsigned values.
- // Must re-flip the top bit if the original read value had it set.
- return temp ^ (raw & (1 << 31));
- }
-
- /**
- * @throws IllegalArgumentException if variable-length value does not terminate
- * after 5 bytes have been read
- * @throws java.io.IOException if {@link java.io.DataInput} throws {@link java.io.IOException}
- * @see #readUnsignedVarLong(java.io.DataInput)
- */
- public static int readUnsignedVarInt(DataInput in) throws IOException {
- int value = 0;
- int i = 0;
- int b;
- while (((b = in.readByte()) & 0x80) != 0) {
- value |= (b & 0x7F) << i;
- i += 7;
- Preconditions.checkArgument(i <= 35, "Variable length quantity is too long (must be <= 35)");
- }
- return value | (b << i);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/main/java/org/apache/mahout/math/VectorWritable.java
----------------------------------------------------------------------
diff --git a/hdfs/src/main/java/org/apache/mahout/math/VectorWritable.java b/hdfs/src/main/java/org/apache/mahout/math/VectorWritable.java
deleted file mode 100644
index 491ae3b..0000000
--- a/hdfs/src/main/java/org/apache/mahout/math/VectorWritable.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.mahout.math;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.Writable;
-import org.apache.mahout.math.Vector.Element;
-
-import com.google.common.base.Preconditions;
-
-public final class VectorWritable extends Configured implements Writable {
-
- public static final int FLAG_DENSE = 0x01;
- public static final int FLAG_SEQUENTIAL = 0x02;
- public static final int FLAG_NAMED = 0x04;
- public static final int FLAG_LAX_PRECISION = 0x08;
- public static final int NUM_FLAGS = 4;
-
- private Vector vector;
- private boolean writesLaxPrecision;
-
- public VectorWritable() {}
-
- public VectorWritable(boolean writesLaxPrecision) {
- setWritesLaxPrecision(writesLaxPrecision);
- }
-
- public VectorWritable(Vector vector) {
- this.vector = vector;
- }
-
- public VectorWritable(Vector vector, boolean writesLaxPrecision) {
- this(vector);
- setWritesLaxPrecision(writesLaxPrecision);
- }
-
- /**
- * @return {@link org.apache.mahout.math.Vector} that this is to write, or has
- * just read
- */
- public Vector get() {
- return vector;
- }
-
- public void set(Vector vector) {
- this.vector = vector;
- }
-
- /**
- * @return true if this is allowed to encode {@link org.apache.mahout.math.Vector}
- * values using fewer bytes, possibly losing precision. In particular this means
- * that floating point values will be encoded as floats, not doubles.
- */
- public boolean isWritesLaxPrecision() {
- return writesLaxPrecision;
- }
-
- public void setWritesLaxPrecision(boolean writesLaxPrecision) {
- this.writesLaxPrecision = writesLaxPrecision;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- writeVector(out, this.vector, this.writesLaxPrecision);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- int flags = in.readByte();
- int size = Varint.readUnsignedVarInt(in);
- readFields(in, (byte) flags, size);
- }
-
- private void readFields(DataInput in, byte flags, int size) throws IOException {
-
- Preconditions.checkArgument(flags >> NUM_FLAGS == 0, "Unknown flags set: %d", Integer.toString(flags, 2));
- boolean dense = (flags & FLAG_DENSE) != 0;
- boolean sequential = (flags & FLAG_SEQUENTIAL) != 0;
- boolean named = (flags & FLAG_NAMED) != 0;
- boolean laxPrecision = (flags & FLAG_LAX_PRECISION) != 0;
-
- Vector v;
- if (dense) {
- double[] values = new double[size];
- for (int i = 0; i < size; i++) {
- values[i] = laxPrecision ? in.readFloat() : in.readDouble();
- }
- v = new DenseVector(values);
- } else {
- int numNonDefaultElements = Varint.readUnsignedVarInt(in);
- v = sequential
- ? new SequentialAccessSparseVector(size, numNonDefaultElements)
- : new RandomAccessSparseVector(size, numNonDefaultElements);
- if (sequential) {
- int lastIndex = 0;
- for (int i = 0; i < numNonDefaultElements; i++) {
- int delta = Varint.readUnsignedVarInt(in);
- int index = lastIndex + delta;
- lastIndex = index;
- double value = laxPrecision ? in.readFloat() : in.readDouble();
- v.setQuick(index, value);
- }
- } else {
- for (int i = 0; i < numNonDefaultElements; i++) {
- int index = Varint.readUnsignedVarInt(in);
- double value = laxPrecision ? in.readFloat() : in.readDouble();
- v.setQuick(index, value);
- }
- }
- }
- if (named) {
- String name = in.readUTF();
- v = new NamedVector(v, name);
- }
- vector = v;
- }
-
- /** Write the vector to the output */
- public static void writeVector(DataOutput out, Vector vector) throws IOException {
- writeVector(out, vector, false);
- }
-
- public static byte flags(Vector vector, boolean laxPrecision) {
- boolean dense = vector.isDense();
- boolean sequential = vector.isSequentialAccess();
- boolean named = vector instanceof NamedVector;
-
- return (byte) ((dense ? FLAG_DENSE : 0)
- | (sequential ? FLAG_SEQUENTIAL : 0)
- | (named ? FLAG_NAMED : 0)
- | (laxPrecision ? FLAG_LAX_PRECISION : 0));
- }
-
- /** Write out type information and size of the vector */
- public static void writeVectorFlagsAndSize(DataOutput out, byte flags, int size) throws IOException {
- out.writeByte(flags);
- Varint.writeUnsignedVarInt(size, out);
- }
-
- public static void writeVector(DataOutput out, Vector vector, boolean laxPrecision) throws IOException {
-
- byte flags = flags(vector, laxPrecision);
-
- writeVectorFlagsAndSize(out, flags, vector.size());
- writeVectorContents(out, vector, flags);
- }
-
- /** Write out contents of the vector */
- public static void writeVectorContents(DataOutput out, Vector vector, byte flags) throws IOException {
-
- boolean dense = (flags & FLAG_DENSE) != 0;
- boolean sequential = (flags & FLAG_SEQUENTIAL) != 0;
- boolean named = (flags & FLAG_NAMED) != 0;
- boolean laxPrecision = (flags & FLAG_LAX_PRECISION) != 0;
-
- if (dense) {
- for (Element element : vector.all()) {
- if (laxPrecision) {
- out.writeFloat((float) element.get());
- } else {
- out.writeDouble(element.get());
- }
- }
- } else {
- Varint.writeUnsignedVarInt(vector.getNumNonZeroElements(), out);
- Iterator<Element> iter = vector.nonZeroes().iterator();
- if (sequential) {
- int lastIndex = 0;
- while (iter.hasNext()) {
- Element element = iter.next();
- if (element.get() == 0) {
- continue;
- }
- int thisIndex = element.index();
- // Delta-code indices:
- Varint.writeUnsignedVarInt(thisIndex - lastIndex, out);
- lastIndex = thisIndex;
- if (laxPrecision) {
- out.writeFloat((float) element.get());
- } else {
- out.writeDouble(element.get());
- }
- }
- } else {
- while (iter.hasNext()) {
- Element element = iter.next();
- if (element.get() == 0) {
- // TODO(robinanil): Fix the damn iterator for the zero element.
- continue;
- }
- Varint.writeUnsignedVarInt(element.index(), out);
- if (laxPrecision) {
- out.writeFloat((float) element.get());
- } else {
- out.writeDouble(element.get());
- }
- }
- }
- }
- if (named) {
- String name = ((NamedVector) vector).getName();
- out.writeUTF(name == null ? "" : name);
- }
- }
-
- public static Vector readVector(DataInput in) throws IOException {
- VectorWritable v = new VectorWritable();
- v.readFields(in);
- return v.get();
- }
-
- public static Vector readVector(DataInput in, byte vectorFlags, int size) throws IOException {
- VectorWritable v = new VectorWritable();
- v.readFields(in, vectorFlags, size);
- return v.get();
- }
-
- public static VectorWritable merge(Iterator<VectorWritable> vectors) {
- return new VectorWritable(mergeToVector(vectors));
- }
-
- public static Vector mergeToVector(Iterator<VectorWritable> vectors) {
- Vector accumulator = vectors.next().get();
- while (vectors.hasNext()) {
- VectorWritable v = vectors.next();
- if (v != null) {
- for (Element nonZeroElement : v.get().nonZeroes()) {
- accumulator.setQuick(nonZeroElement.index(), nonZeroElement.get());
- }
- }
- }
- return accumulator;
- }
-
- @Override
- public boolean equals(Object o) {
- return o instanceof VectorWritable && vector.equals(((VectorWritable) o).get());
- }
-
- @Override
- public int hashCode() {
- return vector.hashCode();
- }
-
- @Override
- public String toString() {
- return vector.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/test/java/org/apache/mahout/math/MatrixWritableTest.java
----------------------------------------------------------------------
diff --git a/hdfs/src/test/java/org/apache/mahout/math/MatrixWritableTest.java b/hdfs/src/test/java/org/apache/mahout/math/MatrixWritableTest.java
deleted file mode 100644
index 31e6947..0000000
--- a/hdfs/src/test/java/org/apache/mahout/math/MatrixWritableTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.math;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.io.Writable;
-import org.junit.Test;
-
-public final class MatrixWritableTest extends MahoutTestCase {
-
- @Test
- public void testSparseMatrixWritable() throws Exception {
- Matrix m = new SparseMatrix(5, 5);
- m.set(1, 2, 3.0);
- m.set(3, 4, 5.0);
- Map<String, Integer> bindings = new HashMap<>();
- bindings.put("A", 0);
- bindings.put("B", 1);
- bindings.put("C", 2);
- bindings.put("D", 3);
- bindings.put("default", 4);
- m.setRowLabelBindings(bindings);
- m.setColumnLabelBindings(bindings);
- doTestMatrixWritableEquals(m);
- }
-
- @Test
- public void testSparseRowMatrixWritable() throws Exception {
- Matrix m = new SparseRowMatrix(5, 5);
- m.set(1, 2, 3.0);
- m.set(3, 4, 5.0);
- Map<String, Integer> bindings = new HashMap<>();
- bindings.put("A", 0);
- bindings.put("B", 1);
- bindings.put("C", 2);
- bindings.put("D", 3);
- bindings.put("default", 4);
- m.setRowLabelBindings(bindings);
- m.setColumnLabelBindings(bindings);
- doTestMatrixWritableEquals(m);
- }
-
- @Test
- public void testDenseMatrixWritable() throws Exception {
- Matrix m = new DenseMatrix(5,5);
- m.set(1, 2, 3.0);
- m.set(3, 4, 5.0);
- Map<String, Integer> bindings = new HashMap<>();
- bindings.put("A", 0);
- bindings.put("B", 1);
- bindings.put("C", 2);
- bindings.put("D", 3);
- bindings.put("default", 4);
- m.setRowLabelBindings(bindings);
- m.setColumnLabelBindings(bindings);
- doTestMatrixWritableEquals(m);
- }
-
- private static void doTestMatrixWritableEquals(Matrix m) throws IOException {
- Writable matrixWritable = new MatrixWritable(m);
- MatrixWritable matrixWritable2 = new MatrixWritable();
- writeAndRead(matrixWritable, matrixWritable2);
- Matrix m2 = matrixWritable2.get();
- compareMatrices(m, m2);
- doCheckBindings(m2.getRowLabelBindings());
- doCheckBindings(m2.getColumnLabelBindings());
- }
-
- private static void compareMatrices(Matrix m, Matrix m2) {
- assertEquals(m.numRows(), m2.numRows());
- assertEquals(m.numCols(), m2.numCols());
- for (int r = 0; r < m.numRows(); r++) {
- for (int c = 0; c < m.numCols(); c++) {
- assertEquals(m.get(r, c), m2.get(r, c), EPSILON);
- }
- }
- Map<String,Integer> bindings = m.getRowLabelBindings();
- Map<String, Integer> bindings2 = m2.getRowLabelBindings();
- assertEquals(bindings == null, bindings2 == null);
- if (bindings != null) {
- assertEquals(bindings.size(), m.numRows());
- assertEquals(bindings.size(), bindings2.size());
- for (Map.Entry<String,Integer> entry : bindings.entrySet()) {
- assertEquals(entry.getValue(), bindings2.get(entry.getKey()));
- }
- }
- bindings = m.getColumnLabelBindings();
- bindings2 = m2.getColumnLabelBindings();
- assertEquals(bindings == null, bindings2 == null);
- if (bindings != null) {
- assertEquals(bindings.size(), bindings2.size());
- for (Map.Entry<String,Integer> entry : bindings.entrySet()) {
- assertEquals(entry.getValue(), bindings2.get(entry.getKey()));
- }
- }
- }
-
- private static void doCheckBindings(Map<String,Integer> labels) {
- assertTrue("Missing label", labels.keySet().contains("A"));
- assertTrue("Missing label", labels.keySet().contains("B"));
- assertTrue("Missing label", labels.keySet().contains("C"));
- assertTrue("Missing label", labels.keySet().contains("D"));
- assertTrue("Missing label", labels.keySet().contains("default"));
- }
-
- private static void writeAndRead(Writable toWrite, Writable toRead) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (DataOutputStream dos = new DataOutputStream(baos)){
- toWrite.write(dos);
- }
-
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- try (DataInputStream dis = new DataInputStream(bais)) {
- toRead.readFields(dis);
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/test/java/org/apache/mahout/math/VarintTest.java
----------------------------------------------------------------------
diff --git a/hdfs/src/test/java/org/apache/mahout/math/VarintTest.java b/hdfs/src/test/java/org/apache/mahout/math/VarintTest.java
deleted file mode 100644
index 0b1a664..0000000
--- a/hdfs/src/test/java/org/apache/mahout/math/VarintTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.math;
-
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-
-/**
- * Tests {@link Varint}.
- */
-public final class VarintTest extends MahoutTestCase {
-
- @Test
- public void testUnsignedLong() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- Varint.writeUnsignedVarLong(0L, out);
- for (long i = 1L; i > 0L && i <= (1L << 62); i <<= 1) {
- Varint.writeUnsignedVarLong(i-1, out);
- Varint.writeUnsignedVarLong(i, out);
- }
- Varint.writeUnsignedVarLong(Long.MAX_VALUE, out);
-
- DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
- assertEquals(0L, Varint.readUnsignedVarLong(in));
- for (long i = 1L; i > 0L && i <= (1L << 62); i <<= 1) {
- assertEquals(i-1, Varint.readUnsignedVarLong(in));
- assertEquals(i, Varint.readUnsignedVarLong(in));
- }
- assertEquals(Long.MAX_VALUE, Varint.readUnsignedVarLong(in));
- }
-
- @Test
- public void testSignedPositiveLong() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- Varint.writeSignedVarLong(0L, out);
- for (long i = 1L; i <= (1L << 61); i <<= 1) {
- Varint.writeSignedVarLong(i-1, out);
- Varint.writeSignedVarLong(i, out);
- }
- Varint.writeSignedVarLong((1L << 62) - 1, out);
- Varint.writeSignedVarLong((1L << 62), out);
- Varint.writeSignedVarLong(Long.MAX_VALUE, out);
-
- DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
- assertEquals(0L, Varint.readSignedVarLong(in));
- for (long i = 1L; i <= (1L << 61); i <<= 1) {
- assertEquals(i-1, Varint.readSignedVarLong(in));
- assertEquals(i, Varint.readSignedVarLong(in));
- }
- assertEquals((1L << 62) - 1, Varint.readSignedVarLong(in));
- assertEquals((1L << 62), Varint.readSignedVarLong(in));
- assertEquals(Long.MAX_VALUE, Varint.readSignedVarLong(in));
- }
-
- @Test
- public void testSignedNegativeLong() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- for (long i = -1L; i >= -(1L << 62); i <<= 1) {
- Varint.writeSignedVarLong(i, out);
- Varint.writeSignedVarLong(i+1, out);
- }
- Varint.writeSignedVarLong(Long.MIN_VALUE, out);
- Varint.writeSignedVarLong(Long.MIN_VALUE+1, out);
- DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
- for (long i = -1L; i >= -(1L << 62); i <<= 1) {
- assertEquals(i, Varint.readSignedVarLong(in));
- assertEquals(i+1, Varint.readSignedVarLong(in));
- }
- assertEquals(Long.MIN_VALUE, Varint.readSignedVarLong(in));
- assertEquals(Long.MIN_VALUE+1, Varint.readSignedVarLong(in));
- }
-
- @Test
- public void testUnsignedInt() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- Varint.writeUnsignedVarInt(0, out);
- for (int i = 1; i > 0 && i <= (1 << 30); i <<= 1) {
- Varint.writeUnsignedVarLong(i-1, out);
- Varint.writeUnsignedVarLong(i, out);
- }
- Varint.writeUnsignedVarLong(Integer.MAX_VALUE, out);
-
- DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
- assertEquals(0, Varint.readUnsignedVarInt(in));
- for (int i = 1; i > 0 && i <= (1 << 30); i <<= 1) {
- assertEquals(i-1, Varint.readUnsignedVarInt(in));
- assertEquals(i, Varint.readUnsignedVarInt(in));
- }
- assertEquals(Integer.MAX_VALUE, Varint.readUnsignedVarInt(in));
- }
-
- @Test
- public void testSignedPositiveInt() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- Varint.writeSignedVarInt(0, out);
- for (int i = 1; i <= (1 << 29); i <<= 1) {
- Varint.writeSignedVarLong(i-1, out);
- Varint.writeSignedVarLong(i, out);
- }
- Varint.writeSignedVarInt((1 << 30) - 1, out);
- Varint.writeSignedVarInt((1 << 30), out);
- Varint.writeSignedVarInt(Integer.MAX_VALUE, out);
-
- DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
- assertEquals(0, Varint.readSignedVarInt(in));
- for (int i = 1; i <= (1 << 29); i <<= 1) {
- assertEquals(i-1, Varint.readSignedVarInt(in));
- assertEquals(i, Varint.readSignedVarInt(in));
- }
- assertEquals((1L << 30) - 1, Varint.readSignedVarInt(in));
- assertEquals((1L << 30), Varint.readSignedVarInt(in));
- assertEquals(Integer.MAX_VALUE, Varint.readSignedVarInt(in));
- }
-
- @Test
- public void testSignedNegativeInt() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- for (int i = -1; i >= -(1 << 30); i <<= 1) {
- Varint.writeSignedVarInt(i, out);
- Varint.writeSignedVarInt(i+1, out);
- }
- Varint.writeSignedVarInt(Integer.MIN_VALUE, out);
- Varint.writeSignedVarInt(Integer.MIN_VALUE+1, out);
- DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
- for (int i = -1; i >= -(1 << 30); i <<= 1) {
- assertEquals(i, Varint.readSignedVarInt(in));
- assertEquals(i+1, Varint.readSignedVarInt(in));
- }
- assertEquals(Integer.MIN_VALUE, Varint.readSignedVarInt(in));
- assertEquals(Integer.MIN_VALUE+1, Varint.readSignedVarInt(in));
- }
-
- @Test
- public void testUnsignedSize() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- int expectedSize = 0;
- for (int exponent = 0; exponent <= 62; exponent++) {
- Varint.writeUnsignedVarLong(1L << exponent, out);
- expectedSize += 1 + exponent / 7;
- assertEquals(expectedSize, baos.size());
- }
- }
-
- @Test
- public void testSignedSize() throws Exception {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput out = new DataOutputStream(baos);
- int expectedSize = 0;
- for (int exponent = 0; exponent <= 61; exponent++) {
- Varint.writeSignedVarLong(1L << exponent, out);
- expectedSize += 1 + ((exponent + 1) / 7);
- assertEquals(expectedSize, baos.size());
- }
- for (int exponent = 0; exponent <= 61; exponent++) {
- Varint.writeSignedVarLong(-(1L << exponent)-1, out);
- expectedSize += 1 + ((exponent + 1) / 7);
- assertEquals(expectedSize, baos.size());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/hdfs/src/test/java/org/apache/mahout/math/VectorWritableTest.java
----------------------------------------------------------------------
diff --git a/hdfs/src/test/java/org/apache/mahout/math/VectorWritableTest.java b/hdfs/src/test/java/org/apache/mahout/math/VectorWritableTest.java
deleted file mode 100644
index 991be6e..0000000
--- a/hdfs/src/test/java/org/apache/mahout/math/VectorWritableTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
- * agreements. See the NOTICE file distributed with this work for additional information regarding
- * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License. You may obtain a
- * copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.mahout.math;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.mahout.math.Vector.Element;
-import org.junit.Test;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.Repeat;
-
-public final class VectorWritableTest extends RandomizedTest {
- private static final int MAX_VECTOR_SIZE = 100;
-
- public void createRandom(Vector v) {
- int size = randomInt(v.size() - 1);
- for (int i = 0; i < size; ++i) {
- v.set(randomInt(v.size() - 1), randomDouble());
- }
-
- int zeros = Math.max(2, size / 4);
- for (Element e : v.nonZeroes()) {
- if (e.index() % zeros == 0) {
- e.set(0.0);
- }
- }
- }
-
- @Test
- @Repeat(iterations = 20)
- public void testViewSequentialAccessSparseVectorWritable() throws Exception {
- Vector v = new SequentialAccessSparseVector(MAX_VECTOR_SIZE);
- createRandom(v);
- Vector view = new VectorView(v, 0, v.size());
- doTestVectorWritableEquals(view);
- }
-
- @Test
- @Repeat(iterations = 20)
- public void testSequentialAccessSparseVectorWritable() throws Exception {
- Vector v = new SequentialAccessSparseVector(MAX_VECTOR_SIZE);
- createRandom(v);
- doTestVectorWritableEquals(v);
- }
-
- @Test
- @Repeat(iterations = 20)
- public void testRandomAccessSparseVectorWritable() throws Exception {
- Vector v = new RandomAccessSparseVector(MAX_VECTOR_SIZE);
- createRandom(v);
- doTestVectorWritableEquals(v);
- }
-
- @Test
- @Repeat(iterations = 20)
- public void testDenseVectorWritable() throws Exception {
- Vector v = new DenseVector(MAX_VECTOR_SIZE);
- createRandom(v);
- doTestVectorWritableEquals(v);
- }
-
- @Test
- @Repeat(iterations = 20)
- public void testNamedVectorWritable() throws Exception {
- Vector v = new DenseVector(MAX_VECTOR_SIZE);
- v = new NamedVector(v, "Victor");
- createRandom(v);
- doTestVectorWritableEquals(v);
- }
-
- private static void doTestVectorWritableEquals(Vector v) throws IOException {
- Writable vectorWritable = new VectorWritable(v);
- VectorWritable vectorWritable2 = new VectorWritable();
- writeAndRead(vectorWritable, vectorWritable2);
- Vector v2 = vectorWritable2.get();
- if (v instanceof NamedVector) {
- assertTrue(v2 instanceof NamedVector);
- NamedVector nv = (NamedVector) v;
- NamedVector nv2 = (NamedVector) v2;
- assertEquals(nv.getName(), nv2.getName());
- assertEquals("Victor", nv.getName());
- }
- assertEquals(v, v2);
- }
-
- private static void writeAndRead(Writable toWrite, Writable toRead) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- try (DataOutputStream dos = new DataOutputStream(baos)){
- toWrite.write(dos);
- }
-
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- try (DataInputStream dis = new DataInputStream(bais)) {
- toRead.readFields(dis);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/bin/prep_asf_mail_archives.sh
----------------------------------------------------------------------
diff --git a/integration/bin/prep_asf_mail_archives.sh b/integration/bin/prep_asf_mail_archives.sh
deleted file mode 100755
index 77f5d13..0000000
--- a/integration/bin/prep_asf_mail_archives.sh
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/bin/bash
-#
-# Performs the setup procedures for clustering the ASF mail archives
-# described in Taming Text.
-#
-# Required Command-line Parameters:
-#
-# $1 - Path to this script's working directory, you will need about
-# 22GB of free space to run this script.
-#
-# $2 - Path to where the ASF Public Archive data is, untarred.
-# If you are running Hadoop and the files are in HDFS, then
-# this will need to be an HDFS path. Default is $1/input
-# $3 - Path to where this script saves the SequenceFile output.
-# If you are running Hadoop and you want the sequence files
-# saved to your HDFS then you need to set this value to an
-# HDFS path and make sure you set HADOOP_HOME so Mahout can
-# find Hadoop. Default is $1/sequence-files
-#
-#
-# Required Environment Variables:
-#
-# MAHOUT_HOME
-# Root directory of your Mahout distribution
-#
-# HADOOP_HOME
-# Only needed if you want to send output to HDFS
-#
-# Example:
-# ./prep_asf_mail_archives.sh /mnt/asf-mail-archives /mnt/asf-archives/asf-mail-archives-7-18-2011 /mnt/asf-mail-archives/output
-#
-# This will download the TAR files from S3, extract them, and then
-# run the Mahout org.apache.mahout.text.SequenceFilesFromMailArchives job
-# to create Hadoop SequenceFiles in /mnt/asf-mail-archives/output
-#
-#/**
-# * Licensed to the Apache Software Foundation (ASF) under one or more
-# * contributor license agreements. See the NOTICE file distributed with
-# * this work for additional information regarding copyright ownership.
-# * The ASF licenses this file to You under the Apache License, Version 2.0
-# * (the "License"); you may not use this file except in compliance with
-# * the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-if [ "$MAHOUT_HOME" = "" ]; then
- echo "Error: MAHOUT_HOME is not set."
- exit 1
-fi
-
-if [ "$1" = "" ]; then
- echo "Error: Please pass the path to your prep directory, such as /mnt/asf-mail-archives.\n\n\tUsage: $0 workingDir inputPath outputPath\n"
- exit 1
-fi
-
-# Location where this script saves files
-PREP_DIR=$1
-
-if [ "$2" != "" ]; then
- SEQFILE_INPUT_DIR=$2
-else
- SEQFILE_INPUT_DIR=$PREP_DIR/input
-fi
-
-
-# Change this to an HDFS path if you are running Hadoop
-if [ "$3" != "" ]; then
- SEQFILE_OUTPUT_DIR=$3
-else
- SEQFILE_OUTPUT_DIR=$PREP_DIR/sequence-files
-fi
-
-# If output sent to HDFS, clear MAHOUT_LOCAL and make sure HADOOP_HOME is set
-if [[ "$SEQFILE_OUTPUT_DIR" = hdfs://* ]]; then
- export MAHOUT_LOCAL=
- if [ "$HADOOP_HOME" = "" ]; then
- echo "Error: HADOOP_HOME must be set if you want to send output to HDFS."
- exit 1
- fi
-else
- export MAHOUT_LOCAL=$PREP_DIR
-fi
-
-echo "Running $0 with:
- PREP_DIR = $PREP_DIR
- SEQFILE_INPUT_DIR = $SEQFILE_INPUT_DIR
- SEQFILE_OUTPUT_DIR = $SEQFILE_OUTPUT_DIR
- MAHOUT_LOCAL = $MAHOUT_LOCAL
- HADOOP_HOME = $HADOOP_HOME"
-
-# Run Mahout in Local mode! Remove this if you want the
-# sequence files stored in your HDFS
-
-
-# convert the extracted gz files into Hadoop SequenceFiles
-echo "Converting extracted directories to SequenceFiles ..."
-$MAHOUT_HOME/bin/mahout org.apache.mahout.text.SequenceFilesFromMailArchives \
---input $SEQFILE_INPUT_DIR --output $SEQFILE_OUTPUT_DIR --subject --body \
--c UTF-8 -chunk 1024 -prefix asf_archives
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/pom.xml
----------------------------------------------------------------------
diff --git a/integration/pom.xml b/integration/pom.xml
deleted file mode 100644
index 5a873a6..0000000
--- a/integration/pom.xml
+++ /dev/null
@@ -1,198 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.mahout</groupId>
- <artifactId>mahout</artifactId>
- <version>0.13.1-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>mahout-integration</artifactId>
- <name>Mahout Integration</name>
- <description>Optional components of Mahout which generally support interaction with third party systems,
- formats, APIs, etc.</description>
-
- <packaging>jar</packaging>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-remote-resources-plugin</artifactId>
- <configuration>
- <appendedResourcesDirectory>../src/main/appended-resources</appendedResourcesDirectory>
- <resourceBundles>
- <resourceBundle>org.apache:apache-jar-resource-bundle:1.4</resourceBundle>
- </resourceBundles>
- <supplementalModels>
- <supplementalModel>supplemental-models.xml</supplementalModel>
- </supplementalModels>
- </configuration>
- </plugin>
-
- <plugin>
- <artifactId>maven-javadoc-plugin</artifactId>
- </plugin>
-
- <plugin>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
-
- </plugins>
-
- </build>
-
- <dependencies>
-
- <!-- own modules -->
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>mahout-hdfs</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>mahout-mr</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>mahout-hdfs</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>mahout-mr</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>mahout-math</artifactId>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>mahout-math</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <!-- 3rd party -->
-
- <dependency>
- <groupId>commons-dbcp</groupId>
- <artifactId>commons-dbcp</artifactId>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>commons-pool</groupId>
- <artifactId>commons-pool</artifactId>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-commons-csv</artifactId>
- <version>3.5.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-benchmark</artifactId>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers-common</artifactId>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>2.11.2</version>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>bson</artifactId>
- <version>2.11.2</version>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.hectorclient</groupId>
- <artifactId>hector-core</artifactId>
- <version>1.1-4</version>
- <optional>true</optional>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-jcl</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.carrotsearch.randomizedtesting</groupId>
- <artifactId>randomizedtesting-runner</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/src/main/java/org/apache/mahout/benchmark/BenchmarkRunner.java
----------------------------------------------------------------------
diff --git a/integration/src/main/java/org/apache/mahout/benchmark/BenchmarkRunner.java b/integration/src/main/java/org/apache/mahout/benchmark/BenchmarkRunner.java
deleted file mode 100644
index 549cf2c..0000000
--- a/integration/src/main/java/org/apache/mahout/benchmark/BenchmarkRunner.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.benchmark;
-
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.mahout.common.RandomUtils;
-import org.apache.mahout.common.TimingStatistics;
-import org.apache.mahout.math.Vector;
-
-import com.google.common.base.Function;
-
-public final class BenchmarkRunner {
- private static final int BUCKET_SIZE = 10000;
- private static final Random R = RandomUtils.getRandom();
- private final long maxTimeUsec;
- private final long leadTimeUsec;
-
- public BenchmarkRunner(long leadTimeMs, long maxTimeMs) {
- maxTimeUsec = TimeUnit.MILLISECONDS.toNanos(maxTimeMs);
- leadTimeUsec = TimeUnit.MILLISECONDS.toNanos(leadTimeMs);
- }
-
- public abstract static class BenchmarkFn implements Function<Integer, Boolean> {
- protected int randIndex() {
- return BenchmarkRunner.randIndex();
- }
-
- protected boolean randBool() {
- return BenchmarkRunner.randBool();
- }
-
- /**
- * Adds a random data dependency so that JVM does not remove dead code.
- */
- protected boolean depends(Vector v) {
- return randIndex() < v.getNumNondefaultElements();
- }
- }
-
- public abstract static class BenchmarkFnD implements Function<Integer, Double> {
- protected int randIndex() {
- return BenchmarkRunner.randIndex();
- }
-
- protected boolean randBool() {
- return BenchmarkRunner.randBool();
- }
-
- /**
- * Adds a random data dependency so that JVM does not remove dead code.
- */
- protected boolean depends(Vector v) {
- return randIndex() < v.getNumNondefaultElements();
- }
- }
-
- private static int randIndex() {
- return R.nextInt(BUCKET_SIZE);
- }
-
- private static boolean randBool() {
- return R.nextBoolean();
- }
-
- public TimingStatistics benchmark(BenchmarkFn function) {
- TimingStatistics stats = new TimingStatistics();
- boolean result = false;
- while (true) {
- int i = R.nextInt(BUCKET_SIZE);
- TimingStatistics.Call call = stats.newCall(leadTimeUsec);
- result = result ^ function.apply(i);
- if (call.end(maxTimeUsec)) {
- break;
- }
- }
- return stats;
- }
-
- public TimingStatistics benchmarkD(BenchmarkFnD function) {
- TimingStatistics stats = new TimingStatistics();
- double result = 0;
- while (true) {
- int i = R.nextInt(BUCKET_SIZE);
- TimingStatistics.Call call = stats.newCall(leadTimeUsec);
- result += function.apply(i);
- if (call.end(maxTimeUsec)) {
- break;
- }
- }
- // print result to prevent hotspot from eliminating deadcode
- System.err.println("Result = " + result);
- return stats;
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/src/main/java/org/apache/mahout/benchmark/CloneBenchmark.java
----------------------------------------------------------------------
diff --git a/integration/src/main/java/org/apache/mahout/benchmark/CloneBenchmark.java b/integration/src/main/java/org/apache/mahout/benchmark/CloneBenchmark.java
deleted file mode 100644
index 5e6ab4d..0000000
--- a/integration/src/main/java/org/apache/mahout/benchmark/CloneBenchmark.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.benchmark;
-
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_SPARSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_SPARSE_VECTOR;
-
-import org.apache.mahout.benchmark.BenchmarkRunner.BenchmarkFn;
-
-public class CloneBenchmark {
- public static final String CLONE = "Clone";
- private final VectorBenchmarks mark;
-
- public CloneBenchmark(VectorBenchmarks mark) {
- this.mark = mark;
- }
-
- public void benchmark() {
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- mark.vectors[0][mark.vIndex(i)] = mark.vectors[0][mark.vIndex(i)].clone();
-
- return depends(mark.vectors[0][mark.vIndex(i)]);
- }
- }), CLONE, DENSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- mark.vectors[1][mark.vIndex(i)] = mark.vectors[1][mark.vIndex(i)].clone();
-
- return depends(mark.vectors[1][mark.vIndex(i)]);
- }
- }), CLONE, RAND_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- mark.vectors[2][mark.vIndex(i)] = mark.vectors[2][mark.vIndex(i)].clone();
-
- return depends(mark.vectors[2][mark.vIndex(i)]);
- }
- }), CLONE, SEQ_SPARSE_VECTOR);
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/src/main/java/org/apache/mahout/benchmark/ClosestCentroidBenchmark.java
----------------------------------------------------------------------
diff --git a/integration/src/main/java/org/apache/mahout/benchmark/ClosestCentroidBenchmark.java b/integration/src/main/java/org/apache/mahout/benchmark/ClosestCentroidBenchmark.java
deleted file mode 100644
index b1c2ded..0000000
--- a/integration/src/main/java/org/apache/mahout/benchmark/ClosestCentroidBenchmark.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.benchmark;
-
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.mahout.common.RandomUtils;
-import org.apache.mahout.common.TimingStatistics;
-import org.apache.mahout.common.distance.DistanceMeasure;
-import org.apache.mahout.math.SparseMatrix;
-import org.apache.mahout.math.Vector;
-
-public class ClosestCentroidBenchmark {
- private final VectorBenchmarks mark;
-
- public ClosestCentroidBenchmark(VectorBenchmarks mark) {
- this.mark = mark;
- }
-
- public void benchmark(DistanceMeasure measure) throws IOException {
- SparseMatrix clusterDistances = new SparseMatrix(mark.numClusters, mark.numClusters);
- for (int i = 0; i < mark.numClusters; i++) {
- for (int j = 0; j < mark.numClusters; j++) {
- double distance = Double.POSITIVE_INFINITY;
- if (i != j) {
- distance = measure.distance(mark.clusters[i], mark.clusters[j]);
- }
- clusterDistances.setQuick(i, j, distance);
- }
- }
-
- long distanceCalculations = 0;
- TimingStatistics stats = new TimingStatistics();
- for (int l = 0; l < mark.loop; l++) {
- TimingStatistics.Call call = stats.newCall(mark.leadTimeUsec);
- for (int i = 0; i < mark.numVectors; i++) {
- Vector vector = mark.vectors[1][mark.vIndex(i)];
- double minDistance = Double.MAX_VALUE;
- for (int k = 0; k < mark.numClusters; k++) {
- double distance = measure.distance(vector, mark.clusters[k]);
- distanceCalculations++;
- if (distance < minDistance) {
- minDistance = distance;
- }
- }
- }
- if (call.end(mark.maxTimeUsec)) {
- break;
- }
- }
- mark.printStats(stats, measure.getClass().getName(), "Closest C w/o Elkan's trick", "distanceCalculations = "
- + distanceCalculations);
-
- distanceCalculations = 0;
- stats = new TimingStatistics();
- Random rand = RandomUtils.getRandom();
- for (int l = 0; l < mark.loop; l++) {
- TimingStatistics.Call call = stats.newCall(mark.leadTimeUsec);
- for (int i = 0; i < mark.numVectors; i++) {
- Vector vector = mark.vectors[1][mark.vIndex(i)];
- int closestCentroid = rand.nextInt(mark.numClusters);
- double dist = measure.distance(vector, mark.clusters[closestCentroid]);
- distanceCalculations++;
- for (int k = 0; k < mark.numClusters; k++) {
- if (closestCentroid != k) {
- double centroidDist = clusterDistances.getQuick(k, closestCentroid);
- if (centroidDist < 2 * dist) {
- dist = measure.distance(vector, mark.clusters[k]);
- closestCentroid = k;
- distanceCalculations++;
- }
- }
- }
- }
- if (call.end(mark.maxTimeUsec)) {
- break;
- }
- }
- mark.printStats(stats, measure.getClass().getName(), "Closest C w/ Elkan's trick", "distanceCalculations = "
- + distanceCalculations);
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/src/main/java/org/apache/mahout/benchmark/DistanceBenchmark.java
----------------------------------------------------------------------
diff --git a/integration/src/main/java/org/apache/mahout/benchmark/DistanceBenchmark.java b/integration/src/main/java/org/apache/mahout/benchmark/DistanceBenchmark.java
deleted file mode 100644
index 25d0ad7..0000000
--- a/integration/src/main/java/org/apache/mahout/benchmark/DistanceBenchmark.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.benchmark;
-
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_FN_RAND;
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_FN_SEQ;
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_FN_DENSE;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_FN_SEQ;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_SPARSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_FN_DENSE;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_FN_RAND;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_SPARSE_VECTOR;
-
-import org.apache.mahout.benchmark.BenchmarkRunner.BenchmarkFnD;
-import org.apache.mahout.common.distance.DistanceMeasure;
-
-public class DistanceBenchmark {
- private final VectorBenchmarks mark;
-
- public DistanceBenchmark(VectorBenchmarks mark) {
- this.mark = mark;
- }
-
- public void benchmark(final DistanceMeasure measure) {
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[0][mark.vIndex(i)], mark.vectors[0][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), DENSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[1][mark.vIndex(i)], mark.vectors[1][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), RAND_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[2][mark.vIndex(i)], mark.vectors[2][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), SEQ_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[0][mark.vIndex(i)], mark.vectors[1][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), DENSE_FN_RAND);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[0][mark.vIndex(i)], mark.vectors[2][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), DENSE_FN_SEQ);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[1][mark.vIndex(i)], mark.vectors[0][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), RAND_FN_DENSE);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[1][mark.vIndex(i)], mark.vectors[2][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), RAND_FN_SEQ);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[2][mark.vIndex(i)], mark.vectors[0][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), SEQ_FN_DENSE);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return measure.distance(mark.vectors[2][mark.vIndex(i)], mark.vectors[1][mark.vIndex(randIndex())]);
- }
- }), measure.getClass().getName(), SEQ_FN_RAND);
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/src/main/java/org/apache/mahout/benchmark/DotBenchmark.java
----------------------------------------------------------------------
diff --git a/integration/src/main/java/org/apache/mahout/benchmark/DotBenchmark.java b/integration/src/main/java/org/apache/mahout/benchmark/DotBenchmark.java
deleted file mode 100644
index fc7f911..0000000
--- a/integration/src/main/java/org/apache/mahout/benchmark/DotBenchmark.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.benchmark;
-
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_FN_RAND;
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_FN_SEQ;
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_FN_DENSE;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_FN_SEQ;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_SPARSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_FN_DENSE;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_FN_RAND;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_SPARSE_VECTOR;
-
-import org.apache.mahout.benchmark.BenchmarkRunner.BenchmarkFn;
-import org.apache.mahout.benchmark.BenchmarkRunner.BenchmarkFnD;
-
-public class DotBenchmark {
- private static final String DOT_PRODUCT = "DotProduct";
- private static final String NORM1 = "Norm1";
- private static final String NORM2 = "Norm2";
- private static final String LOG_NORMALIZE = "LogNormalize";
- private final VectorBenchmarks mark;
-
- public DotBenchmark(VectorBenchmarks mark) {
- this.mark = mark;
- }
-
- public void benchmark() {
- benchmarkDot();
- benchmarkNorm1();
- benchmarkNorm2();
- benchmarkLogNormalize();
- }
-
- private void benchmarkLogNormalize() {
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- return depends(mark.vectors[0][mark.vIndex(i)].logNormalize());
- }
- }), LOG_NORMALIZE, DENSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- return depends(mark.vectors[1][mark.vIndex(i)].logNormalize());
- }
- }), LOG_NORMALIZE, RAND_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- return depends(mark.vectors[2][mark.vIndex(i)].logNormalize());
- }
- }), LOG_NORMALIZE, SEQ_SPARSE_VECTOR);
- }
-
- private void benchmarkNorm1() {
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[0][mark.vIndex(i)].norm(1);
- }
- }), NORM1, DENSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[1][mark.vIndex(i)].norm(1);
- }
- }), NORM1, RAND_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[2][mark.vIndex(i)].norm(1);
- }
- }), NORM1, SEQ_SPARSE_VECTOR);
- }
-
- private void benchmarkNorm2() {
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[0][mark.vIndex(i)].norm(2);
- }
- }), NORM2, DENSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[1][mark.vIndex(i)].norm(2);
- }
- }), NORM2, RAND_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[2][mark.vIndex(i)].norm(2);
- }
- }), NORM2, SEQ_SPARSE_VECTOR);
- }
-
- private void benchmarkDot() {
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[0][mark.vIndex(i)].dot(mark.vectors[0][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, DENSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[1][mark.vIndex(i)].dot(mark.vectors[1][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, RAND_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[2][mark.vIndex(i)].dot(mark.vectors[2][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, SEQ_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[0][mark.vIndex(i)].dot(mark.vectors[1][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, DENSE_FN_RAND);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[0][mark.vIndex(i)].dot(mark.vectors[2][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, DENSE_FN_SEQ);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[1][mark.vIndex(i)].dot(mark.vectors[0][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, RAND_FN_DENSE);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[1][mark.vIndex(i)].dot(mark.vectors[2][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, RAND_FN_SEQ);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[2][mark.vIndex(i)].dot(mark.vectors[0][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, SEQ_FN_DENSE);
-
- mark.printStats(mark.getRunner().benchmarkD(new BenchmarkFnD() {
- @Override
- public Double apply(Integer i) {
- return mark.vectors[2][mark.vIndex(i)].dot(mark.vectors[1][mark.vIndex(randIndex())]);
- }
- }), DOT_PRODUCT, SEQ_FN_RAND);
- }
-
- public static void main(String[] args) {
- VectorBenchmarks mark = new VectorBenchmarks(1000000, 100, 1000, 10, 1);
- mark.createData();
- new DotBenchmark(mark).benchmarkNorm2();
- System.out.println(mark);
- }
-}
http://git-wip-us.apache.org/repos/asf/mahout/blob/99a5358f/integration/src/main/java/org/apache/mahout/benchmark/MinusBenchmark.java
----------------------------------------------------------------------
diff --git a/integration/src/main/java/org/apache/mahout/benchmark/MinusBenchmark.java b/integration/src/main/java/org/apache/mahout/benchmark/MinusBenchmark.java
deleted file mode 100644
index 82fb693..0000000
--- a/integration/src/main/java/org/apache/mahout/benchmark/MinusBenchmark.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.benchmark;
-
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_FN_RAND;
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_FN_SEQ;
-import static org.apache.mahout.benchmark.VectorBenchmarks.DENSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_FN_DENSE;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_FN_SEQ;
-import static org.apache.mahout.benchmark.VectorBenchmarks.RAND_SPARSE_VECTOR;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_FN_DENSE;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_FN_RAND;
-import static org.apache.mahout.benchmark.VectorBenchmarks.SEQ_SPARSE_VECTOR;
-
-import org.apache.mahout.benchmark.BenchmarkRunner.BenchmarkFn;
-import org.apache.mahout.math.Vector;
-
-public class MinusBenchmark {
-
- private static final String MINUS = "Minus";
- private final VectorBenchmarks mark;
-
- public MinusBenchmark(VectorBenchmarks mark) {
- this.mark = mark;
- }
-
- public void benchmark() {
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[0][mark.vIndex(i)].minus(mark.vectors[0][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, DENSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[1][mark.vIndex(i)].minus(mark.vectors[1][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, RAND_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[2][mark.vIndex(i)].minus(mark.vectors[2][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, SEQ_SPARSE_VECTOR);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[0][mark.vIndex(i)].minus(mark.vectors[1][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, DENSE_FN_RAND);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[0][mark.vIndex(i)].minus(mark.vectors[2][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, DENSE_FN_SEQ);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[1][mark.vIndex(i)].minus(mark.vectors[0][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, RAND_FN_DENSE);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[1][mark.vIndex(i)].minus(mark.vectors[2][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, RAND_FN_SEQ);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[2][mark.vIndex(i)].minus(mark.vectors[0][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, SEQ_FN_DENSE);
-
- mark.printStats(mark.getRunner().benchmark(new BenchmarkFn() {
- @Override
- public Boolean apply(Integer i) {
- Vector v = mark.vectors[2][mark.vIndex(i)].minus(mark.vectors[1][mark.vIndex(randIndex())]);
- return depends(v);
- }
- }), MINUS, SEQ_FN_RAND);
- }
-}