You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mrql.apache.org by fe...@apache.org on 2013/10/29 15:25:39 UTC
git commit: MRQL-24: Improve Serializable methods for Spark
Updated Branches:
refs/heads/master 2f7bbacd8 -> 451c3f081
MRQL-24: Improve Serializable methods for Spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-mrql/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mrql/commit/451c3f08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mrql/tree/451c3f08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mrql/diff/451c3f08
Branch: refs/heads/master
Commit: 451c3f08149f59300691d52ea1c7381887ca01f8
Parents: 2f7bbac
Author: fegaras <fe...@cse.uta.edu>
Authored: Tue Oct 29 09:25:06 2013 -0500
Committer: fegaras <fe...@cse.uta.edu>
Committed: Tue Oct 29 09:25:06 2013 -0500
----------------------------------------------------------------------
src/main/java/core/Bag.java | 43 +++++++++++++++++++++++++-------
src/main/java/core/BagIterator.java | 9 +------
src/main/java/core/Environment.java | 1 +
src/main/java/core/Function.java | 6 -----
src/main/java/core/MRContainer.java | 2 +-
src/main/java/core/MRData.java | 11 --------
src/main/java/core/MR_dataset.java | 8 ++----
src/main/java/core/Tuple.java | 2 ++
8 files changed, 41 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/Bag.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/Bag.java b/src/main/java/core/Bag.java
index d0d77c3..b092f30 100644
--- a/src/main/java/core/Bag.java
+++ b/src/main/java/core/Bag.java
@@ -31,13 +31,14 @@ import org.apache.hadoop.fs.*;
* 3) spilled to a local file: can be accessed multiple times
*/
public class Bag extends MRData implements Iterable<MRData> {
+ private final static long serialVersionUID = 64629834894869L;
enum Modes { STREAMED, MATERIALIZED, SPILLED };
- private Modes mode;
- private ArrayList<MRData> content; // content of a materialized bag
- private BagIterator iterator; // iterator for a streamed bag
- private boolean consumed; // true, if the stream has already been used
- private String path; // local path that contains the spilled bag
- private SequenceFile.Writer writer; // the file writer for spiled bags
+ private transient Modes mode;
+ private transient ArrayList<MRData> content; // content of a materialized bag
+ private transient BagIterator iterator; // iterator for a streamed bag
+ private transient boolean consumed; // true, if the stream has already been used
+ private transient String path; // local path that contains the spilled bag
+ private transient SequenceFile.Writer writer; // the file writer for spiled bags
/**
* create an empty bag as an ArrayList
@@ -68,6 +69,17 @@ public class Bag extends MRData implements Iterable<MRData> {
}
/**
+ * in-memory Bag construction (an ArrayList) initialized with data
+ * @param as a vector of MRData to insert in the Bag
+ */
+ public Bag ( final List<MRData> as ) {
+ mode = Modes.MATERIALIZED;
+ content = new ArrayList<MRData>(as.size());
+ for ( MRData a: as )
+ content.add(a);
+ }
+
+ /**
* lazy construction (stream-based) of a Bag
* @param i the Iterator that generates the Bag elements
*/
@@ -188,12 +200,16 @@ public class Bag extends MRData implements Iterable<MRData> {
}
/** make this Bag empty (cache it in memory if necessary) */
- public void clear () throws IOException {
+ public void clear () {
if (materialized())
content.clear();
else if (streamed()) {
if (writer != null)
- writer.close();
+ try {
+ writer.close();
+ } catch (IOException ex) {
+ throw new Error(ex);
+ };
writer = null;
path = null;
mode = Modes.MATERIALIZED;
@@ -471,9 +487,18 @@ public class Bag extends MRData implements Iterable<MRData> {
}
private void readObject ( ObjectInputStream in ) throws IOException, ClassNotFoundException {
- readFields(in);
+ int n = WritableUtils.readVInt(in);
+ mode = Modes.MATERIALIZED;
+ iterator = null;
+ path = null;
+ writer = null;
+ content = new ArrayList<MRData>(n);
+ for ( int i = 0; i < n; i++ )
+ add(MRContainer.read(in));
}
+ private void readObjectNoData () throws ObjectStreamException { };
+
/** compare this Bag with a given Bag by comparing their associated elements */
public int compareTo ( MRData x ) {
Bag xt = (Bag)x;
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/BagIterator.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/BagIterator.java b/src/main/java/core/BagIterator.java
index e87c316..0fd7354 100644
--- a/src/main/java/core/BagIterator.java
+++ b/src/main/java/core/BagIterator.java
@@ -18,17 +18,10 @@
package org.apache.mrql;
import java.util.Iterator;
-import java.io.*;
-
/** an Iterator over Bags */
-abstract public class BagIterator implements Iterator<MRData>, Serializable {
+abstract public class BagIterator implements Iterator<MRData> {
public void remove () {
throw new Error("Bag deletions are not permitted");
}
- public void writeObject(ObjectOutputStream out) throws IOException {}
-
- public void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {}
-
- public void readObjectNoData() throws ObjectStreamException {}
}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/Environment.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/Environment.java b/src/main/java/core/Environment.java
index 2c5ee11..da8dd84 100644
--- a/src/main/java/core/Environment.java
+++ b/src/main/java/core/Environment.java
@@ -20,6 +20,7 @@ package org.apache.mrql;
import java.io.*;
import org.apache.mrql.gen.Tree;
+
/** the run-time environment for in-memory evaluation (binds variables to MRData) */
final public class Environment implements Serializable {
public String name;
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/Function.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/Function.java b/src/main/java/core/Function.java
index 8d700a8..1f1d4e1 100644
--- a/src/main/java/core/Function.java
+++ b/src/main/java/core/Function.java
@@ -31,10 +31,4 @@ abstract public class Function implements Serializable {
* @return the result of evaluating
*/
abstract public MRData eval ( final MRData arg );
-
- public void writeObject(ObjectOutputStream out) throws IOException {}
-
- public void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {}
-
- public void readObjectNoData() throws ObjectStreamException {}
}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/MRContainer.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/MRContainer.java b/src/main/java/core/MRContainer.java
index 1f8d5d9..d72c452 100644
--- a/src/main/java/core/MRContainer.java
+++ b/src/main/java/core/MRContainer.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.io.WritableComparable;
/** A container for MRData that implements read (the deserializer) */
final public class MRContainer implements WritableComparable<MRContainer>, Serializable {
- MRData data;
+ transient MRData data;
public final static byte BOOLEAN = 0, BYTE = 1, SHORT = 2, INT = 3, LONG = 4,
FLOAT = 5, DOUBLE = 6, CHAR = 7, STRING = 8, PAIR = 9, TUPLE = 10, BAG = 11,
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/MRData.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/MRData.java b/src/main/java/core/MRData.java
index 3fef8ec..8c08269 100644
--- a/src/main/java/core/MRData.java
+++ b/src/main/java/core/MRData.java
@@ -24,15 +24,4 @@ import java.io.*;
/** All MRQL data are encoded as MRData (similar to AVRO form) */
public abstract class MRData implements WritableComparable<MRData>, Serializable {
abstract public void materializeAll ();
-
- /* doesn't work
- private void writeObject ( ObjectOutputStream out ) throws IOException {
- write(out);
- }
-
- private void readObject ( ObjectInputStream in ) throws IOException, ClassNotFoundException {
- byte n = in.readByte();
- readFields(in);
- }
- */
}
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/MR_dataset.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/MR_dataset.java b/src/main/java/core/MR_dataset.java
index f384f10..abc81cd 100644
--- a/src/main/java/core/MR_dataset.java
+++ b/src/main/java/core/MR_dataset.java
@@ -34,12 +34,8 @@ final public class MR_dataset extends MRData {
public DataSet dataset () { return dataset; }
- final public void write ( DataOutput out ) throws IOException {
- try {
- Plan.collect(dataset).write(out);
- } catch (Exception ex) {
- throw new Error(ex);
- }
+ public void write ( DataOutput out ) throws IOException {
+ throw new Error("DataSets are not serializable");
}
public void readFields ( DataInput in ) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-mrql/blob/451c3f08/src/main/java/core/Tuple.java
----------------------------------------------------------------------
diff --git a/src/main/java/core/Tuple.java b/src/main/java/core/Tuple.java
index 9825343..bced3dd 100644
--- a/src/main/java/core/Tuple.java
+++ b/src/main/java/core/Tuple.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.fs.*;
/** a container for Tuples */
final public class Tuple extends MRData {
+ private final static long serialVersionUID = 723385754575L;
+
MRData[] tuple;
public Tuple ( int size ) {