You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/09/04 06:51:37 UTC
git commit: TEZ-208. Update Input,
Output and Processor APIs as part of Tez Engine refactor (TEZ-398).
(sseth)
Updated Branches:
refs/heads/TEZ-398 8fca5bb82 -> 5e17f1cc0
TEZ-208. Update Input, Output and Processor APIs as part of Tez
Engine refactor (TEZ-398). (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/5e17f1cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/5e17f1cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/5e17f1cc
Branch: refs/heads/TEZ-398
Commit: 5e17f1cc0b0d6126cc35d1d0faaf850777a7ee1e
Parents: 8fca5bb
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Sep 3 21:49:59 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Sep 3 21:49:59 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/engine/newapi/Input.java | 72 ++++++++++++++++++++
.../tez/engine/newapi/LogicalIOProcessor.java | 45 ++++++++++++
.../apache/tez/engine/newapi/LogicalInput.java | 37 ++++++++++
.../apache/tez/engine/newapi/LogicalOutput.java | 36 ++++++++++
.../org/apache/tez/engine/newapi/Output.java | 72 ++++++++++++++++++++
.../org/apache/tez/engine/newapi/Processor.java | 58 ++++++++++++++++
.../org/apache/tez/engine/newapi/Reader.java | 26 +++++++
.../org/apache/tez/engine/newapi/Writer.java | 26 +++++++
.../org/apache/tez/engine/newapi/KVReader.java | 54 +++++++++++++++
.../org/apache/tez/engine/newapi/KVWriter.java | 38 +++++++++++
10 files changed, 464 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Input.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Input.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Input.java
new file mode 100644
index 0000000..9552d4d
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Input.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Represents an input through which a TezProcessor receives data on an edge.
+ * </p>
+ *
+ * <code>Input</code> classes must have a 0 argument public constructor for Tez
+ * to construct the <code>Input</code>. Tez will take care of initializing and
+ * closing the Input after a {@link Processor} completes. </p>
+ */
+public interface Input {
+
+ /**
+ * Initializes the <code>Input</code>.
+ *
+ * @param inputContext
+ * the {@link TezInputContext}
+ * @return
+ * @throws IOException
+ * if an error occurs
+ */
+ public List<Event> initialize(TezInputContext inputContext)
+ throws IOException;
+
+ /**
+ * Gets an instance of the {@link Reader} for this <code>Output</code>
+ *
+ * @return
+ * @throws IOException
+ * if an error occurs
+ */
+ public Reader getReader() throws IOException;
+
+ /**
+ * Handles user and system generated {@link Events}s, which typically carry
+ * information such as an output being available on the previous vertex.
+ *
+ * @param inputEvents
+ * the list of {@link Event}s
+ */
+ public void handleEvents(List<Event> inputEvents);
+
+ /**
+ * Closes the <code>Input</code>
+ *
+ * @return
+ * @throws IOException
+ * if an error occurs
+ */
+ public List<Event> close() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalIOProcessor.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalIOProcessor.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalIOProcessor.java
new file mode 100644
index 0000000..2ffe060
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalIOProcessor.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Represents a processor which consumes {@link LogicalInput}s and produces
+ * {@link LogicalOutput}s
+ */
+public interface LogicalIOProcessor extends Processor {
+
+ /**
+ * Runs the {@link LogicalProcessor}
+ *
+ * @param inputs
+ * a map of the source vertex name to {@link LogicalInput} - one per
+ * incoming edge.
+ * @param outputs
+ * a map of the destination vertex name to {@link LogicalOutput} -
+ * one per outgoing edge
+ * @throws IOException
+ * if an error occurs
+ */
+ public void run(Map<String, LogicalInput> inputs,
+ Map<String, LogicalOutput> outputs) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalInput.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalInput.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalInput.java
new file mode 100644
index 0000000..554172c
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalInput.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+/**
+ * An @link {@link Input} which handles all incoming physical connections on an
+ * edge. A {@link LogicalIOProcessor} sees a single Logical Input per incoming
+ * edge.
+ */
+public interface LogicalInput extends Input {
+
+ /**
+ * Sets the number of physical inputs that this <code>LogicalInput</code> will
+ * receive. This will be called by the Tez framework before initializing the
+ * <code>LogicalInput</code>
+ *
+ * @param numInputs
+ * the number of physical inputs.
+ */
+ public void setNumPhysicalInputs(int numInputs);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalOutput.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalOutput.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalOutput.java
new file mode 100644
index 0000000..d88e043
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/LogicalOutput.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+/**
+ * An @link {@link Output} which handles all outgoing physical connections on an
+ * edge. A {@link LogicalIOProcessor} sees a single Logical Output per outgoing
+ * edge.
+ */
+public interface LogicalOutput extends Output {
+ /**
+ * Sets the number of physical ouputs that this <code>LogicalOutput</code>
+ * will receive. This will be called by the Tez framework before initializing
+ * the <code>LogicalOutput</code>
+ *
+ * @param numOutputs
+ * the number of physical outputs
+ */
+ public void setNumPhysicalOutputs(int numOutputs);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Output.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Output.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Output.java
new file mode 100644
index 0000000..4c036e6
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Output.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Represents an Output through which a TezProcessor writes information on an
+ * edge. </p>
+ *
+ * <code>Output</code> implementations must have a 0 argument public constructor
+ * for Tez to construct the <code>Output</code>. Tez will take care of
+ * initializing and closing the Input after a {@link Processor} completes. </p>
+ */
+public interface Output {
+
+ /**
+ * Initializes the <code>Output</code>
+ *
+ * @param outputContext
+ * the {@link TezOutputContext}
+ * @return
+ * @throws IOException
+ * if an error occurs
+ */
+ public List<Event> initialize(TezOutputContext outputContext)
+ throws IOException;
+
+ /**
+ * Gets an instance of the {@link Writer} in an <code>Output</code>
+ *
+ * @return
+ * @throws IOException
+ * if an error occurs
+ */
+ public Writer getWriter() throws IOException;
+
+ /**
+ * Handles user and system generated {@link Events}s, which typically carry
+ * information such as a downstream vertex being ready to consume input.
+ *
+ * @param outputEvents
+ * the list of {@link Event}s
+ */
+ public void handleEvents(List<Event> outputEvents);
+
+ /**
+ * Closes the <code>Output</code>
+ *
+ * @return
+ * @throws IOException
+ * if an error occurs
+ */
+ public List<Event> close() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Processor.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Processor.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Processor.java
new file mode 100644
index 0000000..3135cf1
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Processor.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.tez.engine.api.Input;
+import org.apache.tez.engine.api.Output;
+
+/**
+ * {@link Processor} represents the <em>Tez</em> entity responsible for
+ * consuming {@link Input} and producing {@link Output}.
+ */
+public interface Processor {
+
+ /**
+ * Initializes the <code>Processor</code>
+ *
+ * @param processorContext
+ * @throws IOException
+ * if an error occurs
+ */
+ public void initialize(TezProcessorContext processorContext)
+ throws IOException;
+
+ /**
+ * Handles user and system generated {@link Events}s.
+ *
+ * @param processorEvents
+ * the list of {@link Event}s
+ */
+ public void handleEvents(List<Event> processorEvents);
+
+ /**
+ * Closes the <code>Processor</code>
+ *
+ * @throws IOException
+ * if an error occurs
+ */
+ public void close() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Reader.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Reader.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Reader.java
new file mode 100644
index 0000000..8b8750c
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Reader.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+/**
+ * A <code>Reader</code> represents the data being read in an {@link Input}
+ */
+public interface Reader {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Writer.java
----------------------------------------------------------------------
diff --git a/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Writer.java b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Writer.java
new file mode 100644
index 0000000..d8c0b8a
--- /dev/null
+++ b/tez-engine-api/src/main/java/org/apache/tez/engine/newapi/Writer.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+/**
+ * A <code>Writer</code> represents the data being written by an {@link Output}
+ */
+public interface Writer {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVReader.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVReader.java b/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVReader.java
new file mode 100644
index 0000000..68e0f47
--- /dev/null
+++ b/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVReader.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+import java.io.IOException;
+
+/**
+ * A key/value(s) pair based {@link Reader}.
+ */
+public interface KVReader extends Reader {
+
+ /**
+ * Check if there is another key/value(s) pair
+ *
+ * @return true if another key/value(s) pair exists
+ * @throws IOException
+ * if an error occurs
+ */
+ public boolean hasNext() throws IOException;
+
+ /**
+ * Gets the next key.
+ *
+ * @return the next key, or null if none exists
+ * @throws IOException
+ * if an error occurs
+ */
+ public Object getNextKey() throws IOException;
+
+ /**
+ * Get the next values.
+ *
+ * @return an <code>Iterable</code> view of the values for the current key
+ * @throws IOException
+ * if an error occurs
+ */
+ public Iterable<Object> getNextValues() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/5e17f1cc/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVWriter.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVWriter.java b/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVWriter.java
new file mode 100644
index 0000000..f945b63
--- /dev/null
+++ b/tez-engine/src/main/java/org/apache/tez/engine/newapi/KVWriter.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.engine.newapi;
+
+import java.io.IOException;
+
+/**
+ * A key/value(s) pair based {@link Writer}
+ */
+public interface KVWriter {
+ /**
+ * Writes a key/value pair.
+ *
+ * @param key
+ * the key to write
+ * @param value
+ * the value to write
+ * @throws IOException
+ * if an error occurs
+ */
+ public void write(Object key, Object value) throws IOException;
+}
\ No newline at end of file