You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/11/30 23:26:37 UTC
svn commit: r885597 - in /hadoop/pig/branches/load-store-redesign: ./
src/org/apache/pig/ src/org/apache/pig/builtin/
src/org/apache/pig/impl/logicalLayer/optimizer/
Author: pradeepkth
Date: Mon Nov 30 22:26:37 2009
New Revision: 885597
URL: http://svn.apache.org/viewvc?rev=885597&view=rev
Log:
PIG-1072: ReversibleLoadStoreFunc interface should be removed to enable different load and store implementation classes to be used in a reversible manner (rding via pradeepkth)
Modified:
hadoop/pig/branches/load-store-redesign/CHANGES.txt
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ReversibleLoadStoreFunc.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/BinStorage.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/PigStorage.java
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/StreamOptimizer.java
Modified: hadoop/pig/branches/load-store-redesign/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/CHANGES.txt?rev=885597&r1=885596&r2=885597&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/CHANGES.txt (original)
+++ hadoop/pig/branches/load-store-redesign/CHANGES.txt Mon Nov 30 22:26:37 2009
@@ -36,6 +36,10 @@
PIG-1090: Update sources to reflect recent changes in load-store interfaces
(pradeepkth)
+PIG-1072: ReversibleLoadStoreFunc interface should be removed to enable
+different load and store implementation classes to be used in a reversible
+manner (rding via pradeepkth)
+
IMPROVEMENTS
PIG-1053: Consider moving to Hadoop for local mode (ankit.modi via olgan)
Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ReversibleLoadStoreFunc.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ReversibleLoadStoreFunc.java?rev=885597&r1=885596&r2=885597&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ReversibleLoadStoreFunc.java (original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/ReversibleLoadStoreFunc.java Mon Nov 30 22:26:37 2009
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig;
-
-/**
- * This interface is used to implement classes that can perform both
- * Load and Store functionalities in a symmetric fashion (thus reversible).
- *
- * The symmetry property of implementations is used in the optimization
- * engine therefore violation of this property while implementing this
- * interface is likely to result in unexpected output from executions.
- *
- */
-public interface ReversibleLoadStoreFunc {
-
-}
Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/BinStorage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/BinStorage.java?rev=885597&r1=885596&r2=885597&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/BinStorage.java (original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/BinStorage.java Mon Nov 30 22:26:37 2009
@@ -28,7 +28,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
@@ -42,7 +41,6 @@
import org.apache.pig.PigException;
import org.apache.pig.PigWarning;
import org.apache.pig.ResourceSchema;
-import org.apache.pig.ReversibleLoadStoreFunc;
import org.apache.pig.StoreFunc;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
@@ -56,7 +54,7 @@
import org.apache.pig.impl.util.LogUtils;
public class BinStorage extends FileInputLoadFunc
-implements ReversibleLoadStoreFunc, LoadCaster, StoreFunc {
+implements LoadCaster, StoreFunc {
public static final int RECORD_1 = 0x01;
Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/PigStorage.java?rev=885597&r1=885596&r2=885597&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/PigStorage.java (original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/builtin/PigStorage.java Mon Nov 30 22:26:37 2009
@@ -38,11 +38,9 @@
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.pig.FileInputLoadFunc;
-import org.apache.pig.LoadCaster;
import org.apache.pig.LoadFunc;
import org.apache.pig.PigException;
import org.apache.pig.ResourceSchema;
-import org.apache.pig.ReversibleLoadStoreFunc;
import org.apache.pig.StoreFunc;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
@@ -57,9 +55,7 @@
* delimiter is given as a regular expression. See String.split(delimiter) and
* http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/Pattern.html for more information.
*/
-public class PigStorage
-extends FileInputLoadFunc
-implements ReversibleLoadStoreFunc, StoreFunc {
+public class PigStorage extends FileInputLoadFunc implements StoreFunc {
protected RecordReader in = null;
protected RecordWriter writer = null;
protected final Log mLog = LogFactory.getLog(getClass());
Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/StreamOptimizer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/StreamOptimizer.java?rev=885597&r1=885596&r2=885597&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/StreamOptimizer.java (original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/optimizer/StreamOptimizer.java Mon Nov 30 22:26:37 2009
@@ -24,7 +24,6 @@
import org.apache.pig.FuncSpec;
import org.apache.pig.LoadFunc;
import org.apache.pig.PigException;
-import org.apache.pig.ReversibleLoadStoreFunc;
import org.apache.pig.StoreFunc;
import org.apache.pig.builtin.BinaryStorage;
import org.apache.pig.impl.PigContext;
@@ -118,23 +117,7 @@
// LoadFunc and if it does, are they of the same _reversible_
// type?
boolean sameType = false;
- try {
- // Check if the streamStorer is _reversible_ as
- // the inputLoader ...
- if (streamStorer instanceof LoadFunc) {
- // Cast to check if they are of the same type...
- streamStorer.getClass().cast(inputLoader);
- LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
- "inputLoader:" + inputLoader);
- // Now check if they both are reversible...
- if (streamStorer instanceof ReversibleLoadStoreFunc &&
- inputLoader instanceof ReversibleLoadStoreFunc) {
- sameType = true;
- }
- }
- } catch (ClassCastException cce) {
- sameType = false;
- }
+
// Check if both LoadFunc objects belong to the same type and
// are equivalent
if (sameType && streamStorer.equals(inputLoader)) {
@@ -172,22 +155,7 @@
// StoreFunc and if it does, are they of the same _reversible_
// type?
boolean sameType = false;
- try {
- // Check if the streamLoader is _reversible_ as
- // the inputLoader ...
- if (streamLoader instanceof StoreFunc) {
- // Cast to check if they are of the same type...
- streamLoader.getClass().cast(outputStorer);
-
- // Now check if they both are reversible...
- if (streamLoader instanceof ReversibleLoadStoreFunc &&
- outputStorer instanceof ReversibleLoadStoreFunc) {
- sameType = true;
- }
- }
- } catch (ClassCastException cce) {
- sameType = false;
- }
+
// Check if both LoadFunc objects belong to the same type and
// are equivalent
if (sameType && streamLoader.equals(outputStorer)) {