You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2011/05/07 02:15:44 UTC
svn commit: r1100420 [11/19] - in /pig/branches/branch-0.9: ./ src/
src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/
src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/logicalLayer/optimizer/ src/org/apach...
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaCalculator.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaCalculator.java?rev=1100420&r1=1100419&r2=1100420&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaCalculator.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaCalculator.java Sat May 7 00:15:40 2011
@@ -1,311 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.impl.logicalLayer.optimizer;
-
-import org.apache.pig.impl.logicalLayer.*;
-import org.apache.pig.impl.plan.PlanVisitor;
-import org.apache.pig.impl.plan.DependencyOrderWalker;
-import org.apache.pig.impl.plan.VisitorException;
-
-/**
- * A visitor to reset all the schemas in a logical plan.
- */
-public class SchemaCalculator extends LOVisitor {
-
- public SchemaCalculator(LogicalPlan plan) {
- super(plan,
- new DependencyOrderWalker<LogicalOperator, LogicalPlan>(plan));
- }
-
- /**
- * @param binOp
- * the logical binary expression operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(BinaryExpressionOperator binOp)
- throws VisitorException {
- try {
- binOp.getFieldSchema();
- super.visit(binOp);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param uniOp
- * the logical unary operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(UnaryExpressionOperator uniOp) throws VisitorException {
- try {
- uniOp.getFieldSchema();
- super.visit(uniOp);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param cg
- * the logical cogroup operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOCogroup cg) throws VisitorException {
- try {
- cg.getSchema();
- super.visit(cg);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param join
- * the logical join operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOJoin join) throws VisitorException {
- try {
- join.getSchema();
- super.visit(join);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param s
- * the logical sort operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOSort s) throws VisitorException {
- try {
- s.getSchema();
- super.visit(s);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param limit
- * the logical limit operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOLimit limit) throws VisitorException {
- try {
- limit.getSchema();
- super.visit(limit);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param filter
- * the logical filter operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOFilter filter) throws VisitorException {
- try {
- filter.getSchema();
- super.visit(filter);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param split
- * the logical split operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOSplit split) throws VisitorException {
- try {
- split.getSchema();
- super.visit(split);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param forEach
- * the logical foreach operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOForEach forEach) throws VisitorException {
- try {
- super.visit(forEach);
- forEach.getSchema();
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- * Iterate over each expression that is part of the function argument list
- *
- * @param func
- * the user defined function
- * @throws VisitorException
- */
- protected void visit(LOUserFunc func) throws VisitorException {
- try {
- func.getFieldSchema();
- super.visit(func);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- * @param binCond
- * the logical binCond operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOBinCond binCond) throws VisitorException {
- try {
- binCond.getFieldSchema();
- super.visit(binCond);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param cast
- * the logical cast operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LOCast cast) throws VisitorException {
- try {
- cast.getFieldSchema();
- super.visit(cast);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- /**
- *
- * @param regexp
- * the logical regexp operator that has to be visited
- * @throws VisitorException
- */
- protected void visit(LORegexp regexp) throws VisitorException {
- try {
- regexp.getFieldSchema();
- super.visit(regexp);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LOLoad load) throws VisitorException{
- try {
- load.getSchema();
- super.visit(load);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LONative nat) throws VisitorException{
- try {
- nat.getSchema();
- super.visit(nat);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LOStore store) throws VisitorException{
- try {
- store.getSchema();
- super.visit(store);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LOConst c) throws VisitorException{
- try {
- c.getFieldSchema();
- super.visit(c);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LOUnion u) throws VisitorException {
- try {
- u.getSchema();
- super.visit(u);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LOSplitOutput sop) throws VisitorException {
- try {
- sop.getSchema();
- super.visit(sop);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LODistinct dt) throws VisitorException {
- try {
- dt.getSchema();
- super.visit(dt);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LOCross cs) throws VisitorException {
- try {
- cs.getSchema();
- super.visit(cs);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
- protected void visit(LOProject project) throws VisitorException {
- try {
- project.getFieldSchema();
- super.visit(project);
- } catch (FrontendException fe) {
- throw new VisitorException(fe);
- }
- }
-
-}
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java?rev=1100420&r1=1100419&r2=1100420&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java Sat May 7 00:15:40 2011
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.pig.impl.logicalLayer.optimizer;
-
-import org.apache.pig.impl.logicalLayer.*;
-import org.apache.pig.impl.plan.DependencyOrderWalker;
-import org.apache.pig.impl.plan.VisitorException;
-
-/**
- * A visitor to reset all the schemas in a logical plan.
- */
-public class SchemaRemover extends LOVisitor {
-
- public SchemaRemover(LogicalPlan plan) {
- super(plan,
- new DependencyOrderWalker<LogicalOperator, LogicalPlan>(plan));
- }
-
- /**
- * @param binOp
- * the logical binary expression operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(BinaryExpressionOperator binOp)
- throws VisitorException {
- binOp.unsetFieldSchema();
- super.visit(binOp);
- }
-
- /**
- *
- * @param uniOp
- * the logical unary operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(UnaryExpressionOperator uniOp) throws VisitorException {
- uniOp.unsetFieldSchema();
- super.visit(uniOp);
- }
-
- /**
- *
- * @param cg
- * the logical cogroup operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOCogroup cg) throws VisitorException {
- cg.unsetSchema();
- super.visit(cg);
- }
-
- /**
- *
- * @param s
- * the logical sort operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOSort s) throws VisitorException {
- s.unsetSchema();
- super.visit(s);
- }
-
- /**
- *
- * @param limit
- * the logical limit operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOLimit limit) throws VisitorException {
- limit.unsetSchema();
- super.visit(limit);
- }
-
-
- /**
- *
- * @param filter
- * the logical filter operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOFilter filter) throws VisitorException {
- filter.unsetSchema();
- super.visit(filter);
- }
-
- /**
- *
- * @param split
- * the logical split operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOSplit split) throws VisitorException {
- split.unsetSchema();
- super.visit(split);
- }
-
- /**
- *
- * @param forEach
- * the logical foreach operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOForEach forEach) throws VisitorException {
- forEach.unsetSchema();
- super.visit(forEach);
- }
-
- /**
- * Iterate over each expression that is part of the function argument list
- *
- * @param func
- * the user defined function
- * @throws VisitorException
- */
- @Override
- protected void visit(LOUserFunc func) throws VisitorException {
- func.unsetFieldSchema();
- super.visit(func);
- }
-
- /**
- * @param binCond
- * the logical binCond operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOBinCond binCond) throws VisitorException {
- binCond.unsetFieldSchema();
- super.visit(binCond);
- }
-
- /**
- *
- * @param cast
- * the logical cast operator that has to be visited
- * @throws VisitorException
- */
- @Override
- protected void visit(LOCast cast) throws VisitorException {
- cast.unsetFieldSchema();
- super.visit(cast);
- }
-
-
- /**
- *
- * @param regexp
- * the logical regexp operator that has to be visited
- * @throws ParseException
- */
- @Override
- protected void visit(LORegexp regexp) throws VisitorException {
- regexp.unsetFieldSchema();
- super.visit(regexp);
- }
-
- @Override
- protected void visit(LOLoad load) throws VisitorException{
- // Don't remove load's schema, it's not like it will change. And we
- // don't have a way to recover it.
- super.visit(load);
- }
-
- @Override
- protected void visit(LOStore store) throws VisitorException{
- store.unsetSchema();
- super.visit(store);
- }
-
- @Override
- protected void visit(LOConst c) throws VisitorException{
- c.unsetSchema();
- super.visit(c);
- }
-
- @Override
- protected void visit(LOUnion u) throws VisitorException {
- u.unsetSchema();
- super.visit(u);
- }
-
- @Override
- protected void visit(LOSplitOutput sop) throws VisitorException {
- sop.unsetSchema();
- super.visit(sop);
- }
-
- @Override
- protected void visit(LODistinct dt) throws VisitorException {
- dt.unsetSchema();
- super.visit(dt);
- }
-
- @Override
- protected void visit(LOCross cs) throws VisitorException {
- cs.unsetSchema();
- super.visit(cs);
- }
-
- @Override
- protected void visit(LOProject project) throws VisitorException {
- project.unsetFieldSchema();
- super.visit(project);
- }
-
- @Override
- protected void visit(LOJoin join) throws VisitorException {
- join.unsetSchema();
- super.visit(join);
- }
-
- @Override
- protected void visit(ExpressionOperator op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOAdd op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOAnd binOp) throws VisitorException {
- binOp.unsetFieldSchema();
- super.visit(binOp);
- }
-
- @Override
- public void visit(LODivide op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOEqual op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOGreaterThan op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOGreaterThanEqual op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOIsNull uniOp) throws VisitorException {
- uniOp.unsetFieldSchema();
- super.visit(uniOp);
- }
-
- @Override
- public void visit(LOLesserThan op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOLesserThanEqual op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOMapLookup op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOMod op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOMultiply op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LONegative op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LONot uniOp) throws VisitorException {
- uniOp.unsetFieldSchema();
- super.visit(uniOp);
- }
-
- @Override
- public void visit(LONotEqual op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-
- @Override
- public void visit(LOOr binOp) throws VisitorException {
- binOp.unsetFieldSchema();
- super.visit(binOp);
- }
-
- @Override
- public void visit(LOSubtract op) throws VisitorException {
- op.unsetFieldSchema();
- super.visit(op);
- }
-}
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/TypeCastInserter.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/TypeCastInserter.java?rev=1100420&r1=1100419&r2=1100420&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/TypeCastInserter.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/optimizer/TypeCastInserter.java Sat May 7 00:15:40 2011
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pig.impl.logicalLayer.optimizer;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.pig.FuncSpec;
-import org.apache.pig.PigException;
-import org.apache.pig.data.DataType;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.LOCast;
-import org.apache.pig.impl.logicalLayer.LOForEach;
-import org.apache.pig.impl.logicalLayer.LOLoad;
-import org.apache.pig.impl.logicalLayer.LOProject;
-import org.apache.pig.impl.logicalLayer.LOStream;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.impl.plan.optimizer.OptimizerException;
-import org.apache.pig.impl.streaming.StreamingCommand;
-import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec;
-
-/**
- * A visitor to discover if any schema has been specified for a file being
- * loaded. If so, a projection will be injected into the plan to cast the
- * data being loaded to the appropriate types. The optimizer can then come
- * along and move those casts as far down as possible, or in some cases remove
- * them altogether. This visitor does not handle finding the schemas for the
- * file, that has already been done as part of parsing.
- *
- */
-public class TypeCastInserter extends LogicalTransformer {
-
- private String operatorClassName;
-
- public TypeCastInserter(LogicalPlan plan, String operatorClassName) {
- super(plan);
- this.operatorClassName = operatorClassName;
- }
-
- @Override
- public boolean check(List<LogicalOperator> nodes) throws OptimizerException {
- try {
- LogicalOperator op = getOperator(nodes);
- Schema s = op.getSchema();
- if (s == null) return false;
-
- boolean sawOne = false;
- List<Schema.FieldSchema> fss = s.getFields();
- List<Byte> types = new ArrayList<Byte>(s.size());
- Schema determinedSchema = null;
- if(LOLoad.class.getName().equals(operatorClassName)) {
- determinedSchema = ((LOLoad)op).getDeterminedSchema();
- }
- for (int i = 0; i < fss.size(); i++) {
- if (fss.get(i).type != DataType.BYTEARRAY) {
- if(determinedSchema == null ||
- (fss.get(i).type != determinedSchema.getField(i).type)) {
- // Either no schema was determined by loader OR the type
- // from the "determinedSchema" is different
- // from the type specified - so we need to cast
- sawOne = true;
- }
- }
- types.add(fss.get(i).type);
- }
-
- // If all we've found are byte arrays, we don't need a projection.
- return sawOne;
- } catch(OptimizerException oe) {
- throw oe;
- } catch (Exception e) {
- int errCode = 2004;
- String msg = "Internal error while trying to check if type casts are needed";
- throw new OptimizerException(msg, errCode, PigException.BUG, e);
- }
- }
-
- private LogicalOperator getOperator(List<LogicalOperator> nodes) throws FrontendException {
- if((nodes == null) || (nodes.size() <= 0)) {
- int errCode = 2052;
- String msg = "Internal error. Cannot retrieve operator from null or empty list.";
- throw new OptimizerException(msg, errCode, PigException.BUG);
- }
-
- LogicalOperator lo = nodes.get(0);
- if(LOLoad.class.getName().equals(operatorClassName)) {
- if (lo == null || !(lo instanceof LOLoad)) {
- int errCode = 2005;
- String msg = "Expected " + LOLoad.class.getSimpleName()
- + ", got "
- + (lo == null ? lo : lo.getClass().getSimpleName());
- throw new OptimizerException(msg, errCode, PigException.BUG);
- }
-
- return lo;
- } else if(LOStream.class.getName().equals(operatorClassName)){
- if (lo == null || !(lo instanceof LOStream)) {
- int errCode = 2005;
- String msg = "Expected " + LOStream.class.getSimpleName()
- + ", got "
- + (lo == null ? lo : lo.getClass().getSimpleName());
- throw new OptimizerException(msg, errCode, PigException.BUG);
- }
-
- return lo;
- } else {
- // we should never be called with any other operator class name
- int errCode = 1034;
- String msg = "TypeCastInserter invoked with an invalid operator class name:" + operatorClassName;
- throw new OptimizerException(msg, errCode, PigException.INPUT);
- }
-
- }
-
- @Override
- public void transform(List<LogicalOperator> nodes) throws OptimizerException {
- try {
- LogicalOperator lo = getOperator(nodes);
- Schema s = lo.getSchema();
- String scope = lo.getOperatorKey().scope;
- // For every field, build a logical plan. If the field has a type
- // other than byte array, then the plan will be cast(project). Else
- // it will just be project.
- ArrayList<LogicalPlan> genPlans = new ArrayList<LogicalPlan>(s.size());
- ArrayList<Boolean> flattens = new ArrayList<Boolean>(s.size());
- Map<String, Byte> typeChanges = new HashMap<String, Byte>();
- // if we are inserting casts in a load and if the loader
- // implements determineSchema(), insert casts only where necessary
- // Note that in this case, the data coming out of the loader is not
- // a BYTEARRAY but is whatever determineSchema() says it is.
- Schema determinedSchema = null;
- if(LOLoad.class.getName().equals(operatorClassName)) {
- determinedSchema = ((LOLoad)lo).getDeterminedSchema();
- }
- for (int i = 0; i < s.size(); i++) {
- LogicalPlan p = new LogicalPlan();
- genPlans.add(p);
- flattens.add(false);
- List<Integer> toProject = new ArrayList<Integer>(1);
- toProject.add(i);
- LOProject proj = new LOProject(p, OperatorKey.genOpKey(scope),
- lo, toProject);
- p.add(proj);
- Schema.FieldSchema fs = s.getField(i);
- if (fs.type != DataType.BYTEARRAY) {
- if(determinedSchema == null || (fs.type != determinedSchema.getField(i).type)) {
- // Either no schema was determined by loader OR the type
- // from the "determinedSchema" is different
- // from the type specified - so we need to cast
- LOCast cast = new LOCast(p,
- OperatorKey.genOpKey(scope), fs.type);
- cast.setFieldSchema(fs);
- p.add(cast);
- p.connect(proj, cast);
-
- cast.setFieldSchema(fs.clone());
- FuncSpec loadFuncSpec = null;
- if(lo instanceof LOLoad) {
- loadFuncSpec = ((LOLoad)lo).getInputFile().getFuncSpec();
- } else if (lo instanceof LOStream) {
- StreamingCommand command = ((LOStream)lo).getStreamingCommand();
- HandleSpec streamOutputSpec = command.getOutputSpec();
- loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
- } else {
- int errCode = 2006;
- String msg = "TypeCastInserter invoked with an invalid operator class name: " + lo.getClass().getSimpleName();
- throw new OptimizerException(msg, errCode, PigException.BUG);
- }
- cast.setLoadFuncSpec(loadFuncSpec);
- typeChanges.put(fs.canonicalName, fs.type);
- if(determinedSchema == null) {
- // Reset the loads field schema to byte array so that it
- // will reflect reality.
- fs.type = DataType.BYTEARRAY;
- } else {
- // Reset the type to what determinedSchema says it is
- fs.type = determinedSchema.getField(i).type;
- }
- }
- }
- }
-
- // Build a foreach to insert after the load, giving it a cast for each
- // position that has a type other than byte array.
- LOForEach foreach = new LOForEach(mPlan,
- OperatorKey.genOpKey(scope), genPlans, flattens);
- foreach.setAlias(lo.getAlias());
- // Insert the foreach into the plan and patch up the plan.
- insertAfter(lo, foreach, null);
-
- rebuildSchemas();
-
- } catch (OptimizerException oe) {
- throw oe;
- } catch (Exception e) {
- int errCode = 2007;
- String msg = "Unable to insert type casts into plan";
- throw new OptimizerException(msg, errCode, PigException.BUG, e);
- }
- }
-}
-
-
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/package.html
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/package.html?rev=1100420&r1=1100419&r2=1100420&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/package.html (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/package.html Sat May 7 00:15:40 2011
@@ -37,14 +37,5 @@ procedural language where certain statem
other statement in the language rather than being like SQL where the statement
execution tends to be more linear.
-<h2> Notes </h2>
-<p>
-Heads up to developers: when adding a new logical operator to the plan,
-there are a number of classes that need to know about every type of operator.
-These include {@link org.apache.pig.impl.logicalLayer.PlanSetter},
-{@link org.apache.pig.impl.logicalLayer.optimizer.SchemaRemover},
-{@link org.apache.pig.impl.logicalLayer.optimizer.SchemaCalculator}, and
-{@link org.apache.pig.impl.logicalLayer.optimizer.LogicalTransformer}.
-
</body>
</html>