You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2021/02/07 10:00:14 UTC
[incubator-hop] branch master updated: HOP-2504 : Metadata
Injection: Logging levels are not propagating HOP-2505 : Group By : add
support for metadata injection
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new f114fef HOP-2504 : Metadata Injection: Logging levels are not propagating HOP-2505 : Group By : add support for metadata injection
new 9bd98f2 Merge pull request #605 from mattcasters/master
f114fef is described below
commit f114fefbaf180f88d4b88e0e78fcda9a31efd170
Author: Matt Casters <ma...@gmail.com>
AuthorDate: Sat Feb 6 16:55:36 2021 +0100
HOP-2504 : Metadata Injection: Logging levels are not propagating
HOP-2505 : Group By : add support for metadata injection
---
core/src/main/java/org/apache/hop/core/Result.java | 31 ----
.../hop/core/injection/bean/BeanLevelInfo.java | 1 -
.../java/org/apache/hop/pipeline/Pipeline.java | 1 +
.../hop/pipeline/PipelineProfileFactory.java | 45 +++---
.../pipeline/transforms/groupby/Aggregation.java | 118 ++++++++++++++
.../hop/pipeline/transforms/groupby/GroupBy.java | 67 ++++----
.../pipeline/transforms/groupby/GroupByMeta.java | 179 +++++++++------------
.../src/main/java/org/apache/hop/run/HopRun.java | 4 +
.../main/java/org/apache/hop/server/HopServer.java | 4 +-
.../java/org/apache/hop/workflow/Workflow.java | 14 +-
.../org/apache/hop/www/HopServerSingleton.java | 4 +-
.../groupby/messages/messages_en_US.properties | 12 ++
.../xp/HopRunCalculateFilenameExtensionPoint.java | 2 +-
.../pipeline/transforms/metainject/MetaInject.java | 2 +
.../transforms/tableoutput/TableOutput.java | 2 +-
.../csvinput/messages/messages_en_US.properties | 5 +-
.../org/apache/hop/ui/core/widget/ComboVar.java | 6 +-
.../hopgui/file/pipeline/HopGuiPipelineGraph.java | 8 +-
.../hopgui/file/workflow/HopGuiWorkflowGraph.java | 6 +-
.../pipeline/transforms/groupby/GroupByDialog.java | 42 +++--
20 files changed, 316 insertions(+), 237 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/Result.java b/core/src/main/java/org/apache/hop/core/Result.java
index b591755..5e705ae 100644
--- a/core/src/main/java/org/apache/hop/core/Result.java
+++ b/core/src/main/java/org/apache/hop/core/Result.java
@@ -266,37 +266,6 @@ public class Result implements Cloneable {
}
/**
- * Creates a string containing the read/write throughput. Throughput in this case is defined as two measures, number
- * of lines read or written and number of lines read/written per second.
- *
- * @param seconds the number of seconds with which to determine the read/write throughput
- * @return a string containing the read write throughput measures with labelling text
- */
- public String getReadWriteThroughput( int seconds ) {
- String throughput = null;
- if ( seconds != 0 ) {
- String readClause = null, writtenClause = null;
- if ( getNrLinesRead() > 0 ) {
- readClause =
- String.format( "lines read: %d ( %d lines/s)", getNrLinesRead(), ( getNrLinesRead() / seconds ) );
- }
- if ( getNrLinesWritten() > 0 ) {
- writtenClause =
- String.format(
- "%slines written: %d ( %d lines/s)", ( getNrLinesRead() > 0 ? "; " : "" ), getNrLinesWritten(),
- ( getNrLinesWritten() / seconds ) );
- }
- if ( readClause != null || writtenClause != null ) {
- throughput =
- String.format(
- "Pipeline %s%s", ( getNrLinesRead() > 0 ? readClause : "" ), ( getNrLinesWritten() > 0
- ? writtenClause : "" ) );
- }
- }
- return throughput;
- }
-
- /**
* Returns a string representation of the Result object
*
* @see Object#toString()
diff --git a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java
index e0c6c4a..3eb641a 100644
--- a/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java
+++ b/engine/src/main/java/org/apache/hop/core/injection/bean/BeanLevelInfo.java
@@ -126,7 +126,6 @@ class BeanLevelInfo<Meta extends ITransformMeta> {
genericsInfo.put( tps[ i ].getName(), args[ i ] );
}
}
- System.out.println();
}
introspect( info, clazz.getDeclaredFields(), clazz.getDeclaredMethods(), genericsInfo );
diff --git a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
index 604bf6e..8e782ad 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
@@ -2836,6 +2836,7 @@ public abstract class Pipeline implements IVariables, INamedParameters, IHasLogC
*/
public void setLogLevel( LogLevel logLevel ) {
this.logLevel = logLevel;
+ log.setLogLevel( logLevel );
}
/**
diff --git a/engine/src/main/java/org/apache/hop/pipeline/PipelineProfileFactory.java b/engine/src/main/java/org/apache/hop/pipeline/PipelineProfileFactory.java
index 0d37c61..9933332 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/PipelineProfileFactory.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/PipelineProfileFactory.java
@@ -29,6 +29,7 @@ import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.pipeline.transforms.groupby.Aggregation;
import org.apache.hop.pipeline.transforms.groupby.GroupByMeta;
import org.apache.hop.pipeline.transforms.tableinput.TableInputMeta;
@@ -121,38 +122,30 @@ public class PipelineProfileFactory {
nrBooleans++;
}
}
- int nrCalculations =
- nrNumeric
- * numericCalculations.length + nrDates * dateCalculations.length + nrStrings
- * stringCalculations.length + nrBooleans * booleanCalculations.length;
- statsMeta.allocate( 0, nrCalculations );
+ statsMeta.allocate( 0 );
int calcIndex = 0;
for ( int i = 0; i < tableLayout.size(); i++ ) {
IValueMeta valueMeta = tableLayout.getValueMeta( i );
// Numeric data...
//
if ( valueMeta.isNumeric() ) {
- //CHECKSTYLE:Indentation:OFF
- //CHECKSTYLE:LineLength:OFF
for ( int c = 0; c < numericCalculations.length; c++ ) {
- statsMeta.getAggregateField()[ calcIndex ] = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( numericCalculations[ c ] ) + ")";
- statsMeta.getSubjectField()[ calcIndex ] = valueMeta.getName();
- statsMeta.getAggregateType()[ calcIndex ] = numericCalculations[ c ];
- calcIndex++;
+ String aggField = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( numericCalculations[ c ] ) + ")";
+ String aggSubject = valueMeta.getName();
+ int aggType = numericCalculations[ c ];
+ statsMeta.getAggregations().add(new Aggregation(aggField, aggSubject, aggType, null) );
}
}
// String data
//
if ( valueMeta.isString() ) {
- //CHECKSTYLE:Indentation:OFF
- //CHECKSTYLE:LineLength:OFF
for ( int c = 0; c < stringCalculations.length; c++ ) {
- statsMeta.getAggregateField()[ calcIndex ] = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( stringCalculations[ c ] ) + ")";
- statsMeta.getSubjectField()[ calcIndex ] = valueMeta.getName();
- statsMeta.getAggregateType()[ calcIndex ] = stringCalculations[ c ];
- calcIndex++;
+ String aggField = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( stringCalculations[ c ] ) + ")";
+ String aggSubject = valueMeta.getName();
+ int aggType = stringCalculations[ c ];
+ statsMeta.getAggregations().add(new Aggregation(aggField, aggSubject, aggType, null) );
}
}
@@ -160,11 +153,10 @@ public class PipelineProfileFactory {
//
if ( valueMeta.isDate() ) {
for ( int c = 0; c < dateCalculations.length; c++ ) {
- statsMeta.getAggregateField()[ calcIndex ] =
- valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( dateCalculations[ c ] ) + ")";
- statsMeta.getSubjectField()[ calcIndex ] = valueMeta.getName();
- statsMeta.getAggregateType()[ calcIndex ] = dateCalculations[ c ];
- calcIndex++;
+ String aggField = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( dateCalculations[ c ] ) + ")";
+ String aggSubject = valueMeta.getName();
+ int aggType = dateCalculations[ c ];
+ statsMeta.getAggregations().add(new Aggregation(aggField, aggSubject, aggType, null) );
}
}
@@ -172,11 +164,10 @@ public class PipelineProfileFactory {
//
if ( valueMeta.isBoolean() ) {
for ( int c = 0; c < booleanCalculations.length; c++ ) {
- statsMeta.getAggregateField()[ calcIndex ] =
- valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( booleanCalculations[ c ] ) + ")";
- statsMeta.getSubjectField()[ calcIndex ] = valueMeta.getName();
- statsMeta.getAggregateType()[ calcIndex ] = booleanCalculations[ c ];
- calcIndex++;
+ String aggField = valueMeta.getName() + "(" + GroupByMeta.getTypeDesc( booleanCalculations[ c ] ) + ")";
+ String aggSubject = valueMeta.getName();
+ int aggType = booleanCalculations[ c ];
+ statsMeta.getAggregations().add(new Aggregation(aggField, aggSubject, aggType, null) );
}
}
}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/Aggregation.java b/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/Aggregation.java
new file mode 100644
index 0000000..64bb3d7
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/Aggregation.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.pipeline.transforms.groupby;
+
+import org.apache.hop.core.injection.Injection;
+
+public class Aggregation implements Cloneable {
+
+ @Injection( name = "AGG_FIELD", group = "AGGREGATIONS" )
+ private String field;
+
+ @Injection( name = "AGG_SUBJECT", group = "AGGREGATIONS" )
+ private String subject;
+
+ private int type;
+
+ @Injection( name = "AGG_VALUE", group = "AGGREGATIONS" )
+ private String value;
+
+ public Aggregation() {
+ }
+
+ public Aggregation( String field, String subject, int type, String value ) {
+ this.field = field;
+ this.subject = subject;
+ this.type = type;
+ this.value = value;
+ }
+
+ @Override public Aggregation clone() {
+ return new Aggregation(field, subject, type, value);
+ }
+
+ @Injection( name = "AGG_TYPE", group = "AGGREGATIONS" )
+ public void setType(String typeCode) {
+ this.type = GroupByMeta.getType( typeCode );
+ }
+
+ /**
+ * Gets field
+ *
+ * @return value of field
+ */
+ public String getField() {
+ return field;
+ }
+
+ /**
+ * @param field The field to set
+ */
+ public void setField( String field ) {
+ this.field = field;
+ }
+
+ /**
+ * Gets subject
+ *
+ * @return value of subject
+ */
+ public String getSubject() {
+ return subject;
+ }
+
+ /**
+ * @param subject The subject to set
+ */
+ public void setSubject( String subject ) {
+ this.subject = subject;
+ }
+
+ /**
+ * Gets type
+ *
+ * @return value of type
+ */
+ public int getType() {
+ return type;
+ }
+
+ /**
+ * @param type The type to set
+ */
+ public void setType( int type ) {
+ this.type = type;
+ }
+
+ /**
+ * Gets value
+ *
+ * @return value of value
+ */
+ public String getValue() {
+ return value;
+ }
+
+ /**
+ * @param value The value to set
+ */
+ public void setValue( String value ) {
+ this.value = value;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java b/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java
index cb8fca2..7e07e02 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupBy.java
@@ -105,8 +105,8 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
// Do all the work we can beforehand
// Calculate indexes, loop up fields, etc.
//
- data.counts = new long[ meta.getSubjectField().length ];
- data.subjectnrs = new int[ meta.getSubjectField().length ];
+ data.counts = new long[ meta.getAggregations().size() ];
+ data.subjectnrs = new int[ meta.getAggregations().size() ];
data.cumulativeSumSourceIndexes = new ArrayList<>();
data.cumulativeSumTargetIndexes = new ArrayList<>();
@@ -114,28 +114,29 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
data.cumulativeAvgSourceIndexes = new ArrayList<>();
data.cumulativeAvgTargetIndexes = new ArrayList<>();
- for ( int i = 0; i < meta.getSubjectField().length; i++ ) {
- if ( meta.getAggregateType()[ i ] == GroupByMeta.TYPE_GROUP_COUNT_ANY ) {
+ for ( int i = 0; i < meta.getAggregations().size(); i++ ) {
+ Aggregation aggregation = meta.getAggregations().get( i );
+ if ( aggregation.getType() == GroupByMeta.TYPE_GROUP_COUNT_ANY ) {
data.subjectnrs[ i ] = 0;
} else {
- data.subjectnrs[ i ] = data.inputRowMeta.indexOfValue( meta.getSubjectField()[ i ] );
+ data.subjectnrs[ i ] = data.inputRowMeta.indexOfValue( aggregation.getSubject() );
}
if ( ( r != null ) && ( data.subjectnrs[ i ] < 0 ) ) {
logError( BaseMessages.getString( PKG, "GroupBy.Log.AggregateSubjectFieldCouldNotFound",
- meta.getSubjectField()[ i ] ) );
+ aggregation.getSubject() ) );
setErrors( 1 );
stopAll();
return false;
}
- if ( meta.getAggregateType()[ i ] == GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM ) {
+ if ( aggregation.getType() == GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM ) {
data.cumulativeSumSourceIndexes.add( data.subjectnrs[ i ] );
// The position of the target in the output row is the input row size + i
//
data.cumulativeSumTargetIndexes.add( data.inputRowMeta.size() + i );
}
- if ( meta.getAggregateType()[ i ] == GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE ) {
+ if ( aggregation.getType() == GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE ) {
data.cumulativeAvgSourceIndexes.add( data.subjectnrs[ i ] );
// The position of the target in the output row is the input row size + i
@@ -405,12 +406,14 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
*/
@SuppressWarnings( "unchecked" ) void calcAggregate( Object[] row ) throws HopValueException {
for ( int i = 0; i < data.subjectnrs.length; i++ ) {
+ Aggregation aggregation = meta.getAggregations().get( i );
+
Object subj = row[ data.subjectnrs[ i ] ];
IValueMeta subjMeta = data.inputRowMeta.getValueMeta( data.subjectnrs[ i ] );
Object value = data.agg[ i ];
IValueMeta valueMeta = data.aggMeta.getValueMeta( i );
- switch ( meta.getAggregateType()[ i ] ) {
+ switch ( aggregation.getType() ) {
case GroupByMeta.TYPE_GROUP_SUM:
data.agg[ i ] = ValueDataUtil.sum( valueMeta, value, subjMeta, subj );
break;
@@ -448,7 +451,7 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
case GroupByMeta.TYPE_GROUP_COUNT_DISTINCT:
if ( !subjMeta.isNull( subj ) ) {
if ( data.distinctObjs == null ) {
- data.distinctObjs = new Set[ meta.getSubjectField().length ];
+ data.distinctObjs = new Set[ meta.getAggregations().size() ];
}
if ( data.distinctObjs[ i ] == null ) {
data.distinctObjs[ i ] = new TreeSet<>();
@@ -536,8 +539,8 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
case GroupByMeta.TYPE_GROUP_CONCAT_STRING:
if ( !( subj == null ) ) {
String separator = "";
- if ( !Utils.isEmpty( meta.getValueField()[ i ] ) ) {
- separator = resolve( meta.getValueField()[ i ] );
+ if ( !Utils.isEmpty( aggregation.getValue() ) ) {
+ separator = resolve( aggregation.getValue() );
}
StringBuilder sb = (StringBuilder) value;
@@ -559,7 +562,7 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
*
* @param r
*/
- void newAggregate( Object[] r ) {
+ void newAggregate( Object[] r ) throws HopException {
// Put all the counters at 0
for ( int i = 0; i < data.counts.length; i++ ) {
data.counts[ i ] = 0;
@@ -570,10 +573,13 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
data.aggMeta = new RowMeta();
for ( int i = 0; i < data.subjectnrs.length; i++ ) {
+ Aggregation aggregation = meta.getAggregations().get( i );
IValueMeta subjMeta = data.inputRowMeta.getValueMeta( data.subjectnrs[ i ] );
Object v = null;
IValueMeta vMeta = null;
- int aggType = meta.getAggregateType()[ i ];
+ int aggType = aggregation.getType();
+ String fieldName = aggregation.getField();
+
switch ( aggType ) {
case GroupByMeta.TYPE_GROUP_SUM:
case GroupByMeta.TYPE_GROUP_AVERAGE:
@@ -581,28 +587,28 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
case GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE:
if ( subjMeta.isNumeric() ) {
try {
- vMeta = ValueMetaFactory.createValueMeta( meta.getAggregateField()[ i ], subjMeta.getType() );
+ vMeta = ValueMetaFactory.createValueMeta( fieldName, subjMeta.getType() );
} catch ( HopPluginException e ) {
- vMeta = new ValueMetaNone( meta.getAggregateField()[ i ] );
+ vMeta = new ValueMetaNone( fieldName );
}
} else {
- vMeta = new ValueMetaNumber( meta.getAggregateField()[ i ] );
+ vMeta = new ValueMetaNumber( fieldName );
}
break;
case GroupByMeta.TYPE_GROUP_MEDIAN:
case GroupByMeta.TYPE_GROUP_PERCENTILE:
case GroupByMeta.TYPE_GROUP_PERCENTILE_NEAREST_RANK:
- vMeta = new ValueMetaNumber( meta.getAggregateField()[ i ] );
+ vMeta = new ValueMetaNumber( fieldName );
v = new ArrayList<Double>();
break;
case GroupByMeta.TYPE_GROUP_STANDARD_DEVIATION:
case GroupByMeta.TYPE_GROUP_STANDARD_DEVIATION_SAMPLE:
- vMeta = new ValueMetaNumber( meta.getAggregateField()[ i ] );
+ vMeta = new ValueMetaNumber( fieldName );
break;
case GroupByMeta.TYPE_GROUP_COUNT_DISTINCT:
case GroupByMeta.TYPE_GROUP_COUNT_ANY:
case GroupByMeta.TYPE_GROUP_COUNT_ALL:
- vMeta = new ValueMetaInteger( meta.getAggregateField()[ i ] );
+ vMeta = new ValueMetaInteger( fieldName );
break;
case GroupByMeta.TYPE_GROUP_FIRST:
case GroupByMeta.TYPE_GROUP_LAST:
@@ -611,20 +617,20 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
case GroupByMeta.TYPE_GROUP_MIN:
case GroupByMeta.TYPE_GROUP_MAX:
vMeta = subjMeta.clone();
- vMeta.setName( meta.getAggregateField()[ i ] );
+ vMeta.setName( fieldName );
v = r == null ? null : r[ data.subjectnrs[ i ] ];
break;
case GroupByMeta.TYPE_GROUP_CONCAT_COMMA:
- vMeta = new ValueMetaString( meta.getAggregateField()[ i ] );
+ vMeta = new ValueMetaString( fieldName );
v = new StringBuilder();
break;
case GroupByMeta.TYPE_GROUP_CONCAT_STRING:
- vMeta = new ValueMetaString( meta.getAggregateField()[ i ] );
+ vMeta = new ValueMetaString( fieldName );
v = new StringBuilder();
break;
default:
// TODO raise an error here because we cannot continue successfully maybe the UI should validate this
- break;
+ throw new HopException("Please specify an aggregation type for field '"+fieldName+"'");
}
if ( ( subjMeta != null )
@@ -686,8 +692,11 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
Object[] result = new Object[ data.subjectnrs.length ];
for ( int i = 0; i < data.subjectnrs.length; i++ ) {
+ Aggregation aggregation = meta.getAggregations().get(i);
Object ag = data.agg[ i ];
- switch ( meta.getAggregateType()[ i ] ) {
+ int aggType = aggregation.getType();
+ String fieldName = aggregation.getField();
+ switch ( aggType ) {
case GroupByMeta.TYPE_GROUP_SUM:
break;
case GroupByMeta.TYPE_GROUP_AVERAGE:
@@ -698,8 +707,8 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
case GroupByMeta.TYPE_GROUP_MEDIAN:
case GroupByMeta.TYPE_GROUP_PERCENTILE:
double percentile = 50.0;
- if ( meta.getAggregateType()[ i ] == GroupByMeta.TYPE_GROUP_PERCENTILE ) {
- percentile = Double.parseDouble( meta.getValueField()[ i ] );
+ if ( aggType == GroupByMeta.TYPE_GROUP_PERCENTILE ) {
+ percentile = Double.parseDouble( aggregation.getValue() );
}
@SuppressWarnings( "unchecked" )
List<Double> valuesList = (List<Double>) data.agg[ i ];
@@ -711,8 +720,8 @@ public class GroupBy extends BaseTransform<GroupByMeta, GroupByData> implements
break;
case GroupByMeta.TYPE_GROUP_PERCENTILE_NEAREST_RANK:
double percentileValue = 50.0;
- if ( meta.getAggregateType()[ i ] == GroupByMeta.TYPE_GROUP_PERCENTILE_NEAREST_RANK ) {
- percentileValue = Double.parseDouble( meta.getValueField()[ i ] );
+ if ( aggType == GroupByMeta.TYPE_GROUP_PERCENTILE_NEAREST_RANK ) {
+ percentileValue = Double.parseDouble( aggregation.getValue() );
}
@SuppressWarnings( "unchecked" )
List<Double> latenciesList = (List<Double>) data.agg[ i ];
diff --git a/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupByMeta.java b/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupByMeta.java
index 43fd317..e41cae3 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupByMeta.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/transforms/groupby/GroupByMeta.java
@@ -23,6 +23,9 @@ import org.apache.hop.core.ICheckResult;
import org.apache.hop.core.annotations.Transform;
import org.apache.hop.core.exception.HopPluginException;
import org.apache.hop.core.exception.HopXmlException;
+import org.apache.hop.core.injection.Injection;
+import org.apache.hop.core.injection.InjectionDeep;
+import org.apache.hop.core.injection.InjectionSupported;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
@@ -42,6 +45,7 @@ import org.apache.hop.pipeline.transform.ITransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.w3c.dom.Node;
+import java.util.ArrayList;
import java.util.List;
/** Created on 02-jun-2003 */
@@ -53,6 +57,8 @@ import java.util.List;
categoryDescription =
"i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Statistics",
keywords = "")
+@InjectionSupported( localizationPrefix = "GroupByMeta.Injection.",
+groups = {"GROUPS", "AGGREGATIONS"})
public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<GroupBy, GroupByData> {
private static final Class<?> PKG = GroupByMeta.class; // For Translator
@@ -149,12 +155,15 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
};
/** All rows need to pass, adding an extra row at the end of each group/block. */
+ @Injection( name = "PASS_ALL_ROWS" )
private boolean passAllRows;
/** Directory to store the temp files */
+ @Injection( name = "TEMP_DIRECTORY" )
private String directory;
/** Temp files prefix... */
+ @Injection( name = "TEMP_FILE_PREFIX" )
private String prefix;
/** Indicate that some rows don't need to be considered : TODO: make work in GUI & worker */
@@ -167,42 +176,28 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
private String aggregateIgnoredField;
/** Fields to group over */
+ @Injection( name = "GROUP_FIELD", group = "GROUPS" )
private String[] groupField;
- /** Name of aggregate field */
- private String[] aggregateField;
-
- /** Field name to group over */
- private String[] subjectField;
-
- /** Type of aggregate */
- private int[] aggregateType;
-
- /** Value to use as separator for ex */
- private String[] valueField;
+ @InjectionDeep private List<Aggregation> aggregations;
/** Add a linenr in the group, resetting to 0 in a new group. */
+ @Injection( name = "ADD_GROUP_LINENR" )
private boolean addingLineNrInGroup;
/** The fieldname that will contain the added integer field */
+ @Injection( name = "ADD_GROUP_LINENR_FIELD" )
private String lineNrInGroupField;
/** Flag to indicate that we always give back one row. Defaults to true for existing pipelines. */
+ @Injection( name = "ALWAYS_GIVE_ROW" )
private boolean alwaysGivingBackOneRow;
public GroupByMeta() {
super(); // allocate BaseTransformMeta
}
- /** @return Returns the aggregateField. */
- public String[] getAggregateField() {
- return aggregateField;
- }
- /** @param aggregateField The aggregateField to set. */
- public void setAggregateField(String[] aggregateField) {
- this.aggregateField = aggregateField;
- }
/** @return Returns the aggregateIgnored. */
public boolean isAggregateIgnored() {
@@ -224,16 +219,6 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
this.aggregateIgnoredField = aggregateIgnoredField;
}
- /** @return Returns the aggregateType. */
- public int[] getAggregateType() {
- return aggregateType;
- }
-
- /** @param aggregateType The aggregateType to set. */
- public void setAggregateType(int[] aggregateType) {
- this.aggregateType = aggregateType;
- }
-
/** @return Returns the groupField. */
public String[] getGroupField() {
return groupField;
@@ -254,60 +239,35 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
this.passAllRows = passAllRows;
}
- /** @return Returns the subjectField. */
- public String[] getSubjectField() {
- return subjectField;
- }
-
- /** @param subjectField The subjectField to set. */
- public void setSubjectField(String[] subjectField) {
- this.subjectField = subjectField;
- }
-
- /** @return Returns the valueField. */
- public String[] getValueField() {
- return valueField;
- }
-
- /** @param valueField The valueField to set. */
- public void setValueField(String[] valueField) {
- this.valueField = valueField;
- }
-
@Override
public void loadXml(Node transformNode, IHopMetadataProvider metadataProvider)
throws HopXmlException {
readData(transformNode);
}
- public void allocate(int sizegroup, int nrFields) {
- groupField = new String[sizegroup];
- aggregateField = new String[nrFields];
- subjectField = new String[nrFields];
- aggregateType = new int[nrFields];
- valueField = new String[nrFields];
+ public void allocate(int groupSize) {
+ groupField = new String[groupSize];
+ aggregations = new ArrayList<>();
}
@Override
public Object clone() {
- GroupByMeta retval = (GroupByMeta) super.clone();
+ GroupByMeta groupByMeta = (GroupByMeta) super.clone();
- int szGroup = 0, szFields = 0;
+ int szGroup = 0;
if (groupField != null) {
szGroup = groupField.length;
}
- if (valueField != null) {
- szFields = valueField.length;
- }
- retval.allocate(szGroup, szFields);
+ groupByMeta.allocate(szGroup);
+ System.arraycopy(groupField, 0, groupByMeta.groupField, 0, szGroup);
- System.arraycopy(groupField, 0, retval.groupField, 0, szGroup);
- System.arraycopy(aggregateField, 0, retval.aggregateField, 0, szFields);
- System.arraycopy(subjectField, 0, retval.subjectField, 0, szFields);
- System.arraycopy(aggregateType, 0, retval.aggregateType, 0, szFields);
- System.arraycopy(valueField, 0, retval.valueField, 0, szFields);
+ List<Aggregation> aggsCopy = new ArrayList<>();
+ for (Aggregation aggregation : aggregations) {
+ aggsCopy.add(aggregation.clone());
+ }
+ groupByMeta.setAggregations( aggsCopy );
- return retval;
+ return groupByMeta;
}
private void readData(Node transformNode) throws HopXmlException {
@@ -324,33 +284,34 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
"Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "add_linenr"));
lineNrInGroupField = XmlHandler.getTagValue(transformNode, "linenr_fieldname");
- Node groupn = XmlHandler.getSubNode(transformNode, "group");
- Node fields = XmlHandler.getSubNode(transformNode, "fields");
+ Node groupNode = XmlHandler.getSubNode(transformNode, "group");
+ Node fieldsNode = XmlHandler.getSubNode(transformNode, "fields");
- int sizegroup = XmlHandler.countNodes(groupn, "field");
- int nrFields = XmlHandler.countNodes(fields, "field");
+ int nrGroups = XmlHandler.countNodes(groupNode, "field");
+ int nrFields = XmlHandler.countNodes(fieldsNode, "field");
- allocate(sizegroup, nrFields);
+ allocate(nrGroups);
- for (int i = 0; i < sizegroup; i++) {
- Node fnode = XmlHandler.getSubNodeByNr(groupn, "field", i);
+ for (int i = 0; i < nrGroups; i++) {
+ Node fnode = XmlHandler.getSubNodeByNr(groupNode, "field", i);
groupField[i] = XmlHandler.getTagValue(fnode, "name");
}
+ aggregations = new ArrayList<>();
boolean hasNumberOfValues = false;
for (int i = 0; i < nrFields; i++) {
- Node fnode = XmlHandler.getSubNodeByNr(fields, "field", i);
- aggregateField[i] = XmlHandler.getTagValue(fnode, "aggregate");
- subjectField[i] = XmlHandler.getTagValue(fnode, "subject");
- aggregateType[i] = getType(XmlHandler.getTagValue(fnode, "type"));
-
- if (aggregateType[i] == TYPE_GROUP_COUNT_ALL
- || aggregateType[i] == TYPE_GROUP_COUNT_DISTINCT
- || aggregateType[i] == TYPE_GROUP_COUNT_ANY) {
+ Node fnode = XmlHandler.getSubNodeByNr(fieldsNode, "field", i);
+ String aggField = XmlHandler.getTagValue(fnode, "aggregate");
+ String aggSubject = XmlHandler.getTagValue(fnode, "subject");
+ int aggType = getType(XmlHandler.getTagValue(fnode, "type"));
+
+ if (aggType == TYPE_GROUP_COUNT_ALL
+ || aggType == TYPE_GROUP_COUNT_DISTINCT
+ || aggType == TYPE_GROUP_COUNT_ANY) {
hasNumberOfValues = true;
}
-
- valueField[i] = XmlHandler.getTagValue(fnode, "valuefield");
+ String aggValue = XmlHandler.getTagValue(fnode, "valuefield");
+ aggregations.add(new Aggregation(aggField, aggSubject, aggType, aggValue));
}
String giveBackRow = XmlHandler.getTagValue(transformNode, "give_back_row");
@@ -403,9 +364,8 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
aggregateIgnoredField = null;
int sizeGroup = 0;
- int numberOfFields = 0;
- allocate(sizeGroup, numberOfFields);
+ allocate(sizeGroup);
}
@Override
@@ -437,15 +397,16 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
// Re-add aggregates
//
- for (int i = 0; i < subjectField.length; i++) {
- IValueMeta subj = rowMeta.searchValueMeta(subjectField[i]);
- if (subj != null || aggregateType[i] == TYPE_GROUP_COUNT_ANY) {
- String valueName = aggregateField[i];
+ for (Aggregation aggregation : aggregations) {
+ int aggregationType = aggregation.getType();
+ IValueMeta subj = rowMeta.searchValueMeta(aggregation.getSubject());
+ if (subj != null || aggregationType == TYPE_GROUP_COUNT_ANY) {
+ String valueName = aggregation.getField();
int valueType = IValueMeta.TYPE_NONE;
int length = -1;
int precision = -1;
- switch (aggregateType[i]) {
+ switch ( aggregationType ) {
case TYPE_GROUP_SUM:
case TYPE_GROUP_AVERAGE:
case TYPE_GROUP_CUMULATIVE_SUM:
@@ -482,17 +443,17 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
// Change type from integer to number in case off averages for cumulative average
//
- if (aggregateType[i] == TYPE_GROUP_CUMULATIVE_AVERAGE
+ if ( aggregationType == TYPE_GROUP_CUMULATIVE_AVERAGE
&& valueType == IValueMeta.TYPE_INTEGER) {
valueType = IValueMeta.TYPE_NUMBER;
precision = -1;
length = -1;
- } else if (aggregateType[i] == TYPE_GROUP_COUNT_ALL
- || aggregateType[i] == TYPE_GROUP_COUNT_DISTINCT
- || aggregateType[i] == TYPE_GROUP_COUNT_ANY) {
+ } else if ( aggregationType == TYPE_GROUP_COUNT_ALL
+ || aggregationType == TYPE_GROUP_COUNT_DISTINCT
+ || aggregationType == TYPE_GROUP_COUNT_ANY) {
length = IValueMeta.DEFAULT_INTEGER_LENGTH;
precision = 0;
- } else if (aggregateType[i] == TYPE_GROUP_SUM
+ } else if ( aggregationType == TYPE_GROUP_SUM
&& valueType != IValueMeta.TYPE_INTEGER
&& valueType != IValueMeta.TYPE_NUMBER
&& valueType != IValueMeta.TYPE_BIGNUMBER) {
@@ -561,14 +522,14 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
retval.append(" </group>").append(Const.CR);
retval.append(" <fields>").append(Const.CR);
- for (int i = 0; i < subjectField.length; i++) {
+ for (Aggregation aggregation : aggregations) {
retval.append(" <field>").append(Const.CR);
- retval.append(" ").append(XmlHandler.addTagValue("aggregate", aggregateField[i]));
- retval.append(" ").append(XmlHandler.addTagValue("subject", subjectField[i]));
+ retval.append(" ").append(XmlHandler.addTagValue("aggregate", aggregation.getField()));
+ retval.append(" ").append(XmlHandler.addTagValue("subject", aggregation.getSubject()));
retval
.append(" ")
- .append(XmlHandler.addTagValue("type", getTypeDesc(aggregateType[i])));
- retval.append(" ").append(XmlHandler.addTagValue("valuefield", valueField[i]));
+ .append(XmlHandler.addTagValue("type", getTypeDesc(aggregation.getType())));
+ retval.append(" ").append(XmlHandler.addTagValue("valuefield", aggregation.getValue()));
retval.append(" </field>").append(Const.CR);
}
retval.append(" </fields>").append(Const.CR);
@@ -675,4 +636,20 @@ public class GroupByMeta extends BaseTransformMeta implements ITransformMeta<Gro
public PipelineMeta.PipelineType[] getSupportedPipelineTypes() {
return new PipelineMeta.PipelineType[] {PipelineMeta.PipelineType.Normal};
}
+
+ /**
+ * Gets aggregations
+ *
+ * @return value of aggregations
+ */
+ public List<Aggregation> getAggregations() {
+ return aggregations;
+ }
+
+ /**
+ * @param aggregations The aggregations to set
+ */
+ public void setAggregations( List<Aggregation> aggregations ) {
+ this.aggregations = aggregations;
+ }
}
diff --git a/engine/src/main/java/org/apache/hop/run/HopRun.java b/engine/src/main/java/org/apache/hop/run/HopRun.java
index 57d166f..fa59076 100644
--- a/engine/src/main/java/org/apache/hop/run/HopRun.java
+++ b/engine/src/main/java/org/apache/hop/run/HopRun.java
@@ -236,6 +236,8 @@ public class HopRun implements Runnable, IHasHopMetadataProvider {
pipeline.activateParameters(pipeline);
+ log.logMinimal( "Starting pipeline: "+pipelineMeta.getFilename() );
+
// Run it!
//
pipeline.prepareExecution();
@@ -316,6 +318,8 @@ public class HopRun implements Runnable, IHasHopMetadataProvider {
//
workflow.activateParameters(workflow);
+ log.logMinimal( "Starting workflow: "+workflowMeta.getFilename() );
+
workflow.startExecution();
setFinishedWithoutError(workflow.getResult().getResult());
} catch (Exception e) {
diff --git a/engine/src/main/java/org/apache/hop/server/HopServer.java b/engine/src/main/java/org/apache/hop/server/HopServer.java
index 8239e34..88f5572 100644
--- a/engine/src/main/java/org/apache/hop/server/HopServer.java
+++ b/engine/src/main/java/org/apache/hop/server/HopServer.java
@@ -1004,7 +1004,7 @@ public class HopServer extends HopMetadataBase implements Cloneable, IXml, IHopM
}
}
- log.logMinimal( pipelineName, "The remote pipeline has finished." );
+ log.logBasic( pipelineName, "The remote pipeline has finished." );
}
@@ -1073,7 +1073,7 @@ public class HopServer extends HopMetadataBase implements Cloneable, IXml, IHopM
}
}
- log.logMinimal( workflowName, "The remote workflow has finished." );
+ log.logBasic( workflowName, "The remote workflow has finished." );
}
diff --git a/engine/src/main/java/org/apache/hop/workflow/Workflow.java b/engine/src/main/java/org/apache/hop/workflow/Workflow.java
index e0a8dd4..1b4ab30 100644
--- a/engine/src/main/java/org/apache/hop/workflow/Workflow.java
+++ b/engine/src/main/java/org/apache/hop/workflow/Workflow.java
@@ -363,7 +363,7 @@ public abstract class Workflow extends Variables implements IVariables, INamedPa
setStopped( false );
HopEnvironment.setExecutionInformation( this );
- log.logMinimal( BaseMessages.getString( PKG, "Workflow.Comment.WorkflowStarted" ) );
+ log.logBasic( BaseMessages.getString( PKG, "Workflow.Comment.WorkflowStarted" ) );
ExtensionPointHandler.callExtensionPoint( log, this, HopExtensionPoint.WorkflowStart.id, this );
@@ -424,7 +424,7 @@ public abstract class Workflow extends Variables implements IVariables, INamedPa
}
// Save this result...
workflowTracker.addWorkflowTracker( new WorkflowTracker( workflowMeta, jerEnd ) );
- log.logMinimal( BaseMessages.getString( PKG, "Workflow.Comment.WorkflowFinished" ) );
+ log.logBasic( BaseMessages.getString( PKG, "Workflow.Comment.WorkflowFinished" ) );
setActive( false );
if ( !isStopped() ) {
@@ -562,7 +562,8 @@ public abstract class Workflow extends Variables implements IVariables, INamedPa
Thread.currentThread().setContextClassLoader( action.getClass().getClassLoader() );
// Execute this entry...
IAction cloneJei = (IAction) action.clone();
- ( (IVariables) cloneJei ).copyFrom( this );
+ cloneJei.copyFrom( this );
+ cloneJei.getLogChannel().setLogLevel( getLogLevel() );
cloneJei.setMetadataProvider( metadataProvider );
cloneJei.setParentWorkflow( this );
cloneJei.setParentWorkflowMeta( this.getWorkflowMeta() );
@@ -594,12 +595,6 @@ public abstract class Workflow extends Variables implements IVariables, INamedPa
}
}
- if ( cloneJei instanceof ActionPipeline ) {
- String throughput = newResult.getReadWriteThroughput( (int) ( ( end - start ) / 1000 ) );
- if ( throughput != null ) {
- log.logMinimal( throughput );
- }
- }
for ( IActionListener actionListener : actionListeners ) {
actionListener.afterExecution( this, actionMeta, cloneJei, newResult );
}
@@ -1201,6 +1196,7 @@ public abstract class Workflow extends Variables implements IVariables, INamedPa
*/
public void setLogLevel( LogLevel logLevel ) {
this.logLevel = logLevel;
+ log.setLogLevel( logLevel );
}
/**
diff --git a/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java b/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java
index c84534e..323e6c5 100644
--- a/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java
+++ b/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java
@@ -136,7 +136,7 @@ public class HopServerSingleton {
// pipelineMap.deallocateServerSocketPorts(entry);
- log.logMinimal( "Cleaned up pipeline "
+ log.logBasic( "Cleaned up pipeline "
+ entry.getName() + " with id " + entry.getId() + " from " + pipeline.getExecutionStartDate()
+ ", diff=" + diffInMinutes );
}
@@ -163,7 +163,7 @@ public class HopServerSingleton {
workflowMap.removeWorkflow( entry );
- log.logMinimal( "Cleaned up workflow "
+ log.logBasic( "Cleaned up workflow "
+ entry.getName() + " with id " + entry.getId() + " from " + workflow.getExecutionStartDate() );
}
}
diff --git a/engine/src/main/resources/org/apache/hop/pipeline/transforms/groupby/messages/messages_en_US.properties b/engine/src/main/resources/org/apache/hop/pipeline/transforms/groupby/messages/messages_en_US.properties
index 2db9b91..c15cd02 100644
--- a/engine/src/main/resources/org/apache/hop/pipeline/transforms/groupby/messages/messages_en_US.properties
+++ b/engine/src/main/resources/org/apache/hop/pipeline/transforms/groupby/messages/messages_en_US.properties
@@ -78,3 +78,15 @@ GroupByDialog.GroupByWarningDialog.DialogMessage=If the incoming data is not sor
GroupByMeta.TypeGroupLongDesc.MEDIAN=Median
GroupByMeta.TypeGroupLongDesc.COUNT_ANY=Number of rows (without field argument)
GroupByMeta.TypeGroupLongDesc.CONCAT_STRING=Concatenate strings separated by
+
+GroupByMeta.Injection.PASS_ALL_ROWS = Pass all rows?
+GroupByMeta.Injection.TEMP_DIRECTORY=Temporary directory
+GroupByMeta.Injection.TEMP_FILE_PREFIX=Temporary file prefix
+GroupByMeta.Injection.GROUP_FIELD=Group field
+GroupByMeta.Injection.AGG_FIELD=Aggregation field
+GroupByMeta.Injection.AGG_SUBJECT=Aggregation subject
+GroupByMeta.Injection.AGG_TYPE=Aggregation type
+GroupByMeta.Injection.AGG_VALUE=Aggregation value
+GroupByMeta.Injection.ADD_GROUP_LINENR=Add group line number?
+GroupByMeta.Injection.ADD_GROUP_LINENR_FIELD=Group line number field
+GroupByMeta.Injection.ALWAYS_GIVE_ROW=Always give back row?
diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopRunCalculateFilenameExtensionPoint.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopRunCalculateFilenameExtensionPoint.java
index 09a9c71..7763d16 100644
--- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopRunCalculateFilenameExtensionPoint.java
+++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopRunCalculateFilenameExtensionPoint.java
@@ -44,7 +44,7 @@ public class HopRunCalculateFilenameExtensionPoint implements IExtensionPoint<Ho
fileObject = HopVfs.getFileObject( alternativeFilename );
if ( fileObject.exists() ) {
hopRun.setRealFilename(alternativeFilename);
- log.logMinimal( "Relative path filename specified: " + hopRun.getRealFilename() );
+ log.logBasic( "Relative path filename specified: " + hopRun.getRealFilename() );
}
}
} catch ( Exception e ) {
diff --git a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
index dd83d1f..cd611df 100644
--- a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
+++ b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
@@ -147,6 +147,8 @@ public class MetaInject extends BaseTransform<MetaInjectMeta, MetaInjectData> im
getPipeline().addExecutionStoppedListener(e -> injectPipeline.stopAll());
+ injectPipeline.setLogLevel( getLogLevel() );
+
// Parameters get activated below so we need to make sure they have values
//
injectPipeline.prepareExecution( );
diff --git a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java
index 8e70812..ab3be8a 100644
--- a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java
+++ b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java
@@ -492,7 +492,7 @@ public class TableOutput extends BaseTransform<TableOutputMeta, TableOutputData>
// incorrectly processed rows.
//
if ( getTransformMeta().isDoingErrorHandling() && !dbInterface.supportsErrorHandlingOnBatchUpdates() ) {
- log.logMinimal( BaseMessages.getString(
+ log.logBasic( BaseMessages.getString(
PKG, "TableOutput.Warning.ErrorHandlingIsNotFullySupportedWithBatchProcessing" ) );
}
diff --git a/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties b/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties
index 34b30ad..ca63296 100644
--- a/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties
+++ b/plugins/transforms/textfile/src/main/resources/org/apache/hop/pipeline/transforms/csvinput/messages/messages_en_US.properties
@@ -110,4 +110,7 @@ CsvInputMeta.Injection.INPUT_IF_NULL=Field value if null
CsvInputMeta.Injection.ADD_RESULT=Add files to result?
CsvInputMeta.Injection.RUNNING_IN_PARALLEL=Running in parallel?
CsvInputMeta.Injection.FILE_ENCODING=File encoding
-CsvInputMeta.Injection.NEWLINES_IN_FIELDS=Newlines possible in fields?
\ No newline at end of file
+CsvInputMeta.Injection.NEWLINES_IN_FIELDS=Newlines possible in fields?
+CsvInputMeta.Injection.FIELD_TYPE=Field type
+CsvInputMeta.Injection.FIELD_TRIM_TYPE=Field trim type
+
diff --git a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java
index d2bd526..5179826 100644
--- a/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java
+++ b/ui/src/main/java/org/apache/hop/ui/core/widget/ComboVar.java
@@ -208,7 +208,11 @@ public class ComboVar extends Composite {
}
public boolean setFocus() {
- return wCombo.setFocus();
+ if (wCombo != null && !wCombo.isDisposed()) {
+ return wCombo.setFocus();
+ } else {
+ return false;
+ }
}
public void addTraverseListener(TraverseListener tl) {
diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
index 6a656a9..3b79b38 100644
--- a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
+++ b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
@@ -4070,7 +4070,7 @@ public class HopGuiPipelineGraph extends HopGuiAbstractGraph
e);
}
if (pipeline != null) {
- log.logMinimal(
+ log.logBasic(
BaseMessages.getString(PKG, "PipelineLog.Log.LaunchingPipeline")
+ pipeline.getPipelineMeta().getName()
+ "]...");
@@ -4088,7 +4088,7 @@ public class HopGuiPipelineGraph extends HopGuiAbstractGraph
preparePipeline(parentThread);
});
- log.logMinimal(BaseMessages.getString(PKG, "PipelineLog.Log.StartedExecutionOfPipeline"));
+ log.logBasic(BaseMessages.getString(PKG, "PipelineLog.Log.StartedExecutionOfPipeline"));
updateGui();
@@ -4416,7 +4416,7 @@ public class HopGuiPipelineGraph extends HopGuiAbstractGraph
if ((isRunning() && !halting)) {
halting = true;
pipeline.stopAll();
- log.logMinimal(BaseMessages.getString(PKG, "PipelineLog.Log.ProcessingOfPipelineStopped"));
+ log.logBasic(BaseMessages.getString(PKG, "PipelineLog.Log.ProcessingOfPipelineStopped"));
halted = false;
halting = false;
@@ -4536,7 +4536,7 @@ public class HopGuiPipelineGraph extends HopGuiAbstractGraph
private void checkPipelineEnded() {
if (pipeline != null) {
if (pipeline.isFinished() && (isRunning() || halted)) {
- log.logMinimal(BaseMessages.getString(PKG, "PipelineLog.Log.PipelineHasFinished"));
+ log.logBasic(BaseMessages.getString(PKG, "PipelineLog.Log.PipelineHasFinished"));
initialized = false;
halted = false;
diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java
index bbdce04..199d80e 100644
--- a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java
+++ b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java
@@ -1427,7 +1427,7 @@ public class HopGuiWorkflowGraph extends HopGuiAbstractGraph
if ((isRunning() && !halting)) {
halting = true;
workflow.stopExecution();
- log.logMinimal(BaseMessages.getString(PKG, "WorkflowLog.Log.ProcessingOfWorkflowStopped"));
+ log.logBasic(BaseMessages.getString(PKG, "WorkflowLog.Log.ProcessingOfWorkflowStopped"));
halting = false;
@@ -3528,7 +3528,7 @@ public class HopGuiWorkflowGraph extends HopGuiAbstractGraph
return;
}
- log.logMinimal(BaseMessages.getString(PKG, "WorkflowLog.Log.StartingWorkflow"));
+ log.logBasic(BaseMessages.getString(PKG, "WorkflowLog.Log.StartingWorkflow"));
workflowThread = new Thread(() -> workflow.startExecution());
workflowThread.start();
workflowGridDelegate.previousNrItems = -1;
@@ -3594,7 +3594,7 @@ public class HopGuiWorkflowGraph extends HopGuiAbstractGraph
// Do a final check to see if it all ended...
//
if (workflow != null && workflow.isInitialized() && workflow.isFinished()) {
- log.logMinimal(BaseMessages.getString(PKG, "WorkflowLog.Log.WorkflowHasEnded"));
+ log.logBasic(BaseMessages.getString(PKG, "WorkflowLog.Log.WorkflowHasEnded"));
}
updateGui();
}
diff --git a/ui/src/main/java/org/apache/hop/ui/pipeline/transforms/groupby/GroupByDialog.java b/ui/src/main/java/org/apache/hop/ui/pipeline/transforms/groupby/GroupByDialog.java
index 8637cb3..1d46f69 100644
--- a/ui/src/main/java/org/apache/hop/ui/pipeline/transforms/groupby/GroupByDialog.java
+++ b/ui/src/main/java/org/apache/hop/ui/pipeline/transforms/groupby/GroupByDialog.java
@@ -28,10 +28,10 @@ import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.ITransformDialog;
import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.groupby.Aggregation;
import org.apache.hop.pipeline.transforms.groupby.GroupByMeta;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.dialog.MessageDialogWithToggle;
-import org.apache.hop.ui.core.gui.GuiResource;
import org.apache.hop.ui.core.widget.ColumnInfo;
import org.apache.hop.ui.core.widget.TableView;
import org.apache.hop.ui.core.widget.TextVar;
@@ -378,10 +378,10 @@ public class GroupByDialog extends BaseTransformDialog implements ITransformDial
fdlAgg.top = new FormAttachment(wGroup, margin);
wlAgg.setLayoutData(fdlAgg);
- int UpInsCols = 4;
- int UpInsRows = (input.getAggregateField() != null ? input.getAggregateField().length : 1);
+ int nrCols = 4;
+ int nrRows = input.getAggregations().size();
- ciReturn = new ColumnInfo[UpInsCols];
+ ciReturn = new ColumnInfo[nrCols];
ciReturn[0] =
new ColumnInfo(
BaseMessages.getString(PKG, "GroupByDialog.ColumnInfo.Name"),
@@ -412,7 +412,7 @@ public class GroupByDialog extends BaseTransformDialog implements ITransformDial
shell,
SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL,
ciReturn,
- UpInsRows,
+ nrRows,
lsMod,
props);
@@ -568,20 +568,13 @@ public class GroupByDialog extends BaseTransformDialog implements ITransformDial
}
}
- if (input.getAggregateField() != null) {
- for (int i = 0; i < input.getAggregateField().length; i++) {
- TableItem item = wAgg.table.getItem(i);
- if (input.getAggregateField()[i] != null) {
- item.setText(1, input.getAggregateField()[i]);
- }
- if (input.getSubjectField()[i] != null) {
- item.setText(2, input.getSubjectField()[i]);
- }
- item.setText(3, GroupByMeta.getTypeDescLong(input.getAggregateType()[i]));
- if (input.getValueField()[i] != null) {
- item.setText(4, input.getValueField()[i]);
- }
- }
+ int i = 0;
+ for (Aggregation aggregation : input.getAggregations()) {
+ TableItem item = wAgg.table.getItem(i++);
+ item.setText(1, Const.NVL(aggregation.getField(), ""));
+ item.setText(2, Const.NVL(aggregation.getSubject(), ""));
+ item.setText(3, GroupByMeta.getTypeDescLong(aggregation.getType()));
+ item.setText(3, Const.NVL(aggregation.getValue(), ""));
}
wGroup.setRowNums();
@@ -617,7 +610,7 @@ public class GroupByDialog extends BaseTransformDialog implements ITransformDial
input.setAlwaysGivingBackOneRow(wAlwaysAddResult.getSelection());
input.setPassAllRows(wAllRows.getSelection());
- input.allocate(sizegroup, nrFields);
+ input.allocate(sizegroup);
// CHECKSTYLE:Indentation:OFF
for (int i = 0; i < sizegroup; i++) {
@@ -628,10 +621,11 @@ public class GroupByDialog extends BaseTransformDialog implements ITransformDial
// CHECKSTYLE:Indentation:OFF
for (int i = 0; i < nrFields; i++) {
TableItem item = wAgg.getNonEmpty(i);
- input.getAggregateField()[i] = item.getText(1);
- input.getSubjectField()[i] = item.getText(2);
- input.getAggregateType()[i] = GroupByMeta.getType(item.getText(3));
- input.getValueField()[i] = item.getText(4);
+ String aggField = item.getText(1);
+ String aggSubject = item.getText(2);
+ int aggType = GroupByMeta.getType(item.getText(3));
+ String aggValue = item.getText(4);
+ input.getAggregations().add(new Aggregation(aggField, aggSubject, aggType, aggValue));
}
transformName = wTransformName.getText();