You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by am...@apache.org on 2017/07/04 14:48:33 UTC
[1/2] incubator-unomi git commit: UNOMI-101 : Add hasHeader,
hasDeleteColumn flags in importConfig
Repository: incubator-unomi
Updated Branches:
refs/heads/master 061ee51e0 -> d2a480cf5
UNOMI-101 : Add hasHeader, hasDeleteColumn flags in importConfig
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/91e23a29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/91e23a29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/91e23a29
Branch: refs/heads/master
Commit: 91e23a29e8d769cd10deb13f5983cd0a34a161e5
Parents: 061ee51
Author: Abdelkader Midani <am...@apache.org>
Authored: Tue Jul 4 16:47:38 2017 +0200
Committer: Abdelkader Midani <am...@apache.org>
Committed: Tue Jul 4 16:47:38 2017 +0200
----------------------------------------------------------------------
.../unomi/router/api/ImportConfiguration.java | 24 ++++++++++++++++++++
.../unomi/router/api/RouterConstants.java | 3 ++-
.../ImportRouteCompletionProcessor.java | 8 +++++--
.../core/processor/LineSplitProcessor.java | 23 +++++++++++++++----
.../ProfileImportFromSourceRouteBuilder.java | 2 ++
5 files changed, 53 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java
index b961cfb..a6850ef 100644
--- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java
+++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/ImportConfiguration.java
@@ -36,6 +36,8 @@ public class ImportConfiguration extends ImportExportConfiguration {
private String mergingProperty;
private boolean overwriteExistingProfiles = false;
private List<String> propertiesToOverwrite;
+ private boolean hasHeader = false;
+ private boolean hasDeleteColumn = false;
public String getMergingProperty() {
@@ -68,6 +70,11 @@ public class ImportConfiguration extends ImportExportConfiguration {
this.overwriteExistingProfiles = overwriteExistingProfiles;
}
+ /**
+ * Retrieves the import configuration propertiesToOverwrite field.
+ *
+ * @return propertiesToOverwrite list.
+ */
public List<String> getPropertiesToOverwrite() {
return propertiesToOverwrite;
}
@@ -76,4 +83,21 @@ public class ImportConfiguration extends ImportExportConfiguration {
this.propertiesToOverwrite = propertiesToOverwrite;
}
+ /**
+ * Retrieves the hasHeader flag.
+ *
+ * @return true if the file imported by the current config has a header line.
+ */
+ public boolean isHasHeader() {
+ return this.hasHeader;
+ }
+
+ /**
+ * Retrieves the hasDeleteColumn flag.
+ *
+ * @return true if the file imported by the current config has a delete column.
+ */
+ public boolean isHasDeleteColumn() {
+ return this.hasDeleteColumn;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java
index 6f6d5f5..0fca0e0 100644
--- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java
+++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterConstants.java
@@ -48,10 +48,11 @@ public interface RouterConstants {
String DEFAULT_FILE_COLUMN_SEPARATOR = ",";
String DEFAULT_FILE_LINE_SEPARATOR = "\n";
-
String KEY_HISTORY_SIZE = "historySize";
String KEY_CSV_CONTENT = "csvContent";
String KEY_EXECS = "execs";
Object KEY_EXECS_DATE = "date";
Object KEY_EXECS_EXTRACTED = "extractedProfiles";
+
+ String SYSTEM_SCOPE = "integration";
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java
index 8323879..ac39312 100644
--- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java
+++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ImportRouteCompletionProcessor.java
@@ -60,16 +60,20 @@ public class ImportRouteCompletionProcessor implements Processor {
}
}
+ Integer nbTotal = (Integer) exchange.getProperty("CamelSplitSize");
+ if(importConfiguration.isHasHeader()) {
+ nbTotal--;
+ }
+
Map execution = new HashMap();
execution.put("date", ((Date) exchange.getProperty("CamelCreatedTimestamp")).getTime());
- execution.put("totalLinesNb", exchange.getProperty("CamelSplitSize"));
+ execution.put("totalLinesNb", nbTotal);
execution.put("successCount", successCount);
execution.put("failureCount", failureCount);
execution.put("errors", errors);
importConfiguration = (ImportConfiguration) RouterUtils.addExecutionEntry(importConfiguration, execution,executionsHistorySize);
- importConfiguration.getExecutions().add(execution);
//Set running to false, route is complete
if (failureCount > 0 && successCount > 0) {
importConfiguration.setStatus(RouterConstants.CONFIG_STATUS_COMPLETE_WITH_ERRORS);
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
index e047b63..2eadb91 100644
--- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
+++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
@@ -39,10 +39,17 @@ public class LineSplitProcessor implements Processor {
private List<String> propertiesToOverwrite;
private String mergingProperty;
private boolean overwriteExistingProfiles;
+ private boolean hasHeader;
+ private boolean hasDeleteColumn;
private String columnSeparator;
@Override
public void process(Exchange exchange) throws Exception {
+
+ if ((Integer) exchange.getProperty("CamelSplitIndex") == 0 && hasHeader) {
+ exchange.setProperty(Exchange.ROUTE_STOP, Boolean.TRUE);
+ return;
+ }
//In case of one shot import we check the header and overwrite import config
ImportConfiguration importConfigOneShot = (ImportConfiguration) exchange.getIn().getHeader(RouterConstants.HEADER_IMPORT_CONFIG_ONESHOT);
String configType = (String) exchange.getIn().getHeader(RouterConstants.HEADER_CONFIG_TYPE);
@@ -52,12 +59,15 @@ public class LineSplitProcessor implements Processor {
mergingProperty = importConfigOneShot.getMergingProperty();
overwriteExistingProfiles = importConfigOneShot.isOverwriteExistingProfiles();
columnSeparator = importConfigOneShot.getColumnSeparator();
+ hasHeader = importConfigOneShot.isHasHeader();
+ hasDeleteColumn = importConfigOneShot.isHasDeleteColumn();
}
+
String[] profileData = ((String) exchange.getIn().getBody()).split(columnSeparator, -1);
ProfileToImport profileToImport = new ProfileToImport();
profileToImport.setItemId(UUID.randomUUID().toString());
profileToImport.setItemType("profile");
- profileToImport.setScope("system");
+ profileToImport.setScope(RouterConstants.SYSTEM_SCOPE);
if (profileData.length > 0 && StringUtils.isNotBlank(profileData[0])) {
if (fieldsMapping.size() != (profileData.length - 1)) {
throw new BadProfileDataFormatException("The mapping does not match the number of column : line [" + ((Integer) exchange.getProperty("CamelSplitIndex") + 1) + "]", new Throwable("MAPPING_COLUMN_MATCH"));
@@ -72,7 +82,8 @@ public class LineSplitProcessor implements Processor {
profileToImport.setMergingProperty(mergingProperty);
profileToImport.setPropertiesToOverwrite(propertiesToOverwrite);
profileToImport.setOverwriteExistingProfiles(overwriteExistingProfiles);
- if (StringUtils.isNotBlank(profileData[profileData.length - 1]) && Boolean.parseBoolean(profileData[profileData.length - 1].trim())) {
+ if (hasDeleteColumn && StringUtils.isNotBlank(profileData[profileData.length - 1]) &&
+ Boolean.parseBoolean(profileData[profileData.length - 1].trim())) {
profileToImport.setProfileToDelete(true);
}
} else {
@@ -102,8 +113,12 @@ public class LineSplitProcessor implements Processor {
this.overwriteExistingProfiles = overwriteExistingProfiles;
}
- public String getMergingProperty() {
- return this.mergingProperty;
+ public void setHasHeader(boolean hasHeader) {
+ this.hasHeader = hasHeader;
+ }
+
+ public void setHasDeleteColumn(boolean hasDeleteColumn) {
+ this.hasDeleteColumn = hasDeleteColumn;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/91e23a29/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java
index e3ec8bd..06c8513 100644
--- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java
+++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/route/ProfileImportFromSourceRouteBuilder.java
@@ -81,6 +81,8 @@ public class ProfileImportFromSourceRouteBuilder extends RouterAbstractRouteBuil
lineSplitProcessor.setPropertiesToOverwrite(importConfiguration.getPropertiesToOverwrite());
lineSplitProcessor.setMergingProperty(importConfiguration.getMergingProperty());
lineSplitProcessor.setColumnSeparator(importConfiguration.getColumnSeparator());
+ lineSplitProcessor.setHasHeader(importConfiguration.isHasHeader());
+ lineSplitProcessor.setHasDeleteColumn(importConfiguration.isHasDeleteColumn());
String endpoint = (String) importConfiguration.getProperties().get("source");
[2/2] incubator-unomi git commit: UNOMI-102 : Update status on
completion
Posted by am...@apache.org.
UNOMI-102 : Update status on completion
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/d2a480cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/d2a480cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/d2a480cf
Branch: refs/heads/master
Commit: d2a480cf5de2946c1eb9c8b06ae872c53f7ba6ea
Parents: 91e23a2
Author: Abdelkader Midani <am...@apache.org>
Authored: Tue Jul 4 16:48:24 2017 +0200
Committer: Abdelkader Midani <am...@apache.org>
Committed: Tue Jul 4 16:48:24 2017 +0200
----------------------------------------------------------------------
.../unomi/router/core/processor/ExportRouteCompletionProcessor.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d2a480cf/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ExportRouteCompletionProcessor.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ExportRouteCompletionProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ExportRouteCompletionProcessor.java
index 89910b5..d1cbe97 100644
--- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ExportRouteCompletionProcessor.java
+++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/ExportRouteCompletionProcessor.java
@@ -49,6 +49,7 @@ public class ExportRouteCompletionProcessor implements Processor {
ExportConfiguration exportConfiguration = exportConfigurationService.load(exportConfig.getItemId());
exportConfiguration = (ExportConfiguration) RouterUtils.addExecutionEntry(exportConfiguration, execution, executionsHistorySize);
+ exportConfiguration.setStatus(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS);
exportConfigurationService.save(exportConfiguration);