You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by am...@apache.org on 2017/08/10 00:20:43 UTC
[1/5] incubator-unomi git commit: UNOMI-101 : Fix issue with empty
column
Repository: incubator-unomi
Updated Branches:
refs/heads/master 4f1ed654b -> b02863425
UNOMI-101 : Fix issue with empty column
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/d37b0be4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/d37b0be4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/d37b0be4
Branch: refs/heads/master
Commit: d37b0be4800207d197c63ac507174a930c17314e
Parents: 4f1ed65
Author: Abdelkader Midani <am...@apache.org>
Authored: Thu Aug 10 01:19:03 2017 +0200
Committer: Abdelkader Midani <am...@apache.org>
Committed: Thu Aug 10 01:19:03 2017 +0200
----------------------------------------------------------------------
.../unomi/router/core/processor/LineSplitProcessor.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d37b0be4/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
index dcd8374..081a9ef 100644
--- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
+++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/processor/LineSplitProcessor.java
@@ -85,7 +85,12 @@ public class LineSplitProcessor implements Processor {
.withSeparator(columnSeparator.charAt(0))
.build();
+ logger.debug("$$$$ : LineSplitProcessor : BODY : " + (String) exchange.getIn().getBody());
+
String[] profileData = rfc4180Parser.parseLine(((String) exchange.getIn().getBody()));
+
+ logger.debug("$$$$ : LineSplitProcessor : LINE : {}, {}, {}.", profileData[0], profileData[1], profileData[2]);
+
ProfileToImport profileToImport = new ProfileToImport();
profileToImport.setItemId(UUID.randomUUID().toString());
profileToImport.setItemType("profile");
@@ -95,6 +100,7 @@ public class LineSplitProcessor implements Processor {
if (hasDeleteColumn && (fieldsMapping.size() > (profileData.length - 1))) {
throw new BadProfileDataFormatException("The mapping does not match the number of column : line [" + ((Integer) exchange.getProperty("CamelSplitIndex") + 1) + "]", new Throwable("MAPPING_COLUMN_MATCH"));
}
+ logger.debug("$$$$ : LineSplitProcessor : MAPPING : " + fieldsMapping.keySet());
Map<String, Object> properties = new HashMap<>();
for (String fieldMappingKey : fieldsMapping.keySet()) {
PropertyType propertyType = RouterUtils.getPropertyTypeById(profileService.getAllPropertyTypes("profiles"), fieldMappingKey);
@@ -104,15 +110,14 @@ public class LineSplitProcessor implements Processor {
if (propertyType.getValueTypeId().equals("string") || propertyType.getValueTypeId().equals("email")) {
if (BooleanUtils.isTrue(propertyType.isMultivalued())) {
String multivalueArray = profileData[fieldsMapping.get(fieldMappingKey)].trim();
- if (StringUtils.isNoneBlank(multiValueDelimiter) && multiValueDelimiter.length() == 2) {
+ if (StringUtils.isNotBlank(multiValueDelimiter) && multiValueDelimiter.length() == 2) {
multivalueArray = multivalueArray.replaceAll("\\" + multiValueDelimiter.charAt(0), "").replaceAll("\\" + multiValueDelimiter.charAt(1), "");
- multivalueArray = RouterUtils.removeQuotes(multivalueArray);
}
String[] valuesArray = multivalueArray.split("\\" + multiValueSeparator);
properties.put(fieldMappingKey, valuesArray);
} else {
String singleValue = profileData[fieldsMapping.get(fieldMappingKey)].trim();
- properties.put(fieldMappingKey, RouterUtils.removeQuotes(singleValue));
+ properties.put(fieldMappingKey, singleValue);
}
} else if (propertyType.getValueTypeId().equals("boolean")) {
properties.put(fieldMappingKey, new Boolean(profileData[fieldsMapping.get(fieldMappingKey)].trim()));
[2/5] incubator-unomi git commit: UNOMI-101 : Fix null profile
service in recurrent import routes
Posted by am...@apache.org.
UNOMI-101 : Fix null profile service in recurrent import routes
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/2d9a95a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/2d9a95a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/2d9a95a5
Branch: refs/heads/master
Commit: 2d9a95a53cd2c4f59245d60e6ab2eee612c4170f
Parents: d37b0be
Author: Abdelkader Midani <am...@apache.org>
Authored: Thu Aug 10 01:22:16 2017 +0200
Committer: Abdelkader Midani <am...@apache.org>
Committed: Thu Aug 10 01:22:16 2017 +0200
----------------------------------------------------------------------
.../org/apache/unomi/router/core/context/RouterCamelContext.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/2d9a95a5/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java
index 4596af7..302b77d 100644
--- a/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java
+++ b/extensions/router/router-core/src/main/java/org/apache/unomi/router/core/context/RouterCamelContext.java
@@ -191,6 +191,7 @@ public class RouterCamelContext implements SynchronousBundleListener {
ProfileImportFromSourceRouteBuilder builder = new ProfileImportFromSourceRouteBuilder(kafkaProps, configType);
builder.setImportConfigurationList(Arrays.asList(importConfiguration));
builder.setImportConfigurationService(importConfigurationService);
+ builder.setProfileService(profileService);
builder.setAllowedEndpoints(allowedEndpoints);
builder.setJacksonDataFormat(jacksonDataFormat);
builder.setContext(camelContext);
[5/5] incubator-unomi git commit: UNOMI-118 : Surfers test,
profile import, and rename Basic test
Posted by am...@apache.org.
UNOMI-118 : Surfers test, profile import, and rename Basic test
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/b0286342
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/b0286342
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/b0286342
Branch: refs/heads/master
Commit: b02863425db8ce65ceedfbc7cb835507024dd6ee
Parents: 5c99fb2
Author: Abdelkader Midani <am...@apache.org>
Authored: Thu Aug 10 02:20:34 2017 +0200
Committer: Abdelkader Midani <am...@apache.org>
Committed: Thu Aug 10 02:20:34 2017 +0200
----------------------------------------------------------------------
.../java/org/apache/unomi/itests/AllITs.java | 3 +-
.../java/org/apache/unomi/itests/BaseIT.java | 2 +
.../unomi/itests/ProfileImportBasicIT.java | 114 ++++++++++++
.../apache/unomi/itests/ProfileImportIT.java | 107 -----------
.../unomi/itests/ProfileImportSurfersIT.java | 177 +++++++++++++++++++
itests/src/test/resources/2-surfers-test.csv | 37 ++++
6 files changed, 332 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/AllITs.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index c2b6fd3..225513e 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -33,7 +33,8 @@ import org.junit.runners.Suite.SuiteClasses;
ConditionESQueryBuilderIT.class,
SegmentIT.class,
ProfileServiceIT.class,
- ProfileImportIT.class
+ ProfileImportBasicIT.class,
+ ProfileImportSurfersIT.class
})
public class AllITs {
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
index fa56a74..38a2415 100644
--- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
@@ -97,6 +97,8 @@ public abstract class BaseIT {
"src/test/resources/org.apache.unomi.router.cfg")),
replaceConfigurationFile("data/tmp/unomi_oneshot_import_configs/1-basic-test.csv", new File(
"src/test/resources/1-basic-test.csv")),
+ replaceConfigurationFile("data/tmp/recurrent_import/2-surfers-test.csv", new File(
+ "src/test/resources/2-surfers-test.csv")),
keepRuntimeFolder(),
configureConsole().ignoreLocalConsole(),
logLevel(LogLevel.INFO),
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java
new file mode 100644
index 0000000..a147029
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportBasicIT.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests;
+
+import org.apache.unomi.api.PartialList;
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.PropertyType;
+import org.apache.unomi.api.services.ProfileService;
+import org.apache.unomi.router.api.ImportConfiguration;
+import org.apache.unomi.router.api.RouterConstants;
+import org.apache.unomi.router.api.services.ImportExportConfigurationService;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+import org.ops4j.pax.exam.util.Filter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by amidani on 03/08/2017.
+ */
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class ProfileImportBasicIT extends BaseIT {
+
+ @Inject
+ @Filter("(configDiscriminator=IMPORT)")
+ protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService;
+
+ @Inject
+ protected ProfileService profileService;
+ private Logger logger = LoggerFactory.getLogger(ProfileImportBasicIT.class);
+
+
+ @Before
+ public void setUp() {
+
+ /*** Basic Test ***/
+ ImportConfiguration importConfiguration = new ImportConfiguration();
+ importConfiguration.setItemId("1-basic-test");
+ importConfiguration.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_ONESHOT);
+ importConfiguration.setMergingProperty("email");
+ importConfiguration.setOverwriteExistingProfiles(true);
+
+ Map mapping = new HashMap();
+ mapping.put("email", 0);
+ mapping.put("firstName", 1);
+ mapping.put("lastName", 2);
+
+ importConfiguration.getProperties().put("mapping", mapping);
+ importConfiguration.setActive(true);
+
+ importConfigurationService.save(importConfiguration);
+
+ }
+
+ @Test
+ public void testCheckImportConfigList() {
+ List<ImportConfiguration> importConfigurations = importConfigurationService.getAll();
+ Assert.assertEquals(1, importConfigurations.size());
+ }
+
+ @Test
+ public void testImport1Basic() throws IOException, InterruptedException {
+
+ //Wait for the csv to be processed
+ Thread.sleep(10000);
+
+ //Check saved profiles
+ PartialList<Profile> profiles = profileService.findProfilesByPropertyValue("properties.email", "basic1@test.com", 0, 10, null);
+ Assert.assertEquals(3, profileService.getAllProfilesCount());
+ Assert.assertEquals(1, profiles.getList().size());
+ Assert.assertNotNull(profiles.get(0));
+ Assert.assertEquals("Basic1", profiles.get(0).getProperty("firstName"));
+ Assert.assertEquals("User1", profiles.get(0).getProperty("lastName"));
+
+ //Check import config status
+ ImportConfiguration importConfiguration = importConfigurationService.load("1-basic-test");
+ Assert.assertEquals(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS, importConfiguration.getStatus());
+ Assert.assertEquals(1, importConfiguration.getExecutions().size());
+
+ }
+
+ @After
+ public void tearDown() {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java
deleted file mode 100644
index afd3ce9..0000000
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileImportIT.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License
- */
-package org.apache.unomi.itests;
-
-import org.apache.unomi.api.PartialList;
-import org.apache.unomi.api.Profile;
-import org.apache.unomi.api.services.ProfileService;
-import org.apache.unomi.router.api.ImportConfiguration;
-import org.apache.unomi.router.api.RouterConstants;
-import org.apache.unomi.router.api.services.ImportExportConfigurationService;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.ops4j.pax.exam.junit.PaxExam;
-import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
-import org.ops4j.pax.exam.spi.reactors.PerSuite;
-import org.ops4j.pax.exam.util.Filter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created by amidani on 03/08/2017.
- */
-@RunWith(PaxExam.class)
-@ExamReactorStrategy(PerSuite.class)
-public class ProfileImportIT extends BaseIT {
-
- @Inject
- @Filter("(configDiscriminator=IMPORT)")
- protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService;
- @Inject
- protected ProfileService profileService;
- private Logger logger = LoggerFactory.getLogger(ProfileImportIT.class);
-
- @Before
- public void setUp() {
-
- /*** Basic Test ***/
- ImportConfiguration importConfiguration = new ImportConfiguration();
- importConfiguration.setItemId("1-basic-test");
- importConfiguration.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_ONESHOT);
- importConfiguration.setMergingProperty("email");
- importConfiguration.setOverwriteExistingProfiles(true);
-
- Map mapping = new HashMap();
- mapping.put("email", 0);
- mapping.put("firstName", 1);
- mapping.put("lastName", 2);
-
- importConfiguration.getProperties().put("mapping", mapping);
- importConfiguration.setActive(true);
-
- importConfigurationService.save(importConfiguration);
-
- }
-
- @Test
- public void testGetImportConfigList() {
- List<ImportConfiguration> importConfigurations = importConfigurationService.getAll();
- Assert.assertEquals(1, importConfigurations.size());
- }
-
- @Test
- public void testBasic() throws IOException, InterruptedException {
-
- //Wait for the csv to be processed
- Thread.sleep(10000);
-
- //Check saved profiles
- PartialList<Profile> profiles = profileService.findProfilesByPropertyValue("properties.email", "basic1@test.com", 0, 10, null);
- logger.info("$$$$ : Profile count : " + profileService.getAllProfilesCount());
- logger.info("$$$$ : List size : " + profiles.getList().size());
- Assert.assertEquals(3, profileService.getAllProfilesCount());
- Assert.assertEquals(1, profiles.getList().size());
- Assert.assertNotNull(profiles.get(0));
- Assert.assertEquals("Basic1", profiles.get(0).getProperty("firstName"));
- Assert.assertEquals("User1", profiles.get(0).getProperty("lastName"));
-
- //Check import config status
- ImportConfiguration importConfiguration = importConfigurationService.load("1-basic-test");
- Assert.assertEquals(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS, importConfiguration.getStatus());
- Assert.assertEquals(1, importConfiguration.getExecutions().size());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java
----------------------------------------------------------------------
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java
new file mode 100644
index 0000000..1be2a23
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileImportSurfersIT.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthenticationException;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.unomi.api.Metadata;
+import org.apache.unomi.api.PartialList;
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.PropertyType;
+import org.apache.unomi.api.services.ProfileService;
+import org.apache.unomi.router.api.ImportConfiguration;
+import org.apache.unomi.router.api.RouterConstants;
+import org.apache.unomi.router.api.services.ImportExportConfigurationService;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.ops4j.pax.exam.junit.PaxExam;
+import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy;
+import org.ops4j.pax.exam.spi.reactors.PerSuite;
+import org.ops4j.pax.exam.util.Filter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.ws.rs.core.MediaType;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ * Created by amidani on 09/08/2017.
+ */
+@RunWith(PaxExam.class)
+@ExamReactorStrategy(PerSuite.class)
+public class ProfileImportSurfersIT extends BaseIT {
+
+ @Inject
+ @Filter("(configDiscriminator=IMPORT)")
+ protected ImportExportConfigurationService<ImportConfiguration> importConfigurationService;
+
+ @Inject
+ protected ProfileService profileService;
+ private Logger logger = LoggerFactory.getLogger(ProfileImportSurfersIT.class);
+
+
+ @Before
+ public void setUp() throws IOException, AuthenticationException {
+
+ /*** Create Missing Properties ***/
+ PropertyType propertyType = new PropertyType(new Metadata("integration", "alive", "Alive", "Is the person alive?"));
+ propertyType.setValueTypeId("boolean");
+ propertyType.setTagIds(Collections.singleton("basicProfileProperties"));
+ propertyType.setTarget("profiles");
+
+ profileService.setPropertyType(propertyType);
+
+ /*** Surfer Test ***/
+ ImportConfiguration importConfigSurfers = new ImportConfiguration();
+ importConfigSurfers.setItemId("2-surfers-test");
+ importConfigSurfers.setConfigType(RouterConstants.IMPORT_EXPORT_CONFIG_TYPE_RECURRENT);
+ importConfigSurfers.setMergingProperty("linkedInId");
+ importConfigSurfers.setOverwriteExistingProfiles(true);
+ importConfigSurfers.setColumnSeparator(";");
+ importConfigSurfers.setHasHeader(true);
+ importConfigSurfers.setHasDeleteColumn(true);
+
+ Map mappingSurfers = new HashMap();
+ mappingSurfers.put("linkedInId", 0);
+ mappingSurfers.put("lastName", 1);
+ mappingSurfers.put("email", 2);
+ mappingSurfers.put("facebookId", 3);
+ mappingSurfers.put("gender", 4);
+ mappingSurfers.put("alive", 5);
+
+ importConfigSurfers.getProperties().put("mapping", mappingSurfers);
+ File importSurfersFile = new File("data/tmp/recurrent_import/");
+ importConfigSurfers.getProperties().put("source", "file://" + importSurfersFile.getAbsolutePath() + "?fileName=2-surfers-test.csv&consumer.delay=10m&consumer.initialDelay=5s&move=.done");
+ importConfigSurfers.setActive(true);
+
+ ImportConfiguration savedImportConfig = importConfigurationService.save(importConfigSurfers);
+
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+ HttpPut httpPut = new HttpPut(URL + "/configUpdate/importConfigAdmin");
+
+ String json = new ObjectMapper().writeValueAsString(savedImportConfig);
+ StringEntity entity = new StringEntity(json);
+ entity.setContentType(MediaType.APPLICATION_JSON);
+ httpPut.setEntity(entity);
+
+ HttpResponse response = httpclient.execute(httpPut);
+ assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+ httpclient.close();
+
+ }
+
+ @Test
+ public void testCheckAddedProperties() throws IOException, InterruptedException {
+
+ //Wait for data to be processed
+ Thread.sleep(5000);
+
+ PropertyType propAlive = profileService.getPropertyType("alive");
+
+ Assert.assertNotNull(propAlive);
+
+ }
+
+ @Test
+ public void testCheckImportConfigList() {
+
+ List<ImportConfiguration> importConfigurations = importConfigurationService.getAll();
+ Assert.assertEquals(2, importConfigurations.size());
+
+ }
+
+ @Test
+ public void testImport2Surfers() throws IOException, InterruptedException {
+
+ //Wait for data to be processed
+ Thread.sleep(10000);
+
+ Assert.assertEquals(37, profileService.getAllProfilesCount());
+
+ //Profile not to delete
+ PartialList<Profile> jordyProfile = profileService.findProfilesByPropertyValue("properties.email", "jordy@smith.com", 0, 10, null);
+ Assert.assertEquals(1, jordyProfile.getList().size());
+ Assert.assertNotNull(jordyProfile.get(0));
+ Assert.assertEquals("1", jordyProfile.get(0).getProperty("linkedInId"));
+ Assert.assertEquals("Jordy Smith", jordyProfile.get(0).getProperty("lastName"));
+ Assert.assertEquals("999", jordyProfile.get(0).getProperty("facebookId"));
+ Assert.assertEquals("male", jordyProfile.get(0).getProperty("gender"));
+ Assert.assertTrue((Boolean) jordyProfile.get(0).getProperty("alive"));
+
+ //Profile to delete
+ PartialList<Profile> paulineProfile = profileService.findProfilesByPropertyValue("properties.lastName", "Pauline Ado", 0, 10, null);
+ Assert.assertEquals(0, paulineProfile.getList().size());
+
+ //Check import config status
+ ImportConfiguration importConfiguration = importConfigurationService.load("2-surfers-test");
+ Assert.assertEquals(RouterConstants.CONFIG_STATUS_COMPLETE_SUCCESS, importConfiguration.getStatus());
+ Assert.assertEquals(1, importConfiguration.getExecutions().size());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/b0286342/itests/src/test/resources/2-surfers-test.csv
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/2-surfers-test.csv b/itests/src/test/resources/2-surfers-test.csv
new file mode 100644
index 0000000..bee4ef9
--- /dev/null
+++ b/itests/src/test/resources/2-surfers-test.csv
@@ -0,0 +1,37 @@
+Linkedin ID;Last Name;Email;Facebook ID;Gender;Alive;To Delete
+1;Jordy Smith;jordy@smith.com;999;male;TRUE;FALSE
+2;Gabriel Medina;gabriel@medina.com;998;male;TRUE;FALSE
+3;Kolohe Andino;kolohe@andino.com;997;male;TRUE;FALSE
+4;Matt Wilkinson;Matt@Wilkinson.com;996;male;TRUE;FALSE
+5;Michel Bourez;Michel@Bourez.com;995;male;TRUE;FALSE
+6;Kelly Slater;kelly@slater.com;994;male;TRUE;FALSE
+7;Julian Wilson;julian@wilson.com;993;male;TRUE;FALSE
+8;Joel Parkinson;joel@parkinson.com;992;male;TRUE;FALSE
+9;Filipe Toledo;filipe@toledo.com;991;male;TRUE;FALSE
+10;Adriano De Souza;adriano@souza.com;990;male;TRUE;FALSE
+11;Sebastian Zietz;sebastian@zietz.com;989;male;TRUE;TRUE
+12;Josh Kerr;johsh@kerr.com;988;male;TRUE;FALSE
+13;Adrian Buchan;adrian@buchan.com;987;male;TRUE;FALSE
+14;Italo Ferreira;italo@ferreiro.com;986;male;TRUE;FALSE
+15;Mick Fanning;mick@fanning.com;985;male;TRUE;FALSE
+16;Miguel Pupo;miguel@pupo.com;984;male;TRUE;FALSE
+17;Nat Young;nat@young.com;983;male;TRUE;FALSE
+18;Keanu Asing;;982;male;TRUE;FALSE
+19;Tyler Wright;;981;male;TRUE;FALSE
+20;Courtney Conlogue;;980;female;TRUE;FALSE
+21;Carissa Moore;;979;female;TRUE;FALSE
+22;Tatiana Weston-Webb;;978;female;TRUE;FALSE
+23;Johanne Defay;;977;female;TRUE;FALSE
+24;Stephanie Gilmore;;976;female;TRUE;FALSE
+25;Malia Manuel;;975;female;TRUE;FALSE
+26;Sally Fitzgibbons;;974;female;TRUE;FALSE
+27;Sage Erickson;;973;female;TRUE;FALSE
+28;Laura Enever;;972;female;TRUE;FALSE
+29;Nikki Van Dijk;;971;female;TRUE;FALSE
+30;Bianca Buitendag;;970;female;TRUE;FALSE
+31;Coco Ho;;969;female;TRUE;FALSE
+32;Alessa Quizon;;968;female;TRUE;FALSE
+33;Lakey Peterson;;967;female;TRUE;FALSE
+34;Bronte Macaulay;;966;female;TRUE;FALSE
+35;Pauline Ado;;965;female;TRUE;TRUE
+36;Silvana Lima;;964;female;TRUE;FALSE
\ No newline at end of file
[3/5] incubator-unomi git commit: UNOMI-101 : Ignore profiles marked
as "To Delete" if they do not exist
Posted by am...@apache.org.
UNOMI-101 : Ignore profiles marked as "To Delete" if they do not exist
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/3e6739fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/3e6739fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/3e6739fa
Branch: refs/heads/master
Commit: 3e6739faf37693c18147318a0bcf9015475e1917
Parents: 2d9a95a
Author: Abdelkader Midani <am...@apache.org>
Authored: Thu Aug 10 01:23:05 2017 +0200
Committer: Abdelkader Midani <am...@apache.org>
Committed: Thu Aug 10 01:23:05 2017 +0200
----------------------------------------------------------------------
.../apache/unomi/router/services/ProfileImportServiceImpl.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/3e6739fa/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java
index e91ae7e..b7ab45a 100644
--- a/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java
+++ b/extensions/router/router-service/src/main/java/org/apache/unomi/router/services/ProfileImportServiceImpl.java
@@ -68,6 +68,10 @@ public class ProfileImportServiceImpl extends AbstractCustomServiceImpl implemen
profileToImport.getMergingProperty(), profileToImport.getProperties().get(profileToImport.getMergingProperty()).toString());
}
logger.debug("-------------------------------------");
- return persistenceService.save(existingProfile, true);
+ if (!profileToImport.isProfileToDelete()) {
+ return persistenceService.save(existingProfile, true);
+ } else {
+ return false;
+ }
}
}
[4/5] incubator-unomi git commit: UNOMI-101 : Remove unused method
following the use of RFC4180Parser
Posted by am...@apache.org.
UNOMI-101 : Remove unused method following the use of RFC4180Parser
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/5c99fb29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/5c99fb29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/5c99fb29
Branch: refs/heads/master
Commit: 5c99fb29c74456a6e88576c08ac1ffb193bc6c9a
Parents: 3e6739f
Author: Abdelkader Midani <am...@apache.org>
Authored: Thu Aug 10 01:23:59 2017 +0200
Committer: Abdelkader Midani <am...@apache.org>
Committed: Thu Aug 10 01:23:59 2017 +0200
----------------------------------------------------------------------
.../src/main/java/org/apache/unomi/router/api/RouterUtils.java | 6 ------
1 file changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/5c99fb29/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java
----------------------------------------------------------------------
diff --git a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java
index 4584d76..464f908 100644
--- a/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java
+++ b/extensions/router/router-api/src/main/java/org/apache/unomi/router/api/RouterUtils.java
@@ -60,10 +60,4 @@ public class RouterUtils {
return null;
}
- public static String removeQuotes(String propValue) {
- if(propValue.charAt(0) == '"' && propValue.charAt(propValue.length()-1) == '"') {
- return propValue.substring(1, propValue.length()-1);
- }
- return propValue;
- }
}