You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/10/02 01:03:27 UTC
[1/3] usergrid git commit: Added a buffer in front of every JSON
generator.
Repository: usergrid
Updated Branches:
refs/heads/export-feature f4087473d -> ade6e7881
Added a buffer in front of every JSON generator.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/92047328
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/92047328
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/92047328
Branch: refs/heads/export-feature
Commit: 920473287103ac68dca76162e824123a3d455beb
Parents: f408747
Author: George Reyes <gr...@apache.org>
Authored: Wed Sep 30 16:30:51 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Wed Sep 30 16:30:51 2015 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 17 +++++++----------
1 file changed, 7 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/92047328/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 3d5b1a9..9079d65 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -17,7 +17,11 @@
package org.apache.usergrid.management.export;
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilterWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
@@ -581,16 +585,9 @@ public class ExportServiceImpl implements ExportService {
protected JsonGenerator getJsonGenerator( File ephermal ) throws IOException {
//TODO:shouldn't the below be UTF-16?
- JsonGenerator jg = jsonFactory.createGenerator( ephermal, JsonEncoding.UTF8 );
- jg.setPrettyPrinter( new MinimalPrettyPrinter( "" ) );
- jg.setCodec( new ObjectMapper() );
- return jg;
- }
-
- protected JsonGenerator getJsonGenerator( OutputStream outputStream ) throws IOException {
- //TODO:shouldn't the below be UTF-16?
- //TODO:do
- JsonGenerator jg = jsonFactory.createGenerator( outputStream,JsonEncoding.UTF16_LE );
+ FileOutputStream fileOutputStream = new FileOutputStream( ephermal );
+ OutputStream entityOutputStream = new BufferedOutputStream( fileOutputStream );
+ JsonGenerator jg = jsonFactory.createGenerator( entityOutputStream, JsonEncoding.UTF16_LE );
jg.setPrettyPrinter( new MinimalPrettyPrinter( "" ) );
jg.setCodec( new ObjectMapper() );
return jg;
[3/3] usergrid git commit: Added target and filter builder.
Posted by gr...@apache.org.
Added target and filter builder.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ade6e788
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ade6e788
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ade6e788
Branch: refs/heads/export-feature
Commit: ade6e7881af79730ad8732ef655087bd843405c9
Parents: 2c54c32
Author: George Reyes <gr...@apache.org>
Authored: Thu Oct 1 16:03:24 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Thu Oct 1 16:03:24 2015 -0700
----------------------------------------------------------------------
.../management/export/MockS3ExportFilterIT.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ade6e788/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
index e46bf32..3fe537d 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
@@ -221,7 +221,7 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
return payload;
}
- public HashMap<String, Object> targetBuilder() {
+ public Map<String, Object> targetBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> target = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
@@ -237,30 +237,46 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
}
- public HashMap<String, Object> filterBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames,
+ public Map filterBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames,
Set<String> connectionNames){
Map<String, Object> filters = new HashMap<>();
if(query != null)
filters.put( "ql", query.toString());
if(applicationNames != null) {
+ ArrayList<String> appNameList = new ArrayList<>( );
for ( String appName : applicationNames ) {
- filters.put( "apps", < put array of garbage here>)
+ appNameList.add( appName );
}
+ filters.put( "apps", appNameList);
}
if(collectionNames != null) {
+ ArrayList<String> collectionNamesList = new ArrayList<>( );
for ( String colName : collectionNames ) {
- filters.put( "collections", < put array of garbage here>)
+ collectionNamesList.add( colName );
}
+ filters.put( "collections",collectionNamesList);
}
if(connectionNames != null) {
- for ( String connectionNames : connectionNames ) {
- filters.put( "connections", < put array of garbage here>)
+ ArrayList<String> connectionNamesList = new ArrayList<>( );
+ for ( String connectName : connectionNames ) {
+ connectionNamesList.add( connectName );
}
+ filters.put( "connections", connectionNamesList);
+
}
- return
+ return filters;
+
+ }
+
+ public Map payloadBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames,
+ Set<String> connectionNames){
+ Map target = targetBuilder();
+ Map filter = filterBuilder(query,applicationNames,collectionNames,connectionNames);
+
+
}
[2/3] usergrid git commit: Added fixes for new export and deleted
some old stuff.
Posted by gr...@apache.org.
Added fixes for new export and deleted some old stuff.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2c54c329
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2c54c329
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2c54c329
Branch: refs/heads/export-feature
Commit: 2c54c329892cd1c28738f2c5d1bf476b72c40072
Parents: 9204732
Author: George Reyes <gr...@apache.org>
Authored: Thu Oct 1 14:16:02 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Thu Oct 1 14:16:02 2015 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 2 -
.../management/export/MockS3ExportFilterIT.java | 267 +++++++++++++++++++
2 files changed, 267 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2c54c329/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 9079d65..9eb4836 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -439,8 +439,6 @@ public class ExportServiceImpl implements ExportService {
private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String collection, Entity entity )
throws Exception {
- // Write connections
- //saveConnections( entity, em, jg );
// Write dictionaries
saveDictionaries( entity, em, jg );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2c54c329/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
new file mode 100644
index 0000000..e46bf32
--- /dev/null
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.management.export;
+
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.NewOrgAppAdminRule;
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.service.JobSchedulerService;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.entities.JobData;
+import org.apache.usergrid.services.AbstractServiceIT;
+import org.apache.usergrid.setup.ConcurrentProcessSingleton;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.util.concurrent.Service;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * Tests filters in export v2.
+ */
+public class MockS3ExportFilterIT extends AbstractServiceIT {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( MockS3ExportFilterIT.class );
+
+ @Rule
+ public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
+
+
+
+ // app-level data generated only once
+ private UserInfo adminUser;
+ private OrganizationInfo organization;
+ private UUID applicationId;
+
+ @Before
+ public void setup() throws Exception {
+ logger.info( "in setup" );
+
+ // start the scheduler after we're all set up
+ try {
+
+ JobSchedulerService jobScheduler =
+ ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
+ if ( jobScheduler.state() != Service.State.RUNNING ) {
+ jobScheduler.startAsync();
+ jobScheduler.awaitRunning();
+ }
+ }
+ catch ( Exception e ) {
+ logger.warn( "Ignoring error starting jobScheduler, already started?", e );
+ }
+
+ adminUser = newOrgAppAdminRule.getAdminInfo();
+ organization = newOrgAppAdminRule.getOrganizationInfo();
+ applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
+
+ setup.getEntityIndex().refresh( applicationId );
+ }
+
+
+ @Before
+ public void before() {
+ adminUser = newOrgAppAdminRule.getAdminInfo();
+ organization = newOrgAppAdminRule.getOrganizationInfo();
+ applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
+ }
+
+
+ @Test //Connections won't save when run with maven, but on local builds it will.
+ public void test1000ConnectionsToSingleEntity() throws Exception {
+
+ String testFileName ="testConnectionsOnApplicationEndpoint.json";
+
+ S3Export s3Export = new MockS3ExportImpl( testFileName );
+
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder( appName );
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+ // intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ int numberOfEntitiesToBeWritten = 997;
+ entity = new Entity[numberOfEntitiesToBeWritten];
+
+ // creates entities
+ for ( int i = 0; i < numberOfEntitiesToBeWritten; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );
+ entity[i] = em.create( "users", userProperties );
+ }
+
+ for(int i = 1; i<numberOfEntitiesToBeWritten; i++){
+ em.createConnection( em.get( new SimpleEntityRef( "user", entity[0].getUuid() ) ), "testConnections",
+ em.get( new SimpleEntityRef( "user", entity[i].getUuid() ) ) );
+ }
+
+ setup.getEntityIndex().refresh( applicationId );
+
+ Thread.sleep( 1000 );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( jobExecution );
+
+ TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {};
+
+ File exportedFile = new File("entities1"+testFileName);
+ exportedFile.delete();
+ exportedFile = new File("entities2"+testFileName);
+ exportedFile.delete();
+
+ final InputStream in = new FileInputStream( "connections1"+testFileName );
+ try{
+ ObjectMapper mapper = new ObjectMapper();
+ JsonParser jp = new JsonFactory( ).createParser( in );
+ Iterator jsonIterator = mapper.readValues( jp, typeRef);
+ HashMap jsonEntity = (HashMap)jsonIterator.next();
+ HashMap entityConnections =
+ ( HashMap ) ( jsonEntity ).get( entity[0].getUuid().toString() );
+ ArrayList connectionArray = (ArrayList)entityConnections.get( "testconnections" );
+ assertEquals(numberOfEntitiesToBeWritten-1,connectionArray.size());
+ }catch(Exception e){
+ assertTrue(e.getMessage(),false );
+ }
+
+ finally{
+ in.close();
+ }
+
+ // Delete the created connection files
+ exportedFile = new File("connections1"+testFileName);
+ exportedFile.delete();
+ exportedFile = new File("connections2"+testFileName);
+ exportedFile.delete();
+ }
+
+ public JobData jobDataCreator( HashMap<String, Object> payload, UUID exportUUID, S3Export s3Export ) {
+ JobData jobData = new JobData();
+
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", payload );
+ jobData.setProperty( "exportId", exportUUID );
+ jobData.setProperty( "s3Export", s3Export );
+
+ return jobData;
+ }
+
+
+ /*Creates fake payload for testing purposes.*/
+ public HashMap<String, Object> payloadBuilder( String orgOrAppName ) {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> target = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ storage_info.put( "s3_key", "null");
+ storage_info.put( "s3_access_id", "null" );
+ storage_info.put( "bucket_location", "null" );
+
+ target.put( "storage_provider", "s3" );
+ target.put( "storage_info", storage_info );
+
+ payload.put( "target", target );
+ return payload;
+ }
+
+ public HashMap<String, Object> targetBuilder() {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> target = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ storage_info.put( "s3_key", "null");
+ storage_info.put( "s3_access_id", "null" );
+ storage_info.put( "bucket_location", "null" );
+
+ target.put( "storage_provider", "s3" );
+ target.put( "storage_info", storage_info );
+
+ payload.put( "target", target );
+ return payload;
+ }
+
+
+ public HashMap<String, Object> filterBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames,
+ Set<String> connectionNames){
+ Map<String, Object> filters = new HashMap<>();
+ if(query != null)
+ filters.put( "ql", query.toString());
+
+ if(applicationNames != null) {
+ for ( String appName : applicationNames ) {
+ filters.put( "apps", < put array of garbage here>)
+ }
+ }
+ if(collectionNames != null) {
+ for ( String colName : collectionNames ) {
+ filters.put( "collections", < put array of garbage here>)
+ }
+ }
+ if(connectionNames != null) {
+ for ( String connectionNames : connectionNames ) {
+ filters.put( "connections", < put array of garbage here>)
+ }
+ }
+
+ return
+
+
+ }
+
+}