You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/10/26 20:35:25 UTC
[2/5] usergrid git commit: Adding changes for filters
Adding changes for filters
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f51b73cf
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f51b73cf
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f51b73cf
Branch: refs/heads/export-feature
Commit: f51b73cfa618fe0a66ad70210bcd5d95b73e066b
Parents: bce8d21
Author: George Reyes <gr...@apache.org>
Authored: Thu Oct 22 13:04:33 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Thu Oct 22 13:04:33 2015 -0700
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 1 +
.../management/export/ExportFilter.java | 45 ++++
.../management/export/ExportFilterImpl.java | 83 +++++++
.../management/export/ExportServiceImpl.java | 222 ++++++++++++-------
.../usergrid/management/export/export_v2.md | 9 +-
5 files changed, 284 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index afdb8cd..679451a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -326,6 +326,7 @@ public class OrganizationResource extends AbstractContextResource {
throw new NullArgumentException( "Could not find field 's3_key'" );
}
+ //organizationid is added after the fact so that
json.put( "organizationId",organization.getUuid());
jobUUID = exportService.schedule( json );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
new file mode 100644
index 0000000..aff5f26
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.management.export;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.mq.Query;
+
+
+/**
+ * A model of the filters included in usergrid and how they can be added to.
+ */
+public interface ExportFilter {
+ public Set getApplications();
+
+ public Set getCollections();
+
+ public Set getConnections();
+
+ public Query getQuery();
+
+ public void setApplications(Set applications);
+
+ public void setCollections(Set collections);
+
+ public void setConnections(Set connections);
+
+ public void setQuery(Query query);
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java
new file mode 100644
index 0000000..085739e
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.management.export;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.mq.Query;
+
+
+/**
+ * Implementation that also parses the json data to get the filter information.
+ */
+public class ExportFilterImpl implements ExportFilter {
+ public Query query;
+
+ private Set applications;
+
+ private Set collections;
+
+ private Set connections;
+
+ @Override
+ public Set getApplications() {
+ return applications;
+ }
+
+
+ @Override
+ public Set getCollections() {
+ return collections;
+ }
+
+
+ @Override
+ public Set getConnections() {
+ return connections;
+ }
+
+
+ @Override
+ public Query getQuery() {
+ return query;
+ }
+
+
+ @Override
+ public void setApplications( final Set applications ) {
+ this.applications = applications
+ }
+
+
+ @Override
+ public void setCollections( final Set collections ) {
+ this.collections = collections;
+ }
+
+
+ @Override
+ public void setConnections( final Set connections ) {
+ this.connections = connections;
+ }
+
+
+ @Override
+ public void setQuery( final Query query ) {
+ this.query = query;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 9eb4836..2f63096 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -31,6 +31,8 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import javax.management.RuntimeErrorException;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,6 +74,10 @@ public class ExportServiceImpl implements ExportService {
//injected the Entity Manager Factory to access entity manager
protected EntityManagerFactory emf;
+ //EntityManager that will only be used for the management application and updating export entities
+ protected EntityManager em;
+
+
//inject Management Service to access Organization Data
private ManagementService managementService;
@@ -85,6 +91,10 @@ public class ExportServiceImpl implements ExportService {
private JsonFactory jsonFactory = new JsonFactory();
+ public ExportServiceImpl(){
+ em = emf.getEntityManager( emf.getManagementAppId() );
+ }
+
@Override
public UUID schedule( final Map<String, Object> config ) throws Exception {
@@ -195,101 +205,123 @@ public class ExportServiceImpl implements ExportService {
}
+ //This flow that is detailed
+ //The responsibilities of this method is to error check the configuration that is passed to us in job execution
+ //Then it also delegates to the correct type of export.
+
+ //Seperate into two methods, one for error checking and the other for checking the filters?
+
+ //what should this method do? It should handle the flow of export. Aka by looking at this method we should be able to see
+ //the different steps that we need to take in order to do a export.
+
+ //Extract the job data
+ //Determine
@Override
public void doExport( final JobExecution jobExecution ) throws Exception {
- Map<String, Object> config = ( Map<String, Object> ) jobExecution.getJobData().getProperty( "exportInfo" );
- Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" );
- S3Export s3Export = null;
+ final JobData jobData = jobExecution.getJobData();
+
+ Map<String, Object> config = ( Map<String, Object> ) jobData.getProperty( "exportInfo" );
if ( config == null ) {
logger.error( "Export Information passed through is null" );
return;
}
- //get the entity manager for the application, and the entity that this Export corresponds to.
- UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- EntityManager em = emf.getEntityManager( emf.getManagementAppId() );
+ UUID exportId = ( UUID ) jobData.getProperty( EXPORT_ID );
+
+ //TODO:GREY doesn't need to get referenced everytime. Should only be set once and then used everywhere.
+ // EntityManager em = emf.getEntityManager( emf.getManagementAppId() );
Export export = em.get( exportId, Export.class );
//update the entity state to show that the job has officially started.
logger.debug( "Starting export job with uuid: "+export.getUuid() );
export.setState( Export.State.STARTED );
em.update( export );
+
+ //Checks to see if the job was given a different s3 export class. ( Local or Aws )
try {
- if ( s3PlaceHolder != null ) {
- s3Export = ( S3Export ) s3PlaceHolder;
- }
- else {
- s3Export = new AwsS3ExportImpl();
- }
- }
- catch ( Exception e ) {
- logger.error( "S3Export doesn't exist" );
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
+ S3Export s3Export = s3ExportDeterminator( jobData );
+ }catch(Exception e) {
+ updateExportStatus( export, Export.State.FAILED, e.getMessage() );
+ throw e;
}
+
+ //All verification of the job data should be done on the rest tier so at this point we shouldn't need
+ //to error check.
+
+
+
+ //No longer need this specific kind of flow, but what we do need is to check the filters
+ //the filters will tell us how we need to proceed.
+
+
+ //This is defensive programming against anybody who wants to run the export job.
+ //They need to add the organization id or else we won't know where the job came from or what it has
+ //access to.
if ( config.get( "organizationId" ) == null ) {
- logger.error( "No organization could be found" );
- export.setState( Export.State.FAILED );
- em.update( export );
+ logger.error( "No organization uuid was associated with this call. Exiting." );
+ updateExportStatus( export, Export.State.FAILED,"No organization could be found" );
return;
}
- else if ( config.get( "applicationId" ) == null ) {
- //exports All the applications from an organization
- try {
- logger.debug( "starting export of all application from the following org uuid: "+config.get( "organizationId" ).toString() );
- exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
- }
- catch ( Exception e ) {
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
- else if ( config.get( "collectionName" ) == null ) {
- //exports an Application from a single organization
- try {
- logger.debug( "Starting export of application: "+ config.get( "applicationId" ).toString());
- exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ),
- ( UUID ) config.get( "applicationId" ), config, jobExecution, s3Export );
- }
- catch ( Exception e ) {
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
- else {
- try {
- //exports a single collection from an app org combo
- try {
- logger.debug( "Starting export of the following application collection: "+ config.get( "collectionName" ));
- exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
- s3Export );
- }
- catch ( Exception e ) {
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
- catch ( Exception e ) {
- //if for any reason the backing up fails, then update the entity with a failed state.
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
+
+
+ //extracts the filter information
+ parseFilterInformation(jobData);
+
+ //we no longer have a concept of an application id. Just the filters from here on in.
+// else if ( config.get( "applicationId" ) == null ) {
+// //exports All the applications from an organization
+// try {
+// logger.debug( "starting export of all application from the following org uuid: "+config.get( "organizationId" ).toString() );
+// exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
+// }
+// catch ( Exception e ) {
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
+// else if ( config.get( "collectionName" ) == null ) {
+// //exports an Application from a single organization
+// try {
+// logger.debug( "Starting export of application: "+ config.get( "applicationId" ).toString());
+// exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ),
+// ( UUID ) config.get( "applicationId" ), config, jobExecution, s3Export );
+// }
+// catch ( Exception e ) {
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
+// else {
+// try {
+// //exports a single collection from an app org combo
+// try {
+// logger.debug( "Starting export of the following application collection: "+ config.get( "collectionName" ));
+// exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
+// s3Export );
+// }
+// catch ( Exception e ) {
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
+// catch ( Exception e ) {
+// //if for any reason the backing up fails, then update the entity with a failed state.
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
logger.debug( "finished the export job." );
- export.setState( Export.State.FINISHED );
- em.update( export );
+ updateExportStatus( export,Export.State.FINISHED,null );
}
@@ -708,4 +740,46 @@ public class ExportServiceImpl implements ExportService {
return filePointers;
}
+
+ private S3Export s3ExportDeterminator(final JobData jobData){
+ Object s3PlaceHolder = jobData.getProperty( "s3Export" );
+ S3Export s3Export = null;
+
+ try {
+ if ( s3PlaceHolder != null ) {
+ s3Export = ( S3Export ) s3PlaceHolder;
+ }
+ else {
+ s3Export = new AwsS3ExportImpl();
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "S3Export doesn't exist." );
+ throw e;
+ }
+ return s3Export;
+ }
+
+ public void updateExportStatus(Export exportEntity,Export.State exportState,String failureString) throws Exception{
+ if(failureString != null)
+ exportEntity.setErrorMessage( failureString );
+
+ exportEntity.setState( exportState );
+
+ try {
+ em.update( exportEntity );
+ }catch(Exception e){
+ logger.error( "Encountered error updating export entity! " + e.getMessage() );
+ throw e;
+ }
+ }
+
+ //All of this data is vaidated in the rest tier so it can be passed straight through here
+ //TODO: GREY find a way to pass validated object data into the scheduler.
+ public ExportFilter parseFilterInformation(JobData jobData){
+ Map<String,Object> filterData = ( Map<String, Object> ) jobData.getProperty( "filters" );
+ String query = filterData.get( "ql" );
+
+
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
index fb9eb5e..ca86175 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
@@ -4,9 +4,11 @@ Defines how the future iterated export handles what gets exported.
##Endpoints
Endpoints are the same as export v1.
+<!--
+Not needed because the filters handle what we want exported and what application it should be looking at. Not the endpoints.
` POST /management/orgs/<org_name>/apps/<app_name>/collection/<collection_name>/export `
-` POST /management/orgs/<org_name>/apps/<app_name>/export `
+` POST /management/orgs/<org_name>/apps/<app_name>/export `-->
` POST /management/orgs/<org_name>/export`
@@ -15,7 +17,7 @@ Endpoints are the same as export v1.
##What payload to the post endpoints take?
- curl -X POST -i -H 'Authorization: Bearer <your admin token goes here>' 'http://localhost:8080/management/orgs/<org_name>/apps/<app_name>/export' -d
+ curl -X POST -i -H 'Authorization: Bearer <your admin token goes here>' 'http://localhost:8080/management/orgs/<org_name>/export' -d
'{"target":{
"storage_provider":"s3",
"storage_info":{
@@ -50,6 +52,9 @@ There are 4 ways that you can filter data out by.
- Connections
- In order to export specific connections you can list them same as the other filters. If this is filled in then you are only exporting the connections that contain the names listed in the connections json array.
- If you want to export all the connections then delete the filter and it will export all the connections.
+
+####What happens if my data is invalid?
+If you try to export data that doesn't exist in your filter then the call will fail and you will get returned a list of the data that is invalid.
##Data Format for export