You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/10 21:26:14 UTC
[21/50] [abbrv] git commit: Added classes to the branch that were
labeled untracked.
Added classes to the branch that were labeled untracked.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e1ab9df8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e1ab9df8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e1ab9df8
Branch: refs/pull/70/merge
Commit: e1ab9df84e526364ed199dea72988f905add8393
Parents: 22a17dc
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 09:37:04 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 09:37:04 2014 -0800
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/Query.java | 5 +-
.../usergrid/persistence/entities/Export.java | 125 ++
.../usergrid/persistence/entities/Export.java | 124 --
.../rest/management/ManagementResource.java | 15 +-
.../applications/ApplicationResource.java | 84 +-
.../applications/ApplicationResource.java | 277 -----
.../rest/management/ManagementResourceIT.java | 180 ++-
.../apache/usergrid/management/ExportInfo.java | 65 +
.../usergrid/management/export/ExportJob.java | 65 +
.../management/export/ExportService.java | 37 +
.../management/export/ExportServiceImpl.java | 457 +++++++
.../usergrid/management/export/S3Export.java | 16 +
.../management/export/S3ExportImpl.java | 84 ++
.../org/usergrid/management/ExportInfo.java | 65 -
.../usergrid/management/export/ExportJob.java | 64 -
.../management/export/ExportService.java | 37 -
.../management/export/ExportServiceImpl.java | 454 -------
.../usergrid/management/export/S3Export.java | 16 -
.../management/export/S3ExportImpl.java | 84 --
.../resources/usergrid-services-context.xml | 4 +-
.../org/apache/usergrid/ServiceITSetup.java | 3 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 6 +
.../cassandra/ManagementServiceIT.java | 422 +++++++
.../management/cassandra/MockS3ExportImpl.java | 49 +
.../test/java/org/usergrid/ServiceITSetup.java | 47 -
.../java/org/usergrid/ServiceITSetupImpl.java | 135 --
.../cassandra/ManagementServiceIT.java | 1152 ------------------
.../management/cassandra/MockS3ExportImpl.java | 49 -
28 files changed, 1601 insertions(+), 2520 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 01ac231..286cd7f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -36,6 +36,9 @@ import org.antlr.runtime.TokenRewriteStream;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.persistence.Results.Level;
import org.apache.usergrid.persistence.exceptions.QueryParseException;
import org.apache.usergrid.persistence.query.tree.AndOperand;
@@ -51,8 +54,6 @@ import org.apache.usergrid.persistence.query.tree.QueryFilterLexer;
import org.apache.usergrid.persistence.query.tree.QueryFilterParser;
import org.apache.usergrid.utils.JsonUtils;
-import org.apache.commons.lang.StringUtils;
-
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.split;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
new file mode 100644
index 0000000..182b2ae
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
@@ -0,0 +1,125 @@
+package org.apache.usergrid.persistence.entities;
+
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import org.apache.usergrid.persistence.TypedEntity;
+import org.apache.usergrid.persistence.annotations.EntityProperty;
+
+
+/**
+ *Contains state information for an Entity Job
+ *
+ */
+@XmlRootElement
+public class Export extends TypedEntity {
+
+ public static enum State {
+ //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
+ PENDING,STARTED,FAILED,COMPLETED
+ }
+
+ @EntityProperty
+ protected State curState;
+
+ @EntityProperty
+ protected Long queued;
+
+ /** Time send started */
+ @EntityProperty
+ protected Long started;
+
+ /** Time processed */
+ @EntityProperty
+ protected Long finished;
+
+
+ /** Time to expire the exportJob */
+ @EntityProperty
+ protected Long expire;
+
+ /** True if exportJob is canceled */
+ @EntityProperty
+ protected Boolean canceled;
+
+ /** Error message */
+ @EntityProperty
+ protected String errorMessage;
+
+ public Export() {
+ }
+
+ public boolean isExpired () {
+ return (expire != null && expire > System.currentTimeMillis());
+ }
+
+ public Long getStarted() {
+ return started;
+ }
+
+
+ public void setStarted( final Long started ) {
+ this.started = started;
+ }
+
+
+ public Long getFinished() {
+ return finished;
+ }
+
+
+ public void setFinished( final Long finished ) {
+ this.finished = finished;
+ }
+
+
+ public Long getExpire() {
+ return expire;
+ }
+
+
+ public void setExpire( final Long expire ) {
+ this.expire = expire;
+ }
+
+
+ public Boolean getCanceled() {
+ return canceled;
+ }
+
+ //state should moved to a derived state, but it is not there yet.
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public void setState(State setter) {
+ curState = setter;
+ }
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public State getState() { return curState; }
+
+ public void setCanceled( final Boolean canceled ) {
+ this.canceled = canceled;
+ }
+
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+
+ public void setErrorMessage( final String errorMessage ) {
+ this.errorMessage = errorMessage;
+ }
+
+
+ public Long getQueued() {
+ return queued;
+ }
+
+
+ public void setQueued( final Long queued ) {
+ this.queued = queued;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
deleted file mode 100644
index 60d2bcb..0000000
--- a/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
+++ /dev/null
@@ -1,124 +0,0 @@
-package org.usergrid.persistence.entities;
-
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.usergrid.persistence.TypedEntity;
-import org.usergrid.persistence.annotations.EntityProperty;
-
-
-/**
- *Contains state information for an Entity Job
- *
- */
-@XmlRootElement
-public class Export extends TypedEntity {
-
- public static enum State {
- //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
- PENDING,STARTED,FAILED,COMPLETED
- }
-
- @EntityProperty
- protected State curState;
-
- @EntityProperty
- protected Long queued;
-
- /** Time send started */
- @EntityProperty
- protected Long started;
-
- /** Time processed */
- @EntityProperty
- protected Long finished;
-
-
- /** Time to expire the exportJob */
- @EntityProperty
- protected Long expire;
-
- /** True if exportJob is canceled */
- @EntityProperty
- protected Boolean canceled;
-
- /** Error message */
- @EntityProperty
- protected String errorMessage;
-
- public Export() {
- }
-
- public boolean isExpired () {
- return (expire != null && expire > System.currentTimeMillis());
- }
-
- public Long getStarted() {
- return started;
- }
-
-
- public void setStarted( final Long started ) {
- this.started = started;
- }
-
-
- public Long getFinished() {
- return finished;
- }
-
-
- public void setFinished( final Long finished ) {
- this.finished = finished;
- }
-
-
- public Long getExpire() {
- return expire;
- }
-
-
- public void setExpire( final Long expire ) {
- this.expire = expire;
- }
-
-
- public Boolean getCanceled() {
- return canceled;
- }
-
- //state should moved to a derived state, but it is not there yet.
- @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- @EntityProperty
- public void setState(State setter) {
- curState = setter;
- }
- @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- @EntityProperty
- public State getState() { return curState; }
-
- public void setCanceled( final Boolean canceled ) {
- this.canceled = canceled;
- }
-
-
- public String getErrorMessage() {
- return errorMessage;
- }
-
-
- public void setErrorMessage( final String errorMessage ) {
- this.errorMessage = errorMessage;
- }
-
-
- public Long getQueued() {
- return queued;
- }
-
-
- public void setQueued( final Long queued ) {
- this.queued = queued;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 59ceb4e..72e967f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -37,6 +37,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.error.OAuthError;
+import org.apache.amber.oauth2.common.exception.OAuthProblemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.amber.oauth2.common.message.types.GrantType;
+import org.apache.commons.lang.StringUtils;
+import org.apache.shiro.codec.Base64;
+
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.management.exceptions.DisabledAdminUserException;
import org.apache.usergrid.management.exceptions.UnactivatedAdminUserException;
@@ -48,13 +56,6 @@ import org.apache.usergrid.rest.management.users.UsersResource;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.shiro.utils.SubjectUtils;
-import org.apache.amber.oauth2.common.error.OAuthError;
-import org.apache.amber.oauth2.common.exception.OAuthProblemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.amber.oauth2.common.message.types.GrantType;
-import org.apache.commons.lang.StringUtils;
-import org.apache.shiro.codec.Base64;
-
import com.sun.jersey.api.view.Viewable;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index dff0924..5f45326 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -16,6 +16,7 @@
package org.apache.usergrid.rest.management.organizations.applications;
+import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@@ -25,30 +26,44 @@ import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthSystemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.ExportInfo;
import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+import org.apache.usergrid.rest.utils.JSONPUtils;
import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
import org.apache.usergrid.security.providers.SignInAsProvider;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.services.ServiceManager;
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
+import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
+import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
@@ -60,6 +75,8 @@ import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
})
public class ApplicationResource extends AbstractContextResource {
+ @Autowired
+ protected ExportService exportService;
OrganizationInfo organization;
UUID applicationId;
ApplicationInfo application;
@@ -192,4 +209,67 @@ public class ApplicationResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+
+ @POST
+ @Path("export")
+ @Consumes(APPLICATION_JSON)
+ @RequireOrganizationAccess
+ public Response exportPostJson (@Context UriInfo ui,
+ Map<String, Object> json,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
+ throws OAuthSystemException {
+
+
+ OAuthResponse response = null;
+ UUID jobUUID = null;
+ Map<String, String> uuidRet = new HashMap<String, String>( );
+
+ try {
+ //parse the json into some useful object (the config params)
+ ExportInfo objEx = new ExportInfo(json);
+ objEx.setApplicationId( applicationId );
+ jobUUID = exportService.schedule(objEx);
+ uuidRet.put( "jobUUID", jobUUID.toString() );
+
+ }
+ catch (NullPointerException e) {
+ OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
+ .setErrorDescription( "Job Not Created" )
+ .buildJSONMessage();
+
+ return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
+ .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
+ }
+ catch (Exception e) {
+ //TODO:throw descriptive error message and or include on in the response
+ //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
+ return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
+ }
+
+ return Response.status(SC_ACCEPTED).entity(uuidRet).build();
+
+ //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
+ // .entity( wrapWithCallback( "", callback ) ).build();
+ }
+
+ @GET
+ @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
+ public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
+
+ Export entity;
+ try {
+ entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
+ }catch(Exception e) {
+ return Response.status( SC_BAD_REQUEST ).build();
+ }
+ //validate this user owns it
+
+ if (entity == null) {
+ return Response.status(SC_BAD_REQUEST).build();
+ }
+
+ return Response.status(SC_OK).entity(entity.getState()).build();
+ //return Response.status(SC_OK).entity(state).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
deleted file mode 100644
index 5a29500..0000000
--- a/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*******************************************************************************
- * Copyright 2012 Apigee Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.usergrid.rest.management.organizations.applications;
-
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-import org.usergrid.management.ApplicationInfo;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.rest.AbstractContextResource;
-import org.usergrid.rest.ApiResponse;
-import org.usergrid.rest.applications.ServiceResource;
-import org.usergrid.rest.security.annotations.RequireOrganizationAccess;
-import org.usergrid.rest.utils.JSONPUtils;
-import org.usergrid.security.oauth.ClientCredentialsInfo;
-import org.usergrid.security.providers.SignInAsProvider;
-import org.usergrid.security.providers.SignInProviderFactory;
-import org.usergrid.services.ServiceManager;
-
-import org.apache.amber.oauth2.common.exception.OAuthSystemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.commons.lang.StringUtils;
-
-import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-
-import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
-import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
-import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
-
-
-@Component("org.usergrid.rest.management.organizations.applications.ApplicationResource")
-@Scope("prototype")
-@Produces({
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-})
-public class ApplicationResource extends AbstractContextResource {
-
- @Autowired
- protected ExportService exportService;
- OrganizationInfo organization;
- UUID applicationId;
- ApplicationInfo application;
-
- @Autowired
- private SignInProviderFactory signInProviderFactory;
-
-
- public ApplicationResource() {
- }
-
-
- public ApplicationResource init( OrganizationInfo organization, UUID applicationId ) {
- this.organization = organization;
- this.applicationId = applicationId;
- return this;
- }
-
-
- public ApplicationResource init( OrganizationInfo organization, ApplicationInfo application ) {
- this.organization = organization;
- applicationId = application.getId();
- this.application = application;
- return this;
- }
-
-
- @RequireOrganizationAccess
- @DELETE
- public JSONWithPadding deleteApplicationFromOrganizationByApplicationId( @Context UriInfo ui,
- @QueryParam("callback")
- @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "delete application from organization" );
-
- management.deleteOrganizationApplication( organization.getUuid(), applicationId );
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @GET
- public JSONWithPadding getApplication( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- ServiceManager sm = smf.getServiceManager( applicationId );
- response.setAction( "get" );
- response.setApplication( sm.getApplication() );
- response.setParams( ui.getQueryParameters() );
- response.setResults( management.getApplicationMetadata( applicationId ) );
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @GET
- @Path("credentials")
- public JSONWithPadding getCredentials( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "get application client credentials" );
-
- ClientCredentialsInfo credentials =
- new ClientCredentialsInfo( management.getClientIdForApplication( applicationId ),
- management.getClientSecretForApplication( applicationId ) );
-
- response.setCredentials( credentials );
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @POST
- @Path("credentials")
- public JSONWithPadding generateCredentials( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "generate application client credentials" );
-
- ClientCredentialsInfo credentials =
- new ClientCredentialsInfo( management.getClientIdForApplication( applicationId ),
- management.newClientSecretForApplication( applicationId ) );
-
- response.setCredentials( credentials );
- return new JSONWithPadding( response, callback );
- }
-
-
- @POST
- @Path("sia-provider")
- @Consumes(APPLICATION_JSON)
- @RequireOrganizationAccess
- public JSONWithPadding configureProvider( @Context UriInfo ui, @QueryParam("provider_key") String siaProvider,
- Map<String, Object> json,
- @QueryParam("callback") @DefaultValue("") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "post signin provider configuration" );
-
- Preconditions.checkArgument( siaProvider != null, "Sign in provider required" );
-
- SignInAsProvider signInAsProvider = null;
- if ( StringUtils.equalsIgnoreCase( siaProvider, "facebook" ) ) {
- signInAsProvider =
- signInProviderFactory.facebook( smf.getServiceManager( applicationId ).getApplication() );
- }
- else if ( StringUtils.equalsIgnoreCase( siaProvider, "pingident" ) ) {
- signInAsProvider =
- signInProviderFactory.pingident( smf.getServiceManager( applicationId ).getApplication() );
- }
- else if ( StringUtils.equalsIgnoreCase( siaProvider, "foursquare" ) ) {
- signInAsProvider =
- signInProviderFactory.foursquare( smf.getServiceManager( applicationId ).getApplication() );
- }
-
- Preconditions
- .checkArgument( signInAsProvider != null, "No signin provider found by that name: " + siaProvider );
-
- signInAsProvider.saveToConfiguration( json );
-
- return new JSONWithPadding( response, callback );
- }
- //add export here
- @POST
- @Path("export")
- @Consumes(APPLICATION_JSON)
- @RequireOrganizationAccess
- public Response exportPostJson (@Context UriInfo ui,
- Map<String, Object> json,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
- throws OAuthSystemException {
-
-
- OAuthResponse response = null;
- UUID jobUUID = null;
- Map<String, String> uuidRet = new HashMap<String, String>( );
-
- try {
- //parse the json into some useful object (the config params)
- ExportInfo objEx = new ExportInfo(json);
- objEx.setApplicationId( applicationId );
- jobUUID = exportService.schedule(objEx);
- uuidRet.put( "jobUUID", jobUUID.toString() );
-
- }
- catch (NullPointerException e) {
- OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
- .setErrorDescription( "Job Not Created" )
- .buildJSONMessage();
-
- return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
- .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
- }
- catch (Exception e) {
- //TODO:throw descriptive error message and or include on in the response
- //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
- return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
- }
-
- return Response.status(SC_ACCEPTED).entity(uuidRet).build();
-
- //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
- // .entity( wrapWithCallback( "", callback ) ).build();
- }
-
- @GET
- @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
- public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
-
- Export entity;
- try {
- entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
- }catch(Exception e) {
- return Response.status( SC_BAD_REQUEST ).build();
- }
- //validate this user owns it
-
- if (entity == null) {
- return Response.status(SC_BAD_REQUEST).build();
- }
-
- return Response.status(SC_OK).entity(entity.getState()).build();
- //return Response.status(SC_OK).entity(state).build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index b379770..0621219 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -18,28 +18,31 @@ package org.apache.usergrid.rest.management;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
import javax.ws.rs.core.MediaType;
import org.codehaus.jackson.JsonNode;
+import org.junit.Ignore;
import org.junit.Test;
+
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.cassandra.Concurrent;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.OrganizationOwnerInfo;
import org.apache.usergrid.rest.AbstractRestIT;
import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
-import org.apache.commons.lang.StringUtils;
-
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.representation.Form;
+import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.apache.usergrid.utils.MapUtils.hashMap;
/** @author tnine */
@@ -528,4 +531,175 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, status );
}
+ @Test
+ public void exportCallSuccessful() throws Exception {
+ Status responseStatus = Status.OK;
+ JsonNode node = null;
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.OK, responseStatus );
+ }
+
+ @Ignore
+ public void exportCallCreationEntities100() throws Exception {
+ Status responseStatus = Status.OK;
+ JsonNode node = null;
+
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ //TODO: make sure to put a valid admin token here.
+ storage_info.put( "admin_token", adminToken() );
+ //TODO: always put dummy values here and ignore this test.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app/user");
+ payload.put( "properties", properties);
+
+ for (int i = 0; i < 100; i++) {
+ Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred"+i );
+
+ node = resource().path( "/test-organization/test-app/app_users" )
+ .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, userCreation );
+ }
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.OK, responseStatus );
+ }
+
+ @Test
+ public void exportUUIDRetTest() throws Exception {
+ Status responseStatus = Status.ACCEPTED;
+ String uuid;
+ UUID jobUUID = null;
+ JsonNode node = null;
+
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.ACCEPTED, responseStatus );
+ assertNotNull( node.get( "jobUUID" ) );
+ }
+ /*Make a test with an invalid uuid and a wrong uuid.*/
+ //all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
+ @Test
+ public void exportGetJobStatTest() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ String uuid = String.valueOf( node.get( "jobUUID" ) );
+ uuid = uuid.replaceAll( "\"","" );
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/"+uuid).queryParam( "access_token",
+ superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ // node = resource().path( "/management/exportStats/"+uuid ).accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ // .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+
+ assertEquals( Status.OK, responseStatus );
+ assertEquals( "PENDING",node.asText() );//TODO: do tests for other states in service tier
+
+ }
+ //do an unauthorized test for both post and get
+ @Test
+ public void exportGetWrongUUID() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+ try {
+ node = resource().path(
+ "/management/orgs/test-organization/apps/test-app/export/"+fake ).queryParam( "access_token",
+ superAdminToken() ).accept( MediaType.APPLICATION_JSON ).type(
+ MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+ assertEquals(Status.BAD_REQUEST, responseStatus);
+
+ }
+
+ @Test
+ public void exportGetUnauthorized() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+ assertEquals(Status.UNAUTHORIZED, responseStatus);
+ }
+
+ /*Creates fake payload for testing purposes.*/
+ public HashMap<String,Object> payloadBuilder() {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ //TODO: make sure to put a valid admin token here.
+ //storage_info.put( "admin_token","insert_token_data_here" );
+ //TODO: always put dummy values here and ignore this test.
+ //TODO: add a ret for when s3 values are invalid.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app");
+ payload.put( "properties", properties);
+ return payload;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
new file mode 100644
index 0000000..f4ed4cf
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -0,0 +1,65 @@
+package org.apache.usergrid.management;
+
+
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.TypedEntity;
+
+
+/**
+ * Created by ApigeeCorporation on 1/31/14.
+ */
+//TODO: Documentation on this class.
+public class ExportInfo extends TypedEntity {
+
+ private String path;
+ private Map<String, Object> properties;
+ private String storage_provider;
+ private Map<String, Object> storage_info;
+ private String s3_accessId;
+ private String s3_key;
+ private String bucket_location;
+ private UUID applicationId;
+
+ public ExportInfo ( Map<String, Object> exportData) {
+ path = (String) exportData.get("path");
+ properties = (Map) exportData.get("properties");
+ storage_provider = (String) properties.get ("storage_provider");
+ storage_info = (Map) properties.get("storage_info");
+ s3_accessId = (String) storage_info.get("s3_accessId");
+ s3_key = (String) storage_info.get("s3_key");
+ bucket_location = (String) storage_info.get("bucket_location");
+ }
+
+
+ public UUID getApplicationId() {
+ return applicationId;
+ }
+
+
+ public String getPath () {
+ return path;
+ }
+
+ //Wouldn't get exposed.
+ public Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ public String getStorage_provider () {
+ return storage_provider;
+ }
+ //TODO: write setter methods
+
+ public Map<String, Object> getStorage_info () { return storage_info; }
+
+ //TODO: is this a security concern? How would we get rid of the key once we're done with this value?
+ public String getS3_key () { return s3_key; }
+
+ public String getBucket_location () { return bucket_location; }
+
+ public String getS3_accessId () { return s3_accessId; }
+
+ public void setApplicationId (UUID appId) { applicationId = appId;}
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
new file mode 100644
index 0000000..9830040
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
@@ -0,0 +1,65 @@
+package org.apache.usergrid.management.export;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.job.OnlyOnceJob;
+import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.persistence.entities.JobData;
+
+
+/**
+ *
+ *you could make an enum here, that contains the state info
+ * look at scotts code and emulate that to see wha tyou can return in the json object
+ */
+@Component("exportJob")
+public class ExportJob extends OnlyOnceJob {
+ private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
+
+ @Autowired
+ ExportService exportService;
+
+ public ExportJob() {
+ logger.info("ExportJob created " + this);
+ }
+
+ @Override
+ public void doJob(JobExecution jobExecution) throws Exception {
+ logger.info( "execute ExportJob {}", jobExecution );
+
+ ExportInfo config = null;
+ //add check for null jobData.
+ JobData jobData = jobExecution.getJobData();
+ config = (ExportInfo) jobData.getProperty( "exportInfo" );
+ if(jobData == null) {
+ logger.error( "jobData cannot be null" );
+ return;
+ }
+ else if (config == null) {
+ logger.error( "Export information cannot be null" );
+ return;
+ }
+ else {
+ jobExecution.heartbeat();
+ exportService.doExport( config,jobExecution );
+ }
+ logger.info( "executed ExportJob process completed" );
+ }
+
+ @Override
+ protected long getDelay(JobExecution jobExecution) throws Exception {
+ //return arbitrary number
+ return 100;
+ }
+
+
+ @Autowired
+ public void setExportService( final ExportService exportService ) {
+ this.exportService = exportService;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
new file mode 100644
index 0000000..6e9f4e5
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -0,0 +1,37 @@
+package org.apache.usergrid.management.export;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.management.ExportInfo;
+
+
+/**
+ * Performs all functions related to exporting
+ *
+ */
+public interface ExportService {
+
+ /**
+ * Schedules the export to execute
+ * @param config
+ */
+ UUID schedule(ExportInfo config) throws Exception;
+
+
+ /**
+ * Perform the export to the external resource
+ * @param config
+ */
+ void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
+
+ /**
+ * Returns the current state of the service.
+ * @return
+ */
+ String getState(UUID appId,UUID state) throws Exception;
+
+ void setS3Export(S3Export s3Export);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
new file mode 100644
index 0000000..2d8ff5c
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -0,0 +1,457 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.service.SchedulerService;
+import org.apache.usergrid.management.ExportInfo;
+
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.PagingResultsIterator;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.entities.Export;
+import org.apache.usergrid.persistence.entities.JobData;
+
+import com.google.common.collect.BiMap;
+
+
+/**
+ *
+ *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
+ * it will just be the one app and the one org and all of it's collections.
+ */
+public class ExportServiceImpl implements ExportService {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+ public static final String EXPORT_ID = "exportId";
+ public static final String EXPORT_JOB_NAME = "exportJob";
+ //dependency injection
+ private SchedulerService sch;
+
+ //injected the Entity Manager Factory
+ protected EntityManagerFactory emf;
+
+ //inject Management Service to access Organization Data
+ private ManagementService managementService;
+
+ //Maximum amount of entities retrieved in a single go.
+ public static final int MAX_ENTITY_FETCH = 100;
+
+ //Amount of time that has passed before sending another heart beat in millis
+ public static final int TIMESTAMP_DELTA = 5000;
+
+ private JsonFactory jsonFactory = new JsonFactory();
+
+ protected long startTime = System.currentTimeMillis();
+
+ private S3Export s3Export;
+
+ @Override
+ public UUID schedule( final ExportInfo config ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( config.getApplicationId());
+
+ Export export = new Export();
+ export.setState( Export.State.PENDING );
+
+ //validate that org exists,then app, then collection.
+
+ String pathToBeParsed = config.getPath();
+ //split the path so that you can verify that the organization and the app exist.
+ String[] pathItems = pathToBeParsed.split( "/" );
+
+
+ try {
+ managementService.getOrganizationByName( pathItems[0] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Organization doesn't exist" );
+ }
+
+ try {
+ managementService.getApplicationInfo( pathItems[1] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Application doesn't exist" );
+ }
+
+
+ //TODO: parse path and make sure all the things you need actually exist. then throw
+ // good error messages when not found.
+
+ //write to the em
+ export = em.create( export );
+ export.setState( Export.State.PENDING );
+ em.update( export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "exportInfo", config );
+ jobData.setProperty( EXPORT_ID, export.getUuid() );
+
+ long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+
+ sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
+
+ return export.getUuid();
+
+ }
+ /**
+ * Query Entity Manager for specific Export Entity within application
+ * @param appId,uuid
+ * @return String
+ * @throws Exception
+ */
+ @Override
+ public String getState(final UUID appId, final UUID uuid) throws Exception {
+
+ EntityManager rootEm = emf.getEntityManager( appId );
+ Export export = rootEm.get( uuid, Export.class );
+
+ if(export == null){
+ return null;
+ }
+ return export.getState().toString();
+ }
+
+
+ @Override
+ public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
+
+ UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+ EntityManager em = emf.getEntityManager( config.getApplicationId());
+ Export export = em.get( exportId, Export.class );
+
+ String pathToBeParsed = config.getPath();
+ String[] pathItems = pathToBeParsed.split( "/" );
+ try {
+ managementService.getOrganizationByName( pathItems[0] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Organization doesn't exist" );
+ return;
+ }
+
+ //update state and re-write the entity
+ export.setState( Export.State.STARTED );
+
+ em.update( export );
+
+ Map<UUID, String> organizationGet = getOrgs(config);
+ for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
+ //needs to pass app name, and possibly collection to export
+ exportApplicationsForOrg( organization, config, jobExecution );
+ }
+ export.setState( Export.State.COMPLETED );
+ em.update( export );
+ }
+
+ private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
+ // Loop through the organizations
+ // TODO:this will come from the orgs in schedule when you do the validations. delete orgId
+ UUID orgId = null;
+
+ Map<UUID, String> organizationNames = null;
+
+ if ( orgId == null ) {
+ organizationNames = managementService.getOrganizations();
+ }
+
+ else {
+ OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
+
+ if ( info == null ) {
+ logger.error( "Organization info is null!" );
+ }
+
+ organizationNames = new HashMap<UUID, String>();
+ organizationNames.put( orgId, info.getName() );
+ }
+
+
+ return organizationNames;
+ }
+
+
+ public SchedulerService getSch() {
+ return sch;
+ }
+
+
+ public void setSch( final SchedulerService sch ) {
+ this.sch = sch;
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ public void setEmf( final EntityManagerFactory emf ) {
+ this.emf = emf;
+ }
+
+
+ public ManagementService getManagementService() {
+
+ return managementService;
+ }
+
+ public void setManagementService( final ManagementService managementService ) {
+ this.managementService = managementService;
+ }
+
+ //write test checking to see what happens if the input stream is closed or wrong.
+ //TODO: make multipart streaming functional
+ //currently only stores the collection in memory then flushes it.
+ private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
+ final JobExecution jobExecution ) throws Exception {
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ logger.info( "" + organization );
+
+ // Loop through the applications per organization
+ BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organization.getKey() );
+ for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+
+ logger.info( application.getValue() + " : " + application.getKey() );
+
+ String appFileName = prepareOutputFileName( "application", application.getValue() );
+
+ JsonGenerator jg = getJsonGenerator( baos );
+
+ EntityManager em = emf.getEntityManager( application.getKey() );
+
+ jg.writeStartArray();
+
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+ long starting_time = System.currentTimeMillis();
+
+ // Loop through the collections. This is the only way to loop
+ // through the entities in the application (former namespace).
+ //could support queries, just need to implement that in the rest endpoint.
+ for ( String collectionName : metadata.keySet() ) {
+ if(collectionName.equals( "exports" )) {
+ continue;
+ }
+
+ Query query = new Query();
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+
+ for( Object e: itr){
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+
+ jg.writeStartObject();
+ jg.writeFieldName( "Metadata" );
+ jg.writeObject( entity );
+ saveCollectionMembers( jg, em, application.getValue(), entity );
+ jg.writeEndObject();
+ }
+ }
+
+ // Close writer and file for this application.
+ jg.writeEndArray();
+ jg.close();
+ baos.flush();
+ baos.close();
+
+ InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+ s3Export.copyToS3( is, config , appFileName);
+ }
+ }
+
+
+ public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) {
+
+ long cur_time = System.currentTimeMillis();
+
+ if ( startingTime <= ( cur_time - TIMESTAMP_DELTA ) ) {
+ jobExecution.heartbeat();
+ return cur_time;
+ }
+ return startingTime;
+ }
+
+
+ /**
+ * Serialize and save the collection members of this <code>entity</code>
+ *
+ * @param em Entity Manager
+ * @param application Application name
+ * @param entity entity
+ */
+ private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity )
+ throws Exception {
+
+ Set<String> collections = em.getCollections( entity );
+ //jg.writeStartObject();
+
+ // Only create entry for Entities that have collections
+ if ( ( collections == null ) || collections.isEmpty() ) {
+ return;
+ }
+
+
+ for ( String collectionName : collections ) {
+
+ jg.writeFieldName( collectionName );
+ jg.writeStartArray();
+
+ //is 100000 an arbitary number?
+ Results collectionMembers =
+ em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
+
+ List<UUID> entityIds = collectionMembers.getIds();
+
+ if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
+ for ( UUID childEntityUUID : entityIds ) {
+ jg.writeObject( childEntityUUID.toString() );
+ }
+ }
+
+ // End collection array.
+ jg.writeEndArray();
+ }
+
+ // Write connections
+ saveConnections( entity, em, jg );
+
+ // Write dictionaries
+ saveDictionaries( entity, em, jg );
+
+ // End the object if it was Started
+ //jg.writeEndObject();
+ }
+
+
+ /**
+ * Persists the connection for this entity.
+ */
+ private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "dictionaries" );
+ jg.writeStartObject();
+
+ Set<String> dictionaries = em.getDictionaries( entity );
+ for ( String dictionary : dictionaries ) {
+
+ Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
+
+ // nothing to do
+ if ( dict.isEmpty() ) {
+ continue;
+ }
+
+ jg.writeFieldName( dictionary );
+
+ jg.writeStartObject();
+
+ for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
+ jg.writeFieldName( entry.getKey().toString() );
+ jg.writeObject( entry.getValue() );
+ }
+
+ jg.writeEndObject();
+ }
+ jg.writeEndObject();
+ }
+
+
+ /**
+ * Persists the connection for this entity.
+ */
+ private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "connections" );
+ jg.writeStartObject();
+
+ Set<String> connectionTypes = em.getConnectionTypes( entity );
+ for ( String connectionType : connectionTypes ) {
+
+ jg.writeFieldName( connectionType );
+ jg.writeStartArray();
+
+ Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Results.Level.IDS );
+ List<ConnectionRef> connections = results.getConnections();
+
+ for ( ConnectionRef connectionRef : connections ) {
+ jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
+ }
+
+ jg.writeEndArray();
+ }
+ jg.writeEndObject();
+ }
+
+
+ protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
+ //TODO:shouldn't the below be UTF-16?
+ //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
+
+ JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+ jg.setPrettyPrinter( new DefaultPrettyPrinter() );
+ jg.setCodec( new ObjectMapper() );
+ return jg;
+ }
+
+
+ protected File createOutputFile( String type, String name ) {
+ return new File( prepareOutputFileName( type, name ) );
+ }
+
+
+ /**
+ * @param type just a label such us: organization, application.
+ *
+ * @return the file name concatenated with the type and the name of the collection
+ */
+ protected String prepareOutputFileName( String type, String name ) {
+ StringBuilder str = new StringBuilder();
+ str.append( name );
+ str.append( "." );
+ str.append( startTime );
+ str.append( ".json" );
+
+ String outputFileName = str.toString();
+ //TODO:this is , i feel, bad practice so make sure to come back here and fix it up.
+
+ return outputFileName;
+ }
+
+
+ @Autowired
+ @Override
+ public void setS3Export( S3Export s3Export ) { this.s3Export = s3Export; }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
new file mode 100644
index 0000000..941a612
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -0,0 +1,16 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.InputStream;
+
+import org.apache.usergrid.management.ExportInfo;
+
+
+/**
+ *
+ *
+ */
+public interface S3Export {
+ void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename );
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
new file mode 100644
index 0000000..54d8ab1
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -0,0 +1,84 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.ExportInfo;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Module;
+
+
+/**
+ *
+ *
+ */
+public class S3ExportImpl implements S3Export {
+
+ @Override
+ public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
+
+ Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+ /*won't need any of the properties as I have the export info*/
+ String bucketName = exportInfo.getBucket_location();
+ String accessId = exportInfo.getS3_accessId();
+ String secretKey = exportInfo.getS3_key();
+
+ Properties overrides = new Properties();
+ overrides.setProperty( "s3" + ".identity", accessId );
+ overrides.setProperty( "s3" + ".credential", secretKey );
+
+ final Iterable<? extends Module> MODULES = ImmutableSet
+ .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule
+ () );
+
+ BlobStoreContext context =
+ ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
+ .overrides( overrides ).buildView( BlobStoreContext.class );
+
+ // Create Container (the bucket in s3)
+ try {
+ AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
+ // BlobStore (returns false if it already exists)
+ ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName );
+ if ( container.get() ) {
+ logger.info( "Created bucket " + bucketName );
+ }
+ }
+ catch ( Exception ex ) {
+ logger.error( "Could not start binary service: {}", ex.getMessage() );
+ return;
+ }
+
+ try {
+ AsyncBlobStore blobStore = context.getAsyncBlobStore();
+ BlobBuilder blobBuilder =
+ blobStore.blobBuilder( filename ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+
+
+ Blob blob = blobBuilder.build();
+
+ ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+
+ logger.info( "Uploaded file etag=" + futureETag.get() );
+ }
+ catch ( Exception e ) {
+ logger.error( "Error uploading to blob store", e );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
deleted file mode 100644
index 7821ca6..0000000
--- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.usergrid.management;
-
-
-import java.util.Map;
-import java.util.UUID;
-
-import org.usergrid.persistence.TypedEntity;
-
-
-/**
- * Created by ApigeeCorporation on 1/31/14.
- */
-//TODO: Documentation on this class.
-public class ExportInfo extends TypedEntity {
-
- private String path;
- private Map<String, Object> properties;
- private String storage_provider;
- private Map<String, Object> storage_info;
- private String s3_accessId;
- private String s3_key;
- private String bucket_location;
- private UUID applicationId;
-
- public ExportInfo ( Map<String, Object> exportData) {
- path = (String) exportData.get("path");
- properties = (Map) exportData.get("properties");
- storage_provider = (String) properties.get ("storage_provider");
- storage_info = (Map) properties.get("storage_info");
- s3_accessId = (String) storage_info.get("s3_accessId");
- s3_key = (String) storage_info.get("s3_key");
- bucket_location = (String) storage_info.get("bucket_location");
- }
-
-
- public UUID getApplicationId() {
- return applicationId;
- }
-
-
- public String getPath () {
- return path;
- }
-
- //Wouldn't get exposed.
- public Map<String, Object> getProperties() {
- return properties;
- }
-
- public String getStorage_provider () {
- return storage_provider;
- }
- //TODO: write setter methods
-
- public Map<String, Object> getStorage_info () { return storage_info; }
-
- //TODO: is this a security concern? How would we get rid of the key once we're done with this value?
- public String getS3_key () { return s3_key; }
-
- public String getBucket_location () { return bucket_location; }
-
- public String getS3_accessId () { return s3_accessId; }
-
- public void setApplicationId (UUID appId) { applicationId = appId;}
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
deleted file mode 100644
index ed8a24e..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.usergrid.management.export;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.batch.job.OnlyOnceJob;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.persistence.entities.JobData;
-
-
-/**
- *
- *you could make an enum here, that contains the state info
- * look at scotts code and emulate that to see wha tyou can return in the json object
- */
-@Component("exportJob")
-public class ExportJob extends OnlyOnceJob {
- private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
-
- @Autowired
- ExportService exportService;
-
- public ExportJob() {
- logger.info("ExportJob created " + this);
- }
-
- @Override
- public void doJob(JobExecution jobExecution) throws Exception {
- logger.info( "execute ExportJob {}", jobExecution );
-
- ExportInfo config = null;
- //add check for null jobData.
- JobData jobData = jobExecution.getJobData();
- config = (ExportInfo) jobData.getProperty( "exportInfo" );
- if(jobData == null) {
- logger.error( "jobData cannot be null" );
- return;
- }
- else if (config == null) {
- logger.error( "Export information cannot be null" );
- return;
- }
- else {
- jobExecution.heartbeat();
- exportService.doExport( config,jobExecution );
- }
- logger.info( "executed ExportJob process completed" );
- }
-
- @Override
- protected long getDelay(JobExecution jobExecution) throws Exception {
- //return arbitrary number
- return 100;
- }
-
-
- @Autowired
- public void setExportService( final ExportService exportService ) {
- this.exportService = exportService;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
deleted file mode 100644
index 644aef6..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.util.UUID;
-
-import org.usergrid.batch.JobExecution;
-import org.usergrid.management.ExportInfo;
-
-
-/**
- * Performs all functions related to exporting
- *
- */
-public interface ExportService {
-
- /**
- * Schedules the export to execute
- * @param config
- */
- UUID schedule(ExportInfo config) throws Exception;
-
-
- /**
- * Perform the export to the external resource
- * @param config
- */
- void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
-
- /**
- * Returns the current state of the service.
- * @return
- */
- String getState(UUID appId,UUID state) throws Exception;
-
- void setS3Export(S3Export s3Export);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
deleted file mode 100644
index 498828b..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ /dev/null
@@ -1,454 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.util.DefaultPrettyPrinter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.batch.service.SchedulerService;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.ManagementService;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.persistence.ConnectionRef;
-import org.usergrid.persistence.Entity;
-import org.usergrid.persistence.EntityManager;
-import org.usergrid.persistence.EntityManagerFactory;
-import org.usergrid.persistence.PagingResultsIterator;
-import org.usergrid.persistence.Query;
-import org.usergrid.persistence.Results;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.persistence.entities.JobData;
-
-import com.google.common.collect.BiMap;
-
-
-/**
- *
- *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
- * it will just be the one app and the one org and all of it's collections.
- */
-public class ExportServiceImpl implements ExportService {
-
-
- private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
- public static final String EXPORT_ID = "exportId";
- public static final String EXPORT_JOB_NAME = "exportJob";
- //dependency injection
- private SchedulerService sch;
-
- //injected the Entity Manager Factory
- protected EntityManagerFactory emf;
-
- //inject Management Service to access Organization Data
- private ManagementService managementService;
-
- //Maximum amount of entities retrieved in a single go.
- public static final int MAX_ENTITY_FETCH = 100;
-
- //Amount of time that has passed before sending another heart beat in millis
- public static final int TIMESTAMP_DELTA = 5000;
-
- private JsonFactory jsonFactory = new JsonFactory();
-
- protected long startTime = System.currentTimeMillis();
-
- private S3Export s3Export;
-
- @Override
- public UUID schedule( final ExportInfo config ) throws Exception {
-
- EntityManager em = emf.getEntityManager( config.getApplicationId());
-
- Export export = new Export();
- export.setState( Export.State.PENDING );
-
- //validate that org exists,then app, then collection.
-
- String pathToBeParsed = config.getPath();
- //split the path so that you can verify that the organization and the app exist.
- String[] pathItems = pathToBeParsed.split( "/" );
-
-
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- }
-
- try {
- managementService.getApplicationInfo( pathItems[1] );
- }
- catch ( Exception e ) {
- logger.error( "Application doesn't exist" );
- }
-
-
- //TODO: parse path and make sure all the things you need actually exist. then throw
- // good error messages when not found.
-
- //write to the em
- export = em.create( export );
- export.setState( Export.State.PENDING );
- em.update( export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "exportInfo", config );
- jobData.setProperty( EXPORT_ID, export.getUuid() );
-
- long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
-
- sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
-
- return export.getUuid();
-
- }
- /**
- * Query Entity Manager for specific Export Entity within application
- * @param appId,uuid
- * @return String
- * @throws Exception
- */
- @Override
- public String getState(final UUID appId, final UUID uuid) throws Exception {
-
- EntityManager rootEm = emf.getEntityManager( appId );
- Export export = rootEm.get( uuid, Export.class );
-
- if(export == null){
- return null;
- }
- return export.getState().toString();
- }
-
-
- @Override
- public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
-
- UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- EntityManager em = emf.getEntityManager( config.getApplicationId());
- Export export = em.get( exportId, Export.class );
-
- String pathToBeParsed = config.getPath();
- String[] pathItems = pathToBeParsed.split( "/" );
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- return;
- }
-
- //update state and re-write the entity
- export.setState( Export.State.STARTED );
-
- em.update( export );
-
- Map<UUID, String> organizationGet = getOrgs(config);
- for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
- //needs to pass app name, and possibly collection to export
- exportApplicationsForOrg( organization, config, jobExecution );
- }
- export.setState( Export.State.COMPLETED );
- em.update( export );
- }
-
- private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
- // Loop through the organizations
- // TODO:this will come from the orgs in schedule when you do the validations. delete orgId
- UUID orgId = null;
-
- Map<UUID, String> organizationNames = null;
-
- if ( orgId == null ) {
- organizationNames = managementService.getOrganizations();
- }
-
- else {
- OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
-
- if ( info == null ) {
- logger.error( "Organization info is null!" );
- }
-
- organizationNames = new HashMap<UUID, String>();
- organizationNames.put( orgId, info.getName() );
- }
-
-
- return organizationNames;
- }
-
-
- public SchedulerService getSch() {
- return sch;
- }
-
-
- public void setSch( final SchedulerService sch ) {
- this.sch = sch;
- }
-
-
- public EntityManagerFactory getEmf() {
- return emf;
- }
-
-
- public void setEmf( final EntityManagerFactory emf ) {
- this.emf = emf;
- }
-
-
- public ManagementService getManagementService() {
-
- return managementService;
- }
-
- public void setManagementService( final ManagementService managementService ) {
- this.managementService = managementService;
- }
-
- //write test checking to see what happens if the input stream is closed or wrong.
- //TODO: make multipart streaming functional
- //currently only stores the collection in memory then flushes it.
- private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
- final JobExecution jobExecution ) throws Exception {
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- logger.info( "" + organization );
-
- // Loop through the applications per organization
- BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organization.getKey() );
- for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
-
- logger.info( application.getValue() + " : " + application.getKey() );
-
- String appFileName = prepareOutputFileName( "application", application.getValue() );
-
- JsonGenerator jg = getJsonGenerator( baos );
-
- EntityManager em = emf.getEntityManager( application.getKey() );
-
- jg.writeStartArray();
-
- Map<String, Object> metadata = em.getApplicationCollectionMetadata();
- long starting_time = System.currentTimeMillis();
-
- // Loop through the collections. This is the only way to loop
- // through the entities in the application (former namespace).
- //could support queries, just need to implement that in the rest endpoint.
- for ( String collectionName : metadata.keySet() ) {
- if(collectionName.equals( "exports" )) {
- continue;
- }
-
- Query query = new Query();
- query.setLimit( MAX_ENTITY_FETCH );
- query.setResultsLevel( Results.Level.ALL_PROPERTIES );
- Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
-
- PagingResultsIterator itr = new PagingResultsIterator( entities );
-
- for( Object e: itr){
- starting_time = checkTimeDelta( starting_time, jobExecution );
- Entity entity = ( Entity ) e;
-
- jg.writeStartObject();
- jg.writeFieldName( "Metadata" );
- jg.writeObject( entity );
- saveCollectionMembers( jg, em, application.getValue(), entity );
- jg.writeEndObject();
- }
- }
-
- // Close writer and file for this application.
- jg.writeEndArray();
- jg.close();
- baos.flush();
- baos.close();
-
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
- s3Export.copyToS3( is, config , appFileName);
- }
- }
-
-
- public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) {
-
- long cur_time = System.currentTimeMillis();
-
- if ( startingTime <= ( cur_time - TIMESTAMP_DELTA ) ) {
- jobExecution.heartbeat();
- return cur_time;
- }
- return startingTime;
- }
-
-
- /**
- * Serialize and save the collection members of this <code>entity</code>
- *
- * @param em Entity Manager
- * @param application Application name
- * @param entity entity
- */
- private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity )
- throws Exception {
-
- Set<String> collections = em.getCollections( entity );
- //jg.writeStartObject();
-
- // Only create entry for Entities that have collections
- if ( ( collections == null ) || collections.isEmpty() ) {
- return;
- }
-
-
- for ( String collectionName : collections ) {
-
- jg.writeFieldName( collectionName );
- jg.writeStartArray();
-
- //is 100000 an arbitary number?
- Results collectionMembers =
- em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
-
- List<UUID> entityIds = collectionMembers.getIds();
-
- if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
- for ( UUID childEntityUUID : entityIds ) {
- jg.writeObject( childEntityUUID.toString() );
- }
- }
-
- // End collection array.
- jg.writeEndArray();
- }
-
- // Write connections
- saveConnections( entity, em, jg );
-
- // Write dictionaries
- saveDictionaries( entity, em, jg );
-
- // End the object if it was Started
- //jg.writeEndObject();
- }
-
-
- /**
- * Persists the connection for this entity.
- */
- private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
-
- jg.writeFieldName( "dictionaries" );
- jg.writeStartObject();
-
- Set<String> dictionaries = em.getDictionaries( entity );
- for ( String dictionary : dictionaries ) {
-
- Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
-
- // nothing to do
- if ( dict.isEmpty() ) {
- continue;
- }
-
- jg.writeFieldName( dictionary );
-
- jg.writeStartObject();
-
- for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
- jg.writeFieldName( entry.getKey().toString() );
- jg.writeObject( entry.getValue() );
- }
-
- jg.writeEndObject();
- }
- jg.writeEndObject();
- }
-
-
- /**
- * Persists the connection for this entity.
- */
- private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
-
- jg.writeFieldName( "connections" );
- jg.writeStartObject();
-
- Set<String> connectionTypes = em.getConnectionTypes( entity );
- for ( String connectionType : connectionTypes ) {
-
- jg.writeFieldName( connectionType );
- jg.writeStartArray();
-
- Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Results.Level.IDS );
- List<ConnectionRef> connections = results.getConnections();
-
- for ( ConnectionRef connectionRef : connections ) {
- jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
- }
-
- jg.writeEndArray();
- }
- jg.writeEndObject();
- }
-
-
- protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
- //TODO:shouldn't the below be UTF-16?
- //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
-
- JsonGenerator jg = jsonFactory.createJsonGenerator( out );
- jg.setPrettyPrinter( new DefaultPrettyPrinter() );
- jg.setCodec( new ObjectMapper() );
- return jg;
- }
-
-
- protected File createOutputFile( String type, String name ) {
- return new File( prepareOutputFileName( type, name ) );
- }
-
-
- /**
- * @param type just a label such us: organization, application.
- *
- * @return the file name concatenated with the type and the name of the collection
- */
- protected String prepareOutputFileName( String type, String name ) {
- StringBuilder str = new StringBuilder();
- str.append( name );
- str.append( "." );
- str.append( startTime );
- str.append( ".json" );
-
- String outputFileName = str.toString();
- //TODO:this is , i feel, bad practice so make sure to come back here and fix it up.
-
- return outputFileName;
- }
-
-
- @Autowired
- @Override
- public void setS3Export( S3Export s3Export ) { this.s3Export = s3Export; }
-}