You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/02/04 02:44:09 UTC
incubator-usergrid git commit: First pass at creating the interface
for errors
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-import 68894d96e -> ec9356914
First pass at creating the interface for errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ec935691
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ec935691
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ec935691
Branch: refs/heads/two-dot-o-import
Commit: ec935691404b4980e910685ff3c8a3b072765e1d
Parents: 68894d9
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Feb 3 18:44:01 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Feb 3 18:44:01 2015 -0700
----------------------------------------------------------------------
.../persistence/entities/FileImport.java | 31 ++++
.../importer/FileImportStatistics.java | 154 +++++++++++++++++++
2 files changed, 185 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec935691/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
index 815222a..66a0937 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
@@ -61,6 +61,18 @@ public class FileImport extends TypedEntity {
@EntityProperty
protected String lastUpdatedUUID;
+
+
+ /**
+ * File completion Status
+ */
+ @EntityProperty
+ protected long importedEntityCount;
+
+ @EntityProperty
+ protected long failedEntityCount;
+
+
public FileImport() {
}
@@ -148,4 +160,23 @@ public class FileImport extends TypedEntity {
this.fileName = fileName;
}
+
+ public long getImportedEntityCount() {
+ return importedEntityCount;
+ }
+
+
+ public void setImportedEntityCount( final long importedEntityCount ) {
+ this.importedEntityCount = importedEntityCount;
+ }
+
+
+ public long getFailedEntityCount() {
+ return failedEntityCount;
+ }
+
+
+ public void setFailedEntityCount( final long failedEntityCount ) {
+ this.failedEntityCount = failedEntityCount;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec935691/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
new file mode 100644
index 0000000..45a8371
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.management.importer;
+
+
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.entities.FileImport;
+import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
+
+
+/**
+ * Statistics used to track a file import. Only 1 instance of this class should exist per file imported in the cluster.
+ * There is a direct 1-1 mapping of the statistics provided here and the file import status
+ */
+public class FileImportStatistics {
+
+ private final AtomicLong entitiesWritten = new AtomicLong( 0 );
+ private final AtomicLong entitiesFailed = new AtomicLong( 0 );
+
+
+ private final UUID fileImportId;
+ private final EntityManager entityManager;
+
+
+ public FileImportStatistics( final UUID fileImportId, final EntityManager entityManager ) {
+ this.fileImportId = fileImportId;
+ this.entityManager = entityManager;
+ }
+
+
+ /**
+ * Invoke when an entity has been successfully written
+ */
+ public void entityWritten() {
+
+ entitiesWritten.incrementAndGet();
+ }
+
+
+ /**
+ * Invoke when an entity fails to write correctly
+ */
+
+ public void entityFailed( final String message ) {
+ entitiesFailed.incrementAndGet();
+ }
+
+
+ /**
+ * Invoke when the file is completed processing
+ */
+ public void complete() {
+
+ final long failed = entitiesFailed.get();
+ final long written = entitiesWritten.get();
+ final FileImport.State state;
+ final String message;
+
+ if ( failed > 0 ) {
+ state = FileImport.State.FAILED;
+ message = "Successfully imported " + written + " entities";
+ }
+ else {
+ state = FileImport.State.FINISHED;
+ message = "Failed to import " + failed + " entities. Successfully imported " + written + " entities";
+ }
+
+ updateFileImport( written, failed, state, message );
+ }
+
+
+ /**
+ * Invoke when we halt the import with a fatal error that cannot be recovered.
+ */
+ public void fatal( final String message ) {
+
+ final long failed = entitiesFailed.get();
+ final long written = entitiesWritten.get();
+ final FileImport.State state;
+
+ if ( failed > 0 ) {
+ state = FileImport.State.FAILED;
+ }
+ else {
+ state = FileImport.State.FINISHED;
+ }
+
+ updateFileImport( written, failed, state, message );
+ }
+
+
+ /**
+ * Update the file import status with the provided messages
+ *
+ * @param written The number of files written
+ * @param failed The number of files failed
+ * @param state The state to set into the import
+ * @param message The message to set
+ */
+ private void updateFileImport( final long written, final long failed, final FileImport.State state,
+ final String message ) {
+
+ try {
+ FileImport fileImport = entityManager.get( fileImportId, FileImport.class );
+
+ if ( fileImport == null ) {
+ throw new EntityNotFoundException( "Could not file FileImport with id " + fileImportId );
+ }
+
+
+ fileImport.setImportedEntityCount( written );
+ fileImport.setFailedEntityCount( failed );
+ fileImport.setState( state );
+ fileImport.setErrorMessage( message );
+
+ entityManager.update( fileImport );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to persist complete state", e );
+ }
+ }
+
+
+ /**
+ * Returns true if we should stop processing. This will use the following logic
+ *
+ * We've attempted to import over 1k entities After 1k, we have over a 50% failure rate
+ */
+ public boolean stopProcessing() {
+
+ //TODO Dave, George. What algorithm should we use here?
+ return false;
+ }
+}