You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/08/25 21:31:45 UTC
usergrid git commit: Add tool that will iterator a collection and
print UUIDs.
Repository: usergrid
Updated Branches:
refs/heads/hotfix-20160819 fe6bea2bc -> a7639d553
Add tool that will iterator a collection and print UUIDs.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a7639d55
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a7639d55
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a7639d55
Branch: refs/heads/hotfix-20160819
Commit: a7639d55336dd696ea804f166811d6db90225b86
Parents: fe6bea2
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Aug 25 14:31:24 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Aug 25 14:31:24 2016 -0700
----------------------------------------------------------------------
.../usergrid/tools/CollectionIterator.java | 189 +++++++++++++++++++
.../org/apache/usergrid/tools/ExportApp.java | 118 +-----------
.../org/apache/usergrid/tools/ToolBase.java | 2 +
.../usergrid/tools/export/ExportConnection.java | 74 ++++++++
.../usergrid/tools/export/ExportEntity.java | 66 +++++++
5 files changed, 337 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
new file mode 100644
index 0000000..02188cb
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.*;
+import com.google.common.base.Optional;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.astyanax.util.RangeBuilder;
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
+import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
+import org.apache.usergrid.corepersistence.service.CollectionSearch;
+import org.apache.usergrid.corepersistence.service.CollectionService;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.serialization.impl.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.model.entity.*;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+
+import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
+import rx.*;
+
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
+
+
+public class CollectionIterator extends ToolBase {
+
+ private static final Logger logger = LoggerFactory.getLogger( CollectionIterator.class );
+
+ private static final String APPLICATION_ARG = "app";
+
+ private static final String ENTITY_TYPE_ARG = "entityType";
+
+ private EntityManager em;
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = super.createOptions();
+
+
+ Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( true )
+ .withDescription( "application id" ).create( APPLICATION_ARG );
+
+
+ options.addOption( appOption );
+
+ Option collectionOption =
+ OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired( true ).withDescription( "collection name" )
+ .create(ENTITY_TYPE_ARG);
+
+ options.addOption( collectionOption );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+
+ startSpring();
+
+ UUID appToFilter = null;
+ if (!line.getOptionValue(APPLICATION_ARG).isEmpty()) {
+ appToFilter = UUID.fromString(line.getOptionValue(APPLICATION_ARG));
+ }
+
+ String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
+
+ logger.info("Staring Tool: CollectionIterator");
+ logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
+
+ em = emf.getEntityManager( appToFilter );
+ EntityRef headEntity = new SimpleEntityRef("application", appToFilter);
+
+ CollectionService collectionService = injector.getInstance(CollectionService.class);
+ String collectionName = InflectionUtils.pluralize(entityType);
+
+ Query query = new Query();
+ query.setCollection(collectionName);
+ query.setLimit(1000);
+
+ com.google.common.base.Optional<String> queryString = com.google.common.base.Optional.absent();
+
+ CollectionInfo collection = getDefaultSchema().getCollection(headEntity.getType(), collectionName);
+
+ final UUID app = appToFilter;
+
+
+ IdQueryExecutor idQueryExecutor = new IdQueryExecutor(query.getCursor()) {
+ @Override
+ protected rx.Observable<ResultsPage<Id>> buildNewResultsPage(
+ final Optional<String> cursor) {
+
+ final CollectionSearch search =
+ new CollectionSearch(new ApplicationScopeImpl(new SimpleId(app, "application")), new SimpleId(app, "application"), collectionName, collection.getType(), query.getLimit(),
+ queryString, cursor);
+
+ return collectionService.searchCollectionIds(search);
+ }
+ };
+
+ while ( idQueryExecutor.hasNext() ){
+
+ Results results = idQueryExecutor.next();
+
+ List<UUID> ids = results.getIds();
+
+ ids.forEach( uuid -> {
+
+ try {
+ org.apache.usergrid.persistence.Entity retrieved = em.get(new SimpleEntityRef(entityType, uuid));
+
+ long timestamp = 0;
+ String dateString = "NOT TIME-BASED";
+ if (UUIDUtils.isTimeBased(uuid)){
+ timestamp = UUIDUtils.getTimestampInMillis(uuid);
+ Date uuidDate = new Date(timestamp);
+ DateFormat df = new SimpleDateFormat();
+ df.setTimeZone(TimeZone.getTimeZone("GMT"));
+ dateString = df.format(uuidDate) + " GMT";
+ }
+
+ if ( retrieved != null ){
+
+ logger.info("{} - {} - entity data found", uuid, dateString);
+ }else{
+ logger.info("{} - {} - entity data NOT found", uuid, dateString);
+ }
+ } catch (Exception e) {
+ logger.error("{} - exception while trying to load entity data, {} ", uuid, e.getMessage());
+ }
+
+
+ });
+
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
index 985c5f0..db59521 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
@@ -24,6 +24,8 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.tools.export.ExportConnection;
+import org.apache.usergrid.tools.export.ExportEntity;
import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
@@ -195,7 +197,7 @@ public class ExportApp extends ExportingToolBase {
/**
* Emits entities of collection.
*/
- class EntityObservable implements rx.Observable.OnSubscribe<ExportEntity> {
+ private class EntityObservable implements rx.Observable.OnSubscribe<ExportEntity> {
EntityManager em;
String collection;
@@ -264,7 +266,7 @@ public class ExportApp extends ExportingToolBase {
/**
* Emits connections of an entity.
*/
- class ConnectionsObservable implements rx.Observable.OnSubscribe<ExportConnection> {
+ private class ConnectionsObservable implements rx.Observable.OnSubscribe<ExportConnection> {
EntityManager em;
ExportEntity exportEntity;
@@ -330,7 +332,7 @@ public class ExportApp extends ExportingToolBase {
/**
* Writes entities to JSON file.
*/
- class EntityWriteAction implements Action1<ExportEntity> {
+ private class EntityWriteAction implements Action1<ExportEntity> {
public void call(ExportEntity entity) {
@@ -368,7 +370,7 @@ public class ExportApp extends ExportingToolBase {
/**
* Writes connection to JSON file.
*/
- class ConnectionWriteAction implements Action1<ExportConnection> {
+ private class ConnectionWriteAction implements Action1<ExportConnection> {
public void call(ExportConnection conn) {
@@ -432,111 +434,3 @@ public class ExportApp extends ExportingToolBase {
}
}
}
-
-
-/**
- * Represents entity data to be serialized to JSON.
- */
-class ExportEntity {
- private String organization;
- private String application;
- private Entity entity;
- private Map<String, Object> dictionaries;
- public ExportEntity( String organization, String application, Entity entity, Map<String, Object> dictionaries ) {
- this.organization = organization;
- this.application = application;
- this.entity = entity;
- this.dictionaries = dictionaries;
- }
-
- public String getApplication() {
- return application;
- }
-
- public void setApplication(String application) {
- this.application = application;
- }
-
- public Entity getEntity() {
- return entity;
- }
-
- public void setEntity(Entity entity) {
- this.entity = entity;
- }
-
- public Map<String, Object> getDictionaries() {
- return dictionaries;
- }
-
- public void setDictionaries(Map<String, Object> dictionaries) {
- this.dictionaries = dictionaries;
- }
-
- public String getOrganization() {
- return organization;
- }
-
- public void setOrganization(String organization) {
- this.organization = organization;
- }
-}
-
-
-/**
- * Represents connection data to be serialized to JSON.
- */
-class ExportConnection {
- private String organization;
- private String application;
- private String connectionType;
- private UUID sourceUuid;
- private UUID targetUuid;
- public ExportConnection(String organization, String application, String connectionType, UUID sourceUuid, UUID targetUuid) {
- this.organization= organization;
- this.application = application;
- this.connectionType = connectionType;
- this.sourceUuid = sourceUuid;
- this.targetUuid = targetUuid;
- }
-
- public String getApplication() {
- return application;
- }
-
- public void setApplication(String application) {
- this.application = application;
- }
-
- public String getConnectionType() {
- return connectionType;
- }
-
- public void setConnectionType(String connectionType) {
- this.connectionType = connectionType;
- }
-
- public UUID getSourceUuid() {
- return sourceUuid;
- }
-
- public void setSourceUuid(UUID sourceUuid) {
- this.sourceUuid = sourceUuid;
- }
-
- public UUID getTargetUuid() {
- return targetUuid;
- }
-
- public void setTargetUuid(UUID targetUuid) {
- this.targetUuid = targetUuid;
- }
-
- public String getOrganization() {
- return organization;
- }
-
- public void setOrganization(String organization) {
- this.organization = organization;
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index e82e5f8..c33c5f5 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@ -99,6 +99,8 @@ public abstract class ToolBase {
//job scheduler also not needed for tools
System.setProperty("usergrid.scheduler.enabled", "false");
+ System.setProperty("usergrid.cluster.enabled", "false");
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java
new file mode 100644
index 0000000..18b4a26
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools.export;
+
+import java.util.UUID;
+
+public class ExportConnection {
+ private String organization;
+ private String application;
+ private String connectionType;
+ private UUID sourceUuid;
+ private UUID targetUuid;
+ public ExportConnection(String organization, String application, String connectionType, UUID sourceUuid, UUID targetUuid) {
+ this.organization= organization;
+ this.application = application;
+ this.connectionType = connectionType;
+ this.sourceUuid = sourceUuid;
+ this.targetUuid = targetUuid;
+ }
+
+ public String getApplication() {
+ return application;
+ }
+
+ public void setApplication(String application) {
+ this.application = application;
+ }
+
+ public String getConnectionType() {
+ return connectionType;
+ }
+
+ public void setConnectionType(String connectionType) {
+ this.connectionType = connectionType;
+ }
+
+ public UUID getSourceUuid() {
+ return sourceUuid;
+ }
+
+ public void setSourceUuid(UUID sourceUuid) {
+ this.sourceUuid = sourceUuid;
+ }
+
+ public UUID getTargetUuid() {
+ return targetUuid;
+ }
+
+ public void setTargetUuid(UUID targetUuid) {
+ this.targetUuid = targetUuid;
+ }
+
+ public String getOrganization() {
+ return organization;
+ }
+
+ public void setOrganization(String organization) {
+ this.organization = organization;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java
new file mode 100644
index 0000000..800287c
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools.export;
+
+import org.apache.usergrid.persistence.Entity;
+
+import java.util.Map;
+
+public class ExportEntity {
+ private String organization;
+ private String application;
+ private Entity entity;
+ private Map<String, Object> dictionaries;
+ public ExportEntity( String organization, String application, Entity entity, Map<String, Object> dictionaries ) {
+ this.organization = organization;
+ this.application = application;
+ this.entity = entity;
+ this.dictionaries = dictionaries;
+ }
+
+ public String getApplication() {
+ return application;
+ }
+
+ public void setApplication(String application) {
+ this.application = application;
+ }
+
+ public Entity getEntity() {
+ return entity;
+ }
+
+ public void setEntity(Entity entity) {
+ this.entity = entity;
+ }
+
+ public Map<String, Object> getDictionaries() {
+ return dictionaries;
+ }
+
+ public void setDictionaries(Map<String, Object> dictionaries) {
+ this.dictionaries = dictionaries;
+ }
+
+ public String getOrganization() {
+ return organization;
+ }
+
+ public void setOrganization(String organization) {
+ this.organization = organization;
+ }
+}