You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/31 23:00:39 UTC
[01/50] incubator-usergrid git commit: add old type mapping to queries
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-473 0aeaa8815 -> 89af71186
add old type mapping to queries
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cc779b24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cc779b24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cc779b24
Branch: refs/heads/USERGRID-473
Commit: cc779b24c6e8d19d6a20da85d2d625d8407b35eb
Parents: 8fe7267
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 23 10:53:20 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 23 10:53:20 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/SearchType.java | 10 ++++++----
.../apache/usergrid/persistence/index/SearchTypes.java | 3 ++-
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 13 ++++++++-----
.../usergrid/persistence/index/impl/IndexingUtils.java | 2 ++
4 files changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
index 5501c09..c039360 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
@@ -40,10 +40,12 @@ public class SearchType{
return new SearchType( id.getType() );
}
-
-
public String getTypeName(ApplicationScope applicationScope) {
- final String typeName = IndexingUtils.getType(applicationScope, type);
- return typeName;
+ return IndexingUtils.getType(applicationScope, type);
+ }
+
+ public String[] getTypeNames(ApplicationScope applicationScope) {
+ final String[] typeNames = new String[]{type , IndexingUtils.getType(applicationScope, type)};
+ return typeNames;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 1a97da2..9970c62 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -42,10 +42,11 @@ public class SearchTypes {
public String[] getTypeNames(ApplicationScope applicationScope) {
- String[] typeNames = new String[types.length];
+ String[] typeNames = new String[types.length*2];
int i =0 ;
for(String type : types){
typeNames[i++] = IndexingUtils.getType(applicationScope,type);
+ typeNames[i++] = type;
}
return typeNames;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 90b8e35..3b65c74 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -159,8 +159,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
indexes = new String[]{indexIdentifier.getIndex(null)};
}
- container.addDeIndexRequest( new DeIndexRequest( indexes, entityType.getTypeName(applicationScope), indexId ) );
+ String[] typeNames = entityType.getTypeNames(applicationScope);
+ for(String type : typeNames) {
+ container.addDeIndexRequest(new DeIndexRequest(indexes, type, indexId));
+ }
log.debug("Deindexed Entity with index id " + indexId);
return this;
@@ -242,12 +245,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put( ARRAY_PREFIX + field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
@@ -259,12 +262,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof SetField ) {
Set set = ( Set ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put(SET_PREFIX+ field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( set ) ) );
}
else if ( f instanceof EntityObjectField ) {
EntityObject eo = ( EntityObject ) field.getValue();
- entityMap.put( field.getName().toLowerCase(), entityToMap(eo) ); // recursion
+ entityMap.put(EO_PREFIX + field.getName().toLowerCase(), entityToMap(eo) ); // recursion
}
else if ( f instanceof StringField ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index c365ff4..2fb56d1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -40,6 +40,8 @@ public class IndexingUtils {
public static final String BOOLEAN_PREFIX = "bu_";
public static final String ARRAY_PREFIX = "ar_";
+ public static final String SET_PREFIX = "set_";
+ public static final String EO_PREFIX = "eo_";
public static final String SPLITTER = "\\__";
[42/50] incubator-usergrid git commit: remove comment
Posted by sf...@apache.org.
remove comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/33b1b69f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/33b1b69f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/33b1b69f
Branch: refs/heads/USERGRID-473
Commit: 33b1b69fc325c6c4e3bc2fe5628befb6bef2a43d
Parents: 32cb153
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 10:05:51 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 10:05:51 2015 -0600
----------------------------------------------------------------------
.../serialization/impl/migration/EdgeDataMigrationImpl.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/33b1b69f/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index 411e318..2e0412f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -87,9 +87,7 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap( graphNode -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
-
- //TODO: maybe move to retreive all indexes : https://github.com/elastic/elasticsearch/blob/master/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
- //https://www.google.com/webhp?sourceid=chrome-instant&ion=1&espv=2&ie=UTF-8#q=get%20all%20indexes%20elasticsearch
+
//get edges from the source
return edgesFromSourceObservable.edgesFromSource( gm, graphNode.entryNode ).buffer( 1000 )
.doOnNext( edges -> {
[33/50] incubator-usergrid git commit: move impl
Posted by sf...@apache.org.
move impl
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9e8d506c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9e8d506c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9e8d506c
Branch: refs/heads/USERGRID-473
Commit: 9e8d506c3bc5d96a6f24a32f01a3fa7b54531a51
Parents: 4b7b122
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:10:03 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:10:03 2015 -0600
----------------------------------------------------------------------
.../persistence/index/IndexIdentifierImpl.java | 65 -------------------
.../persistence/index/guice/IndexModule.java | 7 +-
.../index/impl/IndexIdentifierImpl.java | 68 ++++++++++++++++++++
3 files changed, 69 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e8d506c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
deleted file mode 100644
index 1d18733..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.index;
-
-import com.google.inject.Inject;
-
-/**
- * Class is used to generate an index name and alias name
- */
-public class IndexIdentifierImpl implements IndexIdentifier {
- private final IndexFig config;
-
- @Inject
- public IndexIdentifierImpl(IndexFig config) {
- this.config = config;
- }
-
- /**
- * Get the alias name
- * @return
- */
- @Override
- public IndexAlias getAlias() {
- return new IndexAlias(config,config.getIndexPrefix());
- }
-
- /**
- * Get index name, send in additional parameter to add incremental indexes
- * @param suffix
- * @return
- */
- @Override
- public String getIndex(String suffix) {
- if (suffix != null) {
- return config.getIndexPrefix() + "_" + suffix;
- } else {
- return config.getIndexPrefix();
- }
- }
-
-
- @Override
- public String toString() {
- return "index id"+config.getIndexPrefix();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e8d506c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index aba49f6..922a916 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -26,13 +26,8 @@ import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
-import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
-import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
+import org.apache.usergrid.persistence.index.impl.*;
import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
import org.apache.usergrid.persistence.index.migration.EsIndexMigrationPlugin;
import org.apache.usergrid.persistence.index.migration.IndexMigration;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e8d506c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
new file mode 100644
index 0000000..0139858
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.index.IndexAlias;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+
+/**
+ * Class is used to generate an index name and alias name
+ */
+public class IndexIdentifierImpl implements IndexIdentifier {
+ private final IndexFig config;
+
+ @Inject
+ public IndexIdentifierImpl(IndexFig config) {
+ this.config = config;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ @Override
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,config.getIndexPrefix());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ @Override
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return config.getIndexPrefix() + "_" + suffix;
+ } else {
+ return config.getIndexPrefix();
+ }
+ }
+
+
+ @Override
+ public String toString() {
+ return "index id"+config.getIndexPrefix();
+ }
+
+}
[32/50] incubator-usergrid git commit: adding test
Posted by sf...@apache.org.
adding test
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4b7b1220
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4b7b1220
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4b7b1220
Branch: refs/heads/USERGRID-473
Commit: 4b7b12203c62c6f6985d5fa21a7e042a24dfc353
Parents: 7976e1d
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:02:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:02:13 2015 -0600
----------------------------------------------------------------------
.../persistence/index/IndexIdentifier.java | 41 ++--------
.../persistence/index/IndexIdentifierImpl.java | 65 ++++++++++++++++
.../persistence/index/guice/IndexModule.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 2 -
.../migration/EsIndexDataMigrationImpl.java | 52 -------------
.../index/migration/LegacyIndexIdentifier.java | 78 ++++++++++++++++++++
.../index/guice/TestIndexModule.java | 26 ++++++-
7 files changed, 174 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 48c48f3..22b82de 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -17,46 +17,15 @@
* * directory of this distribution.
*
*/
-
package org.apache.usergrid.persistence.index;
-import com.google.inject.Inject;
-
/**
- * Class is used to generate an index name and alias name
+ * Classy class class.
*/
-public class IndexIdentifier{
- private final IndexFig config;
-
- @Inject
- public IndexIdentifier(IndexFig config) {
- this.config = config;
- }
-
- /**
- * Get the alias name
- * @return
- */
- public IndexAlias getAlias() {
- return new IndexAlias(config,config.getIndexPrefix());
- }
-
- /**
- * Get index name, send in additional parameter to add incremental indexes
- * @param suffix
- * @return
- */
- public String getIndex(String suffix) {
- if (suffix != null) {
- return config.getIndexPrefix() + "_" + suffix;
- } else {
- return config.getIndexPrefix();
- }
- }
-
+public interface IndexIdentifier {
+ IndexAlias getAlias();
- public String toString() {
- return "index id"+config.getIndexPrefix();
- }
+ String getIndex(String suffix);
+ String toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
new file mode 100644
index 0000000..1d18733
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
@@ -0,0 +1,65 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index;
+
+import com.google.inject.Inject;
+
+/**
+ * Class is used to generate an index name and alias name
+ */
+public class IndexIdentifierImpl implements IndexIdentifier {
+ private final IndexFig config;
+
+ @Inject
+ public IndexIdentifierImpl(IndexFig config) {
+ this.config = config;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ @Override
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,config.getIndexPrefix());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ @Override
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return config.getIndexPrefix() + "_" + suffix;
+ } else {
+ return config.getIndexPrefix();
+ }
+ }
+
+
+ @Override
+ public String toString() {
+ return "index id"+config.getIndexPrefix();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 7ecce54..aba49f6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -56,7 +56,7 @@ public abstract class IndexModule extends AbstractModule {
bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
bind(AliasedEntityIndex.class).to(EsEntityIndexImpl.class);
bind(EntityIndex.class).to(EsEntityIndexImpl.class);
- bind(IndexIdentifier.class);
+ bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 844260f..a22ef38 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -141,8 +141,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
}
-
-
@Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
try {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 13e6526..d6e3812 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -64,7 +64,6 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
- final AtomicInteger integer = new AtomicInteger();
final AdminClient adminClient = provider.getClient().admin();
migrationDataProvider.getData().flatMap(applicationScope -> {
@@ -77,7 +76,6 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
- integer.incrementAndGet();
})
.doOnError(error -> log.error("failed to migrate index", error))
.toBlocking().lastOrDefault(null);
@@ -94,55 +92,5 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
public int getMaxVersion() {
return dataVersion.getImplementationVersion();
}
- /**
- * Class is used to generate an index name and alias name the old way via app name
- */
- public class LegacyIndexIdentifier{
- private final IndexFig config;
- private final ApplicationScope applicationScope;
- public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) {
- this.config = config;
- this.applicationScope = applicationScope;
- }
-
- /**
- * Get the alias name
- * @return
- */
- public IndexAlias getAlias() {
- return new IndexAlias(config,getIndexBase());
- }
-
- /**
- * Get index name, send in additional parameter to add incremental indexes
- * @param suffix
- * @return
- */
- public String getIndex(String suffix) {
- if (suffix != null) {
- return getIndexBase() + "_" + suffix;
- } else {
- return getIndexBase();
- }
- }
-
- /**
- * returns the base name for index which will be used to add an alias and index
- * @return
- */
- private String getIndexBase() {
- StringBuilder sb = new StringBuilder();
- sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
- IndexingUtils.idString(sb, applicationScope.getApplication());
- return sb.toString();
- }
-
-
-
- public String toString() {
- return "application: " + applicationScope.getApplication().getUuid();
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
new file mode 100644
index 0000000..5ab3ac1
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
@@ -0,0 +1,78 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexAlias;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
+
+/**
+ * Class is used to generate an index name and alias name the old way via app name
+ */
+public class LegacyIndexIdentifier implements IndexIdentifier {
+ private final IndexFig config;
+ private final ApplicationScope applicationScope;
+
+ public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) {
+ this.config = config;
+ this.applicationScope = applicationScope;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,getIndexBase());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return getIndexBase() + "_" + suffix;
+ } else {
+ return getIndexBase();
+ }
+ }
+
+ /**
+ * returns the base name for index which will be used to add an alias and index
+ * @return
+ */
+ private String getIndexBase() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
+ IndexingUtils.idString(sb, applicationScope.getApplication());
+ return sb.toString();
+ }
+
+
+
+ public String toString() {
+ return "application: " + applicationScope.getApplication().getUuid();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 0c57b36..6488b16 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,10 +19,22 @@
package org.apache.usergrid.persistence.index.guice;
+import com.amazonaws.services.opsworks.model.App;
+import com.google.inject.Inject;
import com.google.inject.TypeLiteral;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.IndexBufferProducer;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
+import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
+import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
@@ -53,11 +65,23 @@ public class TestIndexModule extends TestModule {
}
public static class TestAllApplicationsObservable implements MigrationDataProvider<ApplicationScope>{
+ final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+
+ @Inject
+ public TestAllApplicationsObservable(final IndexFig config,
+ final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
+ final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexFig indexFig){
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,appScope);
+ EntityIndex entityIndex = new EsEntityIndexImpl(config,indexBatchBufferProducer,provider,indexCache,metricsFactory,indexFig,legacyIndexIdentifier);
+ entityIndex.addIndex(null, 1, 0, indexFig.getWriteConsistencyLevel());
+ }
+
@Override
public Observable<ApplicationScope> getData() {
ApplicationScope[] scopes = new ApplicationScope[]{
- new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"))
+ appScope
};
return Observable.from(scopes);
}
[31/50] incubator-usergrid git commit: refactor into observable
Posted by sf...@apache.org.
refactor into observable
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7976e1d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7976e1d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7976e1d5
Branch: refs/heads/USERGRID-473
Commit: 7976e1d577d9ba3d5f096db33b6984fa5226b7eb
Parents: 2df4013
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 14:38:42 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 14:38:42 2015 -0600
----------------------------------------------------------------------
.../migration/EsIndexDataMigrationImpl.java | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7976e1d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 64b7d29..13e6526 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -33,6 +33,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder
import org.elasticsearch.client.AdminClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import rx.Observable;
import java.util.ArrayList;
import java.util.List;
@@ -64,21 +65,22 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
final AtomicInteger integer = new AtomicInteger();
- migrationDataProvider.getData().doOnNext(applicationScope -> {
- LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,applicationScope);
- String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
- AdminClient adminClient = provider.getClient().admin();
+ final AdminClient adminClient = provider.getClient().admin();
- for (String index : indexes) {
+ migrationDataProvider.getData().flatMap(applicationScope -> {
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
+ String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
+ return Observable.from(indexes);
+ })
+ .doOnNext(index -> {
IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
integer.incrementAndGet();
- }
- })
- .doOnError(error -> log.error("failed to migrate index",error))
- .toBlocking().last();
+ })
+ .doOnError(error -> log.error("failed to migrate index", error))
+ .toBlocking().lastOrDefault(null);
return dataVersion.getImplementationVersion();
}
[07/50] incubator-usergrid git commit: Forgot header
Posted by sf...@apache.org.
Forgot header
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0a2fc00b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0a2fc00b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0a2fc00b
Branch: refs/heads/USERGRID-473
Commit: 0a2fc00b424d302985d73ec0edf6635b8170ad00
Parents: 2670098
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 23 12:18:24 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 23 12:18:24 2015 -0700
----------------------------------------------------------------------
.../endpoints/mgmt/RevokeTokenResource.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0a2fc00b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java
index c727521..dcdd432 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt;
[40/50] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into USERGRID-448-appinfofix
Posted by sf...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-448-appinfofix
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d2be54be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d2be54be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d2be54be
Branch: refs/heads/USERGRID-473
Commit: d2be54be4e57004c55f8be372feb1b9d3974a157
Parents: e611793 a099d0f
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Mar 30 10:19:22 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Mar 30 10:19:22 2015 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 30 +-
.../corepersistence/CpRelationManager.java | 304 +++------
.../usergrid/corepersistence/CpSetup.java | 14 +-
.../results/CollectionRefsVerifier.java | 44 ++
.../CollectionResultsLoaderFactoryImpl.java | 60 ++
.../results/ConnectionRefsVerifier.java | 61 ++
.../ConnectionResultsLoaderFactoryImpl.java | 65 ++
.../results/ElasticSearchQueryExecutor.java | 212 ++++++
.../corepersistence/results/QueryExecutor.java | 37 ++
.../corepersistence/results/RefsVerifier.java | 42 --
.../results/ResultsLoaderFactory.java | 3 +-
.../results/ResultsLoaderFactoryImpl.java | 62 --
.../persistence/MultiQueryIterator.java | 6 +-
.../apache/usergrid/persistence/Results.java | 26 +-
.../cassandra/QueryProcessorImpl.java | 12 +-
.../usergrid/persistence/PathQueryIT.java | 46 +-
.../persistence/query/IteratingQueryIT.java | 38 +-
stack/core/src/test/resources/log4j.properties | 2 +
.../persistence/graph/GraphManagerIT.java | 2 +-
.../index/ApplicationEntityIndex.java | 5 +
.../usergrid/persistence/index/EntityIndex.java | 6 +-
.../usergrid/persistence/index/IndexFig.java | 16 +-
.../index/impl/BufferQueueInMemoryImpl.java | 10 +-
.../impl/EsApplicationEntityIndexImpl.java | 12 +
.../index/impl/EsEntityIndexImpl.java | 9 -
.../index/impl/EsIndexBufferConsumerImpl.java | 74 +--
.../usergrid/persistence/index/query/Query.java | 58 +-
.../persistence/index/utils/ListUtils.java | 3 +-
.../persistence/index/impl/EntityIndexTest.java | 26 +-
.../index/impl/IndexLoadTestsIT.java | 18 +-
.../organizations/OrganizationResource.java | 26 +-
.../applications/ApplicationResource.java | 44 +-
.../applications/imports/ImportsResource.java | 19 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 302 +--------
.../apache/usergrid/rest/SystemResourceIT.java | 10 +-
.../applications/ApplicationResourceIT.java | 4 +-
.../collection/CollectionsResourceIT.java | 19 +-
.../collection/users/OwnershipResourceIT.java | 4 +-
.../collection/users/RetrieveUsersTest.java | 4 +-
.../collection/users/UserResourceIT.java | 4 +-
.../usergrid/rest/management/AccessTokenIT.java | 418 +++++-------
.../usergrid/rest/management/AdminUsersIT.java | 60 +-
.../rest/management/ExportResourceIT.java | 652 +++++++------------
.../rest/management/ImportResourceIT.java | 20 +-
.../rest/management/OrganizationsIT.java | 16 +-
.../organizations/AdminEmailEncodingIT.java | 3 +-
.../test/resource2point0/AbstractRestIT.java | 4 +-
.../rest/test/resource2point0/ClientSetup.java | 30 +-
.../rest/test/resource2point0/RestClient.java | 1 +
.../endpoints/NamedResource.java | 112 +++-
.../endpoints/SetupResource.java | 6 +
.../endpoints/mgmt/ApplicationResource.java | 8 +-
.../endpoints/mgmt/ManagementResource.java | 4 +
.../endpoints/mgmt/MeResource.java | 32 +
.../endpoints/mgmt/PasswordResource.java | 16 -
.../endpoints/mgmt/RevokeTokenResource.java | 32 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 +
.../endpoints/mgmt/TokenResource.java | 25 +-
.../endpoints/mgmt/UserResource.java | 8 +
.../endpoints/mgmt/UsersResource.java | 15 -
.../cassandra/ManagementServiceImpl.java | 21 +-
61 files changed, 1638 insertions(+), 1617 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e2e43ca,3cfc1a4..baac821
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -122,9 -132,11 +125,10 @@@ public class CpEntityManagerFactory imp
this.counterUtils = counterUtils;
this.injector = injector;
this.entityIndex = injector.getInstance(EntityIndex.class);
+ this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
-
- this.orgApplicationCache = new OrgApplicationCacheImpl( this );
+ this.applicationIdCache = new ApplicationIdCacheImpl( this );
}
@@@ -152,9 -164,12 +156,11 @@@
em.getApplication();
}
- entityIndex.initializeIndex();
+ ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) );
+ ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope);
+ applicationEntityIndex.initializeIndex();
entityIndex.refresh();
-
} catch (Exception ex) {
throw new RuntimeException("Fatal error creating system application", ex);
}
@@@ -233,14 -248,15 +239,17 @@@
}
+ /**
+ * @return UUID of newly created Entity of type application_info
+ */
@Override
- public UUID initializeApplication( String organizationName, UUID applicationId, String name,
+ public Entity initializeApplicationV2( String organizationName, final UUID applicationId, String name,
Map<String, Object> properties ) throws Exception {
- EntityManager em = getEntityManager(getManagementAppId());
-
- //Ensure our management system exists before creating our application
++ // Ensure our management system exists before creating our application
+ init();
+
- EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
++ EntityManager em = getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
final String appName = buildAppName( organizationName, name );
@@@ -250,18 -265,63 +259,14 @@@
throw new ApplicationAlreadyExistsException( appName );
}
- // create application info entity
+ ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( applicationId,"application"));
+ ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
+ applicationEntityIndex.initializeIndex();
- try {
- em.create( CpNamingUtils.APPLICATION_INFO, properties );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- throw new ApplicationAlreadyExistsException( appName );
- }
- entityIndex.refresh();
+ getSetup().setupApplicationKeyspace( applicationId, appName );
- final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
-
-
- UUID orgUuid;
-
- if ( !cachedValue.isPresent() ) {
-
-
- // create new org because the specified one does not exist
- final String orgName = organizationName;
-
-
-
- try {
- final Entity orgInfo = em.create( "organization", new HashMap<String, Object>() {{
- put( PROPERTY_NAME, orgName );
- }} );
- orgUuid = orgInfo.getUuid();
- //evit so it's re-loaded later
- orgApplicationCache.evictOrgId( name );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- //swallow, if it exists, just get it
- orgApplicationCache.evictOrgId( organizationName );
- orgUuid = orgApplicationCache.getOrganizationId( organizationName ).get();
- }
-
- } else{
- orgUuid = cachedValue.get();
- }
-
- // create appinfo entry in the system app
- final UUID appId = applicationId;
- final UUID orgId = orgUuid;
- Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
- put( PROPERTY_NAME, appName );
- put( "applicationUuid", appId );
- put( "organizationUuid", orgId );
- }};
-
- try {
- em.create( "appinfo", appInfoMap );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- throw new ApplicationAlreadyExistsException( appName );
- }
- entityIndex.refresh();
-
- // create application entity
if ( properties == null ) {
- properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
+ properties = new TreeMap<>( CASE_INSENSITIVE_ORDER );
}
properties.put( PROPERTY_NAME, appName );
EntityManager appEm = getEntityManager( applicationId);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index 4ab7874,040f60f..b3eef4b
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@@ -98,13 -107,22 +107,10 @@@ public class CpSetup implements Setup
setupStaticKeyspace();
- //force the EMF creation of indexes before creating the default applications
- entityIndex.initializeIndex();
-
- logger.info( "Setting up management app" );
+ injector.getInstance( DataMigrationManager.class ).migrate();
- logger.info( "Setting up default applications" );
-
try {
- emf.initializeApplication( DEFAULT_ORGANIZATION, emf.getDefaultAppId(), DEFAULT_APPLICATION, null );
- }
- catch ( ApplicationAlreadyExistsException ex ) {
- logger.warn( "Application {}/{} already exists", DEFAULT_ORGANIZATION, DEFAULT_APPLICATION );
- }
- catch ( OrganizationAlreadyExistsException oaee ) {
- logger.warn( "Organization {} already exists", DEFAULT_ORGANIZATION );
- }
-
- try {
- emf.initializeApplication( DEFAULT_ORGANIZATION, emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
+ emf.initializeApplicationV2( DEFAULT_ORGANIZATION, emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
}
catch ( ApplicationAlreadyExistsException ex ) {
logger.warn( "Application {}/{} already exists", DEFAULT_ORGANIZATION, MANAGEMENT_APPLICATION );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index ecee76b,d064b97..b2baac9
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@@ -189,10 -200,10 +200,10 @@@ public class EsIndexBufferConsumerImpl
}
while ( true );
}
- } ).subscribeOn(Schedulers.newThread()).doOnNext(new Action1<List<IndexOperationMessage>>() {
+ } ).doOnNext( new Action1<List<IndexOperationMessage>>() {
@Override
- public void call( List<IndexOperationMessage> containerList ) {
- if ( containerList.size() == 0 ) {
+ public void call(List<IndexOperationMessage> containerList) {
+ if (containerList.size() == 0) {
return;
}
@@@ -200,38 -211,22 +211,21 @@@
Timer.Context time = flushTimer.time();
- execute( containerList );
+ execute(containerList);
time.stop();
-
}
- } )
+ })
//ack after we process
- .doOnNext( new Action1<List<IndexOperationMessage>>() {
+ .doOnNext(new Action1<List<IndexOperationMessage>>() {
@Override
- public void call( final List<IndexOperationMessage> indexOperationMessages ) {
- bufferQueue.ack( indexOperationMessages );
+ public void call(final List<IndexOperationMessage> indexOperationMessages) {
+ bufferQueue.ack(indexOperationMessages);
//release so we know we've done processing
- inFlight.addAndGet( -1 * indexOperationMessages.size() );
+ inFlight.addAndGet(-1 * indexOperationMessages.size());
}
- })
- // TODO: implement on error resume next instead of onError
- // .onErrorResumeNext(new Func1<Throwable, Observable<IndexOperationMessage>>() {
- //
- // @Override
- // public Observable<IndexOperationMessage> call(Throwable throwable) {
- // return null;
- // }
- // })
- .doOnError(new Action1<Throwable>() {
- @Override
- public void call(final Throwable throwable) {
- } )
- log.error("An exception occurred when trying to deque and write to elasticsearch. Ignoring",
- throwable);
- indexErrorCounter.inc();
- }
- });
- .subscribeOn( Schedulers.newThread() );
++ } ).subscribeOn( Schedulers.newThread() );
//start in the background
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index 1e7d1f9,d7dcd87..0bebb75
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@@ -21,8 -24,10 +21,9 @@@ import com.google.common.base.Precondit
import com.sun.jersey.api.json.JSONWithPadding;
import org.apache.amber.oauth2.common.exception.OAuthSystemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
+ import org.apache.commons.lang.NullArgumentException;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.export.ExportService;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
index 7d047a3,bb5f958..5755c8e
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
@@@ -107,32 -114,37 +107,33 @@@ public class ImportsResource extends Ab
Map<String, Object> properties;
Map<String, Object> storage_info;
- if ((properties = (Map<String, Object>) json.get("properties")) == null) {
- throw new NullArgumentException("Could not find 'properties'");
++ if ( ( properties = ( Map<String, Object> ) json.get( "properties" ) ) == null ) {
++ throw new NullArgumentException( "Could not find 'properties'" );
+ }
- storage_info = (Map<String, Object>) properties.get("storage_info");
- String storage_provider = (String) properties.get("storage_provider");
- if (storage_provider == null) {
- throw new NullArgumentException("Could not find field 'storage_provider'");
++ storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
++ String storage_provider = ( String ) properties.get( "storage_provider" );
++ if ( storage_provider == null ) {
++ throw new NullArgumentException( "Could not find field 'storage_provider'" );
+ }
- if (storage_info == null) {
- throw new NullArgumentException("Could not find field 'storage_info'");
++ if ( storage_info == null ) {
++ throw new NullArgumentException( "Could not find field 'storage_info'" );
+ }
- String bucketName = (String) storage_info.get("bucket_location");
- if ( ( properties = ( Map<String, Object> ) json.get( "properties" ) ) == null ) {
- throw new NullArgumentException( "Could not find 'properties'" );
- }
- storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
- String storage_provider = ( String ) properties.get( "storage_provider" );
- if ( storage_provider == null ) {
- throw new NullArgumentException( "Could not find field 'storage_provider'" );
- }
- if ( storage_info == null ) {
- throw new NullArgumentException( "Could not find field 'storage_info'" );
- }
-
- String bucketName = ( String ) storage_info.get( "bucket_location" );
-
-
- String accessId = ( String ) storage_info.get( "s3_access_id" );
- String secretKey = ( String ) storage_info.get( "s3_key" );
-
- if ( bucketName == null ) {
- throw new NullArgumentException( "Could not find field 'bucketName'" );
- }
- if ( accessId == null ) {
- throw new NullArgumentException( "Could not find field 's3_access_id'" );
- }
- if ( secretKey == null ) {
++ String bucketName = ( String ) storage_info.get( "bucket_location" );
+
- throw new NullArgumentException( "Could not find field 's3_key'" );
- }
+ String accessId = (String) storage_info.get("s3_access_id");
+ String secretKey = (String) storage_info.get("s3_key");
+ if (bucketName == null) {
+ throw new NullArgumentException("Could not find field 'bucketName'");
+ }
+ if (accessId == null) {
+ throw new NullArgumentException("Could not find field 's3_access_id'");
+ }
+ if (secretKey == null) {
+ throw new NullArgumentException("Could not find field 's3_key'");
+ }
json.put( "organizationId", organization.getUuid() );
json.put( "applicationId", application.getId() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
index a943bba,5e40037..67bc650
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
@@@ -575,8 -584,7 +575,8 @@@ public class ApplicationResourceIT exte
String clientId = orgCredentials.getClientId();
String clientSecret = orgCredentials.getClientSecret();
- Token token = clientSetup.getRestClient().management().token().post(
- new Token("client_credentials", clientId, clientSecret));
- Token token = clientSetup.getRestClient().management().token().post(Token.class,new Token("client_credentials", clientId, clientSecret));
++ Token token = clientSetup.getRestClient().management().token()
++ .post(Token.class,new Token("client_credentials", clientId, clientSecret));
//GET the token endpoint, adding authorization header
Token apiResponse = this.app().token().getResource(false)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
index 1ea9770,355c57a..9b5b09a
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
@@@ -173,11 -171,15 +173,11 @@@ public class ImportResourceIT extends A
Entity payload = payloadBuilder();
// /management/orgs/orgname/apps/appname/import
- Entity entity = this.management()
- .orgs()
- .organization(org)
- .app()
+ Entity entity = this.management().orgs().organization(org).app()
.addToPath(app)
.addToPath("imports")
- .post(payload);
+ .post(Entity.class,payload);
-
assertNotNull(entity);
// test that you can access the organization using the currently set token.
@@@ -644,10 -625,8 +644,10 @@@
}});
}});
- Entity importEntity = this.management().orgs().organization(org).app().addToPath(app).addToPath("imports")
- .post(Entity.class,importPayload);
+ Entity importEntity = this.management().orgs().organization(org).app()
+ .addToPath(app)
+ .addToPath("imports")
- .post(importPayload);
++ .post(Entity.class,importPayload);
int maxRetries = 120;
int retries = 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
index 034c302,90e2c64..c02c48b
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
@@@ -22,7 -22,7 +22,8 @@@ package org.apache.usergrid.rest.test.r
import javax.ws.rs.core.MediaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+ import org.apache.usergrid.rest.test.resource2point0.endpoints.CollectionEndpoint;
import org.apache.usergrid.rest.test.resource2point0.endpoints.NamedResource;
import org.apache.usergrid.rest.test.resource2point0.endpoints.UrlResource;
import org.apache.usergrid.rest.test.resource2point0.model.Application;
@@@ -57,29 -47,17 +58,30 @@@ public class ApplicationResource extend
}
- public void post(Application application) {
- getResource(true).type(MediaType.APPLICATION_JSON_TYPE)
- .accept(MediaType.APPLICATION_JSON).post(application);
+ public ApiResponse post(Application application) {
+ ApiResponse apiResponse =getResource(true).type(MediaType.APPLICATION_JSON_TYPE)
+ .accept(MediaType.APPLICATION_JSON).post(ApiResponse.class,application);
+ return apiResponse;
}
-// public Entity post(Entity payload){
-// ApiResponse response = getResource(true).type( MediaType.APPLICATION_JSON_TYPE ).accept(MediaType.APPLICATION_JSON)
-// .post(ApiResponse.class, payload);
-// return new Entity(response);
-// }
+ public Entity post(Entity payload) {
+
+ String responseString = getResource(true)
+ .type( MediaType.APPLICATION_JSON_TYPE )
+ .accept(MediaType.APPLICATION_JSON)
+ .post(String.class, payload);
+
+ logger.debug("Response from post: " + responseString);
+
+ ApiResponse response;
+ try {
+ response = mapper.readValue(new StringReader(responseString), ApiResponse.class);
+ } catch (IOException e) {
+ throw new RuntimeException("Error parsing response", e);
+ }
+
+ return new Entity(response);
+ }
public Entity get() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2be54be/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index cd8a404,0cf80b4..682044b
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@@ -1445,9 -1477,8 +1445,8 @@@ public class ManagementServiceImpl impl
BiMap<UUID, String> organizations = HashBiMap.create();
EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
- Results results = em.getCollection(
- new SimpleEntityRef( User.ENTITY_TYPE, userId ), Schema.COLLECTION_GROUPS, null, 10000,
- Level.ALL_PROPERTIES, false );
- Results results = em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userId ), "groups", null, 1000,
- Level.ALL_PROPERTIES, false );
++ Results results = em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userId ),
++ Schema.COLLECTION_GROUPS, null, 1000, Level.ALL_PROPERTIES, false );
String path = null;
[24/50] incubator-usergrid git commit: Fixes issue with throwable not
being caught in onSubscribe function
Posted by sf...@apache.org.
Fixes issue with throwable not being caught in onSubscribe function
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2d1c8b8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2d1c8b8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2d1c8b8a
Branch: refs/heads/USERGRID-473
Commit: 2d1c8b8ac7b20b63a11d83adca56839d8b409cca
Parents: beb2a2a
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 26 09:47:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 26 09:47:58 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++++++-------------
1 file changed, 14 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d1c8b8a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 7e64de3..d064b97 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -153,12 +153,16 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
//name our thread so it's easy to see
Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
- List<IndexOperationMessage> drainList;
+
+ List<IndexOperationMessage> drainList = null;
+
do {
- try {
+ Timer.Context timer = produceTimer.time();
+
+
+ try {
- Timer.Context timer = produceTimer.time();
drainList = bufferQueue
.take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
@@ -174,10 +178,15 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
timer.stop();
}
- catch ( Exception e ) {
+ catch ( Throwable t ) {
final long sleepTime = config.getFailureRetryTime();
- log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, e );
+ log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, t );
+
+ if ( drainList != null ) {
+ inFlight.addAndGet( -1 * drainList.size() );
+ }
+
try {
Thread.sleep( sleepTime );
@@ -216,26 +225,6 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
inFlight.addAndGet( -1 * indexOperationMessages.size() );
}
} )
- //catch an unexpected error, then emit an empty list to ensure our subscriber doesn't die
- .onErrorReturn( new Func1<Throwable, List<IndexOperationMessage>>() {
- @Override
- public List<IndexOperationMessage> call( final Throwable throwable ) {
- final long sleepTime = config.getFailureRetryTime();
-
- log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, throwable );
-
- try {
- Thread.sleep( sleepTime );
- }
- catch ( InterruptedException ie ) {
- //swallow
- }
-
- indexErrorCounter.inc();
-
- return Collections.EMPTY_LIST;
- }
- } )
.subscribeOn( Schedulers.newThread() );
[43/50] incubator-usergrid git commit: add comments for interfaces
Posted by sf...@apache.org.
add comments for interfaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c18ba594
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c18ba594
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c18ba594
Branch: refs/heads/USERGRID-473
Commit: c18ba594dc6e88662a30d525b2ddee34d1ed30d7
Parents: 33b1b69
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 10:08:28 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 10:08:28 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexCache.java | 12 +++++++++++-
.../usergrid/persistence/index/IndexIdentifier.java | 16 +++++++++++++++-
2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c18ba594/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
index 253c66d..e852230 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
@@ -20,10 +20,20 @@
package org.apache.usergrid.persistence.index;
/**
- * Classy class class.
+ * Cache for index lookups
*/
public interface IndexCache {
+ /**
+ * get index based on alias name
+ * @param alias
+ * @param aliasType
+ * @return
+ */
String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
+ /**
+ * invalidate cache
+ * @param alias
+ */
void invalidate(IndexAlias alias);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c18ba594/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 22b82de..f393ee3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -20,12 +20,26 @@
package org.apache.usergrid.persistence.index;
/**
- * Classy class class.
+ * Identifier for where an index is in underlying server
*/
public interface IndexIdentifier {
+
+ /**
+ * get the alias name
+ * @return
+ */
IndexAlias getAlias();
+ /**
+ * get index name from suffix
+ * @param suffix
+ * @return
+ */
String getIndex(String suffix);
+ /**
+ * return unique string
+ * @return
+ */
String toString();
}
[25/50] incubator-usergrid git commit: add data migration
Posted by sf...@apache.org.
add data migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f695724b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f695724b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f695724b
Branch: refs/heads/USERGRID-473
Commit: f695724b0417131eb9592a642ae8fa7a29afd8cd
Parents: 8a62d11
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 10:14:12 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 10:14:12 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexAlias.java | 38 ++++++
.../persistence/index/IndexIdentifier.java | 18 ---
.../persistence/index/guice/IndexModule.java | 19 +++
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../persistence/index/impl/EsIndexCache.java | 7 +-
.../impl/SearchRequestBuilderStrategy.java | 5 +-
.../migration/EsIndexDataMigrationImpl.java | 134 +++++++++++++++++++
.../index/migration/EsIndexMigrationPlugin.java | 46 +++++++
.../index/migration/IndexMigration.java | 35 +++++
10 files changed, 281 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java
new file mode 100644
index 0000000..a04f80e
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index;
+
+/**
+ * Abstraction for Index alias names
+ */
+public class IndexAlias{
+ private final String readAlias;
+ private final String writeAlias;
+
+ public IndexAlias(IndexFig indexFig,String indexBase) {
+ this.writeAlias = indexBase + "_write_" + indexFig.getAliasPostfix();
+ this.readAlias = indexBase + "_read_" + indexFig.getAliasPostfix();
+ }
+
+ public String getReadAlias() {
+ return readAlias;
+ }
+
+ public String getWriteAlias() {
+ return writeAlias;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index c659ed6..48c48f3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -55,24 +55,6 @@ public class IndexIdentifier{
}
- public class IndexAlias{
- private final String readAlias;
- private final String writeAlias;
-
- public IndexAlias(IndexFig indexFig,String indexBase) {
- this.writeAlias = indexBase + "_write_" + indexFig.getAliasPostfix();
- this.readAlias = indexBase + "_read_" + indexFig.getAliasPostfix();
- }
-
- public String getReadAlias() {
- return readAlias;
- }
-
- public String getWriteAlias() {
- return writeAlias;
- }
- }
-
public String toString() {
return "index id"+config.getIndexPrefix();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index c9125c5..a42dea8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -19,6 +19,11 @@
package org.apache.usergrid.persistence.index.guice;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -28,6 +33,9 @@ import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
+import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
+import org.apache.usergrid.persistence.index.migration.EsIndexMigrationPlugin;
+import org.apache.usergrid.persistence.index.migration.IndexMigration;
import org.apache.usergrid.persistence.map.guice.MapModule;
import org.apache.usergrid.persistence.queue.guice.QueueModule;
@@ -56,6 +64,17 @@ public class IndexModule extends AbstractModule {
bind( BufferQueue.class).toProvider( QueueProvider.class );
+
+ //wire up the edg migration
+ Multibinder<DataMigration<ApplicationScope>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<ApplicationScope>>() {}, IndexMigration.class );
+
+
+ dataMigrationMultibinder.addBinding().to(EsIndexDataMigrationImpl.class);
+
+
+ //wire up the collection migration plugin
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to(EsIndexMigrationPlugin.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index ca071e2..27791ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -76,7 +76,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
private final EsIndexCache indexCache;
private final IndexFig indexFig;
private final EsProvider esProvider;
- private final IndexIdentifier.IndexAlias alias;
+ private final IndexAlias alias;
private final Timer deleteApplicationTimer;
private final Meter deleteApplicationMeter;
private final SearchRequestBuilderStrategy searchRequest;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 259fa55..cab8ded 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -77,7 +77,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public static final String DEFAULT_TYPE = "_default_";
- private final IndexIdentifier.IndexAlias alias;
+ private final IndexAlias alias;
private final IndexBufferProducer indexBatchBufferProducer;
private final IndexFig indexFig;
private final Timer addTimer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
index ef518dd..9ba89ca 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -27,6 +27,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.persistence.index.IndexAlias;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.cluster.metadata.AliasMetaData;
@@ -98,7 +99,7 @@ public class EsIndexCache {
/**
* Get indexes for an alias
*/
- public String[] getIndexes( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ public String[] getIndexes( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
String[] indexes;
try {
indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
@@ -127,7 +128,7 @@ public class EsIndexCache {
* @param aliasType
* @return
*/
- private String getAliasName( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ private String getAliasName( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
}
@@ -135,7 +136,7 @@ public class EsIndexCache {
/**
* clean up cache
*/
- public void invalidate( IndexIdentifier.IndexAlias alias ) {
+ public void invalidate( IndexAlias alias ) {
aliasIndexCache.invalidate( alias.getWriteAlias() );
aliasIndexCache.invalidate( alias.getReadAlias() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index d146e4c..15796f9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.index.impl;
import com.google.common.base.Preconditions;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.IndexAlias;
import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.SearchTypes;
@@ -53,11 +54,11 @@ public class SearchRequestBuilderStrategy {
private final EsProvider esProvider;
private final ApplicationScope applicationScope;
- private final IndexIdentifier.IndexAlias alias;
+ private final IndexAlias alias;
private final int cursorTimeout;
public static final int MAX_LIMIT = 1000;
- public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexIdentifier.IndexAlias alias, int cursorTimeout){
+ public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){
this.esProvider = esProvider;
this.applicationScope = applicationScope;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
new file mode 100644
index 0000000..b5dab53
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.IndexAlias;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.client.AdminClient;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Classy class class.
+ */
+public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope> {
+
+ private final AliasedEntityIndex entityIndex;
+ private final EsProvider provider;
+ private final IndexFig indexFig;
+ private final IndexIdentifier indexIdentifier;
+ private final EsIndexCache indexCache;
+
+ @Inject
+ public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, EsIndexCache indexCache){
+ this.entityIndex = entityIndex;
+ this.provider = provider;
+ this.indexFig = indexFig;
+ this.indexIdentifier = indexIdentifier;
+ this.indexCache = indexCache;
+ }
+
+ @Override
+ public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
+ migrationDataProvider.getData().doOnNext(applicationScope -> {
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,applicationScope);
+ String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
+ AdminClient adminClient = provider.getClient().admin();
+
+ for (String index : indexes) {
+ IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ // add read alias
+ aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ }
+ });
+ return 0;
+ }
+
+ @Override
+ public boolean supports(int currentVersion) {
+ return false;
+ }
+
+ @Override
+ public int getMaxVersion() {
+ return 0;
+ }
+ /**
+ * Class is used to generate an index name and alias name the old way via app name
+ */
+ public class LegacyIndexIdentifier{
+ private final IndexFig config;
+ private final ApplicationScope applicationScope;
+
+ public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) {
+ this.config = config;
+ this.applicationScope = applicationScope;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,getIndexBase());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return getIndexBase() + "_" + suffix;
+ } else {
+ return getIndexBase();
+ }
+ }
+
+ /**
+ * returns the base name for index which will be used to add an alias and index
+ * @return
+ */
+ private String getIndexBase() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
+ IndexingUtils.idString(sb, applicationScope.getApplication());
+ return sb.toString();
+ }
+
+
+
+ public String toString() {
+ return "application: " + applicationScope.getApplication().getUuid();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
new file mode 100644
index 0000000..a28c701
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.migration.data.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import java.util.Set;
+
+/**
+ * Classy class class.
+ */
+public class EsIndexMigrationPlugin extends AbstractMigrationPlugin<ApplicationScope>{
+
+ @Inject
+ public EsIndexMigrationPlugin(@IndexMigration final Set<DataMigration<ApplicationScope>> entityDataMigrations,
+ final MigrationDataProvider<ApplicationScope> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ){
+ super(entityDataMigrations,entityIdScopeDataMigrationProvider,migrationInfoSerialization);
+ }
+
+ @Override
+ public String getName() {
+ return "index-migration";
+ }
+
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.MIGRATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java
new file mode 100644
index 0000000..c398aac
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Index migration annotation
+ */
+
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface IndexMigration {}
[48/50] incubator-usergrid git commit: separate calls
Posted by sf...@apache.org.
separate calls
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c12aa17c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c12aa17c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c12aa17c
Branch: refs/heads/USERGRID-473
Commit: c12aa17ccabfa02822293f6017612943019ee4b7
Parents: bec7f44
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 16:53:45 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 16:53:45 2015 -0600
----------------------------------------------------------------------
.../java/org/apache/usergrid/corepersistence/CpEntityManager.java | 1 -
.../main/java/org/apache/usergrid/persistence/EntityManager.java | 1 -
.../apache/usergrid/persistence/cassandra/EntityManagerImpl.java | 1 -
3 files changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c12aa17c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 481fff5..efdad6d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,7 +37,6 @@ import com.codahale.metrics.Meter;
import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.elasticsearch.action.ListenableActionFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c12aa17c/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 7108472..23d2e9f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -35,7 +35,6 @@ import org.apache.usergrid.persistence.entities.Role;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query.Level;
-import org.elasticsearch.action.ListenableActionFuture;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c12aa17c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 30872c9..f681191 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -37,7 +37,6 @@ import java.util.UUID;
import javax.annotation.Resource;
import org.apache.usergrid.persistence.index.EntityIndex;
-import org.elasticsearch.action.ListenableActionFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
[08/50] incubator-usergrid git commit: Fixed an issue when multiple
tests are run. Cleaned up and fixed SystemResourceIT
Posted by sf...@apache.org.
Fixed an issue when multiple tests are run.
Cleaned up and fixed SystemResourceIT
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bf04de22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bf04de22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bf04de22
Branch: refs/heads/USERGRID-473
Commit: bf04de22d01fcaeac2fb988bba99c1f828d7a2af
Parents: 0a2fc00
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 23 13:13:14 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 23 13:13:14 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/rest/SystemResourceIT.java | 10 +++-------
.../usergrid/rest/test/resource2point0/ClientSetup.java | 5 ++++-
.../test/resource2point0/endpoints/SetupResource.java | 6 ++++++
3 files changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf04de22/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java
index 5695143..f379672 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/SystemResourceIT.java
@@ -25,6 +25,7 @@ import org.apache.usergrid.rest.test.resource2point0.model.QueryParameters;
import org.apache.usergrid.rest.test.resource2point0.model.Token;
import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
import static org.junit.Assert.assertNotNull;
@@ -36,14 +37,11 @@ public class SystemResourceIT extends AbstractRestIT {
@Test
public void testSystemDatabaseAlreadyRun() {
- //try {
QueryParameters queryParameters = new QueryParameters();
queryParameters.addParam( "access_token",clientSetup.getSuperuserToken().getAccessToken() );
+
Entity result = clientSetup.getRestClient().system().database().setup().get(queryParameters);
-// }catch(UniformInterfaceException uie) {
-// asser
-// }
-//
+
assertNotNull(result);
assertNotNull( "ok" ,(String)result.get( "status" ) );
@@ -53,8 +51,6 @@ public class SystemResourceIT extends AbstractRestIT {
assertNotNull( "ok" ,(String)result.get( "status" ) );
-
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf04de22/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
index 210284b..65f6749 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
@@ -106,11 +106,14 @@ public class ClientSetup implements TestRule {
organization = restClient.management().orgs()
.post( new Organization( orgName, username, username + "@usergrid.com", username,
username, null ) );
+ refreshIndex();
clientCredentials = restClient.management().orgs().organization( orgName ).credentials().get();
+ refreshIndex();
- Token token = restClient.management().token().post(Token.class,new Token(username,username));
+ //restClient.management().token().post(Token.class,new Token(username,password));
restClient.management().orgs().organization(organization.getName()).app().post(new Application(appName));
+ refreshIndex();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf04de22/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/SetupResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/SetupResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/SetupResource.java
index 1639ce5..086a34f 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/SetupResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/SetupResource.java
@@ -25,6 +25,7 @@ import org.apache.usergrid.rest.test.resource2point0.model.Token;
import org.apache.usergrid.rest.test.resource2point0.state.ClientContext;
import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
/**
@@ -37,9 +38,14 @@ public class SetupResource extends NamedResource {
}
public Entity get(QueryParameters queryParameters){
+
WebResource resource = getResource();
resource = addParametersToResource( resource, queryParameters );
+ //added httpBasicauth filter to all setup calls because they all do verification this way.
+ HTTPBasicAuthFilter httpBasicAuthFilter = new HTTPBasicAuthFilter( "superuser","superpassword" );
+ resource.addFilter( httpBasicAuthFilter );
+
return resource.type( MediaType.APPLICATION_JSON_TYPE ).accept( MediaType.APPLICATION_JSON )
.get( Entity.class );
}
[12/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
USERGRID-405
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into USERGRID-405
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e4466e11
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e4466e11
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e4466e11
Branch: refs/heads/USERGRID-473
Commit: e4466e112af89b530e57afc5191835e9ba3673d3
Parents: c0384d5 5937f9f
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Mar 24 12:44:46 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Mar 24 12:44:46 2015 -0400
----------------------------------------------------------------------
.../usergrid/security/providers/PingIdentityProvider.java | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
[41/50] incubator-usergrid git commit: change all application
observable
Posted by sf...@apache.org.
change all application observable
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/32cb153d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/32cb153d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/32cb153d
Branch: refs/heads/USERGRID-473
Commit: 32cb153d1712d112c9ca0eafb775b42b92413c9d
Parents: 6a39add
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 09:41:02 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 09:41:02 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/AllApplicationsObservable.java | 5 -----
.../corepersistence/rx/impl/AbstractGraphVisitorImpl.java | 2 +-
.../rx/impl/AllApplicationsObservableImpl.java | 7 ++-----
.../corepersistence/rx/ApplicationObservableTestIT.java | 2 +-
.../src/main/java/org/apache/usergrid/rest/RootResource.java | 2 +-
5 files changed, 5 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
index 055223e..24f32f3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
@@ -32,9 +32,4 @@ import rx.Observable;
*/
public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
- /**
- * Return all applications in our system
- * @return
- */
- public Observable<ApplicationScope> getAllApplications();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
index b8cb80b..ef66481 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
@@ -58,7 +58,7 @@ public abstract class AbstractGraphVisitorImpl<T> implements MigrationDataProvid
@Override
public Observable<T> getData() {
- return applicationObservable.getAllApplications().flatMap( new Func1<ApplicationScope, Observable<T>>() {
+ return applicationObservable.getData().flatMap( new Func1<ApplicationScope, Observable<T>>() {
@Override
public Observable<T> call( final ApplicationScope applicationScope ) {
return getAllEntities( applicationScope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index aa5b5dc..b57f088 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -69,7 +69,7 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
@Override
- public Observable<ApplicationScope> getAllApplications() {
+ public Observable<ApplicationScope> getData() {
//emit our 3 hard coded applications that are used the manage the system first.
//this way consumers can perform whatever work they need to on the root system first
@@ -138,8 +138,5 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
}
- @Override
- public Observable<ApplicationScope> getData() {
- return getAllApplications();
- }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
index 649f518..9572079 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
@@ -67,7 +67,7 @@ public class ApplicationObservableTestIT extends AbstractCoreIT {
//clean up our wiring
ManagerCache managerCache = SpringResource.getInstance().getBean( Injector.class ).getInstance( ManagerCache.class );
- Observable<ApplicationScope> appObservable = applicationObservable.getAllApplications();
+ Observable<ApplicationScope> appObservable = applicationObservable.getData();
appObservable.doOnNext( new Action1<ApplicationScope>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
index e4086f9..3fa4f74 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
@@ -114,7 +114,7 @@ public class RootResource extends AbstractContextResource implements MetricProce
@QueryParam("deleted") @DefaultValue("false") Boolean deleted,
@QueryParam("callback") @DefaultValue("callback") String callback ) throws URISyntaxException {
- logger.info( "RootResource.getAllApplications" );
+ logger.info( "RootResource.getData" );
ApiResponse response = createApiResponse();
response.setAction( "get applications" );
[50/50] incubator-usergrid git commit: merge from two-o-dev
Posted by sf...@apache.org.
merge from two-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/89af7118
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/89af7118
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/89af7118
Branch: refs/heads/USERGRID-473
Commit: 89af711866c02e0b294a80c584b9e082067cd892
Parents: 0aeaa88 c1643a7
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 31 15:00:21 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 31 15:00:21 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 26 +-
.../dist/init_instance/install_elasticsearch.sh | 22 +-
.../main/groovy/configure_elasticsearch.groovy | 80 ++-
.../src/main/groovy/configure_usergrid.groovy | 12 +-
stack/awscluster/ugcluster-cf.json | 466 +++++++++----
.../main/resources/usergrid-default.properties | 2 +
stack/core/pom.xml | 64 +-
.../batch/service/JobSchedulerService.java | 33 +-
.../batch/service/SchedulerServiceImpl.java | 17 +-
.../AllApplicationsObservable.java | 35 +
.../corepersistence/ApplicationIdCache.java | 52 ++
.../corepersistence/ApplicationIdCacheImpl.java | 135 ++++
.../usergrid/corepersistence/CoreModule.java | 90 ++-
.../corepersistence/CpEntityManager.java | 228 ++++---
.../corepersistence/CpEntityManagerFactory.java | 475 +++++++------
.../corepersistence/CpManagerCache.java | 57 +-
.../corepersistence/CpRelationManager.java | 350 +++-------
.../usergrid/corepersistence/CpSetup.java | 32 +-
.../usergrid/corepersistence/CpWalker.java | 81 +--
.../usergrid/corepersistence/ManagerCache.java | 13 +-
.../corepersistence/OrgApplicationCache.java | 67 --
.../OrgApplicationCacheImpl.java | 181 -----
.../events/EntityDeletedHandler.java | 59 +-
.../events/EntityVersionCreatedHandler.java | 60 +-
.../events/EntityVersionDeletedHandler.java | 104 ++-
.../migration/AppInfoMigrationPlugin.java | 343 ++++++++++
.../migration/AppInfoVersions.java | 40 ++
.../migration/CoreDataVersions.java | 45 ++
.../migration/CoreMigration.java | 53 ++
.../migration/CoreMigrationPlugin.java | 64 ++
.../migration/EntityDataMigration.java | 147 ----
.../migration/EntityTypeMappingMigration.java | 69 +-
.../migration/GraphShardVersionMigration.java | 152 -----
.../migration/MigrationModuleVersionPlugin.java | 145 ++++
.../migration/MigrationSystemVersions.java | 43 ++
.../corepersistence/migration/Versions.java | 14 -
.../results/CollectionRefsVerifier.java | 44 ++
.../CollectionResultsLoaderFactoryImpl.java | 60 ++
.../results/ConnectionRefsVerifier.java | 61 ++
.../ConnectionResultsLoaderFactoryImpl.java | 65 ++
.../results/ElasticSearchQueryExecutor.java | 212 ++++++
.../results/FilteringLoader.java | 55 +-
.../corepersistence/results/QueryExecutor.java | 37 +
.../corepersistence/results/RefsVerifier.java | 42 --
.../results/ResultsLoaderFactory.java | 3 +-
.../results/ResultsLoaderFactoryImpl.java | 62 --
.../rx/AllEntitiesInSystemObservable.java | 101 ---
.../rx/ApplicationObservable.java | 128 ----
.../rx/EdgesFromSourceObservable.java | 63 --
.../rx/EdgesToTargetObservable.java | 63 --
.../corepersistence/rx/TargetIdObservable.java | 66 --
.../rx/impl/AbstractGraphVisitorImpl.java | 107 +++
.../rx/impl/AllApplicationsObservableImpl.java | 140 ++++
.../rx/impl/AllEntitiesInSystemImpl.java | 62 ++
.../rx/impl/AllNodesInGraphImpl.java | 55 ++
.../corepersistence/util/CpEntityMapUtils.java | 19 +-
.../corepersistence/util/CpNamingUtils.java | 36 +-
.../usergrid/exception/ConflictException.java | 36 +
.../usergrid/persistence/EntityManager.java | 28 +-
.../persistence/EntityManagerFactory.java | 51 +-
.../persistence/MultiQueryIterator.java | 6 +-
.../persistence/ObservableIterator.java | 1 +
.../apache/usergrid/persistence/Results.java | 26 +-
.../org/apache/usergrid/persistence/Schema.java | 5 +-
.../cassandra/EntityManagerFactoryImpl.java | 42 +-
.../cassandra/EntityManagerImpl.java | 50 +-
.../cassandra/QueryProcessorImpl.java | 12 +-
.../cassandra/RelationManagerImpl.java | 58 +-
.../persistence/cassandra/SetupImpl.java | 2 -
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
.../main/resources/usergrid-core-context.xml | 1 +
.../org/apache/usergrid/CoreApplication.java | 24 +-
.../java/org/apache/usergrid/CoreITSetup.java | 8 +-
.../org/apache/usergrid/CoreITSetupImpl.java | 54 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 +
.../corepersistence/StaleIndexCleanupTest.java | 143 +++-
.../migration/EntityDataMigrationIT.java | 262 -------
.../migration/EntityTypeMappingMigrationIT.java | 183 ++---
.../migration/GraphShardVersionMigrationIT.java | 226 -------
.../MigrationModuleVersionPluginTest.java | 259 +++++++
.../migration/MigrationTestRule.java | 99 ---
.../migration/TestProgressObserver.java | 71 --
.../rx/AllEntitiesInSystemObservableIT.java | 71 +-
.../rx/ApplicationObservableTestIT.java | 16 +-
.../rx/EdgesFromSourceObservableIT.java | 10 +-
.../rx/EdgesToTargetObservableIT.java | 8 +-
.../rx/TargetIdObservableTestIT.java | 7 +-
.../usergrid/persistence/CollectionIT.java | 80 +--
.../apache/usergrid/persistence/CounterIT.java | 9 +-
.../usergrid/persistence/CountingMutatorIT.java | 4 +-
.../persistence/EntityConnectionsIT.java | 14 +-
.../usergrid/persistence/EntityManagerIT.java | 27 +-
.../org/apache/usergrid/persistence/GeoIT.java | 27 +-
.../persistence/GeoQueryBooleanTest.java | 4 +-
.../apache/usergrid/persistence/IndexIT.java | 14 +-
.../usergrid/persistence/LargeEntityIT.java | 3 +-
.../usergrid/persistence/PathQueryIT.java | 52 +-
.../PerformanceEntityRebuildIndexTest.java | 38 +-
.../usergrid/persistence/PermissionsIT.java | 4 +-
.../cassandra/EntityManagerFactoryImplIT.java | 86 ++-
.../persistence/query/CollectionIoHelper.java | 2 +-
.../persistence/query/ConnectionHelper.java | 2 +-
.../query/IntersectionTransitivePagingIT.java | 2 +-
.../query/IntersectionUnionPagingIT.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 16 +-
stack/core/src/test/resources/log4j.properties | 2 +
.../resources/usergrid-custom-test.properties | 1 +
stack/corepersistence/collection/pom.xml | 14 +-
.../collection/EntityCollectionManager.java | 17 +-
.../EntityCollectionManagerFactory.java | 27 +-
.../collection/EntityCollectionManagerSync.java | 49 --
.../collection/EntityDeletedFactory.java | 34 -
.../collection/EntityVersionCleanupFactory.java | 35 -
.../collection/EntityVersionCreatedFactory.java | 31 -
.../persistence/collection/FieldSet.java | 48 ++
.../persistence/collection/MvccEntity.java | 5 +-
.../persistence/collection/MvccLogEntry.java | 1 +
.../cache/CachedEntityCollectionManager.java | 18 +-
.../collection/event/EntityVersionDeleted.java | 3 +-
.../collection/guice/CollectionModule.java | 65 +-
.../persistence/collection/guice/Write.java | 17 -
.../collection/guice/WriteUpdate.java | 17 -
.../EntityCollectionManagerFactoryImpl.java | 121 ++--
.../impl/EntityCollectionManagerImpl.java | 268 ++++++--
.../impl/EntityCollectionManagerSyncImpl.java | 71 --
.../collection/impl/EntityDeletedTask.java | 49 +-
.../impl/EntityVersionCleanupTask.java | 185 ++---
.../impl/EntityVersionCreatedTask.java | 26 +-
.../impl/EntityVersionTaskFactory.java | 65 ++
.../mvcc/MvccEntitySerializationStrategy.java | 102 ---
.../mvcc/stage/delete/MarkCommit.java | 2 +-
.../mvcc/stage/write/WriteCommit.java | 11 +-
.../collection/mvcc/stage/write/WriteStart.java | 6 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 140 ++--
.../collection/serialization/EntityRepair.java | 38 --
.../MvccEntitySerializationStrategy.java | 119 ++++
.../serialization/OptimisticUpdate.java | 23 -
.../UniqueValueSerializationStrategy.java | 49 +-
.../serialization/UniqueValueSet.java | 4 +
.../impl/CollectionDataVersions.java | 44 ++
.../serialization/impl/EntityRepairImpl.java | 149 ----
.../impl/EntityVersionSerializer.java | 2 -
.../serialization/impl/FieldSerializer.java | 118 ----
.../serialization/impl/MutableFieldSet.java | 63 ++
.../MvccEntitySerializationStrategyImpl.java | 112 ++-
...vccEntitySerializationStrategyProxyImpl.java | 151 +++--
.../MvccEntitySerializationStrategyV1Impl.java | 9 +-
.../MvccEntitySerializationStrategyV2Impl.java | 8 +-
.../MvccEntitySerializationStrategyV3Impl.java | 587 ++++++++++++++++
.../serialization/impl/SerializationModule.java | 85 ++-
.../serialization/impl/UniqueFieldEntry.java | 61 ++
.../impl/UniqueFieldEntrySerializer.java | 141 ++++
.../impl/UniqueFieldRowKeySerializer.java | 122 ++++
.../UniqueValueSerializationStrategyImpl.java | 255 +++++--
.../impl/migration/CollectionMigration.java | 53 ++
.../migration/CollectionMigrationPlugin.java | 68 ++
.../impl/migration/EntityIdScope.java | 49 ++
.../migration/MvccEntityDataMigrationImpl.java | 260 +++++++
.../collection/util/EntityUtils.java | 49 --
.../collection/EntityCollectionManagerIT.java | 174 ++---
.../EntityCollectionManagerSyncIT.java | 193 ------
.../collection/guice/TestCollectionModule.java | 22 +-
.../impl/EntityVersionCleanupTaskTest.java | 678 ++++++++-----------
.../impl/EntityVersionCreatedTaskTest.java | 2 -
.../mvcc/stage/AbstractEntityStageTest.java | 2 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 2 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 2 +-
.../mvcc/stage/write/FieldSerializerTest.java | 52 --
.../write/UniqueFieldRowKeySerializerTest.java | 52 ++
...niqueValueSerializationStrategyImplTest.java | 149 +++-
.../mvcc/stage/write/WriteCommitTest.java | 2 +-
.../mvcc/stage/write/WriteStartTest.java | 6 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../persistence/collection/rx/ParallelTest.java | 10 +-
.../serialization/EntityRepairImplTest.java | 147 ----
.../impl/LogEntryIteratorTest.java | 3 +-
...MvccEntitySerializationStrategyImplTest.java | 121 +---
...cEntitySerializationStrategyProxyV1Test.java | 85 ---
...ntitySerializationStrategyProxyV1_3Test.java | 83 +++
...cEntitySerializationStrategyProxyV2Test.java | 83 ---
...ntitySerializationStrategyProxyV2_3Test.java | 84 +++
...ccEntitySerializationStrategyV1ImplTest.java | 80 ++-
...ccEntitySerializationStrategyV2ImplTest.java | 75 +-
.../MvccEntitySerializationStrategyV2Test.java | 7 +-
...ccEntitySerializationStrategyV3ImplTest.java | 74 ++
.../impl/SerializationComparison.java | 4 +-
.../impl/UniqueFieldEntrySerializerTest.java | 108 +++
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 215 ++++++
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 105 +++
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 99 +++
.../collection/util/InvalidEntityGenerator.java | 1 +
.../collection/util/LogEntryMock.java | 39 +-
.../collection/util/UniqueValueEntryMock.java | 161 +++++
.../collection/util/VersionGenerator.java | 55 ++
stack/corepersistence/common/pom.xml | 15 +-
.../usergrid/persistence/core/CPManager.java | 28 +
.../core/astyanax/CassandraConfig.java | 6 +
.../core/astyanax/CassandraConfigImpl.java | 8 +-
.../persistence/core/astyanax/CassandraFig.java | 9 +-
.../astyanax/DynamicCompositeParserImpl.java | 81 +++
.../core/astyanax/FieldBufferBuilder.java | 15 +
.../core/astyanax/FieldBufferParser.java | 13 +
.../astyanax/MultiKeyColumnNameIterator.java | 4 +-
.../core/astyanax/MultiRowColumnIterator.java | 12 +-
.../persistence/core/future/BetterFuture.java | 43 +-
.../persistence/core/guice/CommonModule.java | 17 +-
.../persistence/core/guice/CurrentImpl.java | 42 --
.../persistence/core/guice/PreviousImpl.java | 42 --
.../core/hystrix/HystrixCassandra.java | 94 ---
.../core/metrics/MetricsFactory.java | 11 +-
.../core/metrics/MetricsFactoryImpl.java | 121 ++--
.../migration/data/AbstractMigrationPlugin.java | 128 ++++
.../core/migration/data/DataMigration.java | 97 +--
.../migration/data/DataMigrationManager.java | 18 +-
.../data/DataMigrationManagerImpl.java | 237 ++++---
.../migration/data/MigrationDataProvider.java | 43 ++
.../core/migration/data/MigrationInfoCache.java | 52 ++
.../migration/data/MigrationInfoCacheImpl.java | 84 +++
.../data/MigrationInfoSerialization.java | 28 +-
.../data/MigrationInfoSerializationImpl.java | 67 +-
.../core/migration/data/MigrationPlugin.java | 56 ++
.../migration/data/MigrationRelationship.java | 100 +++
.../core/migration/data/PluginPhase.java | 42 ++
.../core/migration/data/ProgressObserver.java | 63 ++
.../core/migration/data/VersionedData.java | 38 ++
.../migration/data/VersionedMigrationSet.java | 153 +++++
.../core/scope/ApplicationScope.java | 1 +
.../persistence/core/task/TaskExecutor.java | 4 +-
.../core/astyanax/ColumnNameIteratorTest.java | 7 +-
.../MultiKeyColumnNameIteratorTest.java | 194 +++---
.../astyanax/MultiRowColumnIteratorTest.java | 57 +-
.../core/guice/DataMigrationResetRule.java | 88 +++
.../core/guice/MaxMigrationModule.java | 39 --
.../core/guice/MaxMigrationVersion.java | 40 --
.../core/guice/MigrationManagerRule.java | 7 +
.../core/guice/TestCommonModule.java | 1 +
.../persistence/core/guice/TestModule.java | 5 +-
.../data/DataMigrationManagerImplTest.java | 350 ++++++----
.../data/MigrationInfoSerializationTest.java | 32 +-
.../data/TestMigrationDataProvider.java | 61 ++
.../migration/data/TestProgressObserver.java | 89 +++
.../data/VersionedMigrationSetTest.java | 198 ++++++
.../persistence/core/util/IdGenerator.java | 51 ++
stack/corepersistence/graph/pom.xml | 28 +-
.../persistence/graph/GraphManager.java | 3 +-
.../persistence/graph/GraphManagerFactory.java | 2 +
.../persistence/graph/guice/GraphModule.java | 103 ++-
.../graph/impl/GraphManagerImpl.java | 314 ++++++++-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 18 +-
.../impl/stage/NodeDeleteListenerImpl.java | 18 +-
.../EdgeMetadataSerialization.java | 3 +-
.../graph/serialization/EdgesObservable.java | 33 +
.../graph/serialization/TargetIdObservable.java | 38 ++
.../EdgeMetadataSerializationProxyImpl.java | 161 +++--
.../impl/EdgeMetadataSerializationV1Impl.java | 6 +
.../impl/EdgeMetadataSerializationV2Impl.java | 6 +
.../serialization/impl/EdgesObservableImpl.java | 80 +++
.../serialization/impl/GraphDataVersions.java | 43 ++
.../impl/GraphManagerFactoryImpl.java | 95 +++
.../impl/NodeSerializationImpl.java | 36 +-
.../impl/TargetIdObservableImpl.java | 72 ++
.../impl/migration/EdgeDataMigrationImpl.java | 138 ++++
.../impl/migration/GraphMigration.java | 53 ++
.../impl/migration/GraphMigrationPlugin.java | 69 ++
.../serialization/impl/migration/GraphNode.java | 39 ++
.../shard/count/NodeShardApproximationImpl.java | 4 +-
.../NodeShardCounterSerializationImpl.java | 25 +-
.../shard/impl/NodeShardAllocationImpl.java | 16 +-
.../shard/impl/ShardGroupCompactionImpl.java | 64 +-
.../persistence/graph/GraphManagerIT.java | 36 +-
.../persistence/graph/GraphManagerLoadTest.java | 13 +-
.../graph/GraphManagerShardConsistencyIT.java | 12 +-
.../graph/GraphManagerShardingIT.java | 13 +-
.../graph/GraphManagerStressTest.java | 11 +-
.../usergrid/persistence/graph/SimpleTest.java | 12 +-
.../graph/guice/TestGraphModule.java | 27 +-
.../graph/impl/EdgeDeleteListenerTest.java | 11 +-
.../graph/impl/NodeDeleteListenerTest.java | 9 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 7 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 29 +-
.../EdgeMetaDataSerializationProxyV1Test.java | 14 +-
.../EdgeMetaDataSerializationProxyV2Test.java | 21 +-
.../EdgeMetaDataSerializationV1Test.java | 7 +-
.../EdgeMetaDataSerializationV2Test.java | 7 +-
.../EdgeMetadataSerializationTest.java | 39 +-
.../EdgeSerializationChopTest.java | 7 +-
.../serialization/EdgeSerializationTest.java | 21 +-
.../serialization/NodeSerializationTest.java | 15 +-
.../migration/EdgeDataMigrationImplTest.java | 177 +++++
.../impl/shard/EdgeShardSerializationTest.java | 7 +-
.../impl/shard/NodeShardAllocationTest.java | 27 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../impl/shard/ShardGroupCompactionTest.java | 7 +-
.../shard/count/NodeShardApproximationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 6 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 15 +-
...rceDirectedEdgeDescendingComparatorTest.java | 23 +-
...getDirectedEdgeDescendingComparatorTest.java | 23 +-
.../graph/test/util/EdgeTestUtils.java | 29 +-
.../usergrid/persistence/map/MapManager.java | 10 +
.../persistence/map/guice/MapModule.java | 8 +-
.../persistence/map/impl/MapManagerImpl.java | 8 +
.../persistence/map/impl/MapSerialization.java | 9 +
.../map/impl/MapSerializationImpl.java | 93 +++
.../persistence/map/MapManagerTest.java | 49 +-
.../persistence/map/guice/TestMapModule.java | 3 +-
stack/corepersistence/model/pom.xml | 1 -
.../persistence/model/entity/Entity.java | 23 +-
.../persistence/model/entity/EntityMap.java | 66 ++
.../model/entity/EntityToMapConverter.java | 133 ++++
.../model/entity/MapToEntityConverter.java | 203 ++++++
.../persistence/model/entity/SimpleId.java | 1 -
.../persistence/model/field/AbstractField.java | 8 +
.../persistence/model/field/ListField.java | 2 +-
.../persistence/model/field/StringField.java | 20 +
.../persistence/model/util/EntityUtils.java | 72 ++
stack/corepersistence/pom.xml | 8 +-
stack/corepersistence/queryindex/pom.xml | 80 +--
.../persistence/index/AliasedEntityIndex.java | 7 +-
.../index/ApplicationEntityIndex.java | 57 ++
.../usergrid/persistence/index/EntityIndex.java | 64 +-
.../persistence/index/EntityIndexBatch.java | 5 +
.../persistence/index/EntityIndexFactory.java | 2 +-
.../usergrid/persistence/index/IndexAlias.java | 38 ++
.../persistence/index/IndexBufferConsumer.java | 13 +-
.../persistence/index/IndexBufferProducer.java | 3 +-
.../usergrid/persistence/index/IndexCache.java | 39 ++
.../usergrid/persistence/index/IndexFig.java | 83 ++-
.../persistence/index/IndexIdentifier.java | 63 +-
.../index/IndexOperationMessage.java | 115 +++-
.../usergrid/persistence/index/SearchType.java | 51 ++
.../usergrid/persistence/index/SearchTypes.java | 15 +-
.../persistence/index/guice/IndexModule.java | 49 +-
.../persistence/index/guice/QueueProvider.java | 116 ++++
.../persistence/index/impl/BatchRequest.java | 41 ++
.../persistence/index/impl/BufferQueue.java | 68 ++
.../index/impl/BufferQueueInMemoryImpl.java | 116 ++++
.../index/impl/BufferQueueSQSImpl.java | 307 +++++++++
.../persistence/index/impl/DeIndexRequest.java | 127 ++++
.../index/impl/EntityToMapConverter.java | 173 +++++
.../impl/EsApplicationEntityIndexImpl.java | 307 +++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 270 +-------
.../index/impl/EsEntityIndexFactoryImpl.java | 31 +-
.../index/impl/EsEntityIndexImpl.java | 648 ++++--------------
.../index/impl/EsIndexBufferConsumerImpl.java | 316 ++++++---
.../index/impl/EsIndexBufferProducerImpl.java | 16 +-
.../persistence/index/impl/EsIndexCache.java | 116 ----
.../index/impl/EsIndexCacheImpl.java | 141 ++++
.../persistence/index/impl/EsQueryVistor.java | 88 +--
.../index/impl/IndexIdentifierImpl.java | 68 ++
.../persistence/index/impl/IndexRequest.java | 139 ++++
.../persistence/index/impl/IndexingUtils.java | 150 ++--
.../impl/SearchRequestBuilderStrategy.java | 195 ++++++
.../migration/EsIndexDataMigrationImpl.java | 105 +++
.../index/migration/EsIndexMigrationPlugin.java | 46 ++
.../index/migration/IndexDataVersions.java | 39 ++
.../index/migration/IndexMigration.java | 35 +
.../index/migration/LegacyIndexIdentifier.java | 78 +++
.../index/query/CandidateResults.java | 24 +-
.../persistence/index/query/EntityResults.java | 108 ---
.../usergrid/persistence/index/query/Query.java | 92 +--
.../persistence/index/query/Results.java | 148 ----
.../persistence/index/utils/ListUtils.java | 3 +-
.../persistence/index/guice/IndexTestFig.java | 62 ++
.../index/guice/TestIndexModule.java | 59 +-
.../index/impl/BufferQueueSQSImplTest.java | 173 +++++
.../index/impl/CorePerformanceIT.java | 339 ----------
.../impl/EntityConnectionIndexImplTest.java | 309 ---------
.../index/impl/EntityIndexMapUtils.java | 52 +-
.../persistence/index/impl/EntityIndexTest.java | 251 +++----
.../persistence/index/impl/EsTestUtils.java | 48 --
.../index/impl/IndexLoadTestsIT.java | 390 +++++++++++
.../index/impl/IndexMigrationTest.java | 106 +++
.../persistence/query/tree/GrammarTreeTest.java | 10 +-
.../persistence/queue/QueueManager.java | 4 +-
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 -
.../persistence/queue/guice/QueueModule.java | 17 +-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --
.../persistence/queue/impl/QueueScopeImpl.java | 27 +-
.../queue/impl/SQSQueueManagerImpl.java | 286 ++++----
.../persistence/queue/NoAWSCredsRule.java | 98 +++
.../persistence/queue/QueueManagerTest.java | 27 +-
.../queue/guice/TestQueueModule.java | 3 +-
.../usergrid/mongo/protocol/OpDelete.java | 4 +-
.../apache/usergrid/mongo/protocol/OpQuery.java | 6 +-
.../usergrid/mongo/protocol/OpUpdate.java | 4 +-
stack/pom.xml | 60 +-
stack/rest/pom.xml | 5 +-
.../usergrid/rest/AbstractContextResource.java | 102 +--
.../org/apache/usergrid/rest/IndexResource.java | 5 +-
.../apache/usergrid/rest/MigrateResource.java | 120 +++-
.../org/apache/usergrid/rest/RootResource.java | 8 +-
.../rest/applications/ApplicationResource.java | 38 +-
.../rest/applications/ServiceResource.java | 8 +-
.../exceptions/ConflictExceptionMapper.java | 35 +
.../exceptions/NotFoundExceptionMapper.java | 35 +
.../organizations/OrganizationResource.java | 26 +-
.../applications/ApplicationResource.java | 59 +-
.../applications/ApplicationsResource.java | 40 +-
.../imports/FileIncludesResource.java | 9 -
.../applications/imports/ImportsResource.java | 83 +--
.../organizations/OrganizationResource.java | 23 +-
.../OAuth2AccessTokenSecurityFilter.java | 2 +-
.../rest/test/RefreshIndexResource.java | 17 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 302 +--------
.../apache/usergrid/rest/IndexResourceIT.java | 1 +
.../apache/usergrid/rest/SystemResourceIT.java | 10 +-
.../applications/ApplicationDeleteTest.java | 373 +++++++++-
.../applications/ApplicationResourceIT.java | 63 +-
.../collection/CollectionsResourceIT.java | 19 +-
.../collection/users/OwnershipResourceIT.java | 4 +-
.../collection/users/PermissionsResourceIT.java | 15 -
.../collection/users/RetrieveUsersTest.java | 4 +-
.../collection/users/UserResourceIT.java | 16 +-
.../rest/applications/queries/OrderByTest.java | 2 +-
.../usergrid/rest/management/AccessTokenIT.java | 418 +++++-------
.../usergrid/rest/management/AdminUsersIT.java | 60 +-
.../rest/management/ExportResourceIT.java | 652 ++++++------------
.../rest/management/ImportResourceIT.java | 169 +++--
.../rest/management/OrganizationsIT.java | 16 +-
.../organizations/AdminEmailEncodingIT.java | 3 +-
.../test/resource2point0/AbstractRestIT.java | 4 +-
.../rest/test/resource2point0/ClientSetup.java | 30 +-
.../rest/test/resource2point0/RestClient.java | 1 +
.../endpoints/ApplicationsResource.java | 34 +-
.../endpoints/NamedResource.java | 112 ++-
.../endpoints/OrganizationResource.java | 6 +-
.../endpoints/SetupResource.java | 6 +
.../endpoints/mgmt/ApplicationResource.java | 54 +-
.../endpoints/mgmt/ApplicationsResource.java | 58 ++
.../endpoints/mgmt/ManagementResource.java | 4 +
.../endpoints/mgmt/ManagementResponse.java | 72 ++
.../endpoints/mgmt/MeResource.java | 32 +
.../mgmt/OrganizationApplicationResponse.java | 60 ++
.../endpoints/mgmt/OrganizationResource.java | 5 +-
.../endpoints/mgmt/PasswordResource.java | 16 -
.../endpoints/mgmt/RevokeTokenResource.java | 32 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 +
.../endpoints/mgmt/TokenResource.java | 25 +-
.../endpoints/mgmt/UserResource.java | 8 +
.../endpoints/mgmt/UsersResource.java | 15 -
stack/rest/src/test/resources/log4j.properties | 10 +-
.../resources/testImportCorrect.testCol.1.json | 18 -
...testImportInvalidJson.testApplication.3.json | 153 -----
.../testimport-bad-json-testapp.3.json | 153 +++++
.../src/test/resources/testimport-bad-json.json | 67 ++
.../resources/testimport-correct-testcol.1.json | 18 +
.../test/resources/testimport-test-app.2.json | 56 ++
.../resources/usergrid-custom-test.properties | 2 +
stack/services/pom.xml | 14 +-
.../usergrid/management/ApplicationInfo.java | 4 +
.../usergrid/management/ManagementService.java | 10 +-
.../cassandra/ManagementServiceImpl.java | 253 ++++---
.../management/importer/FileImportJob.java | 6 +-
.../management/importer/FileImportTracker.java | 6 +-
.../usergrid/management/importer/ImportJob.java | 4 +-
.../management/importer/ImportServiceImpl.java | 78 +--
.../usergrid/security/AuthPrincipalType.java | 3 +-
.../providers/PingIdentityProvider.java | 5 +-
.../shiro/PrincipalCredentialsToken.java | 20 +-
.../services/AbstractCollectionService.java | 40 +-
.../services/AbstractConnectionsService.java | 39 +-
.../services/ServiceManagerFactory.java | 4 -
.../notifications/NotificationsService.java | 50 +-
.../services/notifications/QueueListener.java | 5 +-
.../impl/ApplicationQueueManagerImpl.java | 195 +++---
.../usergrid/services/queues/QueueListener.java | 5 +-
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../org/apache/usergrid/ServiceITSetup.java | 3 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 34 +-
.../migration/AppInfoMigrationPluginTest.java | 218 ++++++
.../apache/usergrid/management/EmailFlowIT.java | 8 +-
.../usergrid/management/OrganizationIT.java | 10 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ApplicationCreatorIT.java | 4 +-
.../cassandra/ManagementServiceIT.java | 35 +-
.../management/export/ExportServiceIT.java | 20 +-
.../importer/FileImportTrackerTest.java | 12 +-
.../management/importer/ImportCollectionIT.java | 16 +-
.../importer/ImportConnectionsTest.java | 7 +-
.../security/tokens/TokenServiceIT.java | 2 -
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 2 +-
.../usergrid/services/ServiceFactoryIT.java | 8 +-
.../usergrid/services/ServiceRequestIT.java | 9 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 2 +-
.../notifications/NotifiersServiceIT.java | 6 +
.../apns/NotificationsServiceIT.java | 40 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../src/test/resources/log4j.properties | 6 +-
.../resources/usergrid-custom-test.properties | 2 +
stack/test-utils/pom.xml | 5 -
.../setup/ConcurrentProcessSingleton.java | 49 +-
.../org/apache/usergrid/tools/AppAudit.java | 3 +-
.../org/apache/usergrid/tools/CredExport.java | 2 +-
.../org/apache/usergrid/tools/DupOrgRepair.java | 1 -
.../java/org/apache/usergrid/tools/Import.java | 14 +-
.../usergrid/tools/UniqueIndexCleanup.java | 6 +-
504 files changed, 20395 insertions(+), 12324 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/89af7118/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index cfaf0b1,088359a..d368a7f
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@@ -17,16 -17,44 +17,27 @@@
*/
package org.apache.usergrid.persistence.queue.impl;
- import com.amazonaws.AmazonClientException;
- import com.amazonaws.SDKGlobalConfiguration;
- import com.amazonaws.auth.AWSCredentials;
- import com.amazonaws.auth.AWSCredentialsProvider;
+
+ import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
++import java.util.*;
+ import java.util.concurrent.ExecutionException;
+
++import com.amazonaws.services.sqs.model.*;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import org.apache.usergrid.persistence.queue.Queue;
+ import org.apache.usergrid.persistence.queue.QueueFig;
+ import org.apache.usergrid.persistence.queue.QueueManager;
+ import org.apache.usergrid.persistence.queue.QueueMessage;
+ import org.apache.usergrid.persistence.queue.QueueScope;
+
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQSClient;
- import com.amazonaws.services.sqs.model.*;
-import com.amazonaws.services.sqs.model.BatchResultErrorEntry;
-import com.amazonaws.services.sqs.model.CreateQueueRequest;
-import com.amazonaws.services.sqs.model.CreateQueueResult;
-import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
-import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
-import com.amazonaws.services.sqs.model.DeleteMessageBatchResult;
-import com.amazonaws.services.sqs.model.DeleteMessageRequest;
-import com.amazonaws.services.sqs.model.GetQueueUrlResult;
-import com.amazonaws.services.sqs.model.Message;
-import com.amazonaws.services.sqs.model.MessageAttributeValue;
-import com.amazonaws.services.sqs.model.QueueDoesNotExistException;
-import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
-import com.amazonaws.services.sqs.model.ReceiveMessageResult;
-import com.amazonaws.services.sqs.model.SendMessageBatchRequest;
-import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;
-import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.fasterxml.jackson.databind.ObjectMapper;
- import com.fasterxml.jackson.databind.SerializationFeature;
+ import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+ import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@@ -47,74 -64,62 +47,63 @@@ import com.google.inject.assistedinject
public class SQSQueueManagerImpl implements QueueManager {
private static final Logger LOG = LoggerFactory.getLogger(SQSQueueManagerImpl.class);
- private AmazonSQSClient sqs;
- private QueueScope scope;
- private QueueFig fig;
- private ObjectMapper mapper;
+
- private final QueueScope scope;
- private ObjectMapper mapper;
++ private final QueueScope scope;
++ private ObjectMapper mapper;
+ private final QueueFig fig;
+ private final AmazonSQSClient sqs;
+
private static SmileFactory smileFactory = new SmileFactory();
- private static LoadingCache<SqsLoader, Queue> urlMap = CacheBuilder.newBuilder()
+ private LoadingCache<String, Queue> urlMap = CacheBuilder.newBuilder()
- .maximumSize( 1000 )
- .build( new CacheLoader<String, Queue>() {
- @Override
- public Queue load( String queueName ) throws Exception {
-
- //the amazon client is not thread safe, we need to create one per queue
- Queue queue = null;
- try {
- GetQueueUrlResult result = sqs.getQueueUrl( queueName );
- queue = new Queue( result.getQueueUrl() );
- }catch ( QueueDoesNotExistException queueDoesNotExistException ) {
- //no op, swallow
- LOG.error( "Queue {} does not exist, creating", queueName );
-
- }
- catch ( Exception e ) {
- LOG.error( "failed to get queue from service", e );
- throw e;
- }
- if ( queue == null ) {
- CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueName );
- CreateQueueResult result = sqs.createQueue( createQueueRequest );
- String url = result.getQueueUrl();
- queue = new Queue( url );
- LOG.info( "Created queue with url {}", url );
- }
- return queue;
+ .maximumSize(1000)
- .build(new CacheLoader<SqsLoader, Queue>() {
- @Override
- public Queue load(SqsLoader queueLoader) throws Exception {
- Queue queue = null;
- try {
- GetQueueUrlResult result = queueLoader.getClient().getQueueUrl(queueLoader.getKey());
- queue = new Queue(result.getQueueUrl());
- } catch (QueueDoesNotExistException queueDoesNotExistException) {
- queue = null;
- } catch (Exception e) {
- LOG.error("failed to get queue from service", e);
- throw e;
- }
- if (queue == null) {
- String name = queueLoader.getKey();
- CreateQueueRequest createQueueRequest = new CreateQueueRequest()
- .withQueueName(name);
- CreateQueueResult result = queueLoader.getClient().createQueue(createQueueRequest);
- String queueUrl = result.getQueueUrl();
-
- setDeadLetterQueue(queueLoader.client,queueLoader.config(), queueUrl, name+"_dead");
- queue = new Queue(queueUrl);
- LOG.info("Created queue with url {}", queueUrl);
- }
- return queue;
- }
- }
- );
++ .build(new CacheLoader<String, Queue>() {
++ @Override
++ public Queue load(String queueName) throws Exception {
++
++ //the amazon client is not thread safe, we need to create one per queue
++ Queue queue = null;
++ try {
++ GetQueueUrlResult result = sqs.getQueueUrl(queueName);
++ queue = new Queue(result.getQueueUrl());
++ } catch (QueueDoesNotExistException queueDoesNotExistException) {
++ //no op, swallow
++ LOG.error("Queue {} does not exist, creating", queueName);
++
++ } catch (Exception e) {
++ LOG.error("failed to get queue from service", e);
++ throw e;
+ }
- } );
++ if (queue == null) {
++ CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName(queueName);
++ CreateQueueResult result = sqs.createQueue(createQueueRequest);
++ String url = result.getQueueUrl();
++ queue = new Queue(url);
++ LOG.info("Created queue with url {}", url);
++ setDeadLetterQueue(sqs,fig,url,"dead_letter");
++
++ }
++ return queue;
++ }
++ });
- private static void setDeadLetterQueue(AmazonSQSClient client, QueueFig fig, String queueUrl, String deadLetterName) {
- CreateQueueRequest deadLetterQueueRequest = new CreateQueueRequest()
- .withQueueName(deadLetterName);
- CreateQueueResult deadLetterResult = client.createQueue(deadLetterQueueRequest);
- String deadLetterUrl = deadLetterResult.getQueueUrl();
- String redrivePolicy = String.format("{\"maxReceiveCount\":\"%s\", \"deadLetterTargetArn\":\"%s\"}", fig.getMaxReceiveCount(), deadLetterUrl);
- SetQueueAttributesRequest queueAttributes = new SetQueueAttributesRequest();
- Map<String, String> attributes = new HashMap<>();
- attributes.put("RedrivePolicy", redrivePolicy);
- queueAttributes.setAttributes(attributes);
- queueAttributes.setQueueUrl(queueUrl);
- client.setQueueAttributes(queueAttributes);
- }
@Inject
- public SQSQueueManagerImpl(@Assisted QueueScope scope, QueueFig fig){
- this.fig = fig;
- public SQSQueueManagerImpl( @Assisted QueueScope scope, QueueFig fig ){
++ public SQSQueueManagerImpl(@Assisted QueueScope scope, QueueFig fig) {
this.scope = scope;
+ this.fig = fig;
try {
- UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- this.sqs = new AmazonSQSClient(ugProvider.getCredentials());
- Regions regions = Regions.fromName(fig.getRegion());
- Region region = Region.getRegion(regions);
- sqs.setRegion(region);
+
smileFactory.delegateToTextual(true);
-- mapper = new ObjectMapper( smileFactory );
- mapper.enable(SerializationFeature.INDENT_OUTPUT);
++ mapper = new ObjectMapper(smileFactory);
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.enableDefaultTypingAsProperty(ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class");
- } catch ( Exception e ) {
- LOG.warn("failed to setup SQS",e);
- // throw new RuntimeException("Error setting up mapper", e);
+
+ sqs = createClient();
+
- } catch ( Exception e ) {
++ } catch (Exception e) {
+ throw new RuntimeException("Error setting up mapper", e);
}
}
@@@ -126,7 -134,7 +118,7 @@@
public Queue getQueue() {
try {
- Queue queue = urlMap.get(new SqsLoader(getName(),sqs,fig));
- Queue queue = urlMap.get(getName());
++ Queue queue = urlMap.get(new SqsLoader(getName(), sqs));
return queue;
} catch (ExecutionException ee) {
throw new RuntimeException(ee);
@@@ -135,31 -143,30 +127,30 @@@
@Override
public List<QueueMessage> getMessages(int limit, int transactionTimeout, int waitTime, Class klass) {
-- if(sqs == null){
++ if (sqs == null) {
LOG.error("Sqs is null");
return new ArrayList<>();
}
-- waitTime = waitTime/1000;
++ waitTime = waitTime / 1000;
String url = getQueue().getUrl();
- LOG.info("Getting {} messages from {}", limit, url);
-
- LOG.debug( "Getting {} messages from {}", limit, url);
++ LOG.debug("Getting {} messages from {}", limit, url);
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(url);
receiveMessageRequest.setMaxNumberOfMessages(limit);
receiveMessageRequest.setVisibilityTimeout(transactionTimeout);
receiveMessageRequest.setWaitTimeSeconds(waitTime);
ReceiveMessageResult result = sqs.receiveMessage(receiveMessageRequest);
List<Message> messages = result.getMessages();
- LOG.info("Received {} messages from {}",messages.size(),url);
- LOG.debug( "Received {} messages from {}", messages.size(), url);
++ LOG.debug("Received {} messages from {}", messages.size(), url);
List<QueueMessage> queueMessages = new ArrayList<>(messages.size());
for (Message message : messages) {
-- Object body ;
-- try{
-- body = fromString(message.getBody(),klass);
-- }catch (Exception e){
++ Object body;
++ try {
++ body = fromString(message.getBody(), klass);
++ } catch (Exception e) {
LOG.error("failed to deserialize message", e);
throw new RuntimeException(e);
}
-- QueueMessage queueMessage = new QueueMessage(message.getMessageId(),message.getReceiptHandle(),body,message.getAttributes().get( "type" ));
++ QueueMessage queueMessage = new QueueMessage(message.getMessageId(), message.getReceiptHandle(), body, message.getAttributes().get("type"));
queueMessages.add(queueMessage);
}
return queueMessages;
@@@ -167,20 -174,20 +158,20 @@@
@Override
public void sendMessages(List bodies) throws IOException {
-- if(sqs == null){
++ if (sqs == null) {
LOG.error("Sqs is null");
return;
}
String url = getQueue().getUrl();
- LOG.info("Sending Messages...{} to {}", bodies.size(), url);
- LOG.debug( "Sending Messages...{} to {}", bodies.size(), url);
++ LOG.debug("Sending Messages...{} to {}", bodies.size(), url);
SendMessageBatchRequest request = new SendMessageBatchRequest(url);
List<SendMessageBatchRequestEntry> entries = new ArrayList<>(bodies.size());
-- for(Object body : bodies){
++ for (Object body : bodies) {
SendMessageBatchRequestEntry entry = new SendMessageBatchRequestEntry();
entry.setId(UUID.randomUUID().toString());
-- entry.setMessageBody( toString( body ) );
-- entry.addMessageAttributesEntry( "type",new MessageAttributeValue().withStringValue( "mytype" ) );
++ entry.setMessageBody(toString(body));
++ entry.addMessageAttributesEntry("type", new MessageAttributeValue().withStringValue("mytype"));
entries.add(entry);
}
request.setEntries(entries);
@@@ -190,13 -197,16 +181,16 @@@
@Override
public void sendMessage(Object body) throws IOException {
-- if(sqs == null){
++ if (sqs == null) {
LOG.error("Sqs is null");
return;
}
String url = getQueue().getUrl();
- LOG.info("Sending Message...{} to {}",body.toString(),url);
- SendMessageRequest request = new SendMessageRequest(url,toString((Serializable)body));
- LOG.debug( "Sending Message...{} to {}", body.toString(), url);
++ LOG.debug("Sending Message...{} to {}", body.toString(), url);
+
+ final String stringBody = toString(body);
+
+ SendMessageRequest request = new SendMessageRequest(url, stringBody);
sqs.sendMessage(request);
}
@@@ -204,86 -214,68 +198,92 @@@
@Override
public void commitMessage(QueueMessage queueMessage) {
String url = getQueue().getUrl();
- LOG.info("Commit message {} to queue {}",queueMessage.getMessageId(),url);
- LOG.debug( "Commit message {} to queue {}", queueMessage.getMessageId(), url);
++ LOG.debug("Commit message {} to queue {}", queueMessage.getMessageId(), url);
sqs.deleteMessage(new DeleteMessageRequest()
- .withQueueUrl(url)
- .withReceiptHandle(queueMessage.getHandle()));
+ .withQueueUrl(url)
+ .withReceiptHandle(queueMessage.getHandle()));
}
@Override
public void commitMessages(List<QueueMessage> queueMessages) {
String url = getQueue().getUrl();
- LOG.info("Commit messages {} to queue {}",queueMessages.size(),url);
- LOG.debug( "Commit messages {} to queue {}", queueMessages.size(), url);
++ LOG.debug("Commit messages {} to queue {}", queueMessages.size(), url);
List<DeleteMessageBatchRequestEntry> entries = new ArrayList<>();
-- for(QueueMessage message : queueMessages){
-- entries.add(new DeleteMessageBatchRequestEntry(message.getMessageId(),message.getHandle()));
++ for (QueueMessage message : queueMessages) {
++ entries.add(new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getHandle()));
}
-- DeleteMessageBatchRequest request = new DeleteMessageBatchRequest(url,entries);
++ DeleteMessageBatchRequest request = new DeleteMessageBatchRequest(url, entries);
DeleteMessageBatchResult result = sqs.deleteMessageBatch(request);
boolean successful = result.getFailed().size() <= 0;
-- if(!successful){
-- for( BatchResultErrorEntry failed : result.getFailed()) {
-- LOG.error("Commit failed reason: {} messages id: {}", failed.getMessage(),failed.getId());
++ if (!successful) {
++ for (BatchResultErrorEntry failed : result.getFailed()) {
++ LOG.error("Commit failed reason: {} messages id: {}", failed.getMessage(), failed.getId());
}
}
}
++ private static void setDeadLetterQueue(AmazonSQSClient client, QueueFig fig, String queueUrl, String deadLetterName) {
++ CreateQueueRequest deadLetterQueueRequest = new CreateQueueRequest()
++ .withQueueName(deadLetterName);
++ CreateQueueResult deadLetterResult = client.createQueue(deadLetterQueueRequest);
++ String deadLetterUrl = deadLetterResult.getQueueUrl();
++ String redrivePolicy = String.format("{\"maxReceiveCount\":\"%s\", \"deadLetterTargetArn\":\"%s\"}", fig.getMaxReceiveCount(), deadLetterUrl);
++ SetQueueAttributesRequest queueAttributes = new SetQueueAttributesRequest();
++ Map<String, String> attributes = new HashMap<>();
++ attributes.put("RedrivePolicy", redrivePolicy);
++ queueAttributes.setAttributes(attributes);
++ queueAttributes.setQueueUrl(queueUrl);
++ client.setQueueAttributes(queueAttributes);
++ }
++
-- /** Read the object from Base64 string. */
-- private Object fromString( String s, Class klass ) throws IOException, ClassNotFoundException {
-- Object o = mapper.readValue(s,klass);
++ /**
++ * Read the object from Base64 string.
++ */
++ private Object fromString(String s, Class klass) throws IOException, ClassNotFoundException {
++ Object o = mapper.readValue(s, klass);
return o;
}
-- /** Write the object to a Base64 string. */
-- private String toString( Object o ) throws IOException {
++ /**
++ * Write the object to a Base64 string.
++ */
++ private String toString(Object o) throws IOException {
return mapper.writeValueAsString(o);
}
- public class SqsLoader {
- private final String key;
- private final AmazonSQSClient client;
- private final QueueFig fig;
- public SqsLoader(String key, AmazonSQSClient client,QueueFig fig) {
- this.key = key;
- this.client = client;
- this.fig = fig;
- }
+ /**
+ * Get the region
++ *
+ * @return
+ */
+ private Region getRegion() {
- Regions regions = Regions.fromName( fig.getRegion() );
- Region region = Region.getRegion( regions );
++ Regions regions = Regions.fromName(fig.getRegion());
++ Region region = Region.getRegion(regions);
+ return region;
+ }
- public AmazonSQSClient getClient() {
- return client;
- }
- public String getKey() {
- return key;
- }
+ /**
+ * Create the SQS client for the specified settings
+ */
+ private AmazonSQSClient createClient() {
+ final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- final AmazonSQSClient sqs = new AmazonSQSClient( ugProvider.getCredentials() );
++ final AmazonSQSClient sqs = new AmazonSQSClient(ugProvider.getCredentials());
+ final Region region = getRegion();
- sqs.setRegion( region );
++ sqs.setRegion(region);
- @Override
- public boolean equals(Object o){
- if(o instanceof SqsLoader){
- SqsLoader loader = (SqsLoader)o;
- return loader.getKey().equals(this.getKey());
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- int result = getKey().hashCode();
- return result;
- }
+ return sqs;
+ }
- @Override
- public String toString() {
- return getKey();
- }
++ public QueueFig config() {
++ return fig;
++ }
+
- public QueueFig config(){return fig;}
+
- }
}
[46/50] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-dev
Posted by sf...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/217e7dc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/217e7dc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/217e7dc1
Branch: refs/heads/USERGRID-473
Commit: 217e7dc13297e81e84eafdc6297fbcc1d2eb9bf3
Parents: d2be54b 07cb9df
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Mar 30 15:57:45 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Mar 30 15:57:45 2015 -0400
----------------------------------------------------------------------
.../AllApplicationsObservable.java | 8 +-
.../usergrid/corepersistence/CoreModule.java | 9 +-
.../corepersistence/CpEntityManager.java | 122 +++++-----
.../corepersistence/CpEntityManagerFactory.java | 14 +-
.../rx/impl/AbstractGraphVisitorImpl.java | 2 +-
.../rx/impl/AllApplicationsObservableImpl.java | 2 +-
.../rx/ApplicationObservableTestIT.java | 2 +-
.../core/metrics/MetricsFactory.java | 2 +-
.../data/DataMigrationManagerImpl.java | 9 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../index/ApplicationEntityIndex.java | 20 +-
.../usergrid/persistence/index/EntityIndex.java | 12 +-
.../usergrid/persistence/index/IndexAlias.java | 38 ++++
.../persistence/index/IndexBufferConsumer.java | 2 +-
.../persistence/index/IndexBufferProducer.java | 2 +-
.../usergrid/persistence/index/IndexCache.java | 39 ++++
.../persistence/index/IndexIdentifier.java | 57 +----
.../usergrid/persistence/index/SearchType.java | 12 +-
.../usergrid/persistence/index/SearchTypes.java | 3 +-
.../persistence/index/guice/IndexModule.java | 45 +++-
.../persistence/index/impl/DeIndexRequest.java | 42 ++--
.../index/impl/EntityToMapConverter.java | 173 ++++++++++++++
.../impl/EsApplicationEntityIndexImpl.java | 228 +++++++------------
.../index/impl/EsEntityIndexBatchImpl.java | 219 +-----------------
.../index/impl/EsEntityIndexFactoryImpl.java | 4 +-
.../index/impl/EsEntityIndexImpl.java | 32 ++-
.../persistence/index/impl/EsIndexCache.java | 142 ------------
.../index/impl/EsIndexCacheImpl.java | 141 ++++++++++++
.../index/impl/IndexIdentifierImpl.java | 68 ++++++
.../persistence/index/impl/IndexRequest.java | 24 +-
.../persistence/index/impl/IndexingUtils.java | 10 +-
.../impl/SearchRequestBuilderStrategy.java | 195 ++++++++++++++++
.../migration/EsIndexDataMigrationImpl.java | 105 +++++++++
.../index/migration/EsIndexMigrationPlugin.java | 46 ++++
.../index/migration/IndexDataVersions.java | 39 ++++
.../index/migration/IndexMigration.java | 35 +++
.../index/migration/LegacyIndexIdentifier.java | 78 +++++++
.../index/query/CandidateResults.java | 24 +-
.../usergrid/persistence/index/query/Query.java | 36 +--
.../index/guice/TestIndexModule.java | 51 ++++-
.../index/impl/BufferQueueSQSImplTest.java | 20 +-
.../persistence/index/impl/EntityIndexTest.java | 60 +++--
.../index/impl/IndexLoadTestsIT.java | 4 -
.../index/impl/IndexMigrationTest.java | 106 +++++++++
.../persistence/query/tree/GrammarTreeTest.java | 10 +-
.../org/apache/usergrid/rest/RootResource.java | 2 +-
46 files changed, 1513 insertions(+), 783 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/217e7dc1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index b62cdf8,d994419..62f2d1b
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@@ -16,7 -16,7 +16,8 @@@
package org.apache.usergrid.corepersistence;
+import org.apache.usergrid.corepersistence.migration.*;
+ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.springframework.context.ApplicationContext;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/217e7dc1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index baac821,99204a8..c613e44
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -156,11 -164,10 +156,9 @@@ public class CpEntityManagerFactory imp
em.getApplication();
}
- ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) );
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope);
- applicationEntityIndex.initializeIndex();
+ entityIndex.initialize();
entityIndex.refresh();
-
} catch (Exception ex) {
throw new RuntimeException("Fatal error creating system application", ex);
}
@@@ -259,14 -263,59 +257,10 @@@
throw new ApplicationAlreadyExistsException( appName );
}
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( applicationId,"application"));
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
-
getSetup().setupApplicationKeyspace( applicationId, appName );
- final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
-
-
- UUID orgUuid;
-
- if ( !cachedValue.isPresent() ) {
-
-
- // create new org because the specified one does not exist
- final String orgName = organizationName;
-
-
-
- try {
- final Entity orgInfo = em.create( "organization", new HashMap<String, Object>() {{
- put( PROPERTY_NAME, orgName );
- }} );
- orgUuid = orgInfo.getUuid();
- //evit so it's re-loaded later
- orgApplicationCache.evictOrgId( name );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- //swallow, if it exists, just get it
- orgApplicationCache.evictOrgId( organizationName );
- orgUuid = orgApplicationCache.getOrganizationId( organizationName ).get();
- }
-
- } else{
- orgUuid = cachedValue.get();
- }
-
- // create appinfo entry in the system app
- final UUID appId = applicationId;
- final UUID orgId = orgUuid;
- Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
- put( PROPERTY_NAME, appName );
- put( "applicationUuid", appId );
- put( "organizationUuid", orgId );
- }};
-
- try {
- em.create( "appinfo", appInfoMap );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- throw new ApplicationAlreadyExistsException( appName );
- }
- entityIndex.refresh();
-
- // create application entity
if ( properties == null ) {
- properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
+ properties = new TreeMap<>( CASE_INSENSITIVE_ORDER );
}
properties.put( PROPERTY_NAME, appName );
EntityManager appEm = getEntityManager( applicationId);
@@@ -722,15 -724,9 +716,9 @@@
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
EntityManager em = getEntityManager( appId );
- ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ));
- //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
- em.reindex(po);
-
em.reindex( po );
- logger.info("\n\nRebuilt index for applicationId {} \n", appId );
+ logger.info("\n\nRebuilt index for applicationId {} \n", appId);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/217e7dc1/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index 8dbea19,b57f088..d872b71
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@@ -71,20 -69,21 +71,20 @@@ public class AllApplicationsObservableI
@Override
- public Observable<ApplicationScope> getAllApplications() {
+ public Observable<ApplicationScope> getData() {
- //emit our 3 hard coded applications that are used the manage the system first.
+ //emit our hard coded applications that are used the manage the system first.
//this way consumers can perform whatever work they need to on the root system first
- final Observable<ApplicationScope> systemIds = Observable.from( Arrays
- .asList( getApplicationScope( CpNamingUtils.DEFAULT_APPLICATION_ID ),
+ final Observable<ApplicationScope> systemIds = Observable.from(
+ Arrays.asList(
getApplicationScope( CpNamingUtils.MANAGEMENT_APPLICATION_ID ),
- getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ) ) );
+ getApplicationScope( CpNamingUtils.SYSTEM_APP_ID ))); // still need deprecated system app here
-
- final ApplicationScope appScope = getApplicationScope( CpNamingUtils.SYSTEM_APP_ID );
+ final ApplicationScope appScope = getApplicationScope( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
final CollectionScope appInfoCollectionScope =
- new CollectionScopeImpl( appScope.getApplication(), appScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPINFOS ) );
+ new CollectionScopeImpl( appScope.getApplication(), appScope.getApplication(),
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPLICATION_INFOS) );
final EntityCollectionManager collectionManager =
entityCollectionManagerFactory.createCollectionManager( appInfoCollectionScope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/217e7dc1/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/217e7dc1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 98640c3,ee28939..6b3a11d
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -29,8 -30,12 +30,9 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
-
+ import org.apache.usergrid.persistence.index.migration.IndexDataVersions;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
import org.elasticsearch.action.ActionFuture;
-
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
[35/50] incubator-usergrid git commit: fixed tests
Posted by sf...@apache.org.
fixed tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b98cdce0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b98cdce0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b98cdce0
Branch: refs/heads/USERGRID-473
Commit: b98cdce0f6511cadd5490b7c79617b0789b7a011
Parents: a7e08db
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:48:48 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:48:48 2015 -0600
----------------------------------------------------------------------
.../persistence/index/EsIndexCache.java | 29 -----
.../usergrid/persistence/index/IndexCache.java | 29 +++++
.../persistence/index/guice/IndexModule.java | 2 +-
.../impl/EsApplicationEntityIndexImpl.java | 5 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 4 +-
.../index/impl/EsEntityIndexImpl.java | 17 +--
.../index/impl/EsIndexCacheImpl.java | 2 +-
.../migration/EsIndexDataMigrationImpl.java | 23 ++--
.../index/guice/TestIndexModule.java | 12 +-
.../index/impl/IndexMigrationTest.java | 112 +++++++++++++++++++
10 files changed, 167 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
deleted file mode 100644
index 6427459..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.index;
-
-/**
- * Classy class class.
- */
-public interface EsIndexCache {
- String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
-
- void invalidate(IndexAlias alias);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
new file mode 100644
index 0000000..253c66d
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
@@ -0,0 +1,29 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index;
+
+/**
+ * Classy class class.
+ */
+public interface IndexCache {
+ String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
+
+ void invalidate(IndexAlias alias);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index bb5ec54..6307382 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -51,7 +51,7 @@ public abstract class IndexModule extends AbstractModule {
bind(EntityIndexFactory.class).to(EsEntityIndexFactoryImpl.class);
bind(AliasedEntityIndex.class).to(EsEntityIndexImpl.class);
bind(EntityIndex.class).to(EsEntityIndexImpl.class);
- bind(EsIndexCache.class).to(EsIndexCacheImpl.class);
+ bind(IndexCache.class).to(EsIndexCacheImpl.class);
bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 53286b4..9248980 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -37,7 +37,6 @@ import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
@@ -73,7 +72,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
private final MapManager mapManager;
private final AliasedEntityIndex entityIndex;
private final IndexBufferProducer indexBatchBufferProducer;
- private final EsIndexCache indexCache;
+ private final IndexCache indexCache;
private final IndexFig indexFig;
private final EsProvider esProvider;
private final IndexAlias alias;
@@ -85,7 +84,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
@Inject
public EsApplicationEntityIndexImpl(@Assisted ApplicationScope appScope, final AliasedEntityIndex entityIndex, final IndexFig config,
final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexCache indexCache, final MetricsFactory metricsFactory,
final MapManagerFactory mapManagerFactory, final IndexFig indexFig, final IndexIdentifier indexIdentifier){
this.entityIndex = entityIndex;
this.indexBatchBufferProducer = indexBatchBufferProducer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 426f587..78f0e01 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -39,7 +39,7 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
private final EsProvider provider;
- private final EsIndexCache indexCache;
+ private final IndexCache indexCache;
private final IndexBufferProducer indexBatchBufferProducer;
private final MetricsFactory metricsFactory;
private final MapManagerFactory mapManagerFactory;
@@ -57,7 +57,7 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
} );
@Inject
- public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final EsIndexCache indexCache,
+ public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final IndexCache indexCache,
final IndexBufferProducer indexBatchBufferProducer,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig, final AliasedEntityIndex entityIndex, final IndexIdentifier indexIdentifier ){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a22ef38..055af2a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.VersionedData;
@@ -40,13 +39,7 @@ import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
-import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -60,8 +53,6 @@ import org.elasticsearch.index.query.*;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,7 +83,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private final EsProvider esProvider;
- private final IndexFig config;
//number of times to wait for the index to refresh properly.
private static final int MAX_WAITS = 10;
@@ -107,7 +97,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
private final IndexIdentifier indexIdentifier;
- private EsIndexCache aliasCache;
+ private IndexCache aliasCache;
private Timer mappingTimer;
private Timer refreshTimer;
private Meter refreshIndexMeter;
@@ -117,16 +107,15 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
@Inject
- public EsEntityIndexImpl( final IndexFig config,
+ public EsEntityIndexImpl(
final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexCache indexCache, final MetricsFactory metricsFactory,
final IndexFig indexFig, final IndexIdentifier indexIdentifier ) {
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
this.esProvider = provider;
- this.config = config;
this.alias = indexIdentifier.getAlias();
this.aliasCache = indexCache;
this.addTimer = metricsFactory
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
index 5f5cc8b..2958d5c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
@@ -50,7 +50,7 @@ import com.google.inject.Singleton;
* Cache for Es index operations
*/
@Singleton
-public class EsIndexCacheImpl implements EsIndexCache {
+public class EsIndexCacheImpl implements IndexCache {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final ListeningScheduledExecutorService refreshExecutors;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index fe0cfa1..ebb79dc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -23,22 +23,16 @@ import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.AliasedEntityIndex;
-import org.apache.usergrid.persistence.index.IndexAlias;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.apache.usergrid.persistence.index.EsIndexCache;
+import org.apache.usergrid.persistence.index.IndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
-import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.client.AdminClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
/**
* Classy class class.
*/
@@ -48,12 +42,12 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
private final EsProvider provider;
private final IndexFig indexFig;
private final IndexIdentifier indexIdentifier;
- private final EsIndexCache indexCache;
+ private final IndexCache indexCache;
private final VersionedData dataVersion;
private static final Logger log = LoggerFactory.getLogger(EsIndexDataMigrationImpl.class);
@Inject
- public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, EsIndexCache indexCache){
+ public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, IndexCache indexCache){
this.entityIndex = entityIndex;
this.provider = provider;
this.indexFig = indexFig;
@@ -65,7 +59,9 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
final AdminClient adminClient = provider.getClient().admin();
+ final int latestVersion = dataVersion.getImplementationVersion();
+ observer.start();
migrationDataProvider.getData().flatMap(applicationScope -> {
LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
@@ -76,11 +72,16 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ observer.update(latestVersion,"EsIndexDataMigrationImpl: fixed index: " + index );
+ })
+ .doOnError(error -> {
+ log.error("failed to migrate index", error);
+ observer.failed(latestVersion,"EsIndexDataMigrationImpl: failed to migrate",error);
})
- .doOnError(error -> log.error("failed to migrate index", error))
+ .doOnCompleted(() -> observer.complete())
.toBlocking().lastOrDefault(null);
- return dataVersion.getImplementationVersion();
+ return latestVersion;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 7a99d59..7b26376 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,7 +19,6 @@
package org.apache.usergrid.persistence.index.guice;
-import com.amazonaws.services.opsworks.model.App;
import com.google.inject.Inject;
import com.google.inject.TypeLiteral;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
@@ -29,11 +28,9 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.IndexBufferProducer;
import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.EsIndexCache;
+import org.apache.usergrid.persistence.index.IndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
-import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
@@ -63,17 +60,18 @@ public class TestIndexModule extends TestModule {
});
install( new GuicyFigModule(IndexTestFig.class) );
}
+
public static class TestAllApplicationsObservable implements MigrationDataProvider<ApplicationScope>{
final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
@Inject
- public TestAllApplicationsObservable(final IndexFig config,
+ public TestAllApplicationsObservable(
final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexCache indexCache, final MetricsFactory metricsFactory,
final IndexFig indexFig){
LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,appScope);
- EntityIndex entityIndex = new EsEntityIndexImpl(config,indexBatchBufferProducer,provider,indexCache,metricsFactory,indexFig,legacyIndexIdentifier);
+ EntityIndex entityIndex = new EsEntityIndexImpl(indexBatchBufferProducer,provider,indexCache,metricsFactory,indexFig,legacyIndexIdentifier);
entityIndex.addIndex(null, 1, 0, indexFig.getWriteConsistencyLevel());
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
new file mode 100644
index 0000000..47d81ed
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
@@ -0,0 +1,112 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
+import org.apache.usergrid.persistence.index.migration.IndexDataVersions;
+import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.util.UUID;
+
+import static junit.framework.Assert.fail;
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Classy class class.
+ */
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+public class IndexMigrationTest extends BaseIT{
+
+
+ @Inject
+ public IndexFig fig;
+ @Inject
+ public IndexIdentifier indexIdentifier;
+
+ @Inject
+ public IndexCache indexCache;
+
+ @Inject
+ public EsProvider provider;
+
+ @Inject
+ public MigrationDataProvider<ApplicationScope> applicationScopeMigrationDataProvider;
+
+ @Inject
+ public AliasedEntityIndex ei;
+ @Inject
+ public IndexBufferProducer indexBatchBufferProducer;
+ @Inject
+ public MetricsFactory metricsFactory;
+
+ @Test
+ public void TestMigrate(){
+ EsIndexDataMigrationImpl indexDataMigration = new EsIndexDataMigrationImpl(ei,provider, fig, indexIdentifier,indexCache);
+ ProgressObserver po = new ProgressObserver() {
+ @Override
+ public void start() {
+
+ }
+
+ @Override
+ public void complete() {
+
+ }
+
+ @Override
+ public void failed(int migrationVersion, String reason) {
+ fail(reason);
+ }
+
+ @Override
+ public void failed(int migrationVersion, String reason, Throwable throwable) {
+ fail(reason);
+ }
+
+ @Override
+ public void update(int migrationVersion, String message) {
+
+ }
+ };
+ final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(fig,appScope);
+
+ TestIndexModule.TestAllApplicationsObservable obs = new TestIndexModule.TestAllApplicationsObservable(indexBatchBufferProducer,provider,indexCache,metricsFactory,fig);
+ int version = indexDataMigration.migrate(0, applicationScopeMigrationDataProvider, po );
+ assertEquals(version, IndexDataVersions.SINGLE_INDEX.getVersion());
+ }
+}
+
+
+
[17/50] incubator-usergrid git commit: [USERGRID-348] Added fixes for
ExportResourceIT and changed Export endpoint t use the old method of looking
for payloads so that it is consistent with imports.
Posted by sf...@apache.org.
[USERGRID-348] Added fixes for ExportResourceIT and changed Export endpoint t use the old method of looking for payloads so that it is consistent with imports.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/66fdc612
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/66fdc612
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/66fdc612
Branch: refs/heads/USERGRID-473
Commit: 66fdc61237923809b47d496e3bfbd9db2ca80735
Parents: ee48768
Author: GERey <gr...@apigee.com>
Authored: Tue Mar 24 15:10:07 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Tue Mar 24 15:10:07 2015 -0700
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 26 +-
.../applications/ApplicationResource.java | 44 +-
.../applications/imports/ImportsResource.java | 4 -
.../rest/management/ExportResourceIT.java | 931 ++++++++-----------
.../endpoints/NamedResource.java | 12 +-
5 files changed, 438 insertions(+), 579 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/66fdc612/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index 988af0b..9f32f03 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.rest.management.organizations;
import com.sun.jersey.api.json.JSONWithPadding;
import com.sun.jersey.api.view.Viewable;
import org.apache.amber.oauth2.common.exception.OAuthSystemException;
+import org.apache.commons.lang.NullArgumentException;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.management.ActivationState;
@@ -292,40 +293,45 @@ public class OrganizationResource extends AbstractContextResource {
try {
if((properties = ( Map<String, Object> ) json.get( "properties" )) == null){
- throw new NullPointerException("Could not find 'properties'");
+ throw new NullArgumentException("Could not find 'properties'");
}
storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
String storage_provider = ( String ) properties.get( "storage_provider" );
if(storage_provider == null) {
- throw new NullPointerException( "Could not find field 'storage_provider'" );
+ throw new NullArgumentException( "Could not find field 'storage_provider'" );
}
if(storage_info == null) {
- throw new NullPointerException( "Could not find field 'storage_info'" );
+ throw new NullArgumentException( "Could not find field 'storage_info'" );
}
-
String bucketName = ( String ) storage_info.get( "bucket_location" );
- uac.getAWSAccessKeyIdJson( storage_info );
- uac.getAWSSecretKeyJson( storage_info );
+ String accessId = ( String ) storage_info.get( "s3_access_id" );
+ String secretKey = ( String ) storage_info.get( "s3_key" );
- if(bucketName == null) {
- throw new NullPointerException( "Could not find field 'bucketName'" );
+ if ( bucketName == null ) {
+ throw new NullArgumentException( "Could not find field 'bucketName'" );
+ }
+ if ( accessId == null ) {
+ throw new NullArgumentException( "Could not find field 's3_access_id'" );
}
+ if ( secretKey == null ) {
+ throw new NullArgumentException( "Could not find field 's3_key'" );
+ }
json.put( "organizationId",organization.getUuid());
jobUUID = exportService.schedule( json );
uuidRet.put( "Export Entity", jobUUID.toString() );
}
- catch ( NullPointerException e ) {
+ catch ( NullArgumentException e ) {
return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) )
.entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build();
}
catch ( Exception e ) {
//TODO:throw descriptive error message and or include on in the response
//TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
- return Response.status( SC_INTERNAL_SERVER_ERROR ).type( JSONPUtils.jsonMediaType( callback ) )
+ return Response.status( SC_INTERNAL_SERVER_ERROR ).type( JSONPUtils.jsonMediaType( callback ) )
.entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build();
}
return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/66fdc612/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index 639c41b..d7dcd87 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
import org.apache.amber.oauth2.common.exception.OAuthSystemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.commons.lang.NullArgumentException;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -243,25 +244,31 @@ public class ApplicationResource extends AbstractContextResource {
try {
if((properties = ( Map<String, Object> ) json.get( "properties" )) == null){
- throw new NullPointerException("Could not find 'properties'");
+ throw new NullArgumentException("Could not find 'properties'");
}
storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
String storage_provider = ( String ) properties.get( "storage_provider" );
if(storage_provider == null) {
- throw new NullPointerException( "Could not find field 'storage_provider'" );
+ throw new NullArgumentException( "Could not find field 'storage_provider'" );
}
if(storage_info == null) {
- throw new NullPointerException( "Could not find field 'storage_info'" );
+ throw new NullArgumentException( "Could not find field 'storage_info'" );
}
String bucketName = ( String ) storage_info.get( "bucket_location" );
+ String accessId = ( String ) storage_info.get( "s3_access_id" );
+ String secretKey = ( String ) storage_info.get( "s3_key" );
- uac.getAWSAccessKeyIdJson( storage_info );
- uac.getAWSSecretKeyJson( storage_info );
+ if ( bucketName == null ) {
+ throw new NullArgumentException( "Could not find field 'bucketName'" );
+ }
+ if ( accessId == null ) {
+ throw new NullArgumentException( "Could not find field 's3_access_id'" );
+ }
+ if ( secretKey == null ) {
- if(bucketName == null) {
- throw new NullPointerException( "Could not find field 'bucketName'" );
+ throw new NullArgumentException( "Could not find field 's3_key'" );
}
json.put("organizationId", organization.getUuid());
@@ -270,7 +277,7 @@ public class ApplicationResource extends AbstractContextResource {
jobUUID = exportService.schedule( json );
uuidRet.put( "Export Entity", jobUUID.toString() );
}
- catch ( NullPointerException e ) {
+ catch ( NullArgumentException e ) {
return Response.status( SC_BAD_REQUEST )
.type( JSONPUtils.jsonMediaType( callback ) )
.entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build();
@@ -307,25 +314,30 @@ public class ApplicationResource extends AbstractContextResource {
try {
//checkJsonExportProperties(json);
if((properties = ( Map<String, Object> ) json.get( "properties" )) == null){
- throw new NullPointerException("Could not find 'properties'");
+ throw new NullArgumentException("Could not find 'properties'");
}
storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
String storage_provider = ( String ) properties.get( "storage_provider" );
if(storage_provider == null) {
- throw new NullPointerException( "Could not find field 'storage_provider'" );
+ throw new NullArgumentException( "Could not find field 'storage_provider'" );
}
if(storage_info == null) {
- throw new NullPointerException( "Could not find field 'storage_info'" );
+ throw new NullArgumentException( "Could not find field 'storage_info'" );
}
String bucketName = ( String ) storage_info.get( "bucket_location" );
+ String accessId = ( String ) storage_info.get( "s3_access_id" );
+ String secretKey = ( String ) storage_info.get( "s3_key" );
- //check to make sure that access key and secret key are there.
- uac.getAWSAccessKeyIdJson( storage_info );
- uac.getAWSSecretKeyJson( storage_info );
+ if ( accessId == null ) {
+ throw new NullArgumentException( "Could not find field 's3_access_id'" );
+ }
+ if ( secretKey == null ) {
+ throw new NullArgumentException( "Could not find field 's3_key'" );
+ }
if(bucketName == null) {
- throw new NullPointerException( "Could not find field 'bucketName'" );
+ throw new NullArgumentException( "Could not find field 'bucketName'" );
}
json.put( "organizationId",organization.getUuid() );
@@ -335,7 +347,7 @@ public class ApplicationResource extends AbstractContextResource {
jobUUID = exportService.schedule( json );
uuidRet.put( "Export Entity", jobUUID.toString() );
}
- catch ( NullPointerException e ) {
+ catch ( NullArgumentException e ) {
return Response.status( SC_BAD_REQUEST )
.type( JSONPUtils.jsonMediaType( callback ) )
.entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/66fdc612/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
index 900b7b0..bb5f958 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
@@ -113,10 +113,6 @@ public class ImportsResource extends AbstractContextResource {
Map<String, Object> properties;
Map<String, Object> storage_info;
- // UsergridAwsCredentialsProvider uacp = new UsergridAwsCredentialsProvider();
-
- // try {
- //checkJsonExportProperties(json);
if ( ( properties = ( Map<String, Object> ) json.get( "properties" ) ) == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/66fdc612/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
index 020fce1..0465d69 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
@@ -18,32 +18,24 @@
package org.apache.usergrid.rest.management;
-import com.amazonaws.SDKGlobalConfiguration;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
-import javax.ws.rs.core.MediaType;
-
-
-import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
+import org.junit.Rule;
+import org.junit.Test;
import org.apache.usergrid.rest.TestContextSetup;
+import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
import org.apache.usergrid.rest.test.resource2point0.model.ApiResponse;
-import static org.apache.usergrid.utils.MapUtils.hashMap;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-
public class ExportResourceIT extends AbstractRestIT {
@@ -55,52 +47,6 @@ public class ExportResourceIT extends AbstractRestIT {
}
-//
-// @Ignore( "is this test still valid knowing that the sch. won't run in intelliJ?" )
-// public void exportCallCreationEntities100() throws Exception {
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-// JsonNode node = null;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = new HashMap<String, Object>();
-// Map<String, Object> properties = new HashMap<String, Object>();
-// Map<String, Object> storage_info = new HashMap<String, Object>();
-// //TODO: make sure to put a valid admin token here.
-// //TODO: always put dummy values here and ignore this test.
-//
-//
-// properties.put( "storage_provider", "s3" );
-// properties.put( "storage_info", storage_info );
-//
-// payload.put( "properties", properties );
-//
-// for ( int i = 0; i < 100; i++ ) {
-// Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred" + i );
-//
-// node = mapper.readTree( resource().path( "/test-organization/" + appName + "/app_users" )
-// .queryParam( "access_token", access_token )
-// .accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE )
-// .post( String.class, userCreation ) );
-// }
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/test-organization/apps/" + appName + "/export" )
-// .queryParam( "access_token", adminToken() )
-// .accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals( ClientResponse.Status.OK, responseStatus );
-// }
-//
-//
@Test
public void exportApplicationUUIDRetTest() throws Exception {
@@ -113,9 +59,7 @@ public class ExportResourceIT extends AbstractRestIT {
}
}
-
-
- //
+
@Test
public void exportCollectionUUIDRetTest() throws Exception {
@@ -252,10 +196,9 @@ public class ExportResourceIT extends AbstractRestIT {
// // //do an unauthorized test for both post and get
@Test
public void exportGetWrongUUID() throws Exception {
- ApiResponse exportEntity = null;
UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
try {
- exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ management().orgs().organization( clientSetup.getOrganizationName() )
.addToPath( "export" ).addToPath( fake.toString() ).get( ApiResponse.class );
fail( "Should not have been able to get fake uuid" );
}
@@ -265,486 +208,380 @@ public class ExportResourceIT extends AbstractRestIT {
}
}
-//
-// //
-// @Test
-// public void exportPostApplicationNullPointerProperties() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = new HashMap<String, Object>();
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostOrganizationNullPointerProperties() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = new HashMap<String, Object>();
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// //
-// @Test
-// public void exportPostCollectionNullPointer() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = new HashMap<String, Object>();
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportGetCollectionUnauthorized() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = mapper.readTree( resource()
-// .path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export/" + fake )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportGetApplicationUnauthorized() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export/" + fake )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .get( String.class ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportGetOrganizationUnauthorized() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + fake )
-// .accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostOrganizationNullPointerStorageInfo() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// //remove storage_info field
-// properties.remove( "storage_info" );
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostApplicationNullPointerStorageInfo() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// //remove storage_info field
-// properties.remove( "storage_info" );
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostCollectionNullPointerStorageInfo() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// //remove storage_info field
-// properties.remove( "storage_info" );
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostOrganizationNullPointerStorageProvider() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// //remove storage_info field
-// properties.remove( "storage_provider" );
-//
-//
-// try {
-// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( JsonNode.class, payload );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostApplicationNullPointerStorageProvider() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// //remove storage_info field
-// properties.remove( "storage_provider" );
-//
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostCollectionNullPointerStorageProvider() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// //remove storage_info field
-// properties.remove( "storage_provider" );
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostOrganizationNullPointerStorageVerification() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( "s3_key" );
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( JsonNode.class, payload );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-//
-// payload = payloadBuilder();
-// properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-//
-// try {
-// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( JsonNode.class, payload );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-//
-// payload = payloadBuilder();
-// properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( "bucket_location" );
-//
-// try {
-// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( JsonNode.class, payload );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostApplicationNullPointerStorageVerification() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( "s3_key" );
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-//
-// payload = payloadBuilder();
-// properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-//
-// payload = payloadBuilder();
-// properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( "bucket_location" );
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
-// @Test
-// public void exportPostCollectionNullPointerStorageVerification() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = payloadBuilder();
-// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( "s3_key" );
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-//
-// payload = payloadBuilder();
-// properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// //remove storage_key field
-// storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-//
-// payload = payloadBuilder();
-// properties = ( HashMap<String, Object> ) payload.get( "properties" );
-// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
-// storage_info.remove( "bucket_location" );
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
-//
+
+ //
+ @Test
+ public void exportPostApplicationNullPointerProperties() throws Exception {
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class,
+ new HashMap<String, Object>() );
+ fail( "Should not have passed, The payload is empty." );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+
+ @Test
+ public void exportPostOrganizationNullPointerProperties() throws Exception {
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, new HashMap<String, Object>() );
+ fail( "Should not have passed, The payload is empty." );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ //
+ @Test
+ public void exportPostCollectionNullPointer() throws Exception {
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "collection" ).addToPath( "users" )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, new HashMap<String, Object>() );
+
+ fail( "Should not have passed, The payload is empty." );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportGetCollectionUnauthorized() throws Exception {
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "collection" ).addToPath( "users" )
+ .addToPath( "export" ).addToPath( fake.toString() ).getWithoutToken( ApiResponse.class );
+ fail( "Should not have passed as we didn't have an access token." );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.UNAUTHORIZED, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportGetApplicationUnauthorized() throws Exception {
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).addToPath( fake.toString() ).getWithoutToken( ApiResponse.class );
+ fail( "Should not have passed as we didn't have an access token." );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.UNAUTHORIZED, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportGetOrganizationUnauthorized() throws Exception {
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).addToPath( fake.toString() ).getWithoutToken( ApiResponse.class );
+ fail( "Should not have passed as we didn't have an access token." );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.UNAUTHORIZED, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportPostOrganizationNullPointerStorageInfo() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ //remove storage_info field
+ properties.remove( "storage_info" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+
+ @Test
+ public void exportPostApplicationNullPointerStorageInfo() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ //remove storage_info field
+ properties.remove( "storage_info" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class,
+ payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportPostCollectionNullPointerStorageInfo() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ //remove storage_info field
+ properties.remove( "storage_info" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "collection" ).addToPath( "users" )
+ .addToPath( "export" ).postWithToken( ApiResponse.class,
+ payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportPostOrganizationNullPointerStorageProvider() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ //remove storage_info field
+ properties.remove( "storage_provider" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+
+ @Test
+ public void exportPostApplicationNullPointerStorageProvider() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ //remove storage_info field
+ properties.remove( "storage_provider" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class,
+ payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportPostCollectionNullPointerStorageProvider() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ //remove storage_info field
+ properties.remove( "storage_provider" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "collection" ).addToPath( "users" )
+ .addToPath( "export" ).postWithToken( ApiResponse.class,
+ payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+
+ @Test
+ public void exportPostOrganizationNullPointerStorageVerification() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "s3_key" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+
+ payload = payloadBuilder();
+ properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "s3_access_id");
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+
+ payload = payloadBuilder();
+ properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "bucket_location" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportPostApplicationNullPointerStorageVerification() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "s3_key" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+
+ payload = payloadBuilder();
+ properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "s3_access_id" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+
+ payload = payloadBuilder();
+ properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "bucket_location" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+ @Test
+ public void exportPostCollectionNullPointerStorageVerification() throws Exception {
+ HashMap<String, Object> payload = payloadBuilder();
+ HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "s3_key" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "collection" ).addToPath( "users" )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+
+ payload = payloadBuilder();
+ properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "s3_access_id" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "collection" ).addToPath( "users" )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+
+ payload = payloadBuilder();
+ properties = ( HashMap<String, Object> ) payload.get( "properties" );
+ storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+ //remove storage_key field
+ storage_info.remove( "bucket_location" );
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "collection" ).addToPath( "users" )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payload );
+ fail( "Should not have passed as we were missing an important part of the payload" );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+ }
+ }
+
+
/*Creates fake payload for testing purposes.*/
public HashMap<String, Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
@@ -752,8 +589,8 @@ public class ExportResourceIT extends AbstractRestIT {
Map<String, Object> storage_info = new HashMap<String, Object>();
//TODO: always put dummy values here and ignore this test.
//TODO: add a ret for when s3 values are invalid.
- storage_info.put( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR, "insert key here" );
- storage_info.put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR, "insert access id here" );
+ storage_info.put( "s3_key", "insert key here" );
+ storage_info.put( "s3_access_id", "insert access id here" );
storage_info.put( "bucket_location", "insert bucket name here" );
properties.put( "storage_provider", "s3" );
properties.put( "storage_info", storage_info );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/66fdc612/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
index daf3502..fe9d73e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
@@ -193,8 +193,16 @@ public abstract class NamedResource implements UrlResource {
public <T> T get(Class<T> type) {
GenericType<T> gt = new GenericType<>((Class) type);
return getResource( true ).type(MediaType.APPLICATION_JSON_TYPE)
- .accept( MediaType.APPLICATION_JSON )
- .get( gt.getRawClass() );
+ .accept( MediaType.APPLICATION_JSON )
+ .get( gt.getRawClass() );
+
+ }
+
+ public <T> T getWithoutToken(Class<T> type) {
+ GenericType<T> gt = new GenericType<>((Class) type);
+ return getResource().type(MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON )
+ .get( gt.getRawClass() );
}
[18/50] incubator-usergrid git commit: change query to support legacy
context
Posted by sf...@apache.org.
change query to support legacy context
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/90832ca0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/90832ca0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/90832ca0
Branch: refs/heads/USERGRID-473
Commit: 90832ca0ccab39c6d170d7d42eae3314468f5d17
Parents: e0a79db
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 16:18:56 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 16:18:56 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EntityToMapConverter.java | 8 ++++++--
.../index/impl/EsApplicationEntityIndexImpl.java | 3 ++-
.../usergrid/persistence/index/impl/IndexRequest.java | 4 +---
.../usergrid/persistence/index/impl/IndexingUtils.java | 8 +++++++-
.../apache/usergrid/persistence/index/query/Query.java | 13 +++++++++----
.../persistence/query/tree/GrammarTreeTest.java | 4 ++--
6 files changed, 27 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index 6e72159..fdd9b92 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -16,6 +16,7 @@
*/
package org.apache.usergrid.persistence.index.impl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.*;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
@@ -35,7 +36,7 @@ public class EntityToMapConverter {
* @param entity The entity
* @param context The context this entity appears in
*/
- public static Map convert( final Entity entity, final String context ) {
+ public static Map convert(ApplicationScope applicationScope, final Entity entity, final String context ) {
final Map entityMap = entityToMap( entity );
//add the context for filtering later
@@ -43,7 +44,10 @@ public class EntityToMapConverter {
//but the fieldname we have to prefix because we use query equality to seek this later.
// TODO see if we can make this more declarative
- entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase());
+ entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
+
+ entityMap.put( APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()) );
+
return entityMap;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 1debe3b..ade2286 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -145,10 +145,11 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
+ String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
.setTypes(searchTypes.getTypeNames(applicationScope))
.setScroll(cursorTimeout + "m")
- .setQuery(query.createQueryBuilder(createContextName(applicationScope, indexScope)));
+ .setQuery(query.createQueryBuilder(contexts));
final FilterBuilder fb = query.createFilterBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 9eae481..dc05820 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -55,16 +55,14 @@ public class IndexRequest implements BatchRequest {
}
public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, String context , Entity entity) {
- this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(entity,context), EntityToMapConverter.convert(entity, context));
+ this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(entity,context), EntityToMapConverter.convert(applicationScope,entity, context));
}
public IndexRequest( final String writeAlias, final ApplicationScope applicationScope,SearchType searchType, String documentId, Map<String, Object> data) {
- data.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
this.writeAlias = writeAlias;
this.entityType = searchType.getTypeName(applicationScope);
this.data = data;
this.documentId = documentId;
-
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 2fb56d1..0330bbf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -76,7 +76,13 @@ public class IndexingUtils {
sb.append( scope.getName() );
return sb.toString();
}
-
+ public static String createLegacyContextName(ApplicationScope applicationScope, IndexScope scope ) {
+ StringBuilder sb = new StringBuilder();
+ idString(sb, scope.getOwner());
+ sb.append( SEPARATOR );
+ sb.append( scope.getName() );
+ return sb.toString();
+ }
/**
* Append the id to the string
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 67a1731..438d82e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -60,6 +60,7 @@ import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
+import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@@ -147,7 +148,7 @@ public class Query {
}
- public QueryBuilder createQueryBuilder( final String context ) {
+ public QueryBuilder createQueryBuilder( final String[] contexts ) {
QueryBuilder queryBuilder = null;
@@ -178,14 +179,18 @@ public class Query {
// TODO evaluate performance when it's an all query.
// Do we need to put the context term first for performance?
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for(String context : contexts){
+ boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
+ }
+ boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
if ( queryBuilder != null ) {
- queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
- .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
+ queryBuilder = boolQueryBuilder.must( queryBuilder );
}
//nothing was specified ensure we specify the context in the search
else {
- queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+ queryBuilder = boolQueryBuilder;
}
return queryBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index 34185b3..102e222 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -411,7 +411,7 @@ public class GrammarTreeTest {
@Test
public void selectGeoWithAnd() throws RecognitionException {
String queryString = "select * where location within 20000 of 37,-75 "
- + "and created > 1407776999925 and created < 1407777000266";
+ + "and created > 1407776999925 and created < 1407777000266";
ANTLRStringStream in = new ANTLRStringStream( queryString );
CpQueryFilterLexer lexer = new CpQueryFilterLexer( in );
@@ -429,7 +429,7 @@ public class GrammarTreeTest {
assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
- QueryBuilder qb = query.createQueryBuilder("testcontext");
+ QueryBuilder qb = query.createQueryBuilder(new String[]{"testcontext"});
}
[05/50] incubator-usergrid git commit: Merge branch 'USERGRID-405'
into USERGRID-448-appinfofix
Posted by sf...@apache.org.
Merge branch 'USERGRID-405' into USERGRID-448-appinfofix
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44deb61c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44deb61c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44deb61c
Branch: refs/heads/USERGRID-473
Commit: 44deb61ce4789ec5b0dcee9bf8250788454424b0
Parents: e3d9cc9 c0384d5
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Mar 23 14:34:41 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Mar 23 14:34:41 2015 -0400
----------------------------------------------------------------------
.../batch/service/SchedulerServiceImpl.java | 17 +-
.../corepersistence/CpEntityManager.java | 41 +-
.../corepersistence/CpEntityManagerFactory.java | 70 +--
.../corepersistence/CpManagerCache.java | 6 +-
.../corepersistence/CpRelationManager.java | 19 +-
.../usergrid/corepersistence/CpSetup.java | 6 +-
.../usergrid/corepersistence/ManagerCache.java | 3 +-
.../events/EntityDeletedHandler.java | 3 +-
.../events/EntityVersionDeletedHandler.java | 4 +-
.../results/FilteringLoader.java | 3 +-
.../usergrid/persistence/EntityManager.java | 19 +-
.../cassandra/EntityManagerFactoryImpl.java | 3 +-
.../cassandra/EntityManagerImpl.java | 20 +-
.../org/apache/usergrid/CoreApplication.java | 24 +-
.../java/org/apache/usergrid/CoreITSetup.java | 6 +
.../org/apache/usergrid/CoreITSetupImpl.java | 45 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 ++
.../corepersistence/StaleIndexCleanupTest.java | 87 ++--
.../usergrid/persistence/CollectionIT.java | 80 ++--
.../usergrid/persistence/CountingMutatorIT.java | 4 +-
.../persistence/EntityConnectionsIT.java | 14 +-
.../usergrid/persistence/EntityManagerIT.java | 16 +-
.../org/apache/usergrid/persistence/GeoIT.java | 26 +-
.../persistence/GeoQueryBooleanTest.java | 4 +-
.../apache/usergrid/persistence/IndexIT.java | 14 +-
.../usergrid/persistence/PathQueryIT.java | 6 +-
.../PerformanceEntityRebuildIndexTest.java | 27 +-
.../usergrid/persistence/PermissionsIT.java | 4 +-
.../cassandra/EntityManagerFactoryImplIT.java | 4 +-
.../persistence/query/CollectionIoHelper.java | 2 +-
.../persistence/query/ConnectionHelper.java | 2 +-
.../query/IntersectionTransitivePagingIT.java | 2 +-
.../query/IntersectionUnionPagingIT.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 34 +-
.../resources/usergrid-custom-test.properties | 1 +
stack/corepersistence/queryindex/pom.xml | 57 +--
.../persistence/index/AliasedEntityIndex.java | 7 +-
.../index/ApplicationEntityIndex.java | 50 ++
.../usergrid/persistence/index/EntityIndex.java | 46 +-
.../persistence/index/EntityIndexBatch.java | 5 +
.../persistence/index/EntityIndexFactory.java | 2 +-
.../persistence/index/IndexIdentifier.java | 26 +-
.../usergrid/persistence/index/SearchType.java | 49 ++
.../usergrid/persistence/index/SearchTypes.java | 14 +-
.../persistence/index/guice/IndexModule.java | 5 +-
.../impl/EsApplicationEntityIndexImpl.java | 371 ++++++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 80 ++--
.../index/impl/EsEntityIndexFactoryImpl.java | 27 +-
.../index/impl/EsEntityIndexImpl.java | 478 ++++---------------
.../index/impl/EsIndexBufferConsumerImpl.java | 68 +--
.../persistence/index/impl/EsQueryVistor.java | 88 ++--
.../persistence/index/impl/IndexingUtils.java | 140 +++---
.../usergrid/persistence/index/query/Query.java | 6 +-
.../persistence/index/guice/IndexTestFig.java | 7 +-
.../index/impl/CorePerformanceIT.java | 0
.../impl/EntityConnectionIndexImplTest.java | 0
.../persistence/index/impl/EntityIndexTest.java | 181 +++----
.../index/impl/IndexLoadTestsIT.java | 328 +++++++++++--
.../usergrid/rest/AbstractContextResource.java | 102 ++--
.../org/apache/usergrid/rest/IndexResource.java | 2 +-
.../rest/test/RefreshIndexResource.java | 17 +-
.../resources/usergrid-custom-test.properties | 2 +
.../cassandra/ManagementServiceImpl.java | 1 -
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../apache/usergrid/management/EmailFlowIT.java | 6 +-
.../usergrid/management/OrganizationIT.java | 10 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ManagementServiceIT.java | 22 +-
.../management/export/ExportServiceIT.java | 12 +-
.../management/importer/ImportCollectionIT.java | 15 +-
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 2 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 2 +-
.../apns/NotificationsServiceIT.java | 40 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../resources/usergrid-custom-test.properties | 2 +
78 files changed, 1706 insertions(+), 1253 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 782b273,1388462..2d155a8
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -15,16 -15,33 +15,33 @@@
*/
package org.apache.usergrid.corepersistence;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.TreeMap;
+ import java.util.UUID;
+ import java.util.concurrent.atomic.AtomicBoolean;
+
+ import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.beans.BeansException;
+ import org.springframework.context.ApplicationContext;
+ import org.springframework.context.ApplicationContextAware;
+
import org.apache.commons.lang.StringUtils;
-
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.AbstractEntity;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.exception.ConflictException;
+import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
@@@ -103,9 -128,11 +121,10 @@@ public class CpEntityManagerFactory imp
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
this.injector = injector;
+ this.entityIndex = injector.getInstance(EntityIndex.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
-
- this.orgApplicationCache = new OrgApplicationCacheImpl( this );
+ this.applicationIdCache = new ApplicationIdCacheImpl( this );
}
@@@ -122,19 -149,21 +141,20 @@@
private void init() {
- EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
+ EntityManager em = getEntityManager(getManagementAppId());
try {
- if (em.getApplication() == null) {
- logger.info("Creating system application");
- Map sysAppProps = new HashMap<String, Object>();
- sysAppProps.put(PROPERTY_NAME, "systemapp");
- em.create(CpNamingUtils.SYSTEM_APP_ID, TYPE_APPLICATION, sysAppProps);
+ if ( em.getApplication() == null ) {
+ logger.info("Creating management application");
+ Map mgmtAppProps = new HashMap<String, Object>();
+ mgmtAppProps.put(PROPERTY_NAME, "systemapp");
+ em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps);
em.getApplication();
- em.createIndex();
- em.refreshIndex();
}
+ entityIndex.initializeIndex();
+ entityIndex.refresh();
+
-
} catch (Exception ex) {
throw new RuntimeException("Fatal error creating system application", ex);
}
@@@ -161,9 -190,9 +181,9 @@@
return entityManagers.get( applicationId );
}
catch ( Exception ex ) {
- logger.error("Error getting entity manager", ex);
+ logger.error("Error getting oldAppInfo manager", ex);
}
-- return _getEntityManager( applicationId );
++ return _getEntityManager(applicationId);
}
@@@ -230,40 -260,74 +250,44 @@@
throw new ApplicationAlreadyExistsException( appName );
}
- getSetup().setupApplicationKeyspace( applicationId, appName );
-
-
- final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
-
-
- UUID orgUuid;
-
- if ( !cachedValue.isPresent() ) {
-
-
- // create new org because the specified one does not exist
- final String orgName = organizationName;
-
-
-
- try {
- final Entity orgInfo = em.create( "organization", new HashMap<String, Object>() {{
- put( PROPERTY_NAME, orgName );
- }} );
- orgUuid = orgInfo.getUuid();
- //evit so it's re-loaded later
- orgApplicationCache.evictOrgId( name );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- //swallow, if it exists, just get it
- orgApplicationCache.evictOrgId( organizationName );
- orgUuid = orgApplicationCache.getOrganizationId( organizationName ).get();
- }
-
- } else{
- orgUuid = cachedValue.get();
- }
-
- // create appinfo entry in the system app
- final UUID appId = applicationId;
- final UUID orgId = orgUuid;
- Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
- put( PROPERTY_NAME, appName );
- put( "applicationUuid", appId );
- put( "organizationUuid", orgId );
- }};
+ // create application info entity
- getSetup().setupApplicationKeyspace( applicationId, appName );
+ try {
- em.create( "appinfo", appInfoMap );
++ em.create( CpNamingUtils.APPLICATION_INFO, properties );
+ }
+ catch ( DuplicateUniquePropertyExistsException e ) {
+ throw new ApplicationAlreadyExistsException( appName );
+ }
+ entityIndex.refresh();
- // create application entity
if ( properties == null ) {
- properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
+ properties = new TreeMap<>( CASE_INSENSITIVE_ORDER );
}
properties.put( PROPERTY_NAME, appName );
- EntityManager appEm = getEntityManager( applicationId );
- appEm.create( applicationId, TYPE_APPLICATION, properties );
- appEm.createIndex();
+ EntityManager appEm = getEntityManager( applicationId);
-
+ appEm.create(applicationId, TYPE_APPLICATION, properties);
appEm.resetRoles();
- appEm.refreshIndex();
+ entityIndex.refresh();
- logger.info("Initialized application {}", appName );
+ // create application info entity in the management app
- //evict app Id from cache
- orgApplicationCache.evictAppId( appName );
+ Map<String, Object> appInfoMap = new HashMap<String, Object>() {{
+ put( PROPERTY_NAME, appName );
+ put( PROPERTY_APPLICATION_ID, applicationId );
+ }};
+
+ Entity appInfo;
+ try {
+ appInfo = em.create(CpNamingUtils.APPLICATION_INFO, appInfoMap);
+ } catch (DuplicateUniquePropertyExistsException e) {
+ throw new ApplicationAlreadyExistsException(appName);
+ }
- em.refreshIndex();
+
+ // evict app Id from cache
+ applicationIdCache.evictAppId(appName);
- return applicationId;
+ logger.info("Initialized application {}", appName);
+ return appInfo;
}
@@@ -281,88 -344,48 +305,87 @@@
@Override
public void deleteApplication(UUID applicationId) throws Exception {
- //throw new UnsupportedOperationException("Delete application not supported");
+ // find application_info for application to delete
+
+ EntityManager em = getEntityManager(getManagementAppId());
- // remove old appinfo Entity, which is in the System App's appinfos collection
- EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
- Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString()));
- Results results = em.searchCollection(em.getApplicationRef(), "appinfos", q);
+ final Results results = em.searchCollection(em.getApplicationRef(), CpNamingUtils.APPLICATION_INFOS,
+ Query.fromQL("select * where " + PROPERTY_APPLICATION_ID + " = " + applicationId.toString()));
+ Entity appInfoToDelete = results.getEntity();
- Entity appToDelete = results.getEntity();
- if(appToDelete != null) {
- // create new Entity in deleted_appinfos collection, with same UUID and properties as deleted appinfo
- em.create("deleted_appinfo", appToDelete.getProperties());
- em.delete(appToDelete);
+ // ensure that there is not already a deleted app with the same name
+ final EntityRef alias = em.getAlias(
+ CpNamingUtils.DELETED_APPLICATION_INFO, appInfoToDelete.getName());
+ if ( alias != null ) {
+ throw new ConflictException("Cannot delete app with same name as already deleted app");
}
- entityIndex.refresh();
+
+ // make a copy of the app to delete application_info entity
+ // and put it in a deleted_application_info collection
+
+ Entity deletedApp = em.create(
+ CpNamingUtils.DELETED_APPLICATION_INFO, appInfoToDelete.getProperties());
+
+ // copy its connections too
+
+ final Set<String> connectionTypes = em.getConnectionTypes(appInfoToDelete);
+ for ( String connType : connectionTypes ) {
+ final Results connResults =
+ em.getConnectedEntities(appInfoToDelete, connType, null, Query.Level.ALL_PROPERTIES);
+ for ( Entity entity : connResults.getEntities() ) {
+ em.createConnection( deletedApp, connType, entity );
+ }
+ }
+
+ // delete the app from the application_info collection and delete its index
+
+ em.delete(appInfoToDelete);
- em.refreshIndex();
+
- final EntityIndex entityIndex = managerCache.getEntityIndex(
++ final ApplicationEntityIndex entityIndex = managerCache.getEntityIndex(
+ new ApplicationScopeImpl(new SimpleId(applicationId, TYPE_APPLICATION)));
+
+ applicationIdCache.evictAppId(appInfoToDelete.getName());
-
- entityIndex.deleteIndex();
}
@Override
- public void restoreApplication(UUID applicationId) throws Exception {
+ public Entity restoreApplication(UUID applicationId) throws Exception {
- // remove old delete_appinfos Entity
- EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
- Query q = Query.fromQL(String.format("select * where applicationUuid = '%s'", applicationId.toString()));
- Results results = em.searchCollection(em.getApplicationRef(), "deleted_appinfos", q);
- Entity appToRestore = results.getEntity();
+ // get the deleted_application_info for the deleted app
- if ( appToRestore == null ) {
+ EntityManager em = getEntityManager(getManagementAppId());
+
- final Results results = em.searchCollection(em.getApplicationRef(), CpNamingUtils.DELETED_APPLICATION_INFOS,
++ final Results results = em.searchCollection(
++ em.getApplicationRef(), CpNamingUtils.DELETED_APPLICATION_INFOS,
+ Query.fromQL("select * where " + PROPERTY_APPLICATION_ID + " = " + applicationId.toString()));
+ Entity deletedAppInfo = results.getEntity();
+
+ if ( deletedAppInfo == null ) {
throw new EntityNotFoundException("Cannot restore. Deleted Application not found: " + applicationId );
}
- em.delete( appToRestore );
+ // create application_info for restored app
- // restore entity in appinfo collection
- Map<String, Object> appProps = appToRestore.getProperties();
- appProps.remove("uuid");
- appProps.put("type", "appinfo");
- Entity restoredApp = em.create("appinfo", appToRestore.getProperties());
+ Entity restoredAppInfo = em.create(
+ deletedAppInfo.getUuid(), CpNamingUtils.APPLICATION_INFO, deletedAppInfo.getProperties());
+ // copy connections from deleted app entity
+
+ final Set<String> connectionTypes = em.getConnectionTypes(deletedAppInfo);
+ for ( String connType : connectionTypes ) {
+ final Results connResults =
+ em.getConnectedEntities(deletedAppInfo, connType, null, Query.Level.ALL_PROPERTIES);
+ for ( Entity entity : connResults.getEntities() ) {
+ em.createConnection( restoredAppInfo, connType, entity );
+ }
+ }
+
+ // delete the deleted app entity rebuild the app index
+
+ em.delete(deletedAppInfo);
+ entityIndex.refresh();
- // rebuild the apps index
this.rebuildApplicationIndexes(applicationId, new ProgressObserver() {
@Override
public void onProgress(EntityRef entity) {
@@@ -633,14 -664,13 +656,11 @@@
// refresh special indexes without calling EntityManager refresh because stack overflow
maybeCreateIndexes();
- // system app
+
- for ( EntityIndex index : getManagementIndexes() ) {
- index.refresh();
- }
+ entityIndex.refresh();
}
--
private void maybeCreateIndexes() {
- // system app
if ( indexInitialized.getAndSet( true ) ) {
return;
}
@@@ -651,11 -679,18 +669,11 @@@
}
- private List<EntityIndex> getManagementIndexes() {
+ private List<ApplicationEntityIndex> getManagementIndexes() {
- return Arrays.asList( managerCache.getEntityIndex(
- new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) ) ),
-
- // management app
- managerCache
- .getEntityIndex( new ApplicationScopeImpl( new SimpleId( getManagementAppId(), "application" ) ) ),
-
- // default app TODO: do we need this in two-dot-o
- managerCache
- .getEntityIndex( new ApplicationScopeImpl( new SimpleId( getDefaultAppId(), "application" ) ) ) );
+ return Arrays.asList(
+ managerCache.getEntityIndex( // management app
+ new ApplicationScopeImpl(new SimpleId(getManagementAppId(), "application"))));
}
@@@ -723,13 -750,12 +741,12 @@@
EntityManager em = getEntityManager( appId );
//explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- em.createIndex();
Application app = em.getApplication();
-- em.reindexCollection( po, collectionName, reverse );
++ em.reindexCollection(po, collectionName, reverse);
logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
- new Object[] { app.getName(), appId, collectionName } );
+ new Object[]{app.getName(), appId, collectionName});
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index c5c792b,a9e6cee..4ab7874
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@@ -95,12 -99,24 +99,12 @@@ public class CpSetup implements Setup
setupStaticKeyspace();
//force the EMF creation of indexes before creating the default applications
- emf.refreshIndex();
+ entityIndex.initializeIndex();
- injector.getInstance( DataMigrationManager.class ).migrate();
-
- logger.info( "Setting up default applications" );
-
- try {
- emf.initializeApplication( DEFAULT_ORGANIZATION, emf.getDefaultAppId(), DEFAULT_APPLICATION, null );
- }
- catch ( ApplicationAlreadyExistsException ex ) {
- logger.warn( "Application {}/{} already exists", DEFAULT_ORGANIZATION, DEFAULT_APPLICATION );
- }
- catch ( OrganizationAlreadyExistsException oaee ) {
- logger.warn( "Organization {} already exists", DEFAULT_ORGANIZATION );
- }
+ logger.info( "Setting up management app" );
try {
- emf.initializeApplication( DEFAULT_ORGANIZATION, emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
+ emf.initializeApplicationV2( DEFAULT_ORGANIZATION, emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
}
catch ( ApplicationAlreadyExistsException ex ) {
logger.warn( "Application {}/{} already exists", DEFAULT_ORGANIZATION, MANAGEMENT_APPLICATION );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index cc70a3c,0e1e62a..e8aae01
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@@ -405,10 -403,9 +405,9 @@@ public class EntityManagerFactoryImpl i
return MANAGEMENT_APPLICATION_ID;
}
-
@Override
- public UUID getDefaultAppId() {
- return DEFAULT_APPLICATION_ID;
+ public void refreshIndex() {
- // no op
++ throw new UnsupportedOperationException("Not supported.");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/CoreITSetup.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
index efa6a47,3cc0815..26254b1
--- a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
@@@ -17,32 -17,30 +17,29 @@@
package org.apache.usergrid;
--import java.util.UUID;
--
- import org.apache.usergrid.persistence.Entity;
- import org.apache.usergrid.utils.UUIDUtils;
-import org.apache.usergrid.persistence.index.EntityIndex;
--import org.junit.runner.Description;
--import org.junit.runners.model.Statement;
--import org.slf4j.Logger;
--import org.slf4j.LoggerFactory;
--
++import com.google.inject.Injector;
import org.apache.usergrid.cassandra.SpringResource;
import org.apache.usergrid.mq.QueueManagerFactory;
++import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.cassandra.CassandraService;
--import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
--import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import org.apache.usergrid.setup.ConcurrentProcessSingleton;
import org.apache.usergrid.utils.JsonUtils;
++import org.apache.usergrid.utils.UUIDUtils;
++import org.junit.runner.Description;
++import org.junit.runners.model.Statement;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
--import com.google.inject.Injector;
++import java.util.UUID;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_APPLICATION_ID;
- public class CoreITSetupImpl implements CoreITSetup {
+ public class CoreITSetupImpl implements CoreITSetup, TestEntityIndex {
private static final Logger LOG = LoggerFactory.getLogger( CoreITSetupImpl.class );
+ private final Injector injector;
protected EntityManagerFactory emf;
protected QueueManagerFactory qmf;
@@@ -145,4 -149,24 +144,24 @@@
LOG.info( name + ":\n" + JsonUtils.mapToFormattedJsonString( obj ) );
}
}
+
+ @Override
+ public Injector getInjector() {
+ return injector;
+ }
+
+ @Override
+ public TestEntityIndex getEntityIndex(){
+ return this;
+ }
+
+ @Override
+ public void refresh(){
- try{
++ try {
+ Thread.sleep(50);
- }catch (InterruptedException ie){
++ } catch (InterruptedException ie){
+
+ }
+ getEntityIndex().refresh();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 8363a24,4b284df..9336df8
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@@ -309,13 -311,16 +310,17 @@@ public class PerformanceEntityRebuildIn
// ----------------- delete the system and application indexes
- logger.debug("Deleting app index and system app index");
+ logger.debug("Deleting app index");
deleteIndex( em.getApplicationId() );
-//
-// // deleting sytem app index will interfere with other concurrently running tests
+
++ // ----------------- test that we can read them, should fail
++
++ // deleting sytem app index will interfere with other concurrently running tests
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
-//
-//
-// // ----------------- test that we can read them, should fail
-//
+
+ // ----------------- test that we can read them, should fail
+
logger.debug("Reading data, should fail this time ");
try {
readData( em, "testTypes", entityCount, 3 );
@@@ -377,12 -383,13 +383,13 @@@
Injector injector = SpringResource.getInstance().getBean( Injector.class );
EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
- Id appId = new SimpleId( appUuid, "application");
+ Id appId = new SimpleId( appUuid, Schema.TYPE_APPLICATION );
ApplicationScope scope = new ApplicationScopeImpl( appId );
- EntityIndex ei = eif.createEntityIndex(scope);
- EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
+ ApplicationEntityIndex ei = eif.createApplicationEntityIndex(scope);
+
+ ei.deleteApplication().toBlocking().lastOrDefault(null);
+ app.refreshIndex();
- eeii.deleteIndex();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 5de6f72,4290f23..771a11a
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -18,83 -18,53 +18,44 @@@
package org.apache.usergrid.persistence.index.impl;
- import java.io.IOException;
- import java.util.ArrayList;
- import java.util.List;
- import java.util.UUID;
- import java.util.concurrent.TimeUnit;
- import java.util.concurrent.atomic.AtomicBoolean;
-
+ import com.codahale.metrics.Meter;
+ import com.codahale.metrics.Timer;
+ import com.google.common.collect.ImmutableMap;
+ import com.google.inject.Inject;
+ import com.google.inject.Singleton;
+ import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.ArrayUtils;
+ import org.apache.usergrid.persistence.core.future.BetterFuture;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+ import org.apache.usergrid.persistence.core.util.Health;
+ import org.apache.usergrid.persistence.index.*;
+ import org.apache.usergrid.persistence.index.exceptions.IndexException;
-
+ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
import org.elasticsearch.action.ActionFuture;
-
+ import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
--import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
--import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
--import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
- import org.elasticsearch.action.search.SearchRequestBuilder;
- import org.elasticsearch.action.search.SearchResponse;
- import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-
+ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
- import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.*;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
- import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
- import org.elasticsearch.search.SearchHit;
- import org.elasticsearch.search.SearchHits;
- import org.elasticsearch.search.sort.FieldSortBuilder;
- import org.elasticsearch.search.sort.SortBuilders;
- import org.elasticsearch.search.sort.SortOrder;
-
-import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import org.apache.commons.lang.StringUtils;
- import org.apache.commons.lang3.ArrayUtils;
-
- import org.apache.usergrid.persistence.core.future.BetterFuture;
- import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
- import org.apache.usergrid.persistence.core.scope.ApplicationScope;
- import org.apache.usergrid.persistence.core.util.Health;
- import org.apache.usergrid.persistence.core.util.ValidationUtils;
- import org.apache.usergrid.persistence.index.AliasedEntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexBufferProducer;
- import org.apache.usergrid.persistence.index.IndexFig;
- import org.apache.usergrid.persistence.index.IndexIdentifier;
- import org.apache.usergrid.persistence.index.IndexOperationMessage;
- import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.SearchTypes;
- import org.apache.usergrid.persistence.index.exceptions.IndexException;
- import org.apache.usergrid.persistence.index.query.CandidateResult;
- import org.apache.usergrid.persistence.index.query.CandidateResults;
- import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.map.MapManager;
- import org.apache.usergrid.persistence.map.MapManagerFactory;
- import org.apache.usergrid.persistence.map.MapScope;
- import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
- import org.apache.usergrid.persistence.model.entity.Id;
- import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
- import com.codahale.metrics.Timer;
- import com.google.common.base.Preconditions;
- import com.google.common.collect.ImmutableMap;
- import com.google.inject.Inject;
- import com.google.inject.assistedinject.Assisted;
-
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+ import java.io.IOException;
-import java.util.*;
++import java.util.Arrays;
++import java.util.HashSet;
++import java.util.List;
++import java.util.Set;
/**
@@@ -600,72 -370,16 +361,35 @@@ public class EsEntityIndexImpl implemen
}
};
- doInRetry( retryOperation );
- }
-
-
- @Override
- public int getPendingTasks() {
-
- final PendingClusterTasksResponse tasksResponse = esProvider.getClient().admin()
- .cluster().pendingClusterTasks( new PendingClusterTasksRequest() ).actionGet();
-
- return tasksResponse.pendingTasks().size();
- }
-
-
- @Override
- public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
-
- //since we don't have paging inputs, there's no point in executing a query for paging.
-
- final String context = IndexingUtils.createContextName(scope);
- final SearchTypes searchTypes = SearchTypes.fromTypes(id.getType());
-
- final QueryBuilder queryBuilder =
- QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
-
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(searchTypes.getTypeNames())
- .setScroll(cursorTimeout + "m")
- .setQuery(queryBuilder);
-
- final SearchResponse searchResponse;
- try {
- //Added For Graphite Metrics
- Timer.Context timeEntityIndex = getVersionsTimer.time();
- searchResponse = srb.execute().actionGet();
- timeEntityIndex.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with elasticsearch" );
- failureMonitor.fail( "Unable to execute batch", t);
- throw t;
- }
-
-
- failureMonitor.success();
-
- return parseResults(searchResponse, new Query());
+ doInRetry(retryOperation);
}
+ /**
+ * Completely delete an index.
+ */
+ public void deleteIndex() {
+ AdminClient adminClient = esProvider.getClient().admin();
+
+ DeleteIndexResponse response = adminClient.indices()
- .prepareDelete( indexIdentifier.getIndex(null) ).get();
++ .prepareDelete(indexIdentifier.getIndex(null)).get();
+
- if ( response.isAcknowledged() ) {
- logger.info( "Deleted index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
++ if (response.isAcknowledged()) {
++ logger.info("Deleted index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
+ //invalidate the alias
+ aliasCache.invalidate(alias);
++ } else {
++ logger.info("Failed to delete index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
+ }
- else {
- logger.info( "Failed to delete index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
- }
++ }
++
++
+ public String[] getUniqueIndexes() {
+ Set<String> indexSet = new HashSet<>();
+ List<String> reads = Arrays.asList(getIndexes(AliasType.Read));
+ List<String> writes = Arrays.asList(getIndexes(AliasType.Write));
+ indexSet.addAll(reads);
+ indexSet.addAll(writes);
+ return indexSet.toArray(new String[0]);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 16bb703,854c3e0..cd8a404
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@@ -1605,84 -1636,28 +1605,83 @@@ public class ManagementServiceImpl impl
}
if ( properties == null ) {
- properties = new HashMap<String, Object>();
+ properties = new HashMap<>();
}
+ EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
+
OrganizationInfo organizationInfo = getOrganizationByUuid( organizationId );
+ Entity appInfo = emf.createApplicationV2(
+ organizationInfo.getName(), applicationName, properties);
- em.refreshIndex();
- UUID applicationId = emf.createApplication( organizationInfo.getName(), applicationName, properties );
+ writeUserToken( smf.getManagementAppId(), appInfo,
+ encryptionService.plainTextCredentials(
+ generateOAuthSecretKey( AuthPrincipalType.APPLICATION ),
+ null,
+ smf.getManagementAppId() ) );
- EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
- properties.put( "name", buildAppName( applicationName, organizationInfo ) );
- properties.put( "appUuid", applicationId );
- Entity appInfo = em.create( applicationId, APPLICATION_INFO, properties );
+ UUID applicationId = addApplicationToOrganization( organizationId, appInfo );
+ UserInfo user = null;
+ try {
+ user = SubjectUtils.getUser();
+ }
+ catch ( UnavailableSecurityManagerException e ) {
+ // occurs in the rare case that this is called before the full stack is initialized
+ logger.warn("Error getting user, application created activity will not be created", e);
+ }
+ if ( ( user != null ) && user.isAdminUser() ) {
+ postOrganizationActivity( organizationId, user, "create", appInfo, "Application", applicationName,
+ "<a href=\"mailto:" + user.getEmail() + "\">" + user.getName() + " (" + user.getEmail()
+ + ")</a> created a new application named " + applicationName, null );
+ }
- writeUserToken( smf.getManagementAppId(), appInfo, encryptionService
- .plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.APPLICATION ), null,
- smf.getManagementAppId() ) );
- addApplicationToOrganization( organizationId, applicationId, appInfo );
+
+ return new ApplicationInfo( applicationId, appInfo.getName() );
+ }
+
+
+ @Override
+ public void deleteApplication(UUID applicationId) throws Exception {
+ emf.deleteApplication( applicationId );
+ }
+
+
+ @Override
+ public ApplicationInfo restoreApplication(UUID applicationId) throws Exception {
+
+ ApplicationInfo app = getDeletedApplicationInfo( applicationId );
+ if ( app == null ) {
+ throw new EntityNotFoundException("Deleted application ID " + applicationId + " not found");
+ }
+
+ if ( emf.lookupApplication( app.getName() ) != null ) {
+ throw new ConflictException("Cannot restore application, one with that name already exists.");
+ }
+
+ // restore application_info entity
+
+ EntityManager em = emf.getEntityManager( emf.getManagementAppId() );
+ Entity appInfo = emf.restoreApplication(applicationId);
+
+ // restore token
+
+ writeUserToken( smf.getManagementAppId(), appInfo,
+ encryptionService.plainTextCredentials(
+ generateOAuthSecretKey( AuthPrincipalType.APPLICATION ),
+ null,
+ smf.getManagementAppId() ) );
+
+ String orgName = appInfo.getName().split("/")[0];
+ EntityRef alias = em.getAlias( Group.ENTITY_TYPE, orgName );
+ Entity orgEntity = em.get( alias );
+
+ addApplicationToOrganization( orgEntity.getUuid(), appInfo );
+
+ // create activity
UserInfo user = null;
- // if we call this method before the full stack is initialized
- // we'll get an exception
try {
user = SubjectUtils.getUser();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44deb61c/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
[37/50] incubator-usergrid git commit: Strings in where clause must
be quoted
Posted by sf...@apache.org.
Strings in where clause must be quoted
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2eca36f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2eca36f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2eca36f0
Branch: refs/heads/USERGRID-473
Commit: 2eca36f0af78b61bab24acafbb5bc8858cfe40cf
Parents: efb7d6f
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Mar 30 09:48:37 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Mar 30 09:48:37 2015 -0400
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManagerFactory.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2eca36f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9b7b2bb..e2e43ca 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -359,7 +359,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final Results results = em.searchCollection(
em.getApplicationRef(), CpNamingUtils.DELETED_APPLICATION_INFOS,
- Query.fromQL("select * where " + PROPERTY_APPLICATION_ID + " = " + applicationId.toString()));
+ Query.fromQL("select * where " + PROPERTY_APPLICATION_ID + " = '" + applicationId.toString() + "'"));
Entity deletedAppInfo = results.getEntity();
if ( deletedAppInfo == null ) {
[44/50] incubator-usergrid git commit: add comments for interfaces
Posted by sf...@apache.org.
add comments for interfaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3a318366
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3a318366
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3a318366
Branch: refs/heads/USERGRID-473
Commit: 3a31836627b55aef920e32c8e125d31d70a9dedf
Parents: c18ba59
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 10:14:35 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 10:14:35 2015 -0600
----------------------------------------------------------------------
.../core/metrics/MetricsFactory.java | 2 +-
.../index/ApplicationEntityIndex.java | 2 +-
.../persistence/index/IndexBufferConsumer.java | 2 +-
.../persistence/index/IndexBufferProducer.java | 2 +-
.../usergrid/persistence/index/SearchType.java | 2 +-
.../migration/EsIndexDataMigrationImpl.java | 40 +++++++++++---------
6 files changed, 28 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
index 62a5cb9..b79f33d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.persistence.core.metrics;
import com.codahale.metrics.*;
/**
- * Classy class class.
+ * Get metrics .
*/
public interface MetricsFactory {
MetricRegistry getRegistry();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 59a19eb..cb1b6f0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -24,7 +24,7 @@ import org.apache.usergrid.persistence.index.query.Query;
import rx.Observable;
/**
- * Classy class class.
+ * Entity Index for an Application.
*/
public interface ApplicationEntityIndex {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
index 40c7852..e0fb8b3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.persistence.index;
/**
- * Classy class class.
+ * Buffer index requests
*/
public interface IndexBufferConsumer {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
index 7c8b7e6..a9e6d6f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
@@ -28,7 +28,7 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
/**
- * Classy class class.
+ * Produce to index buffer consumer
*/
public interface IndexBufferProducer {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
index c039360..786219a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
@@ -24,7 +24,7 @@ import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.apache.usergrid.persistence.model.entity.Id;
/**
- * Classy class class.
+ * Get search type
*/
public class SearchType{
private final String type;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index ebb79dc..123527a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.index.IndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.indices.InvalidAliasNameException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -62,24 +63,29 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
final int latestVersion = dataVersion.getImplementationVersion();
observer.start();
- migrationDataProvider.getData().flatMap(applicationScope -> {
- LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
- String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
- return Observable.from(indexes);
- })
- .doOnNext(index -> {
- IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
- aliasesRequestBuilder = adminClient.indices().prepareAliases();
- // add read alias
- aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
- observer.update(latestVersion,"EsIndexDataMigrationImpl: fixed index: " + index );
+ try {
+ migrationDataProvider.getData().flatMap(applicationScope -> {
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
+ String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
+ return Observable.from(indexes);
})
- .doOnError(error -> {
- log.error("failed to migrate index", error);
- observer.failed(latestVersion,"EsIndexDataMigrationImpl: failed to migrate",error);
- })
- .doOnCompleted(() -> observer.complete())
- .toBlocking().lastOrDefault(null);
+ .doOnNext(index -> {
+ IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ // add read alias
+ try {
+ aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ } catch (InvalidAliasNameException e) {
+ log.debug("Failed to add alias due to name conflict",e);
+ }
+ observer.update(latestVersion, "EsIndexDataMigrationImpl: fixed index: " + index);
+ })
+ .doOnCompleted(() -> observer.complete())
+ .toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ log.error("Failed to migrate index", e);
+ throw e;
+ }
return latestVersion;
}
[09/50] incubator-usergrid git commit: [USERGRID-502] Updated BasicIT
to use the new rest test framework. Removed token testing in BasicIt because
token and other various permissions are tested in their respective testing
files. The test didn't add anyth
Posted by sf...@apache.org.
[USERGRID-502] Updated BasicIT to use the new rest test framework.
Removed token testing in BasicIt because token and other various permissions are tested in their respective testing files. The test didn't add anything of value to already existing tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fb908545
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fb908545
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fb908545
Branch: refs/heads/USERGRID-473
Commit: fb90854583fdb9e84a2cf8840d1e3eb68f9cbd49
Parents: bf04de2
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 23 15:13:49 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 23 15:16:09 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/rest/BasicIT.java | 302 ++-----------------
1 file changed, 22 insertions(+), 280 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb908545/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
index 20d0a64..3e0eda5 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
@@ -17,331 +17,73 @@
package org.apache.usergrid.rest;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.core.util.MultivaluedMapImpl;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.rest.test.security.TestAppUser;
-import org.apache.usergrid.rest.test.security.TestUser;
-import org.junit.Rule;
+import java.io.IOException;
+
+import javax.ws.rs.core.MediaType;
+
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.MultivaluedMap;
-import java.io.IOException;
-import java.util.Map;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
+import org.apache.usergrid.rest.test.resource2point0.model.Entity;
+
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
-import static org.apache.commons.lang.StringUtils.isNotBlank;
-import static org.apache.usergrid.utils.MapUtils.hashMap;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
public class BasicIT extends AbstractRestIT {
private static final Logger LOG = LoggerFactory.getLogger( BasicIT.class );
- @Rule
- public TestContextSetup context = new TestContextSetup( this );
public BasicIT() throws Exception {
super();
}
-
- public void tryTest() {
- WebResource webResource = resource();
- String json = webResource.path( "/test/hello" ).accept( MediaType.APPLICATION_JSON ).get( String.class );
- assertTrue( isNotBlank( json ) );
-
- LOG.info( json );
- }
-
-
/**
* For USERGRID-2099 where putting an entity into a generic collection is resulting in a CCE when the name is a UUID
* string.
*/
@Test
public void testGenericCollectionEntityNameUuid() throws Exception {
- JsonNode node = null;
- String orgAppPath = "/"+context.getOrgName()+"/"+context.getAppName();
- TestUser testUser = new TestAppUser( "temp"+ UUIDUtils.newTimeUUID(),
- "password","temp"+UUIDUtils.newTimeUUID()+"@usergrid.com" ).create( context );
-
- //String token = userToken( "ed@anuff.com", "sesame" );
- WebResource resource = resource().path( orgAppPath+"/suspects" )
- .queryParam( "access_token", context.getActiveUser().getToken() );
- node = mapper.readTree( resource.accept( MediaType.APPLICATION_JSON ).post( String.class ));
-
- String uuid = "4dadf156-c82f-4eb7-a437-3e574441c4db";
+ String uuid = UUIDUtils.newTimeUUID().toString();
// Notice for 'name' we replace the dash in uuid string
// with 0's making it no longer conforms to a uuid
- Map<String, String> payload = hashMap( "hair", "brown" ).map( "sex", "male" ).map( "eyes", "green" )
- .map( "name", uuid.replace( '-', '0' ) ).map( "build", "thin" ).map( "height", "6 4" );
+ Entity payload = new Entity();
+ payload.put( "name", uuid.replace( '-', '0' ) );
- node = mapper.readTree( resource.queryParam( "access_token",
- context.getActiveUser().getToken() ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
-
- logNode( node );
+ //Create a user with the payload
+ Entity returnedUser = this.app().collection( "suspects" ).post(payload);
+ assertNotNull( returnedUser );
// Now this should pass with the corrections made to USERGRID-2099 which
// disables conversion of uuid strings into UUID objects in JsonUtils
- payload = hashMap( "hair", "red" ).map( "sex", "female" ).map( "eyes", "blue" ).map( "name", uuid )
- .map( "build", "heavy" ).map( "height", "5 9" );
+ payload.put( "name", uuid );
- node = mapper.readTree( resource.accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( String.class, payload ));
+ returnedUser = this.app().collection( "suspects" ).post(payload);
- logNode( node );
+ assertNotNull( returnedUser );
}
@Test
public void testNonexistentUserAccessViaGuest() throws IOException {
- JsonNode node = null;
try {
WebResource resource = resource();
resource.path( "/test-organization/test-app/users/foobarNonexistent" );
resource.accept( MediaType.APPLICATION_JSON );
- node = mapper.readTree( resource.get( String.class ));
+ mapper.readTree( resource.get( String.class ));
}
catch ( UniformInterfaceException e ) {
- logNode( node );
assertEquals( "Guests should not be able to get a 404", 401, e.getResponse().getStatus() );
}
}
-
-
- @Test
- public void testToken() throws IOException {
- JsonNode node = null;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String username = context.getActiveUser().getUser();
- String password = context.getActiveUser().getPassword();
- String mgmtToken = context.getActiveUser().getToken();
-
-
- // test get token for admin user with bad password
-
- boolean err_thrown = false;
- try {
- node = mapper.readTree( resource().path( "/management/token" ).queryParam( "grant_type", "password" )
- .queryParam( "username", username ).queryParam( "password", "blahblah" )
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
- }
- catch ( UniformInterfaceException e ) {
- assertEquals( "Should receive a 400 Bad Request", 400, e.getResponse().getStatus() );
- err_thrown = true;
- }
- assertTrue( "Error should have been thrown", err_thrown );
-
- // test get token for admin user with correct default password
-
- // test get admin user with token
-
- node = mapper.readTree( resource().path( "/management/users/"+username ).queryParam( "access_token", mgmtToken )
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
-
- logNode( node );
-
- assertEquals( username, node.get( "data" ).get( "organizations" ).get( orgName.toLowerCase() )
- .get( "users" ).get( username ).get("name").textValue());
-
-
- // test login user with incorrect password
-
- err_thrown = false;
- try {
- node = mapper.readTree( resource().path( appName+"/token" ).queryParam( "grant_type", "password" )
- .queryParam( "username", username ).queryParam( "password", "blahblah" )
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
- }
- catch ( UniformInterfaceException e ) {
- assertEquals( "Should receive a 400 Bad Request", 400, e.getResponse().getStatus() );
- err_thrown = true;
- }
- assertTrue( "Error should have been thrown", err_thrown );
-
- // test login user with incorrect pin
-
- err_thrown = false;
- try {
- node = mapper.readTree( resource().path( appName+"/token" ).queryParam( "grant_type", "pin" )
- .queryParam( "username", username ).queryParam( "pin", "4321" )
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
- }
- catch ( UniformInterfaceException e ) {
- assertEquals( "Should receive a 400 Bad Request", 400, e.getResponse().getStatus() );
- err_thrown = true;
- }
- assertTrue( "Error should have been thrown", err_thrown );
-
- // test login user with correct password
- TestUser testUser = new TestAppUser( "temp"+ UUIDUtils.newTimeUUID(),"password",
- "temp"+UUIDUtils.newTimeUUID()+"@usergrid.com" ).create( context );
-
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/token" )
- .queryParam( "grant_type", "password" )
- .queryParam( "username", testUser.getUser() ).queryParam( "password", testUser.getPassword())
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
-
- logNode( node );
-
- String user_access_token = node.get( "access_token" ).textValue();
- assertTrue( isNotBlank( user_access_token ) );
-
- // test get app user collection with insufficient permissions
-
- err_thrown = false;
- try {
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/users" )
- .queryParam( "access_token", user_access_token ).accept( MediaType.APPLICATION_JSON )
- .get( String.class ));
- }
- catch ( UniformInterfaceException e ) {
- if ( e.getResponse().getStatus() != 401 ) {
- throw e;
- }
- err_thrown = true;
- }
-
- // test get app user with sufficient permissions
-
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/users/"+testUser.getUser() )
- .queryParam( "access_token", user_access_token ).accept( MediaType.APPLICATION_JSON )
- .get( String.class ));
- logNode( node );
-
- assertEquals( 1, node.get( "entities" ).size() );
-
- // test get app user collection with bad token
-
- err_thrown = false;
- try {
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/users" )
- .queryParam( "access_token", "blahblahblah" )
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
- }
- catch ( UniformInterfaceException e ) {
- if ( e.getResponse().getStatus() != 401 ) {
- throw e;
- }
- err_thrown = true;
- }
- assertTrue( "Error should have been thrown", err_thrown );
-
- // test get app user collection with no token
-
- err_thrown = false;
- try {
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/users" )
- .accept( MediaType.APPLICATION_JSON )
- .get( String.class ));
- }
- catch ( UniformInterfaceException e ) {
- assertEquals( "Should receive a 401 Unauthorized", 401, e.getResponse().getStatus() );
- err_thrown = true;
- }
- assertTrue( "Error should have been thrown", err_thrown );
-
- // test set app user pin
-
- MultivaluedMap<String, String> formData = new MultivaluedMapImpl();
- formData.add( "pin", "5678" );
- node = mapper.readTree( resource()
- .path( "/"+orgName+"/"+appName+"/users/"+testUser.getUser()+"/setpin" )
- .queryParam( "access_token", user_access_token )
- .type( "application/x-www-form-urlencoded" )
- .post( String.class, formData ));
-
- refreshIndex(orgName, appName);
-
- node = mapper.readTree( resource()
- .path( "/"+orgName+"/"+appName+"/token" )
- .queryParam( "grant_type", "pin" )
- .queryParam( "username", testUser.getUser() )
- .queryParam( "pin", "5678" )
- .accept( MediaType.APPLICATION_JSON )
- .get( String.class ));
-
- logNode( node );
-
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/token" )
- .queryParam( "grant_type", "pin" )
- .queryParam( "username", testUser.getUser() )
- .queryParam( "pin", "5678" )
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
-
- logNode( node );
-
- user_access_token = node.get( "access_token" ).textValue();
- assertTrue( isNotBlank( user_access_token ) );
-
- refreshIndex(orgName, appName);
-
- // test user test extension resource
-
- node = mapper.readTree( resource()
- .path( "/"+orgName+"/"+appName+"/users/"+testUser.getUser()+"/test" )
- .queryParam( "access_token", user_access_token )
- .get( String.class ));
- logNode( node );
-
- // test create user with guest permissions (no token)
-
- String testUsername="burritos"+UUIDUtils.newTimeUUID();
- String testEmail="burritos"+ UUIDUtils.newTimeUUID()+"@usergrid.com";
- String testPassword= "burritos";
- String testPin = "1234";
-
- Map<String, String> payload =
- hashMap( "email", testEmail).map( "username", testUsername ).map( "name", testUsername )
- .map( "password", testPassword ).map( "pin", testPin );
-
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/users" ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
-
- logNode( node );
-
- assertNotNull( node.get( "entities" ) );
- assertNotNull( node.get( "entities" ).get( 0 ) );
- assertNotNull( node.get( "entities" ).get( 0 ).get( "username" ) );
- assertEquals( testUsername, node.get( "entities" ).get( 0 ).get( "username" ).textValue() );
-
- // test create device with guest permissions (no token)
-
- //can't find devices endpoint. I'm not entirely sure this part of valid anymore
-// payload = hashMap( "foo", "bar" );
-//
-// node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/devices/" + UUIDGenerator.newTimeUUID() )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .put( String.class, payload ));
-//
-// logNode( node );
-
- // test create entity with guest permissions (no token), should fail
-
- payload = hashMap( "foo", "bar" );
-
- err_thrown = false;
- try {
- node = mapper.readTree( resource().path( "/"+orgName+"/"+appName+"/items" )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
- }
- catch ( UniformInterfaceException e ) {
- assertEquals( "Should receive a 401 Unauthorized", 401, e.getResponse().getStatus() );
- err_thrown = true;
- }
- assertTrue( "Error should have been thrown", err_thrown );
- }
}
[10/50] incubator-usergrid git commit: [USERGRID-348] Added first
working test under ExportResourseIT. Found bug using management gets with
AppName so I created the ability to get with the stored application uuid
instead.
Posted by sf...@apache.org.
[USERGRID-348] Added first working test under ExportResourseIT. Found bug using management gets with AppName so I created the ability to get with the stored application uuid instead.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2ad1c045
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2ad1c045
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2ad1c045
Branch: refs/heads/USERGRID-473
Commit: 2ad1c045343f746b201187d0f5165ce32099f191
Parents: fb90854
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 23 16:35:57 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 23 16:35:57 2015 -0700
----------------------------------------------------------------------
.../rest/management/ExportResourceIT.java | 1436 +++++++++---------
.../rest/test/resource2point0/ClientSetup.java | 9 +-
.../endpoints/mgmt/ApplicationResource.java | 8 +-
3 files changed, 729 insertions(+), 724 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ad1c045/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
index c8c221b..fbfc2b9 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
@@ -30,8 +30,10 @@ import java.util.UUID;
import javax.ws.rs.core.MediaType;
-import org.apache.usergrid.rest.AbstractRestIT;
+import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
+
import org.apache.usergrid.rest.TestContextSetup;
+import org.apache.usergrid.rest.test.resource2point0.model.ApiResponse;
import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.junit.Assert.assertEquals;
@@ -52,7 +54,6 @@ public class ExportResourceIT extends AbstractRestIT {
}
-
@Test
public void exportCallSuccessful() throws Exception {
ClientResponse.Status responseStatus = ClientResponse.Status.OK;
@@ -60,734 +61,729 @@ public class ExportResourceIT extends AbstractRestIT {
HashMap<String, Object> payload = payloadBuilder();
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
-
- assertEquals( ClientResponse.Status.OK, responseStatus );
- }
-
-
- @Ignore( "is this test still valid knowing that the sch. won't run in intelliJ?" )
- public void exportCallCreationEntities100() throws Exception {
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
- JsonNode node = null;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = new HashMap<String, Object>();
- Map<String, Object> properties = new HashMap<String, Object>();
- Map<String, Object> storage_info = new HashMap<String, Object>();
- //TODO: make sure to put a valid admin token here.
- //TODO: always put dummy values here and ignore this test.
-
-
- properties.put( "storage_provider", "s3" );
- properties.put( "storage_info", storage_info );
-
- payload.put( "properties", properties );
-
- for ( int i = 0; i < 100; i++ ) {
- Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred" + i );
-
- node = mapper.readTree( resource().path( "/test-organization/" + appName + "/app_users" )
- .queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE )
- .post( String.class, userCreation ) );
- }
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/test-organization/apps/" + appName + "/export" )
- .queryParam( "access_token", adminToken() )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
-
- assertEquals( ClientResponse.Status.OK, responseStatus );
- }
-
-
- @Test
- public void exportApplicationUUIDRetTest() throws Exception {
- ClientResponse.Status responseStatus = ClientResponse.Status.ACCEPTED;
- String uuid;
- UUID jobUUID = null;
- JsonNode node = null;
-
- HashMap<String, Object> payload = payloadBuilder();
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
-
- assertEquals( ClientResponse.Status.ACCEPTED, responseStatus );
- assertNotNull( node.get( "Export Entity" ) );
- }
-
-
- //
- @Test
- public void exportCollectionUUIDRetTest() throws Exception {
- ClientResponse.Status responseStatus = ClientResponse.Status.ACCEPTED;
- String uuid;
- UUID jobUUID = null;
- JsonNode node = null;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = payloadBuilder();
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/"+orgName+"/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
-
- assertEquals( ClientResponse.Status.ACCEPTED, responseStatus );
- assertNotNull( node.get( "Export Entity" ) );
- }
-
-
- @Test
- public void exportGetOrganizationJobStatTest() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = payloadBuilder();
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.OK, responseStatus );
-
- String uuid = String.valueOf( node.get( "Export Entity" ) );
- uuid = uuid.replaceAll( "\"", "" );
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
-
-
- assertEquals( ClientResponse.Status.OK, responseStatus );
- assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
- }
-
-
- //all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
- @Test
- public void exportGetApplicationJobStatTest() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = payloadBuilder();
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- String uuid = String.valueOf( node.get( "Export Entity" ) );
- uuid = uuid.replaceAll( "\"", "" );
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
-
-
- assertEquals( ClientResponse.Status.OK, responseStatus );
- assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
- }
-
-
- @Test
- public void exportGetCollectionJobStatTest() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = payloadBuilder();
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- String uuid = String.valueOf( node.get( "Export Entity" ) );
- uuid = uuid.replaceAll( "\"", "" );
try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid()).addToPath( "collection" )
+ .addToPath( "users" ).addToPath( "export" ).postWithToken(ApiResponse.class,payload );
}
catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
}
-
assertEquals( ClientResponse.Status.OK, responseStatus );
- assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
- }
-
-
- // //do an unauthorized test for both post and get
- @Test
- public void exportGetWrongUUID() throws Exception {
- JsonNode node = null;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
- UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + fake )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- //
- @Test
- public void exportPostApplicationNullPointerProperties() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = new HashMap<String, Object>();
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostOrganizationNullPointerProperties() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = new HashMap<String, Object>();
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- //
- @Test
- public void exportPostCollectionNullPointer() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = new HashMap<String, Object>();
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportGetCollectionUnauthorized() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
- UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree( resource()
- .path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export/" + fake )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
- }
-
-
- @Test
- public void exportGetApplicationUnauthorized() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
- UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export/" + fake )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .get( String.class ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
- }
-
-
- @Test
- public void exportGetOrganizationUnauthorized() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
- UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + fake )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
- }
-
-
- @Test
- public void exportPostOrganizationNullPointerStorageInfo() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- //remove storage_info field
- properties.remove( "storage_info" );
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostApplicationNullPointerStorageInfo() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- //remove storage_info field
- properties.remove( "storage_info" );
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostCollectionNullPointerStorageInfo() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- //remove storage_info field
- properties.remove( "storage_info" );
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostOrganizationNullPointerStorageProvider() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- //remove storage_info field
- properties.remove( "storage_provider" );
-
-
- try {
- node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( JsonNode.class, payload );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostApplicationNullPointerStorageProvider() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- //remove storage_info field
- properties.remove( "storage_provider" );
-
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostCollectionNullPointerStorageProvider() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- //remove storage_info field
- properties.remove( "storage_provider" );
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostOrganizationNullPointerStorageVerification() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( "s3_key" );
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( JsonNode.class, payload );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-
- payload = payloadBuilder();
- properties = ( HashMap<String, Object> ) payload.get( "properties" );
- storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-
- try {
- node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( JsonNode.class, payload );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-
- payload = payloadBuilder();
- properties = ( HashMap<String, Object> ) payload.get( "properties" );
- storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( "bucket_location" );
-
- try {
- node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( JsonNode.class, payload );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostApplicationNullPointerStorageVerification() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( "s3_key" );
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-
- payload = payloadBuilder();
- properties = ( HashMap<String, Object> ) payload.get( "properties" );
- storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-
- payload = payloadBuilder();
- properties = ( HashMap<String, Object> ) payload.get( "properties" );
- storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( "bucket_location" );
-
- try {
- node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
- }
-
-
- @Test
- public void exportPostCollectionNullPointerStorageVerification() throws Exception {
- JsonNode node = null;
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-
- String orgName = context.getOrgName();
- String appName = context.getAppName();
- String token = context.getActiveUser().getToken();
-
- HashMap<String, Object> payload = payloadBuilder();
- HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
- HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( "s3_key" );
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-
- payload = payloadBuilder();
- properties = ( HashMap<String, Object> ) payload.get( "properties" );
- storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- //remove storage_key field
- storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-
- payload = payloadBuilder();
- properties = ( HashMap<String, Object> ) payload.get( "properties" );
- storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
- storage_info.remove( "bucket_location" );
-
- try {
- node = mapper.readTree(
- resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
- .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
- assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
}
-
+//
+// @Ignore( "is this test still valid knowing that the sch. won't run in intelliJ?" )
+// public void exportCallCreationEntities100() throws Exception {
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+// JsonNode node = null;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = new HashMap<String, Object>();
+// Map<String, Object> properties = new HashMap<String, Object>();
+// Map<String, Object> storage_info = new HashMap<String, Object>();
+// //TODO: make sure to put a valid admin token here.
+// //TODO: always put dummy values here and ignore this test.
+//
+//
+// properties.put( "storage_provider", "s3" );
+// properties.put( "storage_info", storage_info );
+//
+// payload.put( "properties", properties );
+//
+// for ( int i = 0; i < 100; i++ ) {
+// Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred" + i );
+//
+// node = mapper.readTree( resource().path( "/test-organization/" + appName + "/app_users" )
+// .queryParam( "access_token", access_token )
+// .accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE )
+// .post( String.class, userCreation ) );
+// }
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/test-organization/apps/" + appName + "/export" )
+// .queryParam( "access_token", adminToken() )
+// .accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+//
+// assertEquals( ClientResponse.Status.OK, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportApplicationUUIDRetTest() throws Exception {
+// ClientResponse.Status responseStatus = ClientResponse.Status.ACCEPTED;
+// String uuid;
+// UUID jobUUID = null;
+// JsonNode node = null;
+//
+// HashMap<String, Object> payload = payloadBuilder();
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+//
+// assertEquals( ClientResponse.Status.ACCEPTED, responseStatus );
+// assertNotNull( node.get( "Export Entity" ) );
+// }
+//
+//
+// //
+// @Test
+// public void exportCollectionUUIDRetTest() throws Exception {
+// ClientResponse.Status responseStatus = ClientResponse.Status.ACCEPTED;
+// String uuid;
+// UUID jobUUID = null;
+// JsonNode node = null;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = payloadBuilder();
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/"+orgName+"/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+//
+// assertEquals( ClientResponse.Status.ACCEPTED, responseStatus );
+// assertNotNull( node.get( "Export Entity" ) );
+// }
+//
+//
+// @Test
+// public void exportGetOrganizationJobStatTest() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = payloadBuilder();
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.OK, responseStatus );
+//
+// String uuid = String.valueOf( node.get( "Export Entity" ) );
+// uuid = uuid.replaceAll( "\"", "" );
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+//
+//
+// assertEquals( ClientResponse.Status.OK, responseStatus );
+// assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
+// }
+//
+//
+// //all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
+// @Test
+// public void exportGetApplicationJobStatTest() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = payloadBuilder();
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// String uuid = String.valueOf( node.get( "Export Entity" ) );
+// uuid = uuid.replaceAll( "\"", "" );
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+//
+//
+// assertEquals( ClientResponse.Status.OK, responseStatus );
+// assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
+// }
+//
+//
+// @Test
+// public void exportGetCollectionJobStatTest() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = payloadBuilder();
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// String uuid = String.valueOf( node.get( "Export Entity" ) );
+// uuid = uuid.replaceAll( "\"", "" );
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+//
+//
+// assertEquals( ClientResponse.Status.OK, responseStatus );
+// assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
+// }
+//
+//
+// // //do an unauthorized test for both post and get
+// @Test
+// public void exportGetWrongUUID() throws Exception {
+// JsonNode node = null;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + fake )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// //
+// @Test
+// public void exportPostApplicationNullPointerProperties() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = new HashMap<String, Object>();
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostOrganizationNullPointerProperties() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = new HashMap<String, Object>();
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// //
+// @Test
+// public void exportPostCollectionNullPointer() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = new HashMap<String, Object>();
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportGetCollectionUnauthorized() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = mapper.readTree( resource()
+// .path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export/" + fake )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportGetApplicationUnauthorized() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export/" + fake )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .get( String.class ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportGetOrganizationUnauthorized() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + fake )
+// .accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.UNAUTHORIZED, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostOrganizationNullPointerStorageInfo() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// //remove storage_info field
+// properties.remove( "storage_info" );
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostApplicationNullPointerStorageInfo() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// //remove storage_info field
+// properties.remove( "storage_info" );
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostCollectionNullPointerStorageInfo() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// //remove storage_info field
+// properties.remove( "storage_info" );
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostOrganizationNullPointerStorageProvider() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// //remove storage_info field
+// properties.remove( "storage_provider" );
+//
+//
+// try {
+// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .post( JsonNode.class, payload );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostApplicationNullPointerStorageProvider() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// //remove storage_info field
+// properties.remove( "storage_provider" );
+//
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostCollectionNullPointerStorageProvider() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// //remove storage_info field
+// properties.remove( "storage_provider" );
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostOrganizationNullPointerStorageVerification() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( "s3_key" );
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .post( JsonNode.class, payload );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+//
+// payload = payloadBuilder();
+// properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+//
+// try {
+// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .post( JsonNode.class, payload );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+//
+// payload = payloadBuilder();
+// properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( "bucket_location" );
+//
+// try {
+// node = resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
+// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+// .post( JsonNode.class, payload );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostApplicationNullPointerStorageVerification() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( "s3_key" );
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+//
+// payload = payloadBuilder();
+// properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+//
+// payload = payloadBuilder();
+// properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( "bucket_location" );
+//
+// try {
+// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
+// @Test
+// public void exportPostCollectionNullPointerStorageVerification() throws Exception {
+// JsonNode node = null;
+// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
+//
+// String orgName = context.getOrgName();
+// String appName = context.getAppName();
+// String token = context.getActiveUser().getToken();
+//
+// HashMap<String, Object> payload = payloadBuilder();
+// HashMap<String, Object> properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// HashMap<String, Object> storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( "s3_key" );
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+//
+// payload = payloadBuilder();
+// properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// //remove storage_key field
+// storage_info.remove( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+//
+// payload = payloadBuilder();
+// properties = ( HashMap<String, Object> ) payload.get( "properties" );
+// storage_info = ( HashMap<String, Object> ) properties.get( "storage_info" );
+// storage_info.remove( "bucket_location" );
+//
+// try {
+// node = mapper.readTree(
+// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
+// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
+// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+// }
+// catch ( UniformInterfaceException uie ) {
+// responseStatus = uie.getResponse().getClientResponseStatus();
+// }
+// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
+// }
+//
+//
/*Creates fake payload for testing purposes.*/
public HashMap<String, Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ad1c045/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
index 65f6749..207a962 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/ClientSetup.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.rest.test.resource2point0;
import java.io.IOException;
+import org.apache.usergrid.rest.test.resource2point0.model.ApiResponse;
import org.apache.usergrid.rest.test.resource2point0.model.Application;
import org.apache.usergrid.rest.test.resource2point0.model.Credentials;
import org.apache.usergrid.rest.test.resource2point0.model.Entity;
@@ -48,6 +49,7 @@ public class ClientSetup implements TestRule {
protected String password;
protected String orgName;
protected String appName;
+ protected String appUuid;
protected Token superuserToken;
protected String superuserName = "superuser";
protected String superuserPassword = "superpassword";
@@ -112,7 +114,8 @@ public class ClientSetup implements TestRule {
//restClient.management().token().post(Token.class,new Token(username,password));
- restClient.management().orgs().organization(organization.getName()).app().post(new Application(appName));
+ ApiResponse appResponse = restClient.management().orgs().organization(organization.getName()).app().post(new Application(appName));
+ appUuid = ( String ) appResponse.getEntities().get( 0 ).get( "uuid" );
refreshIndex();
}
@@ -129,6 +132,10 @@ public class ClientSetup implements TestRule {
public String getAppName() {return appName;}
+ public String getAppUuid() {
+ return appUuid;
+ }
+
public Token getSuperuserToken() {
return superuserToken;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ad1c045/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
index be480f8..90e2c64 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt;
import javax.ws.rs.core.MediaType;
+import org.apache.usergrid.rest.test.resource2point0.endpoints.CollectionEndpoint;
import org.apache.usergrid.rest.test.resource2point0.endpoints.NamedResource;
import org.apache.usergrid.rest.test.resource2point0.endpoints.UrlResource;
import org.apache.usergrid.rest.test.resource2point0.model.Application;
@@ -46,9 +47,10 @@ public class ApplicationResource extends NamedResource {
}
- public void post(Application application) {
- getResource(true).type(MediaType.APPLICATION_JSON_TYPE)
- .accept(MediaType.APPLICATION_JSON).post(application);
+ public ApiResponse post(Application application) {
+ ApiResponse apiResponse =getResource(true).type(MediaType.APPLICATION_JSON_TYPE)
+ .accept(MediaType.APPLICATION_JSON).post(ApiResponse.class,application);
+ return apiResponse;
}
// public Entity post(Entity payload){
[13/50] incubator-usergrid git commit: remove context
Posted by sf...@apache.org.
remove context
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a9eafb87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a9eafb87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a9eafb87
Branch: refs/heads/USERGRID-473
Commit: a9eafb8757988cb875f4677eaeeb261615ccc95f
Parents: 1a61e0e
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 11:16:36 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 11:16:36 2015 -0600
----------------------------------------------------------------------
.../data/DataMigrationManagerImpl.java | 9 +-
.../persistence/index/impl/DeIndexRequest.java | 42 ++--
.../index/impl/EntityToMapConverter.java | 169 ++++++++++++++
.../impl/EsApplicationEntityIndexImpl.java | 9 +-
.../index/impl/EsEntityIndexBatchImpl.java | 220 +------------------
.../persistence/index/impl/IndexRequest.java | 19 +-
6 files changed, 221 insertions(+), 247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index ad87724..158315d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -98,13 +98,10 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
/**
* Invoke each plugin to attempt a migration
*/
- for(final MigrationPlugin plugin: executionOrder){
+ executionOrder.forEach(plugin -> {
final ProgressObserver observer = new CassandraProgressObserver(plugin.getName());
-
- plugin.run( observer );
- }
-
-
+ plugin.run(observer);
+ });
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
index 9f3ce66..98aa8cf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
@@ -21,7 +21,13 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Arrays;
+import java.util.UUID;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.client.Client;
@@ -30,6 +36,9 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
+
/**
* Represent the properties required to build an index request
@@ -37,19 +46,21 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
@JsonTypeInfo(use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class")
public class DeIndexRequest implements BatchRequest {
- public String[] indexes;
- public String entityType;
- public String documentId;
+ private String[] entityTypes;
+ private String[] indexes;
+ private String documentId;
- public DeIndexRequest( final String[] indexes, final String entityType, final String documentId) {
- this.indexes = indexes;
- this.entityType = entityType;
- this.documentId = documentId;
+ public DeIndexRequest( ) {
+
}
- public DeIndexRequest() {
+ public DeIndexRequest(String[] indexes, ApplicationScope applicationScope, IndexScope indexScope, Id id, UUID version) {
+ String context = createContextName(applicationScope,indexScope);
+ this.indexes = indexes;
+ this.entityTypes = SearchType.fromId(id).getTypeNames(applicationScope);
+ this.documentId = createIndexDocId(id, version,context);
}
@@ -58,9 +69,10 @@ public class DeIndexRequest implements BatchRequest {
for(final String index: indexes) {
- final DeleteRequestBuilder builder = client.prepareDelete( index, entityType, documentId);
-
- bulkRequest.add( builder );
+ for(String entityType : entityTypes) {
+ final DeleteRequestBuilder builder = client.prepareDelete(index, entityType, documentId);
+ bulkRequest.add(builder);
+ }
}
}
@@ -70,8 +82,8 @@ public class DeIndexRequest implements BatchRequest {
}
- public String getEntityType() {
- return entityType;
+ public String[] getEntityTypes() {
+ return entityTypes;
}
@@ -94,7 +106,7 @@ public class DeIndexRequest implements BatchRequest {
if ( !documentId.equals( that.documentId ) ) {
return false;
}
- if ( !entityType.equals( that.entityType ) ) {
+ if ( !entityTypes.equals( that.entityTypes ) ) {
return false;
}
if ( !Arrays.equals( indexes, that.indexes ) ) {
@@ -108,7 +120,7 @@ public class DeIndexRequest implements BatchRequest {
@Override
public int hashCode() {
int result = Arrays.hashCode( indexes );
- result = 31 * result + entityType.hashCode();
+ result = 31 * result + entityTypes.hashCode();
result = 31 * result + documentId.hashCode();
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
new file mode 100644
index 0000000..6e72159
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.util.*;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+
+/**
+ * Classy class class.
+ */
+public class EntityToMapConverter {
+ /**
+ * Set the entity as a map with the context
+ *
+ * @param entity The entity
+ * @param context The context this entity appears in
+ */
+ public static Map convert( final Entity entity, final String context ) {
+ final Map entityMap = entityToMap( entity );
+
+ //add the context for filtering later
+ entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
+
+ //but the fieldname we have to prefix because we use query equality to seek this later.
+ // TODO see if we can make this more declarative
+ entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase());
+
+ return entityMap;
+ }
+
+
+ /**
+ * Convert Entity to Map and Adding prefixes for types:
+ * <pre>
+ * su_ - String unanalyzed field
+ * sa_ - String analyzed field
+ * go_ - Location field nu_ - Number field
+ * bu_ - Boolean field
+ * </pre>
+ */
+ private static Map entityToMap( EntityObject entity ) {
+
+ Map<String, Object> entityMap = new HashMap<String, Object>();
+
+ for ( Object f : entity.getFields().toArray() ) {
+
+ Field field = ( Field ) f;
+
+
+ if ( f instanceof ArrayField) {
+ List list = ( List ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
+ }
+ else if ( f instanceof ListField) {
+ List list = ( List ) field.getValue();
+ entityMap.put(field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
+
+ if ( !list.isEmpty() ) {
+ if ( list.get( 0 ) instanceof String ) {
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
+ }
+ }
+ }
+ else if ( f instanceof SetField) {
+ Set set = ( Set ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( set ) ) );
+ }
+ else if ( f instanceof EntityObjectField) {
+ EntityObject eo = ( EntityObject ) field.getValue();
+ entityMap.put(EO_PREFIX + field.getName().toLowerCase(), entityToMap(eo) ); // recursion
+ }
+ else if ( f instanceof StringField ) {
+
+ // index in lower case because Usergrid queries are case insensitive
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ ( ( String ) field.getValue() ).toLowerCase() );
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ ( ( String ) field.getValue() ).toLowerCase() );
+ }
+ else if ( f instanceof LocationField ) {
+ LocationField locField = ( LocationField ) f;
+ Map<String, Object> locMap = new HashMap<String, Object>();
+
+ // field names lat and lon trigger ElasticSearch geo location
+ locMap.put( "lat", locField.getValue().getLatitude() );
+ locMap.put( "lon", locField.getValue().getLongitude() );
+ entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
+ }
+ else if( f instanceof DoubleField || f instanceof FloatField){
+ entityMap.put( DOUBLE_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if( f instanceof LongField || f instanceof IntegerField){
+ entityMap.put( LONG_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if ( f instanceof BooleanField ) {
+
+ entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if ( f instanceof UUIDField ) {
+
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ field.getValue().toString().toLowerCase() );
+ }
+ else {
+ entityMap.put( field.getName().toLowerCase(), field.getValue() );
+ }
+ }
+
+ return entityMap;
+ }
+
+
+ private static Collection processCollectionForMap( final Collection c ) {
+ if ( c.isEmpty() ) {
+ return c;
+ }
+ List processed = new ArrayList();
+ Object sample = c.iterator().next();
+
+ if ( sample instanceof Entity ) {
+ for ( Object o : c.toArray() ) {
+ Entity e = ( Entity ) o;
+ processed.add( entityToMap( e ) );
+ }
+ }
+ else if ( sample instanceof List ) {
+ for ( Object o : c.toArray() ) {
+ List list = ( List ) o;
+ processed.add( processCollectionForMap( list ) ); // recursion;
+ }
+ }
+ else if ( sample instanceof Set ) {
+ for ( Object o : c.toArray() ) {
+ Set set = ( Set ) o;
+ processed.add( processCollectionForMap( set ) ); // recursion;
+ }
+ }
+ else {
+ for ( Object o : c.toArray() ) {
+ processed.add( o );
+ }
+ }
+ return processed;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 2d01697..4c3e42c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -148,16 +148,13 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
- final String context = IndexingUtils.createContextName(applicationScope,indexScope);
- final String[] entityTypes = searchTypes.getTypeNames(applicationScope);
- QueryBuilder qb = query.createQueryBuilder(context);
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(entityTypes)
+ .setTypes(searchTypes.getTypeNames(applicationScope))
.setScroll(cursorTimeout + "m")
- .setQuery(qb);
+ .setQuery(query.createQueryBuilder(createContextName(applicationScope, indexScope)));
final FilterBuilder fb = query.createFilterBuilder();
@@ -216,7 +213,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
- this.alias.getReadAlias(), context, entityTypes, srb );
+ this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb );
}
try {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 3a6f019..8f8ac5e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -21,11 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.*;
import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.*;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.client.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,28 +31,7 @@ import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.BooleanField;
-import org.apache.usergrid.persistence.model.field.DoubleField;
-import org.apache.usergrid.persistence.model.field.EntityObjectField;
-import org.apache.usergrid.persistence.model.field.Field;
-import org.apache.usergrid.persistence.model.field.FloatField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.ListField;
-import org.apache.usergrid.persistence.model.field.LocationField;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.SetField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
-import org.apache.usergrid.persistence.model.field.value.EntityObject;
-
-import com.codahale.metrics.*;
-import com.codahale.metrics.Timer;
-
-import rx.Observable;
-import rx.functions.Func1;
-
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -95,33 +70,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
ValidationUtils.verifyEntityWrite( entity );
ValidationUtils.verifyVersion( entity.getVersion() );
//add app id for indexing
-
- final String context = createContextName(applicationScope,indexScope);
-
- if ( log.isDebugEnabled() ) {
- log.debug( "Indexing entity {}:{}\n alias: {}\n" +
- " app: {}\n scope owner: {}\n scope name: {}\n context: {}",
- entity.getId().getType(), entity.getId().getUuid(), alias.getWriteAlias(),
- applicationScope.getApplication(), indexScope.getOwner(), indexScope.getName(), context );
- }
-
- ValidationUtils.verifyEntityWrite( entity );
-
- Map<String, Object> entityAsMap = entityToMap( entity, context );
- //add app id
- entityAsMap.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
- // need prefix here because we index UUIDs as strings
-
- // let caller add these fields if needed
- // entityAsMap.put("created", entity.getId().getUuid().timestamp();
- // entityAsMap.put("updated", entity.getVersion().timestamp());
-
- String indexId = createIndexDocId( entity, context );
-
- log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
- final SearchType entityType =SearchType.fromId(entity.getId());
- container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), entityType.getTypeName(applicationScope), indexId, entityAsMap));
-
+ container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), applicationScope,indexScope, entity));
return this;
}
@@ -133,38 +82,13 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
ValidationUtils.verifyIdentity(id);
ValidationUtils.verifyVersion( version );
- final String context = createContextName(applicationScope,indexScope);
- final SearchType entityType =SearchType.fromId(id);
-
- final String indexId = createIndexDocId( id, version, context );
-
-
- if ( log.isDebugEnabled() ) {
- log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n "
- + "name {} context{}, type {},",
- new Object[] {
- id.getType(),
- id.getUuid(),
- applicationScope.getApplication(),
- indexScope.getOwner(),
- indexScope.getName(),
- context,
- entityType
- } );
- }
-
- String[] indexes = entityIndex.getIndexes(AliasedEntityIndex.AliasType.Read);
+ String[] indexes = entityIndex.getUniqueIndexes();
//get the default index if no alias exists yet
if(indexes == null ||indexes.length == 0){
indexes = new String[]{indexIdentifier.getIndex(null)};
}
-
- String[] typeNames = entityType.getTypeNames(applicationScope);
- for(String type : typeNames) {
- container.addDeIndexRequest(new DeIndexRequest(indexes, type, indexId));
- }
- log.debug("Deindexed Entity with index id " + indexId);
+ container.addDeIndexRequest(new DeIndexRequest(indexes, applicationScope,indexScope,id,version));
return this;
}
@@ -205,142 +129,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
- /**
- * Set the entity as a map with the context
- *
- * @param entity The entity
- * @param context The context this entity appears in
- */
- private static Map entityToMap( final Entity entity, final String context ) {
- final Map entityMap = entityToMap( entity );
-
- //add the context for filtering later
- entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
-
- //but the fieldname we have to prefix because we use query equality to seek this later.
- // TODO see if we can make this more declarative
- entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase());
-
- return entityMap;
- }
- /**
- * Convert Entity to Map and Adding prefixes for types:
- * <pre>
- * su_ - String unanalyzed field
- * sa_ - String analyzed field
- * go_ - Location field nu_ - Number field
- * bu_ - Boolean field
- * </pre>
- */
- private static Map entityToMap( EntityObject entity ) {
-
- Map<String, Object> entityMap = new HashMap<String, Object>();
-
- for ( Object f : entity.getFields().toArray() ) {
-
- Field field = ( Field ) f;
-
-
- if ( f instanceof ArrayField ) {
- List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
- }
- else if ( f instanceof ListField ) {
- List list = ( List ) field.getValue();
- entityMap.put(field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
-
- if ( !list.isEmpty() ) {
- if ( list.get( 0 ) instanceof String ) {
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
- }
- }
- }
- else if ( f instanceof SetField ) {
- Set set = ( Set ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( set ) ) );
- }
- else if ( f instanceof EntityObjectField ) {
- EntityObject eo = ( EntityObject ) field.getValue();
- entityMap.put(EO_PREFIX + field.getName().toLowerCase(), entityToMap(eo) ); // recursion
- }
- else if ( f instanceof StringField ) {
-
- // index in lower case because Usergrid queries are case insensitive
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- ( ( String ) field.getValue() ).toLowerCase() );
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- ( ( String ) field.getValue() ).toLowerCase() );
- }
- else if ( f instanceof LocationField ) {
- LocationField locField = ( LocationField ) f;
- Map<String, Object> locMap = new HashMap<String, Object>();
-
- // field names lat and lon trigger ElasticSearch geo location
- locMap.put( "lat", locField.getValue().getLatitude() );
- locMap.put( "lon", locField.getValue().getLongitude() );
- entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
- }
- else if( f instanceof DoubleField || f instanceof FloatField){
- entityMap.put( DOUBLE_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if( f instanceof LongField || f instanceof IntegerField){
- entityMap.put( LONG_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if ( f instanceof BooleanField ) {
-
- entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if ( f instanceof UUIDField ) {
-
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- field.getValue().toString().toLowerCase() );
- }
- else {
- entityMap.put( field.getName().toLowerCase(), field.getValue() );
- }
- }
-
- return entityMap;
- }
-
-
- private static Collection processCollectionForMap( final Collection c ) {
- if ( c.isEmpty() ) {
- return c;
- }
- List processed = new ArrayList();
- Object sample = c.iterator().next();
-
- if ( sample instanceof Entity ) {
- for ( Object o : c.toArray() ) {
- Entity e = ( Entity ) o;
- processed.add( entityToMap( e ) );
- }
- }
- else if ( sample instanceof List ) {
- for ( Object o : c.toArray() ) {
- List list = ( List ) o;
- processed.add( processCollectionForMap( list ) ); // recursion;
- }
- }
- else if ( sample instanceof Set ) {
- for ( Object o : c.toArray() ) {
- Set set = ( Set ) o;
- processed.add( processCollectionForMap( set ) ); // recursion;
- }
- }
- else {
- for ( Object o : c.toArray() ) {
- processed.add( o );
- }
- }
- return processed;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 4ec4092..23f3d08 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -22,12 +22,20 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Map;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString;
+
/**
* Represent the properties required to build an index request
@@ -42,11 +50,14 @@ public class IndexRequest implements BatchRequest {
public Map<String, Object> data;
- public IndexRequest( final String writeAlias, final String entityType, final String documentId,
- final Map<String, Object> data ) {
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, IndexScope indexScope, Entity entity) {
+ String context = createContextName(applicationScope,indexScope);
+ SearchType searchType = SearchType.fromId(entity.getId());
+ final Map<String, Object> data = EntityToMapConverter.convert(entity,context);
+ data.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
this.writeAlias = writeAlias;
- this.entityType = entityType;
- this.documentId = documentId;
+ this.entityType = searchType.getTypeName(applicationScope);
+ this.documentId = IndexingUtils.createIndexDocId(entity,context);
this.data = data;
}
[03/50] incubator-usergrid git commit: Added /revokeToken and
/revokeTokens resources to the rest test framework Fixed remaining tests in
AccessTokenIT. Extended/fixed some bugs in NamedResource.
Posted by sf...@apache.org.
Added /revokeToken and /revokeTokens resources to the rest test framework
Fixed remaining tests in AccessTokenIT.
Extended/fixed some bugs in NamedResource.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d7734af9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d7734af9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d7734af9
Branch: refs/heads/USERGRID-473
Commit: d7734af907e975baad4887d153db62c330efa2e6
Parents: a33ae23
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 23 11:12:33 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 23 11:12:33 2015 -0700
----------------------------------------------------------------------
.../usergrid/rest/management/AccessTokenIT.java | 406 ++++++++-----------
.../endpoints/NamedResource.java | 29 +-
.../endpoints/mgmt/RevokeTokenResource.java | 16 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 ++
.../endpoints/mgmt/UserResource.java | 8 +
5 files changed, 254 insertions(+), 238 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d7734af9/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java
index 2d7911d..a5adc0e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AccessTokenIT.java
@@ -16,29 +16,25 @@
*/
package org.apache.usergrid.rest.management;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.representation.Form;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+
+import org.junit.Test;
+
import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
-import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
import org.apache.usergrid.rest.test.resource2point0.model.ApiResponse;
import org.apache.usergrid.rest.test.resource2point0.model.Entity;
-import org.apache.usergrid.rest.test.resource2point0.model.Organization;
import org.apache.usergrid.rest.test.resource2point0.model.QueryParameters;
import org.apache.usergrid.rest.test.resource2point0.model.Token;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.representation.Form;
+
import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-
-import org.jclouds.rest.annotations.Api;
-import org.junit.Ignore;
-import org.junit.Test;
+import static org.junit.Assert.fail;
/**
@@ -107,7 +103,7 @@ public class AccessTokenIT extends AbstractRestIT {
queryParameters.addParam( "ttl", String.valueOf(ttl) );
return queryParameters;
}
-
+
@Test
public void meToken() throws Exception {
tokenMeSetup( 0 );
@@ -139,221 +135,171 @@ public class AccessTokenIT extends AbstractRestIT {
assertNotNull(orgProperties.get("name"));
assertNotNull(orgProperties.get("properties"));
}
-//
-// @Test
-// public void meTokenPost() throws Exception {
-// Map<String, String> payload
-// = hashMap("grant_type", "password")
-// .map("username", "test@usergrid.com").map("password", "test");
-//
-// JsonNode node = mapper.readTree(resource()
-// .path("/management/me")
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .post(String.class, payload));
-//
-// logNode(node);
-// String token = node.get("access_token").textValue();
-//
-// assertNotNull(token);
-//
-// refreshIndex("test-organization", "test-app");
-//
-// node = mapper.readTree(resource()
-// .path("/management/me")
-// .queryParam("access_token", token)
-// .accept(MediaType.APPLICATION_JSON)
-// .get(String.class));
-// logNode(node);
-// }
-//
-// @Test
-// public void meTokenPostForm() throws IOException {
-//
-// Form form = new Form();
-// form.add("grant_type", "password");
-// form.add("username", "test@usergrid.com");
-// form.add("password", "test");
-//
-// JsonNode node = mapper.readTree(resource()
-// .path("/management/me")
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_FORM_URLENCODED_TYPE)
-// .entity(form, MediaType.APPLICATION_FORM_URLENCODED_TYPE)
-// .post(String.class));
-//
-// logNode(node);
-// String token = node.get("access_token").textValue();
-//
-// assertNotNull(token);
-//
-// refreshIndex("test-organization", "test-app");
-//
-// node = mapper.readTree(resource()
-// .path("/management/me")
-// .queryParam("access_token", token)
-// .accept(MediaType.APPLICATION_JSON).get(String.class));
-// logNode(node);
-// }
-//
-// @Test
-// public void ttlNan() throws Exception {
-//
-// Map<String, String> payload = hashMap("grant_type", "password")
-// .map("username", "test@usergrid.com")
-// .map("password", "test")
-// .map("ttl", "derp");
-//
-// ClientResponse.Status responseStatus = null;
-// try {
-// resource().path("/management/token")
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .post(String.class, payload);
-// } catch (UniformInterfaceException uie) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals(ClientResponse.Status.BAD_REQUEST, responseStatus);
-// }
-//
-// @Test
-// public void ttlOverMax() throws Exception {
-//
-// Map<String, String> payload = hashMap("grant_type", "password")
-// .map("username", "test@usergrid.com")
-// .map("password", "test")
-// .map("ttl", Long.MAX_VALUE + "");
-//
-// ClientResponse.Status responseStatus = null;
-//
-// try {
-// resource().path("/management/token")
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .post(String.class, payload);
-// } catch (UniformInterfaceException uie) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals(ClientResponse.Status.BAD_REQUEST, responseStatus);
-// }
-//
-// @Test
-// public void revokeToken() throws Exception {
-// String token1 = super.adminToken();
-// String token2 = super.adminToken();
-//
-// JsonNode response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token1)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-//
-// assertEquals("test@usergrid.com", response.get("data").get("email").asText());
-//
-// response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token2)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-//
-// assertEquals("test@usergrid.com", response.get("data").get("email").asText());
-//
-// // now revoke the tokens
-// response = mapper.readTree(resource().path("/management/users/test/revoketokens")
-// .queryParam("access_token", superAdminToken())
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .post(String.class));
-//
-// refreshIndex("test-organization", "test-app");
-//
-// // the tokens shouldn't work
-// ClientResponse.Status status = null;
-//
-// try {
-// response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token1)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-// } catch (UniformInterfaceException uie) {
-// status = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals(ClientResponse.Status.UNAUTHORIZED, status);
-//
-// status = null;
-//
-// try {
-// response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token2)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-// } catch (UniformInterfaceException uie) {
-// status = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals(ClientResponse.Status.UNAUTHORIZED, status);
-//
-// String token3 = super.adminToken();
-// String token4 = super.adminToken();
-//
-// response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token3)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-//
-// assertEquals("test@usergrid.com", response.get("data").get("email").asText());
-//
-// response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token4)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-//
-// assertEquals("test@usergrid.com", response.get("data").get("email").asText());
-//
-// // now revoke the token3
-// response = mapper.readTree(resource().path("/management/users/test/revoketoken")
-// .queryParam("access_token", token3)
-// .queryParam("token", token3)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .post(String.class));
-//
-// // the token3 shouldn't work
-// status = null;
-//
-// try {
-// response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token3)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-// } catch (UniformInterfaceException uie) {
-// status = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals(ClientResponse.Status.UNAUTHORIZED, status);
-//
-// status = null;
-//
-// try {
-// response = mapper.readTree(resource().path("/management/users/test")
-// .queryParam("access_token", token4)
-// .accept(MediaType.APPLICATION_JSON)
-// .type(MediaType.APPLICATION_JSON_TYPE)
-// .get(String.class));
-//
-// status = ClientResponse.Status.OK;
-// } catch (UniformInterfaceException uie) {
-// status = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals(ClientResponse.Status.OK, status);
-// }
+
+
+ /**
+ * Verify that we can POST and GET using the token that was returned.
+ * @throws Exception
+ */
+ @Test
+ public void meTokenPost() throws Exception {
+ Map<String, String> payload
+ = hashMap("grant_type", "password")
+ .map("username", clientSetup.getUsername()).map("password", clientSetup.getPassword());
+
+ Token token = management().me().post( Token.class, payload );
+
+ assertNotNull( token );
+ assertNotNull( token.getAccessToken() );
+ management().token().setToken( token );
+
+ refreshIndex();
+
+ assertNotNull( management().me().get( Token.class ) );
+
+ }
+
+
+ /**
+ * Verifies that we can POST using a form and GET using the token that was returned.
+ * @throws IOException
+ */
+ @Test
+ public void meTokenPostForm() throws IOException {
+
+ Form form = new Form();
+ form.add("grant_type", "password");
+ form.add("username", clientSetup.getUsername());
+ form.add("password", clientSetup.getPassword());
+
+ Token adminToken = management().me().post( Token.class,form );
+
+ assertNotNull( adminToken );
+ assertNotNull( adminToken.getAccessToken() );
+
+ refreshIndex();
+
+ assertNotNull( management().me().get( Token.class ) );
+
+ }
+
+
+ /**
+ * Checks we get approriate response when giving a bad ttl request
+ * @throws Exception
+ */
+ @Test
+ public void ttlNan() throws Exception {
+
+ Map<String, String> payload = hashMap("grant_type", "password")
+ .map("username", clientSetup.getUsername())
+ .map("password", clientSetup.getPassword())
+ .map("ttl", "derp");
+
+ try {
+ management().token().post( Token.class,payload );
+ } catch (UniformInterfaceException uie) {
+ assertEquals(ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus());
+ }
+
+ }
+
+ /**
+ * Checks we get approriate response when giving a bad ttl request
+ * @throws Exception
+ */
+ @Test
+ public void ttlOverMax() throws Exception {
+
+ Map<String, String> payload = hashMap("grant_type", "password")
+ .map("username", clientSetup.getUsername())
+ .map("password", clientSetup.getPassword())
+ .map("ttl", Long.MAX_VALUE + "");
+
+ try {
+ management().token().post( Token.class, payload );
+ } catch (UniformInterfaceException uie) {
+ assertEquals(ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus());
+ }
+
+ }
+
+ /**
+ * Tests that we can revoke all of the tokens that have been assigned to a specific user
+ * @throws Exception
+ */
+ @Test
+ public void revokeTokens() throws Exception {
+ Token token1 = getAdminToken();
+ Token token2 = getAdminToken();
+
+ // using a superuser token, revoke all tokens associated with the admin user
+ management().token().setToken( clientSetup.getSuperuserToken() );
+ management().users().user( clientSetup.getUsername() ).revokeTokens().post( ApiResponse.class );
+
+ refreshIndex();
+
+
+ //test that token 1 doesn't work
+ try {
+ management().token().setToken( token1 );
+ management().users().user( clientSetup.getUsername() ).get();
+ fail( "Token1 should have been revoked" );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.UNAUTHORIZED, uie.getResponse().getClientResponseStatus());
+ }
+
+
+ //test that token 2 doesn't work
+ try {
+ management().token().setToken( token2 );
+ management().users().user( clientSetup.getUsername() ).get();
+ fail( "Token2 should have been revoked" );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.UNAUTHORIZED, uie.getResponse().getClientResponseStatus());
+ }
+ }
+
+ /**
+ * Tests that we can revoke a single token that has been assigned to a specific user
+ * @throws Exception
+ */
+ @Test
+ public void revokeSingleToken() throws Exception {
+ Token token1 = getAdminToken();
+ Token token2 = getAdminToken();
+
+ // using a superuser token, revoke specific token associated with the admin user
+ QueryParameters queryParameters = new QueryParameters();
+ queryParameters.addParam( "token", token1.getAccessToken() );
+
+ management().token().setToken( clientSetup.getSuperuserToken() );
+ management().users().user( clientSetup.getUsername() ).revokeToken().post( ApiResponse.class,queryParameters );
+
+ refreshIndex();
+
+
+ //test that token 1 doesn't work
+ try {
+ management().token().setToken( token1 );
+ management().users().user( clientSetup.getUsername() ).get();
+ fail( "Token1 should have been revoked" );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.UNAUTHORIZED, uie.getResponse().getClientResponseStatus());
+ }
+
+
+ //test that token 2 still works
+ try {
+ management().token().setToken( token2 );
+ management().users().user( clientSetup.getUsername() ).get();
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail( "Token2 shouldn't have been revoked" );
+
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d7734af9/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
index a5ffc9f..daf3502 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/NamedResource.java
@@ -33,6 +33,8 @@ import java.util.Set;
import java.util.UUID;
import javax.ws.rs.core.MediaType;
+import com.sun.jersey.api.representation.Form;
+
/**
@@ -150,6 +152,16 @@ public abstract class NamedResource implements UrlResource {
}
+ public <T> T post(Class<T> type, QueryParameters queryParameters) {
+ WebResource resource = getResource();
+ resource = addParametersToResource(resource, queryParameters);
+ GenericType<T> gt = new GenericType<>((Class) type);
+ return resource.type(MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON )
+ .post(gt.getRawClass());
+
+ }
+
public <T> T postWithToken(Class<T> type, Object requestEntity) {
GenericType<T> gt = new GenericType<>((Class) type);
return getResource(true).type(MediaType.APPLICATION_JSON_TYPE)
@@ -167,14 +179,15 @@ public abstract class NamedResource implements UrlResource {
}
- //Get Resources
-// public Entity get() {
-// WebResource resource = getResource(true);
-//
-// ApiResponse response = resource.type( MediaType.APPLICATION_JSON_TYPE )
-// .accept( MediaType.APPLICATION_JSON ).get( ApiResponse.class);
-// return new Entity(response);
-// }
+ public <T> T post(Class<T> type, Form requestEntity) {
+ GenericType<T> gt = new GenericType<>((Class) type);
+ return getResource()
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_FORM_URLENCODED_TYPE )
+ .entity( requestEntity, MediaType.APPLICATION_FORM_URLENCODED_TYPE )
+ .post( gt.getRawClass() );
+
+ }
//For edge cases like Organizations and Tokens without any payload
public <T> T get(Class<T> type) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d7734af9/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java
new file mode 100644
index 0000000..c727521
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokenResource.java
@@ -0,0 +1,16 @@
+package org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt;
+
+
+import org.apache.usergrid.rest.test.resource2point0.endpoints.NamedResource;
+import org.apache.usergrid.rest.test.resource2point0.endpoints.UrlResource;
+import org.apache.usergrid.rest.test.resource2point0.state.ClientContext;
+
+
+/**
+ * Handles /revokeToken endpoint ( as opposed to revokeTokens
+ */
+public class RevokeTokenResource extends NamedResource {
+ public RevokeTokenResource( final ClientContext context, final UrlResource parent ) {
+ super( "revoketoken", context, parent );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d7734af9/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokensResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokensResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokensResource.java
new file mode 100644
index 0000000..20796ae
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/RevokeTokensResource.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt;
+
+
+import org.apache.usergrid.rest.test.resource2point0.endpoints.NamedResource;
+import org.apache.usergrid.rest.test.resource2point0.endpoints.UrlResource;
+import org.apache.usergrid.rest.test.resource2point0.state.ClientContext;
+
+
+/**
+ * Handles endpoints against /revoketokens
+ */
+public class RevokeTokensResource extends NamedResource {
+ public RevokeTokensResource( final ClientContext context, final UrlResource parent ) {
+ super( "revoketokens", context, parent );
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d7734af9/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/UserResource.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/UserResource.java
index 9af23ce..a52d884 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/UserResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/UserResource.java
@@ -62,6 +62,14 @@ public class UserResource extends NamedResource {
return new OrgResource( context, this );
}
+ public RevokeTokensResource revokeTokens() {
+ return new RevokeTokensResource( context, this );
+ }
+
+ public RevokeTokenResource revokeToken() {
+ return new RevokeTokenResource( context, this );
+ }
+
public Entity get() {
WebResource resource = getResource( true );
ApiResponse response = resource.type( MediaType.APPLICATION_JSON_TYPE )
[11/50] incubator-usergrid git commit: Delete app index on app delete.
Posted by sf...@apache.org.
Delete app index on app delete.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f007f479
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f007f479
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f007f479
Branch: refs/heads/USERGRID-473
Commit: f007f47916a69b17234368572c33f69bc3ada2c7
Parents: 44deb61
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Mar 24 12:44:23 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Mar 24 12:44:23 2015 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 35 ++++++----
.../cassandra/EntityManagerFactoryImplIT.java | 68 +++++++++++---------
2 files changed, 61 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f007f479/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 2d155a8..9b7b2bb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -307,7 +307,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// find application_info for application to delete
- EntityManager em = getEntityManager(getManagementAppId());
+ final EntityManager em = getEntityManager(getManagementAppId());
final Results results = em.searchCollection(em.getApplicationRef(), CpNamingUtils.APPLICATION_INFOS,
Query.fromQL("select * where " + PROPERTY_APPLICATION_ID + " = " + applicationId.toString()));
@@ -344,6 +344,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final ApplicationEntityIndex entityIndex = managerCache.getEntityIndex(
new ApplicationScopeImpl(new SimpleId(applicationId, TYPE_APPLICATION)));
+ entityIndex.deleteApplication();
applicationIdCache.evictAppId(appInfoToDelete.getName());
}
@@ -425,23 +426,29 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public Map<String, UUID> getApplications(boolean deleted) throws Exception {
- Map<String, UUID> appMap = new HashMap<String, UUID>();
+ Map<String, UUID> appMap = new HashMap<>();
- ApplicationScope appScope = CpNamingUtils.getApplicationScope(getManagementAppId());
- GraphManager gm = managerCache.getGraphManager(appScope);
+ ApplicationScope mgmtAppScope = CpNamingUtils.getApplicationScope(getManagementAppId());
+ GraphManager gm = managerCache.getGraphManager(mgmtAppScope);
EntityManager em = getEntityManager(getManagementAppId());
- Application app = em.getApplication();
- Id fromEntityId = new SimpleId( app.getUuid(), app.getType() );
+ Application mgmtApp = em.getApplication();
+ Id fromEntityId = new SimpleId( mgmtApp.getUuid(), mgmtApp.getType() );
final String scopeName;
final String edgeType;
+
if ( deleted ) {
- edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(CpNamingUtils.DELETED_APPLICATION_INFOS);
- scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.DELETED_APPLICATION_INFOS);
+ edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(
+ CpNamingUtils.DELETED_APPLICATION_INFOS);
+ scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(
+ CpNamingUtils.DELETED_APPLICATION_INFOS);
+
} else {
- edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(CpNamingUtils.APPLICATION_INFOS );
- scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(CpNamingUtils.APPLICATION_INFOS);
+ edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(
+ CpNamingUtils.APPLICATION_INFOS );
+ scopeName = CpNamingUtils.getCollectionScopeNameFromCollectionName(
+ CpNamingUtils.APPLICATION_INFOS);
}
logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
@@ -451,7 +458,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
fromEntityId, edgeType, Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING, null ));
- //TODO This is wrong, and will result in OOM if there are too many applications. This needs to stream properly with a buffer
+ // TODO This is wrong, and will result in OOM if there are too many applications.
+ // This needs to stream properly with a buffer
+
Iterator<Edge> iter = edges.toBlocking().getIterator();
while ( iter.hasNext() ) {
@@ -464,8 +473,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
});
CollectionScope collScope = new CollectionScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
+ mgmtAppScope.getApplication(),
+ mgmtAppScope.getApplication(),
scopeName);
org.apache.usergrid.persistence.model.entity.Entity appInfo =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f007f479/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index ff3dc2b..d52f3f6 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -103,13 +103,17 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
@Test
public void testDeleteApplication() throws Exception {
+ int maxRetries = 10;
+
String rand = UUIDGenerator.newTimeUUID().toString();
// create an application with a collection and an entity
- final UUID applicationId = setup.createApplication( "test-org-" + rand, "test-app-" + rand );
+ String appName = "test-app-" + rand;
+ String orgName = "test-org-" + rand;
+ final UUID deletedAppId = setup.createApplication( orgName, appName );
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ EntityManager em = setup.getEmf().getEntityManager(deletedAppId);
Map<String, Object> properties1 = new LinkedHashMap<String, Object>();
properties1.put( "Name", "12 Angry Men" );
@@ -121,16 +125,18 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
properties2.put( "Year", 1992 );
Entity film2 = em.create( "film", properties2 );
- app.refreshIndex();
-
- // TODO: this assertion should work!
- //assertNotNull( "cannot lookup app by name", setup.getEmf().lookupApplication("test-app-" + rand) );
+ for ( int j=0; j<maxRetries; j++ ) {
+ if ( setup.getEmf().lookupApplication( orgName + "/" + appName ) != null ) {
+ break;
+ }
+ Thread.sleep( 500 );
+ }
// delete the application
- setup.getEmf().deleteApplication( applicationId );
+ setup.getEmf().deleteApplication( deletedAppId );
- app.refreshIndex();
+ // wait for it to appear in delete apps list
Func2<UUID, Map<String, UUID> ,Boolean> findApps = new Func2<UUID,Map<String, UUID> ,Boolean>() {
@Override
@@ -148,31 +154,35 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
};
boolean found = false;
- for(int i=0;i<10;i++){
- found = findApps.call(applicationId,emf.getDeletedApplications());
- if(found){
+ for ( int i=0; i<maxRetries; i++) {
+ found = findApps.call( deletedAppId, emf.getDeletedApplications() );
+ if ( found ) {
break;
- } else{
- Thread.sleep(500);
+ } else {
+ Thread.sleep( 500 );
}
}
-
assertTrue( "Deleted app must be found in in deleted apps collection", found );
// attempt to get entities in application's collections in various ways should all fail
- assertNull( setup.getEmf().lookupApplication("test-app-" + rand) );
+ assertNull( setup.getEmf().lookupApplication( orgName + "/" + appName ));
+
+ // app must not be found in apps collection
- Map<String, UUID> appMap = setup.getEmf().getApplications();
- for ( String appName : appMap.keySet() ) {
- UUID appId = appMap.get( appName );
- assertNotEquals( appId, applicationId );
- assertNotEquals( appName, "test-app-" + rand );
+ for ( int i=0; i<maxRetries; i++ ) {
+ found = findApps.call( deletedAppId, emf.getApplications() );
+ if ( found ) {
+ Thread.sleep( 500 );
+ } else {
+ break;
+ }
}
+ assertFalse( "Deleted app must not be found in apps collection", found );
// restore the app
- emf.restoreApplication( applicationId );
+ emf.restoreApplication( deletedAppId );
emf.rebuildAllIndexes(new EntityManagerFactory.ProgressObserver() {
@Override
@@ -184,18 +194,18 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
// test to see that app now works and is happy
// it should not be found in the deleted apps collection
- for(int i=0;i<10;i++){
- found = findApps.call(applicationId,emf.getDeletedApplications());
- if(!found){
+ for ( int i=0; i<maxRetries; i++ ) {
+ found = findApps.call( deletedAppId, emf.getDeletedApplications() );
+ if ( !found ) {
break;
- } else{
- Thread.sleep(500);
+ } else {
+ Thread.sleep( 500 );
}
}
assertFalse("Restored app found in deleted apps collection", found);
- for(int i=0;i<10;i++){
- found = findApps.call(applicationId,setup.getEmf().getApplications());
+ for(int i=0;i<maxRetries;i++){
+ found = findApps.call(deletedAppId,setup.getEmf().getApplications());
if(!found){
break;
} else{
@@ -205,7 +215,7 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
assertTrue("Restored app not found in apps collection", found);
// TODO: this assertion should work!
- //assertNotNull(setup.getEmf().lookupApplication("test-app-" + rand));
+ //assertNotNull(setup.getEmf().lookupApplication( orgName + "/" + appName ));
}
[16/50] incubator-usergrid git commit: factor logic into index request
Posted by sf...@apache.org.
factor logic into index request
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e0a79dbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e0a79dbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e0a79dbb
Branch: refs/heads/USERGRID-473
Commit: e0a79dbb24e1f9060b34c99effc3b966af4eaa56
Parents: a9eafb8
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 13:58:21 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 13:58:21 2015 -0600
----------------------------------------------------------------------
.../index/ApplicationEntityIndex.java | 2 --
.../impl/EsApplicationEntityIndexImpl.java | 6 ------
.../persistence/index/impl/IndexRequest.java | 17 +++++++++++------
.../index/impl/BufferQueueSQSImplTest.java | 20 ++++++++++++--------
.../persistence/index/impl/EntityIndexTest.java | 2 +-
5 files changed, 24 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 4808d8a..dd1c5fa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -19,10 +19,8 @@
*/
package org.apache.usergrid.persistence.index;
-import com.google.common.util.concurrent.ListenableFuture;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
-import org.elasticsearch.action.ListenableActionFuture;
import rx.Observable;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 4c3e42c..1debe3b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -22,11 +22,8 @@ package org.apache.usergrid.persistence.index.impl;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -61,12 +58,9 @@ import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import rx.functions.Action1;
-import rx.schedulers.Schedulers;
import java.util.*;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 23f3d08..9eae481 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -26,6 +26,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
@@ -49,18 +50,22 @@ public class IndexRequest implements BatchRequest {
public Map<String, Object> data;
-
public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, IndexScope indexScope, Entity entity) {
- String context = createContextName(applicationScope,indexScope);
- SearchType searchType = SearchType.fromId(entity.getId());
- final Map<String, Object> data = EntityToMapConverter.convert(entity,context);
+ this(writeAlias, applicationScope, createContextName(applicationScope, indexScope), entity);
+ }
+
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, String context , Entity entity) {
+ this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(entity,context), EntityToMapConverter.convert(entity, context));
+ }
+
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope,SearchType searchType, String documentId, Map<String, Object> data) {
data.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
this.writeAlias = writeAlias;
this.entityType = searchType.getTypeName(applicationScope);
- this.documentId = IndexingUtils.createIndexDocId(entity,context);
this.data = data;
- }
+ this.documentId = documentId;
+ }
/**
* DO NOT DELETE! Required for Jackson
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
index 9a362cb..6fb6021 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
@@ -20,12 +20,15 @@
package org.apache.usergrid.persistence.index.impl;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -90,22 +93,23 @@ public class BufferQueueSQSImplTest {
@Test
public void testMessageIndexing(){
+ ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
- final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testType1", "testDoc1",request1Data );
+ final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", applicationScope, SearchType.fromType("testType1"), "testDoc1",request1Data );
final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
- final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testType2", "testDoc2",request2Data );
+ final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", applicationScope, SearchType.fromType( "testType2"), "testDoc2",request2Data );
//de-index request
- final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, "testType3", "testId3" );
+ final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new IndexScopeImpl(new SimpleId("testId3"),"name3"), new SimpleId("id3"), UUID.randomUUID() );
- final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, "testType4", "testId4" );
+ final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new IndexScopeImpl(new SimpleId("testId4"),"name4"), new SimpleId("id4"), UUID.randomUUID() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 85570f1..6df5a6d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -651,7 +651,7 @@ public class EntityIndexTest extends BaseIT {
for ( int i = 0; i < expectedPages; i++ ) {
//**
- final Query query = Query.fromQL( "select * order by created" );
+ Query query = Query.fromQL( "select * order by created" );
query.setLimit( limit );
if ( cursor != null ) {
[04/50] incubator-usergrid git commit: Merge branch 'USERGRID-405' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-347
Posted by sf...@apache.org.
Merge branch 'USERGRID-405' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-347
# By Shawn Feldman (26) and others
# Via Shawn Feldman (9) and Todd Nine (4)
* 'USERGRID-405' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid: (38 commits)
undo smaller queries
smaller queries
smaller queries
move initialize index
move initialize index
Updated load tests to match new index structure
Updated tests to also contain larger body for measuring performance with more accurate entity size
changing tests to iterate fewer times
add delete back
add delete back
add delay on refresh for tests
add small sleep
search change
Updated surefire to be a single fork to avoid ClassNotFoundException
merge issues
Fixed logic issue in GeoIT test.
adding refresh retry
Fixes test that won't work when the system runs too quickly.
Fixes rebuild bug
Resolves class loader problems during test
...
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d263d36d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d263d36d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d263d36d
Branch: refs/heads/USERGRID-473
Commit: d263d36d877492b52491f2a418a5f58c209826ec
Parents: d7734af 8fe7267
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 23 11:28:57 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 23 11:28:57 2015 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 44 +-
.../batch/service/SchedulerServiceImpl.java | 17 +-
.../corepersistence/CpEntityManager.java | 43 +-
.../corepersistence/CpEntityManagerFactory.java | 72 +--
.../corepersistence/CpManagerCache.java | 6 +-
.../corepersistence/CpRelationManager.java | 43 +-
.../usergrid/corepersistence/CpSetup.java | 16 +-
.../usergrid/corepersistence/CpWalker.java | 81 ++-
.../usergrid/corepersistence/ManagerCache.java | 3 +-
.../events/EntityDeletedHandler.java | 3 +-
.../events/EntityVersionDeletedHandler.java | 72 ++-
.../migration/EntityTypeMappingMigration.java | 41 +-
.../results/FilteringLoader.java | 3 +-
.../usergrid/persistence/EntityManager.java | 19 +-
.../persistence/EntityManagerFactory.java | 2 -
.../persistence/ObservableIterator.java | 1 +
.../cassandra/EntityManagerFactoryImpl.java | 5 -
.../cassandra/EntityManagerImpl.java | 20 +-
.../org/apache/usergrid/CoreApplication.java | 24 +-
.../java/org/apache/usergrid/CoreITSetup.java | 6 +
.../org/apache/usergrid/CoreITSetupImpl.java | 27 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 +
.../corepersistence/StaleIndexCleanupTest.java | 87 ++--
.../migration/EntityTypeMappingMigrationIT.java | 2 +-
.../usergrid/persistence/CollectionIT.java | 80 +--
.../usergrid/persistence/CountingMutatorIT.java | 4 +-
.../persistence/EntityConnectionsIT.java | 14 +-
.../usergrid/persistence/EntityManagerIT.java | 16 +-
.../org/apache/usergrid/persistence/GeoIT.java | 28 +-
.../persistence/GeoQueryBooleanTest.java | 4 +-
.../apache/usergrid/persistence/IndexIT.java | 14 +-
.../usergrid/persistence/PathQueryIT.java | 6 +-
.../PerformanceEntityRebuildIndexTest.java | 39 +-
.../usergrid/persistence/PermissionsIT.java | 4 +-
.../cassandra/EntityManagerFactoryImplIT.java | 4 +-
.../persistence/query/CollectionIoHelper.java | 2 +-
.../persistence/query/ConnectionHelper.java | 2 +-
.../query/IntersectionTransitivePagingIT.java | 2 +-
.../query/IntersectionUnionPagingIT.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 16 +-
.../resources/usergrid-custom-test.properties | 1 +
stack/corepersistence/collection/pom.xml | 14 +-
.../impl/EntityCollectionManagerImpl.java | 10 +-
.../collection/impl/EntityDeletedTask.java | 20 +-
.../impl/EntityVersionCleanupTask.java | 40 +-
.../impl/EntityVersionCreatedTask.java | 26 +-
.../mvcc/stage/write/WriteCommit.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 8 +-
.../MvccEntitySerializationStrategyImpl.java | 92 ++--
.../MvccEntitySerializationStrategyV3Impl.java | 94 ++--
.../UniqueValueSerializationStrategyImpl.java | 8 -
.../migration/MvccEntityDataMigrationImpl.java | 171 +++----
.../collection/util/EntityUtils.java | 72 ---
.../mvcc/stage/AbstractEntityStageTest.java | 2 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 2 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../persistence/collection/rx/ParallelTest.java | 10 +-
...MvccEntitySerializationStrategyImplTest.java | 4 +-
...ccEntitySerializationStrategyV1ImplTest.java | 4 +-
...ccEntitySerializationStrategyV2ImplTest.java | 2 +-
.../impl/SerializationComparison.java | 4 +-
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 2 +-
.../collection/util/InvalidEntityGenerator.java | 1 +
stack/corepersistence/common/pom.xml | 15 +-
.../astyanax/MultiKeyColumnNameIterator.java | 4 +-
.../MultiKeyColumnNameIteratorTest.java | 187 ++++---
.../astyanax/MultiRowColumnIteratorTest.java | 50 +-
.../graph/impl/GraphManagerImpl.java | 6 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 2 +
.../impl/stage/NodeDeleteListenerImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 87 ++--
.../persistence/graph/GraphManagerIT.java | 17 +-
.../graph/GraphManagerShardConsistencyIT.java | 2 +-
.../usergrid/persistence/graph/SimpleTest.java | 12 +-
.../migration/EdgeDataMigrationImplTest.java | 2 +-
stack/corepersistence/model/pom.xml | 1 -
.../persistence/model/util/EntityUtils.java | 72 +++
stack/corepersistence/pom.xml | 8 +-
stack/corepersistence/queryindex/pom.xml | 63 +--
.../persistence/index/AliasedEntityIndex.java | 7 +-
.../index/ApplicationEntityIndex.java | 55 ++
.../usergrid/persistence/index/EntityIndex.java | 50 +-
.../persistence/index/EntityIndexBatch.java | 5 +
.../persistence/index/EntityIndexFactory.java | 2 +-
.../persistence/index/IndexIdentifier.java | 26 +-
.../usergrid/persistence/index/SearchType.java | 49 ++
.../usergrid/persistence/index/SearchTypes.java | 14 +-
.../persistence/index/guice/IndexModule.java | 5 +-
.../impl/EsApplicationEntityIndexImpl.java | 383 ++++++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 80 ++-
.../index/impl/EsEntityIndexFactoryImpl.java | 27 +-
.../index/impl/EsEntityIndexImpl.java | 499 ++++---------------
.../persistence/index/impl/EsQueryVistor.java | 88 ++--
.../persistence/index/impl/IndexingUtils.java | 140 +++---
.../persistence/index/query/EntityResults.java | 108 ----
.../usergrid/persistence/index/query/Query.java | 6 +-
.../persistence/index/query/Results.java | 148 ------
.../persistence/index/utils/ListUtils.java | 6 +-
.../persistence/index/guice/IndexTestFig.java | 62 +++
.../index/guice/TestIndexModule.java | 8 +-
.../index/impl/CorePerformanceIT.java | 339 -------------
.../impl/EntityConnectionIndexImplTest.java | 306 ------------
.../persistence/index/impl/EntityIndexTest.java | 195 +++-----
.../index/impl/IndexLoadTestsIT.java | 394 +++++++++++++++
stack/pom.xml | 16 +-
stack/rest/pom.xml | 1 +
.../usergrid/rest/AbstractContextResource.java | 102 ++--
.../org/apache/usergrid/rest/IndexResource.java | 2 +-
.../rest/test/RefreshIndexResource.java | 17 +-
.../resources/usergrid-custom-test.properties | 2 +
stack/services/pom.xml | 1 +
.../management/importer/ImportServiceImpl.java | 34 +-
.../impl/ApplicationQueueManagerImpl.java | 195 ++++----
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../apache/usergrid/management/EmailFlowIT.java | 6 +-
.../usergrid/management/OrganizationIT.java | 10 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ManagementServiceIT.java | 22 +-
.../management/export/ExportServiceIT.java | 12 +-
.../management/importer/ImportCollectionIT.java | 15 +-
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 2 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 2 +-
.../apns/NotificationsServiceIT.java | 40 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../resources/usergrid-custom-test.properties | 2 +
.../setup/ConcurrentProcessSingleton.java | 16 +-
129 files changed, 2544 insertions(+), 3021 deletions(-)
----------------------------------------------------------------------
[02/50] incubator-usergrid git commit: remove array prefix
Posted by sf...@apache.org.
remove array prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1a61e0ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1a61e0ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1a61e0ee
Branch: refs/heads/USERGRID-473
Commit: 1a61e0eec82da613226396ab8cd533fd5353a89c
Parents: cc779b2
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 23 11:16:41 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 23 11:16:41 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1a61e0ee/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 3b65c74..3a6f019 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -245,12 +245,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put( ARRAY_PREFIX + field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(),
+ entityMap.put(field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
@@ -262,7 +262,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof SetField ) {
Set set = ( Set ) field.getValue();
- entityMap.put(SET_PREFIX+ field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( set ) ) );
}
else if ( f instanceof EntityObjectField ) {
[27/50] incubator-usergrid git commit: add version migration for
indexes
Posted by sf...@apache.org.
add version migration for indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bf0718e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bf0718e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bf0718e6
Branch: refs/heads/USERGRID-473
Commit: bf0718e611bb06d8bf901ba34f5d8172d5ef02bb
Parents: 208ed2b
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 10:53:20 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 10:53:20 2015 -0600
----------------------------------------------------------------------
.../AllApplicationsObservable.java | 3 +-
.../usergrid/corepersistence/CoreModule.java | 9 ++++-
.../rx/impl/AllApplicationsObservableImpl.java | 5 ++-
.../persistence/index/guice/IndexModule.java | 13 ++++++-
.../index/impl/EsEntityIndexImpl.java | 9 ++++-
.../migration/EsIndexDataMigrationImpl.java | 20 ++++++++--
.../index/migration/IndexDataVersions.java | 39 ++++++++++++++++++++
.../index/guice/TestIndexModule.java | 7 +++-
8 files changed, 94 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
index 1187cf2..055223e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
@@ -29,7 +30,7 @@ import rx.Observable;
/**
* Interface for generating an observable of all ApplicationScope
*/
-public interface AllApplicationsObservable {
+public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
/**
* Return all applications in our system
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 2e9b780..d994419 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -16,6 +16,7 @@
package org.apache.usergrid.corepersistence;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.springframework.context.ApplicationContext;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
@@ -96,7 +97,13 @@ public class CoreModule extends AbstractModule {
AllNodesInGraphImpl.class );
}
} );
- install(new IndexModule());
+ install(new IndexModule(){
+ @Override
+ public void configureMigrationProvider() {
+ bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} ).to(
+ AllApplicationsObservable.class );
+ }
+ });
// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
// install(new QueueModule());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index 0fc5452..aa5b5dc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -138,5 +138,8 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
}
-
+ @Override
+ public Observable<ApplicationScope> getData() {
+ return getAllApplications();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index a42dea8..7ecce54 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -42,7 +42,7 @@ import org.apache.usergrid.persistence.queue.guice.QueueModule;
import org.safehaus.guicyfig.GuicyFigModule;
-public class IndexModule extends AbstractModule {
+public abstract class IndexModule extends AbstractModule {
@Override
protected void configure() {
@@ -75,8 +75,17 @@ public class IndexModule extends AbstractModule {
//wire up the collection migration plugin
Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to(EsIndexMigrationPlugin.class);
- }
+ //invoke the migration plugin config
+ configureMigrationProvider();
+ }
+
+ /**
+ * Gives callers the ability to to configure an instance of
+ *
+ * MigrationDataProvider<ApplicationScope> for providing data migrations
+ */
+ public abstract void configureMigrationProvider();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index cab8ded..844260f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -27,10 +27,12 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import org.apache.usergrid.persistence.index.migration.IndexDataVersions;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionFuture;
@@ -71,7 +73,7 @@ import java.util.*;
* Implements index using ElasticSearch Java API.
*/
@Singleton
-public class EsEntityIndexImpl implements AliasedEntityIndex {
+public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
@@ -442,6 +444,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
return Health.RED;
}
+ @Override
+ public int getImplementationVersion() {
+ return IndexDataVersions.SINGLE_INDEX.getVersion();
+ }
+
/**
* Interface for operations.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index b5dab53..2a0bb15 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -20,6 +20,7 @@ import com.google.inject.Inject;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.AliasedEntityIndex;
import org.apache.usergrid.persistence.index.IndexAlias;
@@ -30,9 +31,12 @@ import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* Classy class class.
@@ -44,6 +48,8 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
private final IndexFig indexFig;
private final IndexIdentifier indexIdentifier;
private final EsIndexCache indexCache;
+ private final VersionedData dataVersion;
+ private static final Logger log = LoggerFactory.getLogger(EsIndexDataMigrationImpl.class);
@Inject
public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, EsIndexCache indexCache){
@@ -52,10 +58,12 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
this.indexCache = indexCache;
+ this.dataVersion = (VersionedData) entityIndex;
}
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
+ final AtomicInteger integer = new AtomicInteger();
migrationDataProvider.getData().doOnNext(applicationScope -> {
LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,applicationScope);
String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
@@ -66,19 +74,23 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ integer.incrementAndGet();
}
- });
- return 0;
+ })
+ .doOnError(error -> log.error("failed to migrate index",error))
+ .toBlocking().last();
+
+ return integer.get();
}
@Override
public boolean supports(int currentVersion) {
- return false;
+ return currentVersion < dataVersion.getImplementationVersion();
}
@Override
public int getMaxVersion() {
- return 0;
+ return dataVersion.getImplementationVersion();
}
/**
* Class is used to generate an index name and alias name the old way via app name
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
new file mode 100644
index 0000000..921d86a
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
@@ -0,0 +1,39 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index.migration;
+
+/**
+ * Classy class class.
+ */
+public enum IndexDataVersions {
+ MANY_INDEXES(0),
+ SINGLE_INDEX(1);
+
+ private final int version;
+
+
+ private IndexDataVersions( final int version ) {this.version = version;}
+
+
+ public int getVersion() {
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 4cf46d6..4e6700c 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -33,7 +33,12 @@ public class TestIndexModule extends TestModule {
install( new CommonModule());
// configure collections and our core astyanax framework
- install( new IndexModule() );
+ install( new IndexModule(){
+ @Override
+ public void configureMigrationProvider(){
+
+ }
+ });
install( new GuicyFigModule(IndexTestFig.class) );
}
}
[49/50] incubator-usergrid git commit: separate calls
Posted by sf...@apache.org.
separate calls
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c1643a7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c1643a7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c1643a7f
Branch: refs/heads/USERGRID-473
Commit: c1643a7fff668abab72026fb2dd714bb9eb6245d
Parents: c12aa17
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 17:29:28 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 17:29:28 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/EntityIndex.java | 2 +
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../index/impl/EsEntityIndexBatchImpl.java | 1 +
.../index/impl/EsEntityIndexImpl.java | 57 ++++++++------------
.../index/migration/EsIndexMigrationPlugin.java | 2 +-
5 files changed, 26 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1643a7f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index ff9008e..54a4127 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -70,6 +70,8 @@ public interface EntityIndex extends CPManager {
void initialize();
+
+ boolean shouldInitialize();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1643a7f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 84845d0..eceff90 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -118,7 +118,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
@Override
public EntityIndexBatch createBatch() {
EntityIndexBatch batch = new EsEntityIndexBatchImpl(
- applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier );
+ applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier);
return batch;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1643a7f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index f62648d..6fcb5eb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -69,6 +69,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
IndexValidationUtils.validateIndexScope( indexScope );
ValidationUtils.verifyEntityWrite( entity );
ValidationUtils.verifyVersion( entity.getVersion() );
+
//add app id for indexing
container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), applicationScope,indexScope, entity));
return this;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1643a7f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 993b529..8de4832 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -77,12 +77,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private final Timer addTimer;
private final Timer updateAliasTimer;
-
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
*/
-
-
private final EsProvider esProvider;
//number of times to wait for the index to refresh properly.
@@ -103,7 +100,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private Timer refreshTimer;
private Meter refreshIndexMeter;
-
// private final Timer indexTimer;
@@ -128,23 +124,30 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
this.refreshTimer = metricsFactory
.getTimer(EsEntityIndexImpl.class, "refresh.timer");
this.refreshIndexMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"refresh.meter");
+ if(shouldInitialize()){
+ initialize();
+ }
}
@Override
- public void initialize(){
+ public void initialize() {
final int numberOfShards = indexFig.getNumberOfShards();
final int numberOfReplicas = indexFig.getNumberOfReplicas();
aliasCache.invalidate(alias);
- String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
- String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
-
- if(reads.length==0 || writes.length==0) {
+ if (shouldInitialize()) {
addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
}
}
@Override
+ public boolean shouldInitialize() {
+ String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
+ String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
+ return reads.length==0 || writes.length==0;
+ }
+
+ @Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
try {
//get index name with suffix attached
@@ -216,33 +219,15 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
logger.info("Removed Index Name from Alias=[{}] ACK=[{}]", alias, isAck);
}
- String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
- String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
-
- try {
- aliasesRequestBuilder = adminClient.indices().prepareAliases();
- //Added For Graphite Metrics
- // add read alias
- aliasesRequestBuilder.addAlias(indexName, alias.getReadAlias());
- isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
- logger.info("Created new read aliases ACK=[{}]", isAck);
- } catch (InvalidAliasNameException ie) {
- logger.debug("error creating read", ie);
- }
- try {
- aliasesRequestBuilder = adminClient.indices().prepareAliases();
-
- //Added For Graphite Metrics
- //add write alias
- aliasesRequestBuilder.addAlias(indexName, alias.getWriteAlias());
- isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
- logger.info("Created new write aliases ACK=[{}]", isAck);
-
- } catch (InvalidAliasNameException ie) {
- logger.debug("error creating write", ie);
- }
-
-
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ //Added For Graphite Metrics
+ // add read alias
+ aliasesRequestBuilder.addAlias(indexName, alias.getReadAlias());
+ //Added For Graphite Metrics
+ //add write alias
+ aliasesRequestBuilder.addAlias(indexName, alias.getWriteAlias());
+ isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
+ logger.info("Created new read and write aliases ACK=[{}]", isAck);
aliasCache.invalidate(alias);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c1643a7f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
index a28c701..9ece73b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
@@ -41,6 +41,6 @@ public class EsIndexMigrationPlugin extends AbstractMigrationPlugin<Application
@Override
public PluginPhase getPhase() {
- return PluginPhase.MIGRATE;
+ return PluginPhase.BOOTSTRAP;
}
}
[15/50] incubator-usergrid git commit: [USERGRID-348] Fixed around
half the tests for export rules
Posted by sf...@apache.org.
[USERGRID-348] Fixed around half the tests for export rules
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ee48768e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ee48768e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ee48768e
Branch: refs/heads/USERGRID-473
Commit: ee48768e7cf7c21621899ac0a7d31cf195832c86
Parents: 2ad1c04
Author: GERey <gr...@apigee.com>
Authored: Tue Mar 24 12:25:17 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Tue Mar 24 12:25:17 2015 -0700
----------------------------------------------------------------------
.../rest/management/ExportResourceIT.java | 359 +++++++++----------
1 file changed, 160 insertions(+), 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee48768e/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
index fbfc2b9..020fce1 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
@@ -38,6 +38,7 @@ import org.apache.usergrid.rest.test.resource2point0.model.ApiResponse;
import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
import org.junit.Ignore;
import org.junit.Rule;
@@ -54,26 +55,6 @@ public class ExportResourceIT extends AbstractRestIT {
}
- @Test
- public void exportCallSuccessful() throws Exception {
- ClientResponse.Status responseStatus = ClientResponse.Status.OK;
- JsonNode node = null;
-
- HashMap<String, Object> payload = payloadBuilder();
-
-
- try {
- management().orgs().organization( clientSetup.getOrganizationName() )
- .app().addToPath( clientSetup.getAppUuid()).addToPath( "collection" )
- .addToPath( "users" ).addToPath( "export" ).postWithToken(ApiResponse.class,payload );
- }
- catch ( UniformInterfaceException uie ) {
- responseStatus = uie.getResponse().getClientResponseStatus();
- }
-
- assertEquals( ClientResponse.Status.OK, responseStatus );
- }
-
//
// @Ignore( "is this test still valid knowing that the sch. won't run in intelliJ?" )
// public void exportCallCreationEntities100() throws Exception {
@@ -120,190 +101,170 @@ public class ExportResourceIT extends AbstractRestIT {
// }
//
//
-// @Test
-// public void exportApplicationUUIDRetTest() throws Exception {
-// ClientResponse.Status responseStatus = ClientResponse.Status.ACCEPTED;
-// String uuid;
-// UUID jobUUID = null;
-// JsonNode node = null;
-//
-// HashMap<String, Object> payload = payloadBuilder();
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals( ClientResponse.Status.ACCEPTED, responseStatus );
-// assertNotNull( node.get( "Export Entity" ) );
-// }
-//
-//
-// //
-// @Test
-// public void exportCollectionUUIDRetTest() throws Exception {
-// ClientResponse.Status responseStatus = ClientResponse.Status.ACCEPTED;
-// String uuid;
-// UUID jobUUID = null;
-// JsonNode node = null;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// HashMap<String, Object> payload = payloadBuilder();
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/"+orgName+"/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-//
-// assertEquals( ClientResponse.Status.ACCEPTED, responseStatus );
-// assertNotNull( node.get( "Export Entity" ) );
-// }
-//
-//
-// @Test
-// public void exportGetOrganizationJobStatTest() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = payloadBuilder();
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// try {
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/export" ).queryParam( "access_token", token )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( String.class, payload ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.OK, responseStatus );
-//
-// String uuid = String.valueOf( node.get( "Export Entity" ) );
-// uuid = uuid.replaceAll( "\"", "" );
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-//
-//
-// assertEquals( ClientResponse.Status.OK, responseStatus );
-// assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
-// }
-//
-//
-// //all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
-// @Test
-// public void exportGetApplicationJobStatTest() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = payloadBuilder();
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// String uuid = String.valueOf( node.get( "Export Entity" ) );
-// uuid = uuid.replaceAll( "\"", "" );
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-//
-//
-// assertEquals( ClientResponse.Status.OK, responseStatus );
-// assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
-// }
-//
-//
-// @Test
-// public void exportGetCollectionJobStatTest() throws Exception {
-// JsonNode node = null;
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-//
-// HashMap<String, Object> payload = payloadBuilder();
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// node = mapper.readTree(
-// resource().path( "/management/orgs/" + orgName + "/apps/" + appName + "/collection/users/export" )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
-// String uuid = String.valueOf( node.get( "Export Entity" ) );
-// uuid = uuid.replaceAll( "\"", "" );
-//
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + uuid )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
+ @Test
+ public void exportApplicationUUIDRetTest() throws Exception {
+
+ try {
+ management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() ).addToPath( "export" );
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail("We got back " + uie.getResponse().getClientResponseStatus() + " instead of having a successful call" );
+ }
+
+ }
+
+
+ //
+ @Test
+ public void exportCollectionUUIDRetTest() throws Exception {
+
+ HashMap<String, Object> payload = payloadBuilder();
+ ApiResponse exportEntity = null;
+
+ try {
+
+
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid()).addToPath( "collection" )
+ .addToPath( "users" ).addToPath( "export" ).postWithToken(ApiResponse.class,payloadBuilder() );
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail( "We got back "+uie.getResponse().getClientResponseStatus()+" instead of having a successful call" );
+ }
+
+ assertNotNull( exportEntity );
+ assertNotNull( exportEntity.getProperties().get( "Export Entity" ));
+ }
+
+
+ /**
+ * Check that you can get the org export uuid returned and that you can check the status of the job using that uuid.
+ * @throws Exception
+ */
+ @Test
+ public void exportGetOrganizationJobStatTest() throws Exception {
+
+ ApiResponse exportEntity = null;
+
+
+ try {
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payloadBuilder() );
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail( "We got back "+uie.getResponse().getClientResponseStatus()+" instead of having a successful call" );
+ }
+
+ assertNotNull( exportEntity );
+ String uuid = ( String ) exportEntity.getProperties().get( "Export Entity" );
+ assertNotNull( uuid );
+
+ exportEntity = null;
+ try {
+
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).addToPath( uuid ).get( ApiResponse.class );
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail( "We got back "+uie.getResponse().getClientResponseStatus()+" instead of having a successful call" );
+ }
+
+ assertNotNull( exportEntity );
+ String state = (String) exportEntity.getProperties().get( "state" );
+ assertEquals( "SCHEDULED", state);
+ }
//
//
-// assertEquals( ClientResponse.Status.OK, responseStatus );
-// assertEquals( "SCHEDULED", node.get( "state" ).textValue() );//TODO: do tests for other states in service tier
-// }
+
+
+ /**
+ * Check that you can get the app export uuid returned and that you can check the status of the job using that uuid.
+ * @throws Exception
+ */
+
+ @Test
+ public void exportGetApplicationJobStatTest() throws Exception {
+
+ ApiResponse exportEntity = null;
+
+
+ try {
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid() )
+ .addToPath( "export" ).postWithToken( ApiResponse.class, payloadBuilder() );
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail( "We got back "+uie.getResponse().getClientResponseStatus()+" instead of having a successful call" );
+ }
+
+ assertNotNull( exportEntity );
+ String uuid = ( String ) exportEntity.getProperties().get( "Export Entity" );
+ assertNotNull( uuid );
+
+ exportEntity = null;
+ refreshIndex();
+ try {
+
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).addToPath( uuid ).get( ApiResponse.class );
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail( "We got back "+uie.getResponse().getClientResponseStatus()+" instead of having a successful call" );
+ }
+
+ assertNotNull( exportEntity );
+ String state = (String) exportEntity.getProperties().get( "state" );
+ assertEquals( "SCHEDULED", state);
+ }
+
+
+ @Test
+ public void exportGetCollectionJobStatTest() throws Exception {
+
+ ApiResponse exportEntity = null;
+
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .app().addToPath( clientSetup.getAppUuid()).addToPath( "collection" )
+ .addToPath( "users" ).addToPath( "export" )
+ .postWithToken( ApiResponse.class, payloadBuilder() );
+
+ assertNotNull( exportEntity );
+ String uuid = ( String ) exportEntity.getProperties().get( "Export Entity" );
+ assertNotNull( uuid );
+
+ exportEntity = null;
+ try {
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).addToPath( uuid ).get( ApiResponse.class );
+ }
+ catch ( UniformInterfaceException uie ) {
+ fail( "We got back "+uie.getResponse().getClientResponseStatus()+" instead of having a successful call" );
+ }
+
+
+ assertNotNull( exportEntity );
+ String state = (String) exportEntity.getProperties().get( "state" );
+ assertEquals( "SCHEDULED", state);
+ }
//
//
// // //do an unauthorized test for both post and get
-// @Test
-// public void exportGetWrongUUID() throws Exception {
-// JsonNode node = null;
-//
-// String orgName = context.getOrgName();
-// String appName = context.getAppName();
-// String token = context.getActiveUser().getToken();
-//
-// ClientResponse.Status responseStatus = ClientResponse.Status.OK;
-// UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
-// try {
-// node = mapper.readTree( resource().path( "/management/orgs/" + orgName + "/export/" + fake )
-// .queryParam( "access_token", token ).accept( MediaType.APPLICATION_JSON )
-// .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
-// }
-// catch ( UniformInterfaceException uie ) {
-// responseStatus = uie.getResponse().getClientResponseStatus();
-// }
-// assertEquals( ClientResponse.Status.BAD_REQUEST, responseStatus );
-// }
-//
+ @Test
+ public void exportGetWrongUUID() throws Exception {
+ ApiResponse exportEntity = null;
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+ try {
+ exportEntity = management().orgs().organization( clientSetup.getOrganizationName() )
+ .addToPath( "export" ).addToPath( fake.toString() ).get( ApiResponse.class );
+ fail( "Should not have been able to get fake uuid" );
+ }
+ catch ( UniformInterfaceException uie ) {
+ assertEquals( ClientResponse.Status.BAD_REQUEST, uie.getResponse().getClientResponseStatus() );
+
+ }
+ }
+
//
// //
// @Test
[20/50] incubator-usergrid git commit: change entityindex interface
Posted by sf...@apache.org.
change entityindex interface
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a62d117
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a62d117
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a62d117
Branch: refs/heads/USERGRID-473
Commit: 8a62d1177783612189be49fb924dfead3f348e2f
Parents: e5eb57f
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 25 14:26:10 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 25 14:26:10 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 122 ++++++------
.../index/ApplicationEntityIndex.java | 13 +-
.../impl/EsApplicationEntityIndexImpl.java | 199 +++++++------------
.../impl/SearchRequestBuilderStrategy.java | 197 ++++++++++++++++++
.../index/query/CandidateResults.java | 24 ++-
.../usergrid/persistence/index/query/Query.java | 160 +--------------
.../persistence/index/impl/EntityIndexTest.java | 28 +--
.../persistence/query/tree/GrammarTreeTest.java | 8 +-
8 files changed, 382 insertions(+), 369 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f420d7b..481fff5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -753,14 +753,14 @@ public class CpEntityManager implements EntityManager {
@Override
public void updateApplication( Application app ) throws Exception {
- update( app );
+ update(app);
this.application = app;
}
@Override
public void updateApplication( Map<String, Object> properties ) throws Exception {
- this.updateProperties( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), properties );
+ this.updateProperties(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), properties);
this.application = get( applicationId, Application.class );
}
@@ -857,7 +857,7 @@ public class CpEntityManager implements EntityManager {
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
- return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, alias );
+ return getAlias(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias);
}
@@ -886,7 +886,7 @@ public class CpEntityManager implements EntityManager {
new Object[] { ownerRef, collectionType, aliasValue } );
}
- return results.get( aliasValue );
+ return results.get(aliasValue);
}
@@ -936,7 +936,7 @@ public class CpEntityManager implements EntityManager {
}
- return Collections.<EntityRef>singleton( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+ return Collections.<EntityRef>singleton(new SimpleEntityRef(id.getType(), id.getUuid()));
}
@@ -1003,7 +1003,7 @@ public class CpEntityManager implements EntityManager {
public void setProperty(
EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception {
- setProperty( entityRef, propertyName, propertyValue, false );
+ setProperty(entityRef, propertyName, propertyValue, false);
}
@@ -1021,7 +1021,7 @@ public class CpEntityManager implements EntityManager {
entity.getType(), propertyName, propertyValue );
entity.setProperty( propertyName, propertyValue );
- entity.setProperty( PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis( UUIDUtils.newTimeUUID() ) );
+ entity.setProperty(PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis(UUIDUtils.newTimeUUID()));
update( entity );
}
@@ -1127,7 +1127,7 @@ public class CpEntityManager implements EntityManager {
public void addToDictionary( EntityRef entityRef, String dictionaryName,
Object elementValue ) throws Exception {
- addToDictionary( entityRef, dictionaryName, elementValue, null );
+ addToDictionary(entityRef, dictionaryName, elementValue, null);
}
@@ -1188,7 +1188,7 @@ public class CpEntityManager implements EntityManager {
EntityRef entity = get( entityRef );
UUID timestampUuid = UUIDUtils.newTimeUUID();
- Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
+ Mutator<ByteBuffer> batch = createMutator(cass.getApplicationKeyspace(applicationId), be);
for ( Map.Entry<?, ?> elementValue : elementValues.entrySet() ) {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementValue.getKey(),
@@ -1225,8 +1225,8 @@ public class CpEntityManager implements EntityManager {
boolean coTypeIsBasic = ClassUtils.isBasicType( setCoType );
List<HColumn<ByteBuffer, ByteBuffer>> results =
- cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), be, be );
+ cass.getAllColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), be, be);
for ( HColumn<ByteBuffer, ByteBuffer> result : results ) {
Object name = null;
if ( entityHasDictionary ) {
@@ -1275,7 +1275,7 @@ public class CpEntityManager implements EntityManager {
ApplicationCF dictionaryCf = null;
- boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary( entity.getType(), dictionaryName );
+ boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary(entity.getType(), dictionaryName);
if ( entityHasDictionary ) {
dictionaryCf = ENTITY_DICTIONARIES;
@@ -1285,14 +1285,14 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
+ Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
HColumn<ByteBuffer, ByteBuffer> result =
- cass.getColumn( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
- entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ),
- be, be );
+ cass.getColumn(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName),
+ entityHasDictionary ? bytebuffer(elementName) : DynamicComposite.toByteBuffer(elementName),
+ be, be);
if ( result != null ) {
if ( entityHasDictionary && coTypeIsBasic ) {
@@ -1303,7 +1303,7 @@ public class CpEntityManager implements EntityManager {
}
}
else {
- logger.info( "Results of CpEntityManagerImpl.getDictionaryElementValue is null" );
+ logger.info("Results of CpEntityManagerImpl.getDictionaryElementValue is null");
}
return value;
@@ -1327,8 +1327,8 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
- boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
+ Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
+ boolean coTypeIsBasic = ClassUtils.isBasicType(dictionaryCoType);
ByteBuffer[] columnNames = new ByteBuffer[elementNames.length];
for ( int i = 0; i < elementNames.length; i++ ) {
@@ -1337,8 +1337,8 @@ public class CpEntityManager implements EntityManager {
}
ColumnSlice<ByteBuffer, ByteBuffer> results =
- cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be );
+ cass.getColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), columnNames, be, be);
if ( results != null ) {
values = new HashMap<String, Object>();
for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) {
@@ -1384,7 +1384,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getDictionaries( EntityRef entity ) throws Exception {
- return getDictionaryNames( entity );
+ return getDictionaryNames(entity);
}
@@ -1398,14 +1398,14 @@ public class CpEntityManager implements EntityManager {
@Override
public boolean isCollectionMember( EntityRef owner, String collectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isCollectionMember( collectionName, entity );
+ return getRelationManager( owner ).isCollectionMember(collectionName, entity);
}
@Override
public boolean isConnectionMember( EntityRef owner, String connectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isConnectionMember( connectionName, entity );
+ return getRelationManager( owner ).isConnectionMember(connectionName, entity);
}
@@ -1430,7 +1430,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( get( entityId ))
- .getCollection ( collectionName, query, resultsLevel );
+ .getCollection(collectionName, query, resultsLevel);
}
@@ -1495,8 +1495,8 @@ public class CpEntityManager implements EntityManager {
@Override
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
- return createConnection( connection.getConnectingEntity(), connection.getConnectionType(),
- connection.getConnectedEntity() );
+ return createConnection(connection.getConnectingEntity(), connection.getConnectionType(),
+ connection.getConnectedEntity());
}
@@ -1504,7 +1504,7 @@ public class CpEntityManager implements EntityManager {
public ConnectionRef createConnection( EntityRef connectingEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
- return getRelationManager( connectingEntity ).createConnection( connectionType, connectedEntityRef );
+ return getRelationManager( connectingEntity ).createConnection(connectionType, connectedEntityRef);
}
@@ -1514,7 +1514,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( connectingEntity )
- .createConnection( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
+ .createConnection(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
}
@@ -1540,7 +1540,7 @@ public class CpEntityManager implements EntityManager {
EntityRef pairedEntity, String connectionType, EntityRef connectedEntityRef ) throws Exception {
return getRelationManager( connectingEntity )
- .connectionRef( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
+ .connectionRef(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
}
@@ -1572,7 +1572,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectedEntities( connectionType, connectedEntityType, resultsLevel );
+ .getConnectedEntities(connectionType, connectedEntityType, resultsLevel);
}
@@ -1581,7 +1581,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectingEntities( connectionType, connectedEntityType, resultsLevel );
+ .getConnectingEntities(connectionType, connectedEntityType, resultsLevel);
}
@@ -1589,21 +1589,21 @@ public class CpEntityManager implements EntityManager {
public Results getConnectingEntities( EntityRef entityRef, String connectionType,
String entityType, Level level, int count ) throws Exception {
- return getRelationManager( entityRef ).getConnectingEntities( connectionType, entityType, level, count );
+ return getRelationManager( entityRef ).getConnectingEntities(connectionType, entityType, level, count);
}
@Override
public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception {
- return getRelationManager( connectingEntity ).searchConnectedEntities( query );
+ return getRelationManager( connectingEntity ).searchConnectedEntities(query);
}
@Override
public Set<String> getConnectionIndexes( EntityRef entity, String connectionType ) throws Exception {
- return getRelationManager( entity ).getConnectionIndexes( connectionType );
+ return getRelationManager( entity ).getConnectionIndexes(connectionType);
}
@@ -1829,14 +1829,14 @@ public class CpEntityManager implements EntityManager {
String roleTitle = batchRoleName;
String propertyName = groupId + ":" + batchRoleName;
Map<String, Object> properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
- properties.put( "group", groupId );
+ properties.put("group", groupId);
- Entity entity = batchCreateRole( roleName, roleTitle, inactivity, propertyName, groupId, properties );
+ Entity entity = batchCreateRole(roleName, roleTitle, inactivity, propertyName, groupId, properties);
getRelationManager( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ) )
- .addToCollection( COLLECTION_ROLES, entity );
+ .addToCollection(COLLECTION_ROLES, entity);
- logger.info( "Created role {} with id {} in group {}",
- new String[] { roleName, entity.getUuid().toString(), groupId.toString() } );
+ logger.info("Created role {} with id {} in group {}",
+ new String[]{roleName, entity.getUuid().toString(), groupId.toString()});
return entity;
}
@@ -1848,8 +1848,8 @@ public class CpEntityManager implements EntityManager {
permission = permission.toLowerCase();
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
+ CassandraPersistenceUtils.addInsertToMutator(batch, ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey(groupId, roleName), permission, ByteBuffer.allocate(0), timestamp);
//Adding graphite metrics
Timer.Context timeGroupRolePermission = entGrantGroupPermissionTimer.time();
@@ -1927,7 +1927,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantUserPermission( UUID userId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary( userRef( userId ), DICTIONARY_PERMISSIONS, permission );
+ addToDictionary(userRef(userId), DICTIONARY_PERMISSIONS, permission);
}
@@ -1951,9 +1951,9 @@ public class CpEntityManager implements EntityManager {
roleName = roleName.toLowerCase();
EntityRef userRef = userRef( userId );
EntityRef roleRef = getRoleRef( roleName );
- addToDictionary( userRef, DICTIONARY_ROLENAMES, roleName, roleName );
- addToCollection( userRef, COLLECTION_ROLES, roleRef );
- addToCollection( roleRef, COLLECTION_USERS, userRef );
+ addToDictionary(userRef, DICTIONARY_ROLENAMES, roleName, roleName);
+ addToCollection(userRef, COLLECTION_ROLES, roleRef);
+ addToCollection(roleRef, COLLECTION_USERS, userRef);
}
@@ -1988,8 +1988,8 @@ public class CpEntityManager implements EntityManager {
private EntityRef getRoleRef( String roleName ) throws Exception {
- Results results = this.searchCollection( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ),
- Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.findForProperty( "roleName", roleName ) );
+ Results results = this.searchCollection(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ Schema.defaultCollectionName(Role.ENTITY_TYPE), Query.findForProperty("roleName", roleName));
Iterator<Entity> iterator = results.iterator();
EntityRef roleRef = null;
while ( iterator.hasNext() ) {
@@ -2235,8 +2235,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getCounterNames() throws Exception {
Set<String> names = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
- Set<String> nameSet = cast( getDictionaryAsSet( getApplicationRef(), Schema.DICTIONARY_COUNTERS ) );
- names.addAll( nameSet );
+ Set<String> nameSet = cast(getDictionaryAsSet(getApplicationRef(), Schema.DICTIONARY_COUNTERS));
+ names.addAll(nameSet);
return names;
}
@@ -2300,7 +2300,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
getApplicationScope().getApplication(), collectionName);
- final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collectionScope);
//convert to a string, that's what we store
final Id results = ecm.getIdField( new StringField(
@@ -2313,7 +2313,7 @@ public class CpEntityManager implements EntityManager {
public Entity get( UUID uuid ) throws Exception {
MapManager mm = getMapManagerForTypes();
- String entityType = mm.getString( uuid.toString() );
+ String entityType = mm.getString(uuid.toString());
final Entity entity;
@@ -2335,9 +2335,9 @@ public class CpEntityManager implements EntityManager {
private MapManager getMapManagerForTypes() {
Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION );
- final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner );
+ final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner);
- MapManager mm = managerCache.getMapManager( ms );
+ MapManager mm = managerCache.getMapManager(ms);
return mm;
}
@@ -2440,7 +2440,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Map<String, Role> getUserRolesWithTitles( UUID userId ) throws Exception {
return getRolesWithTitles(
- ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) );
+ ( Set<String> ) cast( getDictionaryAsSet( userRef(userId), DICTIONARY_ROLENAMES ) ) );
}
@@ -2455,8 +2455,8 @@ public class CpEntityManager implements EntityManager {
@Override
public void addGroupToRole( UUID groupId, String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- addToDictionary( groupRef( groupId ), DICTIONARY_ROLENAMES, roleName, roleName );
- addToCollection( groupRef( groupId ), COLLECTION_ROLES, getRoleRef( roleName ) );
+ addToDictionary(groupRef(groupId), DICTIONARY_ROLENAMES, roleName, roleName);
+ addToCollection(groupRef(groupId), COLLECTION_ROLES, getRoleRef(roleName));
}
@@ -2477,7 +2477,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantGroupPermission( UUID groupId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary( groupRef( groupId ), DICTIONARY_PERMISSIONS, permission );
+ addToDictionary(groupRef(groupId), DICTIONARY_PERMISSIONS, permission);
}
@@ -2663,8 +2663,8 @@ public class CpEntityManager implements EntityManager {
catch ( HystrixRuntimeException hre ) {
if ( hre.getCause() instanceof WriteUniqueVerifyException ) {
- WriteUniqueVerifyException wuve = ( WriteUniqueVerifyException ) hre.getCause();
- handleWriteUniqueVerifyException( entity, wuve );
+ WriteUniqueVerifyException wuve = (WriteUniqueVerifyException) hre.getCause();
+ handleWriteUniqueVerifyException(entity, wuve);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index dd1c5fa..2167efa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -39,11 +39,22 @@ public interface ApplicationEntityIndex {
*/
public EntityIndexBatch createBatch();
+ /**
+ * Execute query in Usergrid syntax.
+ */
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query);
/**
* Execute query in Usergrid syntax.
*/
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query );
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, final Query query, final int limit );
+
+ /**
+ * get next page of results
+ * @param cursor
+ * @return
+ */
+ public CandidateResults getNextPage(final String cursor);
/**
* delete all application records
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index ade2286..ca071e2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -46,15 +46,9 @@ import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortBuilders;
-import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -85,6 +79,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
private final IndexIdentifier.IndexAlias alias;
private final Timer deleteApplicationTimer;
private final Meter deleteApplicationMeter;
+ private final SearchRequestBuilderStrategy searchRequest;
private FailureMonitor failureMonitor;
private final int cursorTimeout;
@Inject
@@ -117,6 +112,8 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
this.alias = indexIdentifier.getAlias();
+ this.searchRequest = new SearchRequestBuilderStrategy(esProvider,appScope,alias,cursorTimeout);
+
}
@Override
@@ -139,132 +136,78 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
- final Query query ) {
-
- SearchResponse searchResponse;
-
- if ( query.getCursor() == null ) {
- String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(searchTypes.getTypeNames(applicationScope))
- .setScroll(cursorTimeout + "m")
- .setQuery(query.createQueryBuilder(contexts));
-
- final FilterBuilder fb = query.createFilterBuilder();
-
- //we have post filters, apply them
- if ( fb != null ) {
- logger.debug( " Filter: {} ", fb.toString() );
- srb = srb.setPostFilter( fb );
- }
-
-
- srb = srb.setFrom( 0 ).setSize( query.getLimit() );
-
- for ( Query.SortPredicate sp : query.getSortPredicates() ) {
-
- final SortOrder order;
- if ( sp.getDirection().equals( Query.SortDirection.ASCENDING ) ) {
- order = SortOrder.ASC;
- }
- else {
- order = SortOrder.DESC;
- }
-
- // we do not know the type of the "order by" property and so we do not know what
- // type prefix to use. So, here we add an order by clause for every possible type
- // that you can order by: string, number and boolean and we ask ElasticSearch
- // to ignore any fields that are not present.
-
- final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
- .order( order ).ignoreUnmapped( true );
- srb.addSort( stringSort );
-
- logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query){
+ return search(indexScope,searchTypes,query,10);
+ }
- final String longFieldName = LONG_PREFIX + sp.getPropertyName();
- final FieldSortBuilder longSort = SortBuilders.fieldSort( longFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( longSort );
- logger.debug( " Sort: {} order by {}", longFieldName, order.toString() );
+ @Override
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ SearchResponse searchResponse;
- final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
- final FieldSortBuilder doubleSort = SortBuilders.fieldSort( doubleFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( doubleSort );
- logger.debug( " Sort: {} order by {}", doubleFieldName, order.toString() );
+ SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query, limit);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
+ this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb);
+ }
- final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( booleanSort );
- logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
- }
+ try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearch = searchTimer.time();
+ searchResponse = srb.execute().actionGet();
+ timeSearch.stop();
+ } catch (Throwable t) {
+ logger.error("Unable to communicate with Elasticsearch", t);
+ failureMonitor.fail("Unable to execute batch", t);
+ throw t;
+ }
+ failureMonitor.success();
- if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
- this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb );
- }
+ return parseResults(searchResponse);
+ }
- try {
- //Added For Graphite Metrics
- Timer.Context timeSearch = searchTimer.time();
- searchResponse = srb.execute().actionGet();
- timeSearch.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
+ public CandidateResults getNextPage(final String cursor){
+ SearchResponse searchResponse;
- failureMonitor.success();
+ String userCursorString = cursor;
+ if ( userCursorString.startsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 1 );
+ }
+ if ( userCursorString.endsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
}
- else {
- String userCursorString = query.getCursor();
- if ( userCursorString.startsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 1 );
- }
- if ( userCursorString.endsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
- }
-
- //now get the cursor from the map and validate
- final String esScrollCursor = mapManager.getString( userCursorString );
-
- Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor);
+ //now get the cursor from the map and validate
+ final String esScrollCursor = mapManager.getString(userCursorString);
+ Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor);
- logger.debug( "Executing query with cursor: {} ", esScrollCursor );
- SearchScrollRequestBuilder ssrb = esProvider.getClient()
- .prepareSearchScroll(esScrollCursor).setScroll( cursorTimeout + "m" );
+ logger.debug( "Executing query with cursor: {} ", esScrollCursor );
- try {
- //Added For Graphite Metrics
- Timer.Context timeSearchCursor = cursorTimer.time();
- searchResponse = ssrb.execute().actionGet();
- timeSearchCursor.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with elasticsearch", t );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
+ SearchScrollRequestBuilder ssrb = esProvider.getClient()
+ .prepareSearchScroll(esScrollCursor).setScroll(cursorTimeout + "m");
- failureMonitor.success();
+ try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearchCursor = cursorTimer.time();
+ searchResponse = ssrb.execute().actionGet();
+ timeSearchCursor.stop();
+ }
+ catch ( Throwable t ) {
+ logger.error( "Unable to communicate with elasticsearch", t );
+ failureMonitor.fail( "Unable to execute batch", t );
+ throw t;
}
- return parseResults(searchResponse, query);
+
+ failureMonitor.success();
+ return parseResults(searchResponse);
}
/**
@@ -327,7 +270,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
- private CandidateResults parseResults( final SearchResponse searchResponse, final Query query ) {
+ private CandidateResults parseResults( final SearchResponse searchResponse) {
final SearchHits searchHits = searchResponse.getHits();
final SearchHit[] hits = searchHits.getHits();
@@ -335,41 +278,39 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
logger.debug(" Hit count: {} Total hits: {}", length, searchHits.getTotalHits());
- List<CandidateResult> candidates = new ArrayList<>( length );
+ List<CandidateResult> candidates = new ArrayList<>(length);
- for ( SearchHit hit : hits ) {
+ for (SearchHit hit : hits) {
- String[] idparts = hit.getId().split( SPLITTER );
+ String[] idparts = hit.getId().split(SPLITTER);
String id = idparts[0];
String type = idparts[1];
String version = idparts[2];
- Id entityId = new SimpleId( UUID.fromString(id), type );
+ Id entityId = new SimpleId(UUID.fromString(id), type);
- candidates.add( new CandidateResult( entityId, UUID.fromString( version ) ) );
+ candidates.add(new CandidateResult(entityId, UUID.fromString(version)));
}
- CandidateResults candidateResults = new CandidateResults( query, candidates );
-
- if ( candidates.size() >= query.getLimit() ) {
- //USERGRID-461 our cursor is getting too large, map it to a new time UUID
- //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home
-
- final String userCursorString = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+ CandidateResults candidateResults = new CandidateResults(candidates);
+ if(candidateResults.hasCursor()) {
+ candidateResults.initializeCursor();
final String esScrollCursor = searchResponse.getScrollId();
-
//now set this into our map module
final int minutes = indexFig.getQueryCursorTimeout();
//just truncate it, we'll never hit a long value anyway
- mapManager.putString( userCursorString, esScrollCursor, ( int ) TimeUnit.MINUTES.toSeconds( minutes ) );
+ mapManager.putString(candidateResults.getCursor(), esScrollCursor, (int) TimeUnit.MINUTES.toSeconds(minutes));
- candidateResults.setCursor( userCursorString );
- logger.debug(" User cursor = {}, Cursor = {} ", userCursorString, esScrollCursor);
+ logger.debug(" User cursor = {}, Cursor = {} ", candidateResults.getCursor(), esScrollCursor);
}
return candidateResults;
}
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
new file mode 100644
index 0000000..d146e4c
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -0,0 +1,197 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createLegacyContextName;
+
+/**
+ * Classy class class.
+ */
+
+public class SearchRequestBuilderStrategy {
+
+ private static final Logger logger = LoggerFactory.getLogger(SearchRequestBuilderStrategy.class);
+
+ private final EsProvider esProvider;
+ private final ApplicationScope applicationScope;
+ private final IndexIdentifier.IndexAlias alias;
+ private final int cursorTimeout;
+ public static final int MAX_LIMIT = 1000;
+
+ public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexIdentifier.IndexAlias alias, int cursorTimeout){
+
+ this.esProvider = esProvider;
+ this.applicationScope = applicationScope;
+ this.alias = alias;
+ this.cursorTimeout = cursorTimeout;
+ }
+
+ public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ Preconditions.checkArgument(limit <= MAX_LIMIT, "limit is greater than max "+ MAX_LIMIT);
+
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias())
+ .setTypes(searchTypes.getTypeNames(applicationScope))
+ .setScroll(cursorTimeout + "m")
+ .setQuery(createQueryBuilder( indexScope,query));
+
+ final FilterBuilder fb = createFilterBuilder(query);
+
+ //we have post filters, apply them
+ if (fb != null) {
+ logger.debug(" Filter: {} ", fb.toString());
+ srb = srb.setPostFilter(fb);
+ }
+
+
+ srb = srb.setFrom(0).setSize(limit);
+
+ for (Query.SortPredicate sp : query.getSortPredicates()) {
+
+ final SortOrder order;
+ if (sp.getDirection().equals(Query.SortDirection.ASCENDING)) {
+ order = SortOrder.ASC;
+ } else {
+ order = SortOrder.DESC;
+ }
+
+ // we do not know the type of the "order by" property and so we do not know what
+ // type prefix to use. So, here we add an order by clause for every possible type
+ // that you can order by: string, number and boolean and we ask ElasticSearch
+ // to ignore any fields that are not present.
+
+ final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(stringSort);
+
+ logger.debug(" Sort: {} order by {}", stringFieldName, order.toString());
+
+ final String longFieldName = LONG_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(longSort);
+ logger.debug(" Sort: {} order by {}", longFieldName, order.toString());
+
+
+ final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(doubleSort);
+ logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString());
+
+
+ final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(booleanSort);
+ logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
+ }
+ return srb;
+ }
+
+
+ public QueryBuilder createQueryBuilder( IndexScope indexScope, Query query) {
+ String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
+
+ QueryBuilder queryBuilder = null;
+
+ //we have a root operand. Translate our AST into an ES search
+ if ( query.getRootOperand() != null ) {
+ // In the case of geo only queries, this will return null into the query builder.
+ // Once we start using tiles, we won't need this check any longer, since a geo query
+ // will return a tile query + post filter
+ QueryVisitor v = new EsQueryVistor();
+
+ try {
+ query.getRootOperand().visit( v );
+ }
+ catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+
+
+ queryBuilder = v.getQueryBuilder();
+ }
+
+
+ // Add our filter for context to our query for fast execution.
+ // Fast because it utilizes bitsets internally. See this post for more detail.
+ // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+
+ // TODO evaluate performance when it's an all query.
+ // Do we need to put the context term first for performance?
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for(String context : contexts){
+ boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
+ }
+ boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
+ if ( queryBuilder != null ) {
+ queryBuilder = boolQueryBuilder.must( queryBuilder );
+ }
+
+ //nothing was specified ensure we specify the context in the search
+ else {
+ queryBuilder = boolQueryBuilder;
+ }
+
+ return queryBuilder;
+ }
+
+
+ public FilterBuilder createFilterBuilder(Query query) {
+ FilterBuilder filterBuilder = null;
+
+ if ( query.getRootOperand() != null ) {
+ QueryVisitor v = new EsQueryVistor();
+ try {
+ query.getRootOperand().visit( v );
+
+ } catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+ filterBuilder = v.getFilterBuilder();
+ }
+
+ return filterBuilder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
index be90446..5cb6680 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
@@ -19,12 +19,14 @@ package org.apache.usergrid.persistence.index.query;
import java.util.Iterator;
import java.util.List;
+
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Internal results class, should not be returned as results to a user.
+ * Internal results class, should not be returned as results to a user.
* Only returns candidate entity results
*/
public class CandidateResults implements Iterable<CandidateResult> {
@@ -33,16 +35,25 @@ public class CandidateResults implements Iterable<CandidateResult> {
private String cursor = null;
- private final Query query;
private final List<CandidateResult> candidates;
- public CandidateResults( Query query, List<CandidateResult> candidates ) {
- this.query = query;
+ public CandidateResults( List<CandidateResult> candidates ) {
this.candidates = candidates;
}
+ public void add( List<CandidateResult> candidates ) {
+ this.candidates.addAll( candidates);
+ }
+
+ public void initializeCursor(){
+ //USERGRID-461 our cursor is getting too large, map it to a new time UUID
+ //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home
+
+ cursor = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID(UUIDGenerator.newTimeUUID());
+ }
+
public boolean hasCursor() {
return cursor != null;
@@ -59,11 +70,8 @@ public class CandidateResults implements Iterable<CandidateResult> {
}
- public Query getQuery() {
- return query;
- }
-
+
public int size() {
return candidates.size();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 438d82e..6454b48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -40,10 +40,7 @@ import org.antlr.runtime.Token;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
-import org.apache.usergrid.persistence.index.impl.EsQueryVistor;
-import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.apache.usergrid.persistence.index.query.tree.AndOperand;
import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
@@ -55,15 +52,10 @@ import org.apache.usergrid.persistence.index.query.tree.GreaterThanEqual;
import org.apache.usergrid.persistence.index.query.tree.LessThan;
import org.apache.usergrid.persistence.index.query.tree.LessThanEqual;
import org.apache.usergrid.persistence.index.query.tree.Operand;
-import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,9 +72,7 @@ public class Query {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
}
- public static final int DEFAULT_LIMIT = 10;
- public static final int MAX_LIMIT = 1000;
public static final String PROPERTY_UUID = "uuid";
@@ -90,8 +80,6 @@ public class Query {
private List<SortPredicate> sortPredicates = new ArrayList<SortPredicate>();
private Operand rootOperand;
private UUID startResult;
- private String cursor;
- private int limit = 0;
private Map<String, String> selectAssignments = new LinkedHashMap<String, String>();
private boolean mergeSelectResults = false;
@@ -124,8 +112,6 @@ public class Query {
sortPredicates = q.sortPredicates != null
? new ArrayList<SortPredicate>( q.sortPredicates ) : null;
startResult = q.startResult;
- cursor = q.cursor;
- limit = q.limit;
selectAssignments = q.selectAssignments != null
? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
mergeSelectResults = q.mergeSelectResults;
@@ -148,71 +134,6 @@ public class Query {
}
- public QueryBuilder createQueryBuilder( final String[] contexts ) {
-
-
- QueryBuilder queryBuilder = null;
-
-
- //we have a root operand. Translate our AST into an ES search
- if ( getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
-
- try {
- getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
-
-
- queryBuilder = v.getQueryBuilder();
- }
-
-
- // Add our filter for context to our query for fast execution.
- // Fast because it utilizes bitsets internally. See this post for more detail.
- // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
-
- // TODO evaluate performance when it's an all query.
- // Do we need to put the context term first for performance?
- BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
- for(String context : contexts){
- boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
- }
- boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
- if ( queryBuilder != null ) {
- queryBuilder = boolQueryBuilder.must( queryBuilder );
- }
-
- //nothing was specified ensure we specify the context in the search
- else {
- queryBuilder = boolQueryBuilder;
- }
-
- return queryBuilder;
- }
-
-
- public FilterBuilder createFilterBuilder() {
- FilterBuilder filterBuilder = null;
-
- if ( getRootOperand() != null ) {
- QueryVisitor v = new EsQueryVistor();
- try {
- getRootOperand().visit( v );
-
- } catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
- filterBuilder = v.getFilterBuilder();
- }
-
- return filterBuilder;
- }
/**
@@ -315,7 +236,6 @@ public class Query {
String connection = ListUtils.first( params.get( "connectionType" ) );
UUID start = ListUtils.firstUuid( params.get( "start" ) );
String cursor = ListUtils.first( params.get( "cursor" ) );
- Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
@@ -386,15 +306,7 @@ public class Query {
q.setStartResult( start );
}
- if ( cursor != null ) {
- q = newQueryIfNull( q );
- q.setCursor( cursor );
- }
- if ( limit != null ) {
- q = newQueryIfNull( q );
- q.setLimit( limit );
- }
if ( startTime != null ) {
q = newQueryIfNull( q );
@@ -437,22 +349,21 @@ public class Query {
public static Query searchForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter( propertyName, propertyValue );
+ q.addEqualityFilter(propertyName, propertyValue);
return q;
}
public static Query findForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter( propertyName, propertyValue );
- q.setLimit( 1 );
+ q.addEqualityFilter(propertyName, propertyValue);
return q;
}
public static Query fromUUID( UUID uuid ) {
Query q = new Query();
- q.addIdentifier( Identifier.fromUUID( uuid ) );
+ q.addIdentifier( Identifier.fromUUID(uuid) );
return q;
}
@@ -809,8 +720,8 @@ public class Query {
public Query addContainsFilter( String propName, String keyword ) {
ContainsOperand equality = new ContainsOperand( new ClassicToken( 0, "contains" ) );
- equality.setProperty( propName );
- equality.setLiteral( keyword );
+ equality.setProperty(propName);
+ equality.setLiteral(keyword);
addClause( equality );
@@ -819,8 +730,8 @@ public class Query {
private void addClause( EqualityOperand equals, String propertyName, Object value ) {
- equals.setProperty( propertyName );
- equals.setLiteral( value );
+ equals.setProperty(propertyName);
+ equals.setLiteral(value);
addClause( equals );
}
@@ -882,7 +793,7 @@ public class Query {
}
- public UUID getStartResult() {
+ public UUID getStartResult(String cursor) {
if ( ( startResult == null ) && ( cursor != null ) ) {
byte[] cursorBytes = Base64.decodeBase64( cursor );
if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) {
@@ -893,61 +804,6 @@ public class Query {
}
- public String getCursor() {
- return cursor;
- }
-
-
- public void setCursor( String cursor ) {
- this.cursor = cursor;
- }
-
-
- public Query withCursor( String cursor ) {
- setCursor( cursor );
- return this;
- }
-
-
- public int getLimit() {
- return getLimit( DEFAULT_LIMIT );
- }
-
-
- public int getLimit( int defaultLimit ) {
- if ( limit <= 0 ) {
- if ( defaultLimit > 0 ) {
- return defaultLimit;
- }
- else {
- return DEFAULT_LIMIT;
- }
- }
- return limit;
- }
-
-
- public void setLimit( int limit ) {
-
- // TODO tnine. After users have had time to change their query limits,
- // this needs to be uncommented and enforced.
- // if(limit > MAX_LIMIT){
- // throw new IllegalArgumentException(
- // String.format("Query limit must be <= to %d", MAX_LIMIT));
- // }
-
- if ( limit > MAX_LIMIT ) {
- limit = MAX_LIMIT;
- }
-
- this.limit = limit;
- }
-
-
- public Query withLimit( int limit ) {
- setLimit( limit );
- return this;
- }
public boolean isReversed() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 6df5a6d..2c03635 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -118,7 +118,7 @@ public class EntityIndexTest extends BaseIT {
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ final SearchTypes searchTypes = SearchTypes.fromTypes(entityType);
EntityIndexBatch batch = entityIndex.createBatch();
Entity entity = new Entity( entityType );
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
@@ -143,7 +143,7 @@ public class EntityIndexTest extends BaseIT {
ei.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ testQueries(indexScope, searchTypes, entityIndex);
}
@Test
@@ -219,17 +219,17 @@ public class EntityIndexTest extends BaseIT {
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ final SearchTypes searchTypes = SearchTypes.fromTypes(entityType);
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,0);
ei.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ testQueries(indexScope, searchTypes, entityIndex);
- ei.addIndex("v2", 1,0,"one");
+ ei.addIndex("v2", 1, 0, "one");
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,100);
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json", 101, 100);
ei.refresh();
@@ -364,11 +364,12 @@ public class EntityIndexTest extends BaseIT {
StopWatch timer = new StopWatch();
timer.start();
Query query = Query.fromQL( queryString );
- query.setLimit( 1000 );
- CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query );
+ CandidateResults candidateResults = null;
+ candidateResults = entityIndex.search( scope, searchTypes, query,num+1 );
+
timer.stop();
- assertEquals( num, candidateResults.size() );
+ assertEquals( num,candidateResults.size() );
log.debug( "Query time {}ms", timer.getTime() );
return candidateResults;
}
@@ -652,13 +653,8 @@ public class EntityIndexTest extends BaseIT {
for ( int i = 0; i < expectedPages; i++ ) {
//**
Query query = Query.fromQL( "select * order by created" );
- query.setLimit( limit );
-
- if ( cursor != null ) {
- query.setCursor( cursor );
- }
- final CandidateResults results = entityIndex.search( indexScope, SearchTypes.allTypes(), query );
+ final CandidateResults results = cursor == null ? entityIndex.search( indexScope, SearchTypes.allTypes(), query , limit) : entityIndex.getNextPage(cursor);
assertTrue( results.hasCursor() );
@@ -667,8 +663,6 @@ public class EntityIndexTest extends BaseIT {
assertEquals("Should be 16 bytes as hex", 32, cursor.length());
-
-
assertEquals( 1, results.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index 102e222..792fe3a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -19,6 +19,9 @@
package org.apache.usergrid.persistence.query.tree;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.impl.SearchRequestBuilderStrategy;
import org.apache.usergrid.persistence.index.query.tree.LongLiteral;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterParser;
@@ -43,6 +46,7 @@ import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.junit.Test;
import org.apache.usergrid.persistence.index.query.Query;
import org.elasticsearch.index.query.QueryBuilder;
@@ -429,7 +433,9 @@ public class GrammarTreeTest {
assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
- QueryBuilder qb = query.createQueryBuilder(new String[]{"testcontext"});
+ SearchRequestBuilderStrategy builderStrategy = new SearchRequestBuilderStrategy(null,new ApplicationScopeImpl(new SimpleId("test")),null,100);
+ QueryBuilder qb =builderStrategy.createQueryBuilder(new IndexScopeImpl(new SimpleId("owner"),"app"),query);
+
}
[22/50] incubator-usergrid git commit: Updates iteration logic in the
management service for iterating apps
Posted by sf...@apache.org.
Updates iteration logic in the management service for iterating apps
Reverts connect back removal. This is required for dual indexing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/beb2a2a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/beb2a2a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/beb2a2a5
Branch: refs/heads/USERGRID-473
Commit: beb2a2a5337927f13d9a7c563bb936214eda159b
Parents: c7fa864
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 25 16:59:21 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 25 16:59:21 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 13 ++++++-------
stack/core/src/test/resources/log4j.properties | 2 +-
.../cassandra/ManagementServiceImpl.java | 18 +++++++++++-------
3 files changed, 18 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/beb2a2a5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 7179baf..da39ea9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -702,13 +702,12 @@ public class CpRelationManager implements RelationManager {
// headEntityScope.getOwner().toString(),
// headEntityScope.getName()});
- //TODO T.N. This should even be neccessary any longer, graph maintains 2 edges. .
-// if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
-// getRelationManager( itemEntity ).addToCollection(
-// collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
-// getRelationManager( itemEntity ).addToCollection(
-// collection.getLinkedCollection(), headEntity, false );
-// }
+ if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
+ getRelationManager( itemEntity ).addToCollection(
+ collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
+ getRelationManager( itemEntity ).addToCollection(
+ collection.getLinkedCollection(), headEntity, false );
+ }
return itemEntity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/beb2a2a5/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 0ba16ea..3dee8f6 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -45,7 +45,7 @@ log4j.logger.org.apache.usergrid.persistence.PerformanceEntityRebuildIndexTest=D
log4j.logger.org.apache.usergrid.corepersistence.migration=WARN
-log4j.logger.org.apache.usergrid.persistence.index.impl=DEBUG
+#log4j.logger.org.apache.usergrid.persistence.index.impl=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
#log4j.logger.org.apache.usergrid.corepersistence=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManagerFactory=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/beb2a2a5/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 854c3e0..0cf80b4 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1477,21 +1477,25 @@ public class ManagementServiceImpl implements ManagementService {
BiMap<UUID, String> organizations = HashBiMap.create();
EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
- Results results = em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userId ), "groups", null, 10000,
+ Results results = em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userId ), "groups", null, 1000,
Level.ALL_PROPERTIES, false );
String path = null;
- for ( Entity entity : results.getEntities() ) {
+ do {
+ for ( Entity entity : results.getEntities() ) {
+
+ path = ( String ) entity.getProperty( PROPERTY_PATH );
- path = ( String ) entity.getProperty( PROPERTY_PATH );
+ if ( path != null ) {
+ path = path.toLowerCase();
+ }
- if ( path != null ) {
- path = path.toLowerCase();
+ organizations.put( entity.getUuid(), path );
}
- organizations.put( entity.getUuid(), path );
- }
+ results = results.getNextPageResults();
+ }while(results != null);
return organizations;
}
[29/50] incubator-usergrid git commit: move back to old Query object
Posted by sf...@apache.org.
move back to old Query object
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/96d27b95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/96d27b95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/96d27b95
Branch: refs/heads/USERGRID-473
Commit: 96d27b95336b41aee16180fe80fbf9bc9fb21f4d
Parents: c09e6b4
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 13:34:01 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 13:34:01 2015 -0600
----------------------------------------------------------------------
.../index/ApplicationEntityIndex.java | 6 +-
.../usergrid/persistence/index/EntityIndex.java | 2 +
.../impl/EsApplicationEntityIndexImpl.java | 9 +-
.../impl/SearchRequestBuilderStrategy.java | 11 +-
.../usergrid/persistence/index/query/Query.java | 184 ++++++++++++++++---
5 files changed, 174 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 2167efa..59a19eb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -43,11 +43,9 @@ public interface ApplicationEntityIndex {
* Execute query in Usergrid syntax.
*/
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query);
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit);
+
- /**
- * Execute query in Usergrid syntax.
- */
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, final Query query, final int limit );
/**
* get next page of results
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index af6b013..6783fb0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -39,6 +39,8 @@ import java.util.concurrent.Future;
public interface EntityIndex extends CPManager {
+ public static final int MAX_LIMIT = 1000;
+
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
* @param indexSuffix index name
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 27791ae..603bee4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -136,16 +136,15 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query){
- return search(indexScope,searchTypes,query,10);
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query) {
+ return search(indexScope,searchTypes,query,query.getLimit());
}
-
@Override
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit){
SearchResponse searchResponse;
- SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query, limit);
+ SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query,limit);
if (logger.isDebugEnabled()) {
logger.debug("Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 15796f9..207a7a8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -22,10 +22,7 @@ package org.apache.usergrid.persistence.index.impl;
import com.google.common.base.Preconditions;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.IndexAlias;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
@@ -56,7 +53,6 @@ public class SearchRequestBuilderStrategy {
private final ApplicationScope applicationScope;
private final IndexAlias alias;
private final int cursorTimeout;
- public static final int MAX_LIMIT = 1000;
public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){
@@ -66,8 +62,9 @@ public class SearchRequestBuilderStrategy {
this.cursorTimeout = cursorTimeout;
}
- public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
- Preconditions.checkArgument(limit <= MAX_LIMIT, "limit is greater than max "+ MAX_LIMIT);
+ public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+
+ Preconditions.checkArgument(limit <= EntityIndex.MAX_LIMIT, "limit is greater than max "+ EntityIndex.MAX_LIMIT);
SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias())
.setTypes(searchTypes.getTypeNames(applicationScope))
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index fa7def4..5567382 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -40,7 +40,10 @@ import org.antlr.runtime.Token;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
+import org.apache.usergrid.persistence.index.impl.EsQueryVistor;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.apache.usergrid.persistence.index.query.tree.AndOperand;
import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
@@ -52,10 +55,14 @@ import org.apache.usergrid.persistence.index.query.tree.GreaterThanEqual;
import org.apache.usergrid.persistence.index.query.tree.LessThan;
import org.apache.usergrid.persistence.index.query.tree.LessThanEqual;
import org.apache.usergrid.persistence.index.query.tree.Operand;
+import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,7 +79,9 @@ public class Query {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
}
+ public static final int DEFAULT_LIMIT = 10;
+ public static final int MAX_LIMIT = 1000;
public static final String PROPERTY_UUID = "uuid";
@@ -80,6 +89,8 @@ public class Query {
private List<SortPredicate> sortPredicates = new ArrayList<SortPredicate>();
private Operand rootOperand;
private UUID startResult;
+ private String cursor;
+ private int limit = 0;
private Map<String, String> selectAssignments = new LinkedHashMap<String, String>();
private boolean mergeSelectResults = false;
@@ -111,19 +122,22 @@ public class Query {
* @param q
*/
public Query( Query q ) {
- if (q == null) {
+ if ( q == null ) {
return;
}
+
type = q.type;
sortPredicates = q.sortPredicates != null
- ? new ArrayList<SortPredicate>(q.sortPredicates) : null;
+ ? new ArrayList<>( q.sortPredicates ) : null;
startResult = q.startResult;
+ cursor = q.cursor;
+ limit = q.limit;
selectAssignments = q.selectAssignments != null
- ? new LinkedHashMap<String, String>(q.selectAssignments) : null;
+ ? new LinkedHashMap<>( q.selectAssignments ) : null;
mergeSelectResults = q.mergeSelectResults;
//level = q.level;
connectionType = q.connectionType;
- permissions = q.permissions != null ? new ArrayList<String>(q.permissions) : null;
+ permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null;
reversed = q.reversed;
reversedSet = q.reversedSet;
startTime = q.startTime;
@@ -132,16 +146,76 @@ public class Query {
pad = q.pad;
rootOperand = q.rootOperand;
identifiers = q.identifiers != null
- ? new ArrayList<Identifier>(q.identifiers) : null;
+ ? new ArrayList<>( q.identifiers ) : null;
counterFilters = q.counterFilters != null
- ? new ArrayList<CounterFilterPredicate>(q.counterFilters) : null;
+ ? new ArrayList<>( q.counterFilters ) : null;
collection = q.collection;
-
level = q.level;
}
+ public QueryBuilder createQueryBuilder( final String context ) {
+
+
+ QueryBuilder queryBuilder = null;
+
+
+ //we have a root operand. Translate our AST into an ES search
+ if ( getRootOperand() != null ) {
+ // In the case of geo only queries, this will return null into the query builder.
+ // Once we start using tiles, we won't need this check any longer, since a geo query
+ // will return a tile query + post filter
+ QueryVisitor v = new EsQueryVistor();
+
+ try {
+ getRootOperand().visit( v );
+ }
+ catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+
+
+ queryBuilder = v.getQueryBuilder();
+ }
+
+
+ // Add our filter for context to our query for fast execution.
+ // Fast because it utilizes bitsets internally. See this post for more detail.
+ // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+
+ // TODO evaluate performance when it's an all query.
+ // Do we need to put the context term first for performance?
+ if ( queryBuilder != null ) {
+ queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
+ .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
+ }
+
+ //nothing was specified ensure we specify the context in the search
+ else {
+ queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+ }
+
+ return queryBuilder;
+ }
+
+
+ public FilterBuilder createFilterBuilder() {
+ FilterBuilder filterBuilder = null;
+
+ if ( getRootOperand() != null ) {
+ QueryVisitor v = new EsQueryVistor();
+ try {
+ getRootOperand().visit( v );
+
+ } catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+ filterBuilder = v.getFilterBuilder();
+ }
+
+ return filterBuilder;
+ }
/**
@@ -169,8 +243,8 @@ public class Query {
String qlt = ql.toLowerCase();
if ( !qlt.startsWith( "select" )
- && !qlt.startsWith( "insert" )
- && !qlt.startsWith( "update" ) && !qlt.startsWith( "delete" ) ) {
+ && !qlt.startsWith( "insert" )
+ && !qlt.startsWith( "update" ) && !qlt.startsWith( "delete" ) ) {
if ( qlt.startsWith( "order by" ) ) {
ql = "select * " + ql;
@@ -224,7 +298,7 @@ public class Query {
if ( o instanceof Map ) {
@SuppressWarnings({ "unchecked", "rawtypes" }) Map<String, List<String>> params =
- ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) );
+ ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) );
return fromQueryParams( params );
}
return null;
@@ -232,7 +306,7 @@ public class Query {
public static Query fromQueryParams( Map<String, List<String>> params )
- throws QueryParseException {
+ throws QueryParseException {
Query q = null;
CounterResolution resolution = null;
List<Identifier> identifiers = null;
@@ -243,6 +317,8 @@ public class Query {
Boolean reversed = ListUtils.firstBoolean( params.get( "reversed" ) );
String connection = ListUtils.first( params.get( "connectionType" ) );
UUID start = ListUtils.firstUuid( params.get( "start" ) );
+ String cursor = ListUtils.first( params.get( "cursor" ) );
+ Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
@@ -313,7 +389,15 @@ public class Query {
q.setStartResult( start );
}
+ if ( cursor != null ) {
+ q = newQueryIfNull( q );
+ q.setCursor( cursor );
+ }
+ if ( limit != null ) {
+ q = newQueryIfNull( q );
+ q.setLimit( limit );
+ }
if ( startTime != null ) {
q = newQueryIfNull( q );
@@ -356,21 +440,22 @@ public class Query {
public static Query searchForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter(propertyName, propertyValue);
+ q.addEqualityFilter( propertyName, propertyValue );
return q;
}
public static Query findForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter(propertyName, propertyValue);
+ q.addEqualityFilter( propertyName, propertyValue );
+ q.setLimit( 1 );
return q;
}
public static Query fromUUID( UUID uuid ) {
Query q = new Query();
- q.addIdentifier( Identifier.fromUUID(uuid) );
+ q.addIdentifier( Identifier.fromUUID( uuid ) );
return q;
}
@@ -629,8 +714,8 @@ public class Query {
for ( SortPredicate s : sortPredicates ) {
if ( s.getPropertyName().equals( propertyName ) ) {
logger.error(
- "Attempted to set sort order for " + s.getPropertyName()
- + " more than once, discarding..." );
+ "Attempted to set sort order for " + s.getPropertyName()
+ + " more than once, discarding..." );
return this;
}
}
@@ -727,8 +812,8 @@ public class Query {
public Query addContainsFilter( String propName, String keyword ) {
ContainsOperand equality = new ContainsOperand( new ClassicToken( 0, "contains" ) );
- equality.setProperty(propName);
- equality.setLiteral(keyword);
+ equality.setProperty( propName );
+ equality.setLiteral( keyword );
addClause( equality );
@@ -737,8 +822,8 @@ public class Query {
private void addClause( EqualityOperand equals, String propertyName, Object value ) {
- equals.setProperty(propertyName);
- equals.setLiteral(value);
+ equals.setProperty( propertyName );
+ equals.setLiteral( value );
addClause( equals );
}
@@ -800,7 +885,7 @@ public class Query {
}
- public UUID getStartResult(String cursor) {
+ public UUID getStartResult() {
if ( ( startResult == null ) && ( cursor != null ) ) {
byte[] cursorBytes = Base64.decodeBase64( cursor );
if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) {
@@ -811,6 +896,61 @@ public class Query {
}
+ public String getCursor() {
+ return cursor;
+ }
+
+
+ public void setCursor( String cursor ) {
+ this.cursor = cursor;
+ }
+
+
+ public Query withCursor( String cursor ) {
+ setCursor( cursor );
+ return this;
+ }
+
+
+ public int getLimit() {
+ return getLimit( DEFAULT_LIMIT );
+ }
+
+
+ public int getLimit( int defaultLimit ) {
+ if ( limit <= 0 ) {
+ if ( defaultLimit > 0 ) {
+ return defaultLimit;
+ }
+ else {
+ return DEFAULT_LIMIT;
+ }
+ }
+ return limit;
+ }
+
+
+ public void setLimit( int limit ) {
+
+ // TODO tnine. After users have had time to change their query limits,
+ // this needs to be uncommented and enforced.
+ // if(limit > MAX_LIMIT){
+ // throw new IllegalArgumentException(
+ // String.format("Query limit must be <= to %d", MAX_LIMIT));
+ // }
+
+ if ( limit > MAX_LIMIT ) {
+ limit = MAX_LIMIT;
+ }
+
+ this.limit = limit;
+ }
+
+
+ public Query withLimit( int limit ) {
+ setLimit( limit );
+ return this;
+ }
public boolean isReversed() {
@@ -998,7 +1138,7 @@ public class Query {
public SortPredicate(@JsonProperty("propertyName") String propertyName,
- @JsonProperty("direction") Query.SortDirection direction ) {
+ @JsonProperty("direction") Query.SortDirection direction ) {
if ( propertyName == null ) {
throw new NullPointerException( "Property name was null" );
[19/50] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into USERGRID-501
Posted by sf...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-501
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e5eb57fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e5eb57fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e5eb57fd
Branch: refs/heads/USERGRID-473
Commit: e5eb57fd80525def86211e7a99cdc71be93a8e5a
Parents: 90832ca 66fdc61
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 16:37:55 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 16:37:55 2015 -0600
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 26 +-
.../applications/ApplicationResource.java | 44 +-
.../applications/imports/ImportsResource.java | 4 -
.../java/org/apache/usergrid/rest/BasicIT.java | 302 +--------
.../apache/usergrid/rest/SystemResourceIT.java | 10 +-
.../applications/ApplicationResourceIT.java | 2 +-
.../collection/CollectionsResourceIT.java | 19 +-
.../collection/users/OwnershipResourceIT.java | 4 +-
.../collection/users/RetrieveUsersTest.java | 4 +-
.../collection/users/UserResourceIT.java | 4 +-
.../usergrid/rest/management/AccessTokenIT.java | 418 +++++-------
.../usergrid/rest/management/AdminUsersIT.java | 60 +-
.../rest/management/ExportResourceIT.java | 652 +++++++------------
.../rest/management/ImportResourceIT.java | 20 +-
.../rest/management/OrganizationsIT.java | 16 +-
.../organizations/AdminEmailEncodingIT.java | 3 +-
.../test/resource2point0/AbstractRestIT.java | 4 +-
.../rest/test/resource2point0/ClientSetup.java | 30 +-
.../rest/test/resource2point0/RestClient.java | 1 +
.../endpoints/NamedResource.java | 112 +++-
.../endpoints/SetupResource.java | 6 +
.../endpoints/mgmt/ApplicationResource.java | 18 +-
.../endpoints/mgmt/ManagementResource.java | 4 +
.../endpoints/mgmt/MeResource.java | 32 +
.../endpoints/mgmt/PasswordResource.java | 16 -
.../endpoints/mgmt/RevokeTokenResource.java | 32 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 +
.../endpoints/mgmt/TokenResource.java | 25 +-
.../endpoints/mgmt/UserResource.java | 8 +
.../endpoints/mgmt/UsersResource.java | 15 -
30 files changed, 809 insertions(+), 1115 deletions(-)
----------------------------------------------------------------------
[38/50] incubator-usergrid git commit: Add wait to lookupApplication()
Posted by sf...@apache.org.
Add wait to lookupApplication()
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e6117930
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e6117930
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e6117930
Branch: refs/heads/USERGRID-473
Commit: e61179305c98a58168bfd672ea46ee789d0fe32a
Parents: 2eca36f
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Mar 30 09:50:23 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Mar 30 09:50:23 2015 -0400
----------------------------------------------------------------------
.../cassandra/EntityManagerFactoryImplIT.java | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e6117930/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index d52f3f6..b64f0b5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -166,7 +166,15 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
// attempt to get entities in application's collections in various ways should all fail
- assertNull( setup.getEmf().lookupApplication( orgName + "/" + appName ));
+ for ( int i=0; i<maxRetries; i++ ) {
+ found = ( setup.getEmf().lookupApplication( orgName + "/" + appName ) != null );
+ if ( found ) {
+ Thread.sleep( 500 );
+ } else {
+ break;
+ }
+ }
+ assertFalse( "Lookup of deleted app must fail", found );
// app must not be found in apps collection
@@ -182,6 +190,8 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
// restore the app
+ emf.refreshIndex();
+
emf.restoreApplication( deletedAppId );
emf.rebuildAllIndexes(new EntityManagerFactory.ProgressObserver() {
[28/50] incubator-usergrid git commit: add version migration for
indexes
Posted by sf...@apache.org.
add version migration for indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c09e6b4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c09e6b4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c09e6b4d
Branch: refs/heads/USERGRID-473
Commit: c09e6b4d8d3213c54976184593e6bb592b0c130b
Parents: bf0718e
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 11:27:34 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 11:27:34 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/migration/IndexDataVersions.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c09e6b4d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
index 921d86a..6558339 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
@@ -21,7 +21,7 @@
package org.apache.usergrid.persistence.index.migration;
/**
- * Classy class class.
+ * Index data migration
*/
public enum IndexDataVersions {
MANY_INDEXES(0),
[21/50] incubator-usergrid git commit: Refactored search to
encapsulate repair and load logic
Posted by sf...@apache.org.
Refactored search to encapsulate repair and load logic
Loaders and verifiers are separate for connections and collections
Fixes broken test in GraphManager if writes occur in the same millisecond
Reduces fork count to 1 and uses parallel threads
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c7fa864f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c7fa864f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c7fa864f
Branch: refs/heads/USERGRID-473
Commit: c7fa864f73e742c8dd37071ea57d5b4e2ec31422
Parents: 5937f9f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 25 12:38:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 25 16:45:31 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 316 ++++++-------------
.../results/CollectionRefsVerifier.java | 44 +++
.../CollectionResultsLoaderFactoryImpl.java | 60 ++++
.../results/ConnectionRefsVerifier.java | 61 ++++
.../ConnectionResultsLoaderFactoryImpl.java | 65 ++++
.../results/ElasticSearchQueryExecutor.java | 211 +++++++++++++
.../corepersistence/results/QueryExecutor.java | 37 +++
.../corepersistence/results/RefsVerifier.java | 42 ---
.../results/ResultsLoaderFactory.java | 3 +-
.../results/ResultsLoaderFactoryImpl.java | 62 ----
.../persistence/MultiQueryIterator.java | 6 +-
.../apache/usergrid/persistence/Results.java | 26 +-
.../cassandra/QueryProcessorImpl.java | 12 +-
.../usergrid/persistence/PathQueryIT.java | 46 ++-
stack/core/src/test/resources/log4j.properties | 2 +
.../persistence/graph/GraphManagerIT.java | 9 +-
.../usergrid/persistence/index/IndexFig.java | 16 +-
.../index/impl/BufferQueueInMemoryImpl.java | 10 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 61 ++--
.../usergrid/persistence/index/query/Query.java | 58 ++--
.../persistence/index/utils/ListUtils.java | 2 -
stack/pom.xml | 2 +-
22 files changed, 729 insertions(+), 422 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 2eeee28..7179baf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -29,14 +29,14 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.corepersistence.results.ResultsLoader;
-import org.apache.usergrid.corepersistence.results.ResultsLoaderFactory;
-import org.apache.usergrid.corepersistence.results.ResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
+import org.apache.usergrid.corepersistence.results.QueryExecutor;
+import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.ConnectedEntityRef;
@@ -61,6 +61,7 @@ import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
@@ -80,8 +81,6 @@ import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.Level;
@@ -115,6 +114,7 @@ import me.prettyprint.hector.api.beans.HColumn;
import me.prettyprint.hector.api.mutation.Mutator;
import rx.Observable;
import rx.functions.Action1;
+import rx.functions.Action2;
import rx.functions.Func1;
import static java.util.Arrays.asList;
@@ -186,8 +186,6 @@ public class CpRelationManager implements RelationManager {
private IndexBucketLocator indexBucketLocator;
- private ResultsLoaderFactory resultsLoaderFactory;
-
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
private Timer createConnectionTimer;
@@ -258,7 +256,6 @@ public class CpRelationManager implements RelationManager {
// commented out because it is possible that CP entity has not been created yet
Assert.notNull( cpHeadEntity, "cpHeadEntity cannot be null" );
- this.resultsLoaderFactory = new ResultsLoaderFactoryImpl( managerCache );
return this;
}
@@ -278,7 +275,7 @@ public class CpRelationManager implements RelationManager {
});
Observable<String> types= gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeTypePrefix, null ));
+ new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeTypePrefix, null ) );
Iterator<String> iter = types.toBlockingObservable().getIterator();
while ( iter.hasNext() ) {
@@ -316,63 +313,56 @@ public class CpRelationManager implements RelationManager {
/**
* Gets containing collections and/or connections depending on the edge type you pass in
+ *
* @param limit Max number to return
* @param edgeType Edge type, edge type prefix or null to allow any edge type
* @param fromEntityType Only consider edges from entities of this type
*/
- Map<EntityRef, Set<String>> getContainers( int limit, String edgeType, String fromEntityType ) {
+ Map<EntityRef, Set<String>> getContainers( final int limit, final String edgeType, final String fromEntityType ) {
Map<EntityRef, Set<String>> results = new LinkedHashMap<EntityRef, Set<String>>();
- GraphManager gm = managerCache.getGraphManager(applicationScope);
-
- Iterator<String> edgeTypes = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType(
- cpHeadEntity.getId(), edgeType, null) ).toBlocking().getIterator();
-
- logger.debug("getContainers(): "
- + "Searched for edges of type {}\n to target {}:{}\n in scope {}\n found: {}",
- new Object[] {
- edgeType,
- cpHeadEntity.getId().getType(),
- cpHeadEntity.getId().getUuid(),
- applicationScope.getApplication(),
- edgeTypes.hasNext()
- });
+ final GraphManager gm = managerCache.getGraphManager( applicationScope );
- while ( edgeTypes.hasNext() ) {
+ Observable<Edge> edges =
+ gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeType, null ) )
+ .flatMap( new Func1<String, Observable<Edge>>() {
+ @Override
+ public Observable<Edge> call( final String edgeType ) {
+ return gm.loadEdgesToTarget(
+ new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, null ) );
- String etype = edgeTypes.next();
+ }
+ } );
- Observable<Edge> edges = gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
- cpHeadEntity.getId(), etype, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ));
+ //if our limit is set, take them. Note this logic is still borked, we can't possibly fit everything in memmory
+ if ( limit > -1 ) {
+ edges = edges.take( limit );
+ }
- Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
- while ( iter.hasNext() ) {
- Edge edge = iter.next();
- if ( fromEntityType != null && !fromEntityType.equals( edge.getSourceNode().getType() )) {
- logger.debug("Ignoring edge from entity type {}", edge.getSourceNode().getType());
- continue;
+ return edges.collect( results, new Action2<Map<EntityRef, Set<String>>, Edge>() {
+ @Override
+ public void call( final Map<EntityRef, Set<String>> entityRefSetMap, final Edge edge ) {
+ if ( fromEntityType != null && !fromEntityType.equals( edge.getSourceNode().getType() ) ) {
+ logger.debug( "Ignoring edge from entity type {}", edge.getSourceNode().getType() );
+ return;
}
- EntityRef eref = new SimpleEntityRef(
- edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
+ final EntityRef eref =
+ new SimpleEntityRef( edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
- String name = null;
- if ( CpNamingUtils.isConnectionEdgeType( edge.getType() )) {
+ String name;
+ if ( CpNamingUtils.isConnectionEdgeType( edge.getType() ) ) {
name = CpNamingUtils.getConnectionType( edge.getType() );
- } else {
+ }
+ else {
name = CpNamingUtils.getCollectionName( edge.getType() );
}
- addMapSet( results, eref, name );
- }
-
- if ( limit > 0 && results.keySet().size() >= limit ) {
- break;
+ addMapSet( entityRefSetMap, eref, name );
}
- }
-
- return results;
+ } ).toBlocking().last();
}
@@ -660,9 +650,6 @@ public class CpRelationManager implements RelationManager {
CollectionScope memberScope = getCollectionScopeNameFromEntityType(
applicationScope.getApplication(), itemRef.getType());
- //TODO, this double load should disappear once events are in
- Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
-
if ( memberEntity == null ) {
throw new RuntimeException(
"Unable to load entity uuid=" + itemRef.getUuid() + " type=" + itemRef.getType() );
@@ -694,34 +681,34 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager( applicationScope );
gm.writeEdge( edge ).toBlockingObservable().last();
- logger.debug( "Wrote edgeType {}\n from {}:{}\n to {}:{}\n scope {}:{}",
- new Object[] {
- edgeType,
- cpHeadEntity.getId().getType(),
- cpHeadEntity.getId().getUuid(),
- memberEntity.getId().getType(),
- memberEntity.getId().getUuid(),
- applicationScope.getApplication().getType(),
+
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Wrote edgeType {}\n from {}:{}\n to {}:{}\n scope {}:{}", new Object[] {
+ edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(), memberEntity.getId().getType(),
+ memberEntity.getId().getUuid(), applicationScope.getApplication().getType(),
applicationScope.getApplication().getUuid()
- } );
+ } );
+ }
( ( CpEntityManager ) em ).indexEntityIntoCollection( cpHeadEntity, memberEntity, collName );
- logger.debug( "Added entity {}:{} to collection {}", new Object[] {
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Added entity {}:{} to collection {}", new Object[] {
itemRef.getUuid().toString(), itemRef.getType(), collName
- } );
-
+ } );
+ }
// logger.debug("With head entity scope is {}:{}:{}", new Object[] {
// headEntityScope.getApplication().toString(),
// headEntityScope.getOwner().toString(),
// headEntityScope.getName()});
- if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
- getRelationManager( itemEntity ).addToCollection(
- collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
- getRelationManager( itemEntity ).addToCollection(
- collection.getLinkedCollection(), headEntity, false );
- }
+ //TODO T.N. This should even be neccessary any longer, graph maintains 2 edges. .
+// if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
+// getRelationManager( itemEntity ).addToCollection(
+// collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
+// getRelationManager( itemEntity ).addToCollection(
+// collection.getLinkedCollection(), headEntity, false );
+// }
return itemEntity;
}
@@ -951,65 +938,19 @@ public class CpRelationManager implements RelationManager {
logger.debug( "Searching scope {}:{}",
- indexScope.getOwner().toString(), indexScope.getName() );
+ indexScope.getOwner().toString(), indexScope.getName() );
query.setEntityType( collection.getType() );
query = adjustQuery( query );
- // Because of possible stale entities, which are filtered out by buildResults(),
- // we loop until the we've got enough results to satisfy the query limit.
- int maxQueries = 10; // max re-queries to satisfy query limit
-
- final int originalLimit = query.getLimit();
-
- Results results = null;
- int queryCount = 0;
-
- boolean satisfied = false;
+ final CollectionResultsLoaderFactoryImpl resultsLoaderFactory = new CollectionResultsLoaderFactoryImpl( managerCache );
+ //execute the query and return our next result
+ final QueryExecutor executor = new ElasticSearchQueryExecutor( resultsLoaderFactory, ei, applicationScope, indexScope, types, query );
- while ( !satisfied && queryCount++ < maxQueries ) {
-
- CandidateResults crs = ei.search( indexScope, types, query );
-
- if ( results == null ) {
- logger.debug( "Calling build results 1" );
- results = buildResults( indexScope, query, crs, collName );
- }
- else {
- logger.debug( "Calling build results 2" );
- Results newResults = buildResults(indexScope, query, crs, collName );
- results.merge( newResults );
- }
-
- if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get more
- satisfied = true;
- }
-
- /**
- * In an edge case where we delete stale entities, we could potentially get more results than expected. This will only occur once during the repair phase.
- * We need to ensure that we short circuit if we overflow the requested limit during a repair.
- */
- else if ( results.size() >= originalLimit ) { // got what we need
- satisfied = true;
- }
- else if ( crs.hasCursor() ) {
- satisfied = false;
-
- // need to query for more
- // ask for just what we need to satisfy, don't want to exceed limit
- query.setCursor( results.getCursor() );
- query.setLimit( originalLimit - results.size() );
-
- logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
- originalLimit, query.getLimit(), queryCount
- } );
- }
- }
-
- return results;
+ return executor.next();
}
@@ -1350,6 +1291,7 @@ public class CpRelationManager implements RelationManager {
public Results getConnectedEntities(
String connectionType, String connectedEntityType, Level level ) throws Exception {
+ //until this is refactored properly, we will delegate to a search by query
Results raw = null;
Preconditions.checkNotNull( connectionType, "connectionType cannot be null" );
@@ -1357,50 +1299,11 @@ public class CpRelationManager implements RelationManager {
Query query = new Query();
query.setConnectionType( connectionType );
query.setEntityType( connectedEntityType );
+ query.setResultsLevel( level );
- if ( connectionType == null ) {
- raw = searchConnectedEntities( query );
- }
- else {
-
- headEntity = em.validate( headEntity );
-
-
- IndexScope indexScope = new IndexScopeImpl(
- cpHeadEntity.getId(), CpNamingUtils.getConnectionScopeName( connectionType ) );
-
- final SearchTypes searchTypes = SearchTypes.fromNullableTypes( connectedEntityType );
-
-
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
-
-
- logger.debug("Searching connected entities from scope {}:{}",
- indexScope.getOwner().toString(),
- indexScope.getName());
-
- query = adjustQuery( query );
- CandidateResults crs = ei.search( indexScope, searchTypes, query );
-
- raw = buildResults( indexScope, query, crs, query.getConnectionType() );
- }
-
- if ( Level.ALL_PROPERTIES.equals( level ) ) {
- List<Entity> entities = new ArrayList<Entity>();
- for ( EntityRef ref : raw.getEntities() ) {
- Entity entity = em.get( ref );
- entities.add( entity );
- }
- return Results.fromEntities( entities );
- }
+ return searchConnectedEntities( query );
- List<ConnectionRef> crefs = new ArrayList<ConnectionRef>();
- for ( Entity e : raw.getEntities() ) {
- ConnectionRef cref = new ConnectionRefImpl( headEntity, connectionType, e );
- crefs.add( cref );
- }
- return Results.fromConnections( crefs );
}
@@ -1472,9 +1375,16 @@ public class CpRelationManager implements RelationManager {
} );
query = adjustQuery( query );
- CandidateResults crs = ei.search( indexScope, searchTypes, query );
- return buildConnectionResults( indexScope, query, crs, connection );
+ final ConnectionResultsLoaderFactoryImpl resultsLoaderFactory = new ConnectionResultsLoaderFactoryImpl( managerCache,
+ headEntity, connection );
+
+ final QueryExecutor executor = new ElasticSearchQueryExecutor(resultsLoaderFactory, ei, applicationScope, indexScope, searchTypes, query);
+
+ return executor.next();
+// CandidateResults crs = ei.search( indexScope, searchTypes, query );
+
+// return buildConnectionResults( indexScope, query, crs, connection );
}
@@ -1564,63 +1474,33 @@ public class CpRelationManager implements RelationManager {
return entity;
}
+//
+// private Results buildConnectionResults( final IndexScope indexScope,
+// final Query query, final CandidateResults crs, final String connectionType ) {
+//
+// if ( query.getLevel().equals( Level.ALL_PROPERTIES ) ) {
+// return buildResults( indexScope, query, crs, connectionType );
+// }
+//
+// final EntityRef sourceRef = new SimpleEntityRef( headEntity.getType(), headEntity.getUuid() );
+//
+// List<ConnectionRef> refs = new ArrayList<ConnectionRef>( crs.size() );
+//
+// for ( CandidateResult cr : crs ) {
+//
+// SimpleEntityRef targetRef =
+// new SimpleEntityRef( cr.getId().getType(), cr.getId().getUuid() );
+//
+// final ConnectionRef ref =
+// new ConnectionRefImpl( sourceRef, connectionType, targetRef );
+//
+// refs.add( ref );
+// }
+//
+// return Results.fromConnections( refs );
+// }
- private Results buildConnectionResults( final IndexScope indexScope,
- final Query query, final CandidateResults crs, final String connectionType ) {
-
- if ( query.getLevel().equals( Level.ALL_PROPERTIES ) ) {
- return buildResults( indexScope, query, crs, connectionType );
- }
-
- final EntityRef sourceRef = new SimpleEntityRef( headEntity.getType(), headEntity.getUuid() );
-
- List<ConnectionRef> refs = new ArrayList<ConnectionRef>( crs.size() );
-
- for ( CandidateResult cr : crs ) {
-
- SimpleEntityRef targetRef =
- new SimpleEntityRef( cr.getId().getType(), cr.getId().getUuid() );
-
- final ConnectionRef ref =
- new ConnectionRefImpl( sourceRef, connectionType, targetRef );
-
- refs.add( ref );
- }
-
- return Results.fromConnections( refs );
- }
-
-
- /**
- * Build results from a set of candidates, and discard those that represent stale indexes.
- *
- * @param query Query that was executed
- * @param crs Candidates to be considered for results
- * @param collName Name of collection or null if querying all types
- */
- private Results buildResults( final IndexScope indexScope, final Query query,
- final CandidateResults crs, final String collName ) {
-
- logger.debug( "buildResults() for {} from {} candidates", collName, crs.size() );
-
- //get an instance of our results loader
- final ResultsLoader resultsLoader = this.resultsLoaderFactory.getLoader(
- applicationScope, indexScope, query.getResultsLevel() );
-
- //load the results
- final Results results = resultsLoader.loadResults( crs );
-
- //signal for post processing
- resultsLoader.postProcess();
-
-
- results.setCursor( crs.getCursor() );
- results.setQueryProcessor( new CpQueryProcessor( em, query, headEntity, collName ) );
-
- logger.debug( "Returning results size {}", results.size() );
- return results;
- }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java
new file mode 100644
index 0000000..b74f433
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class CollectionRefsVerifier extends VersionVerifier {
+
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+ List<EntityRef> refs = new ArrayList<EntityRef>(ids.size());
+ for ( Id id : ids ) {
+ refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+ }
+ return Results.fromRefList( refs );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..b79700b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.query.Query;
+
+
+/**
+ * Factory for creating results
+ */
+public class CollectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+ private final ManagerCache managerCache;
+
+
+ public CollectionResultsLoaderFactoryImpl( final ManagerCache managerCache ) {
+ this.managerCache = managerCache;
+ }
+
+
+ @Override
+ public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope scope, final Query.Level resultsLevel ) {
+
+ ResultsVerifier verifier;
+
+ if ( resultsLevel == Query.Level.REFS ) {
+ verifier = new CollectionRefsVerifier();
+ }
+ else if ( resultsLevel == Query.Level.IDS ) {
+// verifier = new RefsVerifier();
+ verifier = new IdsVerifier();
+ }
+ else {
+ verifier = new EntityVerifier(Query.MAX_LIMIT);
+ }
+
+ return new FilteringLoader( managerCache, verifier, applicationScope, scope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java
new file mode 100644
index 0000000..408edd3
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.ConnectedEntityRef;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.ConnectedEntityRefImpl;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+
+
+/**
+ * Verifier for creating connections
+ */
+public class ConnectionRefsVerifier extends VersionVerifier {
+
+
+ private final EntityRef ownerId;
+ private final String connectionType;
+
+
+ public ConnectionRefsVerifier( final EntityRef ownerId, final String connectionType ) {
+ this.ownerId = ownerId;
+ this.connectionType = connectionType;
+ }
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+ List<ConnectionRef> refs = new ArrayList<>();
+ for ( Id id : ids ) {
+ refs.add( new ConnectionRefImpl( ownerId, connectionType, ref(id.getType(), id.getUuid()) ));
+ }
+
+ return Results.fromConnections( refs );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..ba8eb2c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.query.Query;
+
+
+/**
+ * Factory for creating results
+ */
+public class ConnectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+ private final ManagerCache managerCache;
+ private final EntityRef ownerId;
+ private final String connectionType;
+
+
+ public ConnectionResultsLoaderFactoryImpl( final ManagerCache managerCache, final EntityRef ownerId,
+ final String connectionType ) {
+ this.managerCache = managerCache;
+ this.ownerId = ownerId;
+ this.connectionType = connectionType;
+ }
+
+
+ @Override
+ public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope scope, final Query.Level resultsLevel ) {
+
+ ResultsVerifier verifier;
+
+ if ( resultsLevel == Query.Level.REFS ) {
+ verifier = new ConnectionRefsVerifier( ownerId, connectionType );
+ }
+ else if ( resultsLevel == Query.Level.IDS ) {
+ verifier = new ConnectionRefsVerifier( ownerId, connectionType );;
+ }
+ else {
+ verifier = new EntityVerifier(Query.MAX_LIMIT);
+ }
+
+ return new FilteringLoader( managerCache, verifier, applicationScope, scope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
new file mode 100644
index 0000000..535af36
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+
+
+public class ElasticSearchQueryExecutor implements QueryExecutor {
+
+ private static final Logger logger = LoggerFactory.getLogger( ElasticSearchQueryExecutor.class );
+
+ private final ResultsLoaderFactory resultsLoaderFactory;
+
+ private final ApplicationScope applicationScope;
+
+ private final EntityIndex entityIndex;
+
+ private final IndexScope indexScope;
+
+ private final SearchTypes types;
+
+ private final Query query;
+
+
+ private Results currentResults;
+
+ private boolean moreToLoad = true;
+
+
+ public ElasticSearchQueryExecutor( final ResultsLoaderFactory resultsLoaderFactory, final EntityIndex entityIndex,
+ final ApplicationScope applicationScope, final IndexScope indexScope,
+ final SearchTypes types, final Query query ) {
+ this.resultsLoaderFactory = resultsLoaderFactory;
+ this.applicationScope = applicationScope;
+ this.entityIndex = entityIndex;
+ this.indexScope = indexScope;
+ this.types = types;
+
+ //we must deep copy the query passed. Otherwise we will modify it's state with cursors. Won't fix, not relevant
+ //once we start subscribing to streams.
+ this.query = new Query(query);
+ }
+
+
+ @Override
+ public Iterator<Results> iterator() {
+ return this;
+ }
+
+
+ private void loadNextPage() {
+ // Because of possible stale entities, which are filtered out by buildResults(),
+ // we loop until the we've got enough results to satisfy the query limit.
+
+ final int maxQueries = 10; // max re-queries to satisfy query limit
+
+ final int originalLimit = query.getLimit();
+
+ Results results = null;
+ int queryCount = 0;
+
+ boolean satisfied = false;
+
+
+ while ( !satisfied && queryCount++ < maxQueries ) {
+
+ CandidateResults crs = entityIndex.search( indexScope, types, query );
+
+ logger.debug( "Calling build results 1" );
+ results = buildResults( indexScope, query, crs );
+
+ if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get more
+ satisfied = true;
+ }
+
+ /**
+ * In an edge case where we delete stale entities, we could potentially get less results than expected.
+ * This will only occur once during the repair phase.
+ * We need to ensure that we short circuit before we overflow the requested limit during a repair.
+ */
+ else if ( results.size() > 0 ) { // got what we need
+ satisfied = true;
+ }
+ //we didn't load anything, but there was a cursor, this means a read repair occured. We have to short
+ //circuit to avoid over returning the result set
+ else if ( crs.hasCursor() ) {
+ satisfied = false;
+
+ // need to query for more
+ // ask for just what we need to satisfy, don't want to exceed limit
+ query.setCursor( results.getCursor() );
+ query.setLimit( originalLimit - results.size() );
+
+ logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
+ originalLimit, query.getLimit(), queryCount
+ } );
+ }
+ }
+
+ //now set our cursor if we have one for the next iteration
+ if ( results.hasCursor() ) {
+ query.setCursor( results.getCursor() );
+ moreToLoad = true;
+ }
+
+ else {
+ moreToLoad = false;
+ }
+
+
+ //set our current results and the flag
+ this.currentResults = results;
+ }
+
+
+ /**
+ * Build results from a set of candidates, and discard those that represent stale indexes.
+ *
+ * @param query Query that was executed
+ * @param crs Candidates to be considered for results
+ */
+ private Results buildResults( final IndexScope indexScope, final Query query, final CandidateResults crs ) {
+
+ logger.debug( "buildResults() from {} candidates", crs.size() );
+
+ //get an instance of our results loader
+ final ResultsLoader resultsLoader =
+ this.resultsLoaderFactory.getLoader( applicationScope, indexScope, query.getResultsLevel() );
+
+ //load the results
+ final Results results = resultsLoader.loadResults( crs );
+
+ //signal for post processing
+ resultsLoader.postProcess();
+
+
+ results.setCursor( crs.getCursor() );
+
+ //ugly and tight coupling, but we don't have a choice until we finish some refactoring
+ results.setQueryExecutor( this );
+
+ logger.debug( "Returning results size {}", results.size() );
+
+ return results;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+
+ //we've tried to load and it's empty and we have more to load, load the next page
+ if ( currentResults == null ) {
+ //there's nothing left to load, nothing to do
+ if ( !moreToLoad ) {
+ return false;
+ }
+
+ //load the page
+
+ loadNextPage();
+ }
+
+
+ //see if our current results are not null
+ return currentResults != null;
+ }
+
+
+ @Override
+ public Results next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more results present" );
+ }
+
+ final Results toReturn = currentResults;
+
+ currentResults = null;
+
+ return toReturn;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java
new file mode 100644
index 0000000..3afb77f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.Iterator;
+
+import org.apache.usergrid.persistence.Results;
+
+
+/**
+ * A strategy interface for executing queries. Each item in the iterator is a single collection of results
+ * Each implementation should always return 1 element of Results, even if the results are empty.
+ *
+ * QueryExecutor.next() should always return a non-null Results object
+ */
+public interface QueryExecutor extends Iterable<Results>, Iterator<Results> {
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java
deleted file mode 100644
index 096c271..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public class RefsVerifier extends VersionVerifier {
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
- List<EntityRef> refs = new ArrayList<EntityRef>();
- for ( Id id : ids ) {
- refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
- }
- return Results.fromRefList( refs );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
index 14fbf88..3fbfff9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
@@ -37,5 +37,6 @@ public interface ResultsLoaderFactory {
* @param indexScope The index scope used in the search
* @param
*/
- public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope indexScope, final Query.Level resultsLevel );
+ ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope indexScope,
+ final Query.Level resultsLevel );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
deleted file mode 100644
index 892b736..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.query.Query;
-
-import com.google.inject.Inject;
-
-
-/**
- * Factory for creating results
- */
-public class ResultsLoaderFactoryImpl implements ResultsLoaderFactory {
-
- private final ManagerCache managerCache;
-
-
- @Inject
- public ResultsLoaderFactoryImpl( final ManagerCache managerCache ) {
- this.managerCache = managerCache;
- }
-
-
- @Override
- public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope scope, final Query.Level resultsLevel ) {
-
- ResultsVerifier verifier;
-
- if ( resultsLevel == Query.Level.REFS ) {
- verifier = new RefsVerifier();
- }
- else if ( resultsLevel == Query.Level.IDS ) {
- verifier = new RefsVerifier();
- }
- else {
- verifier = new EntityVerifier(Query.MAX_LIMIT);
- }
-
- return new FilteringLoader( managerCache, verifier, applicationScope, scope );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
index 5b64d0b..52e235c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
@@ -24,7 +24,7 @@ import org.apache.usergrid.persistence.index.query.Query.Level;
/**
- * For each in a set of source refs executes a sub-query and provides a unified iterator over
+ * For each in a set of source refs executes a sub-query and provides a unified iterator over
* the union of all results. Honors page sizes for the Query to ensure memory isn't blown out.
*/
public class MultiQueryIterator implements ResultsIterator {
@@ -36,10 +36,6 @@ public class MultiQueryIterator implements ResultsIterator {
private Iterator currentIterator;
- public MultiQueryIterator( Results results, Query query ) {
- this( results.getQueryProcessor().getEntityManager(),
- new PagingResultsIterator( results, Level.IDS ), query );
- }
public MultiQueryIterator( EntityManager entityManager, Iterator<EntityRef> source, Query query ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
index 388b29e..f179000 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
@@ -30,6 +30,7 @@ import java.util.UUID;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.usergrid.corepersistence.results.QueryExecutor;
import org.apache.usergrid.persistence.cassandra.QueryProcessor;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.Level;
@@ -81,9 +82,7 @@ public class Results implements Iterable<Entity> {
Query query;
Object data;
String dataName;
-
- private QueryProcessor queryProcessor;
- private SearchVisitor searchVisitor;
+ private QueryExecutor queryExecutor;
public Results() {
@@ -1268,31 +1267,18 @@ public class Results implements Iterable<Entity> {
}
- protected QueryProcessor getQueryProcessor() {
- return queryProcessor;
- }
-
-
- public void setQueryProcessor( QueryProcessor queryProcessor ) {
- this.queryProcessor = queryProcessor;
- }
-
-
- public void setSearchVisitor( SearchVisitor searchVisitor ) {
- this.searchVisitor = searchVisitor;
+ public void setQueryExecutor(final QueryExecutor queryExecutor){
+ this.queryExecutor = queryExecutor;
}
/** uses cursor to get next batch of Results (returns null if no cursor) */
public Results getNextPageResults() throws Exception {
- if ( !hasCursor() ) {
+ if ( queryExecutor == null || !queryExecutor.hasNext() ) {
return null;
}
- Query q = new Query( query );
- q.setCursor( getCursor() );
- queryProcessor.setQuery( q );
- return queryProcessor.getResults( searchVisitor );
+ return queryExecutor.next();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
index 874ff88..94569d5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
@@ -292,7 +292,7 @@ public class QueryProcessorImpl implements QueryProcessor {
}
}
if (logger.isDebugEnabled()) {
- logger.debug("Getting result for query: [{}], returning entityIds size: {}",
+ logger.debug("Getting result for query: [{}], returning entityIds size: {}",
getQuery(), entityIds.size());
}
@@ -307,8 +307,8 @@ public class QueryProcessorImpl implements QueryProcessor {
results.setCursor( resultsCursor.asString() );
results.setQuery( query );
- results.setQueryProcessor( this );
- results.setSearchVisitor( visitor );
+// results.setQueryProcessor( this );
+// results.setSearchVisitor( visitor );
return results;
}
@@ -621,12 +621,12 @@ public class QueryProcessorImpl implements QueryProcessor {
@Override
public QueryBuilder getQueryBuilder() {
- throw new UnsupportedOperationException("Not supported by this vistor implementation.");
+ throw new UnsupportedOperationException("Not supported by this vistor implementation.");
}
@Override
public FilterBuilder getFilterBuilder() {
- throw new UnsupportedOperationException("Not supported by this vistor implementation.");
+ throw new UnsupportedOperationException("Not supported by this vistor implementation.");
}
}
@@ -724,4 +724,4 @@ public class QueryProcessorImpl implements QueryProcessor {
public EntityManager getEntityManager() {
return em;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
index c56c07f..6ab672a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
@@ -27,8 +27,6 @@ import java.util.UUID;
import org.junit.Test;
-import org.apache.commons.lang3.RandomStringUtils;
-
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.Level;
@@ -181,15 +179,45 @@ public class PathQueryIT extends AbstractCoreIT {
deviceQuery.addFilter( "index >= 4" );
int expectedDeviceQuerySize = 3;
- PathQuery groupsPQ = new PathQuery(new SimpleEntityRef( em.getApplicationRef() ), groupQuery );
- PathQuery usersPQ = groupsPQ.chain( userQuery );
- PathQuery<Entity> devicesPQ = usersPQ.chain( deviceQuery );
- HashSet set = new HashSet( expectedGroupQuerySize * expectedUserQuerySize * expectedDeviceQuerySize );
- Iterator<Entity> i = devicesPQ.iterator( em );
+ final PathQuery groupsPQ = new PathQuery(new SimpleEntityRef( em.getApplicationRef() ), groupQuery );
+
+
+ //test 1 level deep
+ HashSet groupSet = new HashSet( expectedGroupQuerySize );
+ Iterator<Entity> groupsIterator = groupsPQ.iterator( em );
+
+ while ( groupsIterator.hasNext() ) {
+ groupSet.add( groupsIterator.next() );
+ }
+
+ assertEquals( expectedGroupQuerySize, groupSet.size() );
+
+ //test 2 levels
+
+ final PathQuery groupsPQ1 = new PathQuery(new SimpleEntityRef( em.getApplicationRef() ), groupQuery );
+ PathQuery usersPQ1 = groupsPQ1.chain( userQuery );
+ final Iterator<Entity> userIterator = usersPQ1.iterator( em );
+
+ final HashSet userSet = new HashSet( expectedGroupQuerySize * expectedUserQuerySize );
+
+ while ( userIterator.hasNext() ) {
+ userSet.add( userIterator.next() );
+ }
+
+ assertEquals( expectedGroupQuerySize * expectedUserQuerySize, userSet.size() );
+
+
+// ORIGINAL TEST, restore
+ PathQuery groupsPQ2 = new PathQuery(new SimpleEntityRef( em.getApplicationRef() ), groupQuery );
+ PathQuery usersPQ2 = groupsPQ2.chain( userQuery );
+ PathQuery<Entity> devicesPQ2 = usersPQ2.chain( deviceQuery );
+
+ final HashSet deviceSet = new HashSet( expectedGroupQuerySize * expectedUserQuerySize * expectedDeviceQuerySize );
+ Iterator<Entity> i = devicesPQ2.iterator( em );
while ( i.hasNext() ) {
- set.add( i.next() );
+ deviceSet.add( i.next() );
}
- assertEquals( expectedGroupQuerySize * expectedUserQuerySize * expectedDeviceQuerySize, set.size() );
+ assertEquals( expectedGroupQuerySize * expectedUserQuerySize * expectedDeviceQuerySize, deviceSet.size() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 2b156b3..0ba16ea 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -44,6 +44,8 @@ log4j.logger.org.apache.usergrid.persistence.PerformanceEntityRebuildIndexTest=D
#log4j.logger.org.apache.usergrid.persistence=INFO
log4j.logger.org.apache.usergrid.corepersistence.migration=WARN
+
+log4j.logger.org.apache.usergrid.persistence.index.impl=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
#log4j.logger.org.apache.usergrid.corepersistence=DEBUG
#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManagerFactory=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index 69e0422..e129209 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -963,16 +963,19 @@ public abstract class GraphManagerIT {
Id targetId2 = new SimpleId( "target2" );
- Edge edge1 = createEdge( sourceId, "test", targetId1, System.currentTimeMillis() );
+ final long edge1Time = System.currentTimeMillis();
+ final long edge2Time = edge1Time+1;
+
+ Edge edge1 = createEdge( sourceId, "test", targetId1, edge1Time );
gm.writeEdge( edge1 ).toBlocking().singleOrDefault( null );
- Edge edge2 = createEdge( sourceId, "test", targetId2, System.currentTimeMillis() );
+ Edge edge2 = createEdge( sourceId, "test", targetId2, edge2Time );
gm.writeEdge( edge2 ).toBlocking().singleOrDefault( null );
- final long maxVersion = System.currentTimeMillis();
+ final long maxVersion = edge2Time+1;
assertTrue( Long.compare( maxVersion, edge2.getTimestamp() ) > 0 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 6be8234..c7be79d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -24,8 +24,6 @@ import org.safehaus.guicyfig.FigSingleton;
import org.safehaus.guicyfig.GuicyFig;
import org.safehaus.guicyfig.Key;
-import org.apache.usergrid.persistence.index.guice.QueueProvider;
-
@FigSingleton
public interface IndexFig extends GuicyFig {
@@ -93,6 +91,13 @@ public interface IndexFig extends GuicyFig {
*/
public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
+
+
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@Default( "127.0.0.1" )
@@ -115,7 +120,7 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_ALIAS_POSTFIX )
String getAliasPostfix();
- @Default( "1" ) // TODO: does this timeout get extended on each query?
+ @Default( "5" ) // TODO: does this timeout get extended on each query?
@Key( QUERY_CURSOR_TIMEOUT_MINUTES )
int getQueryCursorTimeout();
@@ -193,7 +198,7 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_QUEUE_READ_TIMEOUT )
int getIndexQueueTimeout();
- @Default("2")
+ @Default( "2" )
@Key( ELASTICSEARCH_WORKER_COUNT )
int getWorkerCount();
@@ -201,4 +206,7 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_QUEUE_IMPL )
String getQueueImplementation();
+ @Default( "1000" )
+ @Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
+ long getQueueOfferTimeout();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
index 998c086..b6eaf89 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
@@ -36,18 +36,26 @@ import com.google.inject.Singleton;
@Singleton
public class BufferQueueInMemoryImpl implements BufferQueue {
+
+ private final IndexFig fig;
private final ArrayBlockingQueue<IndexOperationMessage> messages;
@Inject
public BufferQueueInMemoryImpl( final IndexFig fig ) {
+ this.fig = fig;
messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
}
@Override
public void offer( final IndexOperationMessage operation ) {
- messages.offer( operation );
+ try {
+ messages.offer( operation, fig.getQueueOfferTimeout(), TimeUnit.MILLISECONDS );
+ }
+ catch ( InterruptedException e ) {
+ throw new RuntimeException("Unable to offer message to queue", e);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index b31cf39..7e64de3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -19,18 +19,12 @@
*/
package org.apache.usergrid.persistence.index.impl;
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.IndexBufferConsumer;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexOperationMessage;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
import org.elasticsearch.action.WriteConsistencyLevel;
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -39,6 +33,19 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.index.IndexBufferConsumer;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexOperationMessage;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
import rx.Observable;
import rx.Subscriber;
import rx.Subscription;
@@ -47,11 +54,6 @@ import rx.functions.Func1;
import rx.functions.Func2;
import rx.schedulers.Schedulers;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
* Consumer for IndexOperationMessages
@@ -189,7 +191,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
while ( true );
}
- } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
+ } ).doOnNext( new Action1<List<IndexOperationMessage>>() {
@Override
public void call( List<IndexOperationMessage> containerList ) {
if ( containerList.size() == 0 ) {
@@ -203,7 +205,6 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
execute( containerList );
time.stop();
-
}
} )
//ack after we process
@@ -214,15 +215,29 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
//release so we know we've done processing
inFlight.addAndGet( -1 * indexOperationMessages.size() );
}
- } ).doOnError( new Action1<Throwable>() {
+ } )
+ //catch an unexpected error, then emit an empty list to ensure our subscriber doesn't die
+ .onErrorReturn( new Func1<Throwable, List<IndexOperationMessage>>() {
@Override
- public void call( final Throwable throwable ) {
+ public List<IndexOperationMessage> call( final Throwable throwable ) {
+ final long sleepTime = config.getFailureRetryTime();
+
+ log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, throwable );
+
+ try {
+ Thread.sleep( sleepTime );
+ }
+ catch ( InterruptedException ie ) {
+ //swallow
+ }
- log.error( "An exception occurred when trying to deque and write to elasticsearch. Ignoring",
- throwable );
indexErrorCounter.inc();
+
+ return Collections.EMPTY_LIST;
}
- } );
+ } )
+
+ .subscribeOn( Schedulers.newThread() );
//start in the background
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index da68772..9a7a867 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -113,33 +113,41 @@ public class Query {
}
+ /**
+ * Creates a deep copy of a query from another query
+ * @param q
+ */
public Query( Query q ) {
- if ( q != null ) {
- type = q.type;
- sortPredicates = q.sortPredicates != null
- ? new ArrayList<SortPredicate>( q.sortPredicates ) : null;
- startResult = q.startResult;
- cursor = q.cursor;
- limit = q.limit;
- selectAssignments = q.selectAssignments != null
- ? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
- mergeSelectResults = q.mergeSelectResults;
- //level = q.level;
- connectionType = q.connectionType;
- permissions = q.permissions != null ? new ArrayList<String>( q.permissions ) : null;
- reversed = q.reversed;
- reversedSet = q.reversedSet;
- startTime = q.startTime;
- finishTime = q.finishTime;
- resolution = q.resolution;
- pad = q.pad;
- rootOperand = q.rootOperand;
- identifiers = q.identifiers != null
- ? new ArrayList<Identifier>( q.identifiers ) : null;
- counterFilters = q.counterFilters != null
- ? new ArrayList<CounterFilterPredicate>( q.counterFilters ) : null;
- collection = q.collection;
+ if ( q == null ) {
+ return;
}
+
+ type = q.type;
+ sortPredicates = q.sortPredicates != null
+ ? new ArrayList<>( q.sortPredicates ) : null;
+ startResult = q.startResult;
+ cursor = q.cursor;
+ limit = q.limit;
+ selectAssignments = q.selectAssignments != null
+ ? new LinkedHashMap<>( q.selectAssignments ) : null;
+ mergeSelectResults = q.mergeSelectResults;
+ //level = q.level;
+ connectionType = q.connectionType;
+ permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null;
+ reversed = q.reversed;
+ reversedSet = q.reversedSet;
+ startTime = q.startTime;
+ finishTime = q.finishTime;
+ resolution = q.resolution;
+ pad = q.pad;
+ rootOperand = q.rootOperand;
+ identifiers = q.identifiers != null
+ ? new ArrayList<>( q.identifiers ) : null;
+ counterFilters = q.counterFilters != null
+ ? new ArrayList<>( q.counterFilters ) : null;
+ collection = q.collection;
+ level = q.level;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
index 6c7b480..9c768bf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
@@ -23,8 +23,6 @@ import java.util.List;
import java.util.UUID;
import org.apache.commons.lang.math.NumberUtils;
-import org.apache.usergrid.persistence.collection.util.EntityUtils;
-import org.apache.usergrid.persistence.model.entity.Id;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c7fa864f/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index ee3f9cd..aea9d49 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -121,7 +121,7 @@
<!-- only use half the cores on the machine for testing -->
<usergrid.it.parallel>methods</usergrid.it.parallel>
<usergrid.it.reuseForks>true</usergrid.it.reuseForks>
- <usergrid.it.forkCount>4</usergrid.it.forkCount>
+ <usergrid.it.forkCount>1</usergrid.it.forkCount>
<usergrid.it.threads>8</usergrid.it.threads>
<metrics.version>3.0.0</metrics.version>
[39/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-dev
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a099d0fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a099d0fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a099d0fc
Branch: refs/heads/USERGRID-473
Commit: a099d0fcd198b83c07551324d2516e068a20b159
Parents: 6db7ce9 2d1c8b8
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Mar 30 09:58:07 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Mar 30 09:58:07 2015 -0400
----------------------------------------------------------------------
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++++++-------------
1 file changed, 14 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[36/50] incubator-usergrid git commit: fix indexcacheimpl
Posted by sf...@apache.org.
fix indexcacheimpl
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6a39add9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6a39add9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6a39add9
Branch: refs/heads/USERGRID-473
Commit: 6a39add920a052bf629c86df3d30615f59e0c22d
Parents: b98cdce
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 16:04:00 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 16:04:00 2015 -0600
----------------------------------------------------------------------
.../serialization/impl/migration/EdgeDataMigrationImpl.java | 2 ++
.../usergrid/persistence/index/impl/IndexMigrationTest.java | 8 +-------
2 files changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a39add9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index ecb9a9b..411e318 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -88,6 +88,8 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap( graphNode -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
+ //TODO: maybe move to retreive all indexes : https://github.com/elastic/elasticsearch/blob/master/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+ //https://www.google.com/webhp?sourceid=chrome-instant&ion=1&espv=2&ie=UTF-8#q=get%20all%20indexes%20elasticsearch
//get edges from the source
return edgesFromSourceObservable.edgesFromSource( gm, graphNode.entryNode ).buffer( 1000 )
.doOnNext( edges -> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a39add9/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
index 47d81ed..1203e38 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
@@ -60,9 +60,6 @@ public class IndexMigrationTest extends BaseIT{
public EsProvider provider;
@Inject
- public MigrationDataProvider<ApplicationScope> applicationScopeMigrationDataProvider;
-
- @Inject
public AliasedEntityIndex ei;
@Inject
public IndexBufferProducer indexBatchBufferProducer;
@@ -98,12 +95,9 @@ public class IndexMigrationTest extends BaseIT{
}
};
- final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
-
- LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(fig,appScope);
TestIndexModule.TestAllApplicationsObservable obs = new TestIndexModule.TestAllApplicationsObservable(indexBatchBufferProducer,provider,indexCache,metricsFactory,fig);
- int version = indexDataMigration.migrate(0, applicationScopeMigrationDataProvider, po );
+ int version = indexDataMigration.migrate(0, obs, po );
assertEquals(version, IndexDataVersions.SINGLE_INDEX.getVersion());
}
}
[26/50] incubator-usergrid git commit: merge from two-o-dev
Posted by sf...@apache.org.
merge from two-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/208ed2b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/208ed2b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/208ed2b3
Branch: refs/heads/USERGRID-473
Commit: 208ed2b3c93771faf51a41903d617f67cc32b14a
Parents: f695724 6db7ce9
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 10:20:30 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 10:20:30 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 304 ++++++-------------
.../results/CollectionRefsVerifier.java | 44 +++
.../CollectionResultsLoaderFactoryImpl.java | 60 ++++
.../results/ConnectionRefsVerifier.java | 61 ++++
.../ConnectionResultsLoaderFactoryImpl.java | 65 ++++
.../results/ElasticSearchQueryExecutor.java | 212 +++++++++++++
.../corepersistence/results/QueryExecutor.java | 37 +++
.../corepersistence/results/RefsVerifier.java | 42 ---
.../results/ResultsLoaderFactory.java | 3 +-
.../results/ResultsLoaderFactoryImpl.java | 62 ----
.../persistence/MultiQueryIterator.java | 6 +-
.../apache/usergrid/persistence/Results.java | 26 +-
.../cassandra/QueryProcessorImpl.java | 12 +-
.../usergrid/persistence/PathQueryIT.java | 46 ++-
stack/core/src/test/resources/log4j.properties | 2 +
.../persistence/graph/GraphManagerIT.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 16 +-
.../index/impl/BufferQueueInMemoryImpl.java | 10 +-
.../index/impl/EsEntityIndexBatchImpl.java | 2 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 61 ++--
.../usergrid/persistence/index/query/Query.java | 55 ++--
.../persistence/index/utils/ListUtils.java | 3 +-
.../cassandra/ManagementServiceImpl.java | 18 +-
.../providers/PingIdentityProvider.java | 5 +-
24 files changed, 730 insertions(+), 424 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/208ed2b3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 8f8ac5e,90b8e35..f62648d
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -40,7 -65,7 +40,7 @@@ public class EsEntityIndexBatchImpl imp
private final ApplicationScope applicationScope;
-- private final IndexIdentifier.IndexAlias alias;
++ private final IndexAlias alias;
private final IndexIdentifier indexIdentifier;
private final IndexBufferProducer indexBatchBufferProducer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/208ed2b3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 6454b48,6362c0a..fa7def4
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@@ -106,31 -117,86 +106,39 @@@ public class Query
}
+ /**
+ * Creates a deep copy of a query from another query
+ * @param q
+ */
public Query( Query q ) {
- if ( q != null ) {
- type = q.type;
- sortPredicates = q.sortPredicates != null
- ? new ArrayList<SortPredicate>( q.sortPredicates ) : null;
- startResult = q.startResult;
- selectAssignments = q.selectAssignments != null
- ? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
- mergeSelectResults = q.mergeSelectResults;
- //level = q.level;
- connectionType = q.connectionType;
- permissions = q.permissions != null ? new ArrayList<String>( q.permissions ) : null;
- reversed = q.reversed;
- reversedSet = q.reversedSet;
- startTime = q.startTime;
- finishTime = q.finishTime;
- resolution = q.resolution;
- pad = q.pad;
- rootOperand = q.rootOperand;
- identifiers = q.identifiers != null
- ? new ArrayList<Identifier>( q.identifiers ) : null;
- counterFilters = q.counterFilters != null
- ? new ArrayList<CounterFilterPredicate>( q.counterFilters ) : null;
- collection = q.collection;
- if ( q == null ) {
++ if (q == null) {
+ return;
}
-
+ type = q.type;
+ sortPredicates = q.sortPredicates != null
- ? new ArrayList<>( q.sortPredicates ) : null;
++ ? new ArrayList<SortPredicate>(q.sortPredicates) : null;
+ startResult = q.startResult;
- cursor = q.cursor;
- limit = q.limit;
+ selectAssignments = q.selectAssignments != null
- ? new LinkedHashMap<>( q.selectAssignments ) : null;
++ ? new LinkedHashMap<String, String>(q.selectAssignments) : null;
+ mergeSelectResults = q.mergeSelectResults;
+ //level = q.level;
+ connectionType = q.connectionType;
- permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null;
++ permissions = q.permissions != null ? new ArrayList<String>(q.permissions) : null;
+ reversed = q.reversed;
+ reversedSet = q.reversedSet;
+ startTime = q.startTime;
+ finishTime = q.finishTime;
+ resolution = q.resolution;
+ pad = q.pad;
+ rootOperand = q.rootOperand;
+ identifiers = q.identifiers != null
- ? new ArrayList<>( q.identifiers ) : null;
++ ? new ArrayList<Identifier>(q.identifiers) : null;
+ counterFilters = q.counterFilters != null
- ? new ArrayList<>( q.counterFilters ) : null;
++ ? new ArrayList<CounterFilterPredicate>(q.counterFilters) : null;
+ collection = q.collection;
- level = q.level;
-
- }
-
-
- public QueryBuilder createQueryBuilder( final String context ) {
-
-
- QueryBuilder queryBuilder = null;
-
-
- //we have a root operand. Translate our AST into an ES search
- if ( getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
-
- try {
- getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
-
-
- queryBuilder = v.getQueryBuilder();
- }
-
-
- // Add our filter for context to our query for fast execution.
- // Fast because it utilizes bitsets internally. See this post for more detail.
- // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
-
- // TODO evaluate performance when it's an all query.
- // Do we need to put the context term first for performance?
- if ( queryBuilder != null ) {
- queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
- .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
- }
+
- //nothing was specified ensure we specify the context in the search
- else {
- queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
- }
++ level = q.level;
+
- return queryBuilder;
}
@@@ -235,7 -317,8 +243,6 @@@
Boolean reversed = ListUtils.firstBoolean( params.get( "reversed" ) );
String connection = ListUtils.first( params.get( "connectionType" ) );
UUID start = ListUtils.firstUuid( params.get( "start" ) );
-- String cursor = ListUtils.first( params.get( "cursor" ) );
- Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
[47/50] incubator-usergrid git commit: separate calls
Posted by sf...@apache.org.
separate calls
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bec7f44e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bec7f44e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bec7f44e
Branch: refs/heads/USERGRID-473
Commit: bec7f44e79851b443b1bb3c547f21705b540f6f6
Parents: 217e7dc
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 15:48:41 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 15:48:41 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 43 ++++++++++++--------
1 file changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bec7f44e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 6b3a11d..993b529 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -50,6 +50,7 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.indices.InvalidAliasNameException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -207,36 +208,46 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
int count = 0;
IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
- for ( String currentIndex : indexNames ) {
- aliasesRequestBuilder.removeAlias( currentIndex, alias.getWriteAlias() );
+ for (String currentIndex : indexNames) {
+ aliasesRequestBuilder.removeAlias(currentIndex, alias.getWriteAlias());
count++;
}
- if(count>0) {
+ if (count > 0) {
isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
logger.info("Removed Index Name from Alias=[{}] ACK=[{}]", alias, isAck);
}
- aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
+ String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
- //Added For Graphite Metrics
- // add read alias
- aliasesRequestBuilder.addAlias(
- indexName, alias.getReadAlias());
+ try {
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ //Added For Graphite Metrics
+ // add read alias
+ aliasesRequestBuilder.addAlias(indexName, alias.getReadAlias());
+ isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
+ logger.info("Created new read aliases ACK=[{}]", isAck);
+ } catch (InvalidAliasNameException ie) {
+ logger.debug("error creating read", ie);
+ }
+ try {
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
- //Added For Graphite Metrics
- //add write alias
- aliasesRequestBuilder.addAlias(
- indexName, alias.getWriteAlias());
+ //Added For Graphite Metrics
+ //add write alias
+ aliasesRequestBuilder.addAlias(indexName, alias.getWriteAlias());
+ isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
+ logger.info("Created new write aliases ACK=[{}]", isAck);
- isAck = aliasesRequestBuilder.execute().actionGet().isAcknowledged();
+ } catch (InvalidAliasNameException ie) {
+ logger.debug("error creating write", ie);
+ }
- logger.info("Created new aliases ACK=[{}]", isAck);
aliasCache.invalidate(alias);
} catch (Exception e) {
logger.warn("Failed to create alias ", e);
- }
- finally {
+ } finally {
timer.stop();
}
}
[34/50] incubator-usergrid git commit: fix indexcacheimpl
Posted by sf...@apache.org.
fix indexcacheimpl
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a7e08db8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a7e08db8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a7e08db8
Branch: refs/heads/USERGRID-473
Commit: a7e08db85babdb1f4c75c38cee67ba6f03547e32
Parents: 9e8d506
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:17:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:17:13 2015 -0600
----------------------------------------------------------------------
.../persistence/index/EsIndexCache.java | 29 ++++
.../persistence/index/guice/IndexModule.java | 4 +-
.../persistence/index/impl/EsIndexCache.java | 143 -------------------
.../index/impl/EsIndexCacheImpl.java | 141 ++++++++++++++++++
.../migration/EsIndexDataMigrationImpl.java | 2 +-
.../index/guice/TestIndexModule.java | 2 +-
6 files changed, 175 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
new file mode 100644
index 0000000..6427459
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
@@ -0,0 +1,29 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index;
+
+/**
+ * Classy class class.
+ */
+public interface EsIndexCache {
+ String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
+
+ void invalidate(IndexAlias alias);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 922a916..bb5ec54 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -48,9 +48,11 @@ public abstract class IndexModule extends AbstractModule {
install(new MapModule());
install(new QueueModule());
- bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
+ bind(EntityIndexFactory.class).to(EsEntityIndexFactoryImpl.class);
bind(AliasedEntityIndex.class).to(EsEntityIndexImpl.class);
bind(EntityIndex.class).to(EsEntityIndexImpl.class);
+ bind(EsIndexCache.class).to(EsIndexCacheImpl.class);
+
bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
deleted file mode 100644
index 9ba89ca..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.index.IndexAlias;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.index.AliasedEntityIndex;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListenableFutureTask;
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-/**
- * Cache for Es index operations
- */
-@Singleton
-public class EsIndexCache {
-
- private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
- private final ListeningScheduledExecutorService refreshExecutors;
-
- private LoadingCache<String, String[]> aliasIndexCache;
- private EsProvider provider;
-
-
- @Inject
- public EsIndexCache( final EsProvider provider, final IndexFig indexFig ) {
-
- this.refreshExecutors =
- MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool( indexFig.getIndexCacheMaxWorkers() ) );
-
- this.provider = provider;
-
- aliasIndexCache = CacheBuilder.newBuilder().maximumSize( 1000 ).refreshAfterWrite( 5, TimeUnit.MINUTES )
- .build( new CacheLoader<String, String[]>() {
- @Override
- public ListenableFuture<String[]> reload( final String key,
- String[] oldValue )
- throws Exception {
- ListenableFutureTask<String[]> task =
- ListenableFutureTask.create( new Callable<String[]>() {
- public String[] call() {
- return load( key );
- }
- } );
- refreshExecutors.execute( task );
- return task;
- }
-
-
- @Override
- public String[] load( final String aliasName ) {
- return getIndexesFromEs(aliasName);
- }
- } );
- }
-
-
- /**
- * Get indexes for an alias
- */
- public String[] getIndexes( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
- String[] indexes;
- try {
- indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
- }
- catch ( ExecutionException ee ) {
- logger.error( "Failed to retreive indexes", ee );
- throw new RuntimeException( ee );
- }
- return indexes;
- }
-
-
-
- private String[] getIndexesFromEs(final String aliasName){
- final AdminClient adminClient = this.provider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
- adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases();
- return aliasMap.keys().toArray( String.class );
- }
-
-
- /**
- * Get the name of the alias to use
- * @param alias
- * @param aliasType
- * @return
- */
- private String getAliasName( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
- return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
- }
-
-
- /**
- * clean up cache
- */
- public void invalidate( IndexAlias alias ) {
- aliasIndexCache.invalidate( alias.getWriteAlias() );
- aliasIndexCache.invalidate( alias.getReadAlias() );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
new file mode 100644
index 0000000..5f5cc8b
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
@@ -0,0 +1,141 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.persistence.index.*;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ListeningScheduledExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * Cache for Es index operations
+ */
+@Singleton
+public class EsIndexCacheImpl implements EsIndexCache {
+
+ private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
+ private final ListeningScheduledExecutorService refreshExecutors;
+
+ private LoadingCache<String, String[]> aliasIndexCache;
+ private EsProvider provider;
+
+
+ @Inject
+ public EsIndexCacheImpl( final EsProvider provider, final IndexFig indexFig ) {
+
+ this.refreshExecutors =
+ MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool( indexFig.getIndexCacheMaxWorkers() ) );
+
+ this.provider = provider;
+
+ aliasIndexCache = CacheBuilder.newBuilder().maximumSize( 1000 ).refreshAfterWrite( 5, TimeUnit.MINUTES )
+ .build( new CacheLoader<String, String[]>() {
+ @Override
+ public ListenableFuture<String[]> reload( final String key,
+ String[] oldValue )
+ throws Exception {
+ ListenableFutureTask<String[]> task =
+ ListenableFutureTask.create( new Callable<String[]>() {
+ public String[] call() {
+ return load( key );
+ }
+ } );
+ refreshExecutors.execute( task );
+ return task;
+ }
+
+
+ @Override
+ public String[] load( final String aliasName ) {
+ return getIndexesFromEs(aliasName);
+ }
+ } );
+ }
+
+
+ /**
+ * Get indexes for an alias
+ */
+ @Override
+ public String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType) {
+ String[] indexes;
+ try {
+ indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
+ }
+ catch ( ExecutionException ee ) {
+ logger.error( "Failed to retreive indexes", ee );
+ throw new RuntimeException( ee );
+ }
+ return indexes;
+ }
+
+
+
+ private String[] getIndexesFromEs(final String aliasName){
+ final AdminClient adminClient = this.provider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
+ adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases();
+ return aliasMap.keys().toArray( String.class );
+ }
+
+
+ /**
+ * Get the name of the alias to use
+ * @param alias
+ * @param aliasType
+ * @return
+ */
+ private String getAliasName( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
+ }
+
+
+ /**
+ * clean up cache
+ */
+ @Override
+ public void invalidate(IndexAlias alias) {
+ aliasIndexCache.invalidate( alias.getWriteAlias() );
+ aliasIndexCache.invalidate( alias.getReadAlias() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index d6e3812..fe0cfa1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -26,7 +26,7 @@ import org.apache.usergrid.persistence.index.AliasedEntityIndex;
import org.apache.usergrid.persistence.index.IndexAlias;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.EsIndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 6488b16..7a99d59 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -31,7 +31,7 @@ import org.apache.usergrid.persistence.index.IndexBufferProducer;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.EsIndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
[45/50] incubator-usergrid git commit: refactor initiaization index
logic
Posted by sf...@apache.org.
refactor initiaization index logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/07cb9dfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/07cb9dfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/07cb9dfd
Branch: refs/heads/USERGRID-473
Commit: 07cb9dfd6563d974d8c6b9764633be44e389a4d0
Parents: 3a31836
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 11:45:15 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 11:45:15 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 14 +--------
.../index/ApplicationEntityIndex.java | 5 ----
.../usergrid/persistence/index/EntityIndex.java | 10 +++----
.../impl/EsApplicationEntityIndexImpl.java | 12 --------
.../index/impl/EsEntityIndexImpl.java | 13 +++++++++
.../migration/EsIndexDataMigrationImpl.java | 4 ++-
.../persistence/index/impl/EntityIndexTest.java | 30 +++++++++++---------
.../index/impl/IndexLoadTestsIT.java | 4 ---
8 files changed, 38 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3cfc1a4..99204a8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -164,9 +164,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.getApplication();
}
- ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) );
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope);
- applicationEntityIndex.initializeIndex();
+ entityIndex.initialize();
entityIndex.refresh();
@@ -265,10 +263,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
throw new ApplicationAlreadyExistsException( appName );
}
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( applicationId,"application"));
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
-
getSetup().setupApplicationKeyspace( applicationId, appName );
final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
@@ -730,12 +724,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
EntityManager em = getEntityManager( appId );
- ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ));
- //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
- em.reindex(po);
-
em.reindex( po );
logger.info("\n\nRebuilt index for applicationId {} \n", appId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index cb1b6f0..86e97c5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -30,11 +30,6 @@ public interface ApplicationEntityIndex {
/**
- *
- */
- public void initializeIndex();
-
- /**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 6783fb0..ff9008e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -48,28 +48,28 @@ public interface EntityIndex extends CPManager {
* @param replicas
* @param writeConsistency
*/
- public void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency);
+ void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency);
/**
* Refresh the index.
*/
- public void refresh();
+ void refresh();
/**
* Check health of cluster.
*/
- public Health getClusterHealth();
+ Health getClusterHealth();
/**
* Check health of this specific index.
*/
- public Health getIndexHealth();
-
+ Health getIndexHealth();
+ void initialize();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 9248980..84845d0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -116,18 +116,6 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public void initializeIndex() {
- final int numberOfShards = indexFig.getNumberOfShards();
- final int numberOfReplicas = indexFig.getNumberOfReplicas();
- indexCache.invalidate(alias);
- String[] indexes = entityIndex.getUniqueIndexes();
- if(indexes == null || indexes.length==0) {
- entityIndex.addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
- }
-
- }
-
- @Override
public EntityIndexBatch createBatch() {
EntityIndexBatch batch = new EsEntityIndexBatchImpl(
applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 055af2a..ee28939 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -131,6 +131,19 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
}
@Override
+ public void initialize(){
+ final int numberOfShards = indexFig.getNumberOfShards();
+ final int numberOfReplicas = indexFig.getNumberOfReplicas();
+ aliasCache.invalidate(alias);
+ String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
+ String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
+
+ if(reads.length==0 || writes.length==0) {
+ addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
+ }
+ }
+
+ @Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
try {
//get index name with suffix attached
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 123527a..3706722 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -62,6 +62,8 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
final AdminClient adminClient = provider.getClient().admin();
final int latestVersion = dataVersion.getImplementationVersion();
+ entityIndex.initialize();
+
observer.start();
try {
migrationDataProvider.getData().flatMap(applicationScope -> {
@@ -74,7 +76,7 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
try {
- aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias()).get();
} catch (InvalidAliasNameException e) {
log.debug("Failed to add alias due to name conflict",e);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 2c03635..27f0b9d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -30,9 +30,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.index.*;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
+import org.junit.*;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,6 +85,10 @@ public class EntityIndexTest extends BaseIT {
@Rule
public MigrationManagerRule migrationManagerRule;
+ @Before
+ public void setup(){
+ ei.initialize();
+ }
@Test
public void testIndex() throws IOException, InterruptedException {
@@ -94,7 +96,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl(appId);
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl(appId, "things");
@@ -158,7 +160,7 @@ public class EntityIndexTest extends BaseIT {
final ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
final IndexScope indexScope = new IndexScopeImpl(appId, "things");
final String entityType = "thing";
- entityIndex.initializeIndex();
+
final CountDownLatch latch = new CountDownLatch(threads);
final AtomicLong failTime=new AtomicLong(0);
InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
@@ -201,9 +203,9 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
for(int i=0;i<10;i++) {
- entityIndex.initializeIndex();
+
}
}
@@ -215,7 +217,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
@@ -328,7 +330,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
Map entityMap = new HashMap() {{
put( "name", "Ferrari 212 Inter" );
@@ -460,7 +462,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String middleName = "middleName" + UUIDUtils.newTimeUUID();
@@ -509,7 +511,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
entityIndex.createBatch();
// Bill has favorites as string, age as string and retirement goal as number
@@ -581,9 +583,9 @@ public class EntityIndexTest extends BaseIT {
Id ownerId = new SimpleId( "multivaluedtype" );
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
- assertEquals( "index should be ready", Health.GREEN, ei.getIndexHealth() );
+ assertEquals("index should be ready", Health.GREEN, ei.getIndexHealth());
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
ei.refresh();
assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
@@ -603,7 +605,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final EntityIndexBatch batch = entityIndex.createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
index 0275e53..b03f09d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -135,10 +135,6 @@ public class IndexLoadTestsIT extends BaseIT {
final IndexScope indexScope = new IndexScopeImpl( applicationId, "test" );
- final ApplicationEntityIndex appEntityIndex = entityIndexFactory.createApplicationEntityIndex( scope );
-
- appEntityIndex.initializeIndex();
-
batchWriteTPS = metricsFactory.getMeter( IndexLoadTestsIT.class, "write.tps" );
batchWriteTimer = metricsFactory.getTimer( IndexLoadTestsIT.class, "write.timer" );
[14/50] incubator-usergrid git commit: Merge branch 'USERGRID-405'
into USERGRID-448-appinfofix
Posted by sf...@apache.org.
Merge branch 'USERGRID-405' into USERGRID-448-appinfofix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/efb7d6f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/efb7d6f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/efb7d6f5
Branch: refs/heads/USERGRID-473
Commit: efb7d6f553f24ede9fb8f27459e9d59e5156f430
Parents: f007f47 e4466e1
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Mar 24 14:40:02 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Mar 24 14:40:02 2015 -0400
----------------------------------------------------------------------
.../usergrid/security/providers/PingIdentityProvider.java | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
[30/50] incubator-usergrid git commit: fix cursor
Posted by sf...@apache.org.
fix cursor
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2df40137
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2df40137
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2df40137
Branch: refs/heads/USERGRID-473
Commit: 2df40137c9d96ec05f82605e77ebcaeba016a420
Parents: 96d27b9
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 14:23:04 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 14:23:04 2015 -0600
----------------------------------------------------------------------
.../impl/EsApplicationEntityIndexImpl.java | 11 +++++++---
.../migration/EsIndexDataMigrationImpl.java | 2 +-
.../index/guice/TestIndexModule.java | 22 ++++++++++++++++++++
3 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2df40137/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 603bee4..53286b4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -142,6 +142,10 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
@Override
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit){
+ if(query.getCursor()!=null){
+ return getNextPage(query.getCursor());
+ }
+
SearchResponse searchResponse;
SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query,limit);
@@ -291,11 +295,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
candidates.add(new CandidateResult(entityId, UUID.fromString(version)));
}
- CandidateResults candidateResults = new CandidateResults(candidates);
- if(candidateResults.hasCursor()) {
+ final CandidateResults candidateResults = new CandidateResults(candidates);
+ final String esScrollCursor = searchResponse.getScrollId();
+
+ if(esScrollCursor != null) {
candidateResults.initializeCursor();
- final String esScrollCursor = searchResponse.getScrollId();
//now set this into our map module
final int minutes = indexFig.getQueryCursorTimeout();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2df40137/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 2a0bb15..64b7d29 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -80,7 +80,7 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
.doOnError(error -> log.error("failed to migrate index",error))
.toBlocking().last();
- return integer.get();
+ return dataVersion.getImplementationVersion();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2df40137/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 4e6700c..0c57b36 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,10 +19,18 @@
package org.apache.usergrid.persistence.index.guice;
+import com.google.inject.TypeLiteral;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
+import rx.Observable;
+
+import java.util.UUID;
public class TestIndexModule extends TestModule {
@@ -37,8 +45,22 @@ public class TestIndexModule extends TestModule {
@Override
public void configureMigrationProvider(){
+ bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} ).to(
+ TestAllApplicationsObservable.class );
}
});
install( new GuicyFigModule(IndexTestFig.class) );
}
+ public static class TestAllApplicationsObservable implements MigrationDataProvider<ApplicationScope>{
+
+
+ @Override
+ public Observable<ApplicationScope> getData() {
+ ApplicationScope[] scopes = new ApplicationScope[]{
+ new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"))
+ };
+ return Observable.from(scopes);
+ }
+ }
+
}
[06/50] incubator-usergrid git commit: [USERGRID-338] Fixed logic
error in test.
Posted by sf...@apache.org.
[USERGRID-338] Fixed logic error in test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/26700985
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/26700985
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/26700985
Branch: refs/heads/USERGRID-473
Commit: 267009855563a793949826eb76f18f876881f9c2
Parents: d263d36
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 23 12:06:08 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 23 12:06:08 2015 -0700
----------------------------------------------------------------------
.../rest/applications/collection/users/OwnershipResourceIT.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/26700985/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java
index f898cd6..87f2ab7 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/OwnershipResourceIT.java
@@ -238,7 +238,7 @@ public class OwnershipResourceIT extends AbstractRestIT {
refreshIndex();
//Create a restaurant and link it to user1/me
- data = usersResource.entity("me")
+ Entity fourPeaksData = usersResource.entity("me")
.connection("likes").collection( "restaurants" ).entity( "4peaks" ).post();
refreshIndex();
@@ -268,7 +268,7 @@ public class OwnershipResourceIT extends AbstractRestIT {
.collection( "restaurants" );
//Check that we can get the 4peaks restaurant by using its uuid
- String peaksId = data.getUuid().toString();
+ String peaksId = fourPeaksData.getUuid().toString();
data = likeRestaurants.entity(peaksId).get();
assertNotNull( data );
assertEquals("4peaks", data.get("name").toString());
[23/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-dev
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6db7ce93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6db7ce93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6db7ce93
Branch: refs/heads/USERGRID-473
Commit: 6db7ce937b008b26f3f06e9a6fbe295b4b8d7dc3
Parents: 66fdc61 beb2a2a
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 26 09:31:07 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 26 09:31:31 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 304 ++++++-------------
.../results/CollectionRefsVerifier.java | 44 +++
.../CollectionResultsLoaderFactoryImpl.java | 60 ++++
.../results/ConnectionRefsVerifier.java | 61 ++++
.../ConnectionResultsLoaderFactoryImpl.java | 65 ++++
.../results/ElasticSearchQueryExecutor.java | 212 +++++++++++++
.../corepersistence/results/QueryExecutor.java | 37 +++
.../corepersistence/results/RefsVerifier.java | 42 ---
.../results/ResultsLoaderFactory.java | 3 +-
.../results/ResultsLoaderFactoryImpl.java | 62 ----
.../persistence/MultiQueryIterator.java | 6 +-
.../apache/usergrid/persistence/Results.java | 26 +-
.../cassandra/QueryProcessorImpl.java | 12 +-
.../usergrid/persistence/PathQueryIT.java | 46 ++-
stack/core/src/test/resources/log4j.properties | 2 +
.../persistence/graph/GraphManagerIT.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 16 +-
.../index/impl/BufferQueueInMemoryImpl.java | 10 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 61 ++--
.../usergrid/persistence/index/query/Query.java | 58 ++--
.../persistence/index/utils/ListUtils.java | 3 +-
.../cassandra/ManagementServiceImpl.java | 18 +-
.../providers/PingIdentityProvider.java | 5 +-
23 files changed, 731 insertions(+), 424 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6db7ce93/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index f505fa3,da39ea9..5416f19
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -76,9 -76,11 +78,9 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
-import org.apache.usergrid.persistence.index.EntityIndex;
++import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+ import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
- import org.apache.usergrid.persistence.index.query.CandidateResult;
- import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.Level;
@@@ -275,9 -275,9 +275,9 @@@ public class CpRelationManager implemen
});
Observable<String> types= gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeTypePrefix, null ));
+ new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeTypePrefix, null ) );
- Iterator<String> iter = types.toBlockingObservable().getIterator();
+ Iterator<String> iter = types.toBlocking().getIterator();
while ( iter.hasNext() ) {
indexes.add( iter.next() );
}
@@@ -317,59 -318,51 +318,47 @@@
* @param edgeType Edge type, edge type prefix or null to allow any edge type
* @param fromEntityType Only consider edges from entities of this type
*/
- Map<EntityRef, Set<String>> getContainers( int limit, String edgeType, String fromEntityType ) {
-
- Map<EntityRef, Set<String>> results = new LinkedHashMap<EntityRef, Set<String>>();
-
- GraphManager gm = managerCache.getGraphManager(applicationScope);
-
- Iterator<String> edgeTypes = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType(
- cpHeadEntity.getId(), edgeType, null) ).toBlocking().getIterator();
-
- logger.debug("getContainers(): "
- + "Searched for edges of type {}\n to target {}:{}\n in scope {}\n found: {}",
- new Object[] {
- edgeType,
- cpHeadEntity.getId().getType(),
- cpHeadEntity.getId().getUuid(),
- applicationScope.getApplication(),
- edgeTypes.hasNext()
- });
+ Map<EntityRef, Set<String>> getContainers( final int limit, final String edgeType, final String fromEntityType ) {
- while ( edgeTypes.hasNext() ) {
- Map<EntityRef, Set<String>> results = new LinkedHashMap<EntityRef, Set<String>>();
--
- String etype = edgeTypes.next();
+ final GraphManager gm = managerCache.getGraphManager( applicationScope );
- Observable<Edge> edges = gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
- cpHeadEntity.getId(), etype, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ));
+ Observable<Edge> edges =
+ gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeType, null ) )
+ .flatMap( new Func1<String, Observable<Edge>>() {
+ @Override
+ public Observable<Edge> call( final String edgeType ) {
+ return gm.loadEdgesToTarget(
+ new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, null ) );
+
+ }
+ } );
+
+ //if our limit is set, take them. Note this logic is still borked, we can't possibly fit everything in memmory
+ if ( limit > -1 ) {
+ edges = edges.take( limit );
+ }
- Iterator<Edge> iter = edges.toBlocking().getIterator();
- while ( iter.hasNext() ) {
- Edge edge = iter.next();
- if ( fromEntityType != null && !fromEntityType.equals( edge.getSourceNode().getType() )) {
- logger.debug("Ignoring edge from entity type {}", edge.getSourceNode().getType());
- continue;
- return edges.collect( results, new Action2<Map<EntityRef, Set<String>>, Edge>() {
- @Override
- public void call( final Map<EntityRef, Set<String>> entityRefSetMap, final Edge edge ) {
++ return edges.collect( () -> new LinkedHashMap<EntityRef, Set<String>>(), ( entityRefSetMap, edge) -> {
+ if ( fromEntityType != null && !fromEntityType.equals( edge.getSourceNode().getType() ) ) {
+ logger.debug( "Ignoring edge from entity type {}", edge.getSourceNode().getType() );
+ return;
}
- EntityRef eref = new SimpleEntityRef(
- edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
+ final EntityRef eref =
+ new SimpleEntityRef( edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
- String name = null;
- if ( CpNamingUtils.isConnectionEdgeType( edge.getType() )) {
+ String name;
+ if ( CpNamingUtils.isConnectionEdgeType( edge.getType() ) ) {
name = CpNamingUtils.getConnectionType( edge.getType() );
- } else {
+ }
+ else {
name = CpNamingUtils.getCollectionName( edge.getType() );
}
- addMapSet( results, eref, name );
+ addMapSet( entityRefSetMap, eref, name );
}
-
- if ( limit > 0 && results.keySet().size() >= limit ) {
- break;
- }
- }
-
- return results;
- } ).toBlocking().last();
++ ).toBlocking().last();
}
@@@ -689,18 -679,16 +675,16 @@@
// create graph edge connection from head entity to member entity
Edge edge = new SimpleEdge( cpHeadEntity.getId(), edgeType, memberEntity.getId(), uuidHash );
GraphManager gm = managerCache.getGraphManager( applicationScope );
- gm.writeEdge( edge ).toBlockingObservable().last();
+ gm.writeEdge( edge ).toBlocking().last();
- logger.debug( "Wrote edgeType {}\n from {}:{}\n to {}:{}\n scope {}:{}",
- new Object[] {
- edgeType,
- cpHeadEntity.getId().getType(),
- cpHeadEntity.getId().getUuid(),
- memberEntity.getId().getType(),
- memberEntity.getId().getUuid(),
- applicationScope.getApplication().getType(),
+
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Wrote edgeType {}\n from {}:{}\n to {}:{}\n scope {}:{}", new Object[] {
+ edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(), memberEntity.getId().getType(),
+ memberEntity.getId().getUuid(), applicationScope.getApplication().getType(),
applicationScope.getApplication().getUuid()
- } );
+ } );
+ }
( ( CpEntityManager ) em ).indexEntityIntoCollection( cpHeadEntity, memberEntity, collName );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6db7ce93/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
index 0000000,535af36..fea21ae
mode 000000,100644..100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
@@@ -1,0 -1,211 +1,212 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ package org.apache.usergrid.corepersistence.results;
+
+
+ import java.util.Iterator;
+ import java.util.NoSuchElementException;
+
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import org.apache.usergrid.persistence.Results;
+ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
++import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+ import org.apache.usergrid.persistence.index.EntityIndex;
+ import org.apache.usergrid.persistence.index.IndexScope;
+ import org.apache.usergrid.persistence.index.SearchTypes;
+ import org.apache.usergrid.persistence.index.query.CandidateResults;
+ import org.apache.usergrid.persistence.index.query.Query;
+
+
+ public class ElasticSearchQueryExecutor implements QueryExecutor {
+
+ private static final Logger logger = LoggerFactory.getLogger( ElasticSearchQueryExecutor.class );
+
+ private final ResultsLoaderFactory resultsLoaderFactory;
+
+ private final ApplicationScope applicationScope;
+
- private final EntityIndex entityIndex;
++ private final ApplicationEntityIndex entityIndex;
+
+ private final IndexScope indexScope;
+
+ private final SearchTypes types;
+
+ private final Query query;
+
+
+ private Results currentResults;
+
+ private boolean moreToLoad = true;
+
+
- public ElasticSearchQueryExecutor( final ResultsLoaderFactory resultsLoaderFactory, final EntityIndex entityIndex,
++ public ElasticSearchQueryExecutor( final ResultsLoaderFactory resultsLoaderFactory, final ApplicationEntityIndex entityIndex,
+ final ApplicationScope applicationScope, final IndexScope indexScope,
+ final SearchTypes types, final Query query ) {
+ this.resultsLoaderFactory = resultsLoaderFactory;
+ this.applicationScope = applicationScope;
+ this.entityIndex = entityIndex;
+ this.indexScope = indexScope;
+ this.types = types;
+
+ //we must deep copy the query passed. Otherwise we will modify it's state with cursors. Won't fix, not relevant
+ //once we start subscribing to streams.
+ this.query = new Query(query);
+ }
+
+
+ @Override
+ public Iterator<Results> iterator() {
+ return this;
+ }
+
+
+ private void loadNextPage() {
+ // Because of possible stale entities, which are filtered out by buildResults(),
+ // we loop until the we've got enough results to satisfy the query limit.
+
+ final int maxQueries = 10; // max re-queries to satisfy query limit
+
+ final int originalLimit = query.getLimit();
+
+ Results results = null;
+ int queryCount = 0;
+
+ boolean satisfied = false;
+
+
+ while ( !satisfied && queryCount++ < maxQueries ) {
+
+ CandidateResults crs = entityIndex.search( indexScope, types, query );
+
+ logger.debug( "Calling build results 1" );
+ results = buildResults( indexScope, query, crs );
+
+ if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get more
+ satisfied = true;
+ }
+
+ /**
+ * In an edge case where we delete stale entities, we could potentially get less results than expected.
+ * This will only occur once during the repair phase.
+ * We need to ensure that we short circuit before we overflow the requested limit during a repair.
+ */
+ else if ( results.size() > 0 ) { // got what we need
+ satisfied = true;
+ }
+ //we didn't load anything, but there was a cursor, this means a read repair occured. We have to short
+ //circuit to avoid over returning the result set
+ else if ( crs.hasCursor() ) {
+ satisfied = false;
+
+ // need to query for more
+ // ask for just what we need to satisfy, don't want to exceed limit
+ query.setCursor( results.getCursor() );
+ query.setLimit( originalLimit - results.size() );
+
+ logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
+ originalLimit, query.getLimit(), queryCount
+ } );
+ }
+ }
+
+ //now set our cursor if we have one for the next iteration
+ if ( results.hasCursor() ) {
+ query.setCursor( results.getCursor() );
+ moreToLoad = true;
+ }
+
+ else {
+ moreToLoad = false;
+ }
+
+
+ //set our current results and the flag
+ this.currentResults = results;
+ }
+
+
+ /**
+ * Build results from a set of candidates, and discard those that represent stale indexes.
+ *
+ * @param query Query that was executed
+ * @param crs Candidates to be considered for results
+ */
+ private Results buildResults( final IndexScope indexScope, final Query query, final CandidateResults crs ) {
+
+ logger.debug( "buildResults() from {} candidates", crs.size() );
+
+ //get an instance of our results loader
+ final ResultsLoader resultsLoader =
+ this.resultsLoaderFactory.getLoader( applicationScope, indexScope, query.getResultsLevel() );
+
+ //load the results
+ final Results results = resultsLoader.loadResults( crs );
+
+ //signal for post processing
+ resultsLoader.postProcess();
+
+
+ results.setCursor( crs.getCursor() );
+
+ //ugly and tight coupling, but we don't have a choice until we finish some refactoring
+ results.setQueryExecutor( this );
+
+ logger.debug( "Returning results size {}", results.size() );
+
+ return results;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+
+ //we've tried to load and it's empty and we have more to load, load the next page
+ if ( currentResults == null ) {
+ //there's nothing left to load, nothing to do
+ if ( !moreToLoad ) {
+ return false;
+ }
+
+ //load the page
+
+ loadNextPage();
+ }
+
+
+ //see if our current results are not null
+ return currentResults != null;
+ }
+
+
+ @Override
+ public Results next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more results present" );
+ }
+
+ final Results toReturn = currentResults;
+
+ currentResults = null;
+
+ return toReturn;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6db7ce93/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6db7ce93/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6db7ce93/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6db7ce93/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
index d588476,9c768bf..26395b4
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/ListUtils.java
@@@ -25,9 -27,7 +27,8 @@@ import org.apache.commons.lang.math.Num
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import org.apache.commons.lang.math.NumberUtils;
+
public class ListUtils extends org.apache.commons.collections.ListUtils {
private static final Logger LOG = LoggerFactory.getLogger( ListUtils.class );