You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/08/28 00:15:58 UTC
[03/10] usergrid git commit: migration
migration
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c47b02d4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c47b02d4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c47b02d4
Branch: refs/heads/two-dot-o-dev
Commit: c47b02d41910c524957149dbf2bf8f4a3dff051d
Parents: 81e0fba
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Aug 21 19:20:26 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Aug 21 19:20:26 2015 -0700
----------------------------------------------------------------------
.../persistence/index/guice/IndexModule.java | 6 +-
.../EsIndexMappingMigrationPlugin.java | 148 +++++++++++++++++++
.../impl/EsIndexMappingMigrationPluginTest.java | 60 ++++++++
3 files changed, 212 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47b02d4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index b22a461..d03bbf8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import org.apache.usergrid.persistence.index.impl.*;
+import org.apache.usergrid.persistence.index.migration.EsIndexMappingMigrationPlugin;
import org.apache.usergrid.persistence.index.migration.EsIndexMigrationPlugin;
import org.apache.usergrid.persistence.index.migration.IndexMigration;
import org.apache.usergrid.persistence.map.guice.MapModule;
@@ -61,8 +62,9 @@ public abstract class IndexModule extends AbstractModule {
//wire up the collection migration plugin
- Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to(EsIndexMigrationPlugin.class);
-
+ final Multibinder<MigrationPlugin> plugins = Multibinder.newSetBinder(binder(), MigrationPlugin.class);
+ plugins.addBinding().to(EsIndexMigrationPlugin.class);
+ plugins.addBinding().to(EsIndexMappingMigrationPlugin.class);
//invoke the migration plugin config
configureMigrationProvider();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47b02d4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java
new file mode 100644
index 0000000..10acc46
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java
@@ -0,0 +1,148 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Classy class class.
+ */
+public class EsIndexMappingMigrationPlugin implements MigrationPlugin {
+
+ private static final Logger logger = LoggerFactory.getLogger(EsIndexMappingMigrationPlugin.class);
+
+ private final MigrationInfoSerialization migrationInfoSerialization;
+ private final EsProvider provider;
+
+ @Inject
+ public EsIndexMappingMigrationPlugin(
+ final MigrationInfoSerialization migrationInfoSerialization,
+ final EsProvider provider
+ ){
+
+ this.migrationInfoSerialization = migrationInfoSerialization;
+ this.provider = provider;
+ }
+
+ @Override
+ public String getName() {
+ return "index_mapping_migration";
+ }
+
+ @Override
+ public void run(ProgressObserver observer) {
+
+ final int version = migrationInfoSerialization.getVersion(getName());
+
+ if (version == getMaxVersion()) {
+ logger.debug("Skipping Migration Plugin: " + getName());
+ return;
+ }
+
+ try {
+ ActionFuture<GetIndexResponse> responseFuture = provider.getClient().admin().indices().getIndex(new GetIndexRequest());
+ Observable
+ .from(responseFuture)
+ .flatMap(response -> {
+ List<String> indices = Arrays.asList(response.getIndices());
+ return Observable.from(indices);
+ })
+
+ .doOnNext(index -> {
+ createMappings(index);
+ observer.update(getMaxVersion(), "running update for " + index);
+ })
+ .doOnError(t -> {
+ observer.failed(getMaxVersion(),"failed to update",t);
+ })
+ .doOnCompleted(() -> {
+ migrationInfoSerialization.setVersion(getName(), getMaxVersion());
+ observer.complete();
+ })
+ .subscribe();
+
+
+ }catch (Exception ee){
+ observer.failed(getMaxVersion(),"failed to update",ee);
+ throw new RuntimeException(ee);
+ }
+
+ }
+
+
+ /**
+ * Setup ElasticSearch type mappings as a template that applies to all new indexes.
+ * Applies to all indexes that* start with our prefix.
+ */
+ private void createMappings(final String indexName) {
+
+ //Added For Graphite Metrics
+ PutMappingResponse pitr = provider.getClient().admin().indices().preparePutMapping( indexName ).setType( "entity" ).setSource(
+ getMappingsContent() ).execute().actionGet();
+ if ( !pitr.isAcknowledged() ) {
+ throw new RuntimeException( "Unable to create default mappings" );
+ }
+ }
+
+
+ /**
+ * Get the content from our mappings file
+ * @return
+ */
+ private String getMappingsContent(){
+ URL url = Resources.getResource("org/apache/usergrid/persistence/index/usergrid-mappings.json");
+ try {
+ return Resources.toString(url, Charsets.UTF_8);
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to read mappings file", e );
+ }
+ }
+
+ @Override
+ public int getMaxVersion() {
+ return 1;
+ }
+
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.BOOTSTRAP;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47b02d4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java
new file mode 100644
index 0000000..e5d9380
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java
@@ -0,0 +1,60 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.index.migration.EsIndexMappingMigrationPlugin;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Classy class class.
+ */
+
+@RunWith( EsRunner.class )
+@UseModules( { TestIndexModule.class } )
+public class EsIndexMappingMigrationPluginTest extends BaseIT {
+ @Inject
+ public EntityIndexFactory eif;
+ @Inject
+ public MigrationInfoSerialization serialization;
+ @Inject
+ EsProvider provider;
+ @Test
+ public void runMigration(){
+ EsIndexMappingMigrationPlugin plugin = new EsIndexMappingMigrationPlugin(serialization,provider);
+ TestProgressObserver progressObserver = new TestProgressObserver();
+ plugin.run(progressObserver);
+ assertFalse( "Progress observer should not have failed", progressObserver.isFailed() );
+ assertTrue("Progress observer should have update messages", progressObserver.getUpdates().size() > 0);
+
+
+ }
+
+}