[GitHub] [unomi] dgriffon commented on a diff in pull request #458: UNOMI-626: improve migration system to prepare Unomi 2.0.0 data model

[GitBox <gi...@apache.org>]

dgriffon commented on code in PR #458:
URL: https://github.com/apache/unomi/pull/458#discussion_r916745725


##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrateScript.java:
##########
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration;
+
+import groovy.lang.Script;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.Version;
+
+import java.io.IOException;
+import java.net.URL;
+
+public class MigrateScript implements Comparable<MigrateScript> {
+
+
+    private final String script;
+    private Script compiledScript;
+    private final Bundle bundle;
+    private final Version version;
+    private final int priority;
+    private final String name;
+
+    public MigrateScript(URL scriptURL, Bundle bundle) throws IOException {
+        this.bundle = bundle;
+        this.script = IOUtils.toString(scriptURL);
+
+        // parse file name expected format is: migrate-VERSION-PRIORITY-NAME.groovy like: migrate-1.2.1-00-migrateTags.groovy
+        String path = scriptURL.getPath();
+        String[] splitName = path.substring(path.lastIndexOf("/"), path.lastIndexOf(".groovy")).split("-");

Review Comment:
   I would do a format check before doing the parsing, unless it will fail then fail the whole migration



##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java:
##########
@@ -16,120 +16,183 @@
  */
 package org.apache.unomi.shell.migration.actions;
 
+import groovy.lang.GroovyClassLoader;
+import groovy.lang.GroovyShell;
+import groovy.util.GroovyScriptEngine;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.karaf.shell.api.action.Action;
 import org.apache.karaf.shell.api.action.Argument;
 import org.apache.karaf.shell.api.action.Command;
 import org.apache.karaf.shell.api.action.lifecycle.Reference;
 import org.apache.karaf.shell.api.action.lifecycle.Service;
 import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.Migration;
+import org.apache.unomi.shell.migration.MigrateScript;
 import org.apache.unomi.shell.migration.utils.ConsoleUtils;
 import org.apache.unomi.shell.migration.utils.HttpUtils;
-import org.osgi.framework.BundleContext;
-import org.osgi.framework.InvalidSyntaxException;
-import org.osgi.framework.ServiceReference;
-import org.osgi.framework.Version;
-
+import org.osgi.framework.*;
+import org.osgi.framework.wiring.BundleWiring;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
-@Command(scope = "unomi", name = "migrate", description = "This will Migrate your date in ES to be compliant with current version")
+@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version")
 @Service
 public class Migrate implements Action {
+    public static final String CONFIG_ES_ADDRESS = "esAddress";
+    public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates";
 
     @Reference
     Session session;
 
     @Reference
     BundleContext bundleContext;
 
-    @Argument(name = "fromVersionWithoutSuffix", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", multiValued = false, valueToShowInHelp = "1.2.0")
-    private String fromVersionWithoutSuffix;
+    @Argument(index = 0, name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
+    private String originVersion;
 
     public Object execute() throws Exception {
-        if (fromVersionWithoutSuffix == null) {
-            listMigrations();
+        // Load migration scrips
+        Set<MigrateScript> scripts = loadOSGIScripts();
+        scripts.addAll(loadFileSystemScripts());
+
+        if (originVersion == null) {
+            displayMigrations(scripts);
+            ConsoleUtils.printMessage(session, "Select your migration starting point by specifying the current version (e.g. 1.2.0) or the last script that was already run (e.g. 1.2.1)");
             return null;
         }
 
-        String confirmation = ConsoleUtils.askUserWithAuthorizedAnswer(session,"[WARNING] You are about to execute a migration, this a very sensitive operation, are you sure? (yes/no): ", Arrays.asList("yes", "no"));
-        if (confirmation.equalsIgnoreCase("no")) {
-            System.out.println("Migration process aborted");
+        // Check that there is some migration scripts available from given version
+        Version fromVersion = new Version(originVersion);
+        scripts = filterScriptsFromVersion(scripts, fromVersion);
+        if (scripts.size() == 0) {
+            ConsoleUtils.printMessage(session, "No migration scripts available found starting from version: " + originVersion);
             return null;
+        } else {
+            ConsoleUtils.printMessage(session, "The following migration scripts starting from version: " + originVersion + " will be executed.");
+            displayMigrations(scripts);
         }
 
-        System.out.println("Starting migration process from version: " + fromVersionWithoutSuffix);
-
-        Version fromVersion = new Version(fromVersionWithoutSuffix);
-        Version currentVersion = getCurrentVersionWithoutQualifier();
-        System.out.println("current version: " + currentVersion.toString());
-        if (currentVersion.compareTo(fromVersion) <= 0) {
-            System.out.println("From version is same or superior than current version, nothing to migrate.");
+        // Check for user approval before migrate
+        if (ConsoleUtils.askUserWithAuthorizedAnswer(session,
+                "[WARNING] You are about to execute a migration, this a very sensitive operation, are you sure? (yes/no): ",
+                Arrays.asList("yes", "no")).equalsIgnoreCase("no")) {
+            ConsoleUtils.printMessage(session, "Migration process aborted");
             return null;
         }
 
-        CloseableHttpClient httpClient = null;
-        try {
-            httpClient = HttpUtils.initHttpClient(session);
-
-            String esAddress = ConsoleUtils.askUserWithDefaultAnswer(session, "Enter ElasticSearch 7 TARGET address (default = http://localhost:9200): ", "http://localhost:9200");
-
-            for (Migration migration : getMigrations()) {
-                if (fromVersion.compareTo(migration.getToVersion()) < 0) {
-                    String migrateConfirmation = ConsoleUtils.askUserWithAuthorizedAnswer(session,"Starting migration to version " + migration.getToVersion() + ", do you want to proceed? (yes/no): ", Arrays.asList("yes", "no"));
-                    if (migrateConfirmation.equalsIgnoreCase("no")) {
-                        System.out.println("Migration process aborted");
-                        break;
-                    }
-                    migration.execute(session, httpClient, esAddress, bundleContext);
-                    System.out.println("Migration to version " + migration.getToVersion() + " done successfully");
+        // Build conf
+        Map<String, Object> migrationConfig = new HashMap<>();
+        migrationConfig.put(CONFIG_ES_ADDRESS, ConsoleUtils.askUserWithDefaultAnswer(session, "Enter ElasticSearch 7 TARGET address (default = http://localhost:9200): ", "http://localhost:9200"));

Review Comment:
   Do we want to handle right now credentials to connect to ES (as we know cloud will require them)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: dev-unsubscribe@unomi.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org