You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by GitBox <gi...@apache.org> on 2022/05/23 12:12:18 UTC
[GitHub] [unomi] jsinovassin commented on a diff in pull request #427: UNOMI-395 : add migration logic to create scopes entries from scope i…
jsinovassin commented on code in PR #427:
URL: https://github.com/apache/unomi/pull/427#discussion_r879382754
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId = ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\": \"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
httpPost.setEntity(new StringEntity(request));
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName + "\" successfully completed. Total: " +
- responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated") + ".");
+ System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName
+ + "\" successfully completed. Total: " + responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
+ + ".");
} else {
System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName + "\" failed.");
}
}
}
+ private Set<String> getEventIndexes(String indexPrefix) throws IOException {
+ try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(indexPrefix + "-event")).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
+ private void createScopes(Set<String> scopes, String indexPrefix) throws IOException {
+ final StringBuilder body = new StringBuilder();
+ scopes.forEach(scope -> {
+ body.append("{\"index\": {\"_id\": \"" + scope + "\"}}\n");
+ body.append("{\"itemId\": \"" + scope + "\", \"itemType\": \"scope\", \"metadata\": { \"id\": \"" + scope + "\" }}\n");
+ });
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix + "-scope/_bulk");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/x-ndjson");
+
+ httpPost.setEntity(new StringEntity(body.toString()));
+
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope successfully finished");
+ } else {
+ System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope has failed" + response.getStatusLine()
+ .getStatusCode());
+ }
+ }
+ }
+
+ private Set<String> getSetOfScopes(Set<String> indices) throws IOException {
+ String joinedIndices = String.join(",", indices);
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices + "/_search");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/json");
+
+ String request =
+ "{\n" + " \"_source\": false,\n" + " \"size\": 0,\n" + " \"aggs\": {\n" + " \"scopes\": {\n" + " \"terms\": {\n"
+ + " \"field\": \"scope.keyword\"\n" + " }\n" + " },\n" + " \"bucketInfos\": {\n"
+ + " \"stats_bucket\": {\n" + " \"buckets_path\": \"scopes._count\"\n" + " }\n" + " }\n"
+ + " }\n" + "}";
+
+ httpPost.setEntity(new StringEntity(request));
+
+ Set<String> scopes = new HashSet<>();
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Getting the \"scope\" values from the events successfully finished. " + "Number of scope to create: "
+ + responseAsJson.getJSONObject("aggregations").getJSONObject("bucketInfos").get("count").toString());
+ scopes = StreamSupport
+ .stream(responseAsJson.getJSONObject("aggregations").getJSONObject("scopes").getJSONArray("buckets").spliterator(),
+ false).map(bucketElement -> ((JSONObject) bucketElement).getString("key")).collect(Collectors.toSet());
Review Comment:
Yes, it's null safe an exception would be thrown if the attribute is not found and the migration will not crash
##########
tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java:
##########
@@ -126,25 +113,88 @@ private void copyValueScopeToSourceId(final String indexName, final CloseableHtt
httpPost.addHeader("Accept", "application/json");
httpPost.addHeader("Content-Type", "application/json");
- String request = "{\n" +
- " \"script\": {\n" +
- " \"source\": \"ctx._source.sourceId = ctx._source.scope\",\n" +
- " \"lang\": \"painless\"\n" +
- " }\n" +
- "}";
+ String request = "{\n" + " \"script\": {\n" + " \"source\": \"ctx._source.sourceId = ctx._source.scope\",\n"
+ + " \"lang\": \"painless\"\n" + " }\n" + "}";
httpPost.setEntity(new StringEntity(request));
try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName + "\" successfully completed. Total: " +
- responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated") + ".");
+ System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName
+ + "\" successfully completed. Total: " + responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
+ + ".");
} else {
System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName + "\" failed.");
}
}
}
+ private Set<String> getEventIndexes(String indexPrefix) throws IOException {
+ try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(indexPrefix + "-event")).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
+ private void createScopes(Set<String> scopes, String indexPrefix) throws IOException {
+ final StringBuilder body = new StringBuilder();
+ scopes.forEach(scope -> {
+ body.append("{\"index\": {\"_id\": \"" + scope + "\"}}\n");
+ body.append("{\"itemId\": \"" + scope + "\", \"itemType\": \"scope\", \"metadata\": { \"id\": \"" + scope + "\" }}\n");
+ });
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix + "-scope/_bulk");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/x-ndjson");
+
+ httpPost.setEntity(new StringEntity(body.toString()));
+
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope successfully finished");
+ } else {
+ System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope has failed" + response.getStatusLine()
+ .getStatusCode());
+ }
+ }
+ }
+
+ private Set<String> getSetOfScopes(Set<String> indices) throws IOException {
+ String joinedIndices = String.join(",", indices);
+ final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices + "/_search");
+
+ httpPost.addHeader("Accept", "application/json");
+ httpPost.addHeader("Content-Type", "application/json");
+
+ String request =
+ "{\n" + " \"_source\": false,\n" + " \"size\": 0,\n" + " \"aggs\": {\n" + " \"scopes\": {\n" + " \"terms\": {\n"
+ + " \"field\": \"scope.keyword\"\n" + " }\n" + " },\n" + " \"bucketInfos\": {\n"
+ + " \"stats_bucket\": {\n" + " \"buckets_path\": \"scopes._count\"\n" + " }\n" + " }\n"
+ + " }\n" + "}";
+
+ httpPost.setEntity(new StringEntity(request));
+
+ Set<String> scopes = new HashSet<>();
+ try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ System.out.println("Getting the \"scope\" values from the events successfully finished. " + "Number of scope to create: "
+ + responseAsJson.getJSONObject("aggregations").getJSONObject("bucketInfos").get("count").toString());
+ scopes = StreamSupport
+ .stream(responseAsJson.getJSONObject("aggregations").getJSONObject("scopes").getJSONArray("buckets").spliterator(),
+ false).map(bucketElement -> ((JSONObject) bucketElement).getString("key")).collect(Collectors.toSet());
Review Comment:
Yes, it's null safe, an exception would be thrown if the attribute is not found and the migration will not crash
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@unomi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org