You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openmeetings.apache.org by so...@apache.org on 2019/09/19 17:41:18 UTC
[openmeetings] branch 4.0.x updated: [OPENMEETINGS-2113] tree
import should be improved
This is an automated email from the ASF dual-hosted git repository.
solomax pushed a commit to branch 4.0.x
in repository https://gitbox.apache.org/repos/asf/openmeetings.git
The following commit(s) were added to refs/heads/4.0.x by this push:
new dc9da3c [OPENMEETINGS-2113] tree import should be improved
dc9da3c is described below
commit dc9da3c4a55afbd02e17655b8c2a8e9493daf651
Author: Maxim Solodovnik <so...@gmail.com>
AuthorDate: Fri Sep 20 00:05:38 2019 +0700
[OPENMEETINGS-2113] tree import should be improved
---
.../apache/openmeetings/backup/BackupImport.java | 86 ++++++++++++++++------
.../org/apache/openmeetings/backup/FileTree.java | 80 ++++++++++++++++++++
2 files changed, 143 insertions(+), 23 deletions(-)
diff --git a/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java
index aedec16..773cd21 100644
--- a/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java
+++ b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java
@@ -18,6 +18,7 @@
*/
package org.apache.openmeetings.backup;
+import static java.util.UUID.randomUUID;
import static org.apache.openmeetings.db.entity.user.PrivateMessage.INBOX_FOLDER_ID;
import static org.apache.openmeetings.db.entity.user.PrivateMessage.SENT_FOLDER_ID;
import static org.apache.openmeetings.db.entity.user.PrivateMessage.TRASH_FOLDER_ID;
@@ -118,9 +119,11 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
-import java.util.UUID;
+import java.util.TreeMap;
import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -619,7 +622,7 @@ public class BackupImport {
continue;
}
if (u.getType() == User.Type.contact && u.getLogin().length() < minLoginLength) {
- u.setLogin(UUID.randomUUID().toString());
+ u.setLogin(randomUUID().toString());
}
String tz = u.getTimeZoneId();
@@ -802,7 +805,21 @@ public class BackupImport {
}
}
- private boolean isInvalidFile(BaseFileItem file) {
+ private boolean isInvalidFile(BaseFileItem file, final Map<Long, Long> folders) {
+ if (file.isDeleted()) {
+ return true;
+ }
+ if (file.getParentId() != null && file.getParentId() > 0) {
+ Long newFolder = folders.get(file.getParentId());
+ if (newFolder == null) {
+ //folder was deleted
+ return true;
+ } else {
+ file.setParentId(newFolder);
+ }
+ } else {
+ file.setParentId(null);
+ }
if (file.getRoomId() != null) {
Long newRoomId = roomMap.get(file.getRoomId());
if (newRoomId == null) {
@@ -819,18 +836,41 @@ public class BackupImport {
}
return false;
}
+
+ private static <T extends BaseFileItem> FileTree<T> build(List<T> list) {
+ TreeMap<Long, T> items = new TreeMap<>(list.stream().collect(Collectors.toMap(f -> f.getId(), f -> f)));
+ FileTree<T> tree = new FileTree<>();
+ TreeMap<Long, T> remain = new TreeMap<>();
+ int counter = list.size(); //max iterations
+ while (counter > 0 && !items.isEmpty()) {
+ Entry<Long, T> e = items.pollFirstEntry();
+ if (e == null) {
+ break;
+ } else {
+ if (!tree.add(e.getValue())) {
+ remain.put(e.getKey(), e.getValue());
+ }
+ }
+ if (items.isEmpty()) {
+ counter = Math.min(counter - 1, remain.size());
+ items.putAll(remain);
+ remain.clear();
+ }
+ }
+ remain.entrySet().forEach(e -> log.warn("Doungling file/recording: {}", e.getValue()));
+ return tree;
+ }
/*
* ##################### Import Recordings
*/
private void importRecordings(File f) throws Exception {
log.info("Meeting members import complete, starting recordings server import");
+ final Map<Long, Long> folders = new HashMap<>();
List<Recording> list = readRecordingList(f, "flvRecordings.xml", "flvrecordings");
- for (Recording r : list) {
+ FileTree<Recording> tree = build(list);
+ tree.process(r -> isInvalidFile(r, folders), r -> {
Long recId = r.getId();
r.setId(null);
- if (isInvalidFile(r)) {
- continue;
- }
if (r.getMetaData() != null) {
for (RecordingMetaData meta : r.getMetaData()) {
meta.setId(null);
@@ -839,16 +879,19 @@ public class BackupImport {
}
if (!Strings.isEmpty(r.getHash()) && r.getHash().startsWith(RECORDING_FILE_NAME)) {
String name = getFileName(r.getHash());
- r.setHash(UUID.randomUUID().toString());
+ r.setHash(randomUUID().toString());
fileMap.put(String.format(FILE_NAME_FMT, name, EXTENSION_JPG), String.format(FILE_NAME_FMT, r.getHash(), EXTENSION_PNG));
fileMap.put(String.format("%s.%s.%s", name, EXTENSION_FLV, EXTENSION_MP4), String.format(FILE_NAME_FMT, r.getHash(), EXTENSION_MP4));
}
if (Strings.isEmpty(r.getHash())) {
- r.setHash(UUID.randomUUID().toString());
+ r.setHash(randomUUID().toString());
}
r = recordingDao.update(r);
+ if (BaseFileItem.Type.Folder == r.getType()) {
+ folders.put(recId, r.getId());
+ }
fileItemMap.put(recId, r.getId());
- }
+ });
}
/*
@@ -949,24 +992,21 @@ public class BackupImport {
private List<FileItem> importFiles(File f) throws Exception {
log.info("Private message import complete, starting file explorer item import");
List<FileItem> result = new ArrayList<>();
+ final Map<Long, Long> folders = new HashMap<>();
List<FileItem> list = readFileItemList(f, "fileExplorerItems.xml", "fileExplorerItems");
- for (FileItem file : list) {
+ FileTree<FileItem> tree = build(list);
+ tree.process(file -> isInvalidFile(file, folders), file -> {
Long fId = file.getId();
// We need to reset this as openJPA reject to store them otherwise
file.setId(null);
- if (isInvalidFile(file)) {
- continue;
- }
- if (file.getParentId() != null && file.getParentId().longValue() <= 0L) {
- file.setParentId(null);
- }
- if (Strings.isEmpty(file.getHash())) {
- file.setHash(UUID.randomUUID().toString());
- }
+ file.setHash(randomUUID().toString());
file = fileItemDao.update(file);
+ if (BaseFileItem.Type.Folder == file.getType()) {
+ folders.put(fId, file.getId());
+ }
result.add(file);
fileItemMap.put(fId, file.getId());
- }
+ });
return result;
}
@@ -1529,14 +1569,14 @@ public class BackupImport {
if (u.getAddress() != null && u.getAddress().getEmail() != null && User.Type.user == u.getType()) {
if (userEmailMap.containsKey(u.getAddress().getEmail())) {
log.warn("Email is duplicated for user " + u.toString());
- String updateEmail = String.format("modified_by_import_<%s>%s", UUID.randomUUID(), u.getAddress().getEmail());
+ String updateEmail = String.format("modified_by_import_<%s>%s", randomUUID(), u.getAddress().getEmail());
u.getAddress().setEmail(updateEmail);
}
userEmailMap.put(u.getAddress().getEmail(), Integer.valueOf(userEmailMap.size()));
}
if (userLoginMap.containsKey(u.getLogin())) {
log.warn("Login is duplicated for user " + u.toString());
- String updateLogin = String.format("modified_by_import_<%s>%s", UUID.randomUUID(), u.getLogin());
+ String updateLogin = String.format("modified_by_import_<%s>%s", randomUUID(), u.getLogin());
u.setLogin(updateLogin);
}
userLoginMap.put(u.getLogin(), Integer.valueOf(userLoginMap.size()));
diff --git a/openmeetings-install/src/main/java/org/apache/openmeetings/backup/FileTree.java b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/FileTree.java
new file mode 100644
index 0000000..c59ab1f
--- /dev/null
+++ b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/FileTree.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License") + you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.openmeetings.backup;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+import org.apache.openmeetings.db.entity.file.BaseFileItem;
+
+class FileTree<T extends BaseFileItem> {
+ private final T item;
+ private final Map<Long, FileTree<T>> children = new HashMap<>();
+
+ FileTree() {
+ this(null);
+ }
+
+ FileTree(T item) {
+ this.item = item;
+ }
+
+ boolean add(T child) {
+ if (child.getParentId() != null && child.getParentId() < 1) {
+ child.setParentId(null);
+ }
+ if ((item == null && child.getParentId() == null) || (item != null && item.getId().equals(child.getParentId()))) {
+ children.put(child.getId(), new FileTree<>(child));
+ return true;
+ }
+ for (Entry<Long, FileTree<T>> e : children.entrySet()) {
+ if (e.getValue().add(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ void process(Predicate<T> invalid, Consumer<T> consumer) {
+ if (item != null) {
+ if (invalid.test(item)) {
+ return; // we will not process invalid and i's children
+ }
+ consumer.accept(item);
+ }
+ if (!children.isEmpty()) {
+ children.forEach((id, e) -> e.process(invalid, consumer));
+ }
+ }
+
+ @Override
+ public String toString() {
+ String val = "FileTree[type ='" + (item == null ? "root" : item.getType()) + "'";
+ if (item != null) {
+ val += ", name='" + item.getName() + "'";
+ }
+ if (!children.isEmpty()) {
+ val += ", children='" + children + "'";
+ }
+ return val + "]";
+ }
+}