You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openmeetings.apache.org by so...@apache.org on 2019/09/19 17:05:51 UTC
[openmeetings] branch master updated: [OPENMEETINGS-2113] tree
import should be improved
This is an automated email from the ASF dual-hosted git repository.
solomax pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/openmeetings.git
The following commit(s) were added to refs/heads/master by this push:
new 440d0c6 [OPENMEETINGS-2113] tree import should be improved
440d0c6 is described below
commit 440d0c666deda99946aaeba58a1a9d5fe948105b
Author: Maxim Solodovnik <so...@gmail.com>
AuthorDate: Fri Sep 20 00:05:38 2019 +0700
[OPENMEETINGS-2113] tree import should be improved
---
.../apache/openmeetings/backup/BackupImport.java | 70 +++++++++++++++----
.../org/apache/openmeetings/backup/FileTree.java | 80 ++++++++++++++++++++++
2 files changed, 136 insertions(+), 14 deletions(-)
diff --git a/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java
index 1febc7c..d312f64 100644
--- a/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java
+++ b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/BackupImport.java
@@ -116,7 +116,10 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -822,7 +825,21 @@ public class BackupImport {
}
}
- private boolean isInvalidFile(BaseFileItem file) {
+ private boolean isInvalidFile(BaseFileItem file, final Map<Long, Long> folders) {
+ if (file.isDeleted()) {
+ return true;
+ }
+ if (file.getParentId() != null && file.getParentId() > 0) {
+ Long newFolder = folders.get(file.getParentId());
+ if (newFolder == null) {
+ //folder was deleted
+ return true;
+ } else {
+ file.setParentId(newFolder);
+ }
+ } else {
+ file.setParentId(null);
+ }
if (file.getRoomId() != null) {
Long newRoomId = roomMap.get(file.getRoomId());
if (newRoomId == null) {
@@ -839,6 +856,30 @@ public class BackupImport {
}
return false;
}
+
+ private static <T extends BaseFileItem> FileTree<T> build(List<T> list) {
+ TreeMap<Long, T> items = new TreeMap<>(list.stream().collect(Collectors.toMap(f -> f.getId(), f -> f)));
+ FileTree<T> tree = new FileTree<>();
+ TreeMap<Long, T> remain = new TreeMap<>();
+ int counter = list.size(); //max iterations
+ while (counter > 0 && !items.isEmpty()) {
+ Entry<Long, T> e = items.pollFirstEntry();
+ if (e == null) {
+ break;
+ } else {
+ if (!tree.add(e.getValue())) {
+ remain.put(e.getKey(), e.getValue());
+ }
+ }
+ if (items.isEmpty()) {
+ counter = Math.min(counter - 1, remain.size());
+ items.putAll(remain);
+ remain.clear();
+ }
+ }
+ remain.entrySet().forEach(e -> log.warn("Doungling file/recording: {}", e.getValue()));
+ return tree;
+ }
/*
* ##################### Import Recordings
*/
@@ -853,13 +894,12 @@ public class BackupImport {
matcher.bind(Integer.class, IntegerTransform.class);
registry.bind(Date.class, DateConverter.class);
registry.bind(Recording.Status.class, RecordingStatusConverter.class);
+ final Map<Long, Long> folders = new HashMap<>();
List<Recording> list = readList(ser, f, "flvRecordings.xml", "flvrecordings", Recording.class);
- for (Recording r : list) {
+ FileTree<Recording> tree = build(list);
+ tree.process(r -> isInvalidFile(r, folders), r -> {
Long recId = r.getId();
r.setId(null);
- if (isInvalidFile(r)) {
- continue;
- }
if (r.getChunks() != null) {
for (RecordingChunk chunk : r.getChunks()) {
chunk.setId(null);
@@ -876,8 +916,11 @@ public class BackupImport {
hashMap.put(oldHash, r.getHash());
}
r = recordingDao.update(r);
+ if (BaseFileItem.Type.Folder == r.getType()) {
+ folders.put(recId, r.getId());
+ }
fileItemMap.put(recId, r.getId());
- }
+ });
}
/*
@@ -986,24 +1029,23 @@ public class BackupImport {
matcher.bind(Long.class, LongTransform.class);
matcher.bind(Integer.class, IntegerTransform.class);
registry.bind(Date.class, DateConverter.class);
+ final Map<Long, Long> folders = new HashMap<>();
List<FileItem> list = readList(ser, f, "fileExplorerItems.xml", "fileExplorerItems", FileItem.class);
- for (FileItem file : list) {
+ FileTree<FileItem> tree = build(list);
+ tree.process(file -> isInvalidFile(file, folders), file -> {
Long fId = file.getId();
// We need to reset this as openJPA reject to store them otherwise
file.setId(null);
- if (isInvalidFile(file)) {
- continue;
- }
- if (file.getParentId() != null && file.getParentId().longValue() <= 0L) {
- file.setParentId(null);
- }
String oldHash = file.getHash();
file.setHash(randomUUID().toString());
hashMap.put(oldHash, file.getHash());
file = fileItemDao.update(file);
+ if (BaseFileItem.Type.Folder == file.getType()) {
+ folders.put(fId, file.getId());
+ }
result.add(file);
fileItemMap.put(fId, file.getId());
- }
+ });
return result;
}
diff --git a/openmeetings-install/src/main/java/org/apache/openmeetings/backup/FileTree.java b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/FileTree.java
new file mode 100644
index 0000000..c59ab1f
--- /dev/null
+++ b/openmeetings-install/src/main/java/org/apache/openmeetings/backup/FileTree.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License") + you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.openmeetings.backup;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+import org.apache.openmeetings.db.entity.file.BaseFileItem;
+
+class FileTree<T extends BaseFileItem> {
+ private final T item;
+ private final Map<Long, FileTree<T>> children = new HashMap<>();
+
+ FileTree() {
+ this(null);
+ }
+
+ FileTree(T item) {
+ this.item = item;
+ }
+
+ boolean add(T child) {
+ if (child.getParentId() != null && child.getParentId() < 1) {
+ child.setParentId(null);
+ }
+ if ((item == null && child.getParentId() == null) || (item != null && item.getId().equals(child.getParentId()))) {
+ children.put(child.getId(), new FileTree<>(child));
+ return true;
+ }
+ for (Entry<Long, FileTree<T>> e : children.entrySet()) {
+ if (e.getValue().add(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ void process(Predicate<T> invalid, Consumer<T> consumer) {
+ if (item != null) {
+ if (invalid.test(item)) {
+ return; // we will not process invalid and i's children
+ }
+ consumer.accept(item);
+ }
+ if (!children.isEmpty()) {
+ children.forEach((id, e) -> e.process(invalid, consumer));
+ }
+ }
+
+ @Override
+ public String toString() {
+ String val = "FileTree[type ='" + (item == null ? "root" : item.getType()) + "'";
+ if (item != null) {
+ val += ", name='" + item.getName() + "'";
+ }
+ if (!children.isEmpty()) {
+ val += ", children='" + children + "'";
+ }
+ return val + "]";
+ }
+}