You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@felix.apache.org by gn...@apache.org on 2014/03/31 18:12:24 UTC
svn commit: r1583362 - in
/felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal:
DirectoryWatcher.java Scanner.java Watcher.java WatcherScanner.java
Author: gnodet
Date: Mon Mar 31 16:12:24 2014
New Revision: 1583362
URL: http://svn.apache.org/r1583362
Log:
[FELIX-2436] Reduce the number files created by the fileinstall Scanner
Use NIO2 Path watcher, even if we still use a polling mechanism, file watching are delegated to the JVM/OS
Added:
felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Watcher.java
felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/WatcherScanner.java
Modified:
felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/DirectoryWatcher.java
felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Scanner.java
Modified: felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/DirectoryWatcher.java
URL: http://svn.apache.org/viewvc/felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/DirectoryWatcher.java?rev=1583362&r1=1583361&r2=1583362&view=diff
==============================================================================
--- felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/DirectoryWatcher.java (original)
+++ felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/DirectoryWatcher.java Mon Mar 31 16:12:24 2014
@@ -21,7 +21,6 @@ package org.apache.felix.fileinstall.int
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
@@ -42,7 +41,6 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.jar.JarInputStream;
import java.util.jar.Manifest;
-import java.util.regex.Pattern;
import org.apache.felix.fileinstall.ArtifactInstaller;
import org.apache.felix.fileinstall.ArtifactListener;
@@ -106,6 +104,7 @@ public class DirectoryWatcher extends Th
public final static String UPDATE_WITH_LISTENERS = "felix.fileinstall.bundles.updateWithListeners";
public final static String OPTIONAL_SCOPE = "felix.fileinstall.optionalImportRefreshScope";
public final static String FRAGMENT_SCOPE = "felix.fileinstall.fragmentRefreshScope";
+ public final static String DISABLE_NIO2 = "felix.fileinstall.disableNio2";
public final static String SCOPE_NONE = "none";
public final static String SCOPE_MANAGED = "managed";
@@ -132,6 +131,7 @@ public class DirectoryWatcher extends Th
boolean updateWithListeners;
String fragmentScope;
String optionalScope;
+ boolean disableNio2;
// Map of all installed artifacts
final Map<File, Artifact> currentManagedArtifacts = new HashMap<File, Artifact>();
@@ -174,25 +174,18 @@ public class DirectoryWatcher extends Th
updateWithListeners = getBoolean(properties, UPDATE_WITH_LISTENERS, false); // Do not update bundles when listeners are updated
fragmentScope = properties.get(FRAGMENT_SCOPE);
optionalScope = properties.get(OPTIONAL_SCOPE);
+ disableNio2 = getBoolean(properties, DISABLE_NIO2, false);
this.context.addBundleListener(this);
- FilenameFilter flt;
- if (filter != null && filter.length() > 0)
- {
- flt = new FilenameFilter()
- {
- Pattern pattern = Pattern.compile(filter);
- public boolean accept(File dir, String name)
- {
- return pattern.matcher(name).matches();
- }
- };
- }
- else
- {
- flt = null;
+ if (disableNio2) {
+ scanner = new Scanner(watchedDirectory, filter);
+ } else {
+ try {
+ scanner = new WatcherScanner(context, watchedDirectory, filter);
+ } catch (Throwable t) {
+ scanner = new Scanner(watchedDirectory, filter);
+ }
}
- scanner = new Scanner(watchedDirectory, flt);
}
private void verifyWatchedDir()
@@ -769,6 +762,14 @@ public class DirectoryWatcher extends Th
}
try
{
+ scanner.close();
+ }
+ catch (IOException e)
+ {
+ // Ignore
+ }
+ try
+ {
join(10000);
}
catch (InterruptedException ie)
Modified: felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Scanner.java
URL: http://svn.apache.org/viewvc/felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Scanner.java?rev=1583362&r1=1583361&r2=1583362&view=diff
==============================================================================
--- felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Scanner.java (original)
+++ felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Scanner.java Mon Mar 31 16:12:24 2014
@@ -18,6 +18,7 @@
*/
package org.apache.felix.fileinstall.internal;
+import java.io.Closeable;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -25,6 +26,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Pattern;
import java.util.zip.CRC32;
/**
@@ -40,7 +42,7 @@ import java.util.zip.CRC32;
* the change on this file. This allows to not report the change until
* a big copy if complete for example.
*/
-public class Scanner {
+public class Scanner implements Closeable {
final File directory;
final FilenameFilter filter;
@@ -63,12 +65,26 @@ public class Scanner {
* Create a scanner for the specified directory and file filter
*
* @param directory the directory to scan
- * @param filter a filter for file names
+ * @param filterString a filter for file names
*/
- public Scanner(File directory, FilenameFilter filter)
+ public Scanner(File directory, final String filterString)
{
this.directory = canon(directory);
- this.filter = filter;
+ if (filterString != null && filterString.length() > 0)
+ {
+ this.filter = new FilenameFilter()
+ {
+ Pattern pattern = Pattern.compile(filterString);
+ public boolean accept(File dir, String name)
+ {
+ return pattern.matcher(name).matches();
+ }
+ };
+ }
+ else
+ {
+ this.filter = null;
+ }
}
/**
@@ -127,6 +143,9 @@ public class Scanner {
return files;
}
+ public void close() throws IOException {
+ }
+
private static File canon(File file)
{
try
Added: felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Watcher.java
URL: http://svn.apache.org/viewvc/felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Watcher.java?rev=1583362&view=auto
==============================================================================
--- felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Watcher.java (added)
+++ felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/Watcher.java Mon Mar 31 16:12:24 2014
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.felix.fileinstall.internal;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
+import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
+import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
+import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
+import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
+
+/**
+ * A File watching service
+ */
+public abstract class Watcher implements Closeable {
+
+ private Path root;
+ private boolean watch = true;
+ private WatchService watcher;
+ private PathMatcher dirMatcher;
+ private PathMatcher fileMatcher;
+ private final Map<WatchKey, Path> keys = new ConcurrentHashMap<WatchKey, Path>();
+ private volatile long lastModified;
+ private final Map<Path, Boolean> processedMap = new ConcurrentHashMap<Path, Boolean>();
+
+ public void init() throws IOException {
+ if (root == null) {
+ Iterable<Path> rootDirectories = getFileSystem().getRootDirectories();
+ for (Path rootDirectory : rootDirectories) {
+ if (rootDirectory != null) {
+ root = rootDirectory;
+ break;
+ }
+ }
+ }
+ if (!Files.exists(root)) {
+ fail("Root path does not exist: " + root);
+ } else if (!Files.isDirectory(root)) {
+ fail("Root path is not a directory: " + root);
+ }
+ if (watcher == null) {
+ watcher = watch ? getFileSystem().newWatchService() : null;
+ }
+ }
+
+ public void close() throws IOException {
+ if (watcher != null) {
+ watcher.close();
+ }
+ }
+
+ public long getLastModified() {
+ return lastModified;
+ }
+
+ // Properties
+ //-------------------------------------------------------------------------
+
+
+ public void setRootPath(String rootPath) {
+ Path path = new File(rootPath).getAbsoluteFile().toPath();
+ setRoot(path);
+ }
+
+ public void setRootDirectory(File directory) {
+ setRoot(directory.toPath());
+ }
+
+ public Path getRoot() {
+ return root;
+ }
+
+ public void setRoot(Path root) {
+ this.root = root;
+ }
+
+ public boolean isWatch() {
+ return watch;
+ }
+
+ public void setWatch(boolean watch) {
+ this.watch = watch;
+ }
+
+ public WatchService getWatcher() {
+ return watcher;
+ }
+
+ public void setWatcher(WatchService watcher) {
+ this.watcher = watcher;
+ }
+
+ public PathMatcher getDirMatcher() {
+ return dirMatcher;
+ }
+
+ public void setDirMatcher(PathMatcher dirMatcher) {
+ this.dirMatcher = dirMatcher;
+ }
+
+ public PathMatcher getFileMatcher() {
+ return fileMatcher;
+ }
+
+ public void setFileMatcher(PathMatcher fileMatcher) {
+ this.fileMatcher = fileMatcher;
+ }
+
+
+ // Implementation methods
+ //-------------------------------------------------------------------------
+
+ public void rescan() throws IOException {
+ for (WatchKey key : keys.keySet()) {
+ key.cancel();
+ }
+ keys.clear();
+ Files.walkFileTree(root, new FilteringFileVisitor());
+ }
+
+ public void processEvents() {
+ while (true) {
+ WatchKey key = watcher.poll();
+ if (key == null) {
+ break;
+ }
+ Path dir = keys.get(key);
+ if (dir == null) {
+ warn("Could not find key for " + key);
+ continue;
+ }
+
+ for (WatchEvent<?> event : key.pollEvents()) {
+ WatchEvent.Kind kind = event.kind();
+ WatchEvent<Path> ev = (WatchEvent<Path>)event;
+
+ // Context for directory entry event is the file name of entry
+ Path name = ev.context();
+ Path child = dir.resolve(name);
+
+ debug("Processing event {} on path {}", kind, child);
+
+ if (kind == OVERFLOW) {
+// rescan();
+ continue;
+ }
+
+ try {
+ if (kind == ENTRY_CREATE) {
+ if (Files.isDirectory(child, NOFOLLOW_LINKS)) {
+
+ // if directory is created, and watching recursively, then
+ // register it and its sub-directories
+ Files.walkFileTree(child, new FilteringFileVisitor());
+ } else if (Files.isRegularFile(child, NOFOLLOW_LINKS)) {
+ scan(child);
+ }
+ } else if (kind == ENTRY_MODIFY) {
+ if (Files.isRegularFile(child, NOFOLLOW_LINKS)) {
+ scan(child);
+ }
+ } else if (kind == ENTRY_DELETE) {
+ unscan(child);
+ }
+ } catch (IOException x) {
+ // ignore to keep sample readbale
+ x.printStackTrace();
+ }
+ }
+
+ // reset key and remove from set if directory no longer accessible
+ boolean valid = key.reset();
+ if (!valid) {
+ debug("Removing key " + key + " and dir " + dir + " from keys");
+ keys.remove(key);
+
+ // all directories are inaccessible
+ if (keys.isEmpty()) {
+ break;
+ }
+ }
+ }
+ }
+
+ private void scan(final Path file) throws IOException {
+ if (isMatchesFile(file)) {
+ process(file);
+ processedMap.put(file, Boolean.TRUE);
+ }
+ }
+
+ protected boolean isMatchesFile(Path file) {
+ boolean matches = true;
+ if (fileMatcher != null) {
+ Path rel = root.relativize(file);
+ matches = fileMatcher.matches(rel);
+ }
+ return matches;
+ }
+
+ private void unscan(final Path file) throws IOException {
+ if (isMatchesFile(file)) {
+ onRemove(file);
+ lastModified = System.currentTimeMillis();
+ } else {
+ // lets find all the files that now no longer exist
+ List<Path> files = new ArrayList<Path>(processedMap.keySet());
+ for (Path path : files) {
+ if (!Files.exists(path)) {
+ debug("File has been deleted: " + path);
+ processedMap.remove(path);
+ if (isMatchesFile(path)) {
+ onRemove(file);
+ lastModified = System.currentTimeMillis();
+ }
+ }
+ }
+ }
+ }
+
+ private void watch(final Path path) throws IOException {
+ if (watcher != null) {
+ WatchKey key = path.register(watcher, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_DELETE);
+ keys.put(key, path);
+ debug("Watched path " + path + " key " + key);
+ } else {
+ warn("No watcher yet for path " + path);
+ }
+ }
+
+ protected FileSystem getFileSystem() {
+ return FileSystems.getDefault();
+ }
+
+ public class FilteringFileVisitor implements FileVisitor<Path> {
+
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ if (Thread.interrupted()) {
+ throw new InterruptedIOException();
+ }
+ if (dirMatcher != null) {
+ Path rel = root.relativize(dir);
+ if (!"".equals(rel.toString()) && !dirMatcher.matches(rel)) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ }
+ watch(dir);
+ return FileVisitResult.CONTINUE;
+ }
+
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ if (Thread.interrupted()) {
+ throw new InterruptedIOException();
+ }
+ scan(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+ return FileVisitResult.CONTINUE;
+ }
+
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+ return FileVisitResult.CONTINUE;
+ }
+ }
+
+
+ /**
+ * Throws an invalid argument exception after logging a warning
+ * just in case the stack trace gets gobbled up by application containers
+ * like spring or blueprint, at least the error message will be clearly shown in the log
+ *
+ */
+ public void fail(String message) {
+ warn(message);
+ throw new IllegalArgumentException(message);
+ }
+
+ protected abstract void debug(String message, Object... args);
+ protected abstract void warn(String message, Object... args);
+ protected abstract void process(Path path);
+ protected abstract void onRemove(Path path);
+}
Added: felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/WatcherScanner.java
URL: http://svn.apache.org/viewvc/felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/WatcherScanner.java?rev=1583362&view=auto
==============================================================================
--- felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/WatcherScanner.java (added)
+++ felix/trunk/fileinstall/src/main/java/org/apache/felix/fileinstall/internal/WatcherScanner.java Mon Mar 31 16:12:24 2014
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.felix.fileinstall.internal;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.osgi.framework.BundleContext;
+
+public class WatcherScanner extends Scanner {
+
+ BundleContext bundleContext;
+ PathMatcher fileMatcher;
+ Watcher watcher;
+
+ Set<File> changed = new HashSet<File>();
+
+ /**
+ * Create a scanner for the specified directory and file filter
+ *
+ * @param directory the directory to scan
+ * @param filterString a filter for file names
+ */
+ public WatcherScanner(BundleContext bundleContext, File directory, String filterString) throws IOException {
+ super(directory, filterString);
+ this.bundleContext = bundleContext;
+ if (filterString != null) {
+ this.fileMatcher = FileSystems.getDefault().getPathMatcher("regex:" + filterString);
+ } else {
+ this.fileMatcher = null;
+ }
+ this.watcher = new ScannerWatcher();
+ this.watcher.setFileMatcher(fileMatcher);
+ this.watcher.setRootDirectory(this.directory);
+ this.watcher.init();
+ this.watcher.rescan();
+ }
+
+ public Set<File> scan(boolean reportImmediately) {
+ watcher.processEvents();
+ if (changed.isEmpty()) {
+ return new HashSet<File>();
+ }
+ Set<File> files = new HashSet<File>();
+ Set<File> removed = new HashSet<File>();
+ if (reportImmediately) {
+ removed.addAll(storedChecksums.keySet());
+ }
+ for (File file : changed)
+ {
+ long lastChecksum = lastChecksums.get(file) != null ? (Long) lastChecksums.get(file) : 0;
+ long storedChecksum = storedChecksums.get(file) != null ? (Long) storedChecksums.get(file) : 0;
+ long newChecksum = checksum(file);
+ lastChecksums.put(file, newChecksum);
+ if (file.exists()) {
+ // Only handle file when it does not change anymore and it has changed since last reported
+ if ((newChecksum == lastChecksum || reportImmediately)) {
+ if (newChecksum != storedChecksum) {
+ storedChecksums.put(file, newChecksum);
+ files.add(file);
+ } else {
+ changed.remove(file);
+ }
+ if (reportImmediately) {
+ removed.remove(file);
+ }
+ }
+ } else {
+ if (!reportImmediately) {
+ removed.add(file);
+ }
+ }
+ }
+ for (File file : removed)
+ {
+ // Make sure we'll handle a file that has been deleted
+ files.addAll(removed);
+ // Remove no longer used checksums
+ lastChecksums.remove(file);
+ storedChecksums.remove(file);
+ }
+ for (File file : files)
+ {
+ changed.remove(file);
+ }
+
+ return files;
+ }
+
+ public void close() throws IOException {
+ watcher.close();
+ }
+
+ class ScannerWatcher extends Watcher {
+
+ @Override
+ protected void process(Path path) {
+ File file = path.toFile();
+ while (!file.getParentFile().equals(directory)) {
+ file = file.getParentFile();
+ if (file == null) {
+ return;
+ }
+ }
+ changed.add(file);
+ }
+
+ @Override
+ protected void onRemove(Path path) {
+ File file = path.toFile();
+ while (!file.getParentFile().equals(directory)) {
+ file = file.getParentFile();
+ if (file == null) {
+ return;
+ }
+ }
+ changed.add(file);
+ }
+
+ @Override
+ protected void debug(String message, Object... args) {
+ log(Util.Logger.LOG_DEBUG, message, args);
+ }
+
+ @Override
+ protected void warn(String message, Object... args) {
+ log(Util.Logger.LOG_WARNING, message, args);
+ }
+
+ protected void log(int level, String message, Object... args) {
+ String msg = String.format(message, args);
+ Util.log(bundleContext, Util.getGlobalLogLevel(bundleContext),
+ level, msg, null);
+ }
+
+ }
+
+}