You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@netbeans.apache.org by ma...@apache.org on 2021/12/14 19:10:58 UTC
[netbeans] branch master updated: Allow plugins to exclude files from being indexed
This is an automated email from the ASF dual-hosted git repository.
matthiasblaesing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push:
new 81dc21a Allow plugins to exclude files from being indexed
new e991e94 Merge pull request #3209 from matthiasblaesing/indexability_query
81dc21a is described below
commit 81dc21acf30eec2ba7d566e62f4df48db27df9f0
Author: Matthias Bläsing <mb...@doppel-helix.eu>
AuthorDate: Fri Sep 24 19:50:41 2021 +0200
Allow plugins to exclude files from being indexed
This commit adds a query, that allows plugins to prevent files from
being indexed completely or by certain indexers.
One motivator for this is the JS indexer in combination with huge
node_modules directories. In the case of angular/typescript projects the
JS indexer does not help with development, as code assistence is
provided by the typescript LSP server. But the JS indexer is ran anyway
and created huge scan times and big indices. With these new hooks a
plugin could limit the indexing when an angular project is detected.
---
ide/parsing.indexing/apichanges.xml | 28 +
.../parsing/impl/indexing/FileObjectCrawler.java | 39 +-
.../parsing/impl/indexing/FilteringIterable.java | 84 ++
.../parsing/impl/indexing/IndexabilityQuery.java | 194 ++++
.../indexing/IndexabilityQueryContextAccessor.java | 56 ++
...bilitySupport.java => IndexabilitySupport.java} | 120 ++-
.../parsing/impl/indexing/RepositoryUpdater.java | 1009 ++++++++++----------
.../modules/parsing/impl/indexing/TimeStamps.java | 135 +--
.../indexing/IndexabilityQueryImplementation.java | 146 +++
.../impl/indexing/RepositoryUpdaterTest.java | 172 +++-
10 files changed, 1360 insertions(+), 623 deletions(-)
diff --git a/ide/parsing.indexing/apichanges.xml b/ide/parsing.indexing/apichanges.xml
index ee59966..b7fa8c0 100644
--- a/ide/parsing.indexing/apichanges.xml
+++ b/ide/parsing.indexing/apichanges.xml
@@ -87,6 +87,34 @@ is the proper place.
<!-- ACTUAL CHANGES BEGIN HERE: -->
<changes>
+ <change id="IndexabilityQuery">
+ <api name="IndexingAPI"/>
+ <summary>Allow plugins to exclude files from being indexed.</summary>
+ <version major="9" minor="24"/>
+ <date day="14" month="9" year="2021"/>
+ <author login="matthiasblaesing"/>
+ <compatibility source="compatible" binary="compatible" semantic="compatible" addition="yes"/>
+ <description>
+ <p>
+ There are situations where it is desireable to exclude files from the NetBeans indexing infrastructure. Implementors of
+ <a href="@TOP@/org/netbeans/modules/parsing/spi/indexing/IndexabilityQueryImplementation.html">IndexabilityQueryImplementation</a>
+ are queried whether an indexer should be invoked for a given file.
+ </p>
+ <p>
+ Indexing for a file can be rejected verbatim (<code>boolean preventIndexing(FileObject fo)</code>) or more specific by
+ indexer name, URL to the file to be indexed and the UTL of the indexing root.
+ <code>boolean preventIndexing(String indexerName, URL indexable, URL rootUrl)</code>.
+ </p>
+ <p>
+ One such example are Angular projects, where code assistence is provided by the typescript integration by the
+ language server. In these cases the <code>node_modules</code> folder often contains huge amounts of javascript
+ code, that the IDE user does not need to be indexed, but which take a lot of time to parse. A hypothetical
+ plugin could check if an <code>angular.json</code> file is present and then prevent the <code>js</code> indexer
+ from indexing files in the <code>node_modules</code> directory.
+ </p>
+ </description>
+ <class package="org.netbeans.modules.parsing.spi.indexing" name="IndexabilityQueryImplementation"/>
+ </change>
<change id="IndexingTask">
<api name="IndexingAPI"/>
<summary>Allow Parsers to adjust results for indexing</summary>
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FileObjectCrawler.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FileObjectCrawler.java
index 6375450..945c6ec 100644
--- a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FileObjectCrawler.java
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FileObjectCrawler.java
@@ -20,10 +20,7 @@
package org.netbeans.modules.parsing.impl.indexing;
import org.netbeans.modules.parsing.spi.indexing.SuspendStatus;
-import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Comparator;
@@ -54,12 +51,13 @@ final class FileObjectCrawler extends Crawler {
private static final char SEPARATOR = '/'; //NOI18N
private static final Logger LOG = Logger.getLogger(FileObjectCrawler.class.getName());
- /*test*/ static Map<Pair<FileObject,FileObject>,Boolean> mockLinkTypes;
+ @SuppressWarnings("PackageVisibleField") /*test*/
+ static Map<Pair<FileObject,FileObject>,Boolean> mockLinkTypes;
private final FileObject root;
private final ClassPath.Entry entry;
private final FileObject[] files;
-
+
FileObjectCrawler(
@NonNull final FileObject root,
@@ -176,7 +174,7 @@ NEXT_FILE: for(FileObject f : files) {
StringBuilder relativePath = relPaths.get(parent);
if (relativePath != null) {
finished = collect(
- cluster.toArray(new FileObject[cluster.size()]),
+ cluster.toArray(new FileObject[0]),
root,
resources,
allResources,
@@ -273,7 +271,7 @@ NEXT_FILE: for(FileObject f : files) {
if (isCancelled()) {
return false;
}
- if (!fo.isValid() || !isVisible(fo)) {
+ if (!fo.isValid() || !canBeIndexed(fo)) {
continue;
}
@@ -330,14 +328,15 @@ NEXT_FILE: for(FileObject f : files) {
}
}
- private boolean isVisible (final @NonNull FileObject fo) {
+ private boolean canBeIndexed (final @NonNull FileObject fo) {
try {
- return VisibilityQuery.getDefault().isVisible(fo);
+ return VisibilityQuery.getDefault().isVisible(fo)
+ && (! IndexabilityQuery.getInstance().preventIndexing(fo));
} finally {
setListenOnVisibility(true);
}
}
-
+
//Todo: Not exaclty correct. The correct implementation should find if whole root content
//is covered by files. But correct implementation will be very very slow and probably no one
//calls it with such params.
@@ -376,7 +375,7 @@ NEXT_FILE: for(FileObject f : files) {
if (file.getNameExt().equals(pathElement.getNameExt())) {
try {
if (mockLinkTypes != null ?
- mockLinkTypes.get(Pair.<FileObject,FileObject>of(pathElement, file)) :
+ mockLinkTypes.get(Pair.of(pathElement, file)) :
isSameFile(file, pathElement)) {
hasLink = true;
break;
@@ -438,28 +437,28 @@ NEXT_FILE: for(FileObject f : files) {
return PathRelation.EQUAL;
}
}
-
+
private static final class Stats {
public int filesCount;
public long linkCheckTime;
public int linkCount;
- public Map<String, Integer> extensions = new HashMap<String, Integer>();
- public Map<String, Integer> mimeTypes = new HashMap<String, Integer>();
+ public Map<String, Integer> extensions = new HashMap<>();
+ public Map<String, Integer> mimeTypes = new HashMap<>();
public static void inc(Map<String, Integer> m, String k) {
Integer i = m.get(k);
if (i == null) {
m.put(k, 1);
} else {
- m.put(k, i.intValue() + 1);
+ m.put(k, i + 1);
}
}
public static void logHistogram(Level level, Map<String, Integer> data) {
- Map<Integer, Set<String>> sortedMap = new TreeMap<Integer, Set<String>>(REVERSE);
+ Map<Integer, Set<String>> sortedMap = new TreeMap<>(REVERSE);
for(String item : data.keySet()) {
Integer freq = data.get(item);
Set<String> items = sortedMap.get(freq);
if (items == null) {
- items = new TreeSet<String>();
+ items = new TreeSet<>();
sortedMap.put(freq, items);
}
items.add(item);
@@ -470,11 +469,7 @@ NEXT_FILE: for(FileObject f : files) {
}
}
}
- private static final Comparator<Integer> REVERSE = new Comparator<Integer>() {
- public int compare(Integer o1, Integer o2) {
- return -1 * o1.compareTo(o2);
- }
- };
+ private static final Comparator<Integer> REVERSE = (Integer o1, Integer o2) -> -1 * o1.compareTo(o2);
} // End of Stats class
private enum PathRelation {
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FilteringIterable.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FilteringIterable.java
new file mode 100644
index 0000000..8a7f181
--- /dev/null
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/FilteringIterable.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.modules.parsing.impl.indexing;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.function.Function;
+
+/**
+ * Iterable wrapping another iterable, yielding an interator, that only offers
+ * values, for this the supplied filter returns a true value.
+ */
+class FilteringIterable<T> implements Iterable<T> {
+ private final Iterable<? extends T> delegate;
+ private final Function<T,Boolean> filter;
+
+ public FilteringIterable(Iterable<? extends T> delegate, Function<T,Boolean> filter) {
+ this.delegate = delegate;
+ this.filter = filter;
+ }
+
+ @Override
+ public Iterator<T> iterator() {
+ return new FilteringIterator(delegate.iterator(), filter);
+ }
+
+ private static class FilteringIterator<T> implements Iterator<T> {
+ private final Iterator<? extends T> delegate;
+ private final Function<T,Boolean> filter;
+
+ public FilteringIterator(Iterator<? extends T> delegate, Function<T,Boolean> filter) {
+ this.delegate = delegate;
+ this.filter = filter;
+ }
+
+ private T next;
+
+ @Override
+ public boolean hasNext() {
+ fillNextIfPossible();
+ return next != null;
+ }
+
+ @Override
+ public T next() {
+ fillNextIfPossible();
+ if(next == null) {
+ throw new NoSuchElementException();
+ }
+ T value = next;
+ next = null;
+ return value;
+ }
+
+ private void fillNextIfPossible() {
+ if (next == null) {
+ while (delegate.hasNext()) {
+ T nextCandiate = delegate.next();
+ if(filter.apply(nextCandiate)) {
+ next = nextCandiate;
+ break;
+ }
+ }
+ }
+ }
+ }
+}
+
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQuery.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQuery.java
new file mode 100644
index 0000000..1711da0
--- /dev/null
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQuery.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.netbeans.modules.parsing.impl.indexing;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.swing.event.ChangeEvent;
+import javax.swing.event.ChangeListener;
+import org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation;
+import org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation.IndexabilityQueryContext;
+import org.openide.filesystems.FileObject;
+import org.openide.util.Exceptions;
+import org.openide.util.Lookup;
+import org.openide.util.LookupEvent;
+import org.openide.util.LookupListener;
+
+/**
+ * Determine whether files should be skipped by the netbeans indexing infrastructure.
+ *
+ * @see org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation
+ */
+final class IndexabilityQuery {
+ private static final Logger LOG = Logger.getLogger(IndexabilityQuery.class.getName());
+
+ private static final IndexabilityQueryContextAccessor CONTEXT_CREATOR = IndexabilityQueryContextAccessor.getInstance();
+ private static final IndexabilityQuery INSTANCE = new IndexabilityQuery();
+
+ private final ResultListener resultListener = new ResultListener();
+ private final IqiChangedListener vqiListener = new IqiChangedListener();
+
+ private final List<ChangeListener> listeners = new ArrayList<>();
+ private volatile List<IndexabilityQueryImplementation> cachedIqiInstances = null;
+ private Lookup.Result<IndexabilityQueryImplementation> iqiResult = null;
+
+ /**
+ * Get instance of IndexabilityQuery.
+ * @return instance of IndexabilityQuery
+ */
+ public static final IndexabilityQuery getInstance() {
+ return INSTANCE;
+ }
+
+ private IndexabilityQuery() {
+ }
+
+ public boolean preventIndexing(FileObject fileObject) {
+ IndexabilityQueryContext iqc = CONTEXT_CREATOR.createContext(fileObject.toURL(), null, null);
+ for (IndexabilityQueryImplementation iqi : getIqiInstances()) {
+ if (iqi.preventIndexing(iqc)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean preventIndexing (String indexerName, URL indexable, URL rootUrl) {
+ IndexabilityQueryContext iqc = CONTEXT_CREATOR.createContext(indexable, indexerName, rootUrl);
+ for (IndexabilityQueryImplementation iqi : getIqiInstances()) {
+ if (iqi.preventIndexing(iqc)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public String getState() {
+ return getIqiInstances()
+ .stream()
+ .map(iqi -> iqi.getName() + "-" + iqi.getVersion() + "-" + iqi.getStateIdentifier())
+ .collect(Collectors.joining(","));
+ }
+
+ private Set<String> decodeState(String input) {
+ try {
+ return Arrays.stream(input.split(","))
+ .map(s -> s.trim())
+ .collect(Collectors.toSet());
+ } catch (Exception ex) {
+ LOG.log(Level.SEVERE, "Failed to parse IndexabilityQuery state from '" + input + "'", ex);
+ return Collections.EMPTY_SET;
+ }
+ }
+
+ public boolean isSameState(String reference) {
+ return Objects.equals(decodeState(reference), decodeState(getState()));
+ }
+
+ /**
+ * Add a listener to changes.
+ * @param l a listener to add
+ */
+ public void addChangeListener(ChangeListener l) {
+ synchronized (listeners) {
+ listeners.add(l);
+ }
+ }
+
+ /**
+ * Stop listening to changes.
+ * @param l a listener to remove
+ */
+ public void removeChangeListener(ChangeListener l) {
+ synchronized (listeners) {
+ listeners.remove(l);
+ }
+ }
+
+ private void fireChange(ChangeEvent event) {
+ assert event != null;
+ ArrayList<ChangeListener> lists;
+ synchronized (listeners) {
+ lists = new ArrayList<>(listeners);
+ }
+ for (ChangeListener listener : lists) {
+ try {
+ listener.stateChanged(event);
+ } catch (RuntimeException x) {
+ Exceptions.printStackTrace(x);
+ }
+ }
+ }
+
+ @SuppressWarnings("ReturnOfCollectionOrArrayField")
+ private synchronized List<IndexabilityQueryImplementation> getIqiInstances() {
+ if (cachedIqiInstances == null) {
+ iqiResult = Lookup.getDefault().lookupResult(IndexabilityQueryImplementation.class);
+ iqiResult.addLookupListener(resultListener);
+ setupChangeListeners(null, new ArrayList<>(iqiResult.allInstances()));
+ }
+ return cachedIqiInstances;
+ }
+
+ @SuppressWarnings("AssignmentToCollectionOrArrayFieldFromParameter")
+ private synchronized void setupChangeListeners(final List<IndexabilityQueryImplementation> oldVqiInstances, final List<IndexabilityQueryImplementation> newVqiInstances) {
+ if (oldVqiInstances != null) {
+ Set<IndexabilityQueryImplementation> removed = new HashSet<>(oldVqiInstances);
+ removed.removeAll(newVqiInstances);
+ for (IndexabilityQueryImplementation vqi : removed) {
+ vqi.removeChangeListener(vqiListener);
+ }
+ }
+
+ Set<IndexabilityQueryImplementation> added = new HashSet<>(newVqiInstances);
+ if (oldVqiInstances != null) {
+ added.removeAll(oldVqiInstances);
+ }
+ for (IndexabilityQueryImplementation vqi : added) {
+ vqi.addChangeListener(vqiListener);
+ }
+
+ cachedIqiInstances = newVqiInstances;
+ }
+
+ private class ResultListener implements LookupListener {
+ @Override
+ public void resultChanged(LookupEvent ev) {
+ setupChangeListeners(cachedIqiInstances, new ArrayList<>(iqiResult.allInstances()));
+ fireChange(new ChangeEvent(IndexabilityQuery.this));
+ }
+ }
+
+ private class IqiChangedListener implements ChangeListener {
+ @Override
+ public void stateChanged(ChangeEvent e) {
+ fireChange(e);
+ }
+ }
+}
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQueryContextAccessor.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQueryContextAccessor.java
new file mode 100644
index 0000000..5e41276
--- /dev/null
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilityQueryContextAccessor.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.modules.parsing.impl.indexing;
+
+import java.net.URL;
+import org.netbeans.api.annotations.common.NonNull;
+import org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation;
+import org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation.IndexabilityQueryContext;
+import org.openide.util.Exceptions;
+
+public abstract class IndexabilityQueryContextAccessor {
+
+ private static volatile IndexabilityQueryContextAccessor instance;
+
+ public static synchronized IndexabilityQueryContextAccessor getInstance() {
+ if (instance == null) {
+ try {
+ Class.forName(
+ IndexabilityQueryImplementation.IndexabilityQueryContext.class.getName(),
+ true,
+ IndexabilityQueryContextAccessor.class.getClassLoader());
+ assert instance != null;
+ } catch (ClassNotFoundException e) {
+ Exceptions.printStackTrace(e);
+ }
+ }
+ return instance;
+ }
+
+ public static void setInstance(@NonNull final IndexabilityQueryContextAccessor _instance) {
+ assert _instance != null;
+ instance = _instance;
+ }
+
+ public abstract IndexabilityQueryContext createContext(
+ URL indexable,
+ String indexerName,
+ URL root
+ );
+}
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/VisibilitySupport.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilitySupport.java
similarity index 76%
rename from ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/VisibilitySupport.java
rename to ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilitySupport.java
index 5ae5d9f..e9d7c74 100644
--- a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/VisibilitySupport.java
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/IndexabilitySupport.java
@@ -34,7 +34,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.event.ChangeEvent;
@@ -54,32 +53,58 @@ import org.openide.util.RequestProcessor;
*
* @author Tomas Zezula
*/
-class VisibilitySupport implements ChangeListener {
+class IndexabilitySupport {
private static final int VISIBILITY_CHANGE_WINDOW = 1000;
- private static final Logger LOGGER = Logger.getLogger(VisibilitySupport.class.getName());
-
+ private static final Logger LOGGER = Logger.getLogger(IndexabilitySupport.class.getName());
+
//@GuardedBy("visibilityCache")
- private final Map<FileObject,Boolean> visibilityCache = Collections.synchronizedMap(new WeakHashMap<FileObject, Boolean>());
+ private final Map<FileObject,Boolean> visibilityCache = Collections.synchronizedMap(new WeakHashMap<>());
private final SlidingTask visibilityTask;
private final RequestProcessor.Task visibilityChanged;
-
- private VisibilitySupport(
+
+ private final ChangeListener visibilityListener;
+ private final ChangeListener indexabilityListener;
+
+ private IndexabilitySupport(
@NonNull final RepositoryUpdater ru,
@NonNull final RequestProcessor worker) {
this.visibilityTask = new SlidingTask(ru);
this.visibilityChanged = worker.create(this.visibilityTask);
+
+ visibilityListener = (ChangeEvent e) -> {
+ visibilityCache.clear();
+ if (Crawler.listenOnVisibility()) {
+ if (e instanceof VisibilityQueryChangeEvent) {
+ final FileObject[] affectedFiles = ((VisibilityQueryChangeEvent) e).getFileObjects();
+ visibilityTask.localChange(affectedFiles);
+ } else {
+ visibilityTask.globalChange();
+ }
+ visibilityChanged.schedule(VISIBILITY_CHANGE_WINDOW);
+ }
+ };
+
+ indexabilityListener = (ChangeEvent e) -> {
+ // Indexability could invalidate a subset of the indexer,
+ // so there is a valid index state for a file, but it might
+ // contain more/less data than intended after the change
+ visibilityTask.globalChangeFull();
+ visibilityChanged.schedule(VISIBILITY_CHANGE_WINDOW);
+ };
}
void start() {
- VisibilityQuery.getDefault().addChangeListener(this);
+ VisibilityQuery.getDefault().addChangeListener(visibilityListener);
+ IndexabilityQuery.getInstance().addChangeListener(indexabilityListener);
}
-
+
void stop() {
- VisibilityQuery.getDefault().removeChangeListener(this);
+ VisibilityQuery.getDefault().removeChangeListener(visibilityListener);
+ IndexabilityQuery.getInstance().removeChangeListener(indexabilityListener);
}
-
- boolean isVisible(
+
+ boolean canIndex(
@NonNull FileObject file,
@NullAllowed final FileObject root) {
long st = 0L;
@@ -88,7 +113,8 @@ class VisibilitySupport implements ChangeListener {
}
try {
final VisibilityQuery vq = VisibilityQuery.getDefault();
- final Deque<FileObject> fta = new ArrayDeque<FileObject>();
+ final IndexabilityQuery iq = IndexabilityQuery.getInstance();
+ final Deque<FileObject> fta = new ArrayDeque<>();
Boolean vote = null;
boolean folder = false;
while (root != null && !root.equals(file)) {
@@ -99,7 +125,7 @@ class VisibilitySupport implements ChangeListener {
if (folder || file.isFolder()) {
fta.offer(file);
}
- if (!vq.isVisible(file)) {
+ if ((!vq.isVisible(file)) || iq.preventIndexing(file)) {
vote = Boolean.FALSE;
break;
}
@@ -107,7 +133,7 @@ class VisibilitySupport implements ChangeListener {
folder = true;
}
if (vote == null) {
- vote = vq.isVisible(file);
+ vote = vq.isVisible(file) && (! iq.preventIndexing(file));
fta.offer(file);
}
if (!fta.isEmpty()) {
@@ -128,27 +154,14 @@ class VisibilitySupport implements ChangeListener {
}
}
- @Override
- public void stateChanged(ChangeEvent e) {
- visibilityCache.clear();
- if (Crawler.listenOnVisibility()) {
- if (e instanceof VisibilityQueryChangeEvent) {
- final FileObject[] affectedFiles = ((VisibilityQueryChangeEvent)e).getFileObjects();
- visibilityTask.localChange(affectedFiles);
- } else {
- visibilityTask.globalChange();
- }
- visibilityChanged.schedule(VISIBILITY_CHANGE_WINDOW);
- }
- }
-
private static class SlidingTask implements Runnable {
private final RepositoryUpdater ru;
//@GuardedBy("this")
private boolean globalChange;
- private final Set</*@GuardedBy("this")*/FileObject> localChanges = new HashSet<FileObject>();
+ private boolean fullScan;
+ private final Set</*@GuardedBy("this")*/FileObject> localChanges = new HashSet<>();
//@GuardedBy("this")
private LogContext visibilityLogCtx;
@@ -156,6 +169,15 @@ class VisibilitySupport implements ChangeListener {
this.ru = ru;
}
+ synchronized void globalChangeFull() {
+ globalChange = true;
+ fullScan = true;
+
+ if (visibilityLogCtx == null) {
+ visibilityLogCtx = LogContext.create(LogContext.EventType.FILE, null);
+ }
+ }
+
synchronized void globalChange() {
globalChange = true;
@@ -174,19 +196,26 @@ class VisibilitySupport implements ChangeListener {
@Override
public void run() {
final boolean global;
+ final boolean full;
final Collection<FileObject> changedFiles;
final LogContext logCtx;
synchronized (this) {
logCtx = visibilityLogCtx;
visibilityLogCtx = null;
global = globalChange;
+ full = fullScan;
globalChange = false;
- changedFiles = new ArrayList<FileObject>(localChanges);
+ fullScan = false;
+ changedFiles = new ArrayList<>(localChanges);
localChanges.clear();
}
if (global) {
LOGGER.fine ("VisibilityQuery global changed, reindexing"); //NOI18N
- ru.refreshAll(false, false, true, logCtx);
+ if(full) {
+ ru.refreshAll(true, false, true, logCtx);
+ } else {
+ ru.refreshAll(false, false, true, logCtx);
+ }
} else {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.log (
@@ -194,24 +223,23 @@ class VisibilitySupport implements ChangeListener {
"VisibilityQuery changed for {0}, reindexing these files.", //NOI18N
Arrays.asList(changedFiles));
}
- final Map<URI, Collection<URL>> srcShownPerRoot =
- new HashMap<URI, Collection<URL>>();
- final Map<URI, Set<String>> srcHiddenPerRoot =
- new HashMap<URI, Set<String>>();
- final Set<URI> binChangedRoot =
- new HashSet<URI>();
- final Map<URI,TimeStamps> tsPerRoot = new HashMap<URI, TimeStamps>();
+ final Map<URI, Collection<URL>> srcShownPerRoot = new HashMap<>();
+ final Map<URI, Set<String>> srcHiddenPerRoot = new HashMap<>();
+ final Set<URI> binChangedRoot = new HashSet<>();
+ final Map<URI,TimeStamps> tsPerRoot = new HashMap<>();
final VisibilityQuery vq = VisibilityQuery.getDefault();
+ final IndexabilityQuery iq = IndexabilityQuery.getInstance();
for (FileObject chf : changedFiles) {
Pair<URL,FileObject> owner = ru.getOwningSourceRoot(chf);
if (owner != null) {
- final boolean visible = vq.isVisible(chf);
+ final boolean visible = vq.isVisible(chf)
+ && (! iq.preventIndexing(chf));
try {
final URI ownerURI = owner.first().toURI();
if (visible) {
Collection<URL> files = srcShownPerRoot.get(ownerURI);
if (files == null) {
- files = new ArrayList<URL>();
+ files = new ArrayList<>();
srcShownPerRoot.put(ownerURI, files);
}
if (chf.equals(owner.second())) {
@@ -224,7 +252,7 @@ class VisibilitySupport implements ChangeListener {
} else if (owner.second() != null) {
Set<String> files = srcHiddenPerRoot.get(ownerURI);
if (files == null) {
- files = new HashSet<String>();
+ files = new HashSet<>();
srcHiddenPerRoot.put(ownerURI, files);
}
if (chf.isFolder()) {
@@ -238,9 +266,7 @@ class VisibilitySupport implements ChangeListener {
files.add(FileUtil.getRelativePath(owner.second(), chf));
}
}
- } catch (URISyntaxException e) {
- Exceptions.printStackTrace(e);
- } catch (IOException e) {
+ } catch (URISyntaxException | IOException e) {
Exceptions.printStackTrace(e);
}
continue;
@@ -294,12 +320,12 @@ class VisibilitySupport implements ChangeListener {
}
@NonNull
- static VisibilitySupport create(
+ static IndexabilitySupport create(
@NonNull final RepositoryUpdater ru,
@NonNull final RequestProcessor worker) {
Parameters.notNull("ru", ru); //NOI18N
Parameters.notNull("worker", worker); //NOI18N
- return new VisibilitySupport(ru, worker);
+ return new IndexabilitySupport(ru, worker);
}
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdater.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdater.java
index beab9cd..f1f1a07 100644
--- a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdater.java
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdater.java
@@ -25,7 +25,6 @@ import java.io.*;
import java.lang.ref.Reference;
import java.lang.ref.WeakReference;
import java.net.MalformedURLException;
-import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.InvalidParameterException;
@@ -39,12 +38,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import java.util.prefs.Preferences;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.swing.event.DocumentEvent;
@@ -104,7 +103,6 @@ import org.openide.util.BaseUtilities;
import org.openide.util.Cancellable;
import org.openide.util.Exceptions;
import org.openide.util.Lookup;
-import org.openide.util.Mutex.ExceptionAction;
import org.openide.util.NbBundle;
import org.openide.util.NbPreferences;
import org.openide.util.Pair;
@@ -287,21 +285,18 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
final long timeout,
final boolean relaxProtectedMode) throws InterruptedException {
try {
- final Callable<Boolean> call = new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- long ts1 = System.currentTimeMillis();
- long ts2 = ts1;
- //long tout = timeout > 0 ? timeout : 1000;
- do {
- boolean timedOut = !worker.waitUntilFinished(timeout);
- ts2 = System.currentTimeMillis();
- if (timedOut) {
- return false;
- }
- } while (!getIndexingState().isEmpty() && (timeout <= 0 || ts2 - ts1 < timeout));
- return timeout <= 0 || ts2 - ts1 < timeout;
- }
+ final Callable<Boolean> call = () -> {
+ long ts1 = System.currentTimeMillis();
+ long ts2 = ts1;
+
+ do {
+ boolean timedOut = !worker.waitUntilFinished(timeout);
+ ts2 = System.currentTimeMillis();
+ if (timedOut) {
+ return false;
+ }
+ } while (!getIndexingState().isEmpty() && (timeout <= 0 || ts2 - ts1 < timeout));
+ return timeout <= 0 || ts2 - ts1 < timeout;
};
return relaxProtectedMode ?
worker.runOffProtecedMode(call) :
@@ -438,7 +433,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
FileListWork flw = null;
- if (fileUrls != null && fileUrls.size() > 0) {
+ if (fileUrls != null && !fileUrls.isEmpty()) {
Set<FileObject> files = new HashSet<>();
for(URL fileUrl : fileUrls) {
FileObject file = URLMapper.findFileObject(fileUrl);
@@ -453,7 +448,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
}
- if (files.size() > 0) {
+ if (!files.isEmpty()) {
flw = new FileListWork(
scannedRoots2Dependencies,
rootUrl,
@@ -563,7 +558,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
public void suspend() {
- if (notInterruptible) {
+ if (NOT_INTERRUPTIBLE) {
// ignore the request
return;
}
@@ -571,7 +566,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
public void resume() {
- if (notInterruptible) {
+ if (NOT_INTERRUPTIBLE) {
// ignore the request
return;
}
@@ -692,7 +687,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (fo != null && fo.isValid()) {
if (source == null || source) {
root = getOwningSourceRoot(fo);
- if (root != null && visibilitySupport.isVisible(fo, root.second())) {
+ if (root != null && visibilitySupport.canIndex(fo, root.second())) {
if (root.second() == null) {
LOGGER.log(
Level.INFO,
@@ -767,7 +762,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (!processed && (source == null || !source)) {
root = getOwningBinaryRoot(fo);
- if (root != null && visibilitySupport.isVisible(fo, root.second())) {
+ if (root != null && visibilitySupport.canIndex(fo, root.second())) {
final Work wrk = new BinaryWork(
root.first(),
suspendSupport.getSuspendStatus(),
@@ -808,7 +803,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (fo != null && fo.isValid()) {
if (source == null || source) {
root = getOwningSourceRoot (fo);
- if (root != null && visibilitySupport.isVisible(fo,root.second())) {
+ if (root != null && visibilitySupport.canIndex(fo,root.second())) {
if (root.second() == null) {
LOGGER.log(
Level.INFO,
@@ -840,7 +835,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (!processed && (source == null || !source)) {
root = getOwningBinaryRoot(fo);
- if (root != null && visibilitySupport.isVisible(fo,root.second())) {
+ if (root != null && visibilitySupport.canIndex(fo,root.second())) {
final Work wrk = new BinaryWork(
root.first(),
suspendSupport.getSuspendStatus(),
@@ -881,7 +876,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (fo != null) {
if (source == null || source) {
root = getOwningSourceRoot (fo);
- if (root != null && fo.isData() && visibilitySupport.isVisible(fo, root.second())) {
+ if (root != null && fo.isData() && visibilitySupport.canIndex(fo, root.second())) {
String relativePath;
try {
//Root may be deleted -> no root.second available
@@ -908,7 +903,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (!processed && (source == null || !source)) {
root = getOwningBinaryRoot(fo);
- if (root != null && visibilitySupport.isVisible(fo, root.second())) {
+ if (root != null && visibilitySupport.canIndex(fo, root.second())) {
final Work wrk = new BinaryWork(
root.first(),
suspendSupport.getSuspendStatus(),
@@ -989,7 +984,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
}
- if (visibilitySupport.isVisible(newFile,root.second())) {
+ if (visibilitySupport.canIndex(newFile,root.second())) {
final boolean sourceForBinaryRoot = sourcesForBinaryRoots.contains(root.first());
ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
if (entry == null || entry.includes(newFile)) {
@@ -1081,7 +1076,6 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
LogContext.create(LogContext.EventType.INDEXER,null)),
false);
}
- return;
} else if (evt.getPropertyName() != null && evt.getPropertyName().equals(EmbeddingIndexerFactory.class.getName())) {
if (!ignoreIndexerCacheEvents) {
@SuppressWarnings("unchecked")
@@ -1095,7 +1089,6 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
LogContext.create(LogContext.EventType.INDEXER, null)),
false);
}
- return;
}
}
@@ -1110,7 +1103,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
final long version;
final Long lastIndexedVersion;
final Long lastDirtyVersion;
-
+
DocPropsSnapshot(@NonNull final Document doc) {
this.doc = doc;
this.openedInEditor = activeDocProvider.getActiveDocuments().contains(doc);
@@ -1258,7 +1251,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
private static final Logger UI_LOGGER = Logger.getLogger("org.netbeans.ui.indexing"); //NOI18N
private static final RequestProcessor RP = new RequestProcessor("RepositoryUpdater.delay"); //NOI18N
private static final RequestProcessor WORKER = new RequestProcessor("RepositoryUpdater.worker", 1, false, false);
- private static final boolean notInterruptible = Util.getSystemBoolean("netbeans.indexing.notInterruptible", false); //NOI18N
+ private static final boolean NOT_INTERRUPTIBLE = Util.getSystemBoolean("netbeans.indexing.notInterruptible", false); //NOI18N
private static final int FILE_LOCKS_DELAY = BaseUtilities.isWindows() ? 2000 : 1000;
private static final String PROP_LAST_INDEXED_VERSION = RepositoryUpdater.class.getName() + "-last-indexed-document-version"; //NOI18N
private static final String PROP_LAST_DIRTY_VERSION = RepositoryUpdater.class.getName() + "-last-dirty-document-version"; //NOI18N
@@ -1269,35 +1262,35 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
private static final String INDEX_DOWNLOAD_FOLDER = "index-download"; //NOI18N
private static final boolean[] FD_NEW_SFB_ROOT = new boolean[1];
- /* test */ static final List<URL> UNKNOWN_ROOT = Collections.unmodifiableList(new LinkedList<URL>());
- /* test */ static final List<URL> NONEXISTENT_ROOT = Collections.unmodifiableList(new LinkedList<URL>());
+ /* test */ static final List<URL> UNKNOWN_ROOT = Collections.unmodifiableList(new LinkedList<>());
+ /* test */ static final List<URL> NONEXISTENT_ROOT = Collections.unmodifiableList(new LinkedList<>());
/* test */@SuppressWarnings("PackageVisibleField")
volatile static Source unitTestActiveSource;
@org.netbeans.api.annotations.common.SuppressWarnings(
value="DMI_COLLECTION_OF_URLS",
justification="URLs have never host part")
- private final Map<URL, List<URL>>scannedRoots2Dependencies = Collections.synchronizedMap(new TreeMap<URL, List<URL>>(new LexicographicComparator(true)));
+ private final Map<URL, List<URL>>scannedRoots2Dependencies = Collections.synchronizedMap(new TreeMap<>(new LexicographicComparator(true)));
@org.netbeans.api.annotations.common.SuppressWarnings(
value="DMI_COLLECTION_OF_URLS",
justification="URLs have never host part")
- private final Map<URL, List<URL>>scannedBinaries2InvDependencies = Collections.synchronizedMap(new HashMap<URL,List<URL>>());
+ private final Map<URL, List<URL>>scannedBinaries2InvDependencies = Collections.synchronizedMap(new HashMap<>());
@org.netbeans.api.annotations.common.SuppressWarnings(
value="DMI_COLLECTION_OF_URLS",
justification="URLs have never host part")
- private final Map<URL, List<URL>>scannedRoots2Peers = Collections.synchronizedMap(new TreeMap<URL, List<URL>>(new LexicographicComparator(true)));
+ private final Map<URL, List<URL>>scannedRoots2Peers = Collections.synchronizedMap(new TreeMap<>(new LexicographicComparator(true)));
@org.netbeans.api.annotations.common.SuppressWarnings(
value="DMI_COLLECTION_OF_URLS",
justification="URLs have never host part")
- private final Set<URL> incompleteSeenRoots = Collections.synchronizedSet(new HashSet<URL>());
+ private final Set<URL> incompleteSeenRoots = Collections.synchronizedSet(new HashSet<>());
@org.netbeans.api.annotations.common.SuppressWarnings(
value="DMI_COLLECTION_OF_URLS",
justification="URLs have never host part")
- private final Set<URL>scannedUnknown = Collections.synchronizedSet(new HashSet<URL>());
+ private final Set<URL>scannedUnknown = Collections.synchronizedSet(new HashSet<>());
@org.netbeans.api.annotations.common.SuppressWarnings(
value="DMI_COLLECTION_OF_URLS",
justification="URLs have never host part")
- private final Set<URL>sourcesForBinaryRoots = Collections.synchronizedSet(new HashSet<URL>());
+ private final Set<URL>sourcesForBinaryRoots = Collections.synchronizedSet(new HashSet<>());
private volatile State state = State.CREATED;
@@ -1314,7 +1307,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
private final FileChangeListener binaryRootsListener = new FCL(Boolean.FALSE);
private final ThreadLocal<Boolean> inIndexer = new ThreadLocal<>();
- private final VisibilitySupport visibilitySupport = VisibilitySupport.create(this, RP);
+ private final IndexabilitySupport visibilitySupport = IndexabilitySupport.create(this, RP);
private final SuspendSupport suspendSupport = new SuspendSupport(WORKER);
private final AtomicLong scannedRoots2DependenciesLamport = new AtomicLong();
private final ActiveDocumentProvider activeDocProvider;
@@ -1322,7 +1315,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
private final Task worker;
private RepositoryUpdater () {
- LOGGER.log(Level.FINE, "netbeans.indexing.notInterruptible={0}", notInterruptible); //NOI18N
+ LOGGER.log(Level.FINE, "netbeans.indexing.notInterruptible={0}", NOT_INTERRUPTIBLE); //NOI18N
LOGGER.log(Level.FINE, "netbeans.indexing.recursiveListeners={0}", rootsListeners.hasRecursiveListeners()); //NOI18N
LOGGER.log(Level.FINE, "FILE_LOCKS_DELAY={0}", FILE_LOCKS_DELAY); //NOI18N
this.activeDocProvider = Lookup.getDefault().lookup(ActiveDocumentProvider.class);
@@ -2120,14 +2113,11 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
assert action != null;
final List<T> res = new ArrayList<>(1);
try {
- Lookups.executeWith(context, new Runnable() {
- @Override
- public void run() {
- try {
- res.add(action.call());
- } catch (Exception e) {
- RepositoryUpdater.<Void,RuntimeException>sthrow(e);
- }
+ Lookups.executeWith(context, () -> {
+ try {
+ res.add(action.call());
+ } catch (Exception e) {
+ RepositoryUpdater.<Void,RuntimeException>sthrow(e);
}
});
return res.get(0);
@@ -2175,7 +2165,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
//Indexer statistics <IndexerName,{InvocationCount,CumulativeTime}>
//threading: Has to be SynchronizedMap or ConcurrentMap to ensure propper
//visibility. The Work.scanBinaries modifies the map from multiple threads.
- private final Map<String,int[]> indexerStatistics = Collections.<String, int[]>synchronizedMap(new HashMap<String, int[]>());
+ private final Map<String,int[]> indexerStatistics = Collections.<String, int[]>synchronizedMap(new HashMap<>());
private volatile boolean reportIndexerStatistics;
private volatile SourceIndexers sourceIndexers;
@@ -2631,10 +2621,8 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
@NonNull final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> contexts,
@NonNull final UsedIndexables usedIterables
) throws IOException {
- return TaskCache.getDefault().refreshTransaction(new ExceptionAction<Boolean>() {
- public @Override Boolean run() throws IOException {
- return doIndex(resources, allResources, root, sourceForBinaryRoot, indexers, votes, contexts, usedIterables);
- }
+ return TaskCache.getDefault().refreshTransaction(() -> {
+ return doIndex(resources, allResources, root, sourceForBinaryRoot, indexers, votes, contexts, usedIterables);
});
}
@@ -2732,7 +2720,10 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
}
- Iterable<Indexable> indexables = new ProxyIterable<>(indexerIndexablesList);
+ Iterable<Indexable> indexables = new FilteringIterable(
+ new ProxyIterable<>(indexerIndexablesList),
+ indexableFilter(factory, value.second().getRootURI()));
+
allIndexblesSentToIndexers.addAll(indexerIndexablesList);
parkWhileSuspended();
@@ -2740,6 +2731,13 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
return false;
}
+
+ Iterable<Indexable> notIndexables = new FilteringIterable(
+ new ProxyIterable<>(indexerIndexablesList),
+ notIndexableFilter(factory, value.second().getRootURI()));
+
+ factory.filesDeleted(notIndexables, value.second());
+
final CustomIndexer indexer = factory.createIndexer();
logStartIndexer(factory.getIndexerName());
SPIAccessor.getInstance().putProperty(value.second(), ClusteredIndexables.INDEX, usedCi);
@@ -3072,6 +3070,11 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
indexers.bifs);
for(BinaryIndexerFactory f : indexers.bifs) {
+ if(IndexabilityQuery.getInstance().preventIndexing(f.getIndexerName(), root, null)) {
+ f.rootsRemoved(Collections.singleton(root));
+ continue;
+ }
+
parkWhileSuspended();
if (getCancelRequest().isRaised()) {
break;
@@ -3106,6 +3109,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
long et = System.currentTimeMillis();
logIndexerTime(f.getIndexerName(), (int)(et-st));
}
+
return !getCancelRequest().isRaised();
}
@@ -3118,14 +3122,55 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> transactionContexts,
final boolean sourceForBinaryRoot
) throws IOException {
+ IndexabilityQuery iq = IndexabilityQuery.getInstance();
for (final Indexable dirty : files) {
parkWhileSuspended();
if (getCancelRequest().isRaised()) {
return false;
}
- Collection<? extends IndexerCache.IndexerInfo<EmbeddingIndexerFactory>> infos = getIndexerInfos(eifInfosMap, dirty.getMimeType());
- if (infos != null && infos.size() > 0) {
+ Collection<? extends IndexerCache.IndexerInfo<EmbeddingIndexerFactory>> allIndexers =
+ getIndexerInfos(eifInfosMap, dirty.getMimeType());
+ Collection<? extends IndexerCache.IndexerInfo<EmbeddingIndexerFactory>> infos =
+ allIndexers
+ .stream()
+ .filter(i -> ! iq.preventIndexing(
+ i.getIndexerName(),
+ dirty.getURL(),
+ rootURL
+ )
+ )
+ .collect(Collectors.toList());
+
+ allIndexers
+ .stream()
+ .filter(i -> iq.preventIndexing(
+ i.getIndexerName(),
+ dirty.getURL(),
+ rootURL
+ )
+ )
+ .forEach(i -> {
+ try {
+ final Context context = SPIAccessor.getInstance().createContext(
+ cache,
+ rootURL,
+ i.getIndexerName(),
+ i.getIndexerVersion(),
+ null,
+ followUpJob,
+ checkEditor,
+ sourceForBinaryRoot,
+ getSuspendStatus(),
+ getCancelRequest(),
+ logCtx);
+ i.getIndexerFactory().filesDeleted(Collections.singleton(dirty), context);
+ } catch (IOException ex) {
+ LOGGER.log(Level.WARNING, null, ex);
+ }
+ });
+
+ if (infos != null && !infos.isEmpty()) {
final URL url = dirty.getURL();
if (url == null) {
continue;
@@ -3140,14 +3185,14 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
try {
// log parsing for the mimetype:
logStartIndexer(src.getMimeType());
-
+
class T extends UserTask implements IndexingTask {
@Override
public void run(ResultIterator resultIterator) throws Exception {
final String mimeType = resultIterator.getSnapshot().getMimeType();
final Collection<? extends IndexerCache.IndexerInfo<EmbeddingIndexerFactory>> infos = getIndexerInfos(eifInfosMap, mimeType);
- if (infos != null && infos.size() > 0) {
+ if (infos != null && !infos.isEmpty()) {
boolean finished = false;
for (IndexerCache.IndexerInfo<EmbeddingIndexerFactory> info : infos) {
if (getCancelRequest().isRaised()) {
@@ -3254,62 +3299,59 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (rootFo != null) {
final LogContext lctx = getLogContext();
try {
- return runInContext(rootFo, new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
- final boolean permanentUpdate = isSteady();
- assert !TransientUpdateSupport.isTransientUpdate() || !permanentUpdate;
- assert permanentUpdate || (forceRefresh && !files.isEmpty());
-
- final SourceIndexers indexers = getSourceIndexers(false);
- final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> ctxToFinish = new HashMap<>();
- final Map<SourceIndexerFactory,Boolean> invalidatedMap = new IdentityHashMap<>();
- final UsedIndexables usedIterables = new UsedIndexables();
- boolean indexResult = false;
- if (lctx != null) {
- lctx.noteRootScanning(root, false);
- }
- try {
- scanStarted (root, sourceForBinaryRoot, indexers, invalidatedMap, ctxToFinish);
- boolean indexerVeto = false;
- for (Boolean b : invalidatedMap.values()) {
- if (!b) {
- indexerVeto = true;
- break;
- }
- }
- //Find files to index
- final Set<Crawler.TimeStampAction> checkTimeStamps = EnumSet.noneOf(Crawler.TimeStampAction.class);
- if (!forceRefresh) {
- checkTimeStamps.add(Crawler.TimeStampAction.CHECK);
- }
- if (permanentUpdate) {
- checkTimeStamps.add(Crawler.TimeStampAction.UPDATE);
+ return runInContext(rootFo, () -> {
+ final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
+ final boolean permanentUpdate = isSteady();
+ assert !TransientUpdateSupport.isTransientUpdate() || !permanentUpdate;
+ assert permanentUpdate || (forceRefresh && !files.isEmpty());
+
+ final SourceIndexers indexers = getSourceIndexers(false);
+ final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> ctxToFinish = new HashMap<>();
+ final Map<SourceIndexerFactory,Boolean> invalidatedMap = new IdentityHashMap<>();
+ final UsedIndexables usedIterables = new UsedIndexables();
+ boolean indexResult = false;
+ if (lctx != null) {
+ lctx.noteRootScanning(root, false);
+ }
+ try {
+ scanStarted (root, sourceForBinaryRoot, indexers, invalidatedMap, ctxToFinish);
+ boolean indexerVeto = false;
+ for (Boolean b : invalidatedMap.values()) {
+ if (!b) {
+ indexerVeto = true;
+ break;
}
- final Crawler crawler = files.isEmpty() || indexerVeto ?
+ }
+ //Find files to index
+ final Set<Crawler.TimeStampAction> checkTimeStamps = EnumSet.noneOf(Crawler.TimeStampAction.class);
+ if (!forceRefresh) {
+ checkTimeStamps.add(Crawler.TimeStampAction.CHECK);
+ }
+ if (permanentUpdate) {
+ checkTimeStamps.add(Crawler.TimeStampAction.UPDATE);
+ }
+ final Crawler crawler = files.isEmpty() || indexerVeto ?
new FileObjectCrawler(rootFo, checkTimeStamps, entry, getCancelRequest(), getSuspendStatus()) : // rescan the whole root (no timestamp check)
- new FileObjectCrawler(rootFo, files.toArray(new FileObject[files.size()]), checkTimeStamps, entry, getCancelRequest(), getSuspendStatus()); // rescan selected files (no timestamp check)
- if (lctx != null) {
- lctx.startCrawler();
- }
- long t = System.currentTimeMillis();
- final List<Indexable> resources = crawler.getResources();
- if (crawler.isFinished()) {
- logCrawlerTime(crawler, t);
- delete(crawler.getDeletedResources(), ctxToFinish, usedIterables);
- indexResult = index(resources, crawler.getAllResources(), root, sourceForBinaryRoot, indexers, invalidatedMap, ctxToFinish, usedIterables);
- invalidateSources(resources);
- if (indexResult) {
- crawler.storeTimestamps();
- return true;
- }
+ new FileObjectCrawler(rootFo, files.toArray(new FileObject[0]), checkTimeStamps, entry, getCancelRequest(), getSuspendStatus()); // rescan selected files (no timestamp check)
+ if (lctx != null) {
+ lctx.startCrawler();
+ }
+ long t = System.currentTimeMillis();
+ final List<Indexable> resources = crawler.getResources();
+ if (crawler.isFinished()) {
+ logCrawlerTime(crawler, t);
+ delete(crawler.getDeletedResources(), ctxToFinish, usedIterables);
+ indexResult = index(resources, crawler.getAllResources(), root, sourceForBinaryRoot, indexers, invalidatedMap, ctxToFinish, usedIterables);
+ invalidateSources(resources);
+ if (indexResult) {
+ crawler.storeTimestamps();
+ return true;
}
- } finally {
- scanFinished(ctxToFinish.values(), usedIterables, indexResult);
}
- return false;
+ } finally {
+ scanFinished(ctxToFinish.values(), usedIterables, indexResult);
}
+ return false;
});
} catch (IOException ioe) {
LOGGER.log(Level.WARNING, null, ioe);
@@ -3515,7 +3557,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
null :
Source.create(doc);
}
-
+
protected void refreshAffectedDocuments(Set<URL> roots) {
Collection<? extends ActiveDocumentProvider.IndexingAware> toNotify = Lookup.getDefault().lookupAll(ActiveDocumentProvider.IndexingAware.class);
for (ActiveDocumentProvider.IndexingAware o : toNotify) {
@@ -3771,7 +3813,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
super(followUpJob, checkEditor, followUpJob, steady, suspendStatus, logCtx);
assert root != null;
- assert files != null && files.size() > 0;
+ assert files != null && !files.isEmpty();
this.root = root;
this.files.addAll(files);
this.forceRefresh = forceRefresh;
@@ -3950,42 +3992,39 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
public @Override boolean getDone() {
try {
// updateProgress(root);
- final Callable<Boolean> action = new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- LogContext lctx = getLogContext();
- if (lctx != null) {
- lctx.noteRootScanning(root, false);
+ final Callable<Boolean> action = () -> {
+ LogContext lctx = getLogContext();
+ if (lctx != null) {
+ lctx.noteRootScanning(root, false);
+ }
+ try {
+ final List<Indexable> indexables = new ArrayList<>();
+ for(String path : relativePaths) {
+ indexables.add(SPIAccessor.getInstance().create(new DeletedIndexable (root, path)));
}
+ final Map<SourceIndexerFactory,Boolean> votes = new HashMap<>();
+ final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> contexts = new HashMap<>();
+ final UsedIndexables usedIterables = new UsedIndexables();
+ final SourceIndexers indexers = getSourceIndexers(false);
+ final TimeStamps ts = TimeStamps.forRoot(root, false);
try {
- final List<Indexable> indexables = new ArrayList<>();
- for(String path : relativePaths) {
- indexables.add(SPIAccessor.getInstance().create(new DeletedIndexable (root, path)));
- }
- final Map<SourceIndexerFactory,Boolean> votes = new HashMap<>();
- final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> contexts = new HashMap<>();
- final UsedIndexables usedIterables = new UsedIndexables();
- final SourceIndexers indexers = getSourceIndexers(false);
- final TimeStamps ts = TimeStamps.forRoot(root, false);
- try {
- scanStarted(root, false, indexers, votes, contexts);
- delete(indexables, contexts, usedIterables);
- ts.remove(relativePaths);
- } finally {
- final boolean finished = !getCancelRequest().isRaised();
- if (finished) {
- ts.store();
- scanFinished(contexts.values(), usedIterables, finished);
- }
- }
- TEST_LOGGER.log(Level.FINEST, "delete"); //NOI18N
+ scanStarted(root, false, indexers, votes, contexts);
+ delete(indexables, contexts, usedIterables);
+ ts.remove(relativePaths);
} finally {
- if (lctx != null) {
- lctx.finishScannedRoot(root);
+ final boolean finished = !getCancelRequest().isRaised();
+ if (finished) {
+ ts.store();
+ scanFinished(contexts.values(), usedIterables, finished);
}
}
- return true;
+ TEST_LOGGER.log(Level.FINEST, "delete"); //NOI18N
+ } finally {
+ if (lctx != null) {
+ lctx.finishScannedRoot(root);
+ }
}
+ return true;
};
return runInContext(root, action);
} catch (IOException ioe) {
@@ -4095,55 +4134,65 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (!incompleteSeenRoots.contains(root)) {
final FileObject rootFo = URLCache.getInstance().findFileObject(root, true);
if (rootFo != null) {
- final Callable<Boolean> action = new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- long time = System.currentTimeMillis();
- boolean sourceForBinaryRoot = sourcesForBinaryRoots.contains(root);
- final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
- final Crawler crawler = new FileObjectCrawler(rootFo, EnumSet.of(Crawler.TimeStampAction.UPDATE), entry, getCancelRequest(), getSuspendStatus());
- final List<Indexable> resources = crawler.getResources();
- final List<Indexable> deleted = crawler.getDeletedResources();
-
- logCrawlerTime(crawler, time);
- if (crawler.isFinished()) {
- final FileObject cacheRoot = CacheFolder.getDataFolder(
+ final Callable<Boolean> action = () -> {
+ long time = System.currentTimeMillis();
+ boolean sourceForBinaryRoot = sourcesForBinaryRoots.contains(root);
+ final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
+ final Crawler crawler = new FileObjectCrawler(rootFo, EnumSet.of(Crawler.TimeStampAction.UPDATE), entry, getCancelRequest(), getSuspendStatus());
+ final List<Indexable> resources = crawler.getResources();
+ final List<Indexable> deleted = crawler.getDeletedResources();
+
+ logCrawlerTime(crawler, time);
+ if (crawler.isFinished()) {
+ final FileObject cacheRoot = CacheFolder.getDataFolder(
root,
EnumSet.of(CacheFolderProvider.Kind.SOURCES, CacheFolderProvider.Kind.LIBRARIES),
CacheFolderProvider.Mode.CREATE);
- final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> transactionContexts = new HashMap<>();
- final UsedIndexables usedIterables = new UsedIndexables();
- final Map<SourceIndexerFactory,Boolean> votes = new HashMap<>();
+ final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> transactionContexts = new HashMap<>();
+ final UsedIndexables usedIterables = new UsedIndexables();
+ final Map<SourceIndexerFactory,Boolean> votes = new HashMap<>();
+ try {
+ customIndexersScanStarted(root, cacheRoot, sourceForBinaryRoot, cifInfos, votes, transactionContexts);
+ if (!deleted.isEmpty()) {
+ delete(deleted, transactionContexts, usedIterables);
+ }
+ final LinkedList<Iterable<Indexable>> allIndexblesSentToIndexers = new LinkedList<>();
try {
- customIndexersScanStarted(root, cacheRoot, sourceForBinaryRoot, cifInfos, votes, transactionContexts);
- if (deleted.size() > 0) {
- delete(deleted, transactionContexts, usedIterables);
- }
- final LinkedList<Iterable<Indexable>> allIndexblesSentToIndexers = new LinkedList<>();
- try {
- ClusteredIndexables ci = new ClusteredIndexables(resources);
- for(IndexerCache.IndexerInfo<CustomIndexerFactory> cifInfo : cifInfos) {
- List<Iterable<Indexable>> indexerIndexablesList = new LinkedList<>();
- for(String mimeType : cifInfo.getMimeTypes()) {
- indexerIndexablesList.add(ci.getIndexablesFor(mimeType));
- }
- ProxyIterable<Indexable> indexables = new ProxyIterable<>(indexerIndexablesList);
- allIndexblesSentToIndexers.addAll(indexerIndexablesList);
+ ClusteredIndexables ci = new ClusteredIndexables(resources);
+ for(IndexerCache.IndexerInfo<CustomIndexerFactory> cifInfo : cifInfos) {
+ List<Iterable<Indexable>> indexerIndexablesList = new LinkedList<>();
+ for(String mimeType : cifInfo.getMimeTypes()) {
+ indexerIndexablesList.add(ci.getIndexablesFor(mimeType));
+ }
- parkWhileSuspended();
- if (getCancelRequest().isRaised()) {
- return false;
- }
+ allIndexblesSentToIndexers.addAll(indexerIndexablesList);
- final CustomIndexerFactory factory = cifInfo.getIndexerFactory();
- final Pair<String,Integer> indexerKey = Pair.<String,Integer>of(factory.getIndexerName(),factory.getIndexVersion());
- final Pair<SourceIndexerFactory, Context> ctx = transactionContexts.get(indexerKey);
- if (ctx != null) {
- SPIAccessor.getInstance().setAllFilesJob(ctx.second(), true);
- final CustomIndexer indexer = factory.createIndexer();
- if (LOGGER.isLoggable(Level.FINE)) {
- StringBuilder sb = printMimeTypes(cifInfo.getMimeTypes(), new StringBuilder());
- LOGGER.log(
+ parkWhileSuspended();
+ if (getCancelRequest().isRaised()) {
+ return false;
+ }
+
+ final CustomIndexerFactory factory = cifInfo.getIndexerFactory();
+ final Pair<String,Integer> indexerKey = Pair.<String,Integer>of(factory.getIndexerName(),factory.getIndexVersion());
+ final Pair<SourceIndexerFactory, Context> ctx = transactionContexts.get(indexerKey);
+
+ if (ctx != null) {
+ Iterable<Indexable> indexables = new FilteringIterable(
+ new ProxyIterable<>(indexerIndexablesList),
+ indexableFilter(factory, ctx.second().getRootURI()));
+
+ SPIAccessor.getInstance().setAllFilesJob(ctx.second(), true);
+
+ Iterable<Indexable> notIndexables = new FilteringIterable(
+ new ProxyIterable<>(indexerIndexablesList),
+ notIndexableFilter(factory, ctx.second().getRootURI()));
+
+ factory.filesDeleted(notIndexables, ctx.second());
+
+ final CustomIndexer indexer = factory.createIndexer();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ StringBuilder sb = printMimeTypes(cifInfo.getMimeTypes(), new StringBuilder());
+ LOGGER.log(
Level.FINE,
"Reindexing {0} using {1}; mimeTypes={2}", //NOI18N
new Object[]{
@@ -4151,39 +4200,38 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
indexer,
sb
});
- }
- SPIAccessor.getInstance().putProperty(ctx.second(), ClusteredIndexables.INDEX, ci);
- long st = System.currentTimeMillis();
- logStartIndexer(factory.getIndexerName());
- try {
- SPIAccessor.getInstance().index(indexer, indexables, ctx.second());
- } catch (ThreadDeath td) {
- throw td;
- } catch (Throwable t) {
- LOGGER.log(Level.WARNING, null, t);
- }
- long et = System.currentTimeMillis();
- logIndexerTime(factory.getIndexerName(), (int)(et-st));
- } else {
- LOGGER.log(
+ }
+ SPIAccessor.getInstance().putProperty(ctx.second(), ClusteredIndexables.INDEX, ci);
+ long st = System.currentTimeMillis();
+ logStartIndexer(factory.getIndexerName());
+ try {
+ SPIAccessor.getInstance().index(indexer, indexables, ctx.second());
+ } catch (ThreadDeath td) {
+ throw td;
+ } catch (Throwable t) {
+ LOGGER.log(Level.WARNING, null, t);
+ }
+ long et = System.currentTimeMillis();
+ logIndexerTime(factory.getIndexerName(), (int)(et-st));
+ } else {
+ LOGGER.log(
Level.WARNING, "RefreshCifIndices ignored recently added factory: {0}", //NOI18N
indexerKey);
- }
- InjectedTasksSupport.execute();
}
- } finally {
- usedIterables.offerAll(allIndexblesSentToIndexers);
+ InjectedTasksSupport.execute();
}
} finally {
- final boolean commit = !getCancelRequest().isRaised();
- scanFinished(transactionContexts.values(), usedIterables, commit);
- if (commit) {
- crawler.storeTimestamps();
- }
+ usedIterables.offerAll(allIndexblesSentToIndexers);
+ }
+ } finally {
+ final boolean commit = !getCancelRequest().isRaised();
+ scanFinished(transactionContexts.values(), usedIterables, commit);
+ if (commit) {
+ crawler.storeTimestamps();
}
}
- return true;
}
+ return true;
};
if (!runInContext(rootFo, action)) {
return false;
@@ -4303,65 +4351,63 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
if (!incompleteSeenRoots.contains(root)) {
final FileObject rootFo = URLCache.getInstance().findFileObject(root, true);
if (rootFo != null) {
- final Callable<Boolean> action = new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- long t = System.currentTimeMillis();
- boolean sourceForBinaryRoot = sourcesForBinaryRoots.contains(root);
- final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
- Crawler crawler = new FileObjectCrawler(rootFo, EnumSet.of(Crawler.TimeStampAction.UPDATE), entry, getCancelRequest(), getSuspendStatus());
- final List<Indexable> resources = crawler.getResources();
- final List<Indexable> deleted = crawler.getDeletedResources();
-
- logCrawlerTime(crawler, t);
- if (crawler.isFinished()) {
- final FileObject cacheRoot = CacheFolder.getDataFolder(
- root,
- EnumSet.of(CacheFolderProvider.Kind.SOURCES, CacheFolderProvider.Kind.LIBRARIES),
- CacheFolderProvider.Mode.CREATE);
- final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> transactionContexts = new HashMap<>();
- final UsedIndexables usedIterables = new UsedIndexables();
- final Map<SourceIndexerFactory,Boolean> votes = new HashMap<>();
- final Map<String, Collection<IndexerCache.IndexerInfo<EmbeddingIndexerFactory>>> eifInfosMap = new HashMap<>();
- for(IndexerCache.IndexerInfo<EmbeddingIndexerFactory> eifInfo : eifInfos) {
- for (String mimeType : eifInfo.getMimeTypes()) {
- Collection<IndexerInfo<EmbeddingIndexerFactory>> infos = eifInfosMap.get(mimeType);
- if (infos == null) {
- infos = new HashSet<>();
- eifInfosMap.put(mimeType, infos);
- }
- infos.add(eifInfo);
+ final Callable<Boolean> action = () -> {
+ long t = System.currentTimeMillis();
+ boolean sourceForBinaryRoot = sourcesForBinaryRoots.contains(root);
+ final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
+ Crawler crawler = new FileObjectCrawler(rootFo, EnumSet.of(Crawler.TimeStampAction.UPDATE), entry, getCancelRequest(), getSuspendStatus());
+ final List<Indexable> resources = crawler.getResources();
+ final List<Indexable> deleted = crawler.getDeletedResources();
+
+ logCrawlerTime(crawler, t);
+ if (crawler.isFinished()) {
+ final FileObject cacheRoot = CacheFolder.getDataFolder(
+ root,
+ EnumSet.of(CacheFolderProvider.Kind.SOURCES, CacheFolderProvider.Kind.LIBRARIES),
+ CacheFolderProvider.Mode.CREATE);
+ final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> transactionContexts = new HashMap<>();
+ final UsedIndexables usedIterables = new UsedIndexables();
+ final Map<SourceIndexerFactory,Boolean> votes = new HashMap<>();
+ final Map<String, Collection<IndexerCache.IndexerInfo<EmbeddingIndexerFactory>>> eifInfosMap = new HashMap<>();
+ for(IndexerCache.IndexerInfo<EmbeddingIndexerFactory> eifInfo : eifInfos) {
+ for (String mimeType : eifInfo.getMimeTypes()) {
+ Collection<IndexerInfo<EmbeddingIndexerFactory>> infos = eifInfosMap.get(mimeType);
+ if (infos == null) {
+ infos = new HashSet<>();
+ eifInfosMap.put(mimeType, infos);
}
+ infos.add(eifInfo);
}
- SourceAccessor.getINSTANCE().suppressListening(true, !hasToCheckEditor());
+ }
+ SourceAccessor.getINSTANCE().suppressListening(true, !hasToCheckEditor());
+ try {
+ embeddingIndexersScanStarted(root, cacheRoot, sourceForBinaryRoot, eifInfosMap.values(), votes, transactionContexts);
+ if (!deleted.isEmpty()) {
+ delete(deleted, transactionContexts, usedIterables);
+ }
+ final LinkedList<Iterable<Indexable>> allIndexblesSentToIndexers = new LinkedList<>();
try {
- embeddingIndexersScanStarted(root, cacheRoot, sourceForBinaryRoot, eifInfosMap.values(), votes, transactionContexts);
- if (deleted.size() > 0) {
- delete(deleted, transactionContexts, usedIterables);
- }
- final LinkedList<Iterable<Indexable>> allIndexblesSentToIndexers = new LinkedList<>();
- try {
- ClusteredIndexables ci = new ClusteredIndexables(resources);
- for(String mimeType : Util.getAllMimeTypes()) {
- if (getCancelRequest().isRaised()) {
- return false;
- }
+ ClusteredIndexables ci = new ClusteredIndexables(resources);
+ for(String mimeType : Util.getAllMimeTypes()) {
+ if (getCancelRequest().isRaised()) {
+ return false;
+ }
if (!ParserManager.canBeParsed(mimeType)) {
- continue;
- }
+ continue;
+ }
- Iterable<Indexable> indexables = ci.getIndexablesFor(mimeType);
- allIndexblesSentToIndexers.add(indexables);
+ Iterable<Indexable> indexables = ci.getIndexablesFor(mimeType);
+ allIndexblesSentToIndexers.add(indexables);
- long tm1 = System.currentTimeMillis();
- boolean f = indexEmbedding(eifInfosMap, cacheRoot, root, indexables, ci, transactionContexts, sourceForBinaryRoot);
- long tm2 = System.currentTimeMillis();
- if (!f) {
- return false;
- }
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.log(
+ long tm1 = System.currentTimeMillis();
+ boolean f = indexEmbedding(eifInfosMap, cacheRoot, root, indexables, ci, transactionContexts, sourceForBinaryRoot);
+ long tm2 = System.currentTimeMillis();
+ if (!f) {
+ return false;
+ }
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.log(
Level.FINE,
"Indexing {0} embeddables under {1}; took {2}ms", //NOI18N
new Object[]{
@@ -4369,25 +4415,24 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
root,
tm2 - tm1
});
- }
}
- } finally {
- usedIterables.offerAll(allIndexblesSentToIndexers);
}
} finally {
- try {
- final boolean commit = !getCancelRequest().isRaised();
- scanFinished(transactionContexts.values(),usedIterables, commit);
- if (commit) {
- crawler.storeTimestamps();
- }
- } finally {
- SourceAccessor.getINSTANCE().suppressListening(false, false);
+ usedIterables.offerAll(allIndexblesSentToIndexers);
+ }
+ } finally {
+ try {
+ final boolean commit = !getCancelRequest().isRaised();
+ scanFinished(transactionContexts.values(),usedIterables, commit);
+ if (commit) {
+ crawler.storeTimestamps();
}
+ } finally {
+ SourceAccessor.getINSTANCE().suppressListening(false, false);
}
}
- return true;
}
+ return true;
};
if (!runInContext(rootFo, action)) {
return false;
@@ -4485,12 +4530,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
sourcesForBinaryRoots,
false,
false,
- new Callable<SourceIndexers>(){
- @Override
- public SourceIndexers call() throws Exception {
- return getSourceIndexers(false);
- }
- });
+ () -> getSourceIndexers(false));
depCtx.newIncompleteSeenRoots.addAll(this.incompleteSeenRoots);
if (suspectFilesOrFileObjects.isEmpty()) {
@@ -4631,11 +4671,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
// refresh filesystems
- FileSystem.AtomicAction aa = new FileSystem.AtomicAction() {
- public @Override void run() throws IOException {
- FileUtil.refreshFor(File.listRoots());
- }
- };
+ FileSystem.AtomicAction aa = () -> FileUtil.refreshFor(File.listRoots());
// XXX: nested FS.AA don't seem to work, so just ignore evrything
// interceptor.setActiveAtomicAction(aa);
// Probably not needed, the aa calls refreshFor which behaves correctly
@@ -4721,7 +4757,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
private boolean scanRootFiles(
@NullAllowed final Map<URL, Set<FileObject>> files,
@NonNull final Set<URL> incompleteSeenRoots) {
- if (files != null && files.size() > 0) { // #174887
+ if (files != null && !files.isEmpty()) { // #174887
for(Iterator<Map.Entry<URL, Set<FileObject>>> it = files.entrySet().iterator(); it.hasNext(); ) {
Map.Entry<URL, Set<FileObject>> entry = it.next();
URL root = entry.getKey();
@@ -4806,7 +4842,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
int stubCount = 0;
// ignore all files, which reside in IDE installation (in various clusters)
- // the ignored files must be located 1 subdirectory beneath the cluster
+ // the ignored files must be located 1 subdirectory beneath the cluster
// directory
for (URL u : roots) {
String path = u.getPath();
@@ -4895,12 +4931,8 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
sourcesForBinaryRoots,
useInitialState,
refreshNonExistentDeps,
- new Callable<SourceIndexers>(){
- @Override
- public SourceIndexers call() throws Exception {
- return getSourceIndexers(false);
- }
- });
+ () -> getSourceIndexers(false)
+ );
final Collection<URL> newRoots = new HashSet<>();
Collection<? extends URL> c = PathRegistry.getDefault().getSources();
checkRootCollection(c);
@@ -5002,7 +5034,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
final Level logLevel = Level.FINE;
- if (LOGGER.isLoggable(logLevel) && (addedOrChanged.size() > 0 || removed.size() > 0)) {
+ if (LOGGER.isLoggable(logLevel) && (!addedOrChanged.isEmpty() || !removed.isEmpty())) {
LOGGER.log(logLevel, "Changes in dependencies detected:"); //NOI18N
LOGGER.log(logLevel, "initialRoots2Deps({0})=", depCtx.initialRoots2Deps.size()); //NOI18N
LOGGER.log(logLevel, printMap(depCtx.initialRoots2Deps, new StringBuilder()).toString());
@@ -5285,21 +5317,11 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
null : BinaryIndexers.load();
final IndexBinaryWorkPool pool = new IndexBinaryWorkPool(
- new IndexBinaryWorkPool.Function<URL, Boolean>() {
- @Override
- public Boolean apply(URL root) {
- return scanBinary(
- root,
- binaryIndexers,
- scannedRootsCnt);
- }
- },
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- return getCancelRequest().isRaised();
- }
- },
+ (URL root) -> scanBinary(
+ root,
+ binaryIndexers,
+ scannedRootsCnt),
+ () -> getCancelRequest().isRaised(),
ctx.newBinariesToScan);
final long binaryScanStart = System.currentTimeMillis();
final Pair<Boolean,Collection<? extends URL>> res = pool.execute();
@@ -5321,65 +5343,62 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
protected final boolean scanBinary(
@NonNull final URL root,
@NonNull final BinaryIndexers binaryIndexers,
- @NonNull final AtomicInteger scannedRootsCnt) {
+ final AtomicInteger scannedRootsCnt) {
try {
- final Callable<Boolean> action = new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- boolean success = false;
- final long tmStart = System.currentTimeMillis();
- final LinkedHashMap<BinaryIndexerFactory, Context> contexts = new LinkedHashMap<>(binaryIndexers.bifs.size());
- final BitSet startedIndexers = new BitSet(binaryIndexers.bifs.size());
- LogContext lctx = getLogContext();
- if (lctx != null) {
- lctx.noteRootScanning(root, false);
+ final Callable<Boolean> action = () -> {
+ boolean success = false;
+ final long tmStart = System.currentTimeMillis();
+ final LinkedHashMap<BinaryIndexerFactory, Context> contexts = new LinkedHashMap<>(binaryIndexers.bifs.size());
+ final BitSet startedIndexers = new BitSet(binaryIndexers.bifs.size());
+ LogContext lctx = getLogContext();
+ if (lctx != null) {
+ lctx.noteRootScanning(root, false);
+ }
+ try {
+ createBinaryContexts(root, binaryIndexers, contexts);
+ final FileObject rootFo = URLCache.getInstance().findFileObject(root, true);
+ final FileObject file = rootFo == null ? null : FileUtil.getArchiveFile(rootFo);
+ final boolean upToDate;
+ final long currentLastModified;
+ if (file != null) {
+ final Pair<Long,Map<Pair<String,Integer>,Integer>> lastState = ArchiveTimeStamps.getLastModified(root);
+ final boolean indexersUpToDate = checkBinaryIndexers(lastState, contexts);
+ currentLastModified = file.lastModified().getTime();
+ upToDate = indexersUpToDate && lastState.first() == currentLastModified;
+ } else {
+ currentLastModified = -1L;
+ upToDate = false;
}
try {
- createBinaryContexts(root, binaryIndexers, contexts);
- final FileObject rootFo = URLCache.getInstance().findFileObject(root, true);
- final FileObject file = rootFo == null ? null : FileUtil.getArchiveFile(rootFo);
- final boolean upToDate;
- final long currentLastModified;
- if (file != null) {
- final Pair<Long,Map<Pair<String,Integer>,Integer>> lastState = ArchiveTimeStamps.getLastModified(root);
- final boolean indexersUpToDate = checkBinaryIndexers(lastState, contexts);
- currentLastModified = file.lastModified().getTime();
- upToDate = indexersUpToDate && lastState.first() == currentLastModified;
- } else {
- currentLastModified = -1L;
- upToDate = false;
- }
- try {
- binaryScanStarted(root, upToDate, contexts, startedIndexers);
- updateProgress(root, true);
- success = indexBinary(root, binaryIndexers, contexts);
- } finally {
- binaryScanFinished(binaryIndexers, contexts, startedIndexers, success);
- if (success && !upToDate && FileUtil.getArchiveFile(root) != null) {
- ArchiveTimeStamps.setLastModified(root, createBinaryIndexersTimeStamp(currentLastModified,contexts));
- }
- }
+ binaryScanStarted(root, upToDate, contexts, startedIndexers);
+ updateProgress(root, true);
+ success = indexBinary(root, binaryIndexers, contexts);
} finally {
- if (lctx != null) {
- lctx.finishScannedRoot(root);
+ binaryScanFinished(binaryIndexers, contexts, startedIndexers, success);
+ if (success && !upToDate && FileUtil.getArchiveFile(root) != null) {
+ ArchiveTimeStamps.setLastModified(root, createBinaryIndexersTimeStamp(currentLastModified,contexts));
}
- final long time = System.currentTimeMillis() - tmStart;
- if (scannedRootsCnt != null) {
- scannedRootsCnt.incrementAndGet();
- }
- reportRootScan(root, time);
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.log(
+ }
+ } finally {
+ if (lctx != null) {
+ lctx.finishScannedRoot(root);
+ }
+ final long time = System.currentTimeMillis() - tmStart;
+ if (scannedRootsCnt != null) {
+ scannedRootsCnt.incrementAndGet();
+ }
+ reportRootScan(root, time);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.log(
Level.FINE,
"Indexing of: {0} took: {1} ms", //NOI18N
new Object[] {
root,
time
});
- }
}
- return success;
}
+ return success;
};
return runInContext(root, action);
} catch (IOException ioe) {
@@ -5636,131 +5655,128 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
LOGGER.log(Level.FINE, "Scanning sources root: {0}", root); //NOI18N
final FileObject rootFo = URLCache.getInstance().findFileObject(root, true);
if (rootFo != null) {
- final Callable<Boolean> action = new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- final boolean rootSeen = TimeStamps.existForRoot(root);
- final SourceIndexers indexers = getSourceIndexers(false);
- if (isNoRootsScan() && !fullRescan && rootSeen) {
- // We've already seen the root at least once and roots scanning is forcibly turned off
- // so just call indexers with no files to let them know about the root, but perform
- // no indexing.
- return nopCustomIndexers(root, indexers, sourceForBinaryRoot);
- } else {
- LogContext lctx = getLogContext();
- long t = System.currentTimeMillis();
- URL indexURL;
- if (!rootSeen && (indexURL=getRemoteIndexURL(root))!=null) {
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.log(
- Level.FINE,
- "Downloading index for root: {0} from: {1}",
- new Object[]{root, indexURL});
- }
- final FileObject cf = CacheFolder.getCacheFolder();
- assert cf != null;
- final File cacheFolder = FileUtil.toFile(cf);
- assert cacheFolder != null;
- final File downloadFolder = new File (cacheFolder,INDEX_DOWNLOAD_FOLDER); //NOI18N
- if (downloadFolder.exists()) {
- delete (downloadFolder.listFiles());
- } else {
- downloadFolder.mkdir();
- }
- final File packedIndex = download(indexURL, downloadFolder);
- if (packedIndex != null ) {
- unpack(packedIndex, downloadFolder);
- packedIndex.delete();
- if (patchDownloadedIndex(root,BaseUtilities.toURI(downloadFolder).toURL())) {
- final FileObject df = CacheFolder.getDataFolder(
+ final Callable<Boolean> action = () -> {
+ final boolean rootSeen = TimeStamps.existForRoot(root);
+ final SourceIndexers indexers = getSourceIndexers(false);
+ if (isNoRootsScan() && !fullRescan && rootSeen) {
+ // We've already seen the root at least once and roots scanning is forcibly turned off
+ // so just call indexers with no files to let them know about the root, but perform
+ // no indexing.
+ return nopCustomIndexers(root, indexers, sourceForBinaryRoot);
+ } else {
+ LogContext lctx = getLogContext();
+ long t = System.currentTimeMillis();
+ URL indexURL;
+ if (!rootSeen && (indexURL=getRemoteIndexURL(root))!=null) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.log(
+ Level.FINE,
+ "Downloading index for root: {0} from: {1}",
+ new Object[]{root, indexURL});
+ }
+ final FileObject cf = CacheFolder.getCacheFolder();
+ assert cf != null;
+ final File cacheFolder = FileUtil.toFile(cf);
+ assert cacheFolder != null;
+ final File downloadFolder = new File (cacheFolder,INDEX_DOWNLOAD_FOLDER); //NOI18N
+ if (downloadFolder.exists()) {
+ delete (downloadFolder.listFiles());
+ } else {
+ downloadFolder.mkdir();
+ }
+ final File packedIndex = download(indexURL, downloadFolder);
+ if (packedIndex != null ) {
+ unpack(packedIndex, downloadFolder);
+ packedIndex.delete();
+ if (patchDownloadedIndex(root,BaseUtilities.toURI(downloadFolder).toURL())) {
+ final FileObject df = CacheFolder.getDataFolder(
root,
EnumSet.of(CacheFolderProvider.Kind.SOURCES, CacheFolderProvider.Kind.LIBRARIES),
CacheFolderProvider.Mode.CREATE);
- assert df != null;
- final File dataFolder = FileUtil.toFile(df);
- assert dataFolder != null;
- if (dataFolder.exists()) {
- //Some features already forced folder creation
- //delete it to be able to do renameTo
- delete(dataFolder);
- }
- downloadFolder.renameTo(dataFolder);
- final TimeStamps timeStamps = TimeStamps.forRoot(root, false);
- timeStamps.resetToNow();
- timeStamps.store();
- nopCustomIndexers(root, indexers, sourceForBinaryRoot);
- for (Map.Entry<File,Index> e : IndexManager.getOpenIndexes().entrySet()) {
- if (Util.isParentOf(dataFolder, e.getKey())) {
- e.getValue().getStatus(true);
- }
+ assert df != null;
+ final File dataFolder = FileUtil.toFile(df);
+ assert dataFolder != null;
+ if (dataFolder.exists()) {
+ //Some features already forced folder creation
+ //delete it to be able to do renameTo
+ delete(dataFolder);
+ }
+ downloadFolder.renameTo(dataFolder);
+ final TimeStamps timeStamps = TimeStamps.forRoot(root, false);
+ timeStamps.resetToNow();
+ timeStamps.store();
+ nopCustomIndexers(root, indexers, sourceForBinaryRoot);
+ for (Map.Entry<File,Index> e : IndexManager.getOpenIndexes().entrySet()) {
+ if (Util.isParentOf(dataFolder, e.getKey())) {
+ e.getValue().getStatus(true);
}
- return true;
}
+ return true;
}
}
- //todo: optimize for java.io.Files
- final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
- final Set<Crawler.TimeStampAction> checkTimeStamps = EnumSet.of(Crawler.TimeStampAction.UPDATE);
- if (!fullRescan) {
- checkTimeStamps.add(Crawler.TimeStampAction.CHECK);
- }
- final Crawler crawler = new FileObjectCrawler(rootFo, checkTimeStamps, entry, getCancelRequest(), getSuspendStatus());
- final List<Indexable> resources = crawler.getResources();
- final List<Indexable> allResources = crawler.getAllResources();
- final List<Indexable> deleted = crawler.getDeletedResources();
+ }
+ //todo: optimize for java.io.Files
+ final ClassPath.Entry entry = sourceForBinaryRoot ? null : getClassPathEntry(rootFo);
+ final Set<Crawler.TimeStampAction> checkTimeStamps = EnumSet.of(Crawler.TimeStampAction.UPDATE);
+ if (!fullRescan) {
+ checkTimeStamps.add(Crawler.TimeStampAction.CHECK);
+ }
+ final Crawler crawler = new FileObjectCrawler(rootFo, checkTimeStamps, entry, getCancelRequest(), getSuspendStatus());
+ final List<Indexable> resources = crawler.getResources();
+ final List<Indexable> allResources = crawler.getAllResources();
+ final List<Indexable> deleted = crawler.getDeletedResources();
- AbstractRootsWork.this.modifiedResourceCount = resources.size();
- AbstractRootsWork.this.allResourceCount = allResources == null ? 0 : allResources.size();
+ AbstractRootsWork.this.modifiedResourceCount = resources.size();
+ AbstractRootsWork.this.allResourceCount = allResources == null ? 0 : allResources.size();
- logCrawlerTime(crawler, t);
- if (crawler.isFinished()) {
- final Map<SourceIndexerFactory,Boolean> invalidatedMap = new IdentityHashMap<>();
- final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> ctxToFinish = new HashMap<>();
- final UsedIndexables usedIterables = new UsedIndexables();
- boolean indexResult = false;
- try {
- scanStarted (root, sourceForBinaryRoot, indexers, invalidatedMap, ctxToFinish);
- delete(deleted, ctxToFinish, usedIterables);
- final long tm = System.currentTimeMillis();
- final boolean rlAdded = RepositoryUpdater.getDefault().rootsListeners.addSource(root, entry);
- if (recursiveListenersTime != null) {
- recursiveListenersTime[0] = System.currentTimeMillis() - tm;
- }
- if (rlAdded) {
- indexResult = index(
- resources,
- allResources,
- root,
- sourceForBinaryRoot,
- indexers,
- invalidatedMap,
- ctxToFinish,
- usedIterables);
- invalidateSources(resources);
- if (indexResult) {
- crawler.storeTimestamps();
- outOfDateFiles[0] = resources.size();
- deletedFiles[0] = deleted.size();
- if (logStatistics) {
- logStatistics = false;
- if (SFEC_LOGGER.isLoggable(Level.INFO)) {
- LogRecord r = new LogRecord(Level.INFO, "STATS_SCAN_SOURCES"); //NOI18N
- r.setParameters(new Object [] {outOfDateFiles[0] > 0 || deletedFiles[0] > 0});
- r.setResourceBundle(NbBundle.getBundle(RepositoryUpdater.class));
- r.setResourceBundleName(RepositoryUpdater.class.getPackage().getName() + ".Bundle"); //NOI18N
- r.setLoggerName(SFEC_LOGGER.getName());
- SFEC_LOGGER.log(r);
- }
+ logCrawlerTime(crawler, t);
+ if (crawler.isFinished()) {
+ final Map<SourceIndexerFactory,Boolean> invalidatedMap = new IdentityHashMap<>();
+ final Map<Pair<String,Integer>,Pair<SourceIndexerFactory,Context>> ctxToFinish = new HashMap<>();
+ final UsedIndexables usedIterables = new UsedIndexables();
+ boolean indexResult = false;
+ try {
+ scanStarted (root, sourceForBinaryRoot, indexers, invalidatedMap, ctxToFinish);
+ delete(deleted, ctxToFinish, usedIterables);
+ final long tm = System.currentTimeMillis();
+ final boolean rlAdded = RepositoryUpdater.getDefault().rootsListeners.addSource(root, entry);
+ if (recursiveListenersTime != null) {
+ recursiveListenersTime[0] = System.currentTimeMillis() - tm;
+ }
+ if (rlAdded) {
+ indexResult = index(
+ resources,
+ allResources,
+ root,
+ sourceForBinaryRoot,
+ indexers,
+ invalidatedMap,
+ ctxToFinish,
+ usedIterables);
+ invalidateSources(resources);
+ if (indexResult) {
+ crawler.storeTimestamps();
+ outOfDateFiles[0] = resources.size();
+ deletedFiles[0] = deleted.size();
+ if (logStatistics) {
+ logStatistics = false;
+ if (SFEC_LOGGER.isLoggable(Level.INFO)) {
+ LogRecord r = new LogRecord(Level.INFO, "STATS_SCAN_SOURCES"); //NOI18N
+ r.setParameters(new Object [] {outOfDateFiles[0] > 0 || deletedFiles[0] > 0});
+ r.setResourceBundle(NbBundle.getBundle(RepositoryUpdater.class));
+ r.setResourceBundleName(RepositoryUpdater.class.getPackage().getName() + ".Bundle"); //NOI18N
+ r.setLoggerName(SFEC_LOGGER.getName());
+ SFEC_LOGGER.log(r);
}
- return true;
}
+ return true;
}
- } finally {
- scanFinished(ctxToFinish.values(), usedIterables, indexResult);
}
+ } finally {
+ scanFinished(ctxToFinish.values(), usedIterables, indexResult);
}
- return false;
}
+ return false;
}
};
return runInContext(rootFo, action);
@@ -6109,14 +6125,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
@Override
public Void call() throws Exception {
try {
- RunWhenScanFinishedSupport.performScan(
- new Runnable() {
- @Override
- public void run() {
- _run();
- }
- },
- globalLookup);
+ RunWhenScanFinishedSupport.performScan(() -> _run(), globalLookup);
} finally {
synchronized (todo) {
if ((!protectedOwners.isEmpty() && offProtectedMode == null) || todo.isEmpty()) {
@@ -6229,7 +6238,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
private Work getWork () {
synchronized (todo) {
Work w;
- if ((protectedOwners.isEmpty() || offProtectedMode != null) && todo.size() > 0) {
+ if ((protectedOwners.isEmpty() || offProtectedMode != null) && !todo.isEmpty()) {
w = todo.remove(0);
if (w instanceof FileListWork && ((FileListWork) w).isFollowUpJob() && !followUpWorksSorted) {
@@ -6254,7 +6263,7 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
List<URL> sortedRoots;
try {
- sortedRoots = new ArrayList<URL>(BaseUtilities.topologicalSort(toSort.keySet(), getDefault().scannedRoots2Dependencies));
+ sortedRoots = new ArrayList<>(BaseUtilities.topologicalSort(toSort.keySet(), getDefault().scannedRoots2Dependencies));
} catch (TopologicalSortException tse) {
LOGGER.log(Level.INFO, "Cycles detected in classpath roots dependencies, using partial ordering", tse); //NOI18N
@SuppressWarnings("unchecked") List<URL> partialSort = tse.partialSort(); //NOI18N
@@ -6284,23 +6293,21 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
private void scheduleDelayed(
@NonNull final Collection<? extends Runnable> tasks,
final int delay) {
- final Runnable run = new Runnable() {
- public @Override void run() {
- schedule(new Work(false, false, false, true, SuspendSupport.NOP, null) {
- protected @Override boolean getDone() {
- for(Runnable task : tasks) {
- try {
- task.run();
- } catch (ThreadDeath td) {
- throw td;
- } catch (Throwable t) {
- LOGGER.log(Level.WARNING, null, t);
- }
+ final Runnable run = () -> {
+ schedule(new Work(false, false, false, true, SuspendSupport.NOP, null) {
+ protected @Override boolean getDone() {
+ for(Runnable task : tasks) {
+ try {
+ task.run();
+ } catch (ThreadDeath td) {
+ throw td;
+ } catch (Throwable t) {
+ LOGGER.log(Level.WARNING, null, t);
}
- return true;
}
- }, false);
- }
+ return true;
+ }
+ }, false);
};
if (delay == 0) {
//Run now sync
@@ -6681,6 +6688,24 @@ public final class RepositoryUpdater implements PathRegistryListener, PropertyCh
}
+ private static Function<Indexable, Boolean> indexableFilter(final CustomIndexerFactory factory, URL rootUrl) {
+ Function<Indexable, Boolean> canBeIndexed = indexable
+ -> !IndexabilityQuery.getInstance().preventIndexing(
+ factory.getIndexerName(),
+ indexable.getURL(),
+ rootUrl);
+ return canBeIndexed;
+ }
+
+ private static Function<Indexable, Boolean> notIndexableFilter(final CustomIndexerFactory factory, URL rootUrl) {
+ Function<Indexable, Boolean> canBeIndexed = indexable
+ -> IndexabilityQuery.getInstance().preventIndexing(
+ factory.getIndexerName(),
+ indexable.getURL(),
+ rootUrl);
+ return canBeIndexed;
+ }
+
// -----------------------------------------------------------------------
// Methods for tests
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/TimeStamps.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/TimeStamps.java
index 376da19..8028ca7 100644
--- a/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/TimeStamps.java
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/impl/indexing/TimeStamps.java
@@ -55,17 +55,17 @@ public final class TimeStamps {
// -J-Dorg.netbeans.modules.parsing.impl.indexing.TimeStamps.level=FINE
private static final Logger LOG = Logger.getLogger(TimeStamps.class.getName());
private static final String TIME_STAMPS_FILE = "timestamps.properties"; //NOI18N
- private static final String VERSION = "#v2"; //NOI18N
+ private static final String VERSION_2 = "#v2"; //NOI18N
+ private static final String VERSION_3 = "#v3"; //NOI18N
-
private final Implementation impl;
-
+
private TimeStamps(@NonNull final Implementation impl) throws IOException {
assert impl != null;
this.impl = impl;
}
-
+
public boolean checkAndStoreTimestamp(FileObject f, String relativePath) {
return impl.checkAndStoreTimestamp(f, relativePath);
}
@@ -73,11 +73,11 @@ public final class TimeStamps {
public Set<String> getUnseenFiles() {
return impl.getUnseenFiles();
}
-
+
public void store () throws IOException {
impl.store();
}
-
+
void resetToNow() {
final long now = System.currentTimeMillis();
impl.reset(now);
@@ -103,7 +103,7 @@ public final class TimeStamps {
final boolean detectDeletedFiles) throws IOException {
return new TimeStamps(new RegularImpl(root, detectDeletedFiles));
}
-
+
public static TimeStamps changedTransient() throws IOException {
return new TimeStamps(new AllChangedTransientImpl());
}
@@ -118,10 +118,10 @@ public final class TimeStamps {
if (cacheDir != null) {
return new File (FileUtil.toFile(cacheDir),TIME_STAMPS_FILE).exists();
}
-
+
return false;
}
-
+
private static interface Implementation {
boolean checkAndStoreTimestamp(FileObject root, String relativePath);
Set<String> getUnseenFiles();
@@ -131,20 +131,20 @@ public final class TimeStamps {
Collection<? extends String> getEnclosedFiles(@NonNull String relativePath);
void store() throws IOException;
}
-
+
private static final class RegularImpl implements Implementation {
-
+
private final URL root;
- private final LongHashMap<String> timestamps = new LongHashMap<String>();
+ private final LongHashMap<String> timestamps = new LongHashMap<>();
private final Set<String> unseen;
private FileObject rootFoCache;
-
+
private RegularImpl(
@NonNull final URL root,
final boolean detectDeletedFiles) throws IOException {
assert root != null;
this.root = root;
- this.unseen = detectDeletedFiles ? new HashSet<String>() : null;
+ this.unseen = detectDeletedFiles ? new HashSet<>() : null;
load();
}
@@ -183,12 +183,12 @@ public final class TimeStamps {
return isUpToDate;
}
-
+
@Override
public Set<String> getUnseenFiles() {
return unseen;
}
-
+
@Override
public void reset(final long value) {
for (LongHashMap.Entry<String> entry : timestamps.entrySet()) {
@@ -206,7 +206,7 @@ public final class TimeStamps {
@NonNull
@Override
public Collection<? extends String> getEnclosedFiles(@NonNull final String relativePath) {
- final Set<String> res = new HashSet<String>();
+ final Set<String> res = new HashSet<>();
for (String filePath : timestamps.keySet()) {
if (filePath.startsWith(relativePath)) {
res.add(filePath);
@@ -224,14 +224,16 @@ public final class TimeStamps {
final File f = new File(cacheDir, TIME_STAMPS_FILE);
assert f != null;
try {
- final BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8")); //NOI18N
- try {
+ try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8")) //NOI18N
+ ) {
if (unseen != null) {
timestamps.keySet().removeAll(unseen);
}
// write version
- out.write(VERSION); //NOI18N
+ out.write(VERSION_3); //NOI18N
+ out.write(" ");
+ out.write(IndexabilityQuery.getInstance().getState());
out.newLine();
// write data
@@ -243,15 +245,13 @@ public final class TimeStamps {
}
out.flush();
- } finally {
- out.close();
}
} catch (IOException e) {
//In case of IOException props are not stored, everything is scanned next time
LOG.log(Level.FINE, null, e);
}
}
-
+
private void load () throws IOException {
final FileObject cacheFolder = CacheFolder.getDataFolder(
root,
@@ -264,32 +264,64 @@ public final class TimeStamps {
try {
boolean readOldPropertiesFormat = false;
{
- final BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8")); //NOI18N
- try {
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8")) //NOI18N
+ ) {
String line = in.readLine();
- if (line != null && line.startsWith(VERSION)) {
+ if (line != null && line.startsWith(VERSION_2)) {
// it's the new format
- LOG.log(Level.FINE, "{0}: reading {1} timestamps", new Object [] { f.getPath(), VERSION }); //NOI18N
-
- while (null != (line = in.readLine())) {
- int idx = line.indexOf('='); //NOI18N
- if (idx != -1) {
- try {
- final String path = line.substring(0, idx);
- if (!path.isEmpty() && path.charAt(0) != '/') {
- final long ts = Long.parseLong(line.substring(idx + 1));
- timestamps.put(path, ts);
- } else {
- LOG.log(
- Level.WARNING,
- "Invalid timestamp entry {0} in {1}", //NOI18N
- new Object[]{
- path,
- f.getAbsolutePath()
- });
+ LOG.log(Level.FINE, "{0}: reading {1} timestamps", new Object [] { f.getPath(), VERSION_2 }); //NOI18N
+
+ if (IndexabilityQuery.getInstance().isSameState("")) {
+ while (null != (line = in.readLine())) {
+ int idx = line.indexOf('='); //NOI18N
+ if (idx != -1) {
+ try {
+ final String path = line.substring(0, idx);
+ if (!path.isEmpty() && path.charAt(0) != '/') {
+ final long ts = Long.parseLong(line.substring(idx + 1));
+ timestamps.put(path, ts);
+ } else {
+ LOG.log(
+ Level.WARNING,
+ "Invalid timestamp entry {0} in {1}", //NOI18N
+ new Object[]{
+ path,
+ f.getAbsolutePath()
+ });
+ }
+ } catch (NumberFormatException nfe) {
+ LOG.log(Level.FINE, "Invalid timestamp: line={0}, timestamps={1}, exception={2}", new Object[]{line, f.getPath(), nfe}); //NOI18N
+ }
+ }
+ }
+ }
+ } else if (line != null && line.startsWith(VERSION_3)) {
+ // it's the new format
+ LOG.log(Level.FINE, "{0}: reading {1} timestamps", new Object[]{f.getPath(), VERSION_3}); //NOI18N
+
+ String state = line.substring(VERSION_3.length());
+
+ if (IndexabilityQuery.getInstance().isSameState(state)) {
+ while (null != (line = in.readLine())) {
+ int idx = line.indexOf('='); //NOI18N
+ if (idx != -1) {
+ try {
+ final String path = line.substring(0, idx);
+ if (!path.isEmpty() && path.charAt(0) != '/') {
+ final long ts = Long.parseLong(line.substring(idx + 1));
+ timestamps.put(path, ts);
+ } else {
+ LOG.log(
+ Level.WARNING,
+ "Invalid timestamp entry {0} in {1}", //NOI18N
+ new Object[]{
+ path,
+ f.getAbsolutePath()
+ });
+ }
+ } catch (NumberFormatException nfe) {
+ LOG.log(Level.FINE, "Invalid timestamp: line={0}, timestamps={1}, exception={2}", new Object[]{line, f.getPath(), nfe}); //NOI18N
}
- } catch (NumberFormatException nfe) {
- LOG.log(Level.FINE, "Invalid timestamp: line={0}, timestamps={1}, exception={2}", new Object[] { line, f.getPath(), nfe }); //NOI18N
}
}
}
@@ -297,19 +329,14 @@ public final class TimeStamps {
// it's the old format from Properties.store()
readOldPropertiesFormat = true;
}
- } finally {
- in.close();
}
}
if (readOldPropertiesFormat) {
LOG.log(Level.FINE, "{0}: reading old Properties timestamps", f.getPath()); //NOI18N
final Properties p = new Properties();
- final InputStream in = new FileInputStream(f);
- try {
+ try (InputStream in = new FileInputStream(f)) {
p.load(in);
- } finally {
- in.close();
}
for(Map.Entry<Object, Object> entry : p.entrySet()) {
@@ -338,7 +365,7 @@ public final class TimeStamps {
}
LOG.log(Level.FINEST, "---------------------------"); //NOI18N
}
- } catch (Exception e) {
+ } catch (IOException | RuntimeException e) {
// #176001: catching all exceptions, because j.u.Properties can throw IllegalArgumentException
// from its load() method
// In case of any exception props are empty, everything is scanned
@@ -380,5 +407,5 @@ public final class TimeStamps {
public void store() throws IOException {
}
}
-
+
}
diff --git a/ide/parsing.indexing/src/org/netbeans/modules/parsing/spi/indexing/IndexabilityQueryImplementation.java b/ide/parsing.indexing/src/org/netbeans/modules/parsing/spi/indexing/IndexabilityQueryImplementation.java
new file mode 100644
index 0000000..2ac7058
--- /dev/null
+++ b/ide/parsing.indexing/src/org/netbeans/modules/parsing/spi/indexing/IndexabilityQueryImplementation.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.netbeans.modules.parsing.spi.indexing;
+
+import java.net.URL;
+import javax.swing.event.ChangeListener;
+import org.netbeans.modules.parsing.impl.indexing.IndexabilityQueryContextAccessor;
+
+/**
+ * Determine whether files should be skipped for indexing.
+ * <p>
+ * Global lookup is used to find all instances of
+ * IndexabilityQueryImplementation.
+ * </p>
+ * <p>
+ * Threading note: implementors should avoid acquiring locks that might be held
+ * by other threads. Generally treat this interface similarly to SPIs in
+ * {@link org.openide.filesystems} with respect to threading semantics.
+ * </p>
+ *
+ * @see org.netbeans.api.queries.IndexabilityQuery
+ * @since org.netbeans.modules.parsing.indexing 9.24.0
+ */
+public interface IndexabilityQueryImplementation {
+
+ /**
+ * @return name of the IndexabilityQueryImplementation. The allowed
+ * characters are from the range: [A-Za-z0-9$_.+/].
+ **/
+ public String getName();
+
+ /**
+ * @return version of the IndexabilityQueryImplementation, must be
+ * incremented if changes in the implementation would affect indexing
+ * results
+ */
+ public int getVersion();
+
+ /**
+ * It is not the intention of this to represent the configuration,
+ * that should be stored in the configuration, but should make it possible
+ * to identify changes to the configuration. A valid identifier for example
+ * would be a checksum over the configuration. It must be ensured, that
+ * for a given configuration the same stateIdentifier is returned. It must
+ * not change between NetBeans restarts - else a full reindex will be done.
+ *
+ * @return an identifier, that identifies the state of the indexability
+ * query. The allowed characters are from the range: [A-Za-z0-9$_.+/].
+ */
+ public String getStateIdentifier();
+
+ /**
+ * Determine if the supplied indexable should be indexed using the given
+ * indexer (identified either by the name of the indexer or its factory
+ * class).
+ *
+ * @param iqp information about the indexable and circumstances of the
+ * indexing action
+ * @return
+ */
+ default boolean preventIndexing(IndexabilityQueryContext iqp) {
+ return false;
+ }
+
+ /**
+ * Add a listener to changes.
+ *
+ * @param l a listener to add
+ */
+ void addChangeListener(ChangeListener l);
+
+ /**
+ * Stop listening to changes.
+ *
+ * @param l a listener to remove
+ */
+ void removeChangeListener(ChangeListener l);
+
+ public static final class IndexabilityQueryContext {
+
+ static {
+ IndexabilityQueryContextAccessor.setInstance(new Accessor());
+ }
+
+ private final String indexerName;
+ private final URL indexable;
+ private final URL root;
+
+ private IndexabilityQueryContext(
+ URL indexable,
+ String indexerName,
+ URL rootUrl) {
+ this.indexerName = indexerName;
+ this.indexable = indexable;
+ this.root = rootUrl;
+ }
+
+ /**
+ * @return the file that is to be indexed, never {@code null}.
+ */
+ public URL getIndexable() {
+ return this.indexable;
+ }
+
+ /**
+ * @return name of the indexer that is about to index the file. Will be
+ * {@code null} to query general indexability of the supplied indexable.
+ */
+ public String getIndexerName() {
+ return this.indexerName;
+ }
+
+ /**
+ * @return the root of the scanning process, can be {@code null}
+ */
+ public URL getRoot() {
+ return this.root;
+ }
+
+ private static class Accessor extends IndexabilityQueryContextAccessor {
+
+ @Override
+ public IndexabilityQueryContext createContext(URL indexable, String indexerName, URL root) {
+ return new IndexabilityQueryContext(indexable, indexerName, root);
+ }
+
+ }
+ }
+
+}
diff --git a/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdaterTest.java b/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdaterTest.java
index 52337c8..7c70cf8 100644
--- a/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdaterTest.java
+++ b/ide/parsing.indexing/test/unit/src/org/netbeans/modules/parsing/impl/indexing/RepositoryUpdaterTest.java
@@ -100,6 +100,7 @@ import org.netbeans.modules.parsing.spi.indexing.CustomIndexer;
import org.netbeans.modules.parsing.spi.indexing.CustomIndexerFactory;
import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexer;
import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory;
+import org.netbeans.modules.parsing.spi.indexing.IndexabilityQueryImplementation;
import org.netbeans.modules.parsing.spi.indexing.Indexable;
import org.netbeans.modules.parsing.spi.indexing.PathRecognizer;
import org.netbeans.modules.parsing.spi.indexing.support.QuerySupport;
@@ -197,8 +198,9 @@ public class RepositoryUpdaterTest extends IndexingTestBase {
SFBQImpl.class,
ClassPathProviderImpl.class,
Visibility.class,
+ Indexability.class,
IndexDownloaderImpl.class,
- IndexPatcherImpl.class
+ IndexPatcherImpl.class,
})
);
}
@@ -1713,6 +1715,101 @@ public class RepositoryUpdaterTest extends IndexingTestBase {
assertTrue(indexerFactory.indexer.awaitDeleted(TIME));
}
+ public void testIndexabilityQueryInIDERun() throws Exception {
+ final RepositoryUpdater ru = RepositoryUpdater.getDefault();
+ assertEquals(0, ru.getScannedBinaries().size());
+ assertEquals(0, ru.getScannedBinaries().size());
+ assertEquals(0, ru.getScannedUnknowns().size());
+
+ final TestHandler handler = new TestHandler();
+ final Logger logger = Logger.getLogger(RepositoryUpdater.class.getName()+".tests");
+ logger.setLevel (Level.FINEST);
+ logger.addHandler(handler);
+
+ ////1st) Default visibility everything should be scanned
+ indexerFactory.indexer.setExpectedFile(customFiles, new URL[0], new URL[0]);
+ eindexerFactory.indexer.setExpectedFile(embeddedFiles, new URL[0], new URL[0]);
+ final MutableClassPathImplementation mcpi1 = new MutableClassPathImplementation ();
+ mcpi1.addResource(this.srcRootWithFiles1);
+ final ClassPath cp1 = ClassPathFactory.createClassPath(mcpi1);
+ globalPathRegistry_register(SOURCES,new ClassPath[]{cp1});
+ assertTrue (handler.await());
+ assertEquals(0, handler.getBinaries().size());
+ assertEquals(1, handler.getSources().size());
+ assertEquals(this.srcRootWithFiles1.toURL(), handler.getSources().get(0));
+ assertTrue(indexerFactory.indexer.awaitIndex(TIME));
+ assertTrue(eindexerFactory.indexer.awaitIndex());
+
+ //2nd) Change of VisibilityQuery should trigger rescan,
+ // customFiles[1] should be invisible and have been removed from
+ // index
+ final Indexability indexability = Lookup.getDefault().lookup(Indexability.class);
+ assertNotNull(indexability);
+ indexability.registerInvisibles(Collections.singleton(URLMapper.findFileObject(customFiles[1])));
+ handler.reset();
+ indexerFactory.indexer.deletedCounter = 0;
+ eindexerFactory.indexer.deletedCounter = 0;
+ indexerFactory.indexer.setExpectedFile(new URL[0], new URL[] {customFiles[1]}, new URL[0]);
+ eindexerFactory.indexer.setExpectedFile(new URL[0], new URL[0], new URL[0]);
+ assertTrue (handler.await());
+ assertEquals(0, handler.getBinaries().size());
+ assertEquals(1, handler.getSources().size());
+ assertEquals(this.srcRootWithFiles1.toURL(), handler.getSources().get(0));
+ assertTrue(indexerFactory.indexer.awaitIndex(TIME));
+ assertTrue(eindexerFactory.indexer.awaitIndex());
+ assertTrue(indexerFactory.indexer.awaitDeleted(TIME));
+ assertEquals(1, eindexerFactory.indexer.deletedCounter);
+
+ //3rd) Change of VisibilityQuery should trigger rescan, customFiles[1] should be visible again
+ indexability.registerInvisibles(Collections.<FileObject>emptySet());
+ handler.reset();
+ indexerFactory.indexer.setExpectedFile(new URL[]{customFiles[1]}, new URL[0], new URL[0]);
+ eindexerFactory.indexer.setExpectedFile(new URL[0], new URL[0], new URL[0]);
+ assertTrue (handler.await());
+ assertEquals(0, handler.getBinaries().size());
+ assertEquals(1, handler.getSources().size());
+ assertEquals(this.srcRootWithFiles1.toURL(), handler.getSources().get(0));
+ assertTrue(indexerFactory.indexer.awaitIndex(TIME));
+ assertTrue(eindexerFactory.indexer.awaitIndex());
+ assertTrue(indexerFactory.indexer.awaitDeleted(TIME));
+ }
+
+ public void testScannerBasedFilter() throws Exception {
+ final RepositoryUpdater ru = RepositoryUpdater.getDefault();
+ assertEquals(0, ru.getScannedBinaries().size());
+ assertEquals(0, ru.getScannedBinaries().size());
+ assertEquals(0, ru.getScannedUnknowns().size());
+
+ final TestHandler handler = new TestHandler();
+ final Logger logger = Logger.getLogger(RepositoryUpdater.class.getName()+".tests");
+ logger.setLevel (Level.FINEST);
+ logger.addHandler(handler);
+
+ // Block indexing for the "emb" indexer only
+ final Indexability indexability = Lookup.getDefault().lookup(Indexability.class);
+ assertNotNull(indexability);
+ indexability.registerInvisiblesByIndexer(Collections.singleton("emb"));
+
+ indexerFactory.indexer.setExpectedFile(new URL[0], new URL[0], new URL[0]);
+ eindexerFactory.indexer.setExpectedFile(new URL[0], new URL[0], new URL[0]);
+ final MutableClassPathImplementation mcpi1 = new MutableClassPathImplementation ();
+ mcpi1.addResource(this.srcRootWithFiles1);
+ final ClassPath cp1 = ClassPathFactory.createClassPath(mcpi1);
+ globalPathRegistry_register(SOURCES,new ClassPath[]{cp1});
+ assertTrue (handler.await());
+ assertEquals(0, handler.getBinaries().size());
+ assertEquals(1, handler.getSources().size());
+ assertEquals(this.srcRootWithFiles1.toURL(), handler.getSources().get(0));
+ assertTrue(indexerFactory.indexer.awaitIndex(TIME));
+ assertTrue(eindexerFactory.indexer.awaitIndex());
+
+ // srcRootWithFiles1 has to files that are scanned by the foo indexer
+ // and two that are scanned by the emb indexer. The latter is blocked,
+ // so should report 0 files indexed
+ assertEquals(2, indexerFactory.indexer.getIndexCount());
+ assertEquals(0, eindexerFactory.indexer.getIndexCount());
+ }
+
/**
* Test that unknown source roots are registered in the scannedRoots2Dependencies with EMPTY_DEPS
* and when the unknown root registers it's changed to regular dependencies.
@@ -2481,14 +2578,10 @@ public class RepositoryUpdaterTest extends IndexingTestBase {
}
public void reset(final Type t) {
- sources = null;
- binaries = null;
- type = t;
if (t == Type.BATCH) {
- latch = new CountDownLatch(2);
- }
- else {
- latch = new CountDownLatch(1);
+ reset(t, 2);
+ } else {
+ reset(t, 1);
}
}
@@ -2507,10 +2600,12 @@ public class RepositoryUpdaterTest extends IndexingTestBase {
return latch.await(time, TimeUnit.MILLISECONDS);
}
+ @SuppressWarnings("ReturnOfCollectionOrArrayField")
public Set<URL> getBinaries () {
return this.binaries;
}
+ @SuppressWarnings("ReturnOfCollectionOrArrayField")
public List<URL> getSources() {
return this.sources;
}
@@ -3561,6 +3656,67 @@ public class RepositoryUpdaterTest extends IndexingTestBase {
changeSupport.removeChangeListener(l);
}
}
+
+ public static class Indexability implements IndexabilityQueryImplementation {
+
+ private final Collection<FileObject> invisible =
+ Collections.synchronizedList( new ArrayList<>());
+
+ private final Collection<String> blockedIndexers =
+ Collections.synchronizedList( new ArrayList<>());
+
+ private final ChangeSupport changeSupport = new ChangeSupport(this);
+
+ public Indexability() {
+ }
+
+ public void registerInvisibles (final Collection<? extends FileObject> invisibles) {
+ synchronized (invisible) {
+ invisible.clear();
+ invisible.addAll(invisibles);
+ }
+ changeSupport.fireChange();
+ }
+
+ public void registerInvisiblesByIndexer (final Collection<? extends String> blockedIndexersNew) {
+ synchronized (blockedIndexers) {
+ blockedIndexers.clear();
+ blockedIndexers.addAll(blockedIndexersNew);
+ }
+ changeSupport.fireChange();
+ }
+
+ @Override
+ public boolean preventIndexing(IndexabilityQueryContext iqc) {
+ return invisible.contains(URLMapper.findFileObject(iqc.getIndexable()))
+ || blockedIndexers.contains(iqc.getIndexerName());
+ }
+
+ @Override
+ public void addChangeListener(ChangeListener l) {
+ changeSupport.addChangeListener(l);
+ }
+
+ @Override
+ public void removeChangeListener(ChangeListener l) {
+ changeSupport.removeChangeListener(l);
+ }
+
+ @Override
+ public String getName() {
+ return getClass().getName();
+ }
+
+ @Override
+ public int getVersion() {
+ return 1;
+ }
+
+ @Override
+ public String getStateIdentifier() {
+ return "NO_CONFIGURATION";
+ }
+ }
public static class IndexDownloaderImpl implements IndexDownloader {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@netbeans.apache.org
For additional commands, e-mail: commits-help@netbeans.apache.org
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists