You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@netbeans.apache.org by po...@apache.org on 2017/09/03 17:33:34 UTC

[41/51] [partial] incubator-netbeans-jackpot30 git commit: INFRA-15006 Import for http://bits.netbeans.org/download/apache-donation

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/nbproject/suite.properties
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/nbproject/suite.properties b/duplicates/ide/impl/nbproject/suite.properties
new file mode 100644
index 0000000..942e12b
--- /dev/null
+++ b/duplicates/ide/impl/nbproject/suite.properties
@@ -0,0 +1,42 @@
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+#
+# Copyright 2009-2017 Oracle and/or its affiliates. All rights reserved.
+#
+# Oracle and Java are registered trademarks of Oracle and/or its affiliates.
+# Other names may be trademarks of their respective owners.
+#
+# The contents of this file are subject to the terms of either the GNU
+# General Public License Version 2 only ("GPL") or the Common
+# Development and Distribution License("CDDL") (collectively, the
+# "License"). You may not use this file except in compliance with the
+# License. You can obtain a copy of the License at
+# http://www.netbeans.org/cddl-gplv2.html
+# or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+# specific language governing permissions and limitations under the
+# License.  When distributing the software, include this License Header
+# Notice in each file and include the License file at
+# nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the GPL Version 2 section of the License file that
+# accompanied this code. If applicable, add the following below the
+# License Header, with the fields enclosed by brackets [] replaced by
+# your own identifying information:
+# "Portions Copyrighted [year] [name of copyright owner]"
+#
+# Contributor(s):
+#
+# The Original Software is NetBeans. The Initial Developer of the Original
+# Software is Sun Microsystems, Inc. Portions Copyright 2009-2010 Sun
+# Microsystems, Inc. All Rights Reserved.
+#
+# If you wish your version of this file to be governed by only the CDDL
+# or only the GPL Version 2, indicate your decision by adding
+# "[Contributor] elects to include this software in this distribution
+# under the [CDDL or GPL Version 2] license." If you do not indicate a
+# single choice of license, a recipient has the option to distribute
+# your version of this file under either the CDDL, the GPL Version 2 or
+# to extend the choice of license to its licensees as provided above.
+# However, if you add GPL Version 2 code and therefore, elected the GPL
+# Version 2 license, then the option applies only if the new code is
+# made subject to such option by the copyright holder.
+suite.dir=${basedir}/..

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/duplicates/impl/Bundle.properties
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/duplicates/impl/Bundle.properties b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/duplicates/impl/Bundle.properties
new file mode 100644
index 0000000..2c4cb92
--- /dev/null
+++ b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/duplicates/impl/Bundle.properties
@@ -0,0 +1,42 @@
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+#
+# Copyright 2009-2017 Oracle and/or its affiliates. All rights reserved.
+#
+# Oracle and Java are registered trademarks of Oracle and/or its affiliates.
+# Other names may be trademarks of their respective owners.
+#
+# The contents of this file are subject to the terms of either the GNU
+# General Public License Version 2 only ("GPL") or the Common
+# Development and Distribution License("CDDL") (collectively, the
+# "License"). You may not use this file except in compliance with the
+# License. You can obtain a copy of the License at
+# http://www.netbeans.org/cddl-gplv2.html
+# or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+# specific language governing permissions and limitations under the
+# License.  When distributing the software, include this License Header
+# Notice in each file and include the License file at
+# nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the GPL Version 2 section of the License file that
+# accompanied this code. If applicable, add the following below the
+# License Header, with the fields enclosed by brackets [] replaced by
+# your own identifying information:
+# "Portions Copyrighted [year] [name of copyright owner]"
+#
+# Contributor(s):
+#
+# The Original Software is NetBeans. The Initial Developer of the Original
+# Software is Sun Microsystems, Inc. Portions Copyright 2009-2010 Sun
+# Microsystems, Inc. All Rights Reserved.
+#
+# If you wish your version of this file to be governed by only the CDDL
+# or only the GPL Version 2, indicate your decision by adding
+# "[Contributor] elects to include this software in this distribution
+# under the [CDDL or GPL Version 2] license." If you do not indicate a
+# single choice of license, a recipient has the option to distribute
+# your version of this file under either the CDDL, the GPL Version 2 or
+# to extend the choice of license to its licensees as provided above.
+# However, if you add GPL Version 2 code and therefore, elected the GPL
+# Version 2 license, then the option applies only if the new code is
+# made subject to such option by the copyright holder.
+OpenIDE-Module-Name=Jackpot 3.0 Duplicates Impl

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/Bundle.properties
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/Bundle.properties b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/Bundle.properties
new file mode 100644
index 0000000..76b859b
--- /dev/null
+++ b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/Bundle.properties
@@ -0,0 +1,44 @@
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+#
+# Copyright 2009-2017 Oracle and/or its affiliates. All rights reserved.
+#
+# Oracle and Java are registered trademarks of Oracle and/or its affiliates.
+# Other names may be trademarks of their respective owners.
+#
+# The contents of this file are subject to the terms of either the GNU
+# General Public License Version 2 only ("GPL") or the Common
+# Development and Distribution License("CDDL") (collectively, the
+# "License"). You may not use this file except in compliance with the
+# License. You can obtain a copy of the License at
+# http://www.netbeans.org/cddl-gplv2.html
+# or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+# specific language governing permissions and limitations under the
+# License.  When distributing the software, include this License Header
+# Notice in each file and include the License file at
+# nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the GPL Version 2 section of the License file that
+# accompanied this code. If applicable, add the following below the
+# License Header, with the fields enclosed by brackets [] replaced by
+# your own identifying information:
+# "Portions Copyrighted [year] [name of copyright owner]"
+#
+# Contributor(s):
+#
+# The Original Software is NetBeans. The Initial Developer of the Original
+# Software is Sun Microsystems, Inc. Portions Copyright 2009-2010 Sun
+# Microsystems, Inc. All Rights Reserved.
+#
+# If you wish your version of this file to be governed by only the CDDL
+# or only the GPL Version 2, indicate your decision by adding
+# "[Contributor] elects to include this software in this distribution
+# under the [CDDL or GPL Version 2] license." If you do not indicate a
+# single choice of license, a recipient has the option to distribute
+# your version of this file under either the CDDL, the GPL Version 2 or
+# to extend the choice of license to its licensees as provided above.
+# However, if you add GPL Version 2 code and therefore, elected the GPL
+# Version 2 license, then the option applies only if the new code is
+# made subject to such option by the copyright holder.
+CTL_GlobalFindDuplicates=Global Find Duplicates
+DuplicatesListPanel.findMore.text=<html><body><a href="">Look for More</a>
+DuplicatesListPanel.progressLabel.text=

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/ComputeDuplicates.java
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/ComputeDuplicates.java b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/ComputeDuplicates.java
new file mode 100644
index 0000000..be2eeea
--- /dev/null
+++ b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/ComputeDuplicates.java
@@ -0,0 +1,547 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2009-2010 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License.  When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2009-2010 Sun Microsystems, Inc.
+ */
+package org.netbeans.modules.jackpot30.impl.duplicates;
+
+import com.sun.source.tree.CompilationUnitTree;
+import com.sun.source.tree.IdentifierTree;
+import com.sun.source.tree.NewClassTree;
+import com.sun.source.tree.Tree;
+import com.sun.source.tree.VariableTree;
+import com.sun.source.util.SourcePositions;
+import com.sun.source.util.TreePath;
+import com.sun.source.util.TreePathScanner;
+import com.sun.source.util.Trees;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.Modifier;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.FSDirectory;
+import org.netbeans.api.annotations.common.CheckForNull;
+import org.netbeans.api.annotations.common.NonNull;
+import org.netbeans.api.java.classpath.ClassPath;
+import org.netbeans.api.java.classpath.GlobalPathRegistry;
+import org.netbeans.api.java.source.CompilationInfo;
+import org.netbeans.api.progress.ProgressHandle;
+import org.netbeans.modules.jackpot30.common.api.LuceneHelpers.BitSetCollector;
+import org.netbeans.modules.jackpot30.impl.duplicates.indexing.DuplicatesCustomIndexerImpl;
+import org.netbeans.modules.jackpot30.impl.duplicates.indexing.DuplicatesIndex;
+import org.netbeans.modules.parsing.impl.indexing.CacheFolder;
+import org.openide.filesystems.FileObject;
+import org.openide.filesystems.FileUtil;
+import org.openide.filesystems.URLMapper;
+import org.openide.util.Exceptions;
+
+
+/**
+ *
+ * @author lahvac
+ */
+public class ComputeDuplicates {
+
+    public Iterator<? extends DuplicateDescription> computeDuplicatesForAllOpenedProjects(ProgressHandle progress, AtomicBoolean cancel) throws IOException {
+        Set<URL> urls = new HashSet<URL>();
+
+        for (ClassPath cp : GlobalPathRegistry.getDefault().getPaths(ClassPath.SOURCE)) {
+            for (ClassPath.Entry e : cp.entries()) {
+                urls.add(e.getURL());
+            }
+        }
+
+        long start = System.currentTimeMillis();
+        try {
+            return computeDuplicates(urls, progress, cancel);
+        } finally {
+            System.err.println("duplicates for all open projects: " + (System.currentTimeMillis() - start));
+        }
+    }
+
+    public Iterator<? extends DuplicateDescription> computeDuplicates(Set<URL> forURLs, ProgressHandle progress, AtomicBoolean cancel) throws IOException {
+        Map<IndexReader, FileObject> readers2Roots = new LinkedHashMap<IndexReader, FileObject>();
+
+        progress.progress("Updating indices");
+
+        for (URL u : forURLs) {
+            try {
+                //TODO: needs to be removed for server mode
+                new DuplicatesCustomIndexerImpl.FactoryImpl().updateIndex(u, cancel); //TODO: show updating progress to the user
+                
+                File cacheRoot = cacheRoot(u);
+
+                File dir = new File(cacheRoot, DuplicatesIndex.NAME);
+
+                if (dir.listFiles() != null && dir.listFiles().length > 0) {
+                    IndexReader reader = IndexReader.open(FSDirectory.open(dir), true);
+
+                    readers2Roots.put(reader, URLMapper.findFileObject(u));
+                }
+            } catch (IOException ex) {
+                Exceptions.printStackTrace(ex);
+            }
+        }
+        
+        progress.progress("Searching for duplicates");
+
+        MultiReader r = new MultiReader(readers2Roots.keySet().toArray(new IndexReader[0]));
+
+        List<String> dd = new ArrayList<String>(getDuplicatedValues(r, "duplicatesGeneralized", cancel));
+
+        sortHashes(dd);
+
+        //TODO: only show valuable duplicates?:
+//        dd = dd.subList(0, dd.size() / 10 + 1);
+
+        return new DuplicatesIterator(readers2Roots, dd, 2);
+    }
+
+    public static Iterator<? extends DuplicateDescription> XXXduplicatesOf(Map<IndexReader, FileObject> readers2Roots, Collection<String> hashes) {
+        List<String> hashesList = new ArrayList<String>(hashes);
+        sortHashes(hashesList);
+        return new DuplicatesIterator(readers2Roots, hashesList, 1);
+    }
+
+    private static File cacheRoot(URL sourceRoot) throws IOException {
+        FileObject dataFolder = CacheFolder.getDataFolder(sourceRoot);
+        FileObject cacheFO  = dataFolder.getFileObject(DuplicatesIndex.NAME + "/" +DuplicatesIndex.VERSION);
+        File cache = cacheFO != null ? FileUtil.toFile(cacheFO) : null;
+        
+        return cache;
+    }
+    
+    private static final class DuplicatesIterator implements Iterator<DuplicateDescription> {
+        private final Map<IndexReader, FileObject> readers2Roots;
+        private final Iterator<String> duplicateCandidates;
+        private final int minDuplicates;
+        private final List<DuplicateDescription> result = new LinkedList<DuplicateDescription>();
+
+        public DuplicatesIterator(Map<IndexReader, FileObject> readers2Roots, Iterable<String> duplicateCandidates, int minDuplicates) {
+            this.readers2Roots = readers2Roots;
+            this.duplicateCandidates = duplicateCandidates.iterator();
+            this.minDuplicates = minDuplicates;
+        }
+
+        private DuplicateDescription nextDescription() throws IOException {
+        while (duplicateCandidates.hasNext()) {
+            String longest = duplicateCandidates.next();
+            List<Span> foundDuplicates = new LinkedList<Span>();
+
+            Query query = new TermQuery(new Term("duplicatesGeneralized", longest));
+
+            for (Entry<IndexReader, FileObject> e : readers2Roots.entrySet()) {
+                Searcher s = new IndexSearcher(e.getKey());
+                BitSet matchingDocuments = new BitSet(e.getKey().maxDoc());
+                Collector c = new BitSetCollector(matchingDocuments);
+
+                s.search(query, c);
+
+                for (int docNum = matchingDocuments.nextSetBit(0); docNum >= 0; docNum = matchingDocuments.nextSetBit(docNum + 1)) {
+                    final Document doc = e.getKey().document(docNum);
+                    int pos = Arrays.binarySearch(doc.getValues("duplicatesGeneralized"), longest);
+
+                    if (pos < 0) {
+                        continue;
+                    }
+                    
+                    String spanSpec = doc.getValues("duplicatesPositions")[pos];
+                    String relPath = doc.getField("duplicatesPath").stringValue();
+
+                    for (String spanPart : spanSpec.split(";")) {
+                        Span span = Span.of(e.getValue().getFileObject(relPath), spanPart);
+
+                        if (span != null) {
+                            foundDuplicates.add(span);
+                        }
+                    }
+                }
+            }
+
+            if (foundDuplicates.size() >= minDuplicates) {
+                DuplicateDescription current = DuplicateDescription.of(foundDuplicates, getValue(longest), longest);
+                boolean add = true;
+
+                for (Iterator<DuplicateDescription> it = result.iterator(); it.hasNext();) {
+                    DuplicateDescription existing = it.next();
+
+                    if (subsumes(existing, current)) {
+                        add = false;
+                        break;
+                    }
+
+                    if (subsumes(current, existing)) {
+                        //can happen? (note that the duplicates are sorted by value)
+                        it.remove();
+                    }
+                }
+
+                if (add) {
+                    result.add(current);
+                    return current;
+                }
+            }
+
+        }
+        return null;
+        }
+
+        private DuplicateDescription next;
+
+        public boolean hasNext() {
+            if (next == null) {
+                try {
+                    next = nextDescription();
+                } catch (IOException ex) {
+                    Exceptions.printStackTrace(ex);
+                }
+            }
+
+            return next != null;
+        }
+
+        public DuplicateDescription next() {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+
+            DuplicateDescription r = next;
+
+            next = null;
+            return r;
+        }
+
+        public void remove() {
+            throw new UnsupportedOperationException("Not supported.");
+        }
+
+    }
+
+    private static List<String> getDuplicatedValues(IndexReader ir, String field, AtomicBoolean cancel) throws IOException {
+        List<String> values = new ArrayList<String>();
+        TermEnum terms = ir.terms( new Term(field));
+        //while (terms.next()) {
+        do {
+            if (cancel.get()) return Collections.emptyList();
+
+            final Term term =  terms.term();
+
+            if ( !field.equals( term.field() ) ) {
+                break;
+            }
+
+            if (terms.docFreq() < 2) continue;
+
+            values.add(term.text());
+        }
+        while (terms.next());
+        return values;
+    }
+
+    private static long getValue(String encoded) {
+        return Long.parseLong(encoded.substring(encoded.lastIndexOf(":") + 1));
+    }
+
+    private static void sortHashes(List<String> hashes) {
+        Collections.sort(hashes, new Comparator<String>() {
+            public int compare(String arg0, String arg1) {
+                return (int) Math.signum(getValue(arg1) - getValue(arg0));
+            }
+        });
+    }
+    
+    private static boolean subsumes(DuplicateDescription bigger, DuplicateDescription smaller) {
+        Set<FileObject> bFiles = new HashSet<FileObject>();
+
+        for (Span s : bigger.dupes) {
+            bFiles.add(s.file);
+        }
+
+        Set<FileObject> sFiles = new HashSet<FileObject>();
+
+        for (Span s : smaller.dupes) {
+            sFiles.add(s.file);
+        }
+
+        if (!bFiles.equals(sFiles)) return false;
+
+        Span testAgainst = bigger.dupes.get(0);
+
+        for (Span s : smaller.dupes) {
+            if (s.file == testAgainst.file) {
+                if (   (testAgainst.startOff <= s.startOff && testAgainst.endOff > s.endOff)
+                    || (testAgainst.startOff < s.startOff && testAgainst.endOff >= s.endOff)) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    public static Map<String, long[]> encodeGeneralized(CompilationInfo info) {
+        return encodeGeneralized(info.getTrees(), info.getCompilationUnit());
+    }
+
+    public static Map<String, long[]> encodeGeneralized(final Trees trees, final CompilationUnitTree cut) {
+        final SourcePositions sp = trees.getSourcePositions();
+        final Map<String, Collection<Long>> positions = new HashMap<String, Collection<Long>>();
+
+        new TreePathScanner<Void, Void>() {
+            @Override
+            public Void scan(Tree tree, Void p) {
+                if (tree == null) return null;
+                if (getCurrentPath() != null) {
+                    DigestOutputStream baos = null;
+                    PrintWriter out = null;
+                    try {
+                        baos = new DigestOutputStream(new ByteArrayOutputStream(), MessageDigest.getInstance("MD5"));
+                        out = new PrintWriter(new OutputStreamWriter(baos, "UTF-8"));
+                        GeneralizePattern gen = new GeneralizePattern(out, trees);
+                        gen.scan(new TreePath(getCurrentPath(), tree), null);
+                        out.close();
+                        if (gen.value >= MINIMAL_VALUE) {
+                            StringBuilder text = new StringBuilder();
+                            byte[] bytes = baos.getMessageDigest().digest();
+                            for (int cntr = 0; cntr < 4; cntr++) {
+                                text.append(String.format("%02X", bytes[cntr]));
+                            }
+                            text.append(':').append(gen.value);
+                            String enc = text.toString();
+                            Collection<Long> spanSpecs = positions.get(enc);
+                            if (spanSpecs == null) {
+                                positions.put(enc, spanSpecs = new LinkedList<Long>());
+//                            } else {
+//                                spanSpecs.append(";");
+                            }
+                            long start = sp.getStartPosition(cut, tree);
+//                            spanSpecs.append(start).append(":").append(sp.getEndPosition(cut, tree) - start);
+                            spanSpecs.add(start);
+                            spanSpecs.add(sp.getEndPosition(cut, tree));
+                        }
+                    } catch (UnsupportedEncodingException ex) {
+                        Exceptions.printStackTrace(ex);
+                    } catch (NoSuchAlgorithmException ex) {
+                        Exceptions.printStackTrace(ex);
+                    } finally {
+                        try {
+                            baos.close();
+                        } catch (IOException ex) {
+                            Exceptions.printStackTrace(ex);
+                        }
+                        out.close();
+                    }
+                }
+                return super.scan(tree, p);
+            }
+        }.scan(cut, null);
+
+        Map<String, long[]> result = new TreeMap<String, long[]>();
+
+        for (Entry<String, Collection<Long>> e : positions.entrySet()) {
+            long[] spans = new long[e.getValue().size()];
+            int idx = 0;
+
+            for (Long l : e.getValue()) {
+                spans[idx++] = l;
+            }
+
+            result.put(e.getKey(), spans);
+        }
+
+        return result;
+    }
+
+    private static final class GeneralizePattern extends TreePathScanner<Void, Void> {
+
+        public final Map<Tree, Tree> tree2Variable = new HashMap<Tree, Tree>();
+        private final Map<Element, String> element2Variable = new HashMap<Element, String>();
+        private final PrintWriter to;
+        private final Trees javacTrees;
+        private long value;
+
+        private int currentVariableIndex = 0;
+
+        public GeneralizePattern(PrintWriter to, Trees javacTrees) {
+            this.to = to;
+            this.javacTrees = javacTrees;
+        }
+
+        private @NonNull String getVariable(@NonNull Element el) {
+            String var = element2Variable.get(el);
+
+            if (var == null) {
+                element2Variable.put(el, var = "$" + currentVariableIndex++);
+            }
+
+            return var;
+        }
+
+        private boolean shouldBeGeneralized(@NonNull Element el) {
+            if (el.getModifiers().contains(Modifier.PRIVATE)) {
+                return true;
+            }
+
+            switch (el.getKind()) {
+                case LOCAL_VARIABLE:
+                case EXCEPTION_PARAMETER:
+                case PARAMETER:
+                    return true;
+            }
+
+            return false;
+        }
+
+        @Override
+        public Void scan(Tree tree, Void p) {
+            if (tree != null) {
+                to.append(tree.getKind().name());
+                value++;
+            }
+            return super.scan(tree, p);
+        }
+
+        @Override
+        public Void visitIdentifier(IdentifierTree node, Void p) {
+            Element e = javacTrees.getElement(getCurrentPath());
+
+            if (e != null && shouldBeGeneralized(e)) {
+                to.append(getVariable(e));
+                value--;
+                return null;
+            } else {
+                to.append(node.getName());
+            }
+
+            return super.visitIdentifier(node, p);
+        }
+
+        @Override
+        public Void visitVariable(VariableTree node, Void p) {
+            Element e = javacTrees.getElement(getCurrentPath());
+
+            if (e != null && shouldBeGeneralized(e)) {
+                to.append(getVariable(e));
+            } else {
+                to.append(node.getName());
+            }
+
+            return super.visitVariable(node, p);
+        }
+
+        @Override
+        public Void visitNewClass(NewClassTree node, Void p) {
+            return null;
+        }
+
+    }
+
+    private static final int MINIMAL_VALUE = 10;
+
+    public static final class DuplicateDescription {
+
+        public final List<Span> dupes;
+        public final long value;
+        public final String hash;
+
+        private DuplicateDescription(List<Span> dupes, long value, String hash) {
+            this.dupes = dupes;
+            this.value = value;
+            this.hash = hash;
+        }
+
+        public static DuplicateDescription of(List<Span> dupes, long value, String hash) {
+            return new DuplicateDescription(dupes, value, hash);
+        }
+    }
+
+    public static final class Span {
+        public final FileObject file;
+        public final int startOff;
+        public final int endOff;
+
+        public Span(FileObject file, int startOff, int endOff) {
+            this.file = file;
+            this.startOff = startOff;
+            this.endOff = endOff;
+        }
+
+        public static @CheckForNull Span of(FileObject file, String spanSpec) {
+            String[] split = spanSpec.split(":");
+            int start = Integer.valueOf(split[0]);
+            int end = start + Integer.valueOf(split[1]);
+            if (start < 0 || end < 0) return null; //XXX
+
+            return new Span(file, start, end);
+        }
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.form
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.form b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.form
new file mode 100644
index 0000000..c30a99c
--- /dev/null
+++ b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.form
@@ -0,0 +1,189 @@
+<?xml version="1.1" encoding="UTF-8" ?>
+
+<Form version="1.5" maxVersion="1.7" type="org.netbeans.modules.form.forminfo.JPanelFormInfo">
+  <AuxValues>
+    <AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/>
+    <AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
+    <AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
+    <AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="false"/>
+    <AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="true"/>
+    <AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
+    <AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
+    <AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
+    <AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
+  </AuxValues>
+
+  <Layout>
+    <DimensionLayout dim="0">
+      <Group type="103" groupAlignment="0" attributes="0">
+          <Group type="102" alignment="1" attributes="0">
+              <EmptySpace max="-2" attributes="0"/>
+              <Group type="103" groupAlignment="1" attributes="0">
+                  <Component id="mainSplit2" alignment="0" pref="906" max="32767" attributes="0"/>
+                  <Component id="jScrollPane1" alignment="0" pref="906" max="32767" attributes="0"/>
+                  <Component id="jPanel1" alignment="1" min="-2" max="-2" attributes="0"/>
+              </Group>
+              <EmptySpace max="-2" attributes="0"/>
+          </Group>
+      </Group>
+    </DimensionLayout>
+    <DimensionLayout dim="1">
+      <Group type="103" groupAlignment="0" attributes="0">
+          <Group type="102" attributes="0">
+              <EmptySpace min="-2" max="-2" attributes="0"/>
+              <Component id="jScrollPane1" min="-2" pref="67" max="-2" attributes="0"/>
+              <EmptySpace min="-2" max="-2" attributes="0"/>
+              <Component id="mainSplit2" pref="467" max="32767" attributes="0"/>
+              <EmptySpace type="unrelated" min="-2" max="-2" attributes="0"/>
+              <Component id="jPanel1" min="-2" max="-2" attributes="0"/>
+              <EmptySpace min="-2" max="-2" attributes="0"/>
+          </Group>
+      </Group>
+    </DimensionLayout>
+  </Layout>
+  <SubComponents>
+    <Container class="javax.swing.JScrollPane" name="jScrollPane1">
+
+      <Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
+      <SubComponents>
+        <Component class="javax.swing.JList" name="duplicatesList">
+          <Properties>
+            <Property name="model" type="javax.swing.ListModel" editor="org.netbeans.modules.form.editors2.ListModelEditor">
+              <StringArray count="0"/>
+            </Property>
+            <Property name="prototypeCellValue" type="java.lang.Object" editor="org.netbeans.modules.form.RADConnectionPropertyEditor">
+              <Connection code="&quot;9999999999999999999999999999999999999999999999999999999999999999999999&quot;" type="code"/>
+            </Property>
+            <Property name="visibleRowCount" type="int" value="4"/>
+          </Properties>
+        </Component>
+      </SubComponents>
+    </Container>
+    <Container class="javax.swing.JSplitPane" name="mainSplit2">
+      <Properties>
+        <Property name="dividerLocation" type="int" value="400"/>
+      </Properties>
+      <AuxValues>
+        <AuxValue name="JavaCodeGenerator_CreateCodeCustom" type="java.lang.String" value="new BalancedSplitPane()"/>
+      </AuxValues>
+
+      <Layout class="org.netbeans.modules.form.compat2.layouts.support.JSplitPaneSupportLayout"/>
+      <SubComponents>
+        <Container class="javax.swing.JPanel" name="rightPanel">
+          <Constraints>
+            <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.support.JSplitPaneSupportLayout" value="org.netbeans.modules.form.compat2.layouts.support.JSplitPaneSupportLayout$JSplitPaneConstraintsDescription">
+              <JSplitPaneConstraints position="right"/>
+            </Constraint>
+          </Constraints>
+
+          <Layout class="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout"/>
+          <SubComponents>
+            <Component class="javax.swing.JComboBox" name="rightFileList">
+              <Properties>
+                <Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor">
+                  <StringArray count="0"/>
+                </Property>
+              </Properties>
+              <Constraints>
+                <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
+                  <GridBagConstraints gridX="0" gridY="0" gridWidth="1" gridHeight="1" fill="2" ipadX="324" ipadY="0" insetsTop="0" insetsLeft="0" insetsBottom="0" insetsRight="0" anchor="18" weightX="1.0" weightY="0.0"/>
+                </Constraint>
+              </Constraints>
+            </Component>
+            <Container class="javax.swing.JScrollPane" name="jScrollPane3">
+              <AuxValues>
+                <AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
+              </AuxValues>
+              <Constraints>
+                <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
+                  <GridBagConstraints gridX="0" gridY="1" gridWidth="1" gridHeight="1" fill="1" ipadX="0" ipadY="0" insetsTop="6" insetsLeft="0" insetsBottom="0" insetsRight="0" anchor="18" weightX="1.0" weightY="1.0"/>
+                </Constraint>
+              </Constraints>
+
+              <Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
+              <SubComponents>
+                <Component class="javax.swing.JEditorPane" name="right">
+                </Component>
+              </SubComponents>
+            </Container>
+          </SubComponents>
+        </Container>
+        <Container class="javax.swing.JPanel" name="leftPanel">
+          <Constraints>
+            <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.support.JSplitPaneSupportLayout" value="org.netbeans.modules.form.compat2.layouts.support.JSplitPaneSupportLayout$JSplitPaneConstraintsDescription">
+              <JSplitPaneConstraints position="left"/>
+            </Constraint>
+          </Constraints>
+
+          <Layout class="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout"/>
+          <SubComponents>
+            <Container class="javax.swing.JScrollPane" name="jScrollPane2">
+              <AuxValues>
+                <AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
+              </AuxValues>
+              <Constraints>
+                <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
+                  <GridBagConstraints gridX="0" gridY="1" gridWidth="1" gridHeight="1" fill="1" ipadX="0" ipadY="0" insetsTop="6" insetsLeft="0" insetsBottom="0" insetsRight="0" anchor="18" weightX="1.0" weightY="1.0"/>
+                </Constraint>
+              </Constraints>
+
+              <Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
+              <SubComponents>
+                <Component class="javax.swing.JEditorPane" name="left">
+                </Component>
+              </SubComponents>
+            </Container>
+            <Component class="javax.swing.JComboBox" name="leftFileList">
+              <Properties>
+                <Property name="model" type="javax.swing.ComboBoxModel" editor="org.netbeans.modules.form.editors2.ComboBoxModelEditor">
+                  <StringArray count="0"/>
+                </Property>
+              </Properties>
+              <Constraints>
+                <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
+                  <GridBagConstraints gridX="0" gridY="0" gridWidth="1" gridHeight="1" fill="2" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="0" insetsBottom="0" insetsRight="0" anchor="18" weightX="1.0" weightY="0.0"/>
+                </Constraint>
+              </Constraints>
+            </Component>
+          </SubComponents>
+        </Container>
+      </SubComponents>
+    </Container>
+    <Container class="javax.swing.JPanel" name="jPanel1">
+
+      <Layout class="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout"/>
+      <SubComponents>
+        <Component class="javax.swing.JLabel" name="progressLabel">
+          <Properties>
+            <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
+              <ResourceString bundle="org/netbeans/modules/jackpot30/impl/duplicates/Bundle.properties" key="DuplicatesListPanel.progressLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
+            </Property>
+          </Properties>
+          <Constraints>
+            <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
+              <GridBagConstraints gridX="0" gridY="0" gridWidth="1" gridHeight="1" fill="2" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="0" insetsBottom="0" insetsRight="0" anchor="18" weightX="1.0" weightY="0.0"/>
+            </Constraint>
+          </Constraints>
+        </Component>
+        <Component class="javax.swing.JLabel" name="findMore">
+          <Properties>
+            <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
+              <ResourceString bundle="org/netbeans/modules/jackpot30/impl/duplicates/Bundle.properties" key="DuplicatesListPanel.findMore.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
+            </Property>
+            <Property name="cursor" type="java.awt.Cursor" editor="org.netbeans.modules.form.editors2.CursorEditor">
+              <Color id="Hand Cursor"/>
+            </Property>
+          </Properties>
+          <Events>
+            <EventHandler event="mouseClicked" listener="java.awt.event.MouseListener" parameters="java.awt.event.MouseEvent" handler="findMoreMouseClicked"/>
+          </Events>
+          <Constraints>
+            <Constraint layoutClass="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout" value="org.netbeans.modules.form.compat2.layouts.DesignGridBagLayout$GridBagConstraintsDescription">
+              <GridBagConstraints gridX="1" gridY="0" gridWidth="1" gridHeight="1" fill="0" ipadX="0" ipadY="0" insetsTop="0" insetsLeft="6" insetsBottom="0" insetsRight="0" anchor="18" weightX="0.0" weightY="0.0"/>
+            </Constraint>
+          </Constraints>
+        </Component>
+      </SubComponents>
+    </Container>
+  </SubComponents>
+</Form>

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.java
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.java b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.java
new file mode 100644
index 0000000..5415b00
--- /dev/null
+++ b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/DuplicatesListPanel.java
@@ -0,0 +1,464 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License.  When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2009 Sun Microsystems, Inc.
+ */
+package org.netbeans.modules.jackpot30.impl.duplicates;
+
+import java.awt.Color;
+import java.awt.Component;
+import java.awt.Rectangle;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import javax.swing.DefaultComboBoxModel;
+import javax.swing.DefaultListCellRenderer;
+import javax.swing.DefaultListModel;
+import javax.swing.JEditorPane;
+import javax.swing.JList;
+import javax.swing.JSplitPane;
+import javax.swing.SwingUtilities;
+import javax.swing.event.ListSelectionEvent;
+import javax.swing.event.ListSelectionListener;
+import javax.swing.text.AttributeSet;
+import javax.swing.text.BadLocationException;
+import javax.swing.text.StyleConstants;
+import org.netbeans.api.editor.mimelookup.MimePath;
+import org.netbeans.api.editor.settings.AttributesUtilities;
+import org.netbeans.modules.jackpot30.impl.duplicates.ComputeDuplicates.DuplicateDescription;
+import org.netbeans.modules.jackpot30.impl.duplicates.ComputeDuplicates.Span;
+import org.netbeans.spi.editor.highlighting.HighlightsLayer;
+import org.netbeans.spi.editor.highlighting.HighlightsLayerFactory;
+import org.netbeans.spi.editor.highlighting.ZOrder;
+import org.netbeans.spi.editor.highlighting.support.OffsetsBag;
+import org.netbeans.spi.editor.mimelookup.MimeDataProvider;
+import org.openide.filesystems.FileObject;
+import org.openide.filesystems.FileUtil;
+import org.openide.util.Exceptions;
+import org.openide.util.Lookup;
+import org.openide.util.RequestProcessor;
+import org.openide.util.RequestProcessor.Task;
+import org.openide.util.lookup.Lookups;
+import org.openide.util.lookup.ServiceProvider;
+
+/**
+ *
+ * @author lahvac
+ */
+public class DuplicatesListPanel extends javax.swing.JPanel {
+    private final Collection<String> sourceRoots;
+    private final Iterator<? extends DuplicateDescription> dupes;
+
+    private int targetCount;
+
+    public DuplicatesListPanel(Collection<String> sourceRoots, final Iterator<? extends DuplicateDescription> dupes) {
+        this.sourceRoots = sourceRoots;
+        this.dupes = dupes;
+        
+        initComponents();
+
+        left.setContentType("text/x-java");
+        left.putClientProperty(DuplicatesListPanel.class, new OffsetsBag(left.getDocument()));
+        
+        right.setContentType("text/x-java");
+        right.putClientProperty(DuplicatesListPanel.class, new OffsetsBag(right.getDocument()));
+
+        duplicatesList.setModel(new DefaultListModel());
+        duplicatesList.setCellRenderer(new DuplicatesRendererImpl());
+        duplicatesList.getSelectionModel().addListSelectionListener(new ListSelectionListener() {
+            public void valueChanged(ListSelectionEvent arg0) {
+                DuplicateDescription dd = (DuplicateDescription) duplicatesList.getSelectedValue();
+                DefaultComboBoxModel l = new DefaultComboBoxModel();
+                DefaultComboBoxModel r = new DefaultComboBoxModel();
+
+                for (Span s : dd.dupes) {
+                    l.addElement(s);
+                    r.addElement(s);
+                }
+
+                leftFileList.setModel(l);
+                rightFileList.setModel(r);
+
+                leftFileList.setSelectedIndex(0);
+                rightFileList.setSelectedIndex(1);
+            }
+        });
+
+        leftFileList.setRenderer(new SpanRendererImpl());
+        leftFileList.addActionListener(new ActionListener() {
+
+            public void actionPerformed(ActionEvent e) {
+                setSpan(left, (Span) leftFileList.getSelectedItem());
+            }
+        });
+        rightFileList.setRenderer(new SpanRendererImpl());
+        rightFileList.addActionListener(new ActionListener() {
+            public void actionPerformed(ActionEvent e) {
+                setSpan(right, (Span) rightFileList.getSelectedItem());
+            }
+        });
+
+        progressLabel.setText("Looking for duplicates...");
+
+        findMore();
+    }
+
+    /** This method is called from within the constructor to
+     * initialize the form.
+     * WARNING: Do NOT modify this code. The content of this method is
+     * always regenerated by the Form Editor.
+     */
+    @SuppressWarnings("unchecked")
+    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
+    private void initComponents() {
+        java.awt.GridBagConstraints gridBagConstraints;
+
+        jScrollPane1 = new javax.swing.JScrollPane();
+        duplicatesList = new javax.swing.JList();
+        mainSplit2 = new BalancedSplitPane();
+        rightPanel = new javax.swing.JPanel();
+        rightFileList = new javax.swing.JComboBox();
+        jScrollPane3 = new javax.swing.JScrollPane();
+        right = new javax.swing.JEditorPane();
+        leftPanel = new javax.swing.JPanel();
+        jScrollPane2 = new javax.swing.JScrollPane();
+        left = new javax.swing.JEditorPane();
+        leftFileList = new javax.swing.JComboBox();
+        jPanel1 = new javax.swing.JPanel();
+        progressLabel = new javax.swing.JLabel();
+        findMore = new javax.swing.JLabel();
+
+        duplicatesList.setPrototypeCellValue("9999999999999999999999999999999999999999999999999999999999999999999999");
+        duplicatesList.setVisibleRowCount(4);
+        jScrollPane1.setViewportView(duplicatesList);
+
+        mainSplit2.setDividerLocation(400);
+
+        rightPanel.setLayout(new java.awt.GridBagLayout());
+
+        gridBagConstraints = new java.awt.GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 0;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;
+        gridBagConstraints.ipadx = 324;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        rightPanel.add(rightFileList, gridBagConstraints);
+
+        jScrollPane3.setViewportView(right);
+
+        gridBagConstraints = new java.awt.GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 1;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        gridBagConstraints.weighty = 1.0;
+        gridBagConstraints.insets = new java.awt.Insets(6, 0, 0, 0);
+        rightPanel.add(jScrollPane3, gridBagConstraints);
+
+        mainSplit2.setRightComponent(rightPanel);
+
+        leftPanel.setLayout(new java.awt.GridBagLayout());
+
+        jScrollPane2.setViewportView(left);
+
+        gridBagConstraints = new java.awt.GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 1;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        gridBagConstraints.weighty = 1.0;
+        gridBagConstraints.insets = new java.awt.Insets(6, 0, 0, 0);
+        leftPanel.add(jScrollPane2, gridBagConstraints);
+
+        gridBagConstraints = new java.awt.GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 0;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        leftPanel.add(leftFileList, gridBagConstraints);
+
+        mainSplit2.setLeftComponent(leftPanel);
+
+        jPanel1.setLayout(new java.awt.GridBagLayout());
+
+        progressLabel.setText(org.openide.util.NbBundle.getMessage(DuplicatesListPanel.class, "DuplicatesListPanel.progressLabel.text")); // NOI18N
+        gridBagConstraints = new java.awt.GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 0;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        jPanel1.add(progressLabel, gridBagConstraints);
+
+        findMore.setText(org.openide.util.NbBundle.getMessage(DuplicatesListPanel.class, "DuplicatesListPanel.findMore.text")); // NOI18N
+        findMore.setCursor(new java.awt.Cursor(java.awt.Cursor.HAND_CURSOR));
+        findMore.addMouseListener(new java.awt.event.MouseAdapter() {
+            public void mouseClicked(java.awt.event.MouseEvent evt) {
+                findMoreMouseClicked(evt);
+            }
+        });
+        gridBagConstraints = new java.awt.GridBagConstraints();
+        gridBagConstraints.gridx = 1;
+        gridBagConstraints.gridy = 0;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.insets = new java.awt.Insets(0, 6, 0, 0);
+        jPanel1.add(findMore, gridBagConstraints);
+
+        javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
+        this.setLayout(layout);
+        layout.setHorizontalGroup(
+            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
+                .addContainerGap()
+                .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
+                    .addComponent(mainSplit2, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, 906, Short.MAX_VALUE)
+                    .addComponent(jScrollPane1, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, 906, Short.MAX_VALUE)
+                    .addComponent(jPanel1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
+                .addContainerGap())
+        );
+        layout.setVerticalGroup(
+            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
+            .addGroup(layout.createSequentialGroup()
+                .addContainerGap()
+                .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, 67, javax.swing.GroupLayout.PREFERRED_SIZE)
+                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
+                .addComponent(mainSplit2, javax.swing.GroupLayout.DEFAULT_SIZE, 467, Short.MAX_VALUE)
+                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
+                .addComponent(jPanel1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
+                .addContainerGap())
+        );
+    }// </editor-fold>//GEN-END:initComponents
+
+    private void findMoreMouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_findMoreMouseClicked
+        findMore();
+    }//GEN-LAST:event_findMoreMouseClicked
+
+    private void findMore() {
+        targetCount = duplicatesList.getModel().getSize() + 100;
+        findMore.setVisible(false);
+        WORKER.schedule(0);
+    }
+
+    private static String computeCommonPrefix(String origCommonPrefix, FileObject file) {
+        String name = FileUtil.getFileDisplayName(file);
+
+        if (origCommonPrefix == null) return name;
+
+        int len = Math.min(origCommonPrefix.length(), name.length());
+
+        for (int cntr = 0; cntr < len; cntr++) {
+            if (origCommonPrefix.charAt(cntr) != name.charAt(cntr)) {
+                return origCommonPrefix.substring(0, cntr);
+            }
+        }
+
+        return origCommonPrefix;
+    }
+    
+    private static void setSpan(JEditorPane pane, Span s) {
+        try {
+            pane.setText(s.file.asText());
+
+            Rectangle top = pane.modelToView(0);
+            Rectangle start = pane.modelToView(s.startOff);
+            Rectangle end = pane.modelToView(s.endOff);
+
+            if (top != null && start != null && end != null) {
+                Rectangle toScroll = start.union(end);
+
+                pane.scrollRectToVisible(top);
+                pane.scrollRectToVisible(toScroll);
+            }
+
+            OffsetsBag bag = (OffsetsBag) pane.getClientProperty(DuplicatesListPanel.class);
+
+            bag.clear();
+            bag.addHighlight(s.startOff, s.endOff, HIGHLIGHT);
+        } catch (IOException ex) {
+            Exceptions.printStackTrace(ex);
+        } catch (BadLocationException ex) {
+            Exceptions.printStackTrace(ex);
+        }
+    }
+
+    private static final AttributeSet HIGHLIGHT = AttributesUtilities.createImmutable(StyleConstants.Background, new Color(0xDF, 0xDF, 0xDF, 0xff));
+
+    private final class DuplicatesRendererImpl extends DefaultListCellRenderer {
+        @Override
+        public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+            if (!(value instanceof DuplicateDescription)) return super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus);
+            DuplicateDescription dd = (DuplicateDescription) value;
+            Set<FileObject> files = new LinkedHashSet<FileObject>();
+            String commonPrefix = null;
+
+            for (Span s : dd.dupes) {
+                commonPrefix = computeCommonPrefix(commonPrefix, s.file);
+                files.add(s.file);
+            }
+
+            StringBuilder cap = new StringBuilder();
+
+            OUTER: for (FileObject file : files) {
+                String name = FileUtil.getFileDisplayName(file);
+
+                if (cap.length() > 0) {
+                    cap.append("    ");
+                }
+                
+                for (String sr : sourceRoots) {
+                    if (name.startsWith(sr)) {
+                        cap.append(name.substring(Math.max(0, sr.lastIndexOf('/') + 1)));
+                        continue OUTER;
+                    }
+                }
+            }
+
+            return super.getListCellRendererComponent(list, cap.toString(), index, isSelected, cellHasFocus);
+        }
+    }
+
+    private final class SpanRendererImpl extends DefaultListCellRenderer {
+        @Override
+        public Component getListCellRendererComponent(JList list, Object value, int index, boolean isSelected, boolean cellHasFocus) {
+            if (!(value instanceof Span)) {
+                return super.getListCellRendererComponent(list, value, index, isSelected, cellHasFocus);
+            }
+            Span span = (Span) value;
+
+            return super.getListCellRendererComponent(list, FileUtil.getFileDisplayName(span.file), index, isSelected, cellHasFocus);
+        }
+    }
+
+    public static final class HighlightLayerFactoryImpl implements HighlightsLayerFactory {
+        public HighlightsLayer[] createLayers(Context cntxt) {
+            OffsetsBag bag = (OffsetsBag) cntxt.getComponent().getClientProperty(DuplicatesListPanel.class);
+
+            if (bag != null) {
+                return new HighlightsLayer[] {
+                    HighlightsLayer.create(DuplicatesListPanel.class.getName(), ZOrder.CARET_RACK, true, bag)
+                };
+            }
+
+            return new HighlightsLayer[0];
+        }
+    }
+
+    @ServiceProvider(service=MimeDataProvider.class)
+    public static final class MDPI implements MimeDataProvider {
+
+        private static final Lookup L = Lookups.singleton(new HighlightLayerFactoryImpl());
+
+        public Lookup getLookup(MimePath mp) {
+            if (mp.getPath().startsWith("text/x-java")) {
+                return L;
+            }
+
+            return null;
+        }
+        
+    }
+
+    private static final class BalancedSplitPane extends JSplitPane {
+
+        @Override
+        @SuppressWarnings("deprecation")
+        public void reshape(int x, int y, int w, int h) {
+            super.reshape(x, y, w, h);
+            SwingUtilities.invokeLater(new Runnable() {
+                public void run() {
+                    setDividerLocation(0.5);
+                }
+            });
+        }
+
+    }
+
+    // Variables declaration - do not modify//GEN-BEGIN:variables
+    private javax.swing.JList duplicatesList;
+    private javax.swing.JLabel findMore;
+    private javax.swing.JPanel jPanel1;
+    private javax.swing.JScrollPane jScrollPane1;
+    private javax.swing.JScrollPane jScrollPane2;
+    private javax.swing.JScrollPane jScrollPane3;
+    private javax.swing.JEditorPane left;
+    private javax.swing.JComboBox leftFileList;
+    private javax.swing.JPanel leftPanel;
+    private javax.swing.JSplitPane mainSplit2;
+    private javax.swing.JLabel progressLabel;
+    private javax.swing.JEditorPane right;
+    private javax.swing.JComboBox rightFileList;
+    private javax.swing.JPanel rightPanel;
+    // End of variables declaration//GEN-END:variables
+
+    private static final RequestProcessor DEFAULT_WORKER = new RequestProcessor(DuplicatesListPanel.class.getName(), 1, false, false);
+    private final Task WORKER = DEFAULT_WORKER.create(new Runnable() {
+        public void run() {
+            if (dupes.hasNext()) {
+                final DuplicateDescription dd = dupes.next();
+
+                SwingUtilities.invokeLater(new Runnable() {
+
+                    public void run() {
+                        ((DefaultListModel)duplicatesList.getModel()).addElement(dd);
+
+                        int size = duplicatesList.getModel().getSize();
+
+                        if (size == 1) {
+                            duplicatesList.setSelectedIndex(0);
+                        }
+                        
+                        if (size >= targetCount) {
+                            findMore.setVisible(true);
+                            progressLabel.setText("Found " + size + " duplicated snippets.");
+                        } else {
+                            progressLabel.setText("Found " + size + " duplicated snippets and searching...");
+                            WORKER.schedule(0);
+                        }
+                    }
+                });
+            }
+        }
+    });
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/GlobalFindDuplicates.java
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/GlobalFindDuplicates.java b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/GlobalFindDuplicates.java
new file mode 100644
index 0000000..5a35788
--- /dev/null
+++ b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/GlobalFindDuplicates.java
@@ -0,0 +1,179 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2009-2010 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License.  When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2009-2010 Sun Microsystems, Inc.
+ */
+package org.netbeans.modules.jackpot30.impl.duplicates;
+
+import java.awt.Dialog;
+import java.awt.GridBagConstraints;
+import java.awt.GridBagLayout;
+import java.awt.Insets;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.SwingUtilities;
+import org.netbeans.api.java.classpath.ClassPath;
+import org.netbeans.api.java.classpath.GlobalPathRegistry;
+import org.netbeans.api.progress.ProgressHandle;
+import org.netbeans.api.progress.ProgressHandleFactory;
+import org.netbeans.modules.jackpot30.impl.duplicates.ComputeDuplicates.DuplicateDescription;
+import org.openide.DialogDescriptor;
+import org.openide.DialogDisplayer;
+import org.openide.NotifyDescriptor;
+import org.openide.filesystems.FileObject;
+import org.openide.filesystems.FileUtil;
+import org.openide.util.Exceptions;
+import org.openide.util.HelpCtx;
+import org.openide.util.RequestProcessor;
+
+public final class GlobalFindDuplicates implements ActionListener {
+
+    public void actionPerformed(ActionEvent e) {
+        final Iterator<? extends DuplicateDescription>[] dupes = new Iterator[1];
+        final ProgressHandle handle = ProgressHandleFactory.createHandle("Compute Duplicates");
+        JPanel panel = createPanel(handle);
+        final AtomicBoolean cancel = new AtomicBoolean();
+        DialogDescriptor w = new DialogDescriptor(panel, "Computing Duplicates", true, new Object[] {DialogDescriptor.CANCEL_OPTION}, DialogDescriptor.CANCEL_OPTION, DialogDescriptor.DEFAULT_ALIGN, HelpCtx.DEFAULT_HELP, new ActionListener() {
+            public void actionPerformed(ActionEvent e) {
+                cancel.set(true);
+            }
+        });
+
+        w.setClosingOptions(null);
+
+        final Dialog d = DialogDisplayer.getDefault().createDialog(w);
+        final AtomicBoolean done = new AtomicBoolean();
+        final Collection<String> sourceRoots = new LinkedList<String>();
+
+        WORKER.post(new Runnable() {
+            public void run() {
+                try {
+                    for (ClassPath cp : GlobalPathRegistry.getDefault().getPaths(ClassPath.SOURCE)) {
+                        for (ClassPath.Entry e : cp.entries()) {
+                            FileObject root = e.getRoot();
+
+                            if (root == null) continue;
+
+                            sourceRoots.add(FileUtil.getFileDisplayName(root));
+                        }
+                    }
+
+                    dupes[0] = new ComputeDuplicates().computeDuplicatesForAllOpenedProjects(handle, cancel);
+                    done.set(true);
+                } catch (IOException ex) {
+                    Exceptions.printStackTrace(ex);
+                } finally {
+                    handle.finish();
+
+                    SwingUtilities.invokeLater(new Runnable() {
+                        public void run() {
+                            d.setVisible(false);
+                        }
+                    });
+                }
+            }
+        });
+
+        handle.start();
+        handle.progress(" ");
+
+        d.setVisible(true);
+
+        if (!done.get()) {
+            cancel.set(true);
+            return;
+        }
+        
+        if (cancel.get()) return;
+
+        NotifyDescriptor nd = new NotifyDescriptor.Message(new DuplicatesListPanel(sourceRoots, dupes[0]));
+
+        DialogDisplayer.getDefault().notifyLater(nd);
+    }
+
+    private JPanel createPanel(ProgressHandle handle) {
+        JPanel panel = new JPanel(new GridBagLayout());
+        GridBagConstraints gridBagConstraints;
+
+        gridBagConstraints = new GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 0;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        gridBagConstraints.insets = new Insets(6, 6, 0, 6);
+        panel.add(new JLabel("Computing Duplicates - Please Wait"), gridBagConstraints);
+
+        gridBagConstraints = new GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 1;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        gridBagConstraints.insets = new Insets(6, 6, 0, 6);
+        panel.add(ProgressHandleFactory.createProgressComponent(handle), gridBagConstraints);
+
+        gridBagConstraints = new GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 2;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.HORIZONTAL;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        gridBagConstraints.insets = new Insets(6, 6, 6, 6);
+        panel.add(ProgressHandleFactory.createDetailLabelComponent(handle), gridBagConstraints);
+
+        gridBagConstraints = new GridBagConstraints();
+        gridBagConstraints.gridx = 0;
+        gridBagConstraints.gridy = 3;
+        gridBagConstraints.fill = java.awt.GridBagConstraints.BOTH;
+        gridBagConstraints.anchor = java.awt.GridBagConstraints.NORTHWEST;
+        gridBagConstraints.weightx = 1.0;
+        gridBagConstraints.weighty = 1.0;
+        panel.add(new JPanel(), gridBagConstraints);
+
+        return panel;
+    }
+
+    private static final RequestProcessor WORKER = new RequestProcessor(GlobalFindDuplicates.class.getName(), 1);
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-netbeans-jackpot30/blob/9ed0a377/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/hints/FindDuplicates.java
----------------------------------------------------------------------
diff --git a/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/hints/FindDuplicates.java b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/hints/FindDuplicates.java
new file mode 100644
index 0000000..90ad6be
--- /dev/null
+++ b/duplicates/ide/impl/src/org/netbeans/modules/jackpot30/impl/duplicates/hints/FindDuplicates.java
@@ -0,0 +1,133 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License.  When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ *
+ * Contributor(s):
+ *
+ * Portions Copyrighted 2009 Sun Microsystems, Inc.
+ */
+
+package org.netbeans.modules.jackpot30.impl.duplicates.hints;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.netbeans.api.java.source.CancellableTask;
+import org.netbeans.api.java.source.CompilationInfo;
+import org.netbeans.api.java.source.JavaSource.Phase;
+import org.netbeans.api.java.source.JavaSource.Priority;
+import org.netbeans.api.java.source.JavaSourceTaskFactory;
+import org.netbeans.api.java.source.support.EditorAwareJavaSourceTaskFactory;
+import org.netbeans.modules.jackpot30.impl.duplicates.ComputeDuplicates;
+import org.netbeans.modules.jackpot30.impl.duplicates.ComputeDuplicates.DuplicateDescription;
+import org.netbeans.modules.jackpot30.impl.duplicates.indexing.RemoteDuplicatesIndex;
+import org.netbeans.spi.editor.hints.ErrorDescription;
+import org.netbeans.spi.editor.hints.ErrorDescriptionFactory;
+import org.netbeans.spi.editor.hints.HintsController;
+import org.netbeans.spi.editor.hints.Severity;
+import org.openide.filesystems.FileObject;
+import org.openide.util.NbCollections;
+import org.openide.util.lookup.ServiceProvider;
+
+/**
+ *
+ * @author lahvac
+ */
+public class FindDuplicates implements CancellableTask<CompilationInfo> {
+
+    private final AtomicBoolean cancel = new AtomicBoolean();
+    
+    public void run(CompilationInfo info) throws Exception {
+        cancel.set(false);
+
+        long start = System.currentTimeMillis();
+        try {
+            Collection<? extends ErrorDescription> eds = computeErrorDescription(info);
+
+            if (cancel.get()) return;
+
+            if (eds == null) {
+                eds = Collections.emptyList();
+            }
+
+            HintsController.setErrors(info.getFileObject(), FindDuplicates.class.getName(), eds);
+        } finally {
+            long end = System.currentTimeMillis();
+
+            Logger.getLogger("TIMER").log(Level.FINE, "Duplicates in editor", new Object[] {info.getFileObject(), end - start});
+        }
+    }
+
+    private Collection<? extends ErrorDescription> computeErrorDescription(CompilationInfo info) throws Exception {
+        List<ErrorDescription> result = new LinkedList<ErrorDescription>();
+
+        Map<String, long[]> encoded = ComputeDuplicates.encodeGeneralized(info);
+        Iterator<? extends DuplicateDescription> duplicates = RemoteDuplicatesIndex.findDuplicates(encoded, info.getFileObject(), cancel).iterator();
+
+        for (DuplicateDescription dd : NbCollections.iterable(duplicates)) {
+            long[] spans = encoded.get(dd.hash);
+
+            for (int c = 0; c < spans.length; c += 2) {
+                if (cancel.get()) return null;
+                result.add(ErrorDescriptionFactory.createErrorDescription(Severity.WARNING, "Duplicate of code from " + dd.dupes.get(0).file, info.getFileObject(), (int) spans[c], (int) spans[c + 1]));
+            }
+        }
+
+        return result;
+    }
+    
+    public void cancel() {
+        cancel.set(true);
+    }
+
+    @ServiceProvider(service=JavaSourceTaskFactory.class)
+    public static final class FactoryImpl extends EditorAwareJavaSourceTaskFactory {
+
+        public FactoryImpl() {
+            super(Phase.RESOLVED, Priority.LOW);
+        }
+
+        @Override
+        protected CancellableTask<CompilationInfo> createTask(FileObject file) {
+            return new FindDuplicates();
+        }
+        
+    }
+
+}