You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2017/03/31 12:35:36 UTC
[1/2] maven-indexer git commit: MINDEXER-100: forward port 5.x
changes to master
Repository: maven-indexer
Updated Branches:
refs/heads/master a7aeb9f2f -> 6cb2fcbe4
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
new file mode 100644
index 0000000..c5b210a
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
@@ -0,0 +1,170 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Properties;
+import java.util.TimeZone;
+import java.util.regex.Pattern;
+
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+import org.apache.maven.index.reader.ResourceHandler.Resource;
+import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
+
+/**
+ * Reusable code snippets and constants.
+ *
+ * @since 5.1.2
+ */
+public final class Utils
+{
+ private Utils() {
+ // nothing
+ }
+
+ public static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index";
+
+ public static final DateFormat INDEX_DATE_FORMAT;
+
+ static {
+ INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z");
+ INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT"));
+ }
+
+ public static final String FIELD_SEPARATOR = "|";
+
+ public static final String NOT_AVAILABLE = "NA";
+
+ public static final String UINFO = "u";
+
+ public static final String INFO = "i";
+
+ public static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR));
+
+ /**
+ * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream.
+ */
+ public static Properties loadProperties(final InputStream inputStream) throws IOException {
+ try {
+ final Properties properties = new Properties();
+ properties.load(inputStream);
+ return properties;
+ }
+ finally {
+ inputStream.close();
+ }
+ }
+
+ /**
+ * Creates and loads {@link Properties} from provided {@link Resource} if exists, and closes the resource. If not
+ * exists, returns {@code null}.
+ */
+ public static Properties loadProperties(final Resource resource) throws IOException {
+ final InputStream inputStream = resource.read();
+ if (inputStream == null) {
+ return null;
+ }
+ return loadProperties(resource.read());
+ }
+
+ /**
+ * Saves {@link Properties} to provided {@link OutputStream} and closes the stream.
+ */
+ public static void storeProperties(final OutputStream outputStream, final Properties properties) throws IOException {
+ try {
+ properties.store(outputStream, "Maven Indexer Writer");
+ }
+ finally {
+ outputStream.close();
+ }
+ }
+
+
+ /**
+ * Saves {@link Properties} to provided {@link WritableResource} and closes the resource.
+ */
+ public static void storeProperties(final WritableResource writableResource, final Properties properties) throws IOException {
+ try {
+ storeProperties(writableResource.write(), properties);
+ }
+ finally {
+ writableResource.close();
+ }
+ }
+
+ /**
+ * Creates a record of type {@link Type#DESCRIPTOR}.
+ */
+ public static Record descriptor(final String repoId) {
+ HashMap<EntryKey, Object> entries = new HashMap<EntryKey, Object>();
+ entries.put(Record.REPOSITORY_ID, repoId);
+ return new Record(Type.DESCRIPTOR, entries);
+ }
+
+ /**
+ * Creates a record of type {@link Type#ALL_GROUPS}.
+ */
+ public static Record allGroups(final Collection<String> allGroups) {
+ HashMap<EntryKey, Object> entries = new HashMap<EntryKey, Object>();
+ entries.put(Record.ALL_GROUPS, allGroups.toArray(new String[allGroups.size()]));
+ return new Record(Type.ALL_GROUPS, entries);
+ }
+
+ /**
+ * Creates a record of type {@link Type#ROOT_GROUPS}.
+ */
+ public static Record rootGroups(final Collection<String> rootGroups) {
+ HashMap<EntryKey, Object> entries = new HashMap<EntryKey, Object>();
+ entries.put(Record.ROOT_GROUPS, rootGroups.toArray(new String[rootGroups.size()]));
+ return new Record(Type.ROOT_GROUPS, entries);
+ }
+
+ /**
+ * Helper to translate the "NA" (not available) input into {@code null} value.
+ */
+ public static String renvl(final String v) {
+ return NOT_AVAILABLE.equals(v) ? null : v;
+ }
+
+ /**
+ * Helper to translate {@code null} into "NA" (not available) value.
+ */
+ public static String nvl(final String v) {
+ return v == null ? NOT_AVAILABLE : v;
+ }
+
+ /**
+ * Returns the "root group" of given groupId.
+ */
+ public static String rootGroup(final String groupId) {
+ int n = groupId.indexOf('.');
+ if (n > -1) {
+ return groupId.substring(0, n);
+ }
+ return groupId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
new file mode 100644
index 0000000..e3f8ab9
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
@@ -0,0 +1,58 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index
+ * updates are wanted, to store the index state locally, and be able to calculate incremental diffs on next {@link
+ * IndexReader} invocation. Is used by single thread only.
+ *
+ * @see ResourceHandler
+ * @since 5.1.2
+ */
+public interface WritableResourceHandler
+ extends ResourceHandler
+{
+ interface WritableResource
+ extends Resource, Closeable
+ {
+ /**
+ * Returns the {@link OutputStream} stream of the resource, if exists, it will replace the existing content, or if
+ * not exists, the resource will be created. The stream should be closed by caller, otherwise resource leaks might
+ * be introduced. How and when content is written is left to implementation, but it is guaranteed that this method
+ * is called only once, and will be followed by {@link #close()} on the resource itself. Implementation does not
+ * have to be "read consistent", in a way to worry what subsequent {@link #read()} method will return, as mixed
+ * calls will not happen on same instance of resource.
+ */
+ OutputStream write() throws IOException;
+ }
+
+ /**
+ * Returns the {@link WritableResource} with {@code name}. Returned locator should be handled as
+ * resource, is {@link Closeable}.
+ *
+ * @param name Resource name, guaranteed to be non-{@code null} and is FS and URL safe string.
+ */
+ WritableResource locate(String name) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo b/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo
new file mode 100644
index 0000000..9ad81f6
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/packageinfo
@@ -0,0 +1 @@
+version 1.0.0
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
new file mode 100644
index 0000000..68e01f6
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
@@ -0,0 +1,129 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
+
+/**
+ * A trivial caching {@link ResourceHandler} that caches forever during single session (existence of the instance).
+ */
+public class CachingResourceHandler
+ implements ResourceHandler
+{
+ private static final Resource NOT_EXISTING_RESOURCE = new Resource()
+ {
+ public InputStream read() throws IOException {
+ return null;
+ }
+
+ public void close() throws IOException {
+ // nop
+ }
+ };
+
+ private final WritableResourceHandler local;
+
+ private final ResourceHandler remote;
+
+ private final Set<String> notFoundResources;
+
+ public CachingResourceHandler(final WritableResourceHandler local, final ResourceHandler remote) {
+ if (local == null || remote == null) {
+ throw new NullPointerException("null resource handler");
+ }
+ this.local = local;
+ this.remote = remote;
+ this.notFoundResources = new HashSet<String>();
+ }
+
+ public Resource locate(final String name) throws IOException {
+ if (notFoundResources.contains(name)) {
+ return NOT_EXISTING_RESOURCE;
+ }
+ else {
+ return new CachingResource(name);
+ }
+ }
+
+ private class CachingResource
+ implements Resource
+ {
+ private final String name;
+
+ private CachingResource(final String name) {
+ this.name = name;
+ }
+
+ public InputStream read() throws IOException {
+ InputStream inputStream = local.locate(name).read();
+ if (inputStream != null) {
+ return inputStream;
+ }
+ if (cacheLocally(name)) {
+ return local.locate(name).read();
+ }
+ notFoundResources.add(name);
+ return null;
+ }
+
+ private boolean cacheLocally(final String name) throws IOException {
+ final Resource remoteResource = remote.locate(name);
+ final WritableResource localResource = local.locate(name);
+ try {
+ final InputStream inputStream = remoteResource.read();
+ if (inputStream != null) {
+ final OutputStream outputStream = localResource.write();
+ try {
+ int read;
+ byte[] bytes = new byte[8192];
+ while ((read = inputStream.read(bytes)) != -1) {
+ outputStream.write(bytes, 0, read);
+ }
+ outputStream.flush();
+ return true;
+ }
+ finally {
+ outputStream.close();
+ inputStream.close();
+ }
+ }
+ return false;
+ }
+ finally {
+ localResource.close();
+ }
+ }
+
+ public void close() throws IOException {
+ // nop
+ }
+ }
+
+ public void close() throws IOException {
+ remote.close();
+ local.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
new file mode 100644
index 0000000..7d5cc33
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
@@ -0,0 +1,95 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.Type;
+import org.apache.maven.index.reader.ResourceHandler.Resource;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * UT for {@link ChunkReader}
+ */
+public class ChunkReaderTest
+ extends TestSupport
+{
+ @Test
+ public void simple() throws IOException {
+ final ChunkReader chunkReader = new ChunkReader(
+ "full",
+ testResourceHandler("simple").locate("nexus-maven-repository-index.gz").read()
+ );
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType(chunkReader);
+ assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(2));
+ assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
+ }
+
+ @Test
+ public void roundtrip() throws IOException {
+ final Date published;
+ File tempChunkFile = createTempFile("nexus-maven-repository-index.gz");
+ {
+ final Resource resource = testResourceHandler("simple").locate("nexus-maven-repository-index.gz");
+ final ChunkReader chunkReader = new ChunkReader(
+ "full",
+ resource.read()
+ );
+ final ChunkWriter chunkWriter = new ChunkWriter(
+ chunkReader.getName(),
+ new FileOutputStream(tempChunkFile), 1, new Date()
+ );
+ try {
+ chunkWriter.writeChunk(chunkReader.iterator());
+ }
+ finally {
+ chunkWriter.close();
+ chunkReader.close();
+ }
+ published = chunkWriter.getTimestamp();
+ }
+
+ final ChunkReader chunkReader = new ChunkReader(
+ "full",
+ new FileInputStream(tempChunkFile)
+ );
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ assertThat(chunkReader.getTimestamp().getTime(), equalTo(published.getTime()));
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType(chunkReader);
+ assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(2));
+ assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
new file mode 100644
index 0000000..aea0ee1
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
@@ -0,0 +1,87 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * A trivial {@link File} directory handler that does not perform any locking or extra bits, and just serves up files
+ * by name from specified existing directory.
+ */
+public class DirectoryResourceHandler
+ implements WritableResourceHandler
+{
+ private final File rootDirectory;
+
+ public DirectoryResourceHandler(final File rootDirectory) {
+ if (rootDirectory == null) {
+ throw new NullPointerException("null rootDirectory");
+ }
+ if (!rootDirectory.isDirectory()) {
+ throw new IllegalArgumentException("rootDirectory exists and is not a directory");
+ }
+ this.rootDirectory = rootDirectory;
+ }
+
+ public File getRootDirectory() {
+ return rootDirectory;
+ }
+
+ public WritableResource locate(final String name) throws IOException {
+ return new FileResource(new File(rootDirectory, name));
+ }
+
+ public void close() throws IOException {
+ // nop
+ }
+
+ private class FileResource
+ implements WritableResource
+ {
+ private final File file;
+
+ private FileResource(final File file) {
+ this.file = file;
+ }
+
+ public InputStream read() throws IOException {
+ if (file.isFile()) {
+ return new BufferedInputStream(new FileInputStream(file));
+ }
+ return null;
+ }
+
+ public OutputStream write() throws IOException {
+ return new BufferedOutputStream(new FileOutputStream(file));
+ }
+
+ public void close() throws IOException {
+ // nop
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
new file mode 100644
index 0000000..e9ea16d
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java
@@ -0,0 +1,75 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+/**
+ * A trivial HTTP {@link ResourceHandler} that uses {@link URL} to fetch remote content. This implementation does not
+ * handle any advanced cases, like redirects, authentication, etc.
+ */
+public class HttpResourceHandler
+ implements ResourceHandler
+{
+ private final URI root;
+
+ public HttpResourceHandler(final URL root) throws URISyntaxException {
+ if (root == null) {
+ throw new NullPointerException("root URL null");
+ }
+ this.root = root.toURI();
+ }
+
+ public Resource locate(final String name) throws IOException {
+ return new HttpResource(name);
+ }
+
+ private class HttpResource
+ implements Resource
+ {
+ private final String name;
+
+ private HttpResource(final String name) {
+ this.name = name;
+ }
+
+ public InputStream read() throws IOException {
+ URL target = root.resolve(name).toURL();
+ HttpURLConnection conn = (HttpURLConnection) target.openConnection();
+ conn.setRequestMethod("GET");
+ conn.setRequestProperty("User-Agent", "ASF Maven-Indexer-Reader/1.0");
+ return new BufferedInputStream(conn.getInputStream());
+ }
+
+ public void close() throws IOException {
+ // nop
+ }
+ }
+
+ public void close() throws IOException {
+ // nop
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
new file mode 100644
index 0000000..8fc4c34
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
@@ -0,0 +1,209 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.apache.maven.index.reader.TestUtils.expandFunction;
+import static com.google.common.collect.Iterables.transform;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.hamcrest.core.IsNot.not;
+import static org.hamcrest.core.IsNull.nullValue;
+import static org.junit.Assert.assertThat;
+
+/**
+ * UT for {@link IndexReader}
+ */
+public class IndexReaderTest
+ extends TestSupport
+{
+ @Test
+ public void simple() throws IOException {
+ final IndexReader indexReader = new IndexReader(
+ null,
+ testResourceHandler("simple")
+ );
+ try {
+ assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
+ assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(1243533418015L));
+ assertThat(indexReader.isIncremental(), equalTo(false));
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ int chunks = 0;
+ int records = 0;
+ for (ChunkReader chunkReader : indexReader) {
+ chunks++;
+ assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
+ for (Record record : transform(chunkReader, expandFunction)) {
+ records++;
+ }
+ }
+
+ assertThat(chunks, equalTo(1));
+ assertThat(records, equalTo(5));
+ }
+ finally {
+ indexReader.close();
+ }
+ }
+
+ @Test
+ public void simpleWithLocal() throws IOException {
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+ final IndexReader indexReader = new IndexReader(
+ writableResourceHandler,
+ testResourceHandler("simple")
+ );
+ try {
+ assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
+ assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(1243533418015L));
+ assertThat(indexReader.isIncremental(), equalTo(false));
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ int chunks = 0;
+ int records = 0;
+ for (ChunkReader chunkReader : indexReader) {
+ chunks++;
+ assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
+ for (Record record : transform(chunkReader, expandFunction)) {
+ records++;
+ }
+ }
+
+ assertThat(chunks, equalTo(1));
+ assertThat(records, equalTo(5));
+ }
+ finally {
+ indexReader.close();
+ }
+
+ assertThat(writableResourceHandler.locate("nexus-maven-repository-index.properties").read(), not(nullValue()));
+ }
+
+ @Test
+ public void roundtrip() throws IOException {
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+ Date published;
+ {
+ final IndexReader indexReader = new IndexReader(
+ null,
+ testResourceHandler("simple")
+ );
+ final IndexWriter indexWriter = new IndexWriter(
+ writableResourceHandler,
+ indexReader.getIndexId(),
+ false
+ );
+ try {
+ for (ChunkReader chunkReader : indexReader) {
+ indexWriter.writeChunk(chunkReader.iterator());
+ }
+ }
+ finally {
+ indexWriter.close();
+ published = indexWriter.getPublishedTimestamp();
+ indexReader.close();
+ }
+ }
+
+ final IndexReader indexReader = new IndexReader(
+ null,
+ writableResourceHandler
+ );
+ try {
+ assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
+ assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
+ assertThat(indexReader.isIncremental(), equalTo(false));
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ int chunks = 0;
+ int records = 0;
+ for (ChunkReader chunkReader : indexReader) {
+ chunks++;
+ assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
+ for (Record record : transform(chunkReader, expandFunction)) {
+ records++;
+ }
+ }
+
+ assertThat(chunks, equalTo(1));
+ assertThat(records, equalTo(5));
+ }
+ finally {
+ indexReader.close();
+ }
+ }
+
+ /**
+ * This UT is here for demonstration purposes only. Bashing Central is not something you want to do, and risk your
+ * IP address being banned. You were warned!
+ */
+ @Test
+ @Ignore("For eyes only")
+ public void central() throws Exception {
+ // local index location, against which we perform incremental updates
+ final File indexDir = createTempDirectory();
+ // cache of remote, to not rely on HTTP transport possible failures, or, to detect them early
+ final File cacheDir = createTempDirectory();
+
+ final PrintWriter writer = new PrintWriter(System.out, true);
+ final WritableResourceHandler local = new DirectoryResourceHandler(indexDir);
+ final CachingResourceHandler remote = new CachingResourceHandler(
+ new DirectoryResourceHandler(cacheDir),
+ new HttpResourceHandler(new URL("http://repo1.maven.org/maven2/.index/"))
+ );
+ final IndexReader indexReader = new IndexReader(local, remote);
+ try {
+ writer.println("indexRepoId=" + indexReader.getIndexId());
+ writer.println("indexLastPublished=" + indexReader.getPublishedTimestamp());
+ writer.println("isIncremental=" + indexReader.isIncremental());
+ writer.println("indexRequiredChunkNames=" + indexReader.getChunkNames());
+ for (ChunkReader chunkReader : indexReader) {
+ writer.println("chunkName=" + chunkReader.getName());
+ writer.println("chunkVersion=" + chunkReader.getVersion());
+ writer.println("chunkPublished=" + chunkReader.getTimestamp());
+ writer.println("Chunk stats:");
+ Map<Type, Integer> stats = countRecordsByType(chunkReader);
+ for (Map.Entry<Type, Integer> entry : stats.entrySet()) {
+ writer.println(entry.getKey() + " = " + entry.getValue());
+ }
+ writer.println("= = = = = =");
+ }
+ }
+ finally {
+ indexReader.close();
+ remote.close();
+ local.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
new file mode 100644
index 0000000..d1e9bc4
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
@@ -0,0 +1,93 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import static org.apache.maven.index.reader.TestUtils.expandFunction;
+import static com.google.common.collect.Iterables.transform;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * UT for {@link IndexWriter}
+ */
+public class IndexWriterTest
+ extends TestSupport
+{
+ @Test
+ public void roundtrip() throws IOException {
+ IndexReader indexReader;
+ IndexWriter indexWriter;
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+
+ // write it once
+ indexReader = new IndexReader(
+ null,
+ testResourceHandler("simple")
+ );
+ indexWriter = new IndexWriter(
+ writableResourceHandler,
+ indexReader.getIndexId(),
+ false
+ );
+ try {
+ for (ChunkReader chunkReader : indexReader) {
+ indexWriter.writeChunk(chunkReader.iterator());
+ }
+ }
+ finally {
+ indexWriter.close();
+ indexReader.close();
+ }
+
+ // read what we wrote out
+ indexReader = new IndexReader(
+ null,
+ writableResourceHandler
+ );
+ try {
+ assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
+ // assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
+ assertThat(indexReader.isIncremental(), equalTo(false));
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ int chunks = 0;
+ int records = 0;
+ for (ChunkReader chunkReader : indexReader) {
+ chunks++;
+ assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
+ for (Record record : transform(chunkReader, expandFunction)) {
+ records++;
+ }
+ }
+
+ assertThat(chunks, equalTo(1));
+ assertThat(records, equalTo(5));
+ }
+ finally {
+ indexReader.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
new file mode 100644
index 0000000..8794be0
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
@@ -0,0 +1,184 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Test support.
+ */
+public class TestSupport
+{
+ @Rule
+ public TestName testName = new TestName();
+
+ private File tempDir;
+
+ private List<DirectoryResourceHandler> directoryResourceHandlers;
+
+ /**
+ * Creates the temp directory and list for resource handlers.
+ */
+ @Before
+ public void setup() throws IOException {
+ this.tempDir = new File("target/tmp-" + getClass().getSimpleName());
+ this.tempDir.delete();
+ this.tempDir.mkdirs();
+ this.directoryResourceHandlers = new ArrayList<DirectoryResourceHandler>();
+ }
+
+ /**
+ * Closes all the registered resources handlers and deletes the temp directory.
+ */
+ @After
+ public void cleanup() throws IOException {
+ for (DirectoryResourceHandler directoryResourceHandler : directoryResourceHandlers) {
+ directoryResourceHandler.close();
+ }
+ // delete(tempDir);
+ }
+
+ /**
+ * Creates a temp file within {@link #tempDir}.
+ */
+ protected File createTempFile() throws IOException {
+ return File.createTempFile(testName.getMethodName() + "-file", "", tempDir);
+ }
+
+
+ /**
+ * Creates a temp file within {@link #tempDir} with given name.
+ */
+ protected File createTempFile(final String name) throws IOException {
+ return new File(tempDir, name);
+ }
+
+ /**
+ * Creates a temp directory within {@link #tempDir}.
+ */
+ protected File createTempDirectory() throws IOException {
+ File result = File.createTempFile(testName.getMethodName() + "-dir", "", tempDir);
+ result.delete();
+ result.mkdirs();
+ return result;
+ }
+
+ /**
+ * Creates an empty {@link DirectoryResourceHandler}.
+ */
+ protected WritableResourceHandler createWritableResourceHandler() throws IOException {
+ DirectoryResourceHandler result = new DirectoryResourceHandler(createTempDirectory());
+ directoryResourceHandlers.add(result);
+ return result;
+ }
+
+ /**
+ * Creates a "test" {@link ResourceHandler} that contains predefined files, is mapped to test resources under given
+ * name.
+ */
+ protected ResourceHandler testResourceHandler(final String name) throws IOException {
+ DirectoryResourceHandler result = new DirectoryResourceHandler(new File("src/test/resources/" + name));
+ directoryResourceHandlers.add(result);
+ return result;
+ }
+
+ /**
+ * Consumes {@link ChunkReader} and creates a map "by type" with records.
+ */
+ protected Map<Type, List<Record>> loadRecordsByType(final ChunkReader chunkReader) throws IOException {
+ HashMap<Type, List<Record>> stat = new HashMap<Type, List<Record>>();
+ try {
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ final RecordExpander recordExpander = new RecordExpander();
+ for (Map<String, String> rec : chunkReader) {
+ final Record record = recordExpander.apply(rec);
+ if (!stat.containsKey(record.getType())) {
+ stat.put(record.getType(), new ArrayList<Record>());
+ }
+ stat.get(record.getType()).add(record);
+ }
+ }
+ finally {
+ chunkReader.close();
+ }
+ return stat;
+ }
+
+
+ /**
+ * Consumes {@link ChunkReader} and creates a map "by type" with record type counts.
+ */
+ protected Map<Type, Integer> countRecordsByType(final ChunkReader chunkReader) throws IOException {
+ HashMap<Type, Integer> stat = new HashMap<Type, Integer>();
+ try {
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ final RecordExpander recordExpander = new RecordExpander();
+ for (Map<String, String> rec : chunkReader) {
+ final Record record = recordExpander.apply(rec);
+ if (!stat.containsKey(record.getType())) {
+ stat.put(record.getType(), 0);
+ }
+ stat.put(record.getType(), stat.get(record.getType()) + 1);
+ }
+ }
+ finally {
+ chunkReader.close();
+ }
+ return stat;
+ }
+
+ /**
+ * Delete recursively.
+ */
+ private static boolean delete(final File file) {
+ if (file == null) {
+ return false;
+ }
+ if (!file.exists()) {
+ return true;
+ }
+ if (file.isDirectory()) {
+ String[] list = file.list();
+ if (list != null) {
+ for (int i = 0; i < list.length; i++) {
+ File entry = new File(file, list[i]);
+ if (!delete(entry)) {
+ return false;
+ }
+ }
+ }
+ }
+ return file.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
new file mode 100644
index 0000000..348d199
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestUtils.java
@@ -0,0 +1,108 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Map;
+import java.util.TreeSet;
+
+import com.google.common.base.Function;
+import org.apache.maven.index.reader.Record.Type;
+
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.transform;
+import static java.util.Collections.singletonList;
+import static org.apache.maven.index.reader.Utils.allGroups;
+import static org.apache.maven.index.reader.Utils.descriptor;
+import static org.apache.maven.index.reader.Utils.rootGroup;
+import static org.apache.maven.index.reader.Utils.rootGroups;
+
+/**
+ * Helpers to transform records from one to another representation, and, some helpers for publishing using Guava.
+ */
+public final class TestUtils
+{
+ private TestUtils() {
+ // nothing
+ }
+
+ private static final RecordCompactor RECORD_COMPACTOR = new RecordCompactor();
+
+ private static final RecordExpander RECORD_EXPANDER = new RecordExpander();
+
+ public static Function<Record, Map<String, String>> compactFunction =
+ new Function<Record, Map<String, String>>()
+ {
+ public Map<String, String> apply(final Record input) {
+ return RECORD_COMPACTOR.apply(input);
+ }
+ };
+
+ public static Function<Map<String, String>, Record> expandFunction =
+ new Function<Map<String, String>, Record>()
+ {
+ public Record apply(final Map<String, String> input) {
+ return RECORD_EXPANDER.apply(input);
+ }
+ };
+
+ /**
+ * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so
+ * all the caller is needed to provide {@link Iterable} or {@link Record}s <strong>to be</strong> on the index, with
+ * record type {@link Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record
+ * stream, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special
+ * records.
+ */
+ public static Iterable<Record> decorate(final Iterable<Record> iterable,
+ final String repoId)
+ {
+ final TreeSet<String> allGroupsSet = new TreeSet<String>();
+ final TreeSet<String> rootGroupsSet = new TreeSet<String>();
+ return transform(
+ concat(
+ singletonList(descriptor(repoId)),
+ iterable,
+ singletonList(allGroups(allGroupsSet)), // placeholder, will be recreated at the end with proper content
+ singletonList(rootGroups(rootGroupsSet)) // placeholder, will be recreated at the end with proper content
+ ),
+ new Function<Record, Record>()
+ {
+ public Record apply(final Record rec) {
+ if (Type.DESCRIPTOR == rec.getType()) {
+ return rec;
+ }
+ else if (Type.ALL_GROUPS == rec.getType()) {
+ return allGroups(allGroupsSet);
+ }
+ else if (Type.ROOT_GROUPS == rec.getType()) {
+ return rootGroups(rootGroupsSet);
+ }
+ else {
+ final String groupId = rec.get(Record.GROUP_ID);
+ if (groupId != null) {
+ allGroupsSet.add(groupId);
+ rootGroupsSet.add(rootGroup(groupId));
+ }
+ return rec;
+ }
+ }
+ }
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
new file mode 100644
index 0000000..7d48154
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TransformTest.java
@@ -0,0 +1,98 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Test;
+
+import static org.apache.maven.index.reader.TestUtils.decorate;
+import static org.apache.maven.index.reader.TestUtils.compactFunction;
+import static com.google.common.collect.Iterables.transform;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * UT for {@link RecordCompactor} and {@linl RecordExpander}.
+ */
+public class TransformTest
+ extends TestSupport
+{
+ @Test
+ public void decorateAndTransform() throws IOException {
+ final String indexId = "test";
+ final Record r1 = new Record(Type.ARTIFACT_ADD, artifactMap("org.apache"));
+ final Record r2 = new Record(Type.ARTIFACT_ADD, artifactMap("org.foo"));
+ final Record r3 = new Record(Type.ARTIFACT_ADD, artifactMap("com.bar"));
+ Iterable<Map<String, String>> iterable = transform(
+ decorate(Arrays.asList(r1, r2, r3), indexId),
+ compactFunction
+ );
+
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+ try {
+ IndexWriter indexWriter = new IndexWriter(
+ writableResourceHandler,
+ indexId,
+ false
+ );
+ indexWriter.writeChunk(iterable.iterator());
+ indexWriter.close();
+ }
+ finally {
+ writableResourceHandler.close();
+ }
+
+ IndexReader indexReader = new IndexReader(null, writableResourceHandler);
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ ChunkReader chunkReader = indexReader.iterator().next();
+ final Map<Type, List<Record>> recordTypes = loadRecordsByType(chunkReader);
+ assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(3));
+ assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
+
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).get(0).get(Record.ROOT_GROUPS), equalTo(new String[] {"com","org"}));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).get(0).get(Record.ALL_GROUPS), equalTo(new String[] {"com.bar", "org.apache", "org.foo"}));
+ }
+
+ private Map<EntryKey, Object> artifactMap(final String groupId) {
+ final HashMap<EntryKey, Object> result = new HashMap<EntryKey, Object>();
+ result.put(Record.GROUP_ID, groupId);
+ result.put(Record.ARTIFACT_ID, "artifact");
+ result.put(Record.VERSION, "1.0");
+ result.put(Record.PACKAGING, "jar");
+ result.put(Record.FILE_MODIFIED, System.currentTimeMillis());
+ result.put(Record.FILE_SIZE, 123L);
+ result.put(Record.FILE_EXTENSION, "jar");
+ result.put(Record.HAS_SOURCES, Boolean.FALSE);
+ result.put(Record.HAS_JAVADOC, Boolean.FALSE);
+ result.put(Record.HAS_SIGNATURE, Boolean.FALSE);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz
new file mode 100644
index 0000000..490b21c
Binary files /dev/null and b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz differ
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties
new file mode 100644
index 0000000..d1f1b48
--- /dev/null
+++ b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties
@@ -0,0 +1,6 @@
+#Thu May 28 14:56:58 BRT 2009
+nexus.index.time=20090528175658.015 +0000
+nexus.index.chain-id=1243533418968
+nexus.index.id=apache-snapshots-local
+nexus.index.timestamp=20090528175658.015 +0000
+nexus.index.last-incremental=0
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e60ccd1..0bffd83 100644
--- a/pom.xml
+++ b/pom.xml
@@ -238,7 +238,7 @@ under the License.
<artifactId>aether-util</artifactId>
<version>${aether.version}</version>
</dependency>
-
+
<!-- Maven -->
<dependency>
<groupId>org.apache.maven</groupId>
@@ -341,7 +341,7 @@ under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.11</version>
+ <version>4.12</version>
<scope>test</scope>
</dependency>
@@ -378,6 +378,7 @@ under the License.
<modules>
<module>indexer-core</module>
<module>indexer-cli</module>
+ <module>indexer-reader</module>
<module>indexer-examples</module>
</modules>
@@ -440,11 +441,33 @@ under the License.
<artifactId>apache-rat-plugin</artifactId>
<executions>
<execution>
- <id>rat-verify</id>
- <phase>test</phase>
+ <id>rat-check</id>
+ <phase>validate</phase>
<goals>
<goal>check</goal>
</goals>
+ <configuration>
+ <excludes>
+ <exclude>README.md</exclude>
+ <exclude>.gitignore</exclude>
+ <exclude>NOTICE</exclude>
+ <exclude>**/packageinfo</exclude>
+ <exclude>.git/**</exclude>
+ <exclude>.idea/**</exclude>
+ <exclude>**/*.iml</exclude>
+ <!-- exlude some test resources from rat analysis -->
+ <exclude>src/test/**/*.sha1</exclude>
+ <exclude>src/test/**/*.md5</exclude>
+ <exclude>src/test/**/*.xml</exclude>
+ <exclude>src/test/**/*.pom</exclude>
+ <exclude>src/test/**/*.asc</exclude>
+ <exclude>src/test/**/*.properties</exclude>
+ <exclude>src/test/**/*.swc</exclude>
+ <exclude>src/test/**/*.txt</exclude>
+ <exclude>src/test/**/*.filename</exclude>
+ <exclude>src/test/**/.placeholder</exclude>
+ </excludes>
+ </configuration>
</execution>
</executions>
</plugin>
@@ -501,7 +524,7 @@ under the License.
</excludes>
</configuration>
</plugin>
- <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build
itself. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
[2/2] maven-indexer git commit: MINDEXER-100: forward port 5.x
changes to master
Posted by cs...@apache.org.
MINDEXER-100: forward port 5.x changes to master
Squashed from
https://github.com/apache/maven-indexer/pull/14
Project: http://git-wip-us.apache.org/repos/asf/maven-indexer/repo
Commit: http://git-wip-us.apache.org/repos/asf/maven-indexer/commit/6cb2fcbe
Tree: http://git-wip-us.apache.org/repos/asf/maven-indexer/tree/6cb2fcbe
Diff: http://git-wip-us.apache.org/repos/asf/maven-indexer/diff/6cb2fcbe
Branch: refs/heads/master
Commit: 6cb2fcbe47883cd0decac0295493bef6577d7eb4
Parents: a7aeb9f
Author: Tamas Cservenak <ta...@cservenak.net>
Authored: Fri Mar 31 14:31:35 2017 +0200
Committer: Tamas Cservenak <ta...@cservenak.net>
Committed: Fri Mar 31 14:31:35 2017 +0200
----------------------------------------------------------------------
.../maven/index/context/NexusIndexWriter.java | 29 +-
.../maven/index/updater/IndexDataReader.java | 15 +-
.../apache/maven/index/updater/WagonHelper.java | 4 +
.../org.apache.karaf.features.command-2.2.2.pom | 120 +++++-
indexer-reader/README.md | 8 +
indexer-reader/header.txt | 17 +
indexer-reader/pom.xml | 66 ++++
.../apache/maven/index/reader/ChunkReader.java | 259 +++++++++++++
.../apache/maven/index/reader/ChunkWriter.java | 177 +++++++++
.../apache/maven/index/reader/IndexReader.java | 272 +++++++++++++
.../apache/maven/index/reader/IndexWriter.java | 196 ++++++++++
.../org/apache/maven/index/reader/Record.java | 384 +++++++++++++++++++
.../maven/index/reader/RecordCompactor.java | 207 ++++++++++
.../maven/index/reader/RecordExpander.java | 229 +++++++++++
.../maven/index/reader/ResourceHandler.java | 51 +++
.../org/apache/maven/index/reader/Utils.java | 170 ++++++++
.../index/reader/WritableResourceHandler.java | 58 +++
.../org/apache/maven/index/reader/packageinfo | 1 +
.../index/reader/CachingResourceHandler.java | 129 +++++++
.../maven/index/reader/ChunkReaderTest.java | 95 +++++
.../index/reader/DirectoryResourceHandler.java | 87 +++++
.../maven/index/reader/HttpResourceHandler.java | 75 ++++
.../maven/index/reader/IndexReaderTest.java | 209 ++++++++++
.../maven/index/reader/IndexWriterTest.java | 93 +++++
.../apache/maven/index/reader/TestSupport.java | 184 +++++++++
.../apache/maven/index/reader/TestUtils.java | 108 ++++++
.../maven/index/reader/TransformTest.java | 98 +++++
.../simple/nexus-maven-repository-index.gz | Bin 0 -> 319 bytes
.../nexus-maven-repository-index.properties | 6 +
pom.xml | 33 +-
30 files changed, 3351 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java
----------------------------------------------------------------------
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java
index 038eaa8..130ee88 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
@@ -38,16 +39,27 @@ import org.apache.lucene.util.Version;
public class NexusIndexWriter
extends IndexWriter
{
+ public interface IndexWriterConfigFactory {
+ IndexWriterConfig create(Analyzer analyzer);
+ }
+
+ public static IndexWriterConfigFactory CONFIG_FACTORY = new IndexWriterConfigFactory() {
+ public IndexWriterConfig create(final Analyzer analyzer) {
+ IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, analyzer );
+ config.setRAMBufferSizeMB( 2.0 ); // old default
+ config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially
+ config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT);
+ return config;
+ }
+ };
+
@Deprecated
public NexusIndexWriter( final Directory directory, final Analyzer analyzer, boolean create )
throws CorruptIndexException, LockObtainFailedException, IOException
{
- //super( directory, analyzer, create, MaxFieldLength.LIMITED );
- this(directory, new IndexWriterConfig(Version.LUCENE_46, analyzer));
-
- // setSimilarity( new NexusSimilarity() );
+ this(directory, CONFIG_FACTORY.create(analyzer).setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND));
}
-
+
public NexusIndexWriter( final Directory directory, final IndexWriterConfig config )
throws CorruptIndexException, LockObtainFailedException, IOException
{
@@ -58,11 +70,6 @@ public class NexusIndexWriter
public static IndexWriterConfig defaultConfig()
{
- final IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, new NexusAnalyzer() );
- // default open mode is CreateOrAppend which suits us
- config.setRAMBufferSizeMB( 2.0 ); // old default
- config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially
- config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT);
- return config;
+ return CONFIG_FACTORY.create(new NexusAnalyzer());
}
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
----------------------------------------------------------------------
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
index f76200c..f55fbf9 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
@@ -9,7 +9,7 @@ package org.apache.maven.index.updater;
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -41,18 +41,17 @@ import org.apache.maven.index.context.IndexingContext;
/**
* An index data reader used to parse transfer index format.
- *
+ *
* @author Eugene Kuleshov
*/
public class IndexDataReader
{
private final DataInputStream dis;
- public IndexDataReader( InputStream is )
+ public IndexDataReader( final InputStream is )
throws IOException
{
BufferedInputStream bis = new BufferedInputStream( is, 1024 * 8 );
-
// MINDEXER-13
// LightweightHttpWagon may have performed automatic decompression
// Handle it transparently
@@ -61,7 +60,7 @@ public class IndexDataReader
if ( bis.read() == 0x1f && bis.read() == 0x8b ) // GZIPInputStream.GZIP_MAGIC
{
bis.reset();
- data = new GZIPInputStream( bis, 2 * 1024 );
+ data = new GZIPInputStream( bis, 1024 * 8 );
}
else
{
@@ -97,7 +96,7 @@ public class IndexDataReader
}
w.commit();
-
+
w.forceMerge(1);
w.commit();
@@ -317,7 +316,7 @@ public class IndexDataReader
/**
* Reads index content by using a visitor. <br>
* The visitor is called for each read documents after it has been populated with Lucene fields.
- *
+ *
* @param visitor an index data visitor
* @param context indexing context
* @return statistics about read data
@@ -361,7 +360,7 @@ public class IndexDataReader
/**
* Called on each read document. The document is already populated with fields.
- *
+ *
* @param document read document
*/
void visitDocument( Document document );
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java
----------------------------------------------------------------------
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java b/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java
index 0e450c9..40ea1d9 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java
@@ -198,6 +198,7 @@ public class WagonHelper
throws IOException, FileNotFoundException
{
final File target = File.createTempFile( name, "" );
+ target.deleteOnExit();
retrieve( name, target );
return new FileInputStream( target )
{
@@ -220,6 +221,7 @@ public class WagonHelper
}
catch ( AuthorizationException e )
{
+ targetFile.delete();
String msg = "Authorization exception retrieving " + name;
logError( msg, e );
IOException ioException = new IOException( msg );
@@ -228,6 +230,7 @@ public class WagonHelper
}
catch ( ResourceDoesNotExistException e )
{
+ targetFile.delete();
String msg = "Resource " + name + " does not exist";
logError( msg, e );
FileNotFoundException fileNotFoundException = new FileNotFoundException( msg );
@@ -236,6 +239,7 @@ public class WagonHelper
}
catch ( WagonException e )
{
+ targetFile.delete();
String msg = "Transfer for " + name + " failed";
logError( msg, e );
IOException ioException = new IOException( msg + "; " + e.getMessage() );
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom
----------------------------------------------------------------------
diff --git a/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom b/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom
index 67e206f..062383c 100644
--- a/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom
+++ b/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom
@@ -1,5 +1,115 @@
-#Generated by org.apache.felix.bundleplugin
-#Thu Jun 30 15:25:37 NDT 2011
-version=2.2.2
-groupId=org.apache.karaf.features
-artifactId=org.apache.karaf.features.command
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.karaf.features</groupId>
+ <artifactId>features</artifactId>
+ <version>2.2.2</version>
+ </parent>
+ <artifactId>org.apache.karaf.features.command</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Karaf :: Features :: Command</name>
+ <description>
+ This bundle provides the Karaf shell commands to manipulate features.
+ </description>
+ <properties>
+ <appendedResourcesDirectory>${basedir}/../../etc/appended-resources</appendedResourcesDirectory>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.osgi</groupId>
+ <artifactId>org.osgi.compendium</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.features</groupId>
+ <artifactId>org.apache.karaf.features.core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>org.apache.felix.bundlerepository</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.shell</groupId>
+ <artifactId>org.apache.karaf.shell.console</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.shell</groupId>
+ <artifactId>org.apache.karaf.shell.obr</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.osgi</groupId>
+ <artifactId>spring-osgi-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.servicemix.bundles</groupId>
+ <artifactId>org.apache.servicemix.bundles.junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jdk14</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <resources>
+ <resource>
+ <directory>${project.basedir}/src/main/resources</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${project.basedir}/src/main/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>**/*.info</include>
+ </includes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <configuration>
+ <instructions>
+ <Import-Package>
+ !${project.artifactId}*, javax.management, javax.management.loading, org.apache.felix.service.command, org.apache.felix.gogo.commands, org.apache.karaf.shell.console, *
+ </Import-Package>
+ <Private-Package>!*</Private-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/README.md
----------------------------------------------------------------------
diff --git a/indexer-reader/README.md b/indexer-reader/README.md
new file mode 100644
index 0000000..95aa1b8
--- /dev/null
+++ b/indexer-reader/README.md
@@ -0,0 +1,8 @@
+Indexer Reader Notes
+==================
+
+Indexer Reader is a minimal dep-less library that is able to read published (remote)
+index with incremental update support, making this library user able to integrate
+published Maven Indexes into any engine without depending on maven-indexer-core
+and it's transitive dependencies.
+
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/header.txt
----------------------------------------------------------------------
diff --git a/indexer-reader/header.txt b/indexer-reader/header.txt
new file mode 100644
index 0000000..1a2ef73
--- /dev/null
+++ b/indexer-reader/header.txt
@@ -0,0 +1,17 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/pom.xml
----------------------------------------------------------------------
diff --git a/indexer-reader/pom.xml b/indexer-reader/pom.xml
new file mode 100644
index 0000000..3f91847
--- /dev/null
+++ b/indexer-reader/pom.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.maven.indexer</groupId>
+ <artifactId>maven-indexer</artifactId>
+ <version>6.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>indexer-reader</artifactId>
+ <packaging>bundle</packaging>
+
+ <name>Maven :: Indexer Reader</name>
+ <description>
+ Indexer Reader is a minimal dep-less library that is able to read published (remote) index with incremental update
+ support, making user able to integrate published Maven Indexes into any engine without depending on
+ maven-indexer-core and it's transitive dependencies.
+ </description>
+
+ <dependencies>
+ <!-- Test -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>18.0</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <version>3.0.0</version>
+ <extensions>true</extensions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
new file mode 100644
index 0000000..a6c75ac
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
@@ -0,0 +1,259 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UTFDataFormatException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * Maven 2 Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format.
+ *
+ * @since 5.1.2
+ */
+public class ChunkReader
+ implements Closeable, Iterable<Map<String, String>>
+{
+ private final String chunkName;
+
+ private final DataInputStream dataInputStream;
+
+ private final int version;
+
+ private final Date timestamp;
+
+ public ChunkReader(final String chunkName, final InputStream inputStream) throws IOException
+ {
+ this.chunkName = chunkName.trim();
+ this.dataInputStream = new DataInputStream(new GZIPInputStream(inputStream, 2 * 1024));
+ this.version = ((int) dataInputStream.readByte()) & 0xff;
+ this.timestamp = new Date(dataInputStream.readLong());
+ }
+
+ /**
+ * Returns the chunk name.
+ */
+ public String getName() {
+ return chunkName;
+ }
+
+ /**
+ * Returns index version. All releases so far always returned {@code 1}.
+ */
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * Returns the index timestamp of last update of the index.
+ */
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ /**
+ * Returns the {@link Record} iterator.
+ */
+ public Iterator<Map<String, String>> iterator() {
+ try {
+ return new IndexIterator(dataInputStream);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("error", e);
+ }
+ }
+
+ /**
+ * Closes this reader and it's underlying input.
+ */
+ public void close() throws IOException {
+ dataInputStream.close();
+ }
+
+ /**
+ * Low memory footprint index iterator that incrementally parses the underlying stream.
+ */
+ private static class IndexIterator
+ implements Iterator<Map<String, String>>
+ {
+ private final DataInputStream dataInputStream;
+
+ private Map<String, String> nextRecord;
+
+ public IndexIterator(final DataInputStream dataInputStream) throws IOException {
+ this.dataInputStream = dataInputStream;
+ this.nextRecord = nextRecord();
+ }
+
+ public boolean hasNext() {
+ return nextRecord != null;
+ }
+
+ public Map<String, String> next() {
+ if (nextRecord == null) {
+ throw new NoSuchElementException("chunk depleted");
+ }
+ Map<String, String> result = nextRecord;
+ nextRecord = nextRecord();
+ return result;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("remove");
+ }
+
+ private Map<String, String> nextRecord() {
+ try {
+ return readRecord(dataInputStream);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("read error", e);
+ }
+ }
+ }
+
+ /**
+ * Reads and returns next record from the underlying stream, or {@code null} if no more records.
+ */
+ private static Map<String, String> readRecord(final DataInput dataInput)
+ throws IOException
+ {
+ int fieldCount;
+ try {
+ fieldCount = dataInput.readInt();
+ }
+ catch (EOFException ex) {
+ return null; // no more documents
+ }
+
+ Map<String, String> recordMap = new HashMap<String, String>();
+ for (int i = 0; i < fieldCount; i++) {
+ readField(recordMap, dataInput);
+ }
+ return recordMap;
+ }
+
+ private static void readField(final Map<String, String> record, final DataInput dataInput)
+ throws IOException
+ {
+ dataInput.readByte(); // flags: neglect them
+ String name = dataInput.readUTF();
+ String value = readUTF(dataInput);
+ record.put(name, value);
+ }
+
+ private static String readUTF(final DataInput dataInput)
+ throws IOException
+ {
+ int utflen = dataInput.readInt();
+
+ byte[] bytearr;
+ char[] chararr;
+
+ try {
+ bytearr = new byte[utflen];
+ chararr = new char[utflen];
+ }
+ catch (OutOfMemoryError e) {
+ IOException ioex = new IOException("Index data content is corrupt");
+ ioex.initCause(e);
+ throw ioex;
+ }
+
+ int c, char2, char3;
+ int count = 0;
+ int chararr_count = 0;
+
+ dataInput.readFully(bytearr, 0, utflen);
+
+ while (count < utflen) {
+ c = bytearr[count] & 0xff;
+ if (c > 127) {
+ break;
+ }
+ count++;
+ chararr[chararr_count++] = (char) c;
+ }
+
+ while (count < utflen) {
+ c = bytearr[count] & 0xff;
+ switch (c >> 4) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ /* 0xxxxxxx */
+ count++;
+ chararr[chararr_count++] = (char) c;
+ break;
+
+ case 12:
+ case 13:
+ /* 110x xxxx 10xx xxxx */
+ count += 2;
+ if (count > utflen) {
+ throw new UTFDataFormatException("malformed input: partial character at end");
+ }
+ char2 = bytearr[count - 1];
+ if ((char2 & 0xC0) != 0x80) {
+ throw new UTFDataFormatException("malformed input around byte " + count);
+ }
+ chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
+ break;
+
+ case 14:
+ /* 1110 xxxx 10xx xxxx 10xx xxxx */
+ count += 3;
+ if (count > utflen) {
+ throw new UTFDataFormatException("malformed input: partial character at end");
+ }
+ char2 = bytearr[count - 2];
+ char3 = bytearr[count - 1];
+ if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) {
+ throw new UTFDataFormatException("malformed input around byte " + (count - 1));
+ }
+ chararr[chararr_count++] =
+ (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F));
+ break;
+
+ default:
+ /* 10xx xxxx, 1111 xxxx */
+ throw new UTFDataFormatException("malformed input around byte " + count);
+ }
+ }
+
+ // The number of chars produced may be less than utflen
+ return new String(chararr, 0, chararr_count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
new file mode 100644
index 0000000..24e4701
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
@@ -0,0 +1,177 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Maven 2 Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format.
+ *
+ * @since 5.1.2
+ */
+public class ChunkWriter
+ implements Closeable
+{
+ private static final int F_INDEXED = 1;
+
+ private static final int F_TOKENIZED = 2;
+
+ private static final int F_STORED = 4;
+
+ private final String chunkName;
+
+ private final DataOutputStream dataOutputStream;
+
+ private final int version;
+
+ private final Date timestamp;
+
+ public ChunkWriter(final String chunkName, final OutputStream outputStream, final int version, final Date timestamp)
+ throws IOException
+ {
+ this.chunkName = chunkName.trim();
+ this.dataOutputStream = new DataOutputStream(new GZIPOutputStream(outputStream, 2 * 1024));
+ this.version = version;
+ this.timestamp = timestamp;
+
+ dataOutputStream.writeByte(version);
+ dataOutputStream.writeLong(timestamp == null ? -1 : timestamp.getTime());
+ }
+
+ /**
+ * Returns the chunk name.
+ */
+ public String getName() {
+ return chunkName;
+ }
+
+ /**
+ * Returns index version. All releases so far always returned {@code 1}.
+ */
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * Returns the index timestamp of last update of the index.
+ */
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ /**
+ * Writes out the record iterator and returns the written record count.
+ */
+ public int writeChunk(final Iterator<Map<String, String>> iterator) throws IOException {
+ int written = 0;
+ while (iterator.hasNext()) {
+ writeRecord(iterator.next(), dataOutputStream);
+ written++;
+ }
+ return written;
+ }
+
+ /**
+ * Closes this reader and it's underlying input.
+ */
+ public void close() throws IOException {
+ dataOutputStream.close();
+ }
+
+ private static void writeRecord(final Map<String, String> record, final DataOutput dataOutput)
+ throws IOException
+ {
+ dataOutput.writeInt(record.size());
+ for (Map.Entry<String, String> entry : record.entrySet()) {
+ writeField(entry.getKey(), entry.getValue(), dataOutput);
+ }
+ }
+
+ private static void writeField(final String fieldName, final String fieldValue, final DataOutput dataOutput)
+ throws IOException
+ {
+ boolean isIndexed = !(fieldName.equals("i") || fieldName.equals("m"));
+ boolean isTokenized = !(fieldName.equals("i")
+ || fieldName.equals("m")
+ || fieldName.equals("1")
+ || fieldName.equals("px"));
+ int flags = (isIndexed ? F_INDEXED : 0) + (isTokenized ? F_TOKENIZED : 0) + F_STORED;
+ dataOutput.writeByte(flags);
+ dataOutput.writeUTF(fieldName);
+ writeUTF(fieldValue, dataOutput);
+ }
+
+ private static void writeUTF(final String str, final DataOutput dataOutput)
+ throws IOException
+ {
+ int strlen = str.length();
+ int utflen = 0;
+ int c;
+ // use charAt instead of copying String to char array
+ for (int i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ utflen++;
+ }
+ else if (c > 0x07FF) {
+ utflen += 3;
+ }
+ else {
+ utflen += 2;
+ }
+ }
+ dataOutput.writeInt(utflen);
+ byte[] bytearr = new byte[utflen];
+ int count = 0;
+ int i = 0;
+ for (; i < strlen; i++) {
+ c = str.charAt(i);
+ if (!((c >= 0x0001) && (c <= 0x007F))) {
+ break;
+ }
+ bytearr[count++] = (byte) c;
+ }
+ for (; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ bytearr[count++] = (byte) c;
+
+ }
+ else if (c > 0x07FF) {
+ bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
+ }
+ else {
+ bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
+ }
+ }
+ dataOutput.write(bytearr, 0, utflen);
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
new file mode 100644
index 0000000..85c5409
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
@@ -0,0 +1,272 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.maven.index.reader.ResourceHandler.Resource;
+
+import static org.apache.maven.index.reader.Utils.loadProperties;
+import static org.apache.maven.index.reader.Utils.storeProperties;
+
+/**
+ * Maven 2 Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to
+ * read all the required records.
+ *
+ * @since 5.1.2
+ */
+public class IndexReader
+ implements Iterable<ChunkReader>, Closeable
+{
+ private final WritableResourceHandler local;
+
+ private final ResourceHandler remote;
+
+ private final Properties localIndexProperties;
+
+ private final Properties remoteIndexProperties;
+
+ private final String indexId;
+
+ private final Date publishedTimestamp;
+
+ private final boolean incremental;
+
+ private final List<String> chunkNames;
+
+ public IndexReader(final WritableResourceHandler local, final ResourceHandler remote) throws IOException {
+ if (remote == null) {
+ throw new NullPointerException("remote resource handler null");
+ }
+ this.local = local;
+ this.remote = remote;
+ remoteIndexProperties = loadProperties(remote.locate(Utils.INDEX_FILE_PREFIX + ".properties"));
+ if (remoteIndexProperties == null) {
+ throw new IllegalArgumentException("Non-existent remote index");
+ }
+ try {
+ if (local != null) {
+ Properties localProperties = loadProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"));
+ if (localProperties != null) {
+ this.localIndexProperties = localProperties;
+ String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id");
+ String localIndexId = localIndexProperties.getProperty("nexus.index.id");
+ if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) {
+ throw new IllegalArgumentException(
+ "local and remote index IDs does not match or is null: " + localIndexId + ", " +
+ remoteIndexId);
+ }
+ this.indexId = localIndexId;
+ this.incremental = canRetrieveAllChunks();
+ }
+ else {
+ localIndexProperties = null;
+ this.indexId = remoteIndexProperties.getProperty("nexus.index.id");
+ this.incremental = false;
+ }
+ }
+ else {
+ localIndexProperties = null;
+ this.indexId = remoteIndexProperties.getProperty("nexus.index.id");
+ this.incremental = false;
+ }
+ this.publishedTimestamp = Utils.INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp"));
+ this.chunkNames = calculateChunkNames();
+ }
+ catch (ParseException e) {
+ IOException ex = new IOException("Index properties corrupted");
+ ex.initCause(e);
+ throw ex;
+ }
+ }
+
+ /**
+ * Returns the index context ID that published index has set. Usually it is equal to "repository ID" used in {@link
+ * Record.Type#DESCRIPTOR} but does not have to be.
+ */
+ public String getIndexId() {
+ return indexId;
+ }
+
+ /**
+ * Returns the {@link Date} when remote index was last published.
+ */
+ public Date getPublishedTimestamp() {
+ return publishedTimestamp;
+ }
+
+ /**
+ * Returns {@code true} if incremental update is about to happen. If incremental update, the {@link #iterator()} will
+ * return only the diff from the last update.
+ */
+ public boolean isIncremental() {
+ return incremental;
+ }
+
+ /**
+ * Returns unmodifiable list of actual chunks that needs to be pulled from remote {@link ResourceHandler}. Those are
+ * incremental chunks or the big main file, depending on result of {@link #isIncremental()}. Empty list means local
+ * index is up to date, and {@link #iterator()} will return empty iterator.
+ */
+ public List<String> getChunkNames() {
+ return chunkNames;
+ }
+
+ /**
+ * Closes the underlying {@link ResourceHandler}s. In case of incremental update use, it also assumes that user
+ * consumed all the iterator and integrated it, hence, it will update the {@link WritableResourceHandler} contents to
+ * prepare it for future incremental update. If this is not desired (ie. due to aborted update), then this method
+ * should NOT be invoked, but rather the {@link ResourceHandler}s that caller provided in constructor of
+ * this class should be closed manually.
+ */
+ public void close() throws IOException {
+ remote.close();
+ if (local != null) {
+ try {
+ syncLocalWithRemote();
+ }
+ finally {
+ local.close();
+ }
+ }
+ }
+
+ /**
+ * Returns an {@link Iterator} of {@link ChunkReader}s, that if read in sequence, provide all the (incremental)
+ * updates from the index. It is caller responsibility to either consume fully this iterator, or to close current
+ * {@link ChunkReader} if aborting.
+ */
+ public Iterator<ChunkReader> iterator() {
+ return new ChunkReaderIterator(remote, chunkNames.iterator());
+ }
+
+ /**
+ * Stores the remote index properties into local index properties, preparing local {@link WritableResourceHandler}
+ * for future incremental updates.
+ */
+ private void syncLocalWithRemote() throws IOException {
+ storeProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"), remoteIndexProperties);
+ }
+
+ /**
+ * Calculates the chunk names that needs to be fetched.
+ */
+ private List<String> calculateChunkNames() {
+ if (incremental) {
+ ArrayList<String> chunkNames = new ArrayList<String>();
+ int maxCounter = Integer.parseInt(remoteIndexProperties.getProperty("nexus.index.last-incremental"));
+ int currentCounter = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental"));
+ currentCounter++;
+ while (currentCounter <= maxCounter) {
+ chunkNames.add(Utils.INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz");
+ }
+ return Collections.unmodifiableList(chunkNames);
+ }
+ else {
+ return Collections.singletonList(Utils.INDEX_FILE_PREFIX + ".gz");
+ }
+ }
+
+ /**
+ * Verifies incremental update is possible, as all the diff chunks we need are still enlisted in remote properties.
+ */
+ private boolean canRetrieveAllChunks()
+ {
+ String localChainId = localIndexProperties.getProperty("nexus.index.chain-id");
+ String remoteChainId = remoteIndexProperties.getProperty("nexus.index.chain-id");
+
+ // If no chain id, or not the same, do full update
+ if (localChainId == null || remoteChainId == null || !localChainId.equals(remoteChainId)) {
+ return false;
+ }
+
+ try {
+ int localLastIncremental = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental"));
+ String currentLocalCounter = String.valueOf(localLastIncremental);
+ String nextLocalCounter = String.valueOf(localLastIncremental + 1);
+ // check remote props for existence of current or next chunk after local
+ for (Object key : remoteIndexProperties.keySet()) {
+ String sKey = (String) key;
+ if (sKey.startsWith("nexus.index.incremental-")) {
+ String value = remoteIndexProperties.getProperty(sKey);
+ if (currentLocalCounter.equals(value) || nextLocalCounter.equals(value)) {
+ return true;
+ }
+ }
+ }
+ }
+ catch (NumberFormatException e) {
+ // fall through
+ }
+ return false;
+ }
+
+ /**
+ * Internal iterator implementation that lazily opens and closes the returned {@link ChunkReader}s as this iterator
+ * is being consumed.
+ */
+ private static class ChunkReaderIterator
+ implements Iterator<ChunkReader>
+ {
+ private final ResourceHandler resourceHandler;
+
+ private final Iterator<String> chunkNamesIterator;
+
+ private Resource currentResource;
+
+ private ChunkReader currentChunkReader;
+
+ private ChunkReaderIterator(final ResourceHandler resourceHandler, final Iterator<String> chunkNamesIterator) {
+ this.resourceHandler = resourceHandler;
+ this.chunkNamesIterator = chunkNamesIterator;
+ }
+
+ public boolean hasNext() {
+ return chunkNamesIterator.hasNext();
+ }
+
+ public ChunkReader next() {
+ String chunkName = chunkNamesIterator.next();
+ try {
+ if (currentChunkReader != null) {
+ currentChunkReader.close();
+ }
+ currentResource = resourceHandler.locate(chunkName);
+ currentChunkReader = new ChunkReader(chunkName, currentResource.read());
+ return currentChunkReader;
+ }
+ catch (IOException e) {
+ throw new RuntimeException("IO problem while switching chunk readers", e);
+ }
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("remove");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
new file mode 100644
index 0000000..68d4abb
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
@@ -0,0 +1,196 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.apache.maven.index.reader.WritableResourceHandler.WritableResource;
+
+import static org.apache.maven.index.reader.Utils.loadProperties;
+import static org.apache.maven.index.reader.Utils.storeProperties;
+
+/**
+ * Maven 2 Index writer that writes chunk and maintains published property file.
+ * <p/>
+ * <strong>Currently no incremental update is supported, as the deleteion states should be maintained by
+ * caller</strong>. Hence, this writer will always produce the "main" chunk only.
+ *
+ * @since 5.1.2
+ */
+public class IndexWriter
+ implements Closeable
+{
+ private static final int INDEX_V1 = 1;
+
+ private final WritableResourceHandler local;
+
+ private final Properties localIndexProperties;
+
+ private final boolean incremental;
+
+ private final String nextChunkCounter;
+
+ private final String nextChunkName;
+
+ public IndexWriter(final WritableResourceHandler local, final String indexId, final boolean incrementalSupported)
+ throws IOException
+ {
+ if (local == null) {
+ throw new NullPointerException("local resource handler null");
+ }
+ if (indexId == null) {
+ throw new NullPointerException("indexId null");
+ }
+ this.local = local;
+ Properties indexProperties = loadProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"));
+ if (incrementalSupported && indexProperties != null) {
+ this.localIndexProperties = indexProperties;
+ // existing index, this is incremental publish, and we will add new chunk
+ String localIndexId = localIndexProperties.getProperty("nexus.index.id");
+ if (localIndexId == null || !localIndexId.equals(indexId)) {
+ throw new IllegalArgumentException(
+ "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " +
+ indexId);
+ }
+ this.incremental = true;
+ this.nextChunkCounter = calculateNextChunkCounter();
+ this.nextChunkName = Utils.INDEX_FILE_PREFIX + "." + nextChunkCounter + ".gz";
+ }
+ else {
+ // non-existing index, create published index from scratch
+ this.localIndexProperties = new Properties();
+ this.localIndexProperties.setProperty("nexus.index.id", indexId);
+ this.localIndexProperties.setProperty("nexus.index.chain-id", UUID.randomUUID().toString());
+ this.incremental = false;
+ this.nextChunkCounter = null;
+ this.nextChunkName = Utils.INDEX_FILE_PREFIX + ".gz";
+ }
+ }
+
+ /**
+ * Returns the index context ID that published index has set.
+ */
+ public String getIndexId() {
+ return localIndexProperties.getProperty("nexus.index.id");
+ }
+
+ /**
+ * Returns the {@link Date} when index was last published or {@code null} if this is first publishing. In other
+ * words,returns {@code null} when {@link #isIncremental()} returns {@code false}. After this writer is closed, the
+ * return value is updated to "now" (in {@link #close() method}.
+ */
+ public Date getPublishedTimestamp() {
+ try {
+ String timestamp = localIndexProperties.getProperty("nexus.index.timestamp");
+ if (timestamp != null) {
+ return Utils.INDEX_DATE_FORMAT.parse(timestamp);
+ }
+ return null;
+ }
+ catch (ParseException e) {
+ throw new RuntimeException("Corrupt date", e);
+ }
+ }
+
+ /**
+ * Returns {@code true} if incremental publish is about to happen.
+ */
+ public boolean isIncremental() {
+ return incremental;
+ }
+
+ /**
+ * Returns the chain id of published index. If {@link #isIncremental()} is {@code false}, this is the newly generated
+ * chain ID.
+ */
+ public String getChainId() {
+ return localIndexProperties.getProperty("nexus.index.chain-id");
+ }
+
+ /**
+ * Returns the next chunk name about to be published.
+ */
+ public String getNextChunkName() {
+ return nextChunkName;
+ }
+
+ /**
+ * Writes out the record iterator and returns the written record count.
+ */
+ public int writeChunk(final Iterator<Map<String, String>> iterator) throws IOException {
+ int written;
+ WritableResource writableResource = local.locate(nextChunkName);
+ try {
+ final ChunkWriter chunkWriter = new ChunkWriter(nextChunkName, writableResource.write(), INDEX_V1, new Date());
+ try {
+ written = chunkWriter.writeChunk(iterator);
+ }
+ finally {
+ chunkWriter.close();
+ }
+ if (incremental) {
+ // TODO: update main gz file
+ }
+ return written;
+ }
+ finally {
+ writableResource.close();
+ }
+ }
+
+ /**
+ * Closes the underlying {@link ResourceHandler} and synchronizes published index properties, so remote clients
+ * becomes able to consume newly published index. If sync is not desired (ie. due to aborted publish), then this
+ * method should NOT be invoked, but rather the {@link ResourceHandler} that caller provided in constructor of
+ * this class should be closed manually.
+ */
+ public void close() throws IOException {
+ try {
+ if (incremental) {
+ localIndexProperties.setProperty("nexus.index.last-incremental", nextChunkCounter);
+ }
+ localIndexProperties.setProperty("nexus.index.timestamp", Utils.INDEX_DATE_FORMAT.format(new Date()));
+ storeProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"), localIndexProperties);
+ }
+ finally {
+ local.close();
+ }
+ }
+
+ /**
+ * Calculates the chunk names that needs to be fetched.
+ */
+ private String calculateNextChunkCounter() {
+ String lastChunkCounter = localIndexProperties.getProperty("nexus.index.last-incremental");
+ if (lastChunkCounter != null) {
+ return String.valueOf(Integer.parseInt(lastChunkCounter) + 1);
+ }
+ else {
+ return "1";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
new file mode 100644
index 0000000..1746845
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
@@ -0,0 +1,384 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Map;
+
+/**
+ * Maven 2 Index record.
+ *
+ * @since 5.1.2
+ */
+public final class Record
+{
+ public static final class EntryKey<T>
+ {
+ private final String name;
+
+ private final Class<T> proto;
+
+ public EntryKey(final String name, final Class<T> proto) {
+ if (name == null) {
+ throw new NullPointerException("name is null");
+ }
+ if (proto == null) {
+ throw new NullPointerException("proto is null");
+ }
+ this.name = name;
+ this.proto = proto;
+ }
+
+ public T coerce(final Object object) {
+ return (T) proto.cast(object);
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof EntryKey)) {
+ return false;
+ }
+ EntryKey entryKey = (EntryKey) o;
+ return name.equals(entryKey.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "Key{" +
+ "name='" + name + '\'' +
+ ", type=" + proto.getSimpleName() +
+ '}';
+ }
+ }
+
+ /**
+ * Key of repository ID entry, that contains {@link String}.
+ */
+ public static final EntryKey<String> REPOSITORY_ID = new EntryKey<String>("repositoryId", String.class);
+
+ /**
+ * Key of all groups list entry, that contains {@link java.util.List<String>}.
+ */
+ public static final EntryKey<String[]> ALL_GROUPS = new EntryKey<String[]>("allGroups", String[].class);
+
+ /**
+ * Key of root groups list entry, that contains {@link java.util.List<String>}.
+ */
+ public static final EntryKey<String[]> ROOT_GROUPS = new EntryKey<String[]>("rootGroups", String[].class);
+
+ /**
+ * Key of index record modification (added to index or removed from index) timestamp entry, that contains {@link
+ * Long}.
+ */
+ public static final EntryKey<Long> REC_MODIFIED = new EntryKey<Long>("recordModified", Long.class);
+
+ /**
+ * Key of artifact groupId entry, that contains {@link String}.
+ */
+ public static final EntryKey<String> GROUP_ID = new EntryKey<String>("groupId", String.class);
+
+ /**
+ * Key of artifact artifactId entry, that contains {@link String}.
+ */
+ public static final EntryKey<String> ARTIFACT_ID = new EntryKey<String>("artifactId", String.class);
+
+ /**
+ * Key of artifact version entry, that contains {@link String}.
+ */
+ public static final EntryKey<String> VERSION = new EntryKey<String>("version", String.class);
+
+ /**
+ * Key of artifact classifier entry, that contains {@link String}.
+ */
+ public static final EntryKey<String> CLASSIFIER = new EntryKey<String>("classifier", String.class);
+
+ /**
+ * Key of artifact packaging entry, that contains {@link String}.
+ */
+ public static final EntryKey<String> PACKAGING = new EntryKey<String>("packaging", String.class);
+
+ /**
+ * Key of artifact file extension, that contains {@link String}.
+ */
+ public static final EntryKey<String> FILE_EXTENSION = new EntryKey<String>("fileExtension", String.class);
+
+ /**
+ * Key of artifact file last modified timestamp, that contains {@link Long}.
+ */
+ public static final EntryKey<Long> FILE_MODIFIED = new EntryKey<Long>("fileModified", Long.class);
+
+ /**
+ * Key of artifact file size in bytes, that contains {@link Long}.
+ */
+ public static final EntryKey<Long> FILE_SIZE = new EntryKey<Long>("fileSize", Long.class);
+
+ /**
+ * Key of artifact Sources presence flag, that contains {@link Boolean}.
+ */
+ public static final EntryKey<Boolean> HAS_SOURCES = new EntryKey<Boolean>("hasSources", Boolean.class);
+
+ /**
+ * Key of artifact Javadoc presence flag, that contains {@link Boolean}.
+ */
+ public static final EntryKey<Boolean> HAS_JAVADOC = new EntryKey<Boolean>("hasJavadoc", Boolean.class);
+
+ /**
+ * Key of artifact signature presence flag, that contains {@link Boolean}.
+ */
+ public static final EntryKey<Boolean> HAS_SIGNATURE = new EntryKey<Boolean>("hasSignature", Boolean.class);
+
+ /**
+ * Key of artifact name (as set in POM), that contains {@link String}.
+ */
+ public static final EntryKey<String> NAME = new EntryKey<String>("name", String.class);
+
+ /**
+ * Key of artifact description (as set in POM), that contains {@link String}.
+ */
+ public static final EntryKey<String> DESCRIPTION = new EntryKey<String>("description", String.class);
+
+ /**
+ * Key of artifact SHA1 digest, that contains {@link String}.
+ */
+ public static final EntryKey<String> SHA1 = new EntryKey<String>("sha1", String.class);
+
+ /**
+ * Key of artifact contained class names, that contains {@link java.util.List<String>}. Extracted by {@code
+ * JarFileContentsIndexCreator}.
+ */
+ public static final EntryKey<String[]> CLASSNAMES = new EntryKey<String[]>("classNames", String[].class);
+
+ /**
+ * Key of plugin artifact prefix, that contains {@link String}. Extracted by {@code
+ * MavenPluginArtifactInfoIndexCreator}.
+ */
+ public static final EntryKey<String> PLUGIN_PREFIX = new EntryKey<String>("pluginPrefix", String.class);
+
+ /**
+ * Key of plugin artifact goals, that contains {@link java.util.List<String>}. Extracted by {@code
+ * MavenPluginArtifactInfoIndexCreator}.
+ */
+ public static final EntryKey<String[]> PLUGIN_GOALS = new EntryKey<String[]>("pluginGoals", String[].class);
+
+ /**
+ * Key of OSGi "Bundle-SymbolicName" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_BUNDLE_SYMBOLIC_NAME = new EntryKey<String>("Bundle-SymbolicName",
+ String.class);
+
+ /**
+ * Key of OSGi "Bundle-Version" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_BUNDLE_VERSION = new EntryKey<String>("Bundle-Version", String.class);
+
+ /**
+ * Key of OSGi "Export-Package" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_EXPORT_PACKAGE = new EntryKey<String>("Export-Package", String.class);
+
+ /**
+ * Key of OSGi "Export-Service" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_EXPORT_SERVICE = new EntryKey<String>("Export-Service", String.class);
+
+ /**
+ * Key of OSGi "Bundle-Description" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_BUNDLE_DESCRIPTION = new EntryKey<String>("Bundle-Description",
+ String.class);
+
+ /**
+ * Key of OSGi "Bundle-Name" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_BUNDLE_NAME = new EntryKey<String>("Bundle-Name", String.class);
+
+ /**
+ * Key of OSGi "Bundle-License" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_BUNDLE_LICENSE = new EntryKey<String>("Bundle-License", String.class);
+
+ /**
+ * Key of OSGi "Bundle-DocURL" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_EXPORT_DOCURL = new EntryKey<String>("Bundle-DocURL", String.class);
+
+ /**
+ * Key of OSGi "Import-Package" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_IMPORT_PACKAGE = new EntryKey<String>("Import-Package", String.class);
+
+ /**
+ * Key of OSGi "Require-Bundle" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_REQUIRE_BUNDLE = new EntryKey<String>("Require-Bundle", String.class);
+
+ /**
+ * Types of returned records returned from index.
+ */
+ public enum Type
+ {
+ /**
+ * Descriptor record. Can be safely ignored.
+ * Contains following entries:
+ * <ul>
+ * <li>{@link #REPOSITORY_ID}</li>
+ * </ul>
+ */
+ DESCRIPTOR,
+
+ /**
+ * Artifact ADD record. Records of this type should be added to your indexing system.
+ * Contains following entries:
+ * <ul>
+ * <li>{@link #REC_MODIFIED} (when record was added/modified on index)</li>
+ * <li>{@link #GROUP_ID}</li>
+ * <li>{@link #ARTIFACT_ID}</li>
+ * <li>{@link #VERSION}</li>
+ * <li>{@link #CLASSIFIER} (optional)</li>
+ * <li>{@link #FILE_EXTENSION}</li>
+ * <li>{@link #FILE_MODIFIED}</li>
+ * <li>{@link #FILE_SIZE}</li>
+ * <li>{@link #PACKAGING}</li>
+ * <li>{@link #HAS_SOURCES}</li>
+ * <li>{@link #HAS_JAVADOC}</li>
+ * <li>{@link #HAS_SIGNATURE}</li>
+ * <li>{@link #NAME}</li>
+ * <li>{@link #DESCRIPTION}</li>
+ * <li>{@link #SHA1}</li>
+ * <li>{@link #CLASSNAMES} (optional)</li>
+ * <li>{@link #PLUGIN_PREFIX} (optional, for maven-plugins only)</li>
+ * <li>{@link #PLUGIN_GOALS} (optional, for maven-plugins only)</li>
+ * </ul>
+ */
+ ARTIFACT_ADD,
+
+ /**
+ * Artifact REMOVE record. In case of incremental updates, signals that this artifact was removed. Records of this
+ * type should be removed from your indexing system.
+ * Contains following entries:
+ * <ul>
+ * <li>{@link #REC_MODIFIED} (when record was deleted from index)</li>
+ * <li>{@link #GROUP_ID}</li>
+ * <li>{@link #ARTIFACT_ID}</li>
+ * <li>{@link #VERSION}</li>
+ * <li>{@link #CLASSIFIER} (optional)</li>
+ * <li>{@link #FILE_EXTENSION} (if {@link #CLASSIFIER} present)</li>
+ * <li>{@link #PACKAGING} (optional)</li>
+ * </ul>
+ */
+ ARTIFACT_REMOVE,
+
+ /**
+ * Special record, containing all the Maven "groupId"s that are enlisted on the index. Can be safely ignored.
+ * Contains following entries:
+ * <ul>
+ * <li>{@link #ALL_GROUPS}</li>
+ * </ul>
+ */
+ ALL_GROUPS,
+
+ /**
+ * Special record, containing all the root groups of Maven "groupId"s that are enlisted on the index. Can be safely
+ * ignored.
+ * Contains following entries:
+ * <ul>
+ * <li>{@link #ROOT_GROUPS}</li>
+ * </ul>
+ */
+ ROOT_GROUPS
+ }
+
+ private final Type type;
+
+ private final Map<EntryKey, Object> expanded;
+
+ public Record(final Type type, final Map<EntryKey, Object> expanded) {
+ this.type = type;
+ this.expanded = expanded;
+ }
+
+ /**
+ * Returns the {@link Type} of this record. Usually users would be interested in {@link Type#ARTIFACT_ADD} and {@link
+ * Type#ARTIFACT_REMOVE} types only to maintain their own index. Still, indexer offers extra records too, see {@link
+ * Type} for all existing types.
+ */
+ public Type getType() {
+ return type;
+ }
+
+ /**
+ * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption.
+ */
+ public Map<EntryKey, Object> getExpanded() {
+ return expanded;
+ }
+
+ /**
+ * Returns {@code true} if this record contains given {@link EntryKey}.
+ */
+ boolean containsKey(final EntryKey<?> entryKey) { return expanded.containsKey(entryKey); }
+
+ /**
+ * Type safe handy method to get value from expanded map.
+ */
+ public <T> T get(final EntryKey<T> entryKey) {
+ return entryKey.coerce(expanded.get(entryKey));
+ }
+
+ /**
+ * Type safe handy method to put value to expanded map. Accepts {@code null} values, that removes the mapping.
+ */
+ public <T> T put(final EntryKey<T> entryKey, final T value) {
+ if (value == null) {
+ return entryKey.coerce(expanded.remove(entryKey));
+ }
+ else {
+ if (!entryKey.proto.isAssignableFrom(value.getClass())) {
+ throw new IllegalArgumentException("Key " + entryKey + " does not accepts value " + value);
+ }
+ return entryKey.coerce(expanded.put(entryKey, value));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Record{" +
+ "type=" + type +
+ ", expanded=" + expanded +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
new file mode 100644
index 0000000..76fc1e5
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
@@ -0,0 +1,207 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.Type;
+
+import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR;
+import static org.apache.maven.index.reader.Utils.INFO;
+import static org.apache.maven.index.reader.Utils.UINFO;
+import static org.apache.maven.index.reader.Utils.nvl;
+
+/**
+ * Maven 2 Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records.
+ *
+ * @since 5.1.2
+ */
+public class RecordCompactor
+{
+ /**
+ * Compacts {@link Record} into low level MI record with all the encoded fields as physically present in MI binary
+ * chunk.
+ */
+ public Map<String, String> apply(final Record record) {
+ if (Type.DESCRIPTOR == record.getType()) {
+ return compactDescriptor(record);
+ }
+ else if (Type.ALL_GROUPS == record.getType()) {
+ return compactAllGroups(record);
+ }
+ else if (Type.ROOT_GROUPS == record.getType()) {
+ return compactRootGroups(record);
+ }
+ else if (Type.ARTIFACT_REMOVE == record.getType()) {
+ return compactDeletedArtifact(record);
+ }
+ else if (Type.ARTIFACT_ADD == record.getType()) {
+ return compactAddedArtifact(record);
+ }
+ else {
+ throw new IllegalArgumentException("Unknown record: " + record);
+ }
+ }
+
+ private static Map<String, String> compactDescriptor(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ result.put("DESCRIPTOR", "NexusIndex");
+ result.put("IDXINFO", "1.0|" + record.get(Record.REPOSITORY_ID));
+ return result;
+ }
+
+ private static Map<String, String> compactAllGroups(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ result.put("allGroups", "allGroups");
+ putIfNotNullAsStringArray(record.get(Record.ALL_GROUPS), result, "allGroupsList");
+ return result;
+ }
+
+ private static Map<String, String> compactRootGroups(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ result.put("rootGroups", "allGroups");
+ putIfNotNullAsStringArray(record.get(Record.ROOT_GROUPS), result, "rootGroupsList");
+ return result;
+ }
+
+ private static Map<String, String> compactDeletedArtifact(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m");
+ result.put("del", compactUinfo(record));
+ return result;
+ }
+
+ /**
+ * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming.
+ */
+ private static Map<String, String> compactAddedArtifact(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+
+ // Minimal
+ result.put(UINFO, compactUinfo(record));
+
+ StringBuilder info = new StringBuilder();
+ info.append(nvl(record.get(Record.PACKAGING)));
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.FILE_MODIFIED));
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.FILE_SIZE));
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.HAS_SOURCES) ? "1" : "0");
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.HAS_JAVADOC) ? "1" : "0");
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.HAS_SIGNATURE) ? "1" : "0");
+ info.append(FIELD_SEPARATOR);
+ info.append(nvl(record.get(Record.FILE_EXTENSION)));
+ result.put(INFO, info.toString());
+
+ putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m");
+ putIfNotNull(record.get(Record.NAME), result, "n");
+ putIfNotNull(record.get(Record.DESCRIPTION), result, "d");
+ putIfNotNull(record.get(Record.SHA1), result, "1");
+
+ // Jar file contents (optional)
+ putIfNotNullAsStringArray(record.get(Record.CLASSNAMES), result, "classnames");
+
+ // Maven Plugin (optional)
+ putIfNotNull(record.get(Record.PLUGIN_PREFIX), result, "px");
+ putIfNotNullAsStringArray(record.get(Record.PLUGIN_GOALS), result, "gx");
+
+ // OSGi (optional)
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_SYMBOLIC_NAME), result, "Bundle-SymbolicName");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_VERSION), result, "Bundle-Version");
+ putIfNotNull(record.get(Record.OSGI_EXPORT_PACKAGE), result, "Export-Package");
+ putIfNotNull(record.get(Record.OSGI_EXPORT_SERVICE), result, "Export-Service");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_DESCRIPTION), result, "Bundle-Description");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_NAME), result, "Bundle-Name");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_LICENSE), result, "Bundle-License");
+ putIfNotNull(record.get(Record.OSGI_EXPORT_DOCURL), result, "Bundle-DocURL");
+ putIfNotNull(record.get(Record.OSGI_IMPORT_PACKAGE), result, "Import-Package");
+ putIfNotNull(record.get(Record.OSGI_REQUIRE_BUNDLE), result, "Require-Bundle");
+
+ return result;
+ }
+
+ /**
+ * Creates UINFO synthetic field.
+ */
+ private static String compactUinfo(final Record record) {
+ final String classifier = record.get(Record.CLASSIFIER);
+ StringBuilder sb = new StringBuilder();
+ sb.append(record.get(Record.GROUP_ID))
+ .append(FIELD_SEPARATOR)
+ .append(record.get(Record.ARTIFACT_ID))
+ .append(FIELD_SEPARATOR)
+ .append(record.get(Record.VERSION))
+ .append(FIELD_SEPARATOR)
+ .append(nvl(classifier));
+ if (classifier != null) {
+ sb.append(FIELD_SEPARATOR).append(record.get(Record.FILE_EXTENSION));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Helper to put a value from source map into target map, if not null.
+ */
+ private static void putIfNotNull(
+ final String source,
+ final Map<String, String> target,
+ final String targetName)
+ {
+ if (source != null) {
+ target.put(targetName, source);
+ }
+ }
+
+ /**
+ * Helper to put a {@link Long} value from source map into target map, if not null.
+ */
+ private static void putIfNotNullTS(
+ final Long source,
+ final Map<String, String> target,
+ final String targetName)
+ {
+ if (source != null) {
+ target.put(targetName, String.valueOf(source));
+ }
+ }
+
+ /**
+ * Helper to put a array value from source map into target map joined with {@link Utils#FIELD_SEPARATOR}, if not
+ * null.
+ */
+ private static void putIfNotNullAsStringArray(
+ final String[] source,
+ final Map<String, String> target,
+ final String targetName)
+ {
+ if (source != null && source.length > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(source[0]);
+ for (int i = 1; i < source.length; i++) {
+ sb.append(FIELD_SEPARATOR).append(source[i]);
+ }
+ target.put(targetName, sb.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
new file mode 100644
index 0000000..6d7f4be
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
@@ -0,0 +1,229 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+
+import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR;
+import static org.apache.maven.index.reader.Utils.FS_PATTERN;
+import static org.apache.maven.index.reader.Utils.INFO;
+import static org.apache.maven.index.reader.Utils.NOT_AVAILABLE;
+import static org.apache.maven.index.reader.Utils.UINFO;
+import static org.apache.maven.index.reader.Utils.renvl;
+
+/**
+ * Maven 2 Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s.
+ *
+ * @since 5.1.2
+ */
+public class RecordExpander
+{
+ /**
+ * Expands MI low level record into {@link Record}.
+ */
+ public Record apply(final Map<String, String> recordMap) {
+ if (recordMap.containsKey("DESCRIPTOR")) {
+ return expandDescriptor(recordMap);
+ }
+ else if (recordMap.containsKey("allGroups")) {
+ return expandAllGroups(recordMap);
+ }
+ else if (recordMap.containsKey("rootGroups")) {
+ return expandRootGroups(recordMap);
+ }
+ else if (recordMap.containsKey("del")) {
+ return expandDeletedArtifact(recordMap);
+ }
+ else {
+ // Fix up UINFO field wrt MINDEXER-41
+ final String uinfo = recordMap.get(UINFO);
+ final String info = recordMap.get(INFO);
+ if (uinfo != null && !(info == null || info.trim().length() == 0)) {
+ final String[] splitInfo = FS_PATTERN.split(info);
+ if (splitInfo.length > 6) {
+ final String extension = splitInfo[6];
+ if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) {
+ recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension);
+ }
+ }
+ }
+ return expandAddedArtifact(recordMap);
+ }
+ }
+
+ private static Record expandDescriptor(final Map<String, String> raw) {
+ final Record result = new Record(Type.DESCRIPTOR, new HashMap<EntryKey, Object>());
+ String[] r = FS_PATTERN.split(raw.get("IDXINFO"));
+ result.put(Record.REPOSITORY_ID, r[1]);
+ return result;
+ }
+
+ private static Record expandAllGroups(final Map<String, String> raw) {
+ final Record result = new Record(Type.ALL_GROUPS, new HashMap<EntryKey, Object>());
+ putIfNotNullAsStringArray(raw, "allGroupsList", result, Record.ALL_GROUPS);
+ return result;
+ }
+
+ private static Record expandRootGroups(final Map<String, String> raw) {
+ final Record result = new Record(Type.ROOT_GROUPS, new HashMap<EntryKey, Object>());
+ putIfNotNullAsStringArray(raw, "rootGroupsList", result, Record.ROOT_GROUPS);
+ return result;
+ }
+
+ private static Record expandDeletedArtifact(final Map<String, String> raw) {
+ final Record result = new Record(Type.ARTIFACT_REMOVE, new HashMap<EntryKey, Object>());
+ putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED);
+ if (raw.containsKey("del")) {
+ expandUinfo(raw.get("del"), result);
+ }
+ return result;
+ }
+
+ /**
+ * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming.
+ */
+ private static Record expandAddedArtifact(final Map<String, String> raw) {
+ final Record result = new Record(Type.ARTIFACT_ADD, new HashMap<EntryKey, Object>());
+
+ // Minimal
+ expandUinfo(raw.get(UINFO), result);
+ final String info = raw.get(INFO);
+ if (info != null) {
+ String[] r = FS_PATTERN.split(info);
+ result.put(Record.PACKAGING, renvl(r[0]));
+ result.put(Record.FILE_MODIFIED, Long.valueOf(r[1]));
+ result.put(Record.FILE_SIZE, Long.valueOf(r[2]));
+ result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE : Boolean.FALSE);
+ result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE : Boolean.FALSE);
+ result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE : Boolean.FALSE);
+ if (r.length > 6) {
+ result.put(Record.FILE_EXTENSION, r[6]);
+ }
+ else {
+ final String packaging = Record.PACKAGING.coerce(result.get(Record.PACKAGING));
+ if (result.containsKey(Record.CLASSIFIER)
+ || "pom".equals(packaging)
+ || "war".equals(packaging)
+ || "ear".equals(packaging)) {
+ result.put(Record.FILE_EXTENSION, packaging);
+ }
+ else {
+ result.put(Record.FILE_EXTENSION, "jar"); // best guess
+ }
+ }
+ }
+ putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED);
+ putIfNotNull(raw, "n", result, Record.NAME);
+ putIfNotNull(raw, "d", result, Record.DESCRIPTION);
+ putIfNotNull(raw, "1", result, Record.SHA1);
+
+ // Jar file contents (optional)
+ putIfNotNullAsStringArray(raw, "classnames", result, Record.CLASSNAMES);
+
+ // Maven Plugin (optional)
+ putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX);
+ putIfNotNullAsStringArray(raw, "gx", result, Record.PLUGIN_GOALS);
+
+ // OSGi (optional)
+ putIfNotNull(raw, "Bundle-SymbolicName", result, Record.OSGI_BUNDLE_SYMBOLIC_NAME);
+ putIfNotNull(raw, "Bundle-Version", result, Record.OSGI_BUNDLE_VERSION);
+ putIfNotNull(raw, "Export-Package", result, Record.OSGI_EXPORT_PACKAGE);
+ putIfNotNull(raw, "Export-Service", result, Record.OSGI_EXPORT_SERVICE);
+ putIfNotNull(raw, "Bundle-Description", result, Record.OSGI_BUNDLE_DESCRIPTION);
+ putIfNotNull(raw, "Bundle-Name", result, Record.OSGI_BUNDLE_NAME);
+ putIfNotNull(raw, "Bundle-License", result, Record.OSGI_BUNDLE_LICENSE);
+ putIfNotNull(raw, "Bundle-DocURL", result, Record.OSGI_EXPORT_DOCURL);
+ putIfNotNull(raw, "Import-Package", result, Record.OSGI_IMPORT_PACKAGE);
+ putIfNotNull(raw, "Require-Bundle", result, Record.OSGI_REQUIRE_BUNDLE);
+
+ return result;
+ }
+
+ /**
+ * Expands UINFO synthetic field. Handles {@code null} String inputs.
+ */
+ private static void expandUinfo(final String uinfo, final Record result) {
+ if (uinfo != null) {
+ String[] r = FS_PATTERN.split(uinfo);
+ result.put(Record.GROUP_ID, r[0]);
+ result.put(Record.ARTIFACT_ID, r[1]);
+ result.put(Record.VERSION, r[2]);
+ String classifier = renvl(r[3]);
+ if (classifier != null) {
+ result.put(Record.CLASSIFIER, classifier);
+ if (r.length > 4) {
+ result.put(Record.FILE_EXTENSION, r[4]);
+ }
+ }
+ else if (r.length > 4) {
+ result.put(Record.PACKAGING, r[4]);
+ }
+ }
+ }
+
+ /**
+ * Helper to put a value from source map into target map, if not null.
+ */
+ private static void putIfNotNull(
+ final Map<String, String> source,
+ final String sourceName,
+ final Record target,
+ final EntryKey targetName)
+ {
+ String value = source.get(sourceName);
+ if (value != null && value.trim().length() != 0) {
+ target.put(targetName, value);
+ }
+ }
+
+ /**
+ * Helper to put a {@link Long} value from source map into target map, if not null.
+ */
+ private static void putIfNotNullTS(
+ final Map<String, String> source,
+ final String sourceName,
+ final Record target,
+ final EntryKey targetName)
+ {
+ String value = source.get(sourceName);
+ if (value != null && value.trim().length() != 0) {
+ target.put(targetName, Long.valueOf(value));
+ }
+ }
+
+ /**
+ * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null.
+ */
+ private static void putIfNotNullAsStringArray(
+ final Map<String, String> source,
+ final String sourceName,
+ final Record target,
+ final EntryKey targetName)
+ {
+ String value = source.get(sourceName);
+ if (value != null && value.trim().length() != 0) {
+ target.put(targetName, FS_PATTERN.split(value));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
new file mode 100644
index 0000000..244a994
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
@@ -0,0 +1,51 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Maven 2 Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that
+ * implementations could perform any extra activity as FS locking or so (if uses FS as backing store). Is used by single
+ * thread only.
+ *
+ * @since 5.1.2
+ */
+public interface ResourceHandler
+ extends Closeable
+{
+ interface Resource
+ {
+ /**
+ * Returns the {@link InputStream} stream of the resource, if exists, {@code null} otherwise. The stream should
+ * be closed by caller, otherwise resource leaks might be introduced.
+ */
+ InputStream read() throws IOException;
+ }
+
+ /**
+ * Returns the {@link Resource} with {@code name}, non {@code null}.
+ *
+ * @param name Resource name, guaranteed to be non-{@code null} and is FS and URL safe string.
+ */
+ Resource locate(String name) throws IOException;
+}