You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@maven.apache.org by cs...@apache.org on 2015/11/03 12:29:19 UTC
[1/2] maven-indexer git commit: Added index writer,
that writes single chunk for now
Repository: maven-indexer
Updated Branches:
refs/heads/maven-indexer-5.x af8783d8f -> b9c4d9081
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
index edb614a..fb33285 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java
@@ -23,9 +23,11 @@ import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
/**
* A trivial {@link File} directory handler that does not perform any locking or extra bits, and just serves up files
@@ -46,26 +48,24 @@ public class DirectoryResourceHandler
this.rootDirectory = rootDirectory;
}
- public InputStream open(final String name) throws IOException {
- return new BufferedInputStream(new FileInputStream(new File(rootDirectory, name)));
+ public File getRootDirectory() {
+ return rootDirectory;
}
- public void save(final String name, final InputStream inputStream) throws IOException {
+ public InputStream open(final String name) throws IOException {
try {
- final BufferedOutputStream outputStream = new BufferedOutputStream(
- new FileOutputStream(new File(rootDirectory, name)));
- int read;
- byte[] bytes = new byte[8192];
- while ((read = inputStream.read(bytes)) != -1) {
- outputStream.write(bytes, 0, read);
- }
- outputStream.close();
+ return new BufferedInputStream(new FileInputStream(new File(rootDirectory, name)));
}
- finally {
- inputStream.close();
+ catch (FileNotFoundException e) {
+ return null;
}
}
+ public OutputStream openWrite(final String name) throws IOException {
+ return new BufferedOutputStream(
+ new FileOutputStream(new File(rootDirectory, name)));
+ }
+
public void close() throws IOException {
// nop
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
index e5d7df1..153391f 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java
@@ -25,6 +25,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
import java.util.Arrays;
+import java.util.Date;
import org.junit.Ignore;
import org.junit.Test;
@@ -36,12 +37,14 @@ import static org.junit.Assert.assertThat;
* UT for {@link IndexReader}
*/
public class IndexReaderTest
+ extends TestSupport
{
@Test
public void simple() throws IOException {
final IndexReader indexReader = new IndexReader(
null,
- new DirectoryResourceHandler(new File("src/test/resources/")));
+ testResourceHandler("simple")
+ );
try {
assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(1243533418015L));
@@ -54,7 +57,62 @@ public class IndexReaderTest
assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
assertThat(chunkReader.getVersion(), equalTo(1));
assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
- for (Record record : chunkReader) {
+ for (Record record : Iterables.transform(chunkReader, new RecordExpander())) {
+ records++;
+ }
+ }
+
+ assertThat(chunks, equalTo(1));
+ assertThat(records, equalTo(5));
+ }
+ finally {
+ indexReader.close();
+ }
+ }
+
+ @Test
+ public void roundtrip() throws IOException {
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+ Date published;
+ {
+ final IndexReader indexReader = new IndexReader(
+ null,
+ testResourceHandler("simple")
+ );
+ final IndexWriter indexWriter = new IndexWriter(
+ writableResourceHandler,
+ indexReader.getIndexId(),
+ false
+ );
+ try {
+ for (ChunkReader chunkReader : indexReader) {
+ indexWriter.writeChunk(chunkReader.iterator());
+ }
+ }
+ finally {
+ indexWriter.close();
+ published = indexWriter.getPublishedTimestamp();
+ indexReader.close();
+ }
+ }
+
+ final IndexReader indexReader = new IndexReader(
+ null,
+ writableResourceHandler
+ );
+ try {
+ assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
+ assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
+ assertThat(indexReader.isIncremental(), equalTo(false));
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ int chunks = 0;
+ int records = 0;
+ for (ChunkReader chunkReader : indexReader) {
+ chunks++;
+ assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
+ for (Record record : Iterables.transform(chunkReader, new RecordExpander())) {
records++;
}
}
@@ -87,10 +145,8 @@ public class IndexReaderTest
writer.write("chunkVersion=" + chunkReader.getVersion() + "\n");
writer.write("chunkPublished=" + chunkReader.getTimestamp() + "\n");
writer.write("= = = = = = \n");
- for (Record record : chunkReader) {
+ for (Record record : Iterables.transform(chunkReader, new RecordExpander())) {
writer.write(record.getExpanded() + "\n");
- writer.write("--------- \n");
- writer.write(record.getRaw() + "\n");
}
}
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
new file mode 100644
index 0000000..0895c9d
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexWriterTest.java
@@ -0,0 +1,91 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * UT for {@link IndexWriter}
+ */
+public class IndexWriterTest
+ extends TestSupport
+{
+ @Test
+ public void roundtrip() throws IOException {
+ IndexReader indexReader;
+ IndexWriter indexWriter;
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+
+ // write it once
+ indexReader = new IndexReader(
+ null,
+ testResourceHandler("simple")
+ );
+ indexWriter = new IndexWriter(
+ writableResourceHandler,
+ indexReader.getIndexId(),
+ false
+ );
+ try {
+ for (ChunkReader chunkReader : indexReader) {
+ indexWriter.writeChunk(chunkReader.iterator());
+ }
+ }
+ finally {
+ indexWriter.close();
+ indexReader.close();
+ }
+
+ // read what we wrote out
+ indexReader = new IndexReader(
+ null,
+ writableResourceHandler
+ );
+ try {
+ assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local"));
+ // assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(published.getTime()));
+ assertThat(indexReader.isIncremental(), equalTo(false));
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ int chunks = 0;
+ int records = 0;
+ for (ChunkReader chunkReader : indexReader) {
+ chunks++;
+ assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz"));
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ // assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
+ for (Record record : Iterables.transform(chunkReader, new RecordExpander())) {
+ records++;
+ }
+ }
+
+ assertThat(chunks, equalTo(1));
+ assertThat(records, equalTo(5));
+ }
+ finally {
+ indexReader.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java
new file mode 100644
index 0000000..c0b1142
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IterablesTest.java
@@ -0,0 +1,92 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * UT for {@link Iterables}
+ */
+public class IterablesTest
+ extends TestSupport
+{
+ @Test
+ public void decorateAndTransform() throws IOException {
+ final String indexId = "test";
+ final Record r1 = new Record(Type.ARTIFACT_ADD, artifactMap("org.apache"));
+ final Record r2 = new Record(Type.ARTIFACT_ADD, artifactMap("org.foo"));
+ final Record r3 = new Record(Type.ARTIFACT_ADD, artifactMap("com.bar"));
+ Iterable<Map<String, String>> iterable = Iterables.decorateAndTransform(Arrays.asList(r1, r2, r3), indexId);
+
+ WritableResourceHandler writableResourceHandler = createWritableResourceHandler();
+ try {
+ IndexWriter indexWriter = new IndexWriter(
+ writableResourceHandler,
+ indexId,
+ false
+ );
+ indexWriter.writeChunk(iterable.iterator());
+ indexWriter.close();
+ }
+ finally {
+ writableResourceHandler.close();
+ }
+
+ IndexReader indexReader = new IndexReader(null, writableResourceHandler);
+ assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz")));
+ ChunkReader chunkReader = indexReader.iterator().next();
+ final Map<Type, List<Record>> recordTypes = countRecordTypes(chunkReader);
+ assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(3));
+ assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
+
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).get(0).get(Record.ROOT_GROUPS), equalTo(new String[] {"com","org"}));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).get(0).get(Record.ALL_GROUPS), equalTo(new String[] {"com.bar", "org.apache", "org.foo"}));
+ }
+
+ private Map<EntryKey, Object> artifactMap(final String groupId) {
+ final HashMap<EntryKey, Object> result = new HashMap<EntryKey, Object>();
+ result.put(Record.GROUP_ID, groupId);
+ result.put(Record.ARTIFACT_ID, "artifact");
+ result.put(Record.VERSION, "1.0");
+ result.put(Record.PACKAGING, "jar");
+ result.put(Record.FILE_MODIFIED, System.currentTimeMillis());
+ result.put(Record.FILE_SIZE, 123L);
+ result.put(Record.FILE_EXTENSION, "jar");
+ result.put(Record.HAS_SOURCES, Boolean.FALSE);
+ result.put(Record.HAS_JAVADOC, Boolean.FALSE);
+ result.put(Record.HAS_SIGNATURE, Boolean.FALSE);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
new file mode 100644
index 0000000..bcd465e
--- /dev/null
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/TestSupport.java
@@ -0,0 +1,163 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Record.Type;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Test support.
+ */
+public class TestSupport
+{
+ @Rule
+ public TestName testName = new TestName();
+
+ private File tempDir;
+
+ private List<DirectoryResourceHandler> directoryResourceHandlers;
+
+ /**
+ * Creates the temp directory and list for resource handlers.
+ */
+ @Before
+ public void setup() throws IOException {
+ this.tempDir = new File("target/tmp-" + getClass().getSimpleName());
+ this.tempDir.delete();
+ this.tempDir.mkdirs();
+ this.directoryResourceHandlers = new ArrayList<DirectoryResourceHandler>();
+ }
+
+ /**
+ * Closes all the registered resources handlers and deletes the temp directory.
+ */
+ @After
+ public void cleanup() throws IOException {
+ for (DirectoryResourceHandler directoryResourceHandler : directoryResourceHandlers) {
+ directoryResourceHandler.close();
+ }
+ // delete(tempDir);
+ }
+
+ /**
+ * Creates a temp file within {@link #tempDir}.
+ */
+ protected File createTempFile() throws IOException {
+ return File.createTempFile(testName.getMethodName() + "-file", "", tempDir);
+ }
+
+
+ /**
+ * Creates a temp file within {@link #tempDir} with given name.
+ */
+ protected File createTempFile(final String name) throws IOException {
+ return new File(tempDir, name);
+ }
+
+ /**
+ * Creates a temp directory within {@link #tempDir}.
+ */
+ protected File createTempDirectory() throws IOException {
+ File result = File.createTempFile(testName.getMethodName() + "-dir", "", tempDir);
+ result.delete();
+ result.mkdirs();
+ return result;
+ }
+
+ /**
+ * Creates an empty {@link DirectoryResourceHandler}.
+ */
+ protected WritableResourceHandler createWritableResourceHandler() throws IOException {
+ DirectoryResourceHandler result = new DirectoryResourceHandler(createTempDirectory());
+ directoryResourceHandlers.add(result);
+ return result;
+ }
+
+ /**
+ * Creates a "test" {@link ResourceHandler} that contains predefined files, is mapped to test resources under given
+ * name.
+ */
+ protected ResourceHandler testResourceHandler(final String name) throws IOException {
+ DirectoryResourceHandler result = new DirectoryResourceHandler(new File("src/test/resources/" + name));
+ directoryResourceHandlers.add(result);
+ return result;
+ }
+
+ /**
+ * Consumes {@link ChunkReader} and creates "statistics" of index by record type.
+ */
+ protected Map<Type, List<Record>> countRecordTypes(final ChunkReader chunkReader) throws IOException {
+ HashMap<Type, List<Record>> stat = new HashMap<Type, List<Record>>();
+ try {
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ final RecordExpander recordExpander = new RecordExpander();
+ for (Map<String, String> rec : chunkReader) {
+ System.out.println(rec);
+ final Record record = recordExpander.apply(rec);
+ System.out.println(record);
+ if (!stat.containsKey(record.getType())) {
+ stat.put(record.getType(), new ArrayList<Record>());
+ }
+ stat.get(record.getType()).add(record);
+ }
+ }
+ finally {
+ chunkReader.close();
+ }
+ return stat;
+ }
+
+ /**
+ * Delete recursively.
+ */
+ private static boolean delete(final File file) {
+ if (file == null) {
+ return false;
+ }
+ if (!file.exists()) {
+ return true;
+ }
+ if (file.isDirectory()) {
+ String[] list = file.list();
+ if (list != null) {
+ for (int i = 0; i < list.length; i++) {
+ File entry = new File(file, list[i]);
+ if (!delete(entry)) {
+ return false;
+ }
+ }
+ }
+ }
+ return file.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/resources/nexus-maven-repository-index.gz
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/resources/nexus-maven-repository-index.gz b/indexer-reader/src/test/resources/nexus-maven-repository-index.gz
deleted file mode 100644
index 490b21c..0000000
Binary files a/indexer-reader/src/test/resources/nexus-maven-repository-index.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/resources/nexus-maven-repository-index.properties
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/resources/nexus-maven-repository-index.properties b/indexer-reader/src/test/resources/nexus-maven-repository-index.properties
deleted file mode 100644
index d1f1b48..0000000
--- a/indexer-reader/src/test/resources/nexus-maven-repository-index.properties
+++ /dev/null
@@ -1,6 +0,0 @@
-#Thu May 28 14:56:58 BRT 2009
-nexus.index.time=20090528175658.015 +0000
-nexus.index.chain-id=1243533418968
-nexus.index.id=apache-snapshots-local
-nexus.index.timestamp=20090528175658.015 +0000
-nexus.index.last-incremental=0
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz
new file mode 100644
index 0000000..490b21c
Binary files /dev/null and b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.gz differ
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties
new file mode 100644
index 0000000..d1f1b48
--- /dev/null
+++ b/indexer-reader/src/test/resources/simple/nexus-maven-repository-index.properties
@@ -0,0 +1,6 @@
+#Thu May 28 14:56:58 BRT 2009
+nexus.index.time=20090528175658.015 +0000
+nexus.index.chain-id=1243533418968
+nexus.index.id=apache-snapshots-local
+nexus.index.timestamp=20090528175658.015 +0000
+nexus.index.last-incremental=0
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 369fe42..f1464b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -132,7 +132,7 @@ under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.10</version>
+ <version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
[2/2] maven-indexer git commit: Added index writer,
that writes single chunk for now
Posted by cs...@apache.org.
Added index writer, that writes single chunk for now
Project: http://git-wip-us.apache.org/repos/asf/maven-indexer/repo
Commit: http://git-wip-us.apache.org/repos/asf/maven-indexer/commit/b9c4d908
Tree: http://git-wip-us.apache.org/repos/asf/maven-indexer/tree/b9c4d908
Diff: http://git-wip-us.apache.org/repos/asf/maven-indexer/diff/b9c4d908
Branch: refs/heads/maven-indexer-5.x
Commit: b9c4d90810b6174a1851c179ffebf2c3456c4c66
Parents: af8783d
Author: Tamas Cservenak <ta...@cservenak.net>
Authored: Tue Nov 3 12:28:41 2015 +0100
Committer: Tamas Cservenak <ta...@cservenak.net>
Committed: Tue Nov 3 12:28:41 2015 +0100
----------------------------------------------------------------------
.../apache/maven/index/reader/ChunkReader.java | 423 +++++--------------
.../apache/maven/index/reader/ChunkWriter.java | 177 ++++++++
.../apache/maven/index/reader/IndexReader.java | 59 +--
.../apache/maven/index/reader/IndexWriter.java | 196 +++++++++
.../apache/maven/index/reader/Iterables.java | 204 +++++++++
.../org/apache/maven/index/reader/Record.java | 215 ++++++++--
.../maven/index/reader/RecordCompactor.java | 205 +++++++++
.../maven/index/reader/RecordExpander.java | 228 ++++++++++
.../maven/index/reader/ResourceHandler.java | 2 +-
.../org/apache/maven/index/reader/Utils.java | 98 +++++
.../index/reader/WritableResourceHandler.java | 11 +-
.../index/reader/CachingResourceHandler.java | 22 +-
.../maven/index/reader/ChunkReaderTest.java | 71 +++-
.../index/reader/DirectoryResourceHandler.java | 26 +-
.../maven/index/reader/IndexReaderTest.java | 66 ++-
.../maven/index/reader/IndexWriterTest.java | 91 ++++
.../maven/index/reader/IterablesTest.java | 92 ++++
.../apache/maven/index/reader/TestSupport.java | 163 +++++++
.../resources/nexus-maven-repository-index.gz | Bin 319 -> 0 bytes
.../nexus-maven-repository-index.properties | 6 -
.../simple/nexus-maven-repository-index.gz | Bin 0 -> 319 bytes
.../nexus-maven-repository-index.properties | 6 +
pom.xml | 2 +-
23 files changed, 1919 insertions(+), 444 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
index 89434fd..dc6ee79 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java
@@ -20,40 +20,27 @@ package org.apache.maven.index.reader;
*/
import java.io.Closeable;
+import java.io.DataInput;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UTFDataFormatException;
-import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
-import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
-import org.apache.maven.index.reader.Record.Type;
-
/**
- * Maven 2 Index published binary chunk reader.
+ * Maven 2 Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format.
*
* @since 5.1.2
*/
public class ChunkReader
- implements Closeable, Iterable<Record>
+ implements Closeable, Iterable<Map<String, String>>
{
- private static final String FIELD_SEPARATOR = "|";
-
- private static final String NOT_AVAILABLE = "NA";
-
- private static final String UINFO = "u";
-
- private static final String INFO = "i";
-
- private static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR));
-
private final String chunkName;
private final DataInputStream dataInputStream;
@@ -78,14 +65,14 @@ public class ChunkReader
}
/**
- * Returns index getVersion. All releases so far always returned {@code 1}.
+ * Returns index version. All releases so far always returned {@code 1}.
*/
public int getVersion() {
return version;
}
/**
- * Returns the getTimestamp of last update of the index.
+ * Returns the index timestamp of last update of the index.
*/
public Date getTimestamp() {
return timestamp;
@@ -94,7 +81,7 @@ public class ChunkReader
/**
* Returns the {@link Record} iterator.
*/
- public Iterator<Record> iterator() {
+ public Iterator<Map<String, String>> iterator() {
try {
return new IndexIterator(dataInputStream);
}
@@ -114,341 +101,155 @@ public class ChunkReader
* Low memory footprint index iterator that incrementally parses the underlying stream.
*/
private static class IndexIterator
- implements Iterator<Record>
+ implements Iterator<Map<String, String>>
{
private final DataInputStream dataInputStream;
- private Record nextRecord;
+ private Map<String, String> nextRecord;
public IndexIterator(final DataInputStream dataInputStream) throws IOException {
this.dataInputStream = dataInputStream;
- this.nextRecord = readRecord();
+ this.nextRecord = nextRecord();
}
public boolean hasNext() {
return nextRecord != null;
}
- public Record next() {
+ public Map<String, String> next() {
if (nextRecord == null) {
throw new NoSuchElementException("chunk depleted");
}
- Record result = nextRecord;
+ Map<String, String> result = nextRecord;
+ nextRecord = nextRecord();
+ return result;
+ }
+
+ private Map<String, String> nextRecord() {
try {
- nextRecord = readRecord();
- return result;
+ return readRecord(dataInputStream);
}
catch (IOException e) {
throw new RuntimeException("read error", e);
}
}
+ }
- /**
- * Reads and returns next record from the underlying stream, or {@code null} if no more records.
- */
- private Record readRecord()
- throws IOException
- {
- int fieldCount;
- try {
- fieldCount = dataInputStream.readInt();
- }
- catch (EOFException ex) {
- return null; // no more documents
- }
-
- Map<String, String> recordMap = new HashMap<String, String>();
- for (int i = 0; i < fieldCount; i++) {
- readField(recordMap);
- }
-
- if (recordMap.containsKey("DESCRIPTOR")) {
- return new Record(Type.DESCRIPTOR, recordMap, expandDescriptor(recordMap));
- }
- else if (recordMap.containsKey("allGroups")) {
- return new Record(Type.ALL_GROUPS, recordMap, expandAllGroups(recordMap));
- }
- else if (recordMap.containsKey("rootGroups")) {
- return new Record(Type.ROOT_GROUPS, recordMap, expandRootGroups(recordMap));
- }
- else if (recordMap.containsKey("del")) {
- return new Record(Type.ARTIFACT_REMOVE, recordMap, expandDeletedArtifact(recordMap));
- }
- else {
- // Fix up UINFO field wrt MINDEXER-41
- final String uinfo = recordMap.get(UINFO);
- final String info = recordMap.get(INFO);
- if (uinfo != null && !(info == null || info.trim().length() == 0)) {
- final String[] splitInfo = FS_PATTERN.split(info);
- if (splitInfo.length > 6) {
- final String extension = splitInfo[6];
- if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) {
- recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension);
- }
- }
- }
- return new Record(Type.ARTIFACT_ADD, recordMap, expandAddedArtifact(recordMap));
- }
+ /**
+ * Reads and returns next record from the underlying stream, or {@code null} if no more records.
+ */
+ private static Map<String, String> readRecord(final DataInput dataInput)
+ throws IOException
+ {
+ int fieldCount;
+ try {
+ fieldCount = dataInput.readInt();
}
-
- private void readField(final Map<String, String> record)
- throws IOException
- {
- dataInputStream.read(); // flags: neglect them
- String name = dataInputStream.readUTF();
- String value = readUTF();
- record.put(name, value);
+ catch (EOFException ex) {
+ return null; // no more documents
}
- private String readUTF()
- throws IOException
- {
- int utflen = dataInputStream.readInt();
-
- byte[] bytearr;
- char[] chararr;
-
- try {
- bytearr = new byte[utflen];
- chararr = new char[utflen];
- }
- catch (OutOfMemoryError e) {
- IOException ioex = new IOException("Index data content is corrupt");
- ioex.initCause(e);
- throw ioex;
- }
-
- int c, char2, char3;
- int count = 0;
- int chararr_count = 0;
+ Map<String, String> recordMap = new HashMap<String, String>();
+ for (int i = 0; i < fieldCount; i++) {
+ readField(recordMap, dataInput);
+ }
+ return recordMap;
+ }
- dataInputStream.readFully(bytearr, 0, utflen);
+ private static void readField(final Map<String, String> record, final DataInput dataInput)
+ throws IOException
+ {
+ dataInput.readByte(); // flags: neglect them
+ String name = dataInput.readUTF();
+ String value = readUTF(dataInput);
+ record.put(name, value);
+ }
- while (count < utflen) {
- c = bytearr[count] & 0xff;
- if (c > 127) {
- break;
- }
- count++;
- chararr[chararr_count++] = (char) c;
- }
+ private static String readUTF(final DataInput dataInput)
+ throws IOException
+ {
+ int utflen = dataInput.readInt();
- while (count < utflen) {
- c = bytearr[count] & 0xff;
- switch (c >> 4) {
- case 0:
- case 1:
- case 2:
- case 3:
- case 4:
- case 5:
- case 6:
- case 7:
- /* 0xxxxxxx */
- count++;
- chararr[chararr_count++] = (char) c;
- break;
+ byte[] bytearr;
+ char[] chararr;
- case 12:
- case 13:
- /* 110x xxxx 10xx xxxx */
- count += 2;
- if (count > utflen) {
- throw new UTFDataFormatException("malformed input: partial character at end");
- }
- char2 = bytearr[count - 1];
- if ((char2 & 0xC0) != 0x80) {
- throw new UTFDataFormatException("malformed input around byte " + count);
- }
- chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
- break;
-
- case 14:
- /* 1110 xxxx 10xx xxxx 10xx xxxx */
- count += 3;
- if (count > utflen) {
- throw new UTFDataFormatException("malformed input: partial character at end");
- }
- char2 = bytearr[count - 2];
- char3 = bytearr[count - 1];
- if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) {
- throw new UTFDataFormatException("malformed input around byte " + (count - 1));
- }
- chararr[chararr_count++] =
- (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F));
- break;
-
- default:
- /* 10xx xxxx, 1111 xxxx */
- throw new UTFDataFormatException("malformed input around byte " + count);
- }
- }
-
- // The number of chars produced may be less than utflen
- return new String(chararr, 0, chararr_count);
+ try {
+ bytearr = new byte[utflen];
+ chararr = new char[utflen];
}
-
- private Map<String, Object> expandDescriptor(final Map<String, String> raw) {
- final Map<String, Object> result = new HashMap<String, Object>();
- String[] r = FS_PATTERN.split(raw.get("IDXINFO"));
- result.put(Record.REPOSITORY_ID, r[1]);
- return result;
+ catch (OutOfMemoryError e) {
+ IOException ioex = new IOException("Index data content is corrupt");
+ ioex.initCause(e);
+ throw ioex;
}
- private Map<String, Object> expandAllGroups(final Map<String, String> raw) {
- final Map<String, Object> result = new HashMap<String, Object>();
- putIfNotNullAsList(raw, Record.ALL_GROUPS_LIST, result, "allGroups");
- return result;
- }
+ int c, char2, char3;
+ int count = 0;
+ int chararr_count = 0;
- private Map<String, Object> expandRootGroups(final Map<String, String> raw) {
- final Map<String, Object> result = new HashMap<String, Object>();
- putIfNotNullAsList(raw, Record.ROOT_GROUPS_LIST, result, "rootGroups");
- return result;
- }
+ dataInput.readFully(bytearr, 0, utflen);
- private Map<String, Object> expandDeletedArtifact(final Map<String, String> raw) {
- final Map<String, Object> result = new HashMap<String, Object>();
- putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED);
- if (raw.containsKey("del")) {
- expandUinfo(raw.get("del"), result);
+ while (count < utflen) {
+ c = bytearr[count] & 0xff;
+ if (c > 127) {
+ break;
}
- return result;
+ count++;
+ chararr[chararr_count++] = (char) c;
}
- /**
- * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming.
- */
- private Map<String, Object> expandAddedArtifact(final Map<String, String> raw) {
- final Map<String, Object> result = new HashMap<String, Object>();
-
- // Minimal
- expandUinfo(raw.get(UINFO), result);
- final String info = raw.get(INFO);
- if (info != null) {
- String[] r = FS_PATTERN.split(info);
- result.put(Record.PACKAGING, renvl(r[0]));
- result.put(Record.FILE_MODIFIED, Long.valueOf(r[1]));
- result.put(Record.FILE_SIZE, Long.valueOf(r[2]));
- result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString());
- result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString());
- result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString());
- if (r.length > 6) {
- result.put(Record.FILE_EXTENSION, r[6]);
- }
- else {
- final String packaging = raw.get(Record.PACKAGING);
- if (raw.get(Record.CLASSIFIER) != null
- || "pom".equals(packaging)
- || "war".equals(packaging)
- || "ear".equals(packaging)) {
- result.put(Record.FILE_EXTENSION, packaging);
+ while (count < utflen) {
+ c = bytearr[count] & 0xff;
+ switch (c >> 4) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ /* 0xxxxxxx */
+ count++;
+ chararr[chararr_count++] = (char) c;
+ break;
+
+ case 12:
+ case 13:
+ /* 110x xxxx 10xx xxxx */
+ count += 2;
+ if (count > utflen) {
+ throw new UTFDataFormatException("malformed input: partial character at end");
}
- else {
- result.put(Record.FILE_EXTENSION, "jar"); // best guess
+ char2 = bytearr[count - 1];
+ if ((char2 & 0xC0) != 0x80) {
+ throw new UTFDataFormatException("malformed input around byte " + count);
}
- }
- }
- putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED);
- putIfNotNull(raw, "n", result, Record.NAME);
- putIfNotNull(raw, "d", result, Record.DESCRIPTION);
- putIfNotNull(raw, "1", result, Record.SHA1);
-
- // Jar file contents (optional)
- putIfNotNullAsList(raw, "classnames", result, Record.CLASSNAMES);
-
- // Maven Plugin (optional)
- putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX);
- putIfNotNullAsList(raw, "gx", result, Record.PLUGIN_GOALS);
-
- // OSGi (optional)
- putIfNotNull(raw, "Bundle-SymbolicName", result, "Bundle-SymbolicName");
- putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version");
- putIfNotNull(raw, "Export-Package", result, "Export-Package");
- putIfNotNull(raw, "Export-Service", result, "Export-Service");
- putIfNotNull(raw, "Bundle-Description", result, "Bundle-Description");
- putIfNotNull(raw, "Bundle-Name", result, "Bundle-Name");
- putIfNotNull(raw, "Bundle-License", result, "Bundle-License");
- putIfNotNull(raw, "Bundle-DocURL", result, "Bundle-DocURL");
- putIfNotNull(raw, "Import-Package", result, "Import-Package");
- putIfNotNull(raw, "Require-Bundle", result, "Require-Bundle");
- putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version");
-
- return result;
- }
+ chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
+ break;
- /**
- * Expands UINFO synthetic field. Handles {@code null} String inputs.
- */
- private void expandUinfo(final String uinfo, final Map<String, Object> result) {
- if (uinfo != null) {
- String[] r = FS_PATTERN.split(uinfo);
- result.put(Record.GROUP_ID, r[0]);
- result.put(Record.ARTIFACT_ID, r[1]);
- result.put(Record.VERSION, r[2]);
- String classifier = renvl(r[3]);
- if (classifier != null) {
- result.put(Record.CLASSIFIER, classifier);
- if (r.length > 4) {
- result.put(Record.FILE_EXTENSION, r[4]);
+ case 14:
+ /* 1110 xxxx 10xx xxxx 10xx xxxx */
+ count += 3;
+ if (count > utflen) {
+ throw new UTFDataFormatException("malformed input: partial character at end");
}
- }
- else if (r.length > 4) {
- result.put(Record.PACKAGING, r[4]);
- }
- }
- }
- }
-
- /**
- * Helper to put a value from source map into target map, if not null.
- */
- private static void putIfNotNull(
- final Map<String, String> source,
- final String sourceName,
- final Map<String, Object> target,
- final String targetName)
- {
- String value = source.get(sourceName);
- if (value != null && value.trim().length() != 0) {
- target.put(targetName, value);
- }
- }
-
- /**
- * Helper to put a {@link Long} value from source map into target map, if not null.
- */
- private static void putIfNotNullTS(
- final Map<String, String> source,
- final String sourceName,
- final Map<String, Object> target,
- final String targetName)
- {
- String value = source.get(sourceName);
- if (value != null && value.trim().length() != 0) {
- target.put(targetName, Long.valueOf(value));
- }
- }
+ char2 = bytearr[count - 2];
+ char3 = bytearr[count - 1];
+ if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) {
+ throw new UTFDataFormatException("malformed input around byte " + (count - 1));
+ }
+ chararr[chararr_count++] =
+ (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F));
+ break;
- /**
- * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null.
- */
- private static void putIfNotNullAsList(
- final Map<String, String> source,
- final String sourceName,
- final Map<String, Object> target,
- final String targetName)
- {
- String value = source.get(sourceName);
- if (value != null && value.trim().length() != 0) {
- target.put(targetName, Arrays.asList(FS_PATTERN.split(value)));
+ default:
+ /* 10xx xxxx, 1111 xxxx */
+ throw new UTFDataFormatException("malformed input around byte " + count);
+ }
}
- }
- /**
- * Helper to translate the "NA" (not available) input into {@code null} value.
- */
- private static String renvl(final String v) {
- return NOT_AVAILABLE.equals(v) ? null : v;
+ // The number of chars produced may be less than utflen
+ return new String(chararr, 0, chararr_count);
}
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
new file mode 100644
index 0000000..24e4701
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java
@@ -0,0 +1,177 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Maven 2 Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format.
+ *
+ * @since 5.1.2
+ */
+public class ChunkWriter
+ implements Closeable
+{
+ private static final int F_INDEXED = 1;
+
+ private static final int F_TOKENIZED = 2;
+
+ private static final int F_STORED = 4;
+
+ private final String chunkName;
+
+ private final DataOutputStream dataOutputStream;
+
+ private final int version;
+
+ private final Date timestamp;
+
+ public ChunkWriter(final String chunkName, final OutputStream outputStream, final int version, final Date timestamp)
+ throws IOException
+ {
+ this.chunkName = chunkName.trim();
+ this.dataOutputStream = new DataOutputStream(new GZIPOutputStream(outputStream, 2 * 1024));
+ this.version = version;
+ this.timestamp = timestamp;
+
+ dataOutputStream.writeByte(version);
+ dataOutputStream.writeLong(timestamp == null ? -1 : timestamp.getTime());
+ }
+
+ /**
+ * Returns the chunk name.
+ */
+ public String getName() {
+ return chunkName;
+ }
+
+ /**
+ * Returns index version. All releases so far always returned {@code 1}.
+ */
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * Returns the index timestamp of last update of the index.
+ */
+ public Date getTimestamp() {
+ return timestamp;
+ }
+
+ /**
+ * Writes out the record iterator and returns the written record count.
+ */
+ public int writeChunk(final Iterator<Map<String, String>> iterator) throws IOException {
+ int written = 0;
+ while (iterator.hasNext()) {
+ writeRecord(iterator.next(), dataOutputStream);
+ written++;
+ }
+ return written;
+ }
+
+ /**
+ * Closes this reader and it's underlying input.
+ */
+ public void close() throws IOException {
+ dataOutputStream.close();
+ }
+
+ private static void writeRecord(final Map<String, String> record, final DataOutput dataOutput)
+ throws IOException
+ {
+ dataOutput.writeInt(record.size());
+ for (Map.Entry<String, String> entry : record.entrySet()) {
+ writeField(entry.getKey(), entry.getValue(), dataOutput);
+ }
+ }
+
+ private static void writeField(final String fieldName, final String fieldValue, final DataOutput dataOutput)
+ throws IOException
+ {
+ boolean isIndexed = !(fieldName.equals("i") || fieldName.equals("m"));
+ boolean isTokenized = !(fieldName.equals("i")
+ || fieldName.equals("m")
+ || fieldName.equals("1")
+ || fieldName.equals("px"));
+ int flags = (isIndexed ? F_INDEXED : 0) + (isTokenized ? F_TOKENIZED : 0) + F_STORED;
+ dataOutput.writeByte(flags);
+ dataOutput.writeUTF(fieldName);
+ writeUTF(fieldValue, dataOutput);
+ }
+
+ private static void writeUTF(final String str, final DataOutput dataOutput)
+ throws IOException
+ {
+ int strlen = str.length();
+ int utflen = 0;
+ int c;
+ // use charAt instead of copying String to char array
+ for (int i = 0; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ utflen++;
+ }
+ else if (c > 0x07FF) {
+ utflen += 3;
+ }
+ else {
+ utflen += 2;
+ }
+ }
+ dataOutput.writeInt(utflen);
+ byte[] bytearr = new byte[utflen];
+ int count = 0;
+ int i = 0;
+ for (; i < strlen; i++) {
+ c = str.charAt(i);
+ if (!((c >= 0x0001) && (c <= 0x007F))) {
+ break;
+ }
+ bytearr[count++] = (byte) c;
+ }
+ for (; i < strlen; i++) {
+ c = str.charAt(i);
+ if ((c >= 0x0001) && (c <= 0x007F)) {
+ bytearr[count++] = (byte) c;
+
+ }
+ else if (c > 0x07FF) {
+ bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
+ }
+ else {
+ bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
+ bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F));
+ }
+ }
+ dataOutput.write(bytearr, 0, utflen);
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
index 45514b9..d74c366 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java
@@ -19,39 +19,28 @@ package org.apache.maven.index.reader;
* under the License.
*/
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
-import java.io.InputStream;
-import java.text.DateFormat;
+import java.io.OutputStream;
import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
-import java.util.TimeZone;
+
+import static org.apache.maven.index.reader.Utils.loadProperties;
/**
- * Maven 2 Index reader that handles incremental updates if possible.
+ * Maven 2 Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to
+ * read all the required records.
*
* @since 5.1.2
*/
public class IndexReader
implements Iterable<ChunkReader>, Closeable
{
- private static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index";
-
- private static final DateFormat INDEX_DATE_FORMAT;
-
- static {
- INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z");
- INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT"));
- }
-
private final WritableResourceHandler local;
private final ResourceHandler remote;
@@ -74,10 +63,10 @@ public class IndexReader
}
this.local = local;
this.remote = remote;
- remoteIndexProperties = loadProperties(remote.open(INDEX_FILE_PREFIX + ".properties"));
+ remoteIndexProperties = loadProperties(remote.open(Utils.INDEX_FILE_PREFIX + ".properties"));
try {
if (local != null) {
- localIndexProperties = loadProperties(local.open(INDEX_FILE_PREFIX + ".properties"));
+ localIndexProperties = loadProperties(local.open(Utils.INDEX_FILE_PREFIX + ".properties"));
String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id");
String localIndexId = localIndexProperties.getProperty("nexus.index.id");
if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) {
@@ -86,14 +75,16 @@ public class IndexReader
remoteIndexId);
}
this.indexId = localIndexId;
- this.publishedTimestamp = INDEX_DATE_FORMAT.parse(localIndexProperties.getProperty("nexus.index.timestamp"));
+ this.publishedTimestamp = Utils.INDEX_DATE_FORMAT
+ .parse(localIndexProperties.getProperty("nexus.index.timestamp"));
this.incremental = canRetrieveAllChunks();
this.chunkNames = calculateChunkNames();
}
else {
localIndexProperties = null;
this.indexId = remoteIndexProperties.getProperty("nexus.index.id");
- this.publishedTimestamp = INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp"));
+ this.publishedTimestamp = Utils.INDEX_DATE_FORMAT
+ .parse(remoteIndexProperties.getProperty("nexus.index.timestamp"));
this.incremental = false;
this.chunkNames = calculateChunkNames();
}
@@ -170,9 +161,13 @@ public class IndexReader
* for future incremental updates.
*/
private void syncLocalWithRemote() throws IOException {
- final ByteArrayOutputStream bos = new ByteArrayOutputStream();
- remoteIndexProperties.store(bos, "Maven Indexer Reader");
- local.save(INDEX_FILE_PREFIX + ".properties", new ByteArrayInputStream(bos.toByteArray()));
+ final OutputStream outputStream = local.openWrite(Utils.INDEX_FILE_PREFIX + ".properties");
+ try {
+ remoteIndexProperties.store(outputStream, "Maven Indexer Reader");
+ }
+ finally {
+ outputStream.close();
+ }
}
/**
@@ -185,12 +180,12 @@ public class IndexReader
int currentCounter = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental"));
currentCounter++;
while (currentCounter <= maxCounter) {
- chunkNames.add(INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz");
+ chunkNames.add(Utils.INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz");
}
return Collections.unmodifiableList(chunkNames);
}
else {
- return Collections.singletonList(INDEX_FILE_PREFIX + ".gz");
+ return Collections.singletonList(Utils.INDEX_FILE_PREFIX + ".gz");
}
}
@@ -264,18 +259,4 @@ public class IndexReader
}
}
}
-
- /**
- * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream.
- */
- private static Properties loadProperties(final InputStream inputStream) throws IOException {
- try {
- final Properties properties = new Properties();
- properties.load(inputStream);
- return properties;
- }
- finally {
- inputStream.close();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
new file mode 100644
index 0000000..f52cb29
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java
@@ -0,0 +1,196 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.text.ParseException;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import static org.apache.maven.index.reader.Utils.loadProperties;
+
+/**
+ * Maven 2 Index writer that writes chunk and maintains published property file.
+ * <p/>
+ * <strong>Currently no incremental update is supported, as the deleteion states should be maintained by
+ * caller</strong>. Hence, this writer will always produce the "main" chunk only.
+ *
+ * @since 5.1.2
+ */
+public class IndexWriter
+ implements Closeable
+{
+ private static final int INDEX_V1 = 1;
+
+ private final WritableResourceHandler local;
+
+ private final Properties localIndexProperties;
+
+ private final boolean incremental;
+
+ private final String nextChunkCounter;
+
+ private final String nextChunkName;
+
+ public IndexWriter(final WritableResourceHandler local, final String indexId, final boolean incrementalSupported)
+ throws IOException
+ {
+ if (local == null) {
+ throw new NullPointerException("local resource handler null");
+ }
+ if (indexId == null) {
+ throw new NullPointerException("indexId null");
+ }
+ this.local = local;
+ InputStream localIndexPropertiesInputStream = local.open(Utils.INDEX_FILE_PREFIX + ".properties");
+ if (incrementalSupported && localIndexPropertiesInputStream != null) {
+ // existing index, this is incremental publish, and we will add new chunk
+ this.localIndexProperties = loadProperties(local.open(Utils.INDEX_FILE_PREFIX + ".properties"));
+ String localIndexId = localIndexProperties.getProperty("nexus.index.id");
+ if (localIndexId == null || !localIndexId.equals(indexId)) {
+ throw new IllegalArgumentException(
+ "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " +
+ indexId);
+ }
+ this.incremental = true;
+ this.nextChunkCounter = calculateNextChunkCounter();
+ this.nextChunkName = Utils.INDEX_FILE_PREFIX + "." + nextChunkCounter + ".gz";
+ }
+ else {
+ // non-existing index, create published index from scratch
+ this.localIndexProperties = new Properties();
+ this.localIndexProperties.setProperty("nexus.index.id", indexId);
+ this.localIndexProperties.setProperty("nexus.index.chain-id", UUID.randomUUID().toString());
+ this.incremental = false;
+ this.nextChunkCounter = null;
+ this.nextChunkName = Utils.INDEX_FILE_PREFIX + ".gz";
+ }
+ }
+
+ /**
+ * Returns the index context ID that published index has set.
+ */
+ public String getIndexId() {
+ return localIndexProperties.getProperty("nexus.index.id");
+ }
+
+ /**
+ * Returns the {@link Date} when index was last published or {@code null} if this is first publishing. In other
+ * words,returns {@code null} when {@link #isIncremental()} returns {@code false}. After this writer is closed, the
+ * return value is updated to "now" (in {@link #close() method}.
+ */
+ public Date getPublishedTimestamp() {
+ try {
+ String timestamp = localIndexProperties.getProperty("nexus.index.timestamp");
+ if (timestamp != null) {
+ return Utils.INDEX_DATE_FORMAT.parse(timestamp);
+ }
+ return null;
+ }
+ catch (ParseException e) {
+ throw new RuntimeException("Corrupt date", e);
+ }
+ }
+
+ /**
+ * Returns {@code true} if incremental publish is about to happen.
+ */
+ public boolean isIncremental() {
+ return incremental;
+ }
+
+ /**
+ * Returns the chain id of published index. If {@link #isIncremental()} is {@code false}, this is the newly generated
+ * chain ID.
+ */
+ public String getChainId() {
+ return localIndexProperties.getProperty("nexus.index.chain-id");
+ }
+
+ /**
+ * Returns the next chunk name about to be published.
+ */
+ public String getNextChunkName() {
+ return nextChunkName;
+ }
+
+ /**
+ * Writes out the record iterator and returns the written record count.
+ */
+ public int writeChunk(final Iterator<Map<String, String>> iterator) throws IOException {
+ int written;
+ final ChunkWriter chunkWriter = new ChunkWriter(nextChunkName, local.openWrite(nextChunkName), INDEX_V1, new Date());
+ try {
+ written = chunkWriter.writeChunk(iterator);
+ }
+ finally {
+ chunkWriter.close();
+ }
+ if (incremental) {
+ // TODO: update main gz file
+ }
+ return written;
+ }
+
+ /**
+ * Closes the underlying {@link ResourceHandler} and synchronizes published index properties, so remote clients
+ * becomes able to consume newly published index. If sync is not desired (ie. due to aborted publish), then this
+ * method should NOT be invoked, but rather the {@link ResourceHandler} that caller provided in constructor of
+ * this class should be closed manually.
+ */
+ public void close() throws IOException {
+ try {
+ if (incremental) {
+ localIndexProperties.setProperty("nexus.index.last-incremental", nextChunkCounter);
+ }
+ localIndexProperties.setProperty("nexus.index.timestamp", Utils.INDEX_DATE_FORMAT.format(new Date()));
+
+ final OutputStream outputStream = local.openWrite(Utils.INDEX_FILE_PREFIX + ".properties");
+ try {
+ localIndexProperties.store(outputStream, "Maven Indexer Writer");
+ }
+ finally {
+ outputStream.close();
+ }
+ }
+ finally {
+ local.close();
+ }
+ }
+
+ /**
+ * Calculates the chunk names that needs to be fetched.
+ */
+ private String calculateNextChunkCounter() {
+ String lastChunkCounter = localIndexProperties.getProperty("nexus.index.last-incremental");
+ if (lastChunkCounter != null) {
+ return String.valueOf(Integer.parseInt(lastChunkCounter) + 1);
+ }
+ else {
+ return "1";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java
new file mode 100644
index 0000000..78941dc
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Iterables.java
@@ -0,0 +1,204 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.TreeSet;
+
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+
+/**
+ * Helpers to transform {@link Iterable}.
+ *
+ * @since 5.1.2
+ */
+public final class Iterables
+{
+ private Iterables() {
+ // nothing
+ }
+
+ /**
+ * Transforming function.
+ */
+ public interface Function<I, O>
+ {
+ O apply(I rec);
+ }
+
+ /**
+ * Applies {@link Function} to an {@link Iterable} on the fly.
+ */
+ public static <I, O> Iterable<O> transform(final Iterable<I> iterable, final Function<I, O> function) {
+ return new Iterable<O>()
+ {
+ public Iterator<O> iterator() {
+ return new TransformIterator<I, O>(iterable.iterator(), function);
+ }
+ };
+ }
+
+ /**
+ * Helper method, that "decorates" the stream of records to be written out with "special" Maven Indexer records, so
+ * all the caller is needed to provide {@Link Iterable} or {@link Record}s <strong>to be</strong> on the index, with
+ * record type {@link Record.Type#ARTIFACT_ADD}. This method will create the output as "proper" Maven Indexer record
+ * streeam, by adding the {@link Type#DESCRIPTOR}, {@link Type#ROOT_GROUPS} and {@link Type#ALL_GROUPS} special
+ * records.
+ */
+ public static Iterable<Map<String, String>> decorateAndTransform(final Iterable<Record> iterable,
+ final String repoId)
+ {
+ final RecordCompactor recordCompactor = new RecordCompactor();
+ final TreeSet<String> allGroups = new TreeSet<String>();
+ final TreeSet<String> rootGroups = new TreeSet<String>();
+ final ArrayList<Iterator<Record>> iterators = new ArrayList<Iterator<Record>>();
+ iterators.add(Collections.singletonList(descriptor(repoId)).iterator());
+ iterators.add(iterable.iterator());
+ iterators.add(Collections.singletonList(allGroups(allGroups)).iterator());
+ iterators.add(Collections.singletonList(rootGroups(rootGroups)).iterator());
+ return transform(
+ new Iterable<Record>()
+ {
+ public Iterator<Record> iterator() {
+ return new ConcatIterator<Record>(iterators.iterator());
+ }
+ },
+ new Function<Record, Map<String, String>>()
+ {
+ public Map<String, String> apply(final Record rec) {
+ if (Type.DESCRIPTOR == rec.getType()) {
+ return recordCompactor.apply(descriptor(repoId));
+ }
+ else if (Type.ALL_GROUPS == rec.getType()) {
+ return recordCompactor.apply(allGroups(allGroups));
+ }
+ else if (Type.ROOT_GROUPS == rec.getType()) {
+ return recordCompactor.apply(rootGroups(rootGroups));
+ }
+ else {
+ final String groupId = rec.get(Record.GROUP_ID);
+ if (groupId != null) {
+ allGroups.add(groupId);
+ rootGroups.add(Utils.rootGroup(groupId));
+ }
+ return recordCompactor.apply(rec);
+ }
+ }
+ }
+ );
+ }
+
+ private static Record descriptor(final String repoId) {
+ HashMap<EntryKey, Object> entries = new HashMap<EntryKey, Object>();
+ entries.put(Record.REPOSITORY_ID, repoId);
+ return new Record(Type.DESCRIPTOR, entries);
+ }
+
+ private static Record allGroups(final Collection<String> allGroups) {
+ HashMap<EntryKey, Object> entries = new HashMap<EntryKey, Object>();
+ entries.put(Record.ALL_GROUPS, allGroups.toArray(new String[allGroups.size()]));
+ return new Record(Type.ALL_GROUPS, entries);
+ }
+
+ private static Record rootGroups(final Collection<String> rootGroups) {
+ HashMap<EntryKey, Object> entries = new HashMap<EntryKey, Object>();
+ entries.put(Record.ROOT_GROUPS, rootGroups.toArray(new String[rootGroups.size()]));
+ return new Record(Type.ROOT_GROUPS, entries);
+ }
+
+ // ==
+
+ private static final class TransformIterator<I, O>
+ implements Iterator<O>
+ {
+ private final Iterator<I> iterator;
+
+ private final Function<I, O> function;
+
+ private TransformIterator(final Iterator<I> iterator, final Function<I, O> function) {
+ this.iterator = iterator;
+ this.function = function;
+ }
+
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ public O next() {
+ return function.apply(iterator.next());
+ }
+ }
+
+ private static final class ConcatIterator<T>
+ implements Iterator<T>
+ {
+ private final Iterator<Iterator<T>> iterators;
+
+ private Iterator<T> current;
+
+ private T nextElement;
+
+ private ConcatIterator(final Iterator<Iterator<T>> iterators) {
+ this.iterators = iterators;
+ this.nextElement = getNextElement();
+ }
+
+ public boolean hasNext() {
+ return nextElement != null;
+ }
+
+ public T next() {
+ if (nextElement == null) {
+ throw new NoSuchElementException();
+ }
+ T result = nextElement;
+ nextElement = getNextElement();
+ return result;
+ }
+
+ protected T getNextElement() {
+ if ((current == null || !current.hasNext()) && iterators.hasNext()) {
+ current = iterators.next();
+ }
+ while (current != null && !current.hasNext()) {
+ if (!iterators.hasNext()) {
+ current = null;
+ break;
+ }
+ current = iterators.next();
+ }
+ if (current != null) {
+ return current.next();
+ }
+ else {
+ return null;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
index 3354008..1746845 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java
@@ -26,113 +26,225 @@ import java.util.Map;
*
* @since 5.1.2
*/
-public class Record
+public final class Record
{
+ public static final class EntryKey<T>
+ {
+ private final String name;
+
+ private final Class<T> proto;
+
+ public EntryKey(final String name, final Class<T> proto) {
+ if (name == null) {
+ throw new NullPointerException("name is null");
+ }
+ if (proto == null) {
+ throw new NullPointerException("proto is null");
+ }
+ this.name = name;
+ this.proto = proto;
+ }
+
+ public T coerce(final Object object) {
+ return (T) proto.cast(object);
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof EntryKey)) {
+ return false;
+ }
+ EntryKey entryKey = (EntryKey) o;
+ return name.equals(entryKey.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "Key{" +
+ "name='" + name + '\'' +
+ ", type=" + proto.getSimpleName() +
+ '}';
+ }
+ }
+
/**
* Key of repository ID entry, that contains {@link String}.
*/
- public static final String REPOSITORY_ID = "repositoryId";
+ public static final EntryKey<String> REPOSITORY_ID = new EntryKey<String>("repositoryId", String.class);
/**
* Key of all groups list entry, that contains {@link java.util.List<String>}.
*/
- public static final String ALL_GROUPS_LIST = "allGroupsList";
+ public static final EntryKey<String[]> ALL_GROUPS = new EntryKey<String[]>("allGroups", String[].class);
/**
* Key of root groups list entry, that contains {@link java.util.List<String>}.
*/
- public static final String ROOT_GROUPS_LIST = "rootGroupsList";
+ public static final EntryKey<String[]> ROOT_GROUPS = new EntryKey<String[]>("rootGroups", String[].class);
/**
* Key of index record modification (added to index or removed from index) timestamp entry, that contains {@link
* Long}.
*/
- public static final String REC_MODIFIED = "recordModified";
+ public static final EntryKey<Long> REC_MODIFIED = new EntryKey<Long>("recordModified", Long.class);
/**
* Key of artifact groupId entry, that contains {@link String}.
*/
- public static final String GROUP_ID = "groupId";
+ public static final EntryKey<String> GROUP_ID = new EntryKey<String>("groupId", String.class);
/**
* Key of artifact artifactId entry, that contains {@link String}.
*/
- public static final String ARTIFACT_ID = "artifactId";
+ public static final EntryKey<String> ARTIFACT_ID = new EntryKey<String>("artifactId", String.class);
/**
* Key of artifact version entry, that contains {@link String}.
*/
- public static final String VERSION = "version";
+ public static final EntryKey<String> VERSION = new EntryKey<String>("version", String.class);
/**
* Key of artifact classifier entry, that contains {@link String}.
*/
- public static final String CLASSIFIER = "classifier";
+ public static final EntryKey<String> CLASSIFIER = new EntryKey<String>("classifier", String.class);
/**
* Key of artifact packaging entry, that contains {@link String}.
*/
- public static final String PACKAGING = "packaging";
+ public static final EntryKey<String> PACKAGING = new EntryKey<String>("packaging", String.class);
/**
* Key of artifact file extension, that contains {@link String}.
*/
- public static final String FILE_EXTENSION = "fileExtension";
+ public static final EntryKey<String> FILE_EXTENSION = new EntryKey<String>("fileExtension", String.class);
/**
* Key of artifact file last modified timestamp, that contains {@link Long}.
*/
- public static final String FILE_MODIFIED = "fileModified";
+ public static final EntryKey<Long> FILE_MODIFIED = new EntryKey<Long>("fileModified", Long.class);
/**
* Key of artifact file size in bytes, that contains {@link Long}.
*/
- public static final String FILE_SIZE = "fileSize";
+ public static final EntryKey<Long> FILE_SIZE = new EntryKey<Long>("fileSize", Long.class);
/**
* Key of artifact Sources presence flag, that contains {@link Boolean}.
*/
- public static final String HAS_SOURCES = "hasSources";
+ public static final EntryKey<Boolean> HAS_SOURCES = new EntryKey<Boolean>("hasSources", Boolean.class);
/**
* Key of artifact Javadoc presence flag, that contains {@link Boolean}.
*/
- public static final String HAS_JAVADOC = "hasJavadoc";
+ public static final EntryKey<Boolean> HAS_JAVADOC = new EntryKey<Boolean>("hasJavadoc", Boolean.class);
/**
* Key of artifact signature presence flag, that contains {@link Boolean}.
*/
- public static final String HAS_SIGNATURE = "hasSignature";
+ public static final EntryKey<Boolean> HAS_SIGNATURE = new EntryKey<Boolean>("hasSignature", Boolean.class);
/**
* Key of artifact name (as set in POM), that contains {@link String}.
*/
- public static final String NAME = "name";
+ public static final EntryKey<String> NAME = new EntryKey<String>("name", String.class);
/**
* Key of artifact description (as set in POM), that contains {@link String}.
*/
- public static final String DESCRIPTION = "description";
+ public static final EntryKey<String> DESCRIPTION = new EntryKey<String>("description", String.class);
/**
* Key of artifact SHA1 digest, that contains {@link String}.
*/
- public static final String SHA1 = "sha1";
+ public static final EntryKey<String> SHA1 = new EntryKey<String>("sha1", String.class);
+
+ /**
+ * Key of artifact contained class names, that contains {@link java.util.List<String>}. Extracted by {@code
+ * JarFileContentsIndexCreator}.
+ */
+ public static final EntryKey<String[]> CLASSNAMES = new EntryKey<String[]>("classNames", String[].class);
+
+ /**
+ * Key of plugin artifact prefix, that contains {@link String}. Extracted by {@code
+ * MavenPluginArtifactInfoIndexCreator}.
+ */
+ public static final EntryKey<String> PLUGIN_PREFIX = new EntryKey<String>("pluginPrefix", String.class);
+
+ /**
+ * Key of plugin artifact goals, that contains {@link java.util.List<String>}. Extracted by {@code
+ * MavenPluginArtifactInfoIndexCreator}.
+ */
+ public static final EntryKey<String[]> PLUGIN_GOALS = new EntryKey<String[]>("pluginGoals", String[].class);
+
+ /**
+ * Key of OSGi "Bundle-SymbolicName" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_BUNDLE_SYMBOLIC_NAME = new EntryKey<String>("Bundle-SymbolicName",
+ String.class);
+
+ /**
+ * Key of OSGi "Bundle-Version" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_BUNDLE_VERSION = new EntryKey<String>("Bundle-Version", String.class);
+
+ /**
+ * Key of OSGi "Export-Package" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_EXPORT_PACKAGE = new EntryKey<String>("Export-Package", String.class);
+
+ /**
+ * Key of OSGi "Export-Service" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_EXPORT_SERVICE = new EntryKey<String>("Export-Service", String.class);
/**
- * Key of artifact contained class names, that contains {@link java.util.List<String>}.
+ * Key of OSGi "Bundle-Description" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
*/
- public static final String CLASSNAMES = "classNames";
+ public static final EntryKey<String> OSGI_BUNDLE_DESCRIPTION = new EntryKey<String>("Bundle-Description",
+ String.class);
/**
- * Key of plugin artifact prefix, that contains {@link String}.
+ * Key of OSGi "Bundle-Name" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
*/
- public static final String PLUGIN_PREFIX = "pluginPrefix";
+ public static final EntryKey<String> OSGI_BUNDLE_NAME = new EntryKey<String>("Bundle-Name", String.class);
/**
- * Key of plugin artifact goals, that contains {@link java.util.List<String>}.
+ * Key of OSGi "Bundle-License" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
*/
- public static final String PLUGIN_GOALS = "pluginGoals";
+ public static final EntryKey<String> OSGI_BUNDLE_LICENSE = new EntryKey<String>("Bundle-License", String.class);
+
+ /**
+ * Key of OSGi "Bundle-DocURL" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_EXPORT_DOCURL = new EntryKey<String>("Bundle-DocURL", String.class);
+
+ /**
+ * Key of OSGi "Import-Package" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_IMPORT_PACKAGE = new EntryKey<String>("Import-Package", String.class);
+
+ /**
+ * Key of OSGi "Require-Bundle" manifest entry, that contains {@link String}. Extracted by {@code
+ * OsgiArtifactIndexCreator}.
+ */
+ public static final EntryKey<String> OSGI_REQUIRE_BUNDLE = new EntryKey<String>("Require-Bundle", String.class);
/**
* Types of returned records returned from index.
@@ -175,7 +287,7 @@ public class Record
ARTIFACT_ADD,
/**
- * Artifact REMOTE record. In case of incremental updates, notes that this artifact was removed. Records of this
+ * Artifact REMOVE record. In case of incremental updates, signals that this artifact was removed. Records of this
* type should be removed from your indexing system.
* Contains following entries:
* <ul>
@@ -194,7 +306,7 @@ public class Record
* Special record, containing all the Maven "groupId"s that are enlisted on the index. Can be safely ignored.
* Contains following entries:
* <ul>
- * <li>{@link #ALL_GROUPS_LIST}</li>
+ * <li>{@link #ALL_GROUPS}</li>
* </ul>
*/
ALL_GROUPS,
@@ -204,7 +316,7 @@ public class Record
* ignored.
* Contains following entries:
* <ul>
- * <li>{@link #ROOT_GROUPS_LIST}</li>
+ * <li>{@link #ROOT_GROUPS}</li>
* </ul>
*/
ROOT_GROUPS
@@ -212,13 +324,10 @@ public class Record
private final Type type;
- private final Map<String, String> raw;
+ private final Map<EntryKey, Object> expanded;
- private final Map<String, Object> expanded;
-
- public Record(final Type type, final Map<String, String> raw, final Map<String, Object> expanded) {
+ public Record(final Type type, final Map<EntryKey, Object> expanded) {
this.type = type;
- this.raw = raw;
this.expanded = expanded;
}
@@ -232,16 +341,44 @@ public class Record
}
/**
- * Returns the "raw", Maven Indexer specific record as a {@link Map}.
+ * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption.
*/
- public Map<String, String> getRaw() {
- return raw;
+ public Map<EntryKey, Object> getExpanded() {
+ return expanded;
}
/**
- * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption.
+ * Returns {@code true} if this record contains given {@link EntryKey}.
*/
- public Map<String, Object> getExpanded() {
- return expanded;
+ boolean containsKey(final EntryKey<?> entryKey) { return expanded.containsKey(entryKey); }
+
+ /**
+ * Type safe handy method to get value from expanded map.
+ */
+ public <T> T get(final EntryKey<T> entryKey) {
+ return entryKey.coerce(expanded.get(entryKey));
+ }
+
+ /**
+ * Type safe handy method to put value to expanded map. Accepts {@code null} values, that removes the mapping.
+ */
+ public <T> T put(final EntryKey<T> entryKey, final T value) {
+ if (value == null) {
+ return entryKey.coerce(expanded.remove(entryKey));
+ }
+ else {
+ if (!entryKey.proto.isAssignableFrom(value.getClass())) {
+ throw new IllegalArgumentException("Key " + entryKey + " does not accepts value " + value);
+ }
+ return entryKey.coerce(expanded.put(entryKey, value));
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Record{" +
+ "type=" + type +
+ ", expanded=" + expanded +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
new file mode 100644
index 0000000..3fcfd25
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java
@@ -0,0 +1,205 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Iterables.Function;
+import org.apache.maven.index.reader.Record.Type;
+
+import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR;
+import static org.apache.maven.index.reader.Utils.INFO;
+import static org.apache.maven.index.reader.Utils.UINFO;
+import static org.apache.maven.index.reader.Utils.nvl;
+
+/**
+ * Maven 2 Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records.
+ *
+ * @since 5.1.2
+ */
+public class RecordCompactor
+ implements Function<Record, Map<String, String>>
+{
+ public Map<String, String> apply(final Record record) {
+ if (Type.DESCRIPTOR == record.getType()) {
+ return compactDescriptor(record);
+ }
+ else if (Type.ALL_GROUPS == record.getType()) {
+ return compactAllGroups(record);
+ }
+ else if (Type.ROOT_GROUPS == record.getType()) {
+ return compactRootGroups(record);
+ }
+ else if (Type.ARTIFACT_REMOVE == record.getType()) {
+ return compactDeletedArtifact(record);
+ }
+ else if (Type.ARTIFACT_ADD == record.getType()) {
+ return compactAddedArtifact(record);
+ }
+ else {
+ throw new IllegalArgumentException("Unknown record: " + record);
+ }
+ }
+
+ private static Map<String, String> compactDescriptor(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ result.put("DESCRIPTOR", "NexusIndex");
+ result.put("IDXINFO", "1.0|" + record.get(Record.REPOSITORY_ID));
+ return result;
+ }
+
+ private static Map<String, String> compactAllGroups(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ result.put("allGroups", "allGroups");
+ putIfNotNullAsStringArray(record.get(Record.ALL_GROUPS), result, "allGroupsList");
+ return result;
+ }
+
+ private static Map<String, String> compactRootGroups(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ result.put("rootGroups", "allGroups");
+ putIfNotNullAsStringArray(record.get(Record.ROOT_GROUPS), result, "rootGroupsList");
+ return result;
+ }
+
+ private static Map<String, String> compactDeletedArtifact(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+ putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m");
+ result.put("del", compactUinfo(record));
+ return result;
+ }
+
+ /**
+ * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming.
+ */
+ private static Map<String, String> compactAddedArtifact(final Record record) {
+ final Map<String, String> result = new HashMap<String, String>();
+
+ // Minimal
+ result.put(UINFO, compactUinfo(record));
+
+ StringBuilder info = new StringBuilder();
+ info.append(nvl(record.get(Record.PACKAGING)));
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.FILE_MODIFIED));
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.FILE_SIZE));
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.HAS_SOURCES) ? "1" : "0");
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.HAS_JAVADOC) ? "1" : "0");
+ info.append(FIELD_SEPARATOR);
+ info.append(record.get(Record.HAS_SIGNATURE) ? "1" : "0");
+ info.append(FIELD_SEPARATOR);
+ info.append(nvl(record.get(Record.FILE_EXTENSION)));
+ result.put(INFO, info.toString());
+
+ putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m");
+ putIfNotNull(record.get(Record.NAME), result, "n");
+ putIfNotNull(record.get(Record.DESCRIPTION), result, "d");
+ putIfNotNull(record.get(Record.SHA1), result, "1");
+
+ // Jar file contents (optional)
+ putIfNotNullAsStringArray(record.get(Record.CLASSNAMES), result, "classnames");
+
+ // Maven Plugin (optional)
+ putIfNotNull(record.get(Record.PLUGIN_PREFIX), result, "px");
+ putIfNotNullAsStringArray(record.get(Record.PLUGIN_GOALS), result, "gx");
+
+ // OSGi (optional)
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_SYMBOLIC_NAME), result, "Bundle-SymbolicName");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_VERSION), result, "Bundle-Version");
+ putIfNotNull(record.get(Record.OSGI_EXPORT_PACKAGE), result, "Export-Package");
+ putIfNotNull(record.get(Record.OSGI_EXPORT_SERVICE), result, "Export-Service");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_DESCRIPTION), result, "Bundle-Description");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_NAME), result, "Bundle-Name");
+ putIfNotNull(record.get(Record.OSGI_BUNDLE_LICENSE), result, "Bundle-License");
+ putIfNotNull(record.get(Record.OSGI_EXPORT_DOCURL), result, "Bundle-DocURL");
+ putIfNotNull(record.get(Record.OSGI_IMPORT_PACKAGE), result, "Import-Package");
+ putIfNotNull(record.get(Record.OSGI_REQUIRE_BUNDLE), result, "Require-Bundle");
+
+ return result;
+ }
+
+ /**
+ * Creates UINFO synthetic field.
+ */
+ private static String compactUinfo(final Record record) {
+ final String classifier = record.get(Record.CLASSIFIER);
+ StringBuilder sb = new StringBuilder();
+ sb.append(record.get(Record.GROUP_ID))
+ .append(FIELD_SEPARATOR)
+ .append(record.get(Record.ARTIFACT_ID))
+ .append(FIELD_SEPARATOR)
+ .append(record.get(Record.VERSION))
+ .append(FIELD_SEPARATOR)
+ .append(nvl(classifier));
+ if (classifier != null) {
+ sb.append(record.get(Record.FILE_EXTENSION));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Helper to put a value from source map into target map, if not null.
+ */
+ private static void putIfNotNull(
+ final String source,
+ final Map<String, String> target,
+ final String targetName)
+ {
+ if (source != null) {
+ target.put(targetName, source);
+ }
+ }
+
+ /**
+ * Helper to put a {@link Long} value from source map into target map, if not null.
+ */
+ private static void putIfNotNullTS(
+ final Long source,
+ final Map<String, String> target,
+ final String targetName)
+ {
+ if (source != null) {
+ target.put(targetName, String.valueOf(source));
+ }
+ }
+
+ /**
+ * Helper to put a array value from source map into target map joined with {@link Utils#FIELD_SEPARATOR}, if not
+ * null.
+ */
+ private static void putIfNotNullAsStringArray(
+ final String[] source,
+ final Map<String, String> target,
+ final String targetName)
+ {
+ if (source != null && source.length > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(source[0]);
+ for (int i = 1; i < source.length; i++) {
+ sb.append(FIELD_SEPARATOR).append(source[i]);
+ }
+ target.put(targetName, sb.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
new file mode 100644
index 0000000..83f6ca7
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java
@@ -0,0 +1,228 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.maven.index.reader.Iterables.Function;
+import org.apache.maven.index.reader.Record.EntryKey;
+import org.apache.maven.index.reader.Record.Type;
+
+import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR;
+import static org.apache.maven.index.reader.Utils.FS_PATTERN;
+import static org.apache.maven.index.reader.Utils.INFO;
+import static org.apache.maven.index.reader.Utils.NOT_AVAILABLE;
+import static org.apache.maven.index.reader.Utils.UINFO;
+import static org.apache.maven.index.reader.Utils.renvl;
+
+/**
+ * Maven 2 Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s.
+ *
+ * @since 5.1.2
+ */
+public class RecordExpander
+ implements Function<Map<String, String>, Record>
+{
+ public Record apply(final Map<String, String> recordMap) {
+ if (recordMap.containsKey("DESCRIPTOR")) {
+ return expandDescriptor(recordMap);
+ }
+ else if (recordMap.containsKey("allGroups")) {
+ return expandAllGroups(recordMap);
+ }
+ else if (recordMap.containsKey("rootGroups")) {
+ return expandRootGroups(recordMap);
+ }
+ else if (recordMap.containsKey("del")) {
+ return expandDeletedArtifact(recordMap);
+ }
+ else {
+ // Fix up UINFO field wrt MINDEXER-41
+ final String uinfo = recordMap.get(UINFO);
+ final String info = recordMap.get(INFO);
+ if (uinfo != null && !(info == null || info.trim().length() == 0)) {
+ final String[] splitInfo = FS_PATTERN.split(info);
+ if (splitInfo.length > 6) {
+ final String extension = splitInfo[6];
+ if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) {
+ recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension);
+ }
+ }
+ }
+ return expandAddedArtifact(recordMap);
+ }
+ }
+
+ private static Record expandDescriptor(final Map<String, String> raw) {
+ final Record result = new Record(Type.DESCRIPTOR, new HashMap<EntryKey, Object>());
+ String[] r = FS_PATTERN.split(raw.get("IDXINFO"));
+ result.put(Record.REPOSITORY_ID, r[1]);
+ return result;
+ }
+
+ private static Record expandAllGroups(final Map<String, String> raw) {
+ final Record result = new Record(Type.ALL_GROUPS, new HashMap<EntryKey, Object>());
+ putIfNotNullAsStringArray(raw, "allGroupsList", result, Record.ALL_GROUPS);
+ return result;
+ }
+
+ private static Record expandRootGroups(final Map<String, String> raw) {
+ final Record result = new Record(Type.ROOT_GROUPS, new HashMap<EntryKey, Object>());
+ putIfNotNullAsStringArray(raw, "rootGroupsList", result, Record.ROOT_GROUPS);
+ return result;
+ }
+
+ private static Record expandDeletedArtifact(final Map<String, String> raw) {
+ final Record result = new Record(Type.ARTIFACT_REMOVE, new HashMap<EntryKey, Object>());
+ putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED);
+ if (raw.containsKey("del")) {
+ expandUinfo(raw.get("del"), result);
+ }
+ return result;
+ }
+
+ /**
+ * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming.
+ */
+ private static Record expandAddedArtifact(final Map<String, String> raw) {
+ final Record result = new Record(Type.ARTIFACT_ADD, new HashMap<EntryKey, Object>());
+
+ // Minimal
+ expandUinfo(raw.get(UINFO), result);
+ final String info = raw.get(INFO);
+ if (info != null) {
+ String[] r = FS_PATTERN.split(info);
+ result.put(Record.PACKAGING, renvl(r[0]));
+ result.put(Record.FILE_MODIFIED, Long.valueOf(r[1]));
+ result.put(Record.FILE_SIZE, Long.valueOf(r[2]));
+ result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE : Boolean.FALSE);
+ result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE : Boolean.FALSE);
+ result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE : Boolean.FALSE);
+ if (r.length > 6) {
+ result.put(Record.FILE_EXTENSION, r[6]);
+ }
+ else {
+ final String packaging = Record.PACKAGING.coerce(result.get(Record.PACKAGING));
+ if (result.containsKey(Record.CLASSIFIER)
+ || "pom".equals(packaging)
+ || "war".equals(packaging)
+ || "ear".equals(packaging)) {
+ result.put(Record.FILE_EXTENSION, packaging);
+ }
+ else {
+ result.put(Record.FILE_EXTENSION, "jar"); // best guess
+ }
+ }
+ }
+ putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED);
+ putIfNotNull(raw, "n", result, Record.NAME);
+ putIfNotNull(raw, "d", result, Record.DESCRIPTION);
+ putIfNotNull(raw, "1", result, Record.SHA1);
+
+ // Jar file contents (optional)
+ putIfNotNullAsStringArray(raw, "classnames", result, Record.CLASSNAMES);
+
+ // Maven Plugin (optional)
+ putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX);
+ putIfNotNullAsStringArray(raw, "gx", result, Record.PLUGIN_GOALS);
+
+ // OSGi (optional)
+ putIfNotNull(raw, "Bundle-SymbolicName", result, Record.OSGI_BUNDLE_SYMBOLIC_NAME);
+ putIfNotNull(raw, "Bundle-Version", result, Record.OSGI_BUNDLE_VERSION);
+ putIfNotNull(raw, "Export-Package", result, Record.OSGI_EXPORT_PACKAGE);
+ putIfNotNull(raw, "Export-Service", result, Record.OSGI_EXPORT_SERVICE);
+ putIfNotNull(raw, "Bundle-Description", result, Record.OSGI_BUNDLE_DESCRIPTION);
+ putIfNotNull(raw, "Bundle-Name", result, Record.OSGI_BUNDLE_NAME);
+ putIfNotNull(raw, "Bundle-License", result, Record.OSGI_BUNDLE_LICENSE);
+ putIfNotNull(raw, "Bundle-DocURL", result, Record.OSGI_EXPORT_DOCURL);
+ putIfNotNull(raw, "Import-Package", result, Record.OSGI_IMPORT_PACKAGE);
+ putIfNotNull(raw, "Require-Bundle", result, Record.OSGI_REQUIRE_BUNDLE);
+
+ return result;
+ }
+
+ /**
+ * Expands UINFO synthetic field. Handles {@code null} String inputs.
+ */
+ private static void expandUinfo(final String uinfo, final Record result) {
+ if (uinfo != null) {
+ String[] r = FS_PATTERN.split(uinfo);
+ result.put(Record.GROUP_ID, r[0]);
+ result.put(Record.ARTIFACT_ID, r[1]);
+ result.put(Record.VERSION, r[2]);
+ String classifier = renvl(r[3]);
+ if (classifier != null) {
+ result.put(Record.CLASSIFIER, classifier);
+ if (r.length > 4) {
+ result.put(Record.FILE_EXTENSION, r[4]);
+ }
+ }
+ else if (r.length > 4) {
+ result.put(Record.PACKAGING, r[4]);
+ }
+ }
+ }
+
+ /**
+ * Helper to put a value from source map into target map, if not null.
+ */
+ private static void putIfNotNull(
+ final Map<String, String> source,
+ final String sourceName,
+ final Record target,
+ final EntryKey targetName)
+ {
+ String value = source.get(sourceName);
+ if (value != null && value.trim().length() != 0) {
+ target.put(targetName, value);
+ }
+ }
+
+ /**
+ * Helper to put a {@link Long} value from source map into target map, if not null.
+ */
+ private static void putIfNotNullTS(
+ final Map<String, String> source,
+ final String sourceName,
+ final Record target,
+ final EntryKey targetName)
+ {
+ String value = source.get(sourceName);
+ if (value != null && value.trim().length() != 0) {
+ target.put(targetName, Long.valueOf(value));
+ }
+ }
+
+ /**
+ * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null.
+ */
+ private static void putIfNotNullAsStringArray(
+ final Map<String, String> source,
+ final String sourceName,
+ final Record target,
+ final EntryKey targetName)
+ {
+ String value = source.get(sourceName);
+ if (value != null && value.trim().length() != 0) {
+ target.put(targetName, FS_PATTERN.split(value));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
index 9780067..fa6b6a7 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java
@@ -37,7 +37,7 @@ public interface ResourceHandler
{
/**
* Returns the {@link InputStream} of resource with {@code name} or {@code null} if no such resource. Closing the
- * stream is the responsibility of the caller.
+ * stream is the responsibility of the caller. The stream should be buffered if possible.
*
* @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string.
*/
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
new file mode 100644
index 0000000..798c7a1
--- /dev/null
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Utils.java
@@ -0,0 +1,98 @@
+package org.apache.maven.index.reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Properties;
+import java.util.TimeZone;
+import java.util.regex.Pattern;
+
+/**
+ * Reusable code snippets and constants.
+ *
+ * @since 5.1.2
+ */
+public final class Utils
+{
+ private Utils() {
+ // nothing
+ }
+
+ static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index";
+
+ static final DateFormat INDEX_DATE_FORMAT;
+
+ static {
+ INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z");
+ INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT"));
+ }
+
+ static final String FIELD_SEPARATOR = "|";
+
+ static final String NOT_AVAILABLE = "NA";
+
+ static final String UINFO = "u";
+
+ static final String INFO = "i";
+
+ static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR));
+
+ /**
+ * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream.
+ */
+ static Properties loadProperties(final InputStream inputStream) throws IOException {
+ try {
+ final Properties properties = new Properties();
+ properties.load(inputStream);
+ return properties;
+ }
+ finally {
+ inputStream.close();
+ }
+ }
+
+ /**
+ * Helper to translate the "NA" (not available) input into {@code null} value.
+ */
+ static String renvl(final String v) {
+ return NOT_AVAILABLE.equals(v) ? null : v;
+ }
+
+ /**
+ * Helper to translate {@code null} into "NA" (not available) value.
+ */
+ static String nvl(final String v) {
+ return v == null ? NOT_AVAILABLE : v;
+ }
+
+ /**
+ * Returns the "root group" of given groupId.
+ */
+ static String rootGroup(final String groupId) {
+ int n = groupId.indexOf('.');
+ if (n > -1) {
+ return groupId.substring(0, n);
+ }
+ return groupId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
index 7fe3896..7d1ff03 100644
--- a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
+++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java
@@ -20,7 +20,7 @@ package org.apache.maven.index.reader;
*/
import java.io.IOException;
-import java.io.InputStream;
+import java.io.OutputStream;
/**
* Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index
@@ -34,11 +34,10 @@ public interface WritableResourceHandler
extends ResourceHandler
{
/**
- * Stores (creates or overwrites if resource with name exists) the resource under {@code name} with content provided
- * by the stream. The {@link InputStream} should be closed when method returns.
+ * Returns the {@link OutputStream} of resource with {@code name}, never {@code null}. Closing the stream is the
+ * responsibility of the caller. The stream should be buffered if possible.
*
- * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string.
- * @param inputStream the content of the resource, guaranteed to be non-{@code null}.
+ * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string.
*/
- void save(final String name, final InputStream inputStream) throws IOException;
+ OutputStream openWrite(final String name) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
index 6afe222..d1f7a64 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java
@@ -21,6 +21,7 @@ package org.apache.maven.index.reader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
/**
* A trivial caching {@link ResourceHandler} that caches forever during single session (existence of the instance).
@@ -49,7 +50,7 @@ public class CachingResourceHandler
if (inputStream == null) {
return null;
}
- local.save(name, inputStream);
+ writeLocal(name, inputStream);
return local.open(name);
}
@@ -57,4 +58,23 @@ public class CachingResourceHandler
remote.close();
local.close();
}
+
+ private void writeLocal(final String name, final InputStream inputStream) throws IOException {
+ try {
+ final OutputStream outputStream = local.openWrite(name);
+ try {
+ int read;
+ byte[] bytes = new byte[8192];
+ while ((read = inputStream.read(bytes)) != -1) {
+ outputStream.write(bytes, 0, read);
+ }
+ }
+ finally {
+ outputStream.close();
+ }
+ }
+ finally {
+ inputStream.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/b9c4d908/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
----------------------------------------------------------------------
diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
index 0d1915d..097d44d 100644
--- a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
+++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java
@@ -19,14 +19,18 @@ package org.apache.maven.index.reader;
* under the License.
*/
+import java.io.File;
import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
-import java.util.HashMap;
+import java.util.Date;
+import java.util.List;
import java.util.Map;
import org.apache.maven.index.reader.Record.Type;
import org.junit.Test;
+import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertThat;
@@ -34,33 +38,56 @@ import static org.junit.Assert.assertThat;
* UT for {@link ChunkReader}
*/
public class ChunkReaderTest
+ extends TestSupport
{
@Test
public void simple() throws IOException {
- final Map<Type, Integer> recordTypes = new HashMap<Type, Integer>();
- recordTypes.put(Type.DESCRIPTOR, 0);
- recordTypes.put(Type.ROOT_GROUPS, 0);
- recordTypes.put(Type.ALL_GROUPS, 0);
- recordTypes.put(Type.ARTIFACT_ADD, 0);
- recordTypes.put(Type.ARTIFACT_REMOVE, 0);
+ final ChunkReader chunkReader = new ChunkReader(
+ "full",
+ testResourceHandler("simple").open("nexus-maven-repository-index.gz")
+ );
+ final Map<Type, List<Record>> recordTypes = countRecordTypes(chunkReader);
+ assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(2));
+ assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
+ }
- final ChunkReader chunkReader = new ChunkReader("full",
- new FileInputStream("src/test/resources/nexus-maven-repository-index.gz"));
- try {
- assertThat(chunkReader.getVersion(), equalTo(1));
- assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L));
- for (Record record : chunkReader) {
- recordTypes.put(record.getType(), recordTypes.get(record.getType()) + 1);
+ @Test
+ public void roundtrip() throws IOException {
+ final Date published;
+ File tempChunkFile = createTempFile("nexus-maven-repository-index.gz");
+ {
+ final ChunkReader chunkReader = new ChunkReader(
+ "full",
+ testResourceHandler("simple").open("nexus-maven-repository-index.gz")
+ );
+ final ChunkWriter chunkWriter = new ChunkWriter(
+ chunkReader.getName(),
+ new FileOutputStream(tempChunkFile), 1, new Date()
+ );
+ try {
+ chunkWriter.writeChunk(chunkReader.iterator());
}
- }
- finally {
- chunkReader.close();
+ finally {
+ chunkWriter.close();
+ chunkReader.close();
+ }
+ published = chunkWriter.getTimestamp();
}
- assertThat(recordTypes.get(Type.DESCRIPTOR), equalTo(1));
- assertThat(recordTypes.get(Type.ROOT_GROUPS), equalTo(1));
- assertThat(recordTypes.get(Type.ALL_GROUPS), equalTo(1));
- assertThat(recordTypes.get(Type.ARTIFACT_ADD), equalTo(2));
- assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), equalTo(0));
+ final ChunkReader chunkReader = new ChunkReader(
+ "full",
+ new FileInputStream(tempChunkFile)
+ );
+ assertThat(chunkReader.getVersion(), equalTo(1));
+ assertThat(chunkReader.getTimestamp().getTime(), equalTo(published.getTime()));
+ final Map<Type, List<Record>> recordTypes = countRecordTypes(chunkReader);
+ assertThat(recordTypes.get(Type.DESCRIPTOR).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ROOT_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ALL_GROUPS).size(), equalTo(1));
+ assertThat(recordTypes.get(Type.ARTIFACT_ADD).size(), equalTo(2));
+ assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), nullValue());
}
}