You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@archiva.apache.org by ma...@apache.org on 2019/08/25 11:15:28 UTC
[archiva] branch feature/storage_refactoring updated (ecec848 ->
232ac43)
This is an automated email from the ASF dual-hosted git repository.
martin_s pushed a change to branch feature/storage_refactoring
in repository https://gitbox.apache.org/repos/asf/archiva.git.
from ecec848 Changing checksum handling in metadata
new 63cbe9b Adding stream methods for artifact search
new 232ac43 Adding generic checksum storage for artifacts
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
archiva-modules/metadata/metadata-model/pom.xml | 4 +
.../archiva/metadata/model/ArtifactMetadata.java | 7 +-
.../repository/AbstractMetadataRepository.java | 38 ++++-
.../metadata/repository/MetadataRepository.java | 127 ++++++++++++---
.../repository/AbstractMetadataRepositoryTest.java | 61 +++++++-
.../cassandra/CassandraMetadataRepository.java | 56 +++++--
.../repository/file/FileMetadataRepository.java | 171 +++++++++++++++------
.../file/FileMetadataRepositoryTest.java | 2 +
.../metadata/repository/jcr/JcrConstants.java | 1 +
.../repository/jcr/JcrMetadataRepository.java | 157 +++++++++++++++++--
.../repository/jcr/OakRepositoryFactory.java | 7 +-
.../archiva/metadata/repository/jcr/jcr-schema.cnd | 29 ++--
.../repository/jcr/JcrMetadataRepositoryTest.java | 6 +-
13 files changed, 551 insertions(+), 115 deletions(-)
[archiva] 02/02: Adding generic checksum storage for artifacts
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
martin_s pushed a commit to branch feature/storage_refactoring
in repository https://gitbox.apache.org/repos/asf/archiva.git
commit 232ac43077037ad349ae62e9c254dcc18378b1d3
Author: Martin Stockhammer <ma...@apache.org>
AuthorDate: Sun Aug 25 13:14:59 2019 +0200
Adding generic checksum storage for artifacts
---
.../repository/AbstractMetadataRepository.java | 10 +-
.../metadata/repository/MetadataRepository.java | 62 +++++----
.../repository/AbstractMetadataRepositoryTest.java | 15 +-
.../cassandra/CassandraMetadataRepository.java | 49 ++++++-
.../repository/file/FileMetadataRepository.java | 3 +-
.../metadata/repository/jcr/JcrConstants.java | 1 +
.../repository/jcr/JcrMetadataRepository.java | 155 +++++++++++++++++++--
.../repository/jcr/OakRepositoryFactory.java | 7 +-
.../archiva/metadata/repository/jcr/jcr-schema.cnd | 29 ++--
.../repository/jcr/JcrMetadataRepositoryTest.java | 6 +-
10 files changed, 267 insertions(+), 70 deletions(-)
diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java
index bcb3e83..248e194 100644
--- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java
+++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java
@@ -344,10 +344,6 @@ public abstract class AbstractMetadataRepository
throw new UnsupportedOperationException();
}
- protected QueryParameter getParameterOrDefault(QueryParameter queryParameter) {
- return queryParameter == null ? new QueryParameter( ) : queryParameter;
- }
-
@Override
public <T extends MetadataFacet> Stream<T> getMetadataFacetStream( RepositorySession session, String repositoryId, Class<T> facetClazz ) throws MetadataRepositoryException
{
@@ -409,13 +405,13 @@ public abstract class AbstractMetadataRepository
@Override
public List<ArtifactMetadata> getArtifactsByDateRange(RepositorySession session, String repoId, ZonedDateTime startTime, ZonedDateTime endTime)
throws MetadataRepositoryException {
- return getArtifactsByDateRange(session, repoId, startTime, endTime, getParameterOrDefault( null ));
+ return getArtifactsByDateRange(session, repoId, startTime, endTime, new QueryParameter( ));
}
@Override
public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repositoryId ) throws MetadataResolutionException
{
- return getArtifactStream( session, repositoryId, getParameterOrDefault( null ) );
+ return getArtifactStream( session, repositoryId, new QueryParameter( ) );
}
@Override
@@ -423,7 +419,7 @@ public abstract class AbstractMetadataRepository
final String namespace, final String projectId,
final String projectVersion) throws MetadataResolutionException
{
- return getArtifactStream( session,repoId,namespace, projectId, projectVersion, getParameterOrDefault( null ));
+ return getArtifactStream( session,repoId,namespace, projectId, projectVersion, new QueryParameter( ));
}
@Override
diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java
index da54715..c267b36 100644
--- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java
+++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java
@@ -107,7 +107,7 @@ public interface MetadataRepository
* @param project the project metadata to create or update
* @throws MetadataRepositoryException if the update fails
*/
- void updateProject( RepositorySession session, String repositoryId, ProjectMetadata project )
+ void updateProject( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull ProjectMetadata project )
throws MetadataRepositoryException;
/**
@@ -121,8 +121,9 @@ public interface MetadataRepository
* @param artifactMeta Information about the artifact itself.
* @throws MetadataRepositoryException if something goes wrong during update.
*/
- void updateArtifact( RepositorySession session, String repositoryId, String namespace, String projectId, String projectVersion,
- ArtifactMetadata artifactMeta )
+ void updateArtifact( @Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nonnull String namespace, @Nonnull String projectId, @Nonnull String projectVersion,
+ @Nonnull ArtifactMetadata artifactMeta )
throws MetadataRepositoryException;
/**
@@ -136,8 +137,9 @@ public interface MetadataRepository
* @param versionMetadata The metadata for the version
* @throws MetadataRepositoryException if something goes wrong during update
*/
- void updateProjectVersion( RepositorySession session, String repositoryId, String namespace, String projectId,
- ProjectVersionMetadata versionMetadata )
+ void updateProjectVersion( @Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nonnull String namespace, @Nonnull String projectId,
+ @Nonnull ProjectVersionMetadata versionMetadata )
throws MetadataRepositoryException;
/**
@@ -149,7 +151,7 @@ public interface MetadataRepository
* @param namespace The namespace ('.' separated)
* @throws MetadataRepositoryException if something goes wrong during update
*/
- void updateNamespace( RepositorySession session, String repositoryId, String namespace )
+ void updateNamespace( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull String namespace )
throws MetadataRepositoryException;
/**
@@ -161,7 +163,7 @@ public interface MetadataRepository
* @return The list of facet names, or an empty list, if there are no facets stored on this repository for the given facet id.
* @throws MetadataRepositoryException if something goes wrong
*/
- List<String> getMetadataFacets( RepositorySession session, String repositoryId, String facetId )
+ List<String> getMetadataFacets( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull String facetId )
throws MetadataRepositoryException;
@@ -182,7 +184,8 @@ public interface MetadataRepository
* @throws MetadataRepositoryException
* @since 3.0
*/
- <T extends MetadataFacet> Stream<T> getMetadataFacetStream( RepositorySession session, String repositoryId, Class<T> facetClazz)
+ <T extends MetadataFacet> Stream<T> getMetadataFacetStream( @Nonnull RepositorySession session,
+ @Nonnull String repositoryId, @Nonnull Class<T> facetClazz)
throws MetadataRepositoryException;
/**
@@ -198,7 +201,9 @@ public interface MetadataRepository
* @throws MetadataRepositoryException
* @since 3.0
*/
- <T extends MetadataFacet> Stream<T> getMetadataFacetStream(RepositorySession session, String repositoryId, Class<T> facetClazz, QueryParameter queryParameter)
+ <T extends MetadataFacet> Stream<T> getMetadataFacetStream(@Nonnull RepositorySession session,
+ @Nonnull String repositoryId, @Nonnull Class<T> facetClazz,
+ @Nonnull QueryParameter queryParameter)
throws MetadataRepositoryException;
/**
@@ -212,7 +217,7 @@ public interface MetadataRepository
* @throws MetadataRepositoryException if something goes wrong
* @since 1.4-M4
*/
- boolean hasMetadataFacet( RepositorySession session, String repositoryId, String facetId )
+ boolean hasMetadataFacet( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull String facetId )
throws MetadataRepositoryException;
/**
@@ -226,7 +231,8 @@ public interface MetadataRepository
* @return The facet values
* @throws MetadataRepositoryException if something goes wrong.
*/
- MetadataFacet getMetadataFacet( RepositorySession session, String repositoryId, String facetId, String name )
+ MetadataFacet getMetadataFacet( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull String facetId,
+ @Nonnull String name )
throws MetadataRepositoryException;
/**
@@ -243,7 +249,8 @@ public interface MetadataRepository
* @throws MetadataRepositoryException if the data cannot be retrieved from the backend
* @since 3.0
*/
- <T extends MetadataFacet> T getMetadataFacet(RepositorySession session, String repositoryId, Class<T> clazz, String name)
+ <T extends MetadataFacet> T getMetadataFacet(@Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nonnull Class<T> clazz, @Nonnull String name)
throws MetadataRepositoryException;
/**
@@ -254,7 +261,8 @@ public interface MetadataRepository
* @param metadataFacet The facet to add
* @throws MetadataRepositoryException if the facet cannot be stored.
*/
- void addMetadataFacet( RepositorySession session, String repositoryId, MetadataFacet metadataFacet )
+ void addMetadataFacet( @Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nonnull MetadataFacet metadataFacet )
throws MetadataRepositoryException;
/**
@@ -265,7 +273,7 @@ public interface MetadataRepository
* @param facetId The facet id
* @throws MetadataRepositoryException if the removal fails
*/
- void removeMetadataFacets( RepositorySession session, String repositoryId, String facetId )
+ void removeMetadataFacets( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull String facetId )
throws MetadataRepositoryException;
/**
@@ -276,7 +284,7 @@ public interface MetadataRepository
* @param facetId The facet id
* @param name The facet name or path
*/
- void removeMetadataFacet( RepositorySession session, String repositoryId, String facetId, String name )
+ void removeMetadataFacet( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull String facetId, @Nonnull String name )
throws MetadataRepositoryException;
@@ -285,7 +293,8 @@ public interface MetadataRepository
* uses default query parameters.
*
*/
- List<ArtifactMetadata> getArtifactsByDateRange(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime )
+ List<ArtifactMetadata> getArtifactsByDateRange( @Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nullable ZonedDateTime startTime, @Nullable ZonedDateTime endTime )
throws MetadataRepositoryException;
/**
@@ -303,7 +312,9 @@ public interface MetadataRepository
* @throws MetadataRepositoryException if the query fails.
* @since 3.0
*/
- List<ArtifactMetadata> getArtifactsByDateRange(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter )
+ List<ArtifactMetadata> getArtifactsByDateRange(@Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nullable ZonedDateTime startTime, @Nullable ZonedDateTime endTime,
+ @Nonnull QueryParameter queryParameter )
throws MetadataRepositoryException;
@@ -320,7 +331,8 @@ public interface MetadataRepository
* @throws MetadataRepositoryException
* @since 3.0
*/
- Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime )
+ Stream<ArtifactMetadata> getArtifactByDateRangeStream( @Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nullable ZonedDateTime startTime, @Nullable ZonedDateTime endTime )
throws MetadataRepositoryException;
/**
@@ -337,8 +349,9 @@ public interface MetadataRepository
* @throws MetadataRepositoryException
* @since 3.0
*/
- Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId,
- ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter)
+ Stream<ArtifactMetadata> getArtifactByDateRangeStream( @Nonnull RepositorySession session, @Nonnull String repositoryId,
+ @Nullable ZonedDateTime startTime, @Nullable ZonedDateTime endTime,
+ @Nonnull QueryParameter queryParameter)
throws MetadataRepositoryException;
@@ -351,7 +364,7 @@ public interface MetadataRepository
* @return The list of artifacts that match the given checksum.
* @throws MetadataRepositoryException
*/
- List<ArtifactMetadata> getArtifactsByChecksum(RepositorySession session, String repositoryId, String checksum )
+ List<ArtifactMetadata> getArtifactsByChecksum(@Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull String checksum )
throws MetadataRepositoryException;
/**
@@ -365,7 +378,8 @@ public interface MetadataRepository
* @return a list of artifacts
* @throws MetadataRepositoryException
*/
- List<ArtifactMetadata> getArtifactsByProjectVersionMetadata( RepositorySession session, String key, String value, String repositoryId )
+ List<ArtifactMetadata> getArtifactsByProjectVersionMetadata( @Nonnull RepositorySession session, @Nonnull String key, @Nonnull String value,
+ @Nullable String repositoryId )
throws MetadataRepositoryException;
/**
@@ -467,7 +481,7 @@ public interface MetadataRepository
* @return A stream of artifact metadata objects for each artifact found in the repository.
* @since 3.0
*/
- Stream<ArtifactMetadata> getArtifactStream( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nullable QueryParameter queryParameter )
+ Stream<ArtifactMetadata> getArtifactStream( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nonnull QueryParameter queryParameter )
throws MetadataResolutionException;
/**
@@ -500,7 +514,7 @@ public interface MetadataRepository
*/
Stream<ArtifactMetadata> getArtifactStream( @Nonnull RepositorySession session, @Nonnull String repoId,
@Nonnull String namespace, @Nonnull String projectId,
- @Nonnull String projectVersion, @Nullable QueryParameter queryParameter )
+ @Nonnull String projectVersion, @Nonnull QueryParameter queryParameter )
throws MetadataResolutionException;
/**
diff --git a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
index 6fb1e75..e24f456 100644
--- a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
+++ b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
@@ -1207,19 +1207,22 @@ public abstract class AbstractMetadataRepositoryTest
public void testGetArtifactStream( )
throws Exception
{
+ ArtifactMetadata artifact1 = createArtifact( );
+ ArtifactMetadata artifact2 = createArtifact( "pom" );
try ( RepositorySession session = getSessionFactory( ).createSession( ) )
{
- ArtifactMetadata artifact1 = createArtifact( );
- ArtifactMetadata artifact2 = createArtifact( "pom" );
+
getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact1 );
getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact2 );
-
+ }
+ try ( RepositorySession session = getSessionFactory( ).createSession( ) ) {
tryAssert( ( ) -> {
Stream<ArtifactMetadata> artifacts =
getRepository( ).getArtifactStream( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION );
assertNotNull( artifacts );
List<ArtifactMetadata> actual = artifacts
.sorted( ( o1, o2 ) -> o1.getId( ).compareTo( o2.getId( ) ) ).collect( Collectors.toList( ) );
+ assertEquals( 2, actual.size( ) );
assertEquals( Arrays.asList( artifact1, artifact2 ), actual );
} );
@@ -1628,8 +1631,10 @@ public abstract class AbstractMetadataRepositoryTest
getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact );
session.save( );
- assertEquals( Collections.singletonList( artifact ),
- new ArrayList<>( getRepository( ).getArtifactsByChecksum( session, TEST_REPO_ID, TEST_SHA256 ) ) );
+ tryAssert( () ->
+ assertEquals( Collections.singletonList( artifact ),
+ new ArrayList<>( getRepository( ).getArtifactsByChecksum( session, TEST_REPO_ID, TEST_SHA256 ) ))
+ );
}
}
diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
index 844bba7..7637135 100644
--- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
@@ -2095,7 +2095,7 @@ public class CassandraMetadataRepository
}
}
- return mapArtifactMetadataToArtifact( metadataFacetResult, artifactMetadatas );
+ return mapArtifactFacetToArtifact( metadataFacetResult, artifactMetadatas );
}
@Override
@@ -2354,13 +2354,13 @@ public class CassandraMetadataRepository
.addEqualsExpression( PROJECT_VERSION.toString(), projectVersion ) //
.execute();
- return mapArtifactMetadataToArtifact(result, artifactMetadatas);
+ return mapArtifactFacetToArtifact(result, artifactMetadatas);
}
/**
* Attach metadata to each of the ArtifactMetadata objects
*/
- private List<ArtifactMetadata> mapArtifactMetadataToArtifact(QueryResult<OrderedRows<String, String, String>> result, List<ArtifactMetadata> artifactMetadatas) {
+ private List<ArtifactMetadata> mapArtifactFacetToArtifact( QueryResult<OrderedRows<String, String, String>> result, List<ArtifactMetadata> artifactMetadatas) {
if ( result.get() == null || result.get().getCount() < 1 )
{
return artifactMetadatas;
@@ -2475,7 +2475,8 @@ public class CassandraMetadataRepository
* any property.
*/
@Override
- public List<ArtifactMetadata> searchArtifacts( RepositorySession session, String repositoryId, String text, boolean exact )
+ public List<ArtifactMetadata> searchArtifacts( final RepositorySession session, final String repositoryId,
+ final String text, final boolean exact )
throws MetadataRepositoryException
{
return getArtifactsByMetadata( session, null, text, repositoryId );
@@ -2485,7 +2486,8 @@ public class CassandraMetadataRepository
* The exact parameter is ignored as we can't do non exact searches in Cassandra
*/
@Override
- public List<ArtifactMetadata> searchArtifacts( RepositorySession session, String repositoryId, String key, String text, boolean exact )
+ public List<ArtifactMetadata> searchArtifacts( final RepositorySession session, final String repositoryId,
+ final String key, final String text, final boolean exact )
throws MetadataRepositoryException
{
// TODO optimize
@@ -2494,4 +2496,41 @@ public class CassandraMetadataRepository
artifacts.addAll( getArtifactsByProperty( session, key, text, repositoryId ) );
return artifacts;
}
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repositoryId,
+ final QueryParameter queryParameter ) throws MetadataResolutionException
+ {
+ RangeSlicesQuery<String, String, String> query = HFactory //
+ .createRangeSlicesQuery( keyspace, ss, ss, ss ) //
+ .setColumnFamily( cassandraArchivaManager.getArtifactMetadataFamilyName( ) ) //
+ .setColumnNames( ArtifactMetadataModel.COLUMNS ); //
+
+ query = query.addEqualsExpression( REPOSITORY_NAME.toString(), repositoryId );
+
+ QueryResult<OrderedRows<String, String, String>> result = query.execute();
+
+ try
+ {
+ return StreamSupport.stream( createResultSpliterator( result, ( Row<String, String, String> row, ArtifactMetadata last ) ->
+ mapArtifactMetadataStringColumnSlice( row.getKey( ), row.getColumnSlice( ) ) ), false )
+ .skip( queryParameter.getOffset( ) ).limit( queryParameter.getLimit( ) );
+ }
+ catch ( MetadataRepositoryException e )
+ {
+ throw new MetadataResolutionException( e.getMessage( ), e );
+ }
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repoId,
+ final String namespace, final String projectId, final String projectVersion,
+ final QueryParameter queryParameter ) throws MetadataResolutionException
+ {
+ // Currently we have to align the facets with the artifacts, which means querying artifacts, querying facets and combining them.
+ // I so no stream friendly way to do this, so we just use the collection based method and return the stream.
+ // TODO: Maybe we can query the facets for each artifact separately, but not sure, if this affects performance significantly
+ // We need some data to verify this.
+ return getArtifacts( session, repoId, namespace, projectId, projectVersion ).stream( ).skip( queryParameter.getOffset( ) ).limit( queryParameter.getLimit( ) );
+ }
}
diff --git a/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java b/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java
index 4191945..4719d92 100644
--- a/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java
@@ -1214,10 +1214,9 @@ public class FileMetadataRepository
@Override
public Stream<ArtifactMetadata> getArtifactStream( @Nonnull final RepositorySession session, @Nonnull final String repositoryId,
- @Nullable QueryParameter queryParameter ) throws MetadataResolutionException
+ @Nonnull QueryParameter queryParameter ) throws MetadataResolutionException
{
- queryParameter = getParameterOrDefault( queryParameter );
return getAllNamespacesStream( session, repositoryId ).filter( Objects::nonNull ).flatMap( ns ->
{
try
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrConstants.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrConstants.java
index 46caef9..d0119c7 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrConstants.java
+++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrConstants.java
@@ -45,6 +45,7 @@ public interface JcrConstants
String MIXIN_META_LICENSE = "archiva:meta_license";
String MIXIN_META_MAILINGLIST = "archiva:meta_mailinglist";
String DEPENDENCY_NODE_TYPE = "archiva:dependency";
+ String CHECKSUM_NODE_TYPE = "archiva:checksum";
// Must be alphabetically ordered!
String[] PROJECT_VERSION_VERSION_PROPERTIES = {"ci.system","ci.url", "description", "incomplete", "issue.system","issue.url", "name", "org.name", "org.url", "url", "scm.connection", "scm.developerConnection", "scm.url"};
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
index 25b11de..a9d0a85 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
@@ -20,6 +20,7 @@ package org.apache.archiva.metadata.repository.jcr;
*/
import com.google.common.collect.ImmutableMap;
+import org.apache.archiva.checksum.ChecksumAlgorithm;
import org.apache.archiva.metadata.QueryParameter;
import org.apache.archiva.metadata.model.*;
import org.apache.archiva.metadata.model.maven2.MavenArtifactFacet;
@@ -42,6 +43,7 @@ import org.slf4j.LoggerFactory;
import javax.jcr.NamespaceRegistry;
import javax.jcr.Node;
+import javax.jcr.NodeIterator;
import javax.jcr.PathNotFoundException;
import javax.jcr.Property;
import javax.jcr.Repository;
@@ -196,8 +198,22 @@ public class JcrMetadataRepository
node.setProperty( "whenGathered", cal );
node.setProperty( "size", artifactMeta.getSize() );
- node.setProperty( "md5", artifactMeta.getMd5() );
- node.setProperty( "sha1", artifactMeta.getSha1() );
+
+ int idx=0;
+ Node cslistNode = getOrAddNodeByPath( node, "checksums" );
+ NodeIterator nit = cslistNode.getNodes("*");
+ while (nit.hasNext()) {
+ Node csNode = nit.nextNode();
+ if (csNode.isNodeType( CHECKSUM_NODE_TYPE )) {
+ csNode.remove();
+ }
+ }
+ for ( Map.Entry<ChecksumAlgorithm, String> entry : artifactMeta.getChecksums().entrySet()) {
+ String type = entry.getKey( ).name( );
+ Node csNode = cslistNode.addNode( type , CHECKSUM_NODE_TYPE);
+ csNode.setProperty( "type", type );
+ csNode.setProperty( "value", entry.getValue( ) );
+ }
node.setProperty( "version", artifactMeta.getVersion() );
@@ -836,7 +852,7 @@ public class JcrMetadataRepository
final Session jcrSession = getSession( session );
List<ArtifactMetadata> artifacts;
- String q = getArtifactQuery( repositoryId ) + " AND ([sha1] = $checksum OR [md5] = $checksum)";
+ String q = getArtifactQuery( repositoryId ).append(" AND ([artifact].[checksums/*/value] = $checksum)").toString();
try
{
@@ -1688,20 +1704,33 @@ public class JcrMetadataRepository
artifact.setSize( artifactNode.getProperty( "size" ).getLong() );
}
- if ( artifactNode.hasProperty( "md5" ) )
- {
- artifact.setMd5( artifactNode.getProperty( "md5" ).getString() );
- }
-
- if ( artifactNode.hasProperty( "sha1" ) )
- {
- artifact.setSha1( artifactNode.getProperty( "sha1" ).getString() );
+ Node cslistNode = getOrAddNodeByPath( artifactNode, "checksums" );
+ NodeIterator csNodeIt = cslistNode.getNodes( "*" );
+ while (csNodeIt.hasNext()) {
+ Node csNode = csNodeIt.nextNode( );
+ if (csNode.isNodeType( CHECKSUM_NODE_TYPE ))
+ {
+ addChecksum( artifact, csNode );
+ }
}
retrieveFacetProperties( artifact, artifactNode );
return artifact;
}
+ private void addChecksum(ArtifactMetadata artifact, Node n) {
+ try
+ {
+ ChecksumAlgorithm alg = ChecksumAlgorithm.valueOf( n.getProperty( "type" ).getString() );
+ String value = n.getProperty( "value" ).getString( );
+ artifact.setChecksum( alg, value );
+ }
+ catch ( Throwable e )
+ {
+ log.error( "Could not set checksum from node {}", n );
+ }
+ }
+
private static String getPropertyString( Node node, String name )
throws RepositoryException
{
@@ -1996,4 +2025,108 @@ public class JcrMetadataRepository
{
return repository.login(new SimpleCredentials( "admin", "admin".toCharArray() ) );
}
+
+ private static boolean isArtifactNodeType(Node n) {
+ try
+ {
+ return n != null && n.isNodeType( ARTIFACT_NODE_TYPE );
+ }
+ catch ( RepositoryException e )
+ {
+ return false;
+ }
+ }
+
+ private Optional<ArtifactMetadata> getArtifactOptional(final String repositoryId, final Node n) {
+ try
+ {
+ return Optional.ofNullable( getArtifactFromNode( repositoryId, n ) );
+ }
+ catch ( RepositoryException e )
+ {
+ return Optional.empty( );
+ }
+ }
+
+ private Optional<ArtifactMetadata> getArtifactOptional(final String repositoryId, final Row row) {
+ try
+ {
+ return Optional.ofNullable( getArtifactFromNode( repositoryId, row.getNode( "artifact" ) ) );
+ }
+ catch ( RepositoryException e )
+ {
+ return Optional.empty( );
+ }
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repositoryId,
+ final String namespace, final String projectId, final String projectVersion,
+ final QueryParameter queryParameter ) throws MetadataResolutionException
+ {
+ final Session jcrSession;
+ try
+ {
+ jcrSession = getSession( session );
+ }
+ catch ( MetadataRepositoryException e )
+ {
+ throw new MetadataResolutionException( e.getMessage( ) );
+ }
+
+ try
+ {
+ Node root = jcrSession.getRootNode();
+ String path = getProjectVersionPath( repositoryId, namespace, projectId, projectVersion );
+
+ if ( root.hasNode( path ) )
+ {
+ Node node = root.getNode( path );
+ return StreamSupport.stream( JcrUtils.getChildNodes( node ).spliterator( ), false ).filter(JcrMetadataRepository::isArtifactNodeType)
+ .map( n -> getArtifactOptional( repositoryId, n ) )
+ .map( Optional::get ).skip( queryParameter.getOffset( ) ).limit( queryParameter.getLimit( ) );
+ } else {
+ return Stream.empty( );
+ }
+ }
+ catch ( RepositoryException e )
+ {
+ throw new MetadataResolutionException( e.getMessage(), e );
+ }
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repositoryId,
+ final QueryParameter queryParameter ) throws MetadataResolutionException
+ {
+ final Session jcrSession;
+ try
+ {
+ jcrSession = getSession( session );
+ }
+ catch ( MetadataRepositoryException e )
+ {
+ throw new MetadataResolutionException( e.getMessage( ), e );
+ }
+ List<ArtifactMetadata> artifacts;
+
+ String q = getArtifactQuery( repositoryId ).toString();
+
+ try
+ {
+ Query query = jcrSession.getWorkspace().getQueryManager().createQuery( q, Query.JCR_SQL2 );
+ QueryResult result = query.execute();
+
+ return StreamSupport.stream( createResultSpliterator( result, ( Row row ) ->
+ getArtifactOptional( repositoryId, row ) ), false )
+ .map(Optional::get)
+ .skip( queryParameter.getOffset( ) ).limit( queryParameter.getLimit( ) );
+
+ }
+ catch ( RepositoryException | MetadataRepositoryException e )
+ {
+ throw new MetadataResolutionException( e.getMessage(), e );
+ }
+
+ }
}
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/OakRepositoryFactory.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/OakRepositoryFactory.java
index 56f17b4..d303048 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/OakRepositoryFactory.java
+++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/OakRepositoryFactory.java
@@ -485,7 +485,12 @@ public class OakRepositoryFactory
initBaseRule(idxBuilder.indexRule( ARTIFACT_NODE_TYPE ))
.property( "whenGathered" ).type("Date").propertyIndex().analyzed().ordered()
.property("size").type("Long").propertyIndex().analyzed().ordered()
- .property("version").propertyIndex().analyzed().ordered();
+ .property("version").propertyIndex().analyzed().ordered()
+ .property("checksums/*/value").propertyIndex();
+
+ initBaseRule( idxBuilder.indexRule( CHECKSUM_NODE_TYPE ) )
+ .property("type").propertyIndex()
+ .property("value").propertyIndex();
initRegexAll( idxBuilder.indexRule( FACET_NODE_TYPE ) )
.property("archiva:facetId").propertyIndex().analyzed().ordered()
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/resources/org/apache/archiva/metadata/repository/jcr/jcr-schema.cnd b/archiva-modules/plugins/metadata-store-jcr/src/main/resources/org/apache/archiva/metadata/repository/jcr/jcr-schema.cnd
index ce33089..7fdb678 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/main/resources/org/apache/archiva/metadata/repository/jcr/jcr-schema.cnd
+++ b/archiva-modules/plugins/metadata-store-jcr/src/main/resources/org/apache/archiva/metadata/repository/jcr/jcr-schema.cnd
@@ -26,16 +26,16 @@
+ content (archiva:content) primary
[archiva:content] > archiva:base mixin
- + * (archiva:namespace) multiple
+ + * (archiva:namespace)
[archiva:namespace] > archiva:base mixin
- namespace (string)
- + * (archiva:namespace) multiple
- + * (archiva:project) multiple
+ + * (archiva:namespace)
+ + * (archiva:project)
[archiva:project] > archiva:base mixin
- name (string)
- + * (archiva:projectVersion) multiple
+ + * (archiva:projectVersion)
[archiva:meta_scm] mixin
- scm.connection (string)
@@ -79,7 +79,10 @@
- optional (boolean)
[archiva:dependencies] mixin
- + * (archiva:dependency) multiple
+ + * (archiva:dependency)
+
+[archiva:checksums]
+ + * (archiva:checksum)
[archiva:checksum]
- type (string)
@@ -90,20 +93,18 @@
- description (string)
- url (uri)
- incomplete (boolean)
- + * (archiva:artifact) multiple
- + license (archiva:meta_license) multiple
- + mailinglist (archiva:meta_mailinglist) multiple
- + dependencies (archiva:dependencies)
- + * (archiva:facet) multiple
+ + * (archiva:artifact)
+ + * (archiva:meta_license)
+ + * (archiva:meta_mailinglist)
+ + * (archiva:dependencies)
+ + * (archiva:facet)
[archiva:artifact] > archiva:base mixin
- whenGathered (date)
- size (long)
- - md5 (string)
- - sha1 (string)
- version (string)
- + checksum (archiva:checksum) multiple
- + * (archiva:facet) multiple
+ + * (archiva:checksum)
+ + * (archiva:facet)
[archiva:facet] > archiva:base mixin
- archiva:facetId
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepositoryTest.java b/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepositoryTest.java
index 58bb441..9f7bd8c 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepositoryTest.java
+++ b/archiva-modules/plugins/metadata-store-jcr/src/test/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepositoryTest.java
@@ -90,8 +90,12 @@ public class JcrMetadataRepositoryTest
}
@Before
- public void setup() throws MetadataRepositoryException, RepositoryException, MetadataSessionException
+ @Override
+ public void setUp() throws Exception
{
+ super.setUp();
+ super.assertMaxTries=5;
+ super.assertRetrySleepMs = 500;
try( JcrRepositorySession session = (JcrRepositorySession) getSessionFactory().createSession() ) {
Session jcrSession = session.getJcrSession( );
if (jcrSession.itemExists( "/repositories/test" ))
[archiva] 01/02: Adding stream methods for artifact search
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
martin_s pushed a commit to branch feature/storage_refactoring
in repository https://gitbox.apache.org/repos/asf/archiva.git
commit 63cbe9b685a17483cdc88dbfe833fa1e14094843
Author: Martin Stockhammer <ma...@apache.org>
AuthorDate: Sat Aug 24 22:11:44 2019 +0200
Adding stream methods for artifact search
---
archiva-modules/metadata/metadata-model/pom.xml | 4 +
.../archiva/metadata/model/ArtifactMetadata.java | 7 +-
.../repository/AbstractMetadataRepository.java | 42 ++++-
.../metadata/repository/MetadataRepository.java | 73 ++++++++-
.../repository/AbstractMetadataRepositoryTest.java | 56 ++++++-
.../cassandra/CassandraMetadataRepository.java | 7 +-
.../repository/file/FileMetadataRepository.java | 172 +++++++++++++++------
.../file/FileMetadataRepositoryTest.java | 2 +
.../repository/jcr/JcrMetadataRepository.java | 2 +-
9 files changed, 302 insertions(+), 63 deletions(-)
diff --git a/archiva-modules/metadata/metadata-model/pom.xml b/archiva-modules/metadata/metadata-model/pom.xml
index 4d81ad9..b4894b3 100644
--- a/archiva-modules/metadata/metadata-model/pom.xml
+++ b/archiva-modules/metadata/metadata-model/pom.xml
@@ -40,6 +40,10 @@
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-jpa_2.0_spec</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-collections4</artifactId>
+ </dependency>
</dependencies>
diff --git a/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java b/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java
index fd44feb..d8e9290 100644
--- a/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java
+++ b/archiva-modules/metadata/metadata-model/src/main/java/org/apache/archiva/metadata/model/ArtifactMetadata.java
@@ -20,6 +20,7 @@ package org.apache.archiva.metadata.model;
*/
import org.apache.archiva.checksum.ChecksumAlgorithm;
+import org.apache.commons.collections4.bidimap.DualHashBidiMap;
import sun.reflect.generics.repository.MethodRepository;
import javax.xml.bind.annotation.XmlRootElement;
@@ -99,7 +100,7 @@ public class ArtifactMetadata
/**
* The list of checksums.
*/
- private Map<ChecksumAlgorithm, String> checksums = new HashMap<>();
+ private Map<ChecksumAlgorithm, String> checksums = new DualHashBidiMap<>( );
private String toStringValue = "";
private int lastHash = 0;
@@ -177,6 +178,10 @@ public class ArtifactMetadata
return this.checksums;
}
+ public boolean hasChecksum(String checksum) {
+ return this.checksums.containsValue( checksum );
+ }
+
public void setChecksums(Map<ChecksumAlgorithm,String> checksums) {
this.checksums = checksums;
}
diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java
index c4e2710..bcb3e83 100644
--- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java
+++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/AbstractMetadataRepository.java
@@ -28,6 +28,8 @@ import org.apache.archiva.metadata.model.ProjectVersionMetadata;
import org.apache.archiva.metadata.model.ProjectVersionReference;
import org.apache.commons.collections4.ComparatorUtils;
+import javax.annotation.Nonnull;
+import javax.management.Query;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.stream.Stream;
@@ -342,6 +344,10 @@ public abstract class AbstractMetadataRepository
throw new UnsupportedOperationException();
}
+ protected QueryParameter getParameterOrDefault(QueryParameter queryParameter) {
+ return queryParameter == null ? new QueryParameter( ) : queryParameter;
+ }
+
@Override
public <T extends MetadataFacet> Stream<T> getMetadataFacetStream( RepositorySession session, String repositoryId, Class<T> facetClazz ) throws MetadataRepositoryException
{
@@ -349,9 +355,9 @@ public abstract class AbstractMetadataRepository
}
@Override
- public Stream<ArtifactMetadata> getArtifactsByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime ) throws MetadataRepositoryException
+ public Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime ) throws MetadataRepositoryException
{
- return getArtifactsByDateRangeStream( session, repositoryId, startTime, endTime, new QueryParameter());
+ return getArtifactByDateRangeStream( session, repositoryId, startTime, endTime, new QueryParameter());
}
@Override
@@ -376,7 +382,7 @@ public abstract class AbstractMetadataRepository
@Override
- public Stream<ArtifactMetadata> getArtifactsByDateRangeStream(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException
+ public Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException
{
throw new UnsupportedOperationException();
}
@@ -403,7 +409,35 @@ public abstract class AbstractMetadataRepository
@Override
public List<ArtifactMetadata> getArtifactsByDateRange(RepositorySession session, String repoId, ZonedDateTime startTime, ZonedDateTime endTime)
throws MetadataRepositoryException {
- return getArtifactsByDateRange(session, repoId, startTime, endTime, new QueryParameter());
+ return getArtifactsByDateRange(session, repoId, startTime, endTime, getParameterOrDefault( null ));
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repositoryId ) throws MetadataResolutionException
+ {
+ return getArtifactStream( session, repositoryId, getParameterOrDefault( null ) );
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repoId,
+ final String namespace, final String projectId,
+ final String projectVersion) throws MetadataResolutionException
+ {
+ return getArtifactStream( session,repoId,namespace, projectId, projectVersion, getParameterOrDefault( null ));
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repositoryId, QueryParameter queryParameter ) throws MetadataResolutionException
+ {
+ throw new UnsupportedOperationException( );
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repoId,
+ final String namespace, final String projectId,
+ final String projectVersion, final QueryParameter queryParameter ) throws MetadataResolutionException
+ {
+ throw new UnsupportedOperationException( );
}
}
diff --git a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java
index e679704..da54715 100644
--- a/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java
+++ b/archiva-modules/metadata/metadata-repository-api/src/main/java/org/apache/archiva/metadata/repository/MetadataRepository.java
@@ -26,7 +26,8 @@ import org.apache.archiva.metadata.model.ProjectMetadata;
import org.apache.archiva.metadata.model.ProjectVersionMetadata;
import org.apache.archiva.metadata.model.ProjectVersionReference;
-import java.time.ZoneId;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.time.ZonedDateTime;
import java.util.Collection;
import java.util.List;
@@ -319,7 +320,7 @@ public interface MetadataRepository
* @throws MetadataRepositoryException
* @since 3.0
*/
- Stream<ArtifactMetadata> getArtifactsByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime )
+ Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime )
throws MetadataRepositoryException;
/**
@@ -336,7 +337,7 @@ public interface MetadataRepository
* @throws MetadataRepositoryException
* @since 3.0
*/
- Stream<ArtifactMetadata> getArtifactsByDateRangeStream(RepositorySession session, String repositoryId,
+ Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId,
ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter)
throws MetadataRepositoryException;
@@ -454,6 +455,72 @@ public interface MetadataRepository
throws MetadataRepositoryException;
/**
+ * Returns a stream of artifacts that are stored in the given repository. The number and order of elements in the stream
+ * is defined by the <code>queryParameter</code>.
+ * The efficiency of ordering of elements is dependent on the implementation.
+ * There may be some implementations that have to put a hard limit on the elements returned.
+ * If there are no <code>sortFields</code> defined in the query parameter, the order of elements in the stream is undefined and depends
+ * on the implementation.
+ *
+ * @param session
+ * @param repositoryId
+ * @return A stream of artifact metadata objects for each artifact found in the repository.
+ * @since 3.0
+ */
+ Stream<ArtifactMetadata> getArtifactStream( @Nonnull RepositorySession session, @Nonnull String repositoryId, @Nullable QueryParameter queryParameter )
+ throws MetadataResolutionException;
+
+ /**
+ * Returns a stream of all the artifacts in the given repository using default query parameter.
+ * The order of the artifacts returned in the stream depends on the implementation.
+ * The number of elements in the stream is unlimited, but there may be some implementations that have to put a hard
+ * limit on the elements returned.
+ * For further information see {@link #getArtifactStream(RepositorySession, String, QueryParameter)}
+ *
+ * @param session The repository session
+ * @param repositoryId The repository id
+ * @return A (unlimited) stream of artifact metadata elements that are found in this repository
+ * @since 3.0
+ * @see #getArtifactStream(RepositorySession, String, QueryParameter)
+ */
+ Stream<ArtifactMetadata> getArtifactStream( @Nonnull RepositorySession session, @Nonnull String repositoryId)
+ throws MetadataResolutionException;
+
+ /**
+ * Returns a stream of artifacts found for the given artifact coordinates and using the <code>queryParameter</code>
+ *
+ * @param session The repository session. May not be <code>null</code>.
+ * @param repoId The repository id. May not be <code>null</code>.
+ * @param namespace The namespace. May not be <code>null</code>.
+ * @param projectId The project id. May not be <code>null</code>.
+ * @param projectVersion The project version. May not be <code>null</code>.
+ * @return A stream of artifact metadata object. Order and number of elements returned, depends on the <code>queryParameter</code>.
+ * @since 3.0
+ * @throws MetadataResolutionException if there are no elements for the given artifact coordinates.
+ */
+ Stream<ArtifactMetadata> getArtifactStream( @Nonnull RepositorySession session, @Nonnull String repoId,
+ @Nonnull String namespace, @Nonnull String projectId,
+ @Nonnull String projectVersion, @Nullable QueryParameter queryParameter )
+ throws MetadataResolutionException;
+
+ /**
+ * Returns a stream of artifacts found for the given artifact coordinates. The order of elements returned, depends on the
+ * implementation.
+ *
+ * @param session The repository session. May not be <code>null</code>.
+ * @param repoId The repository id. May not be <code>null</code>.
+ * @param namespace The namespace. May not be <code>null</code>.
+ * @param projectId The project id. May not be <code>null</code>.
+ * @param projectVersion The project version. May not be <code>null</code>.
+ * @return A stream of artifact metadata object. Order and number of elements returned, depends on the <code>queryParameter</code>.
+ * @since 3.0
+ * @throws MetadataResolutionException if there are no elements for the given artifact coordinates.
+ */
+ Stream<ArtifactMetadata> getArtifactStream( @Nonnull RepositorySession session, @Nonnull String repoId,
+ @Nonnull String namespace, @Nonnull String projectId,
+ @Nonnull String projectVersion)
+ throws MetadataResolutionException;
+ /**
* basically just checking it exists not complete data returned
*
*
diff --git a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
index c777097..6fb1e75 100644
--- a/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
+++ b/archiva-modules/metadata/metadata-repository-api/src/test/java/org/apache/archiva/metadata/repository/AbstractMetadataRepositoryTest.java
@@ -20,6 +20,7 @@ package org.apache.archiva.metadata.repository;
*/
import junit.framework.TestCase;
+import org.apache.archiva.checksum.ChecksumAlgorithm;
import org.apache.archiva.metadata.QueryParameter;
import org.apache.archiva.metadata.generic.GenericMetadataFacet;
import org.apache.archiva.metadata.generic.GenericMetadataFacetFactory;
@@ -70,6 +71,8 @@ public abstract class AbstractMetadataRepositoryTest
private static final String UNKNOWN = "unknown";
+ private static final String TEST_SHA256 = "e43857b4e75e04a09d167564ca9a4636e5d233035483ba4ecf1243e34325d565";
+
private static final String TEST_MD5 = "bd4a9b642562547754086de2dab26b7d";
private static final String TEST_SHA1 = "2e5daf0201ddeb068a62d5e08da18657ab2c6be9";
@@ -989,7 +992,7 @@ public abstract class AbstractMetadataRepositoryTest
ZonedDateTime upper = artifact2.getWhenGathered().plusSeconds(10);
tryAssert( ( ) -> {
- Stream<ArtifactMetadata> stream = getRepository().getArtifactsByDateRangeStream(session, TEST_REPO_ID, lower, upper, new QueryParameter());
+ Stream<ArtifactMetadata> stream = getRepository().getArtifactByDateRangeStream(session, TEST_REPO_ID, lower, upper, new QueryParameter());
assertNotNull(stream);
List<ArtifactMetadata> artifacts = stream.collect(Collectors.toList());
@@ -1201,6 +1204,29 @@ public abstract class AbstractMetadataRepositoryTest
}
@Test
+ public void testGetArtifactStream( )
+ throws Exception
+ {
+ try ( RepositorySession session = getSessionFactory( ).createSession( ) )
+ {
+ ArtifactMetadata artifact1 = createArtifact( );
+ ArtifactMetadata artifact2 = createArtifact( "pom" );
+ getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact1 );
+ getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact2 );
+
+ tryAssert( ( ) -> {
+ Stream<ArtifactMetadata> artifacts =
+ getRepository( ).getArtifactStream( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION );
+ assertNotNull( artifacts );
+ List<ArtifactMetadata> actual = artifacts
+ .sorted( ( o1, o2 ) -> o1.getId( ).compareTo( o2.getId( ) ) ).collect( Collectors.toList( ) );
+ assertEquals( Arrays.asList( artifact1, artifact2 ), actual );
+ } );
+
+ }
+ }
+
+ @Test
public void testGetArtifactVersions( )
throws Exception
{
@@ -1592,6 +1618,23 @@ public abstract class AbstractMetadataRepositoryTest
}
@Test
+ public void testGetArtifactsByChecksumSingleResultSha256( )
+ throws Exception
+ {
+ try ( RepositorySession session = getSessionFactory( ).createSession( ) )
+ {
+ ArtifactMetadata artifact = createArtifact( );
+ artifact.setChecksum( ChecksumAlgorithm.SHA256, TEST_SHA256);
+ getRepository( ).updateArtifact( session, TEST_REPO_ID, TEST_NAMESPACE, TEST_PROJECT, TEST_PROJECT_VERSION, artifact );
+ session.save( );
+
+ assertEquals( Collections.singletonList( artifact ),
+ new ArrayList<>( getRepository( ).getArtifactsByChecksum( session, TEST_REPO_ID, TEST_SHA256 ) ) );
+
+ }
+ }
+
+ @Test
public void testGetArtifactsByChecksumSingleResultMd5( )
throws Exception
{
@@ -1635,10 +1678,13 @@ public abstract class AbstractMetadataRepositoryTest
getRepository( ).updateArtifact( session, TEST_REPO_ID, namespace, TEST_PROJECT, TEST_PROJECT_VERSION, artifact );
session.save( );
- tryAssert( ( ) -> assertEquals( Collections.singletonList( artifact ),
- new ArrayList<>( getRepository( ).getArtifactsByChecksum( session, TEST_REPO_ID, TEST_SHA1 ) ) ) );
- tryAssert( ( ) -> assertEquals( Collections.singletonList( artifact ),
- new ArrayList<>( getRepository( ).getArtifactsByChecksum( session, TEST_REPO_ID, TEST_MD5 ) ) ) );
+ tryAssert( ( ) ->
+ {
+ assertEquals( Collections.singletonList( artifact ),
+ new ArrayList<>( getRepository( ).getArtifactsByChecksum( session, TEST_REPO_ID, TEST_SHA1 ) ) );
+ assertEquals( Collections.singletonList( artifact ),
+ new ArrayList<>( getRepository( ).getArtifactsByChecksum( session, TEST_REPO_ID, TEST_MD5 ) ) );
+ });
}
}
diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
index 768a94b..844bba7 100644
--- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
@@ -72,7 +72,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Instant;
-import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.function.BiFunction;
@@ -1918,7 +1917,7 @@ public class CassandraMetadataRepository
}
/**
- * For documentation see {@link MetadataRepository#getArtifactsByDateRangeStream(RepositorySession, String, ZonedDateTime, ZonedDateTime, QueryParameter)}
+ * For documentation see {@link MetadataRepository#getArtifactByDateRangeStream(RepositorySession, String, ZonedDateTime, ZonedDateTime, QueryParameter)}
*
* This implementation orders the stream. It does not order the query in the backend.
*
@@ -1929,10 +1928,10 @@ public class CassandraMetadataRepository
* @param queryParameter Additional parameters for the query that affect ordering and number of returned results.
* @return
* @throws MetadataRepositoryException
- * @see MetadataRepository#getArtifactsByDateRangeStream
+ * @see MetadataRepository#getArtifactByDateRangeStream
*/
@Override
- public Stream<ArtifactMetadata> getArtifactsByDateRangeStream(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException
+ public Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException
{
Comparator<ArtifactMetadata> comp = getArtifactMetadataComparator(queryParameter, "whenGathered");
return getArtifactsByDateRange(session, repositoryId, startTime, endTime, queryParameter).stream().sorted(comp).skip(queryParameter.getOffset()).limit(queryParameter.getLimit());
diff --git a/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java b/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java
index 272f536..4191945 100644
--- a/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-file/src/main/java/org/apache/archiva/metadata/repository/file/FileMetadataRepository.java
@@ -19,14 +19,18 @@ package org.apache.archiva.metadata.repository.file;
* under the License.
*/
+import org.apache.archiva.checksum.ChecksumAlgorithm;
import org.apache.archiva.configuration.ArchivaConfiguration;
import org.apache.archiva.configuration.ManagedRepositoryConfiguration;
import org.apache.archiva.metadata.QueryParameter;
import org.apache.archiva.metadata.model.*;
import org.apache.archiva.metadata.repository.*;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -413,7 +417,7 @@ public class FileMetadataRepository
* @throws MetadataRepositoryException
*/
@Override
- public Stream<ArtifactMetadata> getArtifactsByDateRangeStream(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException {
+ public Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException {
try {
List<ArtifactMetadata> artifacts = new ArrayList<>();
for (String ns : getRootNamespaces(session, repositoryId)) {
@@ -452,6 +456,7 @@ public class FileMetadataRepository
}
}
+
@Override
public Collection<ArtifactMetadata> getArtifacts(RepositorySession session, String repoId, String namespace, String projectId,
String projectVersion)
@@ -491,10 +496,9 @@ public class FileMetadataRepository
artifact.setWhenGathered(ZonedDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(value)), ZoneId.of("GMT")));
} else if ("version".equals(field)) {
artifact.setVersion(value);
- } else if ("md5".equals(field)) {
- artifact.setMd5(value);
- } else if ("sha1".equals(field)) {
- artifact.setSha1(value);
+ } else if (field.startsWith("checksum")) {
+ String algorithmStr = StringUtils.removeStart( name, "artifact:checksum:"+id+":");
+ artifact.setChecksum( ChecksumAlgorithm.valueOf( algorithmStr ), value );
} else if ("facetIds".equals(field)) {
if (value.length() > 0) {
String propertyPrefix = "artifact:facet:" + id + ":";
@@ -566,11 +570,9 @@ public class FileMetadataRepository
// alternatively, we could build a referential tree in the content repository, however it would need some levels
// of depth to avoid being too broad to be useful (eg. /repository/checksums/a/ab/abcdef1234567)
- List<ArtifactMetadata> artifacts = new ArrayList<>();
- for (String ns : getRootNamespaces(session, repositoryId)) {
- getArtifactsByChecksum(session, artifacts, repositoryId, ns, checksum);
- }
- return artifacts;
+ return getArtifactStream( session, repositoryId ).filter(
+ a -> a.hasChecksum( checksum )
+ ).collect( Collectors.toList() );
} catch (MetadataResolutionException e) {
throw new MetadataRepositoryException(e.getMessage(), e);
}
@@ -607,8 +609,8 @@ public class FileMetadataRepository
properties.remove("artifact:updated:" + id);
properties.remove("artifact:whenGathered:" + id);
properties.remove("artifact:size:" + id);
- properties.remove("artifact:md5:" + id);
- properties.remove("artifact:sha1:" + id);
+ artifactMetadata.getChecksums().entrySet().stream().forEach( entry ->
+ properties.remove( "artifact:checksum:"+id+":"+entry.getKey().name() ));
properties.remove("artifact:version:" + id);
properties.remove("artifact:facetIds:" + id);
@@ -638,15 +640,15 @@ public class FileMetadataRepository
properties.remove("artifact:updated:" + id);
properties.remove("artifact:whenGathered:" + id);
properties.remove("artifact:size:" + id);
- properties.remove("artifact:md5:" + id);
- properties.remove("artifact:sha1:" + id);
properties.remove("artifact:version:" + id);
properties.remove("artifact:facetIds:" + id);
- String prefix = "artifact:facet:" + id + ":";
- for (Object key : new ArrayList<>(properties.keySet())) {
- String property = (String) key;
- if (property.startsWith(prefix)) {
+ String facetPrefix = "artifact:facet:" + id + ":";
+ String checksumPrefix = "artifact:checksum:"+id+":";
+ for (String property : properties.stringPropertyNames()) {
+ if (property.startsWith( checksumPrefix )) {
+ properties.remove( property );
+ } else if (property.startsWith(facetPrefix)) {
properties.remove(property);
}
}
@@ -687,27 +689,6 @@ public class FileMetadataRepository
}
}
- private void getArtifactsByChecksum(RepositorySession session, List<ArtifactMetadata> artifacts, String repositoryId, String ns,
- String checksum)
- throws MetadataRepositoryException {
- try {
- for (String namespace : getNamespaces(session, repositoryId, ns)) {
- getArtifactsByChecksum(session, artifacts, repositoryId, ns + "." + namespace, checksum);
- }
-
- for (String project : getProjects(session, repositoryId, ns)) {
- for (String version : getProjectVersions(session, repositoryId, ns, project)) {
- for (ArtifactMetadata artifact : getArtifacts(session, repositoryId, ns, project, version)) {
- if (checksum.equals(artifact.getMd5()) || checksum.equals(artifact.getSha1())) {
- artifacts.add(artifact);
- }
- }
- }
- }
- } catch (MetadataResolutionException e) {
- throw new MetadataRepositoryException(e.getMessage(), e);
- }
- }
@Override
public List<ArtifactMetadata> getArtifactsByProjectVersionMetadata(RepositorySession session, String key, String value, String repositoryId)
@@ -771,12 +752,8 @@ public class FileMetadataRepository
properties.setProperty("artifact:whenGathered:" + id,
Long.toString(artifact.getWhenGathered().toInstant().toEpochMilli()));
properties.setProperty("artifact:size:" + id, Long.toString(artifact.getSize()));
- if (artifact.getMd5() != null) {
- properties.setProperty("artifact:md5:" + id, artifact.getMd5());
- }
- if (artifact.getSha1() != null) {
- properties.setProperty("artifact:sha1:" + id, artifact.getSha1());
- }
+ artifact.getChecksums().entrySet().stream().forEach( entry ->
+ properties.setProperty( "artifact:checksum:"+id+":"+entry.getKey().name(), entry.getValue() ));
properties.setProperty("artifact:version:" + id, artifact.getVersion());
Set<String> facetIds = new LinkedHashSet<>(artifact.getFacetIds());
@@ -1043,6 +1020,32 @@ public class FileMetadataRepository
return getNamespaces(session, repoId, null);
}
+ private Stream<String> getAllNamespacesStream(RepositorySession session, String repoId) {
+ Path directory = null;
+ try
+ {
+ directory = getDirectory(repoId);
+ }
+ catch ( IOException e )
+ {
+ return Stream.empty( );
+ }
+ if (!(Files.exists(directory) && Files.isDirectory(directory))) {
+ return Stream.empty( );
+ }
+ final String searchFile = NAMESPACE_METADATA_KEY + ".properties";
+ try
+ {
+ return Files.list(directory).filter(Files::isDirectory).filter(path ->
+ Files.exists(path.resolve(searchFile))
+ ).map(path -> path.getFileName().toString());
+ }
+ catch ( IOException e )
+ {
+ return Stream.empty( );
+ }
+ }
+
@Override
public Collection<String> getNamespaces(RepositorySession session, String repoId, String baseNamespace)
throws MetadataResolutionException {
@@ -1182,6 +1185,85 @@ public class FileMetadataRepository
}
}
+ private class ArtifactCoordinates {
+ final String namespace;
+ final String project;
+ final String version;
+
+ public ArtifactCoordinates(String namespace, String project, String version) {
+ this.namespace = namespace;
+ this.project = project;
+ this.version = version;
+ }
+
+ public String getNamespace( )
+ {
+ return namespace;
+ }
+
+ public String getProject( )
+ {
+ return project;
+ }
+
+ public String getVersion( )
+ {
+ return version;
+ }
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( @Nonnull final RepositorySession session, @Nonnull final String repositoryId,
+ @Nullable QueryParameter queryParameter ) throws MetadataResolutionException
+ {
+
+ queryParameter = getParameterOrDefault( queryParameter );
+ return getAllNamespacesStream( session, repositoryId ).filter( Objects::nonNull ).flatMap( ns ->
+ {
+ try
+ {
+ return getProjects( session, repositoryId, ns ).stream( ).map( proj ->
+ new ArtifactCoordinates( ns, proj, null ) );
+ }
+ catch ( MetadataResolutionException e )
+ {
+ return null;
+ }
+ }
+ ).filter( Objects::nonNull ).flatMap( artifactCoordinates ->
+ {
+ try
+ {
+ return getProjectVersions( session, repositoryId, artifactCoordinates.getNamespace( ), artifactCoordinates.getProject( ) )
+ .stream( ).map(version -> new ArtifactCoordinates( artifactCoordinates.getNamespace(), artifactCoordinates.getProject(), version ));
+ }
+ catch ( MetadataResolutionException e )
+ {
+ return null;
+ }
+ }
+ ).filter( Objects::nonNull ).flatMap( ac ->
+ {
+ try
+ {
+ return getArtifactStream( session, repositoryId, ac.getNamespace(), ac.getProject(), ac.getVersion() );
+ }
+ catch ( MetadataResolutionException e )
+ {
+ return null;
+ }
+ }
+ ).filter( Objects::nonNull );
+ }
+
+ @Override
+ public Stream<ArtifactMetadata> getArtifactStream( final RepositorySession session, final String repoId,
+ final String namespace, final String projectId,
+ final String projectVersion ) throws MetadataResolutionException
+ {
+ return getArtifacts( session, repoId, namespace, projectId, projectVersion ).stream( );
+ }
+
private void getArtifacts(RepositorySession session, List<ArtifactMetadata> artifacts, String repoId, String ns)
throws MetadataResolutionException {
for (String namespace : getNamespaces(session, repoId, ns)) {
diff --git a/archiva-modules/plugins/metadata-store-file/src/test/java/org/apache/archiva/metadata/repository/file/FileMetadataRepositoryTest.java b/archiva-modules/plugins/metadata-store-file/src/test/java/org/apache/archiva/metadata/repository/file/FileMetadataRepositoryTest.java
index 41497ac..053b283 100644
--- a/archiva-modules/plugins/metadata-store-file/src/test/java/org/apache/archiva/metadata/repository/file/FileMetadataRepositoryTest.java
+++ b/archiva-modules/plugins/metadata-store-file/src/test/java/org/apache/archiva/metadata/repository/file/FileMetadataRepositoryTest.java
@@ -64,6 +64,8 @@ public class FileMetadataRepositoryTest
throws Exception
{
super.setUp();
+ assertMaxTries = 1;
+ assertRetrySleepMs = 10;
Path directory = Paths.get( "target/test-repositories" );
if (Files.exists(directory))
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
index 101b633..25b11de 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
@@ -815,7 +815,7 @@ public class JcrMetadataRepository
}
@Override
- public Stream<ArtifactMetadata> getArtifactsByDateRangeStream(RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException
+ public Stream<ArtifactMetadata> getArtifactByDateRangeStream( RepositorySession session, String repositoryId, ZonedDateTime startTime, ZonedDateTime endTime, QueryParameter queryParameter) throws MetadataRepositoryException
{
final Session jcrSession = getSession( session );
QueryResult result = queryArtifactByDateRange(jcrSession, repositoryId, startTime, endTime, queryParameter);