You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@archiva.apache.org by ca...@apache.org on 2014/12/01 17:30:07 UTC
[2/4] archiva git commit: [MRM-1390] Implement search methods for
generic metadata and properties in Cassandra store
[MRM-1390] Implement search methods for generic metadata and properties in Cassandra store
Some caveats with Cassandra implementation:
We can't search artifacts by any (wildcard) property, so searchArtifacts(text,...) just calls getArtifactsByMetadata
The exact parameter is ignored as we can't do non exact searches in Cassandra
Project: http://git-wip-us.apache.org/repos/asf/archiva/repo
Commit: http://git-wip-us.apache.org/repos/asf/archiva/commit/4e5d1183
Tree: http://git-wip-us.apache.org/repos/asf/archiva/tree/4e5d1183
Diff: http://git-wip-us.apache.org/repos/asf/archiva/diff/4e5d1183
Branch: refs/heads/master
Commit: 4e5d1183c31651680e8266198f41c4464647b28d
Parents: e4da1fa
Author: Carlos Sanchez <ca...@apache.org>
Authored: Sun Nov 23 11:34:24 2014 +0100
Committer: Carlos Sanchez <ca...@apache.org>
Committed: Mon Dec 1 16:48:31 2014 +0100
----------------------------------------------------------------------
.../cassandra/CassandraMetadataRepository.java | 141 +++++++++++++++----
.../repository/cassandra/CassandraUtils.java | 8 +-
.../cassandra/model/MetadataFacetModel.java | 5 +
.../CassandraMetadataRepositoryTest.java | 98 -------------
.../repository/jcr/JcrMetadataRepository.java | 3 +
5 files changed, 127 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/archiva/blob/4e5d1183/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
----------------------------------------------------------------------
diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
index 779c19f..268fd15 100644
--- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepository.java
@@ -77,6 +77,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -1826,25 +1827,114 @@ public class CassandraMetadataRepository
return artifactMetadataMap.values();
}
+ /**
+ * Project version and artifact level metadata are stored in the same place, no distinctions in Cassandra
+ * implementation, just calls {@link #getArtifactsByMetadata(String, String, String)}
+ */
@Override
public List<ArtifactMetadata> getArtifactsByProjectVersionMetadata( String key, String value, String repositoryId )
throws MetadataRepositoryException
{
- throw new UnsupportedOperationException( "not yet implemented in Cassandra backend" );
+ return getArtifactsByMetadata( key, value, repositoryId );
}
@Override
public List<ArtifactMetadata> getArtifactsByMetadata( String key, String value, String repositoryId )
throws MetadataRepositoryException
{
- throw new UnsupportedOperationException( "not yet implemented in Cassandra backend" );
+ RangeSlicesQuery<String, String, String> query =
+ HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
+ .setColumnFamily( cassandraArchivaManager.getMetadataFacetFamilyName() ) //
+ .setColumnNames( MetadataFacetModel.COLUMNS ) //
+ .addEqualsExpression( VALUE.toString(), value );
+
+ if ( key != null )
+ {
+ query.addEqualsExpression( KEY.toString(), key ); //
+ }
+ if ( repositoryId != null )
+ {
+ query.addEqualsExpression( "repositoryName", repositoryId );
+ }
+
+ QueryResult<OrderedRows<String, String, String>> metadataFacetResult = query.execute();
+ if ( metadataFacetResult.get() == null || metadataFacetResult.get().getCount() < 1 )
+ {
+ return Collections.emptyList();
+ }
+
+ List<ArtifactMetadata> artifactMetadatas = new LinkedList<ArtifactMetadata>();
+
+ // TODO doing multiple queries, there should be a way to get all the artifactMetadatas for any number of
+ // projects
+ for ( Row<String, String, String> row : metadataFacetResult.get() )
+ {
+ QueryResult<OrderedRows<String, String, String>> artifactMetadataResult =
+ HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
+ .setColumnFamily( cassandraArchivaManager.getArtifactMetadataFamilyName() ) //
+ .setColumnNames( ArtifactMetadataModel.COLUMNS ) //
+ .setRowCount( Integer.MAX_VALUE ) //
+ .addEqualsExpression( REPOSITORY_NAME.toString(),
+ getStringValue( row.getColumnSlice(), REPOSITORY_NAME ) ) //
+ .addEqualsExpression( NAMESPACE_ID.toString(), getStringValue( row.getColumnSlice(), NAMESPACE_ID ) ) //
+ .addEqualsExpression( PROJECT.toString(), getStringValue( row.getColumnSlice(), PROJECT_ID ) ) //
+ .addEqualsExpression( PROJECT_VERSION.toString(),
+ getStringValue( row.getColumnSlice(), PROJECT_VERSION ) ) //
+ .execute();
+
+ if ( artifactMetadataResult.get() == null || artifactMetadataResult.get().getCount() < 1 )
+ {
+ return Collections.emptyList();
+ }
+
+ for ( Row<String, String, String> artifactMetadataRow : artifactMetadataResult.get() )
+ {
+ artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( artifactMetadataRow.getColumnSlice() ) );
+ }
+ }
+
+ return mapArtifactMetadataToArtifact( metadataFacetResult, artifactMetadatas );
}
@Override
public List<ArtifactMetadata> getArtifactsByProperty( String key, String value, String repositoryId )
throws MetadataRepositoryException
{
- throw new UnsupportedOperationException( "getArtifactsByProperty not yet implemented in Cassandra backend" );
+ QueryResult<OrderedRows<String, String, String>> result =
+ HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
+ .setColumnFamily( cassandraArchivaManager.getProjectVersionMetadataFamilyName() ) //
+ .setColumnNames( PROJECT_ID.toString(), REPOSITORY_NAME.toString(), NAMESPACE_ID.toString(),
+ PROJECT_VERSION.toString() ) //
+ .addEqualsExpression( key, value ) //
+ .execute();
+
+ int count = result.get().getCount();
+
+ if ( count < 1 )
+ {
+ return Collections.emptyList();
+ }
+
+ List<ArtifactMetadata> artifacts = new LinkedList<ArtifactMetadata>();
+
+ for ( Row<String, String, String> row : result.get() )
+ {
+ // TODO doing multiple queries, there should be a way to get all the artifactMetadatas for any number of
+ // projects
+ try
+ {
+ artifacts.addAll( getArtifacts( getStringValue( row.getColumnSlice(), REPOSITORY_NAME ),
+ getStringValue( row.getColumnSlice(), NAMESPACE_ID ),
+ getStringValue( row.getColumnSlice(), PROJECT_ID ),
+ getStringValue( row.getColumnSlice(), PROJECT_VERSION ) ) );
+ }
+ catch ( MetadataResolutionException e )
+ {
+ // never raised
+ throw new IllegalStateException( e );
+ }
+ }
+ return artifacts;
}
@Override
@@ -2046,29 +2136,12 @@ public class CassandraMetadataRepository
for ( Row<String, String, String> row : result.get() )
{
- ColumnSlice<String, String> columnSlice = row.getColumnSlice();
- ArtifactMetadata artifactMetadata = new ArtifactMetadata();
- artifactMetadata.setNamespace( getStringValue( columnSlice, NAMESPACE_ID.toString() ) );
- artifactMetadata.setSize( getAsLongValue( columnSlice, SIZE.toString() ) );
- artifactMetadata.setId( getStringValue( columnSlice, ID.toString() ) );
- artifactMetadata.setFileLastModified( getAsLongValue( columnSlice, FILE_LAST_MODIFIED.toString() ) );
- artifactMetadata.setMd5( getStringValue( columnSlice, MD5.toString() ) );
- artifactMetadata.setProject( getStringValue( columnSlice, PROJECT.toString() ) );
- artifactMetadata.setProjectVersion( getStringValue( columnSlice, PROJECT_VERSION.toString() ) );
- artifactMetadata.setRepositoryId( repoId );
- artifactMetadata.setSha1( getStringValue( columnSlice, SHA1.toString() ) );
- artifactMetadata.setVersion( getStringValue( columnSlice, VERSION.toString() ) );
- Long whenGathered = getAsLongValue( columnSlice, WHEN_GATHERED.toString() );
- if ( whenGathered != null )
- {
- artifactMetadata.setWhenGathered( new Date( whenGathered ) );
- }
- artifactMetadatas.add( artifactMetadata );
+ artifactMetadatas.add( mapArtifactMetadataStringColumnSlice( row.getColumnSlice() ) );
}
result = HFactory.createRangeSlicesQuery( keyspace, ss, ss, ss ) //
.setColumnFamily( cassandraArchivaManager.getMetadataFacetFamilyName() ) //
- .setColumnNames( FACET_ID.toString(), NAME.toString(), VALUE.toString(), KEY.toString(), PROJECT_VERSION.toString() ) //
+ .setColumnNames( MetadataFacetModel.COLUMNS ) //
.setRowCount( Integer.MAX_VALUE ) //
.addEqualsExpression( REPOSITORY_NAME.toString(), repoId ) //
.addEqualsExpression( NAMESPACE_ID.toString(), namespace ) //
@@ -2076,6 +2149,13 @@ public class CassandraMetadataRepository
.addEqualsExpression( PROJECT_VERSION.toString(), projectVersion ) //
.execute();
+ return mapArtifactMetadataToArtifact(result, artifactMetadatas);
+ }
+
+ /**
+ * Attach metadata to each of the ArtifactMetadata objects
+ */
+ private List<ArtifactMetadata> mapArtifactMetadataToArtifact(QueryResult<OrderedRows<String, String, String>> result, List<ArtifactMetadata> artifactMetadatas) {
if ( result.get() == null || result.get().getCount() < 1 )
{
return artifactMetadatas;
@@ -2147,8 +2227,6 @@ public class CassandraMetadataRepository
}
}
}
-
-
}
return artifactMetadatas;
@@ -2198,17 +2276,28 @@ public class CassandraMetadataRepository
return ModelMapperHolder.MODEL_MAPPER;
}
+ /**
+ * This implementation just calls getArtifactsByMetadata( null, text, repositoryId ). We can't search artifacts by
+ * any property.
+ */
@Override
public List<ArtifactMetadata> searchArtifacts( String text, String repositoryId, boolean exact )
throws MetadataRepositoryException
{
- throw new UnsupportedOperationException( "searchArtifacts not yet implemented in Cassandra backend" );
+ return getArtifactsByMetadata( null, text, repositoryId );
}
+ /**
+ * The exact parameter is ignored as we can't do non exact searches in Cassandra
+ */
@Override
public List<ArtifactMetadata> searchArtifacts( String key, String text, String repositoryId, boolean exact )
throws MetadataRepositoryException
{
- throw new UnsupportedOperationException( "searchArtifacts not yet implemented in Cassandra backend" );
+ // TODO optimize
+ List<ArtifactMetadata> artifacts = new LinkedList<ArtifactMetadata>();
+ artifacts.addAll( getArtifactsByMetadata( key, text, repositoryId ) );
+ artifacts.addAll( getArtifactsByProperty( key, text, repositoryId ) );
+ return artifacts;
}
}
http://git-wip-us.apache.org/repos/asf/archiva/blob/4e5d1183/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java
----------------------------------------------------------------------
diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java
index 6366fe9..f3de42b 100644
--- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java
+++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/CassandraUtils.java
@@ -95,18 +95,18 @@ public class CassandraUtils
return hColumn == null ? null : hColumn.getValue();
}
- public static Long getLongValue( ColumnSlice<String, Long> columnSlice, String columnName )
+ public static Long getLongValue( ColumnSlice<String, ?> columnSlice, String columnName )
{
if ( StringUtils.isEmpty( columnName ) )
{
return null;
}
- HColumn<String, Long> hColumn = columnSlice.getColumnByName( columnName );
+ HColumn<String, Long> hColumn = (HColumn<String, Long>) columnSlice.getColumnByName( columnName );
return hColumn == null ? null : hColumn.getValue();
}
- public static String getAsStringValue( ColumnSlice<String, Long> columnSlice, String columnName )
+ public static <T> String getAsStringValue( ColumnSlice<String, T> columnSlice, String columnName )
{
StringSerializer ss = StringSerializer.get();
if ( StringUtils.isEmpty( columnName ) )
@@ -114,7 +114,7 @@ public class CassandraUtils
return null;
}
- HColumn<String, Long> hColumn = columnSlice.getColumnByName( columnName );
+ HColumn<String, T> hColumn = columnSlice.getColumnByName( columnName );
return hColumn == null ? null : ss.fromByteBuffer( hColumn.getValueBytes() );
}
http://git-wip-us.apache.org/repos/asf/archiva/blob/4e5d1183/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java
----------------------------------------------------------------------
diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java
index ae494e1..a2347bf 100644
--- a/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java
+++ b/archiva-modules/plugins/metadata-store-cassandra/src/main/java/org/apache/archiva/metadata/repository/cassandra/model/MetadataFacetModel.java
@@ -19,6 +19,8 @@ package org.apache.archiva.metadata.repository.cassandra.model;
* under the License.
*/
+import static org.apache.archiva.metadata.repository.cassandra.model.ColumnNames.*;
+
import org.apache.archiva.metadata.repository.cassandra.CassandraUtils;
/**
@@ -29,6 +31,9 @@ import org.apache.archiva.metadata.repository.cassandra.CassandraUtils;
*/
public class MetadataFacetModel
{
+ public static final String[] COLUMNS = new String[] { FACET_ID.toString(), KEY.toString(), VALUE.toString(),
+ REPOSITORY_NAME.toString(), NAMESPACE_ID.toString(), PROJECT_ID.toString(), PROJECT_VERSION.toString() };
+
private String facetId;
private String key;
http://git-wip-us.apache.org/repos/asf/archiva/blob/4e5d1183/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
index 97b3718..b4f0cdb 100644
--- a/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
+++ b/archiva-modules/plugins/metadata-store-cassandra/src/test/java/org/apache/archiva/metadata/repository/cassandra/CassandraMetadataRepositoryTest.java
@@ -25,7 +25,6 @@ import org.apache.archiva.metadata.repository.cassandra.model.ProjectVersionMeta
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import javax.inject.Inject;
@@ -69,103 +68,6 @@ public class CassandraMetadataRepositoryTest
clearReposAndNamespace( cassandraArchivaManager );
}
- @Override
- @Ignore
- public void testGetArtifactsByProjectVersionMetadata()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testGetArtifactsByProjectVersionMetadataNoRepository()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testGetArtifactsByProjectVersionMetadataAllRepositories()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testGetArtifactsByMetadataAllRepositories()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testGetArtifactsByPropertySingleResult()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testSearchArtifactsByKey()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testSearchArtifactsByKeyExact()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testSearchArtifactsFullText()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testSearchArtifactsFullTextExact()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testSearchArtifactsByFacetKeyAllRepos()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testSearchArtifactsByFacetKey()
- throws Exception
- {
- // TODO not implemented
- }
-
- @Override
- @Ignore
- public void testSearchArtifactsFullTextByFacet()
- throws Exception
- {
- // TODO not implemented
- }
-
-
/**
* ensure all dependant tables are cleaned up (mailinglist, license, dependencies)
*
http://git-wip-us.apache.org/repos/asf/archiva/blob/4e5d1183/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
----------------------------------------------------------------------
diff --git a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
index d4c79b6..8fecba8 100644
--- a/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
+++ b/archiva-modules/plugins/metadata-store-jcr/src/main/java/org/apache/archiva/metadata/repository/jcr/JcrMetadataRepository.java
@@ -1403,6 +1403,9 @@ public class JcrMetadataRepository
}
+ /**
+ * Exact is ignored as we can't do exact search in any property, we need a key
+ */
@Override
public List<ArtifactMetadata> searchArtifacts( String text, String repositoryId, boolean exact )
throws MetadataRepositoryException