You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@archiva.apache.org by br...@apache.org on 2009/12/04 02:22:44 UTC
svn commit: r887027 - in
/archiva/trunk/archiva-modules/archiva-scheduled/src:
main/java/org/apache/maven/archiva/scheduled/executors/
test/java/org/apache/maven/archiva/scheduled/executors/
Author: brett
Date: Fri Dec 4 01:22:43 2009
New Revision: 887027
URL: http://svn.apache.org/viewvc?rev=887027&view=rev
Log:
[MRM-1294] avoid starting the search period at twice the duration since the start of the previous scan
Modified:
archiva/trunk/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java
archiva/trunk/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java
Modified: archiva/trunk/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java
URL: http://svn.apache.org/viewvc/archiva/trunk/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java?rev=887027&r1=887026&r2=887027&view=diff
==============================================================================
--- archiva/trunk/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java (original)
+++ archiva/trunk/archiva-modules/archiva-scheduled/src/main/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutor.java Fri Dec 4 01:22:43 2009
@@ -19,6 +19,10 @@
* under the License.
*/
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
@@ -46,29 +50,23 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
/**
- * ArchivaRepositoryScanningTaskExecutor
+ * ArchivaRepositoryScanningTaskExecutor
*
* @version $Id$
- *
- * @plexus.component
- * role="org.codehaus.plexus.taskqueue.execution.TaskExecutor"
- * role-hint="repository-scanning"
+ * @plexus.component role="org.codehaus.plexus.taskqueue.execution.TaskExecutor"
+ * role-hint="repository-scanning"
*/
public class ArchivaRepositoryScanningTaskExecutor
implements TaskExecutor, Initializable
{
private Logger log = LoggerFactory.getLogger( ArchivaRepositoryScanningTaskExecutor.class );
-
+
/**
* @plexus.requirement role-hint="jdo"
*/
private ArchivaDAO dao;
-
+
/**
* @plexus.requirement
*/
@@ -76,16 +74,16 @@
/**
* The repository scanner component.
- *
+ *
* @plexus.requirement
*/
private RepositoryScanner repoScanner;
-
+
/**
* @plexus.requirement
*/
private RepositoryContentConsumers consumers;
-
+
private Task task;
public void initialize()
@@ -99,18 +97,19 @@
throws TaskExecutionException
{
this.task = task;
-
+
RepositoryTask repoTask = (RepositoryTask) task;
-
+
if ( StringUtils.isBlank( repoTask.getRepositoryId() ) )
{
- throw new TaskExecutionException("Unable to execute RepositoryTask with blank repository Id.");
+ throw new TaskExecutionException( "Unable to execute RepositoryTask with blank repository Id." );
}
- ManagedRepositoryConfiguration arepo = archivaConfiguration.getConfiguration().findManagedRepositoryById( repoTask.getRepositoryId() );
-
+ ManagedRepositoryConfiguration arepo =
+ archivaConfiguration.getConfiguration().findManagedRepositoryById( repoTask.getRepositoryId() );
+
// execute consumers on resource file if set
- if( repoTask.getResourceFile() != null )
+ if ( repoTask.getResourceFile() != null )
{
log.debug( "Executing task from queue with job name: " + repoTask );
consumers.executeConsumers( arepo, repoTask.getResourceFile(), repoTask.isUpdateRelatedArtifacts() );
@@ -118,50 +117,50 @@
else
{
log.info( "Executing task from queue with job name: " + repoTask );
-
+
// otherwise, execute consumers on whole repository
try
- {
+ {
if ( arepo == null )
{
- throw new TaskExecutionException( "Unable to execute RepositoryTask with invalid repository id: " + repoTask.getRepositoryId() );
+ throw new TaskExecutionException(
+ "Unable to execute RepositoryTask with invalid repository id: " + repoTask.getRepositoryId() );
}
-
+
long sinceWhen = RepositoryScanner.FRESH_SCAN;
-
- List<RepositoryContentStatistics> results = (List<RepositoryContentStatistics>) dao.query( new MostRecentRepositoryScanStatistics( arepo.getId() ) );
-
+
+ List<RepositoryContentStatistics> results = (List<RepositoryContentStatistics>) dao.query(
+ new MostRecentRepositoryScanStatistics( arepo.getId() ) );
+
if ( CollectionUtils.isNotEmpty( results ) )
{
RepositoryContentStatistics lastStats = results.get( 0 );
- if( !repoTask.isScanAll() )
+ if ( !repoTask.isScanAll() )
{
- sinceWhen = lastStats.getWhenGathered().getTime() + lastStats.getDuration();
+ sinceWhen = lastStats.getWhenGathered().getTime();
}
}
-
+
RepositoryScanStatistics stats = repoScanner.scan( arepo, sinceWhen );
-
+
log.info( "Finished repository task: " + stats.toDump( arepo ) );
-
- RepositoryContentStatistics dbstats = constructRepositoryStatistics( arepo, sinceWhen, results, stats );
-
- dao.getRepositoryContentStatisticsDAO().saveRepositoryContentStatistics( dbstats );
-
+
+ RepositoryContentStatistics dbstats = constructRepositoryStatistics( arepo, stats );
+
+ dao.getRepositoryContentStatisticsDAO().saveRepositoryContentStatistics( dbstats );
+
this.task = null;
}
catch ( RepositoryException e )
- {
+ {
throw new TaskExecutionException( "Repository error when executing repository job.", e );
- }
+ }
}
}
@SuppressWarnings("unchecked")
private RepositoryContentStatistics constructRepositoryStatistics( ManagedRepositoryConfiguration arepo,
- long sinceWhen,
- List<RepositoryContentStatistics> results,
- RepositoryScanStatistics stats )
+ RepositoryScanStatistics stats )
{
// I hate jpox and modello <-- and so do I
RepositoryContentStatistics dbstats = new RepositoryContentStatistics();
@@ -170,12 +169,12 @@
dbstats.setRepositoryId( stats.getRepositoryId() );
dbstats.setTotalFileCount( stats.getTotalFileCount() );
dbstats.setWhenGathered( stats.getWhenGathered() );
-
+
// total artifact count
try
{
- List<ArchivaArtifact> artifacts = dao.getArtifactDAO().queryArtifacts(
- new ArtifactsByRepositoryConstraint( arepo.getId(), stats.getWhenGathered(), "groupId", true ) );
+ List<ArchivaArtifact> artifacts = dao.getArtifactDAO().queryArtifacts(
+ new ArtifactsByRepositoryConstraint( arepo.getId(), stats.getWhenGathered(), "groupId", true ) );
dbstats.setTotalArtifactCount( artifacts.size() );
}
catch ( ObjectNotFoundException oe )
@@ -183,27 +182,28 @@
log.error( "Object not found in the database : " + oe.getMessage() );
}
catch ( ArchivaDatabaseException ae )
- {
+ {
log.error( "Error occurred while querying artifacts for artifact count : " + ae.getMessage() );
}
-
+
// total repo size
long size = FileUtils.sizeOfDirectory( new File( arepo.getLocation() ) );
dbstats.setTotalSize( size );
-
- // total unique groups
+
+ // total unique groups
List<String> repos = new ArrayList<String>();
- repos.add( arepo.getId() );
-
+ repos.add( arepo.getId() );
+
List<String> groupIds = (List<String>) dao.query( new UniqueGroupIdConstraint( repos ) );
dbstats.setTotalGroupCount( groupIds.size() );
-
- List<Object[]> artifactIds = (List<Object[]>) dao.query( new UniqueArtifactIdConstraint( arepo.getId(), true ) );
+
+ List<Object[]> artifactIds =
+ (List<Object[]>) dao.query( new UniqueArtifactIdConstraint( arepo.getId(), true ) );
dbstats.setTotalProjectCount( artifactIds.size() );
-
+
return dbstats;
- }
-
+ }
+
public Task getCurrentTaskInExecution()
{
return task;
Modified: archiva/trunk/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java
URL: http://svn.apache.org/viewvc/archiva/trunk/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java?rev=887027&r1=887026&r2=887027&view=diff
==============================================================================
--- archiva/trunk/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java (original)
+++ archiva/trunk/archiva-modules/archiva-scheduled/src/test/java/org/apache/maven/archiva/scheduled/executors/ArchivaRepositoryScanningTaskExecutorTest.java Fri Dec 4 01:22:43 2009
@@ -264,6 +264,65 @@
assertEquals( 43687, updatedStats.getTotalSize() );
}
+ public void testExecutorScanOnlyNewArtifactsChangeTimes()
+ throws Exception
+ {
+ RepositoryTask repoTask = new RepositoryTask();
+
+ repoTask.setRepositoryId( TEST_REPO_ID );
+ repoTask.setScanAll( false );
+
+ RepositoryContentStatistics stats = new RepositoryContentStatistics();
+ stats.setDuration( 1234567 );
+ stats.setNewFileCount( 31 );
+ stats.setRepositoryId( TEST_REPO_ID );
+ stats.setTotalArtifactCount( 8 );
+ stats.setTotalFileCount( 31 );
+ stats.setTotalGroupCount( 3 );
+ stats.setTotalProjectCount( 5 );
+ stats.setTotalSize( 38545 );
+ stats.setWhenGathered( Calendar.getInstance().getTime() );
+
+ dao.getRepositoryContentStatisticsDAO().saveRepositoryContentStatistics( stats );
+
+ File newArtifactGroup = new File( repoDir, "org/apache/archiva" );
+
+ FileUtils.copyDirectoryStructure( new File( getBasedir(), "target/test-classes/test-repo/org/apache/archiva" ),
+ newArtifactGroup );
+
+ // update last modified date
+ new File( newArtifactGroup, "archiva-index-methods-jar-test/1.0/pom.xml" ).setLastModified(
+ Calendar.getInstance().getTimeInMillis() + 1000 );
+ new File( newArtifactGroup,
+ "archiva-index-methods-jar-test/1.0/archiva-index-methods-jar-test-1.0.jar" ).setLastModified(
+ Calendar.getInstance().getTimeInMillis() + 1000 );
+
+ assertTrue( newArtifactGroup.exists() );
+
+ // scan using the really long previous duration
+ taskExecutor.executeTask( repoTask );
+
+ // check no artifacts processed
+ ArtifactDAO adao = dao.getArtifactDAO();
+ List<ArchivaArtifact> unprocessedResultList = adao.queryArtifacts( new ArtifactsProcessedConstraint( false ) );
+ assertNotNull( unprocessedResultList );
+ assertEquals( "Incorrect number of unprocessed artifacts detected. One new artifact should have been found.", 1,
+ unprocessedResultList.size() );
+
+ // check correctness of new stats
+ List<RepositoryContentStatistics> results =
+ (List<RepositoryContentStatistics>) dao.query( new MostRecentRepositoryScanStatistics( TEST_REPO_ID ) );
+ RepositoryContentStatistics newStats = results.get( 0 );
+ assertEquals( 2, newStats.getNewFileCount() );
+ assertEquals( TEST_REPO_ID, newStats.getRepositoryId() );
+ assertEquals( 33, newStats.getTotalFileCount() );
+ // TODO: can't test these as they weren't stored in the database
+// assertEquals( 8, newStats.getTotalArtifactCount() );
+// assertEquals( 3, newStats.getTotalGroupCount() );
+// assertEquals( 5, newStats.getTotalProjectCount() );
+ assertEquals( 43687, newStats.getTotalSize() );
+ }
+
public void testExecutorForceScanAll()
throws Exception
{