You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ace.apache.org by an...@apache.org on 2013/10/31 21:43:15 UTC
svn commit: r1537630 - in /ace/trunk:
org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/
org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/
org.apache.ace.gogo/ org.apache.ace.gogo/src/org/apache/ace/go...
Author: angelos
Date: Thu Oct 31 20:43:14 2013
New Revision: 1537630
URL: http://svn.apache.org/r1537630
Log:
ACE-378 We can now limit server logs on file-based log stores.
Added:
ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/
ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java
ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg
ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg
Modified:
ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java
ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java
ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java
ace/trunk/org.apache.ace.gogo/bnd.bnd
ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java
ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java
ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java
ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java
ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java
ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java
ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java
ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java
ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java
ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java
Modified: ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java (original)
+++ ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java Thu Oct 31 20:43:14 2013
@@ -129,6 +129,8 @@ public class LogAuthenticationTest exten
configure("org.apache.ace.scheduler",
"org.apache.ace.configurator.useradmin.task.UpdateUserAdminTask", "100");
+ configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+
configure(DiscoveryConstants.DISCOVERY_PID,
DiscoveryConstants.DISCOVERY_URL_KEY, baseURL);
configure(IdentificationConstants.IDENTIFICATION_PID,
Modified: ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java (original)
+++ ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java Thu Oct 31 20:43:14 2013
@@ -97,6 +97,8 @@ public class ObrAuthenticationTest exten
RepositoryConstants.REPOSITORY_CUSTOMER, "apache",
RepositoryConstants.REPOSITORY_MASTER, "true");
+ configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+
configure("org.apache.ace.configurator.useradmin.task.UpdateUserAdminTask",
"repositoryName", "users",
"repositoryCustomer", "apache");
Modified: ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java (original)
+++ ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java Thu Oct 31 20:43:14 2013
@@ -179,6 +179,8 @@ public abstract class BaseRepositoryAdmi
getService(SessionFactory.class).createSession("test-session-ID", null);
+ configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+
configureFactory("org.apache.ace.log.server.store.factory",
"name", "auditlog", "authentication.enabled", "false");
}
Modified: ace/trunk/org.apache.ace.gogo/bnd.bnd
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.gogo/bnd.bnd?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.gogo/bnd.bnd (original)
+++ ace/trunk/org.apache.ace.gogo/bnd.bnd Thu Oct 31 20:43:14 2013
@@ -4,7 +4,8 @@
org.osgi.impl.bundle.repoindex.lib,\
org.apache.felix.dependencymanager,\
org.apache.felix.gogo.runtime,\
- org.apache.ace.bnd.repository;version=latest
+ org.apache.ace.bnd.repository;version=latest,\
+ org.apache.ace.log.server.store.api;version=latest
Bundle-Name: Apache ACE Gogo commands
Bundle-Description: Provides Gogo commands for working with ACE
@@ -75,5 +76,6 @@ Private-Package: org.apache.ace.bnd.repo
org.osgi.impl.bundle.bindex.*;-split-package:=merge-last,\
org.osgi.impl.bundle.obr.*;-split-package:=merge-last,\
org.xmlpull.v1;-split-package:=first,\
- org.kxml2.io;-split-package:=first
+ org.kxml2.io;-split-package:=first,\
+ org.apache.ace.gogo.log
\ No newline at end of file
Modified: ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java (original)
+++ ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java Thu Oct 31 20:43:14 2013
@@ -22,10 +22,12 @@ import java.util.Properties;
import org.apache.ace.gogo.execute.ExecuteCommands;
import org.apache.ace.gogo.execute.ScriptExecutor;
+import org.apache.ace.gogo.log.LogCommands;
import org.apache.ace.gogo.math.MathCommands;
import org.apache.ace.gogo.misc.MiscCommands;
import org.apache.ace.gogo.queue.QueueCommands;
import org.apache.ace.gogo.repo.RepoCommands;
+import org.apache.ace.log.server.store.LogStore;
import org.apache.felix.dm.DependencyActivatorBase;
import org.apache.felix.dm.DependencyManager;
import org.apache.felix.service.command.CommandProcessor;
@@ -63,6 +65,13 @@ public class Activator extends Dependenc
.setService(CommandProcessor.class)
.setRequired(true)));
+ manager.add(createComponent()
+ .setInterface(Object.class.getName(), createProps(LogCommands.SCOPE, LogCommands.FUNCTIONS))
+ .setImplementation(LogCommands.class)
+ .add(createServiceDependency()
+ .setService(LogStore.class)
+ .setRequired(true)));
+
String script = System.getProperty("ace.gogo.script");
if (script != null) {
long delay = Long.getLong("ace.gogo.script.delay", 300L);
Added: ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java?rev=1537630&view=auto
==============================================================================
--- ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java (added)
+++ ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java Thu Oct 31 20:43:14 2013
@@ -0,0 +1,20 @@
+package org.apache.ace.gogo.log;
+
+import org.apache.ace.log.server.store.LogStore;
+import org.apache.felix.service.command.Descriptor;
+
+public class LogCommands {
+
+ public final static String SCOPE = "ace-log";
+ public final static String[] FUNCTIONS = new String[] { "cleanup" };
+
+ // Injected by Felix DM...
+ private volatile LogStore m_logStore;
+
+ @Descriptor("Apply the configured maximum to all existing logs")
+ public void cleanup() throws Exception {
+ m_logStore.clean();
+ System.out.println("All logfiles processed");
+ }
+
+}
Modified: ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java (original)
+++ ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java Thu Oct 31 20:43:14 2013
@@ -77,6 +77,8 @@ public class LogIntegrationTest extends
configure("org.apache.ace.deployment.servlet",
HttpConstants.ENDPOINT, DEPLOYMENT, "authentication.enabled", "false");
+ configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+
configureFactory("org.apache.ace.log.server.servlet.factory",
"name", "auditlog",
HttpConstants.ENDPOINT, AUDITLOG, "authentication.enabled", "false");
Modified: ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java (original)
+++ ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java Thu Oct 31 20:43:14 2013
@@ -165,6 +165,8 @@ public class MongoLogStoreTest extends I
@Override
protected void configureProvisionedServices() throws Exception {
+ configure("org.apache.ace.log.server.store.mongo", "MaxEvents", "0");
+
configureFactory("org.amdatu.mongo", "dbName", "ace");
configureFactory("org.apache.ace.log.server.store.factory", "name", "serverlog");
}
Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java (original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java Thu Oct 31 20:43:14 2013
@@ -90,4 +90,12 @@ public interface LogStore
* @throws java.io.IOException in case of any error.
*/
public List<Descriptor> getDescriptors() throws IOException;
+
+ /**
+ * Cleanup the events in the store. This method will check each target and log in this store and remove all
+ * events exceeding the maximum number of events that can be configured for this store.
+ *
+ * @throws IOException in case of any error
+ */
+ public void clean() throws IOException;
}
\ No newline at end of file
Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java (original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java Thu Oct 31 20:43:14 2013
@@ -36,6 +36,7 @@ import org.osgi.service.event.EventAdmin
import org.osgi.service.log.LogService;
public class Activator extends DependencyActivatorBase implements ManagedServiceFactory {
+ public static final String PID = "org.apache.ace.log.server.store.filebased";
private static final String LOG_NAME = "name";
private DependencyManager m_manager;
@@ -94,7 +95,8 @@ public class Activator extends Dependenc
service = m_manager.createComponent()
.setInterface(LogStore.class.getName(), props)
.setImplementation(new LogStoreImpl(baseDir, name))
- .add(createServiceDependency().setService(EventAdmin.class).setRequired(false));
+ .add(createServiceDependency().setService(EventAdmin.class).setRequired(false))
+ .add(createConfigurationDependency().setPid(PID));
m_instances.put(pid, service);
m_manager.add(service);
} else {
Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java (original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java Thu Oct 31 20:43:14 2013
@@ -29,27 +29,39 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Dictionary;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.ace.feedback.Descriptor;
import org.apache.ace.feedback.Event;
import org.apache.ace.log.server.store.LogStore;
import org.apache.ace.range.Range;
import org.apache.ace.range.SortedRangeSet;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.cm.ManagedService;
import org.osgi.service.event.EventAdmin;
/**
* A simple implementation of the LogStore interface.
*/
-public class LogStoreImpl implements LogStore {
+public class LogStoreImpl implements LogStore, ManagedService {
+
+ private static final String MAXIMUM_NUMBER_OF_EVENTS = "MaxEvents";
private volatile EventAdmin m_eventAdmin; /* Injected by dependency manager */
// the dir to store logs in - init is in the start method
private final File m_dir;
private final String m_name;
+ private int m_maxEvents = 0;
+
+ private final ConcurrentMap<String, Set<Long>> m_locks = new ConcurrentHashMap<String, Set<Long>>();
+ private final Map<String, Long> m_fileToID = new HashMap<String, Long>();
public LogStoreImpl(File baseDir, String name) {
m_name = name;
@@ -57,8 +69,7 @@ public class LogStoreImpl implements Log
}
/*
- * init the dir in which to store logs in - thows IllegalArgumentException
- * if we can't get it.
+ * init the dir in which to store logs in - thows IllegalArgumentException if we can't get it.
*/
protected void start() throws IOException {
if (!m_dir.isDirectory() && !m_dir.mkdirs()) {
@@ -69,15 +80,35 @@ public class LogStoreImpl implements Log
/**
* @see org.apache.ace.log.server.store.LogStore#get(org.apache.ace.log.Descriptor)
*/
- public synchronized List<Event> get(Descriptor descriptor)
- throws IOException {
+ public List<Event> get(Descriptor descriptor)
+ throws IOException {
+ try {
+ obtainLock(descriptor.getTargetID(), descriptor.getStoreID());
+ return getInternal(descriptor);
+ }
+ finally {
+ releaseLock(descriptor.getTargetID(), descriptor.getStoreID());
+ }
+ }
+
+ /**
+ * Retrieve the events that match the given descriptor. This method relies on external locking, the caller should
+ * take care of that.
+ *
+ * @param descriptor
+ * the events to retrieve
+ * @return the events that match
+ * @throws IOException
+ * if anything goes wrong
+ */
+ private List<Event> getInternal(Descriptor descriptor) throws IOException {
final List<Event> result = new ArrayList<Event>();
final SortedRangeSet set = descriptor.getRangeSet();
BufferedReader in = null;
try {
File log = new File(new File(m_dir,
- targetIDToFilename(descriptor.getTargetID())),
- String.valueOf(descriptor.getStoreID()));
+ targetIDToFilename(descriptor.getTargetID())),
+ String.valueOf(descriptor.getStoreID()));
if (!log.isFile()) {
return result;
}
@@ -85,12 +116,13 @@ public class LogStoreImpl implements Log
String file = log.getAbsolutePath();
long counter = 0;
for (String line = in.readLine(); line != null; line = in
- .readLine()) {
+ .readLine()) {
Event event = new Event(line);
long id = event.getID();
if ((counter != -1) && ++counter == id) {
- } else {
+ }
+ else {
counter = -1;
}
if (set.contains(id)) {
@@ -99,15 +131,16 @@ public class LogStoreImpl implements Log
}
if (counter < 1) {
m_fileToID.remove(file);
- } else {
+ }
+ else {
m_fileToID.put(file, counter);
}
- }
+ }
finally {
if (in != null) {
try {
in.close();
- }
+ }
catch (Exception ex) {
// Not much we can do
}
@@ -116,23 +149,21 @@ public class LogStoreImpl implements Log
return result;
}
- private final Map<String, Long> m_fileToID = new HashMap<String, Long>();
-
/**
* @see org.apache.ace.log.server.store.LogStore#getDescriptor(String, long)
*/
- public synchronized Descriptor getDescriptor(String targetID, long logID)
- throws IOException {
+ public Descriptor getDescriptor(String targetID, long logID)
+ throws IOException {
Long high = m_fileToID.get(new File(new File(m_dir,
- targetIDToFilename(targetID)), String.valueOf(logID))
- .getAbsolutePath());
+ targetIDToFilename(targetID)), String.valueOf(logID))
+ .getAbsolutePath());
if (high != null) {
Range r = new Range(1, high);
return new Descriptor(targetID, logID, new SortedRangeSet(
- r.toRepresentation()));
+ r.toRepresentation()));
}
List<Event> events = get(new Descriptor(targetID, logID,
- SortedRangeSet.FULL_SET));
+ SortedRangeSet.FULL_SET));
long[] idsArray = new long[events.size()];
int i = 0;
@@ -146,7 +177,7 @@ public class LogStoreImpl implements Log
* @see org.apache.ace.log.server.store.LogStore#getDescriptors(String)
*/
public List<Descriptor> getDescriptors(String targetID)
- throws IOException {
+ throws IOException {
File dir = new File(m_dir, targetIDToFilename(targetID));
List<Descriptor> result = new ArrayList<Descriptor>();
if (!dir.isDirectory()) {
@@ -178,13 +209,20 @@ public class LogStoreImpl implements Log
Map<String, Map<Long, List<Event>>> sorted = sort(events);
for (String targetID : sorted.keySet()) {
for (Long logID : sorted.get(targetID).keySet()) {
- put(targetID, logID, sorted.get(targetID).get(logID));
+ try {
+ obtainLock(targetID, logID);
+ put(targetID, logID, sorted.get(targetID).get(logID));
+ }
+ finally {
+ releaseLock(targetID, logID);
+ }
}
}
}
/**
- * Add a list of events to the log of the given ids.
+ * Add a list of events to the log of the given ids. This method relies on external locking, the caller should take
+ * care of that.
*
* @param targetID
* the id of the target to append to its log.
@@ -195,8 +233,8 @@ public class LogStoreImpl implements Log
* @throws java.io.IOException
* in case of any error.
*/
- protected synchronized void put(String targetID, Long logID,
- List<Event> list) throws IOException {
+ protected void put(String targetID, Long logID,
+ List<Event> list) throws IOException {
if ((list == null) || (list.size() == 0)) {
// nothing to add, so return
return;
@@ -206,7 +244,7 @@ public class LogStoreImpl implements Log
// 2. we need to insert events in the existing file (meaning we have to
// rewrite basically the whole file)
String file = new File(new File(m_dir, targetIDToFilename(targetID)),
- String.valueOf(logID)).getAbsolutePath();
+ String.valueOf(logID)).getAbsolutePath();
Long highest = m_fileToID.get(file);
boolean cached = false;
if (highest != null) {
@@ -216,15 +254,20 @@ public class LogStoreImpl implements Log
}
List<Event> events = null;
if (!cached) {
- events = get(new Descriptor(targetID, logID,
- SortedRangeSet.FULL_SET));
+ events = getInternal(new Descriptor(targetID, logID,
+ SortedRangeSet.FULL_SET));
// remove duplicates first
list.removeAll(events);
}
- if (list.size() == 0) {
- // nothing to add anymore, so return
+ boolean removeEvents = false;
+ if (m_maxEvents > 0 && m_maxEvents < list.size() + events.size()) {
+ removeEvents = true;
+ }
+
+ if (!removeEvents && list.size() == 0) {
+ // nothing to add or remove anymore, so return
return;
}
@@ -234,26 +277,34 @@ public class LogStoreImpl implements Log
if (!dir.isDirectory() && !dir.mkdirs()) {
throw new IOException("Unable to create backup store.");
}
- if (cached
- || ((events.size() == 0) || (events.get(events.size() - 1)
- .getID() < list.get(0).getID()))) {
- // we can append to the existing file
+ if (!removeEvents && (cached
+ || ((events.size() == 0) || (events.get(events.size() - 1)
+ .getID() < list.get(0).getID())))) {
+ // we can append to the existing file without need to remove records
out = new PrintWriter(new FileWriter(new File(dir,
- logID.toString()), true));
- } else {
+ logID.toString()), true));
+ }
+ else {
// we have to merge the lists
list.addAll(events);
// and sort
Collections.sort(list);
+ // and remove if necessary
+ for (int i = 0; i < m_maxEvents; i++) {
+ if (list.size() > 0) {
+ list.remove(0);
+ }
+ }
out = new PrintWriter(new FileWriter(new File(dir,
- logID.toString())));
+ logID.toString())));
}
long high = 0;
for (Event event : list) {
out.println(event.toRepresentation());
if (high < event.getID()) {
high = event.getID();
- } else {
+ }
+ else {
high = Long.MAX_VALUE;
}
// send (eventadmin)event about a new (log)event being stored
@@ -264,14 +315,15 @@ public class LogStoreImpl implements Log
}
if ((cached) && (high < Long.MAX_VALUE)) {
m_fileToID.put(file, new Long(high));
- } else {
+ }
+ else {
m_fileToID.remove(file);
}
- }
+ }
finally {
try {
out.close();
- }
+ }
catch (Exception ex) {
// Not much we can do
}
@@ -279,20 +331,18 @@ public class LogStoreImpl implements Log
}
/**
- * Sort the given list of events into a map of maps according to the
- * targetID and the logID of each event.
+ * Sort the given list of events into a map of maps according to the targetID and the logID of each event.
*
* @param events
* a list of events to sort.
- * @return a map of maps that maps target ids to a map that maps log ids to
- * a list of events that have those ids.
+ * @return a map of maps that maps target ids to a map that maps log ids to a list of events that have those ids.
*/
@SuppressWarnings("boxing")
protected Map<String, Map<Long, List<Event>>> sort(List<Event> events) {
Map<String, Map<Long, List<Event>>> result = new HashMap<String, Map<Long, List<Event>>>();
for (Event event : events) {
Map<Long, List<Event>> target = result
- .get(event.getTargetID());
+ .get(event.getTargetID());
if (target == null) {
target = new HashMap<Long, List<Event>>();
@@ -316,7 +366,7 @@ public class LogStoreImpl implements Log
private <T> T notNull(T target) throws IOException {
if (target == null) {
throw new IOException(
- "Unknown IO error while trying to access the store.");
+ "Unknown IO error while trying to access the store.");
}
return target;
}
@@ -331,7 +381,7 @@ public class LogStoreImpl implements Log
String result = null;
try {
result = new String(bytes, "UTF-8");
- }
+ }
catch (UnsupportedEncodingException e) {
// UTF-8 is a mandatory encoding; this will never happen.
}
@@ -346,15 +396,124 @@ public class LogStoreImpl implements Log
String hexValue = Integer.toHexString(b.intValue());
if (hexValue.length() % 2 == 0) {
result.append(hexValue);
- } else {
+ }
+ else {
result.append('0').append(hexValue);
}
}
- }
+ }
catch (UnsupportedEncodingException e) {
// UTF-8 is a mandatory encoding; this will never happen.
}
return result.toString();
}
-}
\ No newline at end of file
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void updated(Dictionary settings) throws ConfigurationException {
+ if (settings != null) {
+ String maximumNumberOfEvents = (String) settings.get(MAXIMUM_NUMBER_OF_EVENTS);
+ if (maximumNumberOfEvents != null) {
+ try {
+ m_maxEvents = Integer.parseInt(maximumNumberOfEvents);
+ }
+ catch (NumberFormatException nfe) {
+ throw new ConfigurationException(MAXIMUM_NUMBER_OF_EVENTS, "is not a number");
+ }
+ }
+ }
+ }
+
+ @Override
+ public void clean() throws IOException {
+ // check if we event might have to cleanup anything
+ if (m_maxEvents <= 0) {
+ return;
+ }
+ // create a list of unique targets and their logs
+ Map<String, Set<Long>> allTargetsAndLogs = new HashMap<String, Set<Long>>();
+ for (Descriptor descriptor : getDescriptors()) {
+ Set<Long> logs = allTargetsAndLogs.get(descriptor.getTargetID());
+ if (logs == null) {
+ logs = new HashSet<Long>();
+ allTargetsAndLogs.put(descriptor.getTargetID(), logs);
+ }
+ logs.add(descriptor.getStoreID());
+ }
+
+ // cleanup per log
+ for (String targetID : allTargetsAndLogs.keySet()) {
+ for (Long logId : allTargetsAndLogs.get(targetID)) {
+ clean(targetID, logId);
+ }
+ }
+ }
+
+ private void clean(String targetID, Long logID) throws IOException {
+ try {
+ obtainLock(targetID, logID);
+ List<Event> events = getInternal(new Descriptor(targetID, logID, SortedRangeSet.FULL_SET));
+ if (events.size() > m_maxEvents) {
+ for (int i = 0; i < m_maxEvents; i++) {
+ if (events.size() > 0) {
+ events.remove(0);
+ }
+ }
+ }
+ put(targetID, logID, events);
+ }
+ finally {
+ releaseLock(targetID, logID);
+ }
+ }
+
+ private void obtainLock(String targetID, long logID) throws IOException {
+ Set<Long> newLockedLogs = new HashSet<Long>();
+ Set<Long> lockedLogs = m_locks.putIfAbsent(targetID, newLockedLogs);
+ if (lockedLogs == null) {
+ lockedLogs = newLockedLogs;
+ }
+ boolean alreadyLocked;
+
+ synchronized (lockedLogs) {
+ alreadyLocked = lockedLogs.contains(logID);
+ if (!alreadyLocked) {
+ // lock it now, we are working on it
+ lockedLogs.add(logID);
+ }
+ }
+
+ // try to obtain the lock if we could not lock it on the first try
+ if (alreadyLocked) {
+ int nrOfTries = 1;
+ while (alreadyLocked && nrOfTries < 10) {
+ try {
+ Thread.sleep(20);
+ }
+ catch (InterruptedException e) {
+ break;
+ }
+ nrOfTries++;
+ synchronized (lockedLogs) {
+ alreadyLocked = lockedLogs.contains(logID);
+ if (!alreadyLocked) {
+ // lock it now, we are working on it
+ lockedLogs.add(logID);
+ }
+ }
+ }
+ // if the log is still locked throw an exception
+ if (alreadyLocked) {
+ throw new IOException("Could not obtain a lock for the store " + logID + " of target " + targetID);
+ }
+ }
+ }
+
+ private void releaseLock(String targetID, Long logID) {
+ Set<Long> lockedLogs = m_locks.get(targetID);
+ synchronized (lockedLogs) {
+ lockedLogs.remove(logID);
+ }
+ }
+}
Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java (original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java Thu Oct 31 20:43:14 2013
@@ -36,6 +36,8 @@ import org.osgi.service.event.EventAdmin
import org.osgi.service.log.LogService;
public class Activator extends DependencyActivatorBase implements ManagedServiceFactory {
+ public static final String PID = "org.apache.ace.log.server.store.mongo";
+
private static final String LOG_NAME = "name";
private DependencyManager m_manager;
@@ -83,7 +85,8 @@ public class Activator extends Dependenc
.setInterface(LogStore.class.getName(), props)
.setImplementation(new MongoLogStore(name))
.add(createServiceDependency().setService(EventAdmin.class).setRequired(false))
- .add(createServiceDependency().setService(MongoDBService.class).setRequired(true));
+ .add(createServiceDependency().setService(MongoDBService.class).setRequired(true))
+ .add(createConfigurationDependency().setPid(PID));
m_instances.put(pid, service);
m_manager.add(service);
} else {
Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java (original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java Thu Oct 31 20:43:14 2013
@@ -2,6 +2,7 @@ package org.apache.ace.log.server.store.
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Dictionary;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -15,6 +16,8 @@ import org.apache.ace.feedback.Event;
import org.apache.ace.log.server.store.LogStore;
import org.apache.ace.range.Range;
import org.apache.ace.range.SortedRangeSet;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.cm.ManagedService;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
@@ -23,9 +26,12 @@ import com.mongodb.DBObject;
import com.mongodb.MapReduceCommand.OutputType;
import com.mongodb.MapReduceOutput;
-public class MongoLogStore implements LogStore {
+public class MongoLogStore implements LogStore, ManagedService {
+ private static final String MAXIMUM_NUMBER_OF_EVENTS = "MaxEvents";
+
private final String m_logname;
private volatile MongoDBService m_mongoDBService;
+ private int m_maxEvents = 0;
public MongoLogStore(String logname) {
this.m_logname = logname;
@@ -91,6 +97,7 @@ public class MongoLogStore implements Lo
@Override
public void put(List<Event> events) throws IOException {
+ // TODO : if m_max_events > 0 then make sure there are no more than m_maxEvents
DBCollection collection = m_mongoDBService.getDB().getCollection(m_logname);
for (Event event : events) {
@@ -142,5 +149,24 @@ public class MongoLogStore implements Lo
public List<Descriptor> getDescriptors() throws IOException {
return getDescriptors(null);
}
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void updated(Dictionary settings) throws ConfigurationException {
+ if (settings != null) {
+ String maximumNumberOfEvents = (String) settings.get(MAXIMUM_NUMBER_OF_EVENTS);
+ if (maximumNumberOfEvents != null) {
+ try {
+ m_maxEvents = Integer.parseInt(maximumNumberOfEvents);
+ } catch (NumberFormatException nfe) {
+ throw new ConfigurationException(MAXIMUM_NUMBER_OF_EVENTS, "is not a number");
+ }
+ }
+ }
+ }
+ @Override
+ public void clean() throws IOException {
+ // TODO : if m_max_events > 0 then remove all events from the mongo store where there are more than m_maxEvents
+ }
}
Modified: ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java (original)
+++ ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java Thu Oct 31 20:43:14 2013
@@ -129,6 +129,9 @@ public class LogServletTest {
public void put(List<Event> events) throws IOException {
m_events = events;
}
+ public void clean() throws IOException {
+ throw new UnsupportedOperationException("not implemented");
+ }
}
private class MockServletOutputStream extends ServletOutputStream {
Modified: ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java (original)
+++ ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java Thu Oct 31 20:43:14 2013
@@ -23,10 +23,12 @@ import static org.apache.ace.test.utils.
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Dictionary;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import org.apache.ace.feedback.AuditEvent;
@@ -39,6 +41,8 @@ import org.testng.annotations.BeforeMeth
import org.testng.annotations.Test;
public class ServerLogStoreTester {
+ private static final String MAXIMUM_NUMBER_OF_EVENTS = "MaxEvents";
+
private LogStoreImpl m_logStore;
private File m_dir;
@@ -104,6 +108,92 @@ public class ServerLogStoreTester {
assert m_logStore.getDescriptors(targetID).size() == 1 : "We expect to find a single event: expected 1, found " + m_logStore.getDescriptors(targetID).size();
}
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test(groups = { TestUtils.UNIT })
+ public void testMaximumNumberOfEvents() throws Exception {
+ Dictionary settings = new Properties();
+ settings.put(MAXIMUM_NUMBER_OF_EVENTS, "1");
+ m_logStore.updated(settings);
+
+ List<Event> events = new ArrayList<Event>();
+ for (String target : new String[] { "target"}) {
+ for (long log : new long[] { 1 }) {
+ for (long id : new long[] { 1, 2 }) {
+ events.add(new Event(target, log, id, System.currentTimeMillis(), AuditEvent.FRAMEWORK_STARTED, new HashMap<String, String>()));
+ }
+ }
+ }
+
+ m_logStore.put(events);
+
+ List<Descriptor> allDescriptors = m_logStore.getDescriptors();
+ assert allDescriptors.size() == 1 : "Expected only one descriptor, found: " + allDescriptors.size();
+ for (Descriptor range : allDescriptors) {
+ List<Descriptor> allLogsForTarget = m_logStore.getDescriptors(range.getTargetID());
+ for (Descriptor range2 : allLogsForTarget) {
+ List<Event> getEvents = m_logStore.get(m_logStore.getDescriptor(range2.getTargetID(), range2.getStoreID()));
+ assert getEvents.size() == 1 : "Only one event expected, found " + getEvents.size();
+ }
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test(groups = { TestUtils.UNIT })
+ public void testMaximumNumberOfEventsMultipleLogs() throws Exception {
+ Dictionary settings = new Properties();
+ settings.put(MAXIMUM_NUMBER_OF_EVENTS, "1");
+ m_logStore.updated(settings);
+
+ List<Event> events = new ArrayList<Event>();
+ for (String target : new String[] { "target"}) {
+ for (long log : new long[] { 1,2 }) {
+ for (long id : new long[] { 1, 2 }) {
+ events.add(new Event(target, log, id, System.currentTimeMillis(), AuditEvent.FRAMEWORK_STARTED, new HashMap<String, String>()));
+ }
+ }
+ }
+
+ m_logStore.put(events);
+ List<Descriptor> allDescriptors = m_logStore.getDescriptors();
+ assert allDescriptors.size() == 2 : "Expected two descriptor, found: " + allDescriptors.size();
+ for (Descriptor range : allDescriptors) {
+ List<Descriptor> allLogsForTarget = m_logStore.getDescriptors(range.getTargetID());
+ for (Descriptor range2 : allLogsForTarget) {
+ List<Event> getEvents = m_logStore.get(m_logStore.getDescriptor(range2.getTargetID(), range2.getStoreID()));
+ assert getEvents.size() == 1 : "Only one event expected, found " + getEvents.size();
+ }
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test(groups = { TestUtils.UNIT })
+ public void testClean() throws Exception {
+ List<Event> events = new ArrayList<Event>();
+ for (String target : new String[] { "target"}) {
+ for (long log : new long[] { 1,2 }) {
+ for (long id : new long[] { 1, 2 }) {
+ events.add(new Event(target, log, id, System.currentTimeMillis(), AuditEvent.FRAMEWORK_STARTED, new HashMap<String, String>()));
+ }
+ }
+ }
+ m_logStore.put(events);
+
+ Dictionary settings = new Properties();
+ settings.put(MAXIMUM_NUMBER_OF_EVENTS, "1");
+ m_logStore.updated(settings);
+
+ m_logStore.clean();
+ List<Descriptor> allDescriptors = m_logStore.getDescriptors();
+ assert allDescriptors.size() == 2 : "Expected two descriptor, found: " + allDescriptors.size();
+ for (Descriptor range : allDescriptors) {
+ List<Descriptor> allLogsForTarget = m_logStore.getDescriptors(range.getTargetID());
+ for (Descriptor range2 : allLogsForTarget) {
+ List<Event> getEvents = m_logStore.get(m_logStore.getDescriptor(range2.getTargetID(), range2.getStoreID()));
+ assert getEvents.size() == 1 : "Only one event expected, found " + getEvents.size();
+ }
+ }
+ }
+
private void delete(File root) {
if (root.isDirectory()) {
for (File child : root.listFiles()) {
Added: ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg
URL: http://svn.apache.org/viewvc/ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg?rev=1537630&view=auto
==============================================================================
--- ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg (added)
+++ ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg Thu Oct 31 20:43:14 2013
@@ -0,0 +1 @@
+MaxEvents=0
\ No newline at end of file
Added: ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg
URL: http://svn.apache.org/viewvc/ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg?rev=1537630&view=auto
==============================================================================
--- ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg (added)
+++ ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg Thu Oct 31 20:43:14 2013
@@ -0,0 +1 @@
+MaxEvents=0
\ No newline at end of file