You are viewing a plain text version of this content. The canonical link for it is here.
Posted to tashi-commits@incubator.apache.org by st...@apache.org on 2011/12/20 23:44:29 UTC
svn commit: r1221528 - in /incubator/tashi/trunk: Makefile
etc/Accounting.cfg etc/Agent.cfg etc/TashiDefaults.cfg
src/tashi/accounting/ src/tashi/accounting/accounting.py
src/tashi/clustermanager/clustermanagerservice.py
Author: stroucki
Date: Tue Dec 20 23:44:28 2011
New Revision: 1221528
URL: http://svn.apache.org/viewvc?rev=1221528&view=rev
Log:
accounting.py: Add separate optional accounting server
clustermanagerservice.py: Remove chatty accounting messages
configs: add sample configs for Agent and Accounting to log to separate files
Makefile: Add accounting server to Makefile
Added:
incubator/tashi/trunk/etc/Accounting.cfg
incubator/tashi/trunk/etc/Agent.cfg
incubator/tashi/trunk/src/tashi/accounting/
incubator/tashi/trunk/src/tashi/accounting/accounting.py (with props)
Modified:
incubator/tashi/trunk/Makefile
incubator/tashi/trunk/etc/TashiDefaults.cfg
incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py
Modified: incubator/tashi/trunk/Makefile
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/Makefile?rev=1221528&r1=1221527&r2=1221528&view=diff
==============================================================================
--- incubator/tashi/trunk/Makefile (original)
+++ incubator/tashi/trunk/Makefile Tue Dec 20 23:44:28 2011
@@ -65,15 +65,20 @@ src/utils/nmd: src/utils/nmd.py
rmnmd:
echo Removing nmd...; rm -f bin/nmd.py
-bin: bindir bin/clustermanager.py bin/nodemanager.py bin/tashi-client.py bin/primitive.py bin/zoni-cli.py
+bin: bindir bin/clustermanager.py bin/nodemanager.py bin/tashi-client.py bin/primitive.py bin/zoni-cli.py bin/accounting.py
bindir:
if test ! -d bin; then mkdir bin; fi
-rmbin: rmclustermanager rmnodemanager rmtashi-client rmprimitive rmzoni-cli
+rmbin: rmclustermanager rmnodemanager rmtashi-client rmprimitive rmzoni-cli rmaccounting
if test -d bin; then rmdir bin; fi
bin/getInstances:
if test ! -e bin/getInstances; then (echo "Generating client symlinks..."; cd bin; PYTHONPATH=../src ../src/tashi/client/client.py --makesyms); fi
rmclients:
if test -e bin/getInstances; then (echo Removing client symlinks...; cd bin; PYTHONPATH=../src ../src/tashi/client/client.py --rmsyms; cd ..); fi
+bin/accounting.py: src/tashi/accounting/accounting.py
+ @echo Symlinking in Accounting server...
+ (cd bin; ln -s ../src/tashi/accounting/accounting.py .)
+rmaccounting:
+ if test -e bin/accounting.py; then echo Removing Accounting server symlink...; rm bin/accounting.py; fi
bin/clustermanager.py: src/tashi/clustermanager/clustermanager.py
@echo Symlinking in clustermanager...
(cd bin; ln -s ../src/tashi/clustermanager/clustermanager.py .)
Added: incubator/tashi/trunk/etc/Accounting.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/etc/Accounting.cfg?rev=1221528&view=auto
==============================================================================
--- incubator/tashi/trunk/etc/Accounting.cfg (added)
+++ incubator/tashi/trunk/etc/Accounting.cfg Tue Dec 20 23:44:28 2011
@@ -0,0 +1,18 @@
+[handler_fileHandler]
+class = FileHandler
+level = NOTSET
+formatter = standardFormatter
+args = ("/var/log/accounting.log",)
+
+[handlers]
+#keys = consoleHandler,publisherHandler,fileHandler
+keys = consoleHandler,fileHandler
+
+[loggers]
+keys = root
+
+[logger_root]
+level = DEBUG
+#handlers = consoleHandler,publisherHandler,fileHandler,syslogHandler
+handlers = fileHandler
+propagate = 1
Added: incubator/tashi/trunk/etc/Agent.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/etc/Agent.cfg?rev=1221528&view=auto
==============================================================================
--- incubator/tashi/trunk/etc/Agent.cfg (added)
+++ incubator/tashi/trunk/etc/Agent.cfg Tue Dec 20 23:44:28 2011
@@ -0,0 +1,18 @@
+[handler_fileHandler]
+class = FileHandler
+level = NOTSET
+formatter = standardFormatter
+args = ("/var/log/agent.log",)
+
+[handlers]
+#keys = consoleHandler,publisherHandler,fileHandler
+keys = consoleHandler,fileHandler
+
+[loggers]
+keys = root
+
+[logger_root]
+level = DEBUG
+#handlers = consoleHandler,publisherHandler,fileHandler,syslogHandler
+handlers = consoleHandler,fileHandler
+propagate = 1
Modified: incubator/tashi/trunk/etc/TashiDefaults.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/etc/TashiDefaults.cfg?rev=1221528&r1=1221527&r2=1221528&view=diff
==============================================================================
--- incubator/tashi/trunk/etc/TashiDefaults.cfg (original)
+++ incubator/tashi/trunk/etc/TashiDefaults.cfg Tue Dec 20 23:44:28 2011
@@ -87,6 +87,10 @@ file = /var/tmp/cm.dat
uri = mysql://root@clustermanager/tashi
password = changeme
+# Accounting portion
+[Accounting]
+publisher = tashi.messaging.GangliaPublisher
+
# NodeManger portion
[NodeManager]
dfs = tashi.dfs.Vfs
Added: incubator/tashi/trunk/src/tashi/accounting/accounting.py
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/src/tashi/accounting/accounting.py?rev=1221528&view=auto
==============================================================================
--- incubator/tashi/trunk/src/tashi/accounting/accounting.py (added)
+++ incubator/tashi/trunk/src/tashi/accounting/accounting.py Tue Dec 20 23:44:28 2011
@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from socket import gethostname
+import os
+import socket
+import sys
+import threading
+import time
+import random
+import logging.config
+
+from tashi.rpycservices.rpyctypes import *
+from tashi.util import getConfig, createClient, instantiateImplementation, boolean
+import tashi
+
+class Accounting(object):
+ def __init__(self, config, cmclient):
+ self.config = config
+ self.cm = cmclient
+ self.hooks = []
+ self.log = logging.getLogger(__file__)
+
+ items = self.config.items("Accounting")
+ items.sort()
+ for item in items:
+ (name, value) = item
+ name = name.lower()
+ if (name.startswith("hook")):
+ try:
+ self.hooks.append(instantiateImplementation(value, config, cmclient, False))
+ except:
+ self.log.exception("Failed to load hook %s" % (value))
+
+ def start(self):
+ while True:
+ try:
+ instances = self.cm.getInstances()
+ for instance in instances:
+ # XXXstroucki this currently duplicates what the CM was doing.
+ # perhaps implement a diff-like log?
+ self.log.info('Accounting: id %d host %d vmId %d user %d cores %d memory %d' % (instance.id, instance.hostId, instance.vmId, instance.userId, instance.cores, instance.memory))
+ except:
+ self.log.warning("Accounting iteration failed")
+
+ # wait to do the next iteration
+ # XXXstroucki make this configurable?
+ time.sleep(60)
+
+def main():
+ (config, configFiles) = getConfig(["Accounting"])
+ publisher = instantiateImplementation(config.get("Accounting", "publisher"), config)
+ tashi.publisher = publisher
+ cmclient = createClient(config)
+ logging.config.fileConfig(configFiles)
+ accounting = Accounting(config, cmclient)
+ accounting.start()
+
+if __name__ == "__main__":
+ main()
Propchange: incubator/tashi/trunk/src/tashi/accounting/accounting.py
------------------------------------------------------------------------------
svn:executable = *
Modified: incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py?rev=1221528&r1=1221527&r2=1221528&view=diff
==============================================================================
--- incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py (original)
+++ incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py Tue Dec 20 23:44:28 2011
@@ -236,6 +236,8 @@ class ClusterManagerService(object):
except:
self.log.exception('monitorCluster iteration failed')
# XXXrgass too chatty. Remove
+ # XXXstroucki the risk is that a deadlock in obtaining
+ # data could prevent this loop from continuing.
#self.log.info("Sleeping for %d seconds" % sleepFor)
time.sleep(sleepFor)
@@ -479,10 +481,8 @@ class ClusterManagerService(object):
oldHost.state = HostState.Normal
# let the host communicate what it is running
- # XXXrgass - This is too chatty for the console, I think we should remove this.
- # XXXstroucki - My install depends on this, but I output to log files. This should be handled by a separate accounting server in future.
+ # and note that the information is not stale
for instance in instances:
- self.log.info('Accounting: id %d host %d vmId %d user %d cores %d memory %d' % (instance.id, host.id, instance.vmId, instance.userId, instance.cores, instance.memory))
self.instanceLastContactTime.setdefault(instance.id, 0)
self.data.releaseHost(oldHost)