You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@deltacloud.apache.org by tc...@redhat.com on 2010/12/10 14:06:34 UTC
[PATCH core] Allow setting the provider on the server side, and implement region overriding in the ec2 driver based off of the provider (rev 2).
From: Tobias Crawley <tc...@redhat.com>
The provider can now be set on the commandline to deltacloudd via -P/--provider
(putting it in ENV['API_PROVIDER']), and also from the client via a header
(putting it in Thread.current[:provider]).
The ec2 driver is currently the only driver that understands providers, and
looks in Thread.current[:provider], then ENV['API_PROVIDER'], then falls back
to a default (currently 'us-east-1', the AWS default).
---
server/bin/deltacloudd | 8 +++++++-
server/lib/deltacloud/drivers/ec2/ec2_driver.rb | 21 ++++++++++++++++++---
server/lib/sinatra/rack_driver_select.rb | 20 ++++++++++++++------
3 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/server/bin/deltacloudd b/server/bin/deltacloudd
index ed7883b..70be6ca 100755
--- a/server/bin/deltacloudd
+++ b/server/bin/deltacloudd
@@ -36,6 +36,9 @@ BANNER
opts.on( '-p', '--port PORT', 'Use PORT (default: 3001)') do |port|
ENV["API_PORT"] = port
end
+ opts.on( '-P', '--provider PROVIDER', 'Use PROVIDER (default is set in the driver)') do |provider|
+ ENV['API_PROVIDER'] = provider
+ end
opts.on( '-e', '--env ENV', 'Environment (default: "development")') { |env| options[:env] = env }
opts.on( '-h', '--help', '') { options[:help] = true }
end
@@ -55,7 +58,10 @@ end
ENV["API_HOST"] = "localhost" unless ENV["API_HOST"]
ENV["API_PORT"] = "3001" unless ENV["API_PORT"]
-puts "Starting Deltacloud API :: #{ENV["API_DRIVER"]} :: http://#{ENV["API_HOST"]}:#{ENV["API_PORT"]}/api"
+msg = "Starting Deltacloud API :: #{ENV["API_DRIVER"]} "
+msg << ":: #{ENV['API_PROVIDER']} " if ENV['API_PROVIDER']
+msg << ":: http://#{ENV["API_HOST"]}:#{ENV["API_PORT"]}/api"
+puts msg
puts
dirname="#{File.dirname(__FILE__)}/.."
diff --git a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
index efbfcc8..5f0e8f4 100644
--- a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
+++ b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
@@ -30,7 +30,7 @@ class EC2Driver < Deltacloud::BaseDriver
def supported_collections
DEFAULT_COLLECTIONS + [ :keys, :buckets, :load_balancers ]
end
-
+
feature :instances, :user_data
feature :instances, :authentication_key
feature :instances, :security_group
@@ -106,6 +106,8 @@ class EC2Driver < Deltacloud::BaseDriver
stopped.to( :finish ) .automatically
end
+ DEFAULT_REGION = 'us-east-1'
+
#
# Images
#
@@ -504,9 +506,9 @@ class EC2Driver < Deltacloud::BaseDriver
def new_client(credentials, provider_type = :ec2)
opts = {
:access_key_id => credentials.user,
- :secret_access_key => credentials.password
+ :secret_access_key => credentials.password,
+ :server => endpoint_for_service(provider_type)
}
- opts[:server] = ENV['DCLOUD_EC2_URL'] if ENV['DCLOUD_EC2_URL']
safely do
case provider_type
when :ec2
@@ -517,6 +519,19 @@ class EC2Driver < Deltacloud::BaseDriver
end
end
+ def endpoint_for_service(service)
+ url = ""
+ url << case service
+ when :ec2
+ 'ec2.'
+ when :elb
+ 'elasticloadbalancing.'
+ end
+ url << (Thread.current[:provider] || ENV['API_PROVIDER'] || DEFAULT_REGION)
+ url << '.amazonaws.com'
+ url
+ end
+
def convert_load_balancer(credentials, loadbalancer)
balancer_realms = loadbalancer.AvailabilityZones.member.collect do |m|
realm(credentials, m)
diff --git a/server/lib/sinatra/rack_driver_select.rb b/server/lib/sinatra/rack_driver_select.rb
index f00a2c8..3e7d038 100644
--- a/server/lib/sinatra/rack_driver_select.rb
+++ b/server/lib/sinatra/rack_driver_select.rb
@@ -5,17 +5,25 @@ class RackDriverSelect
@opts = opts
end
+ HEADER_TO_ENV_MAP = {
+ 'HTTP_X_DELTACLOUD_DRIVER' => :driver,
+ 'HTTP_X_DELTACLOUD_PROVIDER' => :provider
+ }
+
def call(env)
- original_driver = Thread.current[:driver]
- new_driver = extract_driver(env)
- Thread.current[:driver] = new_driver if new_driver
+ original_settings = { }
+ HEADER_TO_ENV_MAP.each do |header, name|
+ original_settings[name] = Thread.current[name]
+ new_setting = extract_header(env, header)
+ Thread.current[name] = new_setting if new_setting
+ end
@app.call(env)
ensure
- Thread.current[:driver] = original_driver
+ original_settings.each { |name, value| Thread.current[name] = value }
end
- def extract_driver(env)
- driver_name = env['HTTP_X_DELTACLOUD_DRIVER'].downcase if env['HTTP_X_DELTACLOUD_DRIVER']
+ def extract_header(env, header)
+ env[header].downcase if env[header]
end
end
--
1.7.3.2
Re: [PATCH core] Allow setting the provider on the server side, and implement region overriding in the ec2 driver based off of the provider (rev 2).
Posted by Toby Crawley <tc...@redhat.com>.
On Dec 15, 2010, at 7:44 PM, David Lutterkort wrote:
> On Tue, 2010-12-14 at 23:06 -0500, Toby Crawley wrote:
>> On 12/14/2010 08:00 PM, David Lutterkort wrote:
>>> On Fri, 2010-12-10 at 08:06 -0500, tcrawley@redhat.com wrote:
>>>> From: Tobias Crawley<tc...@redhat.com>
>>>>
>>>> The provider can now be set on the commandline to deltacloudd via -P/--provider
>>>> (putting it in ENV['API_PROVIDER']), and also from the client via a header
>>>> (putting it in Thread.current[:provider]).
>>>>
>>>> The ec2 driver is currently the only driver that understands providers, and
>>>> looks in Thread.current[:provider], then ENV['API_PROVIDER'], then falls back
>>>> to a default (currently 'us-east-1', the AWS default).
>>>
>>> Sorry for being late to the party; the word 'provider' seems very
>>> unclear to me. In addition, the driver does some scary URL surgery that
>>> seems less than robust to me.
>>>
>>> I'd prefer if we rename this mechanism into 'endpoint' (and
>>> ENV['API_ENDPOINT']) with the expectation that the entire URL that we
>>> should talk to is passed in. With that, it also becomes possible to use
>>> the EC2 driver against a private Eucalyptus instance.
>>
>> I'm not attached to 'provider', and would be fine calling it 'endpoint'.
>
> The main thing I wanted to convey with the name change is that the value
> of the parameter is the URL for the backend cloud - I don't think that
> using a keyword will work for other backends (like RAX and OpenStack)
>
> But you are right - AWS has made quite a mess of this, and for AWS we
> will need to use a keyword rather than an actual URL, and map that
> keyword to the actual endpoint URL for EC2, S3 and ELB.
>
> So, for naming: the name should remain API_PROVIDER; we'll have to
> document the right values for it for each driver separately.
I jumped the gun a bit, and submitted a patch yesterday that changed 'provider' to 'endpoint', and added a lookup table for the default endpoints. I'll fix that and submit a new patch.
>
>> If we do keep the endpoint as simply the region name (or allow it to be a url or region name), then maybe this would be better implemented
>> as a simple lookup table:
>>
>> EC2_ENDPOINTS = {
>> :ec2 => {
>> 'us-east-1' => 'ec2.us-east-1.amazonaws.com',
>> ...
>> },
>> :s3 => {
>> 'us-east-1' => 's3.amazonaws.com',
>> 'eu-west-1' => 's3.amazonaws.com',
>> 'ap-southeast-1' => 's3-ap-southeast-1.amazonaws.com',
>> ...
>> },
>> ...
>> }
>
> That seems to me the best solution - with a proper error message when
> nonsense is passed in for API_PROVIDER (like API_PROVIDER=eu-east-1)
Yesterday's patch just attempts to use the provider as an endpoint if it is not found in the lookup table. I'll add a TODO comment there for now about raising an error, and think about the best way to do that.
>
> For S3, there's a few gotchas: (1) there's no us-east-1; we should just
> map the US Standard zone to us-east-1 (2) the request to S3 must set the
> location constraint corresponding to the endpoint.
The lookup table for s3 currently has:
's3' => {
'ap-southeast-1' => 's3-ap-southeast-1.amazonaws.com',
'eu-west-1' => 's3.amazonaws.com',
'us-east-1' => 's3.amazonaws.com',
'us-west-1' => 's3-us-west-1.amazonaws.com'
}
But I think the right thing to do is always use s3.amazonaws.com, no matter what the requested provider - there is no requirement to use a region specific endpoint like there is for the other AWS services. The bucket/blob code already has a provision for setting the location constraint per request, so we don't need to deal with that with the provider code.
I'll make this change as well, and submit a new patch set.
>
> David
>
>
---
Toby Crawley
tcrawley@redhat.com
Re: [PATCH core] Allow setting the provider on the server side,
and implement region overriding in the ec2 driver based off of the provider
(rev 2).
Posted by David Lutterkort <lu...@redhat.com>.
On Tue, 2010-12-14 at 23:06 -0500, Toby Crawley wrote:
> On 12/14/2010 08:00 PM, David Lutterkort wrote:
> > On Fri, 2010-12-10 at 08:06 -0500, tcrawley@redhat.com wrote:
> >> From: Tobias Crawley<tc...@redhat.com>
> >>
> >> The provider can now be set on the commandline to deltacloudd via -P/--provider
> >> (putting it in ENV['API_PROVIDER']), and also from the client via a header
> >> (putting it in Thread.current[:provider]).
> >>
> >> The ec2 driver is currently the only driver that understands providers, and
> >> looks in Thread.current[:provider], then ENV['API_PROVIDER'], then falls back
> >> to a default (currently 'us-east-1', the AWS default).
> >
> > Sorry for being late to the party; the word 'provider' seems very
> > unclear to me. In addition, the driver does some scary URL surgery that
> > seems less than robust to me.
> >
> > I'd prefer if we rename this mechanism into 'endpoint' (and
> > ENV['API_ENDPOINT']) with the expectation that the entire URL that we
> > should talk to is passed in. With that, it also becomes possible to use
> > the EC2 driver against a private Eucalyptus instance.
>
> I'm not attached to 'provider', and would be fine calling it 'endpoint'.
The main thing I wanted to convey with the name change is that the value
of the parameter is the URL for the backend cloud - I don't think that
using a keyword will work for other backends (like RAX and OpenStack)
But you are right - AWS has made quite a mess of this, and for AWS we
will need to use a keyword rather than an actual URL, and map that
keyword to the actual endpoint URL for EC2, S3 and ELB.
So, for naming: the name should remain API_PROVIDER; we'll have to
document the right values for it for each driver separately.
> If we do keep the endpoint as simply the region name (or allow it to be a url or region name), then maybe this would be better implemented
> as a simple lookup table:
>
> EC2_ENDPOINTS = {
> :ec2 => {
> 'us-east-1' => 'ec2.us-east-1.amazonaws.com',
> ...
> },
> :s3 => {
> 'us-east-1' => 's3.amazonaws.com',
> 'eu-west-1' => 's3.amazonaws.com',
> 'ap-southeast-1' => 's3-ap-southeast-1.amazonaws.com',
> ...
> },
> ...
> }
That seems to me the best solution - with a proper error message when
nonsense is passed in for API_PROVIDER (like API_PROVIDER=eu-east-1)
For S3, there's a few gotchas: (1) there's no us-east-1; we should just
map the US Standard zone to us-east-1 (2) the request to S3 must set the
location constraint corresponding to the endpoint.
David
Re: [PATCH core] Allow setting the provider on the server side, and
implement region overriding in the ec2 driver based off of the provider (rev
2).
Posted by Toby Crawley <tc...@redhat.com>.
On 12/14/2010 08:00 PM, David Lutterkort wrote:
> On Fri, 2010-12-10 at 08:06 -0500, tcrawley@redhat.com wrote:
>> From: Tobias Crawley<tc...@redhat.com>
>>
>> The provider can now be set on the commandline to deltacloudd via -P/--provider
>> (putting it in ENV['API_PROVIDER']), and also from the client via a header
>> (putting it in Thread.current[:provider]).
>>
>> The ec2 driver is currently the only driver that understands providers, and
>> looks in Thread.current[:provider], then ENV['API_PROVIDER'], then falls back
>> to a default (currently 'us-east-1', the AWS default).
>
> Sorry for being late to the party; the word 'provider' seems very
> unclear to me. In addition, the driver does some scary URL surgery that
> seems less than robust to me.
>
> I'd prefer if we rename this mechanism into 'endpoint' (and
> ENV['API_ENDPOINT']) with the expectation that the entire URL that we
> should talk to is passed in. With that, it also becomes possible to use
> the EC2 driver against a private Eucalyptus instance.
I'm not attached to 'provider', and would be fine calling it 'endpoint'.
The issue with passing in a url instead of just region name (in the ec2 case) is that each client (user) application would have to have code
to construct the url, and would have to pass a different url to use different services - one with ec2 calls, a different one with elb calls,
and yet another one for s3 calls (the code below doesn't include the logic to construct s3 urls - that's needed for the new bucket support
in the new ec2 driver. See [1] for all the different service/region urls for ec2). This would require the user code to know which endpoint
to request for each call, which would not always be obvious. For the case where the the dc core is configured to work against one
driver/endpoint and driver/endpoint overriding is not used, it would still need to be configured with three different endpoint urls somehow.
Figuring out the correct url really belongs in the gem that connects to aws. The 'aws' gem itself may have the logic to figure them out -
I'll have to check and see.
In the Eucalyptus case, it may be a better idea to actually have a euca_driver that extends ec2_driver, with the euca driver overriding any
bits it needs to. It could then have its own logic for constructing urls, with any data needed to construct the endpoint defined in
API_ENDPOINT and friends.
Another option is to allow the endpoint to be a region, which we use it to create/discover a url, or a url that we use outright. That would
allow the ec2 driver to be simple enough to use when talking to ec2, and be able to talk to euca without too much modification (we would
still need some way to specify to the 'static' driver/endpoint dc core the endpoint for each service, if euca expects that).
[1] http://aws.amazon.com/articles/3912?_encoding=UTF8&jiveRedirect=1
>
>> + def endpoint_for_service(service)
>> + url = ""
>> + url<< case service
>> + when :ec2
>> + 'ec2.'
>> + when :elb
>> + 'elasticloadbalancing.'
>> + end
>> + url<< (Thread.current[:provider] || ENV['API_PROVIDER'] || DEFAULT_REGION)
>> + url<< '.amazonaws.com'
>> + url
>> + end
>
> This is the code I object to; and as I said, the URL surgery restricts
> unnecessarily how the EC2 driver can be used.
>
This code gets even more complicated when s3 is added to the mix:
def endpoint_for_service(service)
region = (Thread.current[:provider] || ENV['API_PROVIDER'] || DEFAULT_REGION)
append = true
url = ""
url << case service
when :ec2
'ec2.'
when :elb
'elasticloadbalancing.'
when :s3
case region
when 'us-east-1', 'eu-west-1'
's3'
append = false
else
's3-'
end
end
url << region if append
url << '.amazonaws.com'
url
end
If we do keep the endpoint as simply the region name (or allow it to be a url or region name), then maybe this would be better implemented
as a simple lookup table:
EC2_ENDPOINTS = {
:ec2 => {
'us-east-1' => 'ec2.us-east-1.amazonaws.com',
...
},
:s3 => {
'us-east-1' => 's3.amazonaws.com',
'eu-west-1' => 's3.amazonaws.com',
'ap-southeast-1' => 's3-ap-southeast-1.amazonaws.com',
...
},
...
}
Toby
> David
>
>
Re: [PATCH core] Allow setting the provider on the server side,
and implement region overriding in the ec2 driver based off of the provider
(rev 2).
Posted by David Lutterkort <lu...@redhat.com>.
On Fri, 2010-12-10 at 08:06 -0500, tcrawley@redhat.com wrote:
> From: Tobias Crawley <tc...@redhat.com>
>
> The provider can now be set on the commandline to deltacloudd via -P/--provider
> (putting it in ENV['API_PROVIDER']), and also from the client via a header
> (putting it in Thread.current[:provider]).
>
> The ec2 driver is currently the only driver that understands providers, and
> looks in Thread.current[:provider], then ENV['API_PROVIDER'], then falls back
> to a default (currently 'us-east-1', the AWS default).
Sorry for being late to the party; the word 'provider' seems very
unclear to me. In addition, the driver does some scary URL surgery that
seems less than robust to me.
I'd prefer if we rename this mechanism into 'endpoint' (and
ENV['API_ENDPOINT']) with the expectation that the entire URL that we
should talk to is passed in. With that, it also becomes possible to use
the EC2 driver against a private Eucalyptus instance.
> + def endpoint_for_service(service)
> + url = ""
> + url << case service
> + when :ec2
> + 'ec2.'
> + when :elb
> + 'elasticloadbalancing.'
> + end
> + url << (Thread.current[:provider] || ENV['API_PROVIDER'] || DEFAULT_REGION)
> + url << '.amazonaws.com'
> + url
> + end
This is the code I object to; and as I said, the URL surgery restricts
unnecessarily how the EC2 driver can be used.
David
Re: [PATCH core] Allow setting the provider on the server side,
and implement region overriding in the ec2 driver based off of the
provider (rev 2).
Posted by Michal Fojtik <mf...@redhat.com>.
On 10/12/10 08:06 -0500, tcrawley@redhat.com wrote:
>From: Tobias Crawley <tc...@redhat.com>
>
>The provider can now be set on the commandline to deltacloudd via -P/--provider
>(putting it in ENV['API_PROVIDER']), and also from the client via a header
>(putting it in Thread.current[:provider]).
>
>The ec2 driver is currently the only driver that understands providers, and
>looks in Thread.current[:provider], then ENV['API_PROVIDER'], then falls back
>to a default (currently 'us-east-1', the AWS default).
Hi,
For testing I used this script:
<snip>
require 'rubygems'
require 'lib/deltacloud'
client=DeltaCloud.new('mockuser', 'mockpassword', 'http://localhost:3001/api')
client.use_driver('ec2', :username => 'xxxx', :password => 'xxxx', :provider => 'ap-southeast-1').realms.collect { |r| r.name }
=>
[
[0] "ap-southeast-1a",
[1] "ap-southeast-1b"
]
client.use_driver('ec2', :username => 'xxxx', :password => 'xxxx', :provider => 'eu-west-1').realms.collect { |r| r.name }
=>
[
[0] "eu-west-1a",
[1] "eu-west-1b"
]
</snip>
So as you can see, it works perfectly. ACK and thanks for this addition!
-- Michal
>---
> server/bin/deltacloudd | 8 +++++++-
> server/lib/deltacloud/drivers/ec2/ec2_driver.rb | 21 ++++++++++++++++++---
> server/lib/sinatra/rack_driver_select.rb | 20 ++++++++++++++------
> 3 files changed, 39 insertions(+), 10 deletions(-)
>
>diff --git a/server/bin/deltacloudd b/server/bin/deltacloudd
>index ed7883b..70be6ca 100755
>--- a/server/bin/deltacloudd
>+++ b/server/bin/deltacloudd
>@@ -36,6 +36,9 @@ BANNER
> opts.on( '-p', '--port PORT', 'Use PORT (default: 3001)') do |port|
> ENV["API_PORT"] = port
> end
>+ opts.on( '-P', '--provider PROVIDER', 'Use PROVIDER (default is set in the driver)') do |provider|
>+ ENV['API_PROVIDER'] = provider
>+ end
> opts.on( '-e', '--env ENV', 'Environment (default: "development")') { |env| options[:env] = env }
> opts.on( '-h', '--help', '') { options[:help] = true }
> end
>@@ -55,7 +58,10 @@ end
> ENV["API_HOST"] = "localhost" unless ENV["API_HOST"]
> ENV["API_PORT"] = "3001" unless ENV["API_PORT"]
>
>-puts "Starting Deltacloud API :: #{ENV["API_DRIVER"]} :: http://#{ENV["API_HOST"]}:#{ENV["API_PORT"]}/api"
>+msg = "Starting Deltacloud API :: #{ENV["API_DRIVER"]} "
>+msg << ":: #{ENV['API_PROVIDER']} " if ENV['API_PROVIDER']
>+msg << ":: http://#{ENV["API_HOST"]}:#{ENV["API_PORT"]}/api"
>+puts msg
> puts
>
> dirname="#{File.dirname(__FILE__)}/.."
>diff --git a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
>index efbfcc8..5f0e8f4 100644
>--- a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
>+++ b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
>@@ -30,7 +30,7 @@ class EC2Driver < Deltacloud::BaseDriver
> def supported_collections
> DEFAULT_COLLECTIONS + [ :keys, :buckets, :load_balancers ]
> end
>-
>+
> feature :instances, :user_data
> feature :instances, :authentication_key
> feature :instances, :security_group
>@@ -106,6 +106,8 @@ class EC2Driver < Deltacloud::BaseDriver
> stopped.to( :finish ) .automatically
> end
>
>+ DEFAULT_REGION = 'us-east-1'
>+
> #
> # Images
> #
>@@ -504,9 +506,9 @@ class EC2Driver < Deltacloud::BaseDriver
> def new_client(credentials, provider_type = :ec2)
> opts = {
> :access_key_id => credentials.user,
>- :secret_access_key => credentials.password
>+ :secret_access_key => credentials.password,
>+ :server => endpoint_for_service(provider_type)
> }
>- opts[:server] = ENV['DCLOUD_EC2_URL'] if ENV['DCLOUD_EC2_URL']
> safely do
> case provider_type
> when :ec2
>@@ -517,6 +519,19 @@ class EC2Driver < Deltacloud::BaseDriver
> end
> end
>
>+ def endpoint_for_service(service)
>+ url = ""
>+ url << case service
>+ when :ec2
>+ 'ec2.'
>+ when :elb
>+ 'elasticloadbalancing.'
>+ end
>+ url << (Thread.current[:provider] || ENV['API_PROVIDER'] || DEFAULT_REGION)
>+ url << '.amazonaws.com'
>+ url
>+ end
>+
> def convert_load_balancer(credentials, loadbalancer)
> balancer_realms = loadbalancer.AvailabilityZones.member.collect do |m|
> realm(credentials, m)
>diff --git a/server/lib/sinatra/rack_driver_select.rb b/server/lib/sinatra/rack_driver_select.rb
>index f00a2c8..3e7d038 100644
>--- a/server/lib/sinatra/rack_driver_select.rb
>+++ b/server/lib/sinatra/rack_driver_select.rb
>@@ -5,17 +5,25 @@ class RackDriverSelect
> @opts = opts
> end
>
>+ HEADER_TO_ENV_MAP = {
>+ 'HTTP_X_DELTACLOUD_DRIVER' => :driver,
>+ 'HTTP_X_DELTACLOUD_PROVIDER' => :provider
>+ }
>+
> def call(env)
>- original_driver = Thread.current[:driver]
>- new_driver = extract_driver(env)
>- Thread.current[:driver] = new_driver if new_driver
>+ original_settings = { }
>+ HEADER_TO_ENV_MAP.each do |header, name|
>+ original_settings[name] = Thread.current[name]
>+ new_setting = extract_header(env, header)
>+ Thread.current[name] = new_setting if new_setting
>+ end
> @app.call(env)
> ensure
>- Thread.current[:driver] = original_driver
>+ original_settings.each { |name, value| Thread.current[name] = value }
> end
>
>- def extract_driver(env)
>- driver_name = env['HTTP_X_DELTACLOUD_DRIVER'].downcase if env['HTTP_X_DELTACLOUD_DRIVER']
>+ def extract_header(env, header)
>+ env[header].downcase if env[header]
> end
>
> end
>--
>1.7.3.2
>
--
--------------------------------------------------------
Michal Fojtik, mfojtik@redhat.com
Deltacloud API: http://deltacloud.org
--------------------------------------------------------