You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@deltacloud.apache.org by ma...@redhat.com on 2012/02/23 14:28:10 UTC
[PATCH 3/3] Updated openstack driver for openstack compute v2.0 API (openstack compute release 'diablo')
From: marios <ma...@redhat.com>
Signed-off-by: marios <ma...@redhat.com>
---
.../drivers/openstack/openstack_driver.rb | 295 +++++++++++++++++++-
1 files changed, 289 insertions(+), 6 deletions(-)
diff --git a/server/lib/deltacloud/drivers/openstack/openstack_driver.rb b/server/lib/deltacloud/drivers/openstack/openstack_driver.rb
index d09edf2..da0dc54 100644
--- a/server/lib/deltacloud/drivers/openstack/openstack_driver.rb
+++ b/server/lib/deltacloud/drivers/openstack/openstack_driver.rb
@@ -14,16 +14,22 @@
# under the License.
#
-require 'deltacloud/drivers/rackspace/rackspace_driver.rb'
+require 'deltacloud/base_driver'
+require 'openstack/compute'
+require 'tempfile'
module Deltacloud
module Drivers
module Openstack
- class OpenstackDriver < Rackspace::RackspaceDriver
+ class OpenstackDriver < Deltacloud::BaseDriver
feature :instances, :user_name
feature :instances, :authentication_password
feature :instances, :user_files
+ def supported_collections
+ DEFAULT_COLLECTIONS - [ :storage_snapshots, :storage_volumes ] #+ [ :buckets ]
+ end
+
define_instance_states do
start.to( :pending ) .on( :create )
pending.to( :running ) .automatically
@@ -33,15 +39,292 @@ module Deltacloud
stopped.to( :finish ) .automatically
end
- def new_client(credentials)
+ def hardware_profiles(credentials, opts = {})
+ os = new_client(credentials)
+ results = []
+ safely do
+ if opts[:id]
+ flavor = os.flavor(opts[:id])
+ results << convert_from_flavor(flavor)
+ else
+ results = os.flavors.collect do |f|
+ convert_from_flavor(f)
+ end
+ end
+ filter_hardware_profiles(results, opts)
+ end
+ end
+
+ def images(credentials, opts=nil)
+ os = new_client(credentials)
+ owner = extract_user_tenant(credentials.user).first
+ results = []
+ safely do
+ if(opts[:id])
+ img = os.get_image(opts[:id])
+ results << convert_from_image(img, owner)
+ else
+ results = os.list_images.collect do |img|
+ convert_from_image(img, owner)
+ end
+ end
+ end
+ filter_on(results, :id, opts)
+ filter_on(results, :owner_id, opts)
+ end
+
+ def create_image(credentials, opts)
+ os = new_client(credentials)
+ owner = extract_user_tenant(credentials.user).first
+ safely do
+ server = os.get_server(opts[:id])
+ image_name = opts[:name] || "#{server.name}_#{Time.now}"
+ img = server.create_image(:name=>image_name)
+ convert_from_image(img, owner)
+ end
+ end
+
+ def destroy_image(credentials, image_id)
+ os = new_client(credentials)
+ safely do
+ image = os.get_image(image_id)
+ unless image.delete!
+ raise "ERROR: Cannot delete image with ID:#{image_id}"
+ end
+ end
+ end
+
+ def realms(credentials, opts=nil)
+ os = new_client(credentials)
+ limits = ""
+ safely do
+ lim = os.limits
+ limits << "ABSOLUTE >> Max. Instances: #{lim[:absolute][:maxTotalInstances]} Max. RAM: #{lim[:absolute][:maxTotalRAMSize]} || "
+ lim[:rate].each do |rate|
+ if rate[:regex] =~ /servers/
+ limits << "SERVERS >> Total: #{rate[:limit].first[:value]} Remaining: #{rate[:limit].first[:remaining]} Time Unit: per #{rate[:limit].first[:unit]}"
+ end
+ end
+ end
+ [ Realm.new( { :id=>'default',
+ :name=>'default',
+ :limit => limits,
+ :state=>'AVAILABLE' })]
+ end
+
+ def instances(credentials, opts={})
+ os = new_client(credentials)
+ insts = []
+ owner = extract_user_tenant(credentials.user).first
+ safely do
+ begin
+ if opts[:id]
+ server = os.get_server(opts[:id].to_i)
+ insts << convert_from_server(server, owner)
+ else
+ insts = os.list_servers_detail.collect do |server|
+ convert_from_server(server, owner)
+ end
+ end
+ rescue OpenStack::Compute::Exception::ItemNotFound
+ end
+ end
+ insts = filter_on( insts, :id, opts )
+ insts = filter_on( insts, :state, opts )
+ insts
+ end
+
+ def create_instance(credentials, image_id, opts)
+ os = new_client( credentials )
+ result = nil
+#opts[:personality]: path1='server_path1'. content1='contents1', path2='server_path2', content2='contents2' etc
+ params = extract_personality(opts)
+# ref_prefix = get_prefix(os)
+ params[:name] = opts[:name] || Time.now.to_s
+ params[:imageRef] = image_id
+ params[:flavorRef] = (opts[:hwp_id] && opts[:hwp_id].length>0) ?
+ opts[:hwp_id] : hardware_profiles(credentials).first
+ if opts[:password] && opts[:password].length > 0
+ params[:adminPass]=opts[:password]
+ end
+ safely do
+ server = os.create_server(params)
+ result = convert_from_server(server, extract_user_tenant(credentials.user).first)
+ end
+ result
+ end
+
+ def reboot_instance(credentials, instance_id)
+ os = new_client(credentials)
+ safely do
+ server = os.get_server(instance_id.to_i)
+ server.reboot! # sends a hard reboot (power cycle) - could instead server.reboot("SOFT")
+ convert_from_server(server, extract_user_tenant(credentials.user).first)
+ end
+ end
+
+ def destroy_instance(credentials, instance_id)
+ os = new_client(credentials)
+ safely do
+ server = os.get_server(instance_id.to_i)
+ server.delete!
+ convert_from_server(server, extract_user_tenant(credentials.user).first)
+ end
+ end
+
+ alias_method :stop_instance, :destroy_instance
+
+ def valid_credentials?(credentials)
+ begin
+ new_client(credentials)
+ rescue
+ return false
+ end
+ true
+ end
+
+ def buckets(credentials, opts={})
+
+ end
+
+ def create_bucket(credentials, name, opts={})
+
+ end
+
+ def delete_bucket(credentials, name, opts={})
+
+ end
+
+ def blobs(credentials, opts={})
+
+ end
+
+ def blob_data(credentials, bucket, blob, opts={})
+
+ end
+
+ def create_blob(credentials, bucket, blob, data, opts={})
+
+ end
+
+ def delete_blob(credentials, bucket, blob, opts={})
+
+ end
+
+ def blob_metadata(credentials, opts={})
+
+ end
+
+ def update_blob_metadata(credentials, opts={})
+
+ end
+
+ def blob_stream_connection(params)
+
+ end
+
+private
+
+ #credentials.name == "username+tenant_name"
+ def new_client(credentials, buckets=false)
+ user_name, tenant_name = extract_user_tenant(credentials.user)
+ safely do
+ if buckets
+#TODO CloudFiles::Connection.new(:username => user_name, :api_key => credentials.password)
+ else
+ OpenStack::Compute::Connection.new(:username => user_name, :api_key => credentials.password, :authtenant => tenant_name, :auth_url => api_provider)
+ end
+ end
+ end
+
+ def cloudfiles_client(credentials)
safely do
- CloudServers::Connection.new(:username => credentials.user, :api_key => credentials.password, :auth_url => api_provider)
+ CloudFiles::Connection.new(:username => credentials.user, :api_key => credentials.password)
+ end
+ end
+
+#NOTE: for the convert_from_foo methods below... openstack-compute
+#gives Hash for 'flavors' but OpenStack::Compute::Flavor for 'flavor'
+#hence the use of 'send' to deal with both cases and save duplication
+
+ def convert_from_flavor(flavor)
+ op = (flavor.class == Hash)? :fetch : :send
+ HardwareProfile.new(flavor.send(op, :id).to_s) do
+ architecture 'x86_64'
+ memory flavor.send(op, :ram).to_i
+ storage flavor.send(op, :disk).to_i
+ cpu flavor.send(op, :vcpus).to_i
+ end
+ end
+
+ def convert_from_image(image, owner)
+ op = (image.class == Hash)? :fetch : :send
+ Image.new({
+ :id => image.send(op, :id),
+ :name => image.send(op, :name),
+ :description => image.send(op, :name),
+ :owner_id => owner,
+ :state => image.send(op, :status),
+ :architecture => 'x86_64'
+ })
+ end
+
+ def convert_from_server(server, owner)
+ op = (server.class == Hash)? :fetch : :send
+ image = server.send(op, :image)
+ flavor = server.send(op, :flavor)
+ begin
+ password = server.send(op, :adminPass) || ""
+ rescue IndexError
+ password = ""
+ end
+ inst = Instance.new(
+ :id => server.send(op, :id).to_s,
+ :realm_id => 'default',
+ :owner_id => owner,
+ :description => server.send(op, :name),
+ :name => server.send(op, :name),
+ :state => (server.send(op, :status) == 'ACTIVE') ? 'RUNNING' : 'PENDING',
+ :architecture => 'x86_64',
+ :image_id => image[:id] || image["id"],
+ :instance_profile => InstanceProfile::new(flavor[:id] || flavor["id"]),
+ :public_addresses => convert_server_addresses(server, :public),
+ :private_addresses => convert_server_addresses(server, :private),
+ :username => 'root',
+ :password => password
+ )
+ inst.actions = instance_actions_for(inst.state)
+ inst.create_image = 'RUNNING'.eql?(inst.state)
+ inst
+ end
+
+ def convert_server_addresses(server, type)
+ op, address_label = (server.class == Hash)? [:fetch, :addr] : [:send, :address]
+ addresses = (server.send(op, :addresses)[type] || []).collect do |addr|
+ type = (addr.send(op, :version) == 4)? :ipv4 : :ipv6
+ InstanceAddress.new(addr.send(op, address_label), {:type=>type} )
+ end
+ end
+
+ def extract_user_tenant(user_tenant)
+ tokens = user_tenant.split("+")
+ return tokens.first, tokens.last
+ end
+
+ #IN: path1='server_path1'. content1='contents1', path2='server_path2', content2='contents2' etc
+ #OUT:{local_path=>server_path, local_path1=>server_path2 etc}
+ def extract_personality(opts)
+ personality_hash = opts.inject({}) do |result, (opt_k,opt_v)|
+ if opt_k.to_s =~ /^path([1-5]+)/
+ tempfile = Tempfile.new("os_personality_local_#{$1}")
+ tempfile.write(opts[:"content#{$1}"])
+ result[tempfile.path]=opts[:"path#{$1}"]
+ end
+ result
end
end
- private :new_client
end
end
end
end
-
--
1.7.6.5
Re: [PATCH 3/3] Updated openstack driver for openstack compute v2.0
API (openstack compute release 'diablo')
Posted by David Lutterkort <lu...@redhat.com>.
Hi Marios,
On Tue, 2012-02-28 at 18:11 +0200, marios@redhat.com wrote:
> On 24/02/12 02:06, David Lutterkort wrote:
> yes, based on the Auth url provided as api_provider - i.e. the way the
> openstack gem does it. in theory, the openstack gem handles this so the
> driver should work for both... we need some testing against both
> versions and put in safety/begin-rescue clauses to handle the edge
> cases. I put in a private boolean method 'api_v2' for convenience in
> new_client (whether or not to blow up when its v2 and the client didn't
> provide both username and authtenant).
Good, I didn't realize that about the client.
> >> + def destroy_image(credentials, image_id)
> >> + os = new_client(credentials)
> >> + safely do
> >> + image = os.get_image(image_id)
> >> + unless image.delete!
> >> + raise "ERROR: Cannot delete image with ID:#{image_id}"
> >
> > What error does that lead to in the API ? Can we pass on more
> > informative details based on what the backend tells us ?
> >
>
> actually, looking at the gem code some more
> [/gems/openstack-compute-1.1.7/lib/openstack/compute/image.rb:65]:
>
> the safely..do will handle that. image.delete! will blow up unless the
> operation is successful. In other words, the image.delete! method
> returns true only if successful. Otherwise it raises an exception based
> on the response from the Openstack server. This exception will be
> propagated to the client because of safely..do. The actual errors that
> can be raised here as defined by the API are numerous:
> http://docs.openstack.org/api/openstack-compute/2/content/Delete_Image-d1e4957.html
> (computeFault (400, 500, …), serviceUnavailable (503), unauthorized
> (401), forbidden (403), badRequest (400), badMethod (405), overLimit
> (413), itemNotFound (404) )
Ok; nice. Just wanted to make sure we think of our poor users ;)
David
Re: [PATCH 3/3] Updated openstack driver for openstack compute v2.0
API (openstack compute release 'diablo')
Posted by "marios@redhat.com" <ma...@redhat.com>.
many thanks for taking the time. Comments inline (and new patches on
list with the fixes below):
On 24/02/12 02:06, David Lutterkort wrote:
> Hi Marios,
>
> On Thu, 2012-02-23 at 15:28 +0200, marios@redhat.com wrote:
>> From: marios <ma...@redhat.com>
>>
>>
>> Signed-off-by: marios <ma...@redhat.com>
>> ---
>
> * We also need an update to the gemspec to add the openstack gems
done
> * When supplying the wrong credentials, you get a server error
> with a stacktrace going to new_client - we need to produce at
> least a 401
done
> * When I click on an image (/api/images/1358), I get a 500
> "undefined method `each' for nil:NilClass" with a stacktrace
> ending in server/views/images/show.html.haml:26:in
> `evaluate_source'
done (yes, this was due to the addition of 'hardware_profiles' to
images, which was only added after I rebased my branch and made patches
for the list. i.e., the haml was doing something like
@image.hardware_profiles which was nil... added now)
>
> More inline:
>
>> .../drivers/openstack/openstack_driver.rb | 295 +++++++++++++++++++-
>> 1 files changed, 289 insertions(+), 6 deletions(-)
>>
>> + class OpenstackDriver < Deltacloud::BaseDriver
>
> What do we do about backward compat ? I am not sure how widely the v1.0
> API is deployed, but we now leave people who won't immediately upgrade
> to v2 in the dust. Is there any way to dynamically determine which
> version of the API we're talking to ?
yes, based on the Auth url provided as api_provider - i.e. the way the
openstack gem does it. in theory, the openstack gem handles this so the
driver should work for both... we need some testing against both
versions and put in safety/begin-rescue clauses to handle the edge
cases. I put in a private boolean method 'api_v2' for convenience in
new_client (whether or not to blow up when its v2 and the client didn't
provide both username and authtenant).
>
>> define_instance_states do
>> start.to( :pending ) .on( :create )
>> pending.to( :running ) .automatically
>
> Is this still up-to-date with v2 ?
>
yes as far as I can see there are no changes here
>> @@ -33,15 +39,292 @@ module Deltacloud
>> +
>> + def images(credentials, opts=nil)
>> + os = new_client(credentials)
>> + owner = extract_user_tenant(credentials.user).first
>
> Couldn't we just use os.authuser/os.authtenant here ? That way, we could
> get rid of extract_user_tenant
>
good point, though we still need to extract it for 'new_client' - but
that can be done once in the new_client method and we don't need
'extract_user_tenant' as seperate method
>> + results = []
>> + safely do
>> + if(opts[:id])
>> + img = os.get_image(opts[:id])
>> + results << convert_from_image(img, owner)
>> + else
>> + results = os.list_images.collect do |img|
>> + convert_from_image(img, owner)
>> + end
>> + end
>> + end
>> + filter_on(results, :id, opts)
>
> That's not needed (I fear we have that in lots of drivers)
>
DONE... yeah, this is a remnant from when we used to 'get all' and then
filter for opts[:id] ... i guess now we have a if(opts[:id]) clause in
all drivers.
>> + def destroy_image(credentials, image_id)
>> + os = new_client(credentials)
>> + safely do
>> + image = os.get_image(image_id)
>> + unless image.delete!
>> + raise "ERROR: Cannot delete image with ID:#{image_id}"
>
> What error does that lead to in the API ? Can we pass on more
> informative details based on what the backend tells us ?
>
actually, looking at the gem code some more
[/gems/openstack-compute-1.1.7/lib/openstack/compute/image.rb:65]:
the safely..do will handle that. image.delete! will blow up unless the
operation is successful. In other words, the image.delete! method
returns true only if successful. Otherwise it raises an exception based
on the response from the Openstack server. This exception will be
propagated to the client because of safely..do. The actual errors that
can be raised here as defined by the API are numerous:
http://docs.openstack.org/api/openstack-compute/2/content/Delete_Image-d1e4957.html
(computeFault (400, 500, …), serviceUnavailable (503), unauthorized
(401), forbidden (403), badRequest (400), badMethod (405), overLimit
(413), itemNotFound (404) )
>> + def realms(credentials, opts=nil)
>> + os = new_client(credentials)
>> + limits = ""
>> + safely do
>> + lim = os.limits
>> + limits << "ABSOLUTE >> Max. Instances: #{lim[:absolute][:maxTotalInstances]} Max. RAM: #{lim[:absolute][:maxTotalRAMSize]} || "
>> + lim[:rate].each do |rate|
>> + if rate[:regex] =~ /servers/
>> + limits << "SERVERS OpenStack_2>> Total: #{rate[:limit].first[:value]} Remaining: #{rate[:limit].first[:remaining]} Time Unit: per #{rate[:limit].first[:unit]}"
>> + end
>> + end
>> + end
>> + [ Realm.new( { :id=>'default',
>> + :name=>'default',
>> + :limit => limits,
>> + :state=>'AVAILABLE' })]
>
> I had completely forgotten we had limits in realms; we should probably
> have a closer look at how we report that information, so that it's
> consumable in the XML. But that can be a separate cleanup patch.
>
yes... for now I put a ![CDATA[] around the limit xml (as far as grep is
concerned... this is the only driver we have limits in right now)
>> + def instances(credentials, opts={})
>> + os = new_client(credentials)
>> + insts = []
>> + owner = extract_user_tenant(credentials.user).first
>> + safely do
>> + begin
>> + if opts[:id]
>> + server = os.get_server(opts[:id].to_i)
>> + insts << convert_from_server(server, owner)
>> + else
>> + insts = os.list_servers_detail.collect do |server|
>> + convert_from_server(server, owner)
>> + end
>> + end
>> + rescue OpenStack::Compute::Exception::ItemNotFound
>
> Shouldn't we report a 404 in that case ? And can ItemNotFound happen
> even when opts[:id] is not set ?
>
yeah sorry this must have been remnant of debugging. I let safely do...
handle this and put a " on /Exception::ItemNotFound/ do" in the
driver 'on exceptions'
>> +#NOTE: for the convert_from_foo methods below... openstack-compute
>> +#gives Hash for 'flavors' but OpenStack::Compute::Flavor for 'flavor'
>> +#hence the use of 'send' to deal with both cases and save duplication
>
> Eeeew ... nice hack around that; is there any way this can be addressed
> upstream ? Could be as simple as implementing
> OpenStack::Compute::Flavor.[]
>
>> + def extract_user_tenant(user_tenant)
>> + tokens = user_tenant.split("+")
>> + return tokens.first, tokens.last
>
> If we get more than two tokens from the split, we should complain very
> loudly; otherwise, people will get a 401 and not really understand why.
>
yes done, with the added clause that '<2 tokens AND apiv2' , i.e. its
acceptable to have <2 tokens if this is api v1, cos we don't need
authtenant in that case.
marios
> David
>
Re: [PATCH 3/3] Updated openstack driver for openstack compute v2.0
API (openstack compute release 'diablo')
Posted by David Lutterkort <lu...@redhat.com>.
Hi Marios,
On Thu, 2012-02-23 at 15:28 +0200, marios@redhat.com wrote:
> From: marios <ma...@redhat.com>
>
>
> Signed-off-by: marios <ma...@redhat.com>
> ---
ACK. Nice to see this take shape; some comments:
* We also need an update to the gemspec to add the openstack gems
* When supplying the wrong credentials, you get a server error
with a stacktrace going to new_client - we need to produce at
least a 401
* When I click on an image (/api/images/1358), I get a 500
"undefined method `each' for nil:NilClass" with a stacktrace
ending in server/views/images/show.html.haml:26:in
`evaluate_source'
More inline:
> .../drivers/openstack/openstack_driver.rb | 295 +++++++++++++++++++-
> 1 files changed, 289 insertions(+), 6 deletions(-)
>
> diff --git a/server/lib/deltacloud/drivers/openstack/openstack_driver.rb b/server/lib/deltacloud/drivers/openstack/openstack_driver.rb
> index d09edf2..da0dc54 100644
> --- a/server/lib/deltacloud/drivers/openstack/openstack_driver.rb
> +++ b/server/lib/deltacloud/drivers/openstack/openstack_driver.rb
> @@ -14,16 +14,22 @@
> # under the License.
> #
>
> -require 'deltacloud/drivers/rackspace/rackspace_driver.rb'
> +require 'deltacloud/base_driver'
> +require 'openstack/compute'
> +require 'tempfile'
> module Deltacloud
> module Drivers
> module Openstack
> - class OpenstackDriver < Rackspace::RackspaceDriver
> + class OpenstackDriver < Deltacloud::BaseDriver
What do we do about backward compat ? I am not sure how widely the v1.0
API is deployed, but we now leave people who won't immediately upgrade
to v2 in the dust. Is there any way to dynamically determine which
version of the API we're talking to ?
> define_instance_states do
> start.to( :pending ) .on( :create )
> pending.to( :running ) .automatically
Is this still up-to-date with v2 ?
> @@ -33,15 +39,292 @@ module Deltacloud
> +
> + def images(credentials, opts=nil)
> + os = new_client(credentials)
> + owner = extract_user_tenant(credentials.user).first
Couldn't we just use os.authuser/os.authtenant here ? That way, we could
get rid of extract_user_tenant
> + results = []
> + safely do
> + if(opts[:id])
> + img = os.get_image(opts[:id])
> + results << convert_from_image(img, owner)
> + else
> + results = os.list_images.collect do |img|
> + convert_from_image(img, owner)
> + end
> + end
> + end
> + filter_on(results, :id, opts)
That's not needed (I fear we have that in lots of drivers)
> + def destroy_image(credentials, image_id)
> + os = new_client(credentials)
> + safely do
> + image = os.get_image(image_id)
> + unless image.delete!
> + raise "ERROR: Cannot delete image with ID:#{image_id}"
What error does that lead to in the API ? Can we pass on more
informative details based on what the backend tells us ?
> + def realms(credentials, opts=nil)
> + os = new_client(credentials)
> + limits = ""
> + safely do
> + lim = os.limits
> + limits << "ABSOLUTE >> Max. Instances: #{lim[:absolute][:maxTotalInstances]} Max. RAM: #{lim[:absolute][:maxTotalRAMSize]} || "
> + lim[:rate].each do |rate|
> + if rate[:regex] =~ /servers/
> + limits << "SERVERS >> Total: #{rate[:limit].first[:value]} Remaining: #{rate[:limit].first[:remaining]} Time Unit: per #{rate[:limit].first[:unit]}"
> + end
> + end
> + end
> + [ Realm.new( { :id=>'default',
> + :name=>'default',
> + :limit => limits,
> + :state=>'AVAILABLE' })]
I had completely forgotten we had limits in realms; we should probably
have a closer look at how we report that information, so that it's
consumable in the XML. But that can be a separate cleanup patch.
> + def instances(credentials, opts={})
> + os = new_client(credentials)
> + insts = []
> + owner = extract_user_tenant(credentials.user).first
> + safely do
> + begin
> + if opts[:id]
> + server = os.get_server(opts[:id].to_i)
> + insts << convert_from_server(server, owner)
> + else
> + insts = os.list_servers_detail.collect do |server|
> + convert_from_server(server, owner)
> + end
> + end
> + rescue OpenStack::Compute::Exception::ItemNotFound
Shouldn't we report a 404 in that case ? And can ItemNotFound happen
even when opts[:id] is not set ?
> +#NOTE: for the convert_from_foo methods below... openstack-compute
> +#gives Hash for 'flavors' but OpenStack::Compute::Flavor for 'flavor'
> +#hence the use of 'send' to deal with both cases and save duplication
Eeeew ... nice hack around that; is there any way this can be addressed
upstream ? Could be as simple as implementing
OpenStack::Compute::Flavor.[]
> + def extract_user_tenant(user_tenant)
> + tokens = user_tenant.split("+")
> + return tokens.first, tokens.last
If we get more than two tokens from the split, we should complain very
loudly; otherwise, people will get a 401 and not really understand why.
David