You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@deltacloud.apache.org by ma...@redhat.com on 2010/08/13 23:30:11 UTC
Basic Blobstore functions - read containers and blobs
This patch series implements basic bloblstore functionality - at present you can list all your containers and inspect individual blobs for Amazon S3 and EC2. This is mainly to generate discussion (e.g. attributes of a blob? - are we happy with the terms 'containers' and 'blobs' etc) before moving to full blobstore implementation. For containers we do a fudge on content negotiation: 'GET /api/containers/myContainer/myFile.txt' gives html by default, but you can request 'GET /api/containers/myContainer/myFile.txt?format=xml' (json not done yet) - eventually this will be using Accept header.
There is a problem with the s3 gem from AWS which means you can't use it with European buckets but will suffice for now.
Re: Basic Blobstore functions - read containers and blobs
Posted by "marios@redhat.com" <ma...@redhat.com>.
On 13/08/10 22:30, mandreou@redhat.com wrote:
> individual blobs for Amazon S3 and EC2
sorry that should be Amazon S3 and Rackspace Cloudfiles
['PATCH' 2/3] basic blobstore functions - collections, respond_to hack [2/3 one more follows]
Posted by ma...@redhat.com.
From: marios <ma...@redhat.com>
---
server/lib/sinatra/respond_to.rb | 3 +-
server/server.rb | 39 ++++++++++++++++++++++++++++++++++++++
2 files changed, 41 insertions(+), 1 deletions(-)
diff --git a/server/lib/sinatra/respond_to.rb b/server/lib/sinatra/respond_to.rb
index f4704ea..f87628e 100644
--- a/server/lib/sinatra/respond_to.rb
+++ b/server/lib/sinatra/respond_to.rb
@@ -43,7 +43,8 @@ module Sinatra
app.before do
# Let through sinatra image urls in development
next if self.class.development? && request.path_info =~ %r{/__sinatra__/.*?.png}
-
+ next if request.path_info =~ (/\/api\/containers\/*/)
+
unless options.static? && options.public? && (request.get? || request.head?) && static_file?(request.path_info)
rpi = request.path_info.sub(%r{\.([^\./]+)$}, '')
diff --git a/server/server.rb b/server/server.rb
index 1373fa3..e0a61c1 100644
--- a/server/server.rb
+++ b/server/server.rb
@@ -352,3 +352,42 @@ collection :keys do
end
end
+
+VALID_RESPONSE_FORMATS = ['xml', 'XML', 'html', 'HTML', 'json', 'JSON']
+#--
+#* F * I * X * M * E - will ultimately use Accept header to do this
+#--
+get '/api/containers/:container/:blob' do
+ response_format = params['format'] unless (params['format'].nil? || !VALID_RESPONSE_FORMATS.include?(params['format']))
+ response_format ||= 'html'
+ @blob = driver.blob(credentials, { :id => params[:blob], 'container' => params[:container]})
+ if @blob
+ respond_to do |format|
+ case response_format
+ when /html/i then format.html { haml :'blobs/show' }
+ when /xml/i then format.xml { haml :'blobs/show' }
+ when /json/i then format.json { convert_to_json(blobs, @blob) }
+ end
+ end
+ else
+ report_error(404, 'not_found')
+ end
+end
+
+collection :containers do
+ description "Cloud Storage Containers - aka buckets|directories|folders"
+
+ operation :index do
+ description "List containers associated with this account"
+ param :id, :string
+ param :name, :string
+ param :size, :string
+ control { filter_all(:containers) }
+ end
+
+ operation :show do
+ description "Show container"
+ param :id, :string
+ control { show(:container) }
+ end
+end
\ No newline at end of file
--
1.7.2.1
['PATCH' 1/3] basic blobstore functions - driver methods and models here (container, blob) [1/3 two more follow]
Posted by ma...@redhat.com.
From: marios <ma...@redhat.com>
---
server/deltacloud.rb | 2 +
server/lib/deltacloud/base_driver/base_driver.rb | 21 +++++++
server/lib/deltacloud/drivers/ec2/ec2_driver.rb | 62 +++++++++++++++++++-
.../drivers/rackspace/rackspace_driver.rb | 56 ++++++++++++++++++
server/lib/deltacloud/models/blob.rb | 26 ++++++++
server/lib/deltacloud/models/container.rb | 29 +++++++++
6 files changed, 195 insertions(+), 1 deletions(-)
create mode 100644 server/lib/deltacloud/models/blob.rb
create mode 100644 server/lib/deltacloud/models/container.rb
diff --git a/server/deltacloud.rb b/server/deltacloud.rb
index 6799b2f..6ad4afc 100644
--- a/server/deltacloud.rb
+++ b/server/deltacloud.rb
@@ -13,6 +13,8 @@ require 'deltacloud/models/key'
require 'deltacloud/models/instance_profile'
require 'deltacloud/models/storage_snapshot'
require 'deltacloud/models/storage_volume'
+require 'deltacloud/models/container'
+require 'deltacloud/models/blob'
require 'deltacloud/validation'
require 'deltacloud/helpers'
diff --git a/server/lib/deltacloud/base_driver/base_driver.rb b/server/lib/deltacloud/base_driver/base_driver.rb
index 88563a5..c21ccfa 100644
--- a/server/lib/deltacloud/base_driver/base_driver.rb
+++ b/server/lib/deltacloud/base_driver/base_driver.rb
@@ -185,6 +185,27 @@ module Deltacloud
[]
end
+ def containers(credentials, opts = nil)
+ #list of containers belonging to account
+ []
+ end
+
+ def container(credentials, opts = nil)
+ #list of objects within container
+ list = containers(credentials, opts)
+ return list.first unless list.empty?
+ nil
+ end
+
+ def blobs(credentials, opts = nil)
+ []
+ end
+
+ def blob(credentials, opts = nil)
+ list = blobs(credentials, opts)
+ return list.first unless list.empty?
+ end
+
def filter_on(collection, attribute, opts)
return collection if opts.nil?
return collection if opts[attribute].nil?
diff --git a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
index 909eca3..5ad745b 100644
--- a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
+++ b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
@@ -19,6 +19,7 @@
require 'deltacloud/base_driver'
require 'AWS'
+require 'aws/s3' #FIXME NEED A NEW S3 GEM Amazon gem has issues with European buckets (some kind of naming/addressing problem)...
class Instance
attr_accessor :keyname
@@ -36,7 +37,7 @@ module Deltacloud
class EC2Driver < Deltacloud::BaseDriver
def supported_collections
- DEFAULT_COLLECTIONS + [ :keys ]
+ DEFAULT_COLLECTIONS + [ :keys, :containers ]
end
feature :instances, :user_data
@@ -276,6 +277,37 @@ class EC2Driver < Deltacloud::BaseDriver
end
snapshots
end
+
+#--
+# Containers (buckets in S3 storage)
+#-- get back a list of your buckets from the s3 service
+ def containers(credentials, opts)
+ container_list = []
+ s3_connect(credentials) #establish a connection with the S3 service
+ bucket_list = AWS::S3::Service.buckets
+ bucket_list.each do |current|
+ container_list << convert_storage_container(current)
+ end
+ container_list = filter_on(container_list, :id, opts)
+ container_list
+ end
+
+#--
+# BLOBS get a list of blobs for a given bucket
+#--
+ def blobs(credentials, opts = nil)
+ unless opts['container'] then
+ raise Deltacloud::Validation::Failure.new(Deltacloud::Validation::Param.new(["container"]), "Error - need container name to retrieve the blob list. You said bucket->#{opts['container']}.")
+ end
+ s3_connect(credentials)
+ s3_bucket = AWS::S3::Bucket.find(opts['container'])
+ blobs = []
+ s3_bucket.objects.each do |s3_object|
+ blobs << convert_storage_object(s3_object)
+ end
+ blobs = filter_on(blobs, :id, opts)
+ blobs
+ end
def key(credentials, opts=nil)
keys(credentials, opts).first
@@ -395,6 +427,34 @@ class EC2Driver < Deltacloud::BaseDriver
} )
end
+ def s3_connect(credentials)
+ AWS::S3::Base.establish_connection!(:access_key_id => credentials.user, :secret_access_key => credentials.password)
+ end
+
+ def convert_storage_container(s3_bucket)
+ #get blob list:
+ blob_list = []
+ s3_bucket.objects.each do |s3_object|
+ blob_list << s3_object.key
+ end
+ #can use AWS::S3::Owner.current.display_name or current.id
+ Container.new( { :id => s3_bucket.name,
+ :name => s3_bucket.name,
+ :size => s3_bucket.size,
+ :blob_list => blob_list
+ }
+ )
+ end
+
+ def convert_storage_object(s3_object)
+ Blob.new({ :id => s3_object.key,
+ :container => s3_object.bucket.name.to_s,
+ :content_length => s3_object.about['content-length'],
+ :content_type => s3_object.about['content-type'],
+ :last_modified => s3_object.about['last-modified']
+ })
+ end
+
def catched_exceptions_list
{
:auth => [ AWS::AuthFailure ],
diff --git a/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb b/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
index 6d6ba0b..e6fbf97 100644
--- a/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
+++ b/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
@@ -18,6 +18,7 @@
require 'deltacloud/base_driver'
require 'deltacloud/drivers/rackspace/rackspace_client'
+require 'cloudfiles'
module Deltacloud
module Drivers
@@ -25,6 +26,11 @@ module Deltacloud
class RackspaceDriver < Deltacloud::BaseDriver
+ def supported_collections
+ DEFAULT_COLLECTIONS + [ :containers ]
+ end
+
+
feature :instances, :user_name
def hardware_profiles(credentials, opts = nil)
@@ -148,6 +154,56 @@ class RackspaceDriver < Deltacloud::BaseDriver
inst
end
+#--
+# Containers
+#--
+ def containers(credentials, opts)
+ container_list = []
+ cf = cloudfiles_client(credentials)
+ cf.containers.each do |container_name|
+ current = cf.container(container_name)
+ container_list << convert_container(current)
+ end
+ container_list = filter_on(container_list, :id, opts)
+ container_list
+ end
+
+ def blobs(credentials, opts)
+ unless opts['container'] then
+ raise Deltacloud::Validation::Failure.new(Deltacloud::Validation::Param.new(["container"]), "Error - need container name to retrieve the blob list. You said bucket->#{opts['container']}.")
+ end
+ cf = cloudfiles_client(credentials)
+ blobs = []
+ cf_container = cf.container(opts['container'])
+ cf_container.objects.each do |object_name|
+ blobs << convert_object(cf_container.object(object_name))
+ end
+ blobs = filter_on(blobs, :id, opts)
+ blobs
+ end
+
+ def convert_container(cf_container)
+ Container.new({ :id => cf_container.name,
+ :name => cf_container.name,
+ :size => cf_container.count,
+ :blob_list => cf_container.objects
+ })
+ end
+
+ def convert_object(cf_object)
+ Blob.new({ :id => cf_object.name,
+ :container => cf_object.container.name,
+ :content_length => cf_object.bytes,
+ :content_type => cf_object.content_type,
+ :last_modified => cf_object.last_modified
+ })
+ end
+
+ def cloudfiles_client(credentials)
+ CloudFiles::Connection.new(credentials.user, credentials.password)
+ end
+
+
def new_client(credentials)
safely do
return RackspaceClient.new(credentials.user, credentials.password)
diff --git a/server/lib/deltacloud/models/blob.rb b/server/lib/deltacloud/models/blob.rb
new file mode 100644
index 0000000..080b426
--- /dev/null
+++ b/server/lib/deltacloud/models/blob.rb
@@ -0,0 +1,26 @@
+#
+# Copyright (C) 2009 Red Hat, Inc.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership. The
+# ASF licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+class Blob < BaseModel
+ #already has an id from basemodel (for the key)
+ attr_accessor :container
+ attr_accessor :content_length
+ attr_accessor :content_type
+ attr_accessor :last_modified
+ attr_accessor :content
+end
\ No newline at end of file
diff --git a/server/lib/deltacloud/models/container.rb b/server/lib/deltacloud/models/container.rb
new file mode 100644
index 0000000..5696061
--- /dev/null
+++ b/server/lib/deltacloud/models/container.rb
@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2009 Red Hat, Inc.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership. The
+# ASF licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+class Container < BaseModel
+
+ attr_accessor :name
+ attr_accessor :size
+ attr_accessor :blob_list
+ #size is number of objects
+ #>> b = Bucket.find('mariosbucket1')
+ #=> #<AWS::S3::Bucket:0xb74159d0 @object_cache=[],
+ # @attributes={"prefix"=>nil, "name"=>"mariosbucket1", "marker"=>nil,
+ # "max_keys"=>1000, "is_truncated"=>false, "xmlns"=>"http://s3.amazonaws.com/doc/2006-03-01/"}>
+end
\ No newline at end of file
--
1.7.2.1
['PATCH' 3/3] basic blobstore functions - views haml for container, blob [3/3]
Posted by ma...@redhat.com.
From: marios <ma...@redhat.com>
---
server/views/blobs/show.html.haml | 18 ++++++++++++++++++
server/views/blobs/show.xml.haml | 5 +++++
server/views/containers/index.html.haml | 27 +++++++++++++++++++++++++++
server/views/containers/index.xml.haml | 8 ++++++++
server/views/containers/show.html.haml | 16 ++++++++++++++++
server/views/containers/show.xml.haml | 9 +++++++++
6 files changed, 83 insertions(+), 0 deletions(-)
create mode 100644 server/views/blobs/show.html.haml
create mode 100644 server/views/blobs/show.xml.haml
create mode 100644 server/views/containers/index.html.haml
create mode 100644 server/views/containers/index.xml.haml
create mode 100644 server/views/containers/show.html.haml
create mode 100644 server/views/containers/show.xml.haml
diff --git a/server/views/blobs/show.html.haml b/server/views/blobs/show.html.haml
new file mode 100644
index 0000000..f038b8e
--- /dev/null
+++ b/server/views/blobs/show.html.haml
@@ -0,0 +1,18 @@
+%h1 Blob
+%h2
+ = @blob.id
+
+%dl
+ %dt Container
+ %dd
+ = @blob.container
+ %dt Content_Length
+ %dd
+ = @blob.content_length
+ %dt Content_Type
+ %dd
+ = @blob.content_type
+ %dt Last_Modified
+ %dd
+ = @blob.last_modified
+
diff --git a/server/views/blobs/show.xml.haml b/server/views/blobs/show.xml.haml
new file mode 100644
index 0000000..0f8597e
--- /dev/null
+++ b/server/views/blobs/show.xml.haml
@@ -0,0 +1,5 @@
+!!! XML
+%blob{:href => container_url(@blob.container) + '/' + @blob.id, :id => @blob.id}
+ - @blob.attributes.select{ |attr| attr!=:id }.each do |attribute|
+ -haml_tag(attribute, :<) do
+ - haml_concat @blob.send(attribute)
\ No newline at end of file
diff --git a/server/views/containers/index.html.haml b/server/views/containers/index.html.haml
new file mode 100644
index 0000000..a199ace
--- /dev/null
+++ b/server/views/containers/index.html.haml
@@ -0,0 +1,27 @@
+%h1
+ Containers
+
+%table.display
+ %thead
+ %tr
+ %th
+ ID
+ %th
+ Name
+ %th
+ Size
+ %th
+ Blob List
+
+ %tbody
+ - @containers.each do |container|
+ %tr
+ %td
+ = link_to container.id, container_url(container.id)
+ %td
+ = container.name
+ %td
+ = container.size
+ %td
+ -container.blob_list.each do |blob|
+ = blob
\ No newline at end of file
diff --git a/server/views/containers/index.xml.haml b/server/views/containers/index.xml.haml
new file mode 100644
index 0000000..86c4349
--- /dev/null
+++ b/server/views/containers/index.xml.haml
@@ -0,0 +1,8 @@
+!!! XML
+%containers
+ - @elements.each do |container|
+ %container{:href => container_url(container.id), :id => container.id}
+ - container.attributes.select{ |attr| attr!=:id }.each do |attribute|
+ - haml_tag("#{attribute}".tr('-', '_'), :<) do
+ - haml_concat container.send(attribute)
+
diff --git a/server/views/containers/show.html.haml b/server/views/containers/show.html.haml
new file mode 100644
index 0000000..dea2603
--- /dev/null
+++ b/server/views/containers/show.html.haml
@@ -0,0 +1,16 @@
+%h1 Container
+%h2
+ = @container.id
+
+%dl
+ %di
+ %dt Name
+ %dd
+ = @container.name
+ %dt Size
+ %dd
+ = @container.size
+ %dt Objects
+ %dd
+ -@container.blob_list.each do |blob|
+ = link_to blob, container_url(@container.name) + '/' + blob
\ No newline at end of file
diff --git a/server/views/containers/show.xml.haml b/server/views/containers/show.xml.haml
new file mode 100644
index 0000000..df27074
--- /dev/null
+++ b/server/views/containers/show.xml.haml
@@ -0,0 +1,9 @@
+!!! XML
+%container{:href => container_url(@container.id), :id => @container.id}
+ - @container.attributes.select{ |attr| attr!=:id }.each do |attribute|
+ - unless attribute == :blob_list
+ -haml_tag(attribute, :<) do
+ - haml_concat @container.send(attribute)
+ - @container.blob_list.each do |blb|
+ %blob{:href => container_url(@container.id + blb), :id => blb}
+
\ No newline at end of file
--
1.7.2.1