You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@deltacloud.apache.org by ma...@redhat.com on 2011/01/28 19:15:06 UTC
[PATCH] Introduces blob metadata - Adds ability to specify arbitrary user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during blob creation
From: marios <ma...@redhat.com>
---
server/lib/deltacloud/base_driver/base_driver.rb | 4 +-
.../lib/deltacloud/drivers/azure/azure_driver.rb | 47 +++++++++++++-------
server/lib/deltacloud/drivers/ec2/ec2_driver.rb | 23 ++++++----
server/lib/deltacloud/drivers/mock/mock_driver.rb | 10 ++--
.../drivers/rackspace/rackspace_driver.rb | 37 ++++++++++------
server/lib/deltacloud/helpers/blob_stream.rb | 17 +++++++
server/lib/deltacloud/models/blob.rb | 1 +
server/public/javascripts/application.js | 35 +++++++++++++++
server/server.rb | 18 +++++++-
server/views/blobs/new.html.haml | 18 +++++++-
server/views/blobs/show.html.haml | 6 +++
server/views/blobs/show.xml.haml | 6 ++-
12 files changed, 173 insertions(+), 49 deletions(-)
diff --git a/server/lib/deltacloud/base_driver/base_driver.rb b/server/lib/deltacloud/base_driver/base_driver.rb
index bf06e56..2442385 100644
--- a/server/lib/deltacloud/base_driver/base_driver.rb
+++ b/server/lib/deltacloud/base_driver/base_driver.rb
@@ -186,12 +186,12 @@ module Deltacloud
storage_snapshots(credentials, opts).first if has_capability?(:storage_snapshots)
end
- def bucket(credentials, opts = nil)
+ def bucket(credentials, opts = {})
#list of objects within bucket
buckets(credentials, opts).first if has_capability?(:buckets)
end
- def blob(credentials, opts = nil)
+ def blob(credentials, opts = {})
blobs(credentials, opts).first if has_capability?(:blobs)
end
diff --git a/server/lib/deltacloud/drivers/azure/azure_driver.rb b/server/lib/deltacloud/drivers/azure/azure_driver.rb
index abbd353..dce63dd 100644
--- a/server/lib/deltacloud/drivers/azure/azure_driver.rb
+++ b/server/lib/deltacloud/drivers/azure/azure_driver.rb
@@ -32,7 +32,7 @@ class AzureDriver < Deltacloud::BaseDriver
#--
# Buckets
#--
- def buckets(credentials, opts)
+ def buckets(credentials, opts={})
buckets = []
azure_connect(credentials)
safely do
@@ -46,7 +46,7 @@ class AzureDriver < Deltacloud::BaseDriver
#--
# Create bucket
#--
- def create_bucket(credentials, name, opts)
+ def create_bucket(credentials, name, opts={})
#for whatever reason, bucket names MUST be lowercase...
#http://msdn.microsoft.com/en-us/library/dd135715.aspx
name.downcase!
@@ -62,7 +62,7 @@ class AzureDriver < Deltacloud::BaseDriver
#--
# Delete bucket
#--
- def delete_bucket(credentials, name, opts)
+ def delete_bucket(credentials, name, opts={})
azure_connect(credentials)
safely do
WAZ::Blobs::Container.find(name).destroy!
@@ -72,20 +72,25 @@ class AzureDriver < Deltacloud::BaseDriver
#--
# Blobs
#--
- def blobs(credentials, opts)
+ def blobs(credentials, opts={})
blob_list = []
azure_connect(credentials)
safely do
the_bucket = WAZ::Blobs::Container.find(opts['bucket'])
- the_bucket.blobs.each do |waz_blob|
- blob_list << convert_blob(waz_blob)
- end
- end
+ if(opts[:id])
+ the_blob = the_bucket[opts[:id]]
+ blob_list << convert_blob(the_blob) unless the_blob.nil?
+ else
+ the_bucket.blobs.each do |waz_blob|
+ blob_list << convert_blob(waz_blob)
+ end #each.do
+ end #if
+ end #safely do
blob_list = filter_on(blob_list, :id, opts)
blob_list
end
- def blob_data(credentials, bucket_id, blob_id, opts)
+ def blob_data(credentials, bucket_id, blob_id, opts={})
azure_connect(credentials)
# WAZ get blob data methods cant accept blocks for 'streaming'... FIXME
yield WAZ::Blobs::Container.find(bucket_id)[blob_id].value
@@ -94,23 +99,28 @@ class AzureDriver < Deltacloud::BaseDriver
#--
# Create Blob
#--
- def create_blob(credentials, bucket_id, blob_id, blob_data, opts=nil)
+ def create_blob(credentials, bucket_id, blob_id, blob_data, opts={})
azure_connect(credentials)
- #get a handle to the bucket in order to put there
- the_bucket = WAZ::Blobs::Container.find(bucket_id)
- the_bucket.store(blob_id, blob_data[:tempfile], blob_data[:type])
+ #insert azure-specific header for user metadata ... x-ms-meta-kEY = VALUE
+ opts.gsub_keys("HTTP_X_Deltacloud_Blobmeta_", "x-ms-meta-")
+ safely do
+ #get a handle to the bucket in order to put there
+ the_bucket = WAZ::Blobs::Container.find(bucket_id)
+ the_bucket.store(blob_id, blob_data[:tempfile], blob_data[:type], opts)
+ end
Blob.new( { :id => blob_id,
:bucket => bucket_id,
:content_lengh => blob_data[:tempfile].length,
:content_type => blob_data[:type],
- :last_modified => ''
+ :last_modified => '',
+ :user_metadata => opts.gsub_keys('x-ms-meta-','')
} )
end
#--
# Delete Blob
#--
- def delete_blob(credentials, bucket_id, blob_id, opts=nil)
+ def delete_blob(credentials, bucket_id, blob_id, opts={})
azure_connect(credentials)
#get a handle to bucket and blob, and destroy!
the_bucket = WAZ::Blobs::Container.find(bucket_id)
@@ -142,11 +152,16 @@ class AzureDriver < Deltacloud::BaseDriver
def convert_blob(waz_blob)
url = waz_blob.url.split('/')
bucket = url[url.length-2] #FIXME
+ #get only user defined metadata
+ blob_metadata = waz_blob.metadata.reject{|k,v| (!(k.to_s.match('x_ms_meta')))}
+ #strip off the x_ms_meta_ from each key
+ blob_metadata.gsub_keys('x_ms_meta_', '')
Blob.new({ :id => waz_blob.name,
:bucket => bucket,
:content_length => waz_blob.metadata[:content_length],
:content_type => waz_blob.metadata[:content_type],
- :last_modified => waz_blob.metadata[:last_modified]
+ :last_modified => waz_blob.metadata[:last_modified],
+ :user_metadata => blob_metadata
})
end
diff --git a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
index bcd848e..2d0194f 100644
--- a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
+++ b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
@@ -324,7 +324,7 @@ module Deltacloud
end
- def buckets(credentials, opts)
+ def buckets(credentials, opts={})
buckets = []
safely do
s3_client = new_client(credentials, :s3)
@@ -357,7 +357,7 @@ module Deltacloud
end
end
- def blobs(credentials, opts = nil)
+ def blobs(credentials, opts = {})
s3_client = new_client(credentials, :s3)
blobs = []
safely do
@@ -373,25 +373,29 @@ module Deltacloud
#--
# Create Blob
#--
- def create_blob(credentials, bucket_id, blob_id, data = nil, opts = nil)
+ def create_blob(credentials, bucket_id, blob_id, data = nil, opts = {})
s3_client = new_client(credentials, :s3)
#data is a construct with the temporary file created by server @.tempfile
#also file[:type] will give us the content-type
res = nil
# File stream needs to be reopened in binary mode for whatever reason
file = File::open(data[:tempfile].path, 'rb')
+ #insert ec2-specific header for user metadata ... x-amz-meta-KEY = VALUE
+ opts.gsub_keys('HTTP_X_Deltacloud_Blobmeta_', 'x-amz-meta-')
+ opts["Content-Type"] = data[:type]
safely do
res = s3_client.interface.put(bucket_id,
blob_id,
file,
- {"Content-Type" => data[:type]})
+ opts)
end
#create a new Blob object and return that
Blob.new( { :id => blob_id,
:bucket => bucket_id,
:content_length => data[:tempfile].length,
:content_type => data[:type],
- :last_modified => ''
+ :last_modified => '',
+ :user_metadata => opts.select{|k,v| k.match(/^x-amz-meta-/)}
}
)
end
@@ -399,7 +403,7 @@ module Deltacloud
#--
# Delete Blob
#--
- def delete_blob(credentials, bucket_id, blob_id, opts=nil)
+ def delete_blob(credentials, bucket_id, blob_id, opts={})
s3_client = new_client(credentials, :s3)
safely do
s3_client.interface.delete(bucket_id, blob_id)
@@ -407,7 +411,7 @@ module Deltacloud
end
- def blob_data(credentials, bucket_id, blob_id, opts)
+ def blob_data(credentials, bucket_id, blob_id, opts={})
s3_client = new_client(credentials, :s3)
safely do
s3_client.interface.get(bucket_id, blob_id) do |chunk|
@@ -522,7 +526,7 @@ module Deltacloud
'us-east-1' => 'elasticloadbalancing.us-east-1.amazonaws.com',
'us-west-1' => 'elasticloadbalancing.us-west-1.amazonaws.com'
},
-
+
's3' => {
'us-east-1' => 's3.amazonaws.com',
'us-west-1' => 's3-us-west-1.amazonaws.com',
@@ -587,7 +591,8 @@ module Deltacloud
:bucket => s3_object.bucket.name.to_s,
:content_length => s3_object.headers['content-length'],
:content_type => s3_object.headers['content-type'],
- :last_modified => s3_object.last_modified
+ :last_modified => s3_object.last_modified,
+ :user_metadata => s3_object.meta_headers
)
end
diff --git a/server/lib/deltacloud/drivers/mock/mock_driver.rb b/server/lib/deltacloud/drivers/mock/mock_driver.rb
index da2931b..a36d569 100644
--- a/server/lib/deltacloud/drivers/mock/mock_driver.rb
+++ b/server/lib/deltacloud/drivers/mock/mock_driver.rb
@@ -318,7 +318,7 @@ class MockDriver < Deltacloud::BaseDriver
#--
# Buckets
#--
- def buckets(credentials, opts=nil)
+ def buckets(credentials, opts={})
check_credentials(credentials)
buckets=[]
Dir[ "#{@storage_root}/buckets/*.yml" ].each do |bucket_file|
@@ -334,7 +334,7 @@ class MockDriver < Deltacloud::BaseDriver
#--
# Create bucket
#--
- def create_bucket(credentials, name, opts=nil)
+ def create_bucket(credentials, name, opts={})
check_credentials(credentials)
bucket = {
:id => name,
@@ -349,7 +349,7 @@ class MockDriver < Deltacloud::BaseDriver
#--
# Delete bucket
#--
- def delete_bucket(credentials, name, opts=nil)
+ def delete_bucket(credentials, name, opts={})
bucket = bucket(credentials, {:id => name})
unless (bucket.size == "0")
raise Deltacloud::BackendError.new(403, self.class.to_s, "bucket-not-empty", "delete operation not valid for non-empty bucket")
@@ -362,7 +362,7 @@ class MockDriver < Deltacloud::BaseDriver
#--
# Blobs
#--
- def blobs(credentials, opts = nil)
+ def blobs(credentials, opts = {})
check_credentials(credentials)
blobs=[]
Dir[ "#{@storage_root}/buckets/blobs/*.yml" ].each do |blob_file|
@@ -378,7 +378,7 @@ class MockDriver < Deltacloud::BaseDriver
#--
# Blob content
#--
- def blob_data(credentials, bucket_id, blob_id, opts = nil)
+ def blob_data(credentials, bucket_id, blob_id, opts = {})
check_credentials(credentials)
blob=nil
Dir[ "#{@storage_root}/buckets/blobs/*.yml" ].each do |blob_file|
diff --git a/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb b/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
index e40fb9c..8e6612b 100644
--- a/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
+++ b/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
@@ -166,7 +166,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
#--
# Buckets
#--
- def buckets(credentials, opts)
+ def buckets(credentials, opts = {})
bucket_list = []
cf = cloudfiles_client(credentials)
safely do
@@ -182,7 +182,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
#--
# Create Bucket
#--
- def create_bucket(credentials, name, opts)
+ def create_bucket(credentials, name, opts = {})
bucket = nil
cf = cloudfiles_client(credentials)
safely do
@@ -195,7 +195,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
#--
# Delete Bucket
#--
- def delete_bucket(credentials, name, opts)
+ def delete_bucket(credentials, name, opts = {})
cf = cloudfiles_client(credentials)
safely do
cf.delete_container(name)
@@ -205,7 +205,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
#--
# Blobs
#--
- def blobs(credentials, opts)
+ def blobs(credentials, opts = {})
cf = cloudfiles_client(credentials)
blobs = []
safely do
@@ -221,7 +221,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
#-
# Blob data
#-
- def blob_data(credentials, bucket_id, blob_id, opts)
+ def blob_data(credentials, bucket_id, blob_id, opts = {})
cf = cloudfiles_client(credentials)
cf.container(bucket_id).object(blob_id).data_stream do |chunk|
yield chunk
@@ -231,18 +231,26 @@ class RackspaceDriver < Deltacloud::BaseDriver
#--
# Create Blob
#--
- def create_blob(credentials, bucket_id, blob_id, blob_data, opts=nil)
+ def create_blob(credentials, bucket_id, blob_id, blob_data, opts={})
cf = cloudfiles_client(credentials)
- #must first create the object using cloudfiles_client.create_object
- #then can write using object.write(data)
- object = cf.container(bucket_id).create_object(blob_id)
- #blob_data is a construct with data in .tempfile and content-type in {:type}
- res = object.write(blob_data[:tempfile], {'Content-Type' => blob_data[:type]})
+ user_meta = opts.select{|k,v| k.match(/^META-/)}
+ #insert ec2-specific header for user metadata ... X-Object-Meta-KEY = VALUE
+ opts.gsub_keys("HTTP_X_Deltacloud_Blobmeta_", "X-Object-Meta-")
+ opts['Content-Type'] = blob_data[:type]
+ object = nil
+ safely do
+ #must first create the object using cloudfiles_client.create_object
+ #then can write using object.write(data)
+ object = cf.container(bucket_id).create_object(blob_id)
+ #blob_data is a construct with data in .tempfile and content-type in {:type}
+ res = object.write(blob_data[:tempfile], opts)
+ end
Blob.new( { :id => object.name,
:bucket => object.container.name,
:content_length => blob_data[:tempfile].length,
:content_type => blob_data[:type],
- :last_modified => ''
+ :last_modified => '',
+ :user_metadata => opts.select{|k,v| k.match(/^X-Object-Meta-/)}
}
)
end
@@ -250,7 +258,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
#--
# Delete Blob
#--
- def delete_blob(credentials, bucket_id, blob_id, opts=nil)
+ def delete_blob(credentials, bucket_id, blob_id, opts={})
cf = cloudfiles_client(credentials)
cf.container(bucket_id).delete_object(blob_id)
end
@@ -292,7 +300,8 @@ private
:bucket => cf_object.container.name,
:content_length => cf_object.bytes,
:content_type => cf_object.content_type,
- :last_modified => cf_object.last_modified
+ :last_modified => cf_object.last_modified,
+ :user_metadata => cf_object.metadata
})
end
diff --git a/server/lib/deltacloud/helpers/blob_stream.rb b/server/lib/deltacloud/helpers/blob_stream.rb
index a99e6c2..6a8a2c5 100644
--- a/server/lib/deltacloud/helpers/blob_stream.rb
+++ b/server/lib/deltacloud/helpers/blob_stream.rb
@@ -62,3 +62,20 @@ rescue LoadError => e
end
end
end
+
+class Hash
+
+ def gsub_keys(pattern, replacement)
+ remove = []
+ self.each_key do |key|
+ if key.to_s.match(pattern)
+ new_key = key.to_s.gsub(pattern, replacement)
+ self[new_key] = self[key]
+ remove << key
+ end #key.match
+ end # each_key do
+ #remove the original keys
+ self.delete_if{|k,v| remove.include?(k)}
+ end #def
+
+end #class
diff --git a/server/lib/deltacloud/models/blob.rb b/server/lib/deltacloud/models/blob.rb
index dfa67fe..0bf473b 100644
--- a/server/lib/deltacloud/models/blob.rb
+++ b/server/lib/deltacloud/models/blob.rb
@@ -24,5 +24,6 @@ class Blob < BaseModel
attr_accessor :content_type
attr_accessor :last_modified
attr_accessor :content
+ attr_accessor :user_metadata
end
diff --git a/server/public/javascripts/application.js b/server/public/javascripts/application.js
index 5ee0f7f..95c9bc2 100644
--- a/server/public/javascripts/application.js
+++ b/server/public/javascripts/application.js
@@ -16,3 +16,38 @@ $(document).ready(function() {
}
})
+
+function more_fields()
+{
+ //increment the hidden input that captures how many meta_data are passed
+ var meta_params = document.getElementsByName('meta_params')
+ current_number_params = eval(meta_params[0].value)+1
+ meta_params[0].value = current_number_params
+ var new_meta = document.getElementById('metadata_holder').cloneNode(true);
+ new_meta.id = 'metadata_holder' + current_number_params;
+ new_meta.style.display = 'block';
+ var nodes = new_meta.childNodes;
+ for (var i=0;i < nodes.length;i++) {
+ var theName = nodes[i].name;
+ if (theName)
+ nodes[i].name = theName + current_number_params;
+ }
+ var insertHere = document.getElementById('metadata_holder');
+ insertHere.parentNode.insertBefore(new_meta,insertHere);
+}
+
+function less_fields()
+{
+ var meta_params = document.getElementsByName('meta_params')
+ current_val = eval(meta_params[0].value)
+ if (current_val == 0)
+ {
+ return;
+ }
+ else
+ {
+ var theDiv = document.getElementById('metadata_holder'+current_val)
+ theDiv.parentNode.removeChild(theDiv)
+ meta_params[0].value = eval(current_val)-1
+ }
+}
diff --git a/server/server.rb b/server/server.rb
index 043cbee..8cdabc6 100644
--- a/server/server.rb
+++ b/server/server.rb
@@ -626,7 +626,23 @@ post '/api/buckets/:bucket' do
bucket_id = params[:bucket]
blob_id = params['blob_id']
blob_data = params['blob_data']
- @blob = driver.create_blob(credentials, bucket_id, blob_id, blob_data )
+ user_meta = {}
+#first try get blob_metadata from params (i.e., passed by http form post, e.g. browser)
+ max = params[:meta_params]
+ if(max)
+ (1..max.to_i).each do |i|
+ key = params[:"meta_name#{i}"]
+ key = "HTTP_X_Deltacloud_Blobmeta_#{key}"
+ value = params[:"meta_value#{i}"]
+ user_meta[key] = value
+ end #max.each do
+ else #can try to get blob_metadata from http headers
+ meta_array = request.env.select{|k,v| k.match(/^HTTP[-_]X[-_]Deltacloud[-_]Blobmeta[-_]/i)}
+ meta_array.each do |k,v|
+ user_meta[k] = v
+ end
+ end #end if
+ @blob = driver.create_blob(credentials, bucket_id, blob_id, blob_data, user_meta)
respond_to do |format|
format.html { haml :"blobs/show"}
format.xml { haml :"blobs/show" }
diff --git a/server/views/blobs/new.html.haml b/server/views/blobs/new.html.haml
index feb94e5..a075f0a 100644
--- a/server/views/blobs/new.html.haml
+++ b/server/views/blobs/new.html.haml
@@ -4,7 +4,23 @@
%label
Blob Name:
%input{ :name => 'blob_id', :size => 512}/
+ %label
Blob Data:
- %input{ :type => "file", :name => 'blob_data', :size => 50}/
%br
+ %input{ :type => "file", :name => 'blob_data', :size => 50}/
+ %br
+ %br
+ %input{ :type => "hidden", :name => "meta_params", :value => "0"}
+ %a{ :href => "javascript:;", :onclick => "more_fields();"} Add Metadata
+ %div{ :id => "metadata_holder", :style => "display: none;"}
+ %label
+ Metadata Key:
+ %input{ :type => "text", :name => "meta_name", :value => ""}/
+ %label
+ Metadata Value:
+ %input{ :type => "text", :name => "meta_value", :value => ""}/
+ %br
+ %a{ :href => "javascript:;", :onclick => "less_fields();"} Less Metadata
+ %br
+ %br
%input{ :type => :submit, :name => "commit", :value => "create"}/
diff --git a/server/views/blobs/show.html.haml b/server/views/blobs/show.html.haml
index e5c2cee..99c090a 100644
--- a/server/views/blobs/show.html.haml
+++ b/server/views/blobs/show.html.haml
@@ -19,6 +19,12 @@
%dt Content
%dd
=link_to 'Blob content', bucket_url(@blob.bucket) + '/' + @blob.id + '/content'
+ %dt User_Metadata
+ %dd
+ -@blob.user_metadata.each do |k,v|
+ %b #{k} :
+ #{v}
+ %br
%dt Delete this Blob
%dd
%form{ :action => bucket_url(@blob.bucket) + '/' + @blob.id , :method => :post }
diff --git a/server/views/blobs/show.xml.haml b/server/views/blobs/show.xml.haml
index bc499ca..34b9cca 100644
--- a/server/views/blobs/show.xml.haml
+++ b/server/views/blobs/show.xml.haml
@@ -1,7 +1,11 @@
!!! XML
%blob{:href => bucket_url(@blob.bucket) + '/' + @blob.id, :id => @blob.id}
- - @blob.attributes.select{ |attr| attr!=:id}.each do |attribute|
+ - @blob.attributes.select{ |attr| (attr!=:id && attr!=:user_metadata) }.each do |attribute|
- unless attribute == :content
- haml_tag(attribute, :<) do
- haml_concat @blob.send(attribute)
+ %user_metadata
+ - @blob.user_metadata.each do |k, v|
+ %entry{:key => k}
+ #{cdata v}
%content{:href => bucket_url(@blob.bucket) + '/' + @blob.id + '/content'}
--
1.7.3.4
Re: [PATCH] Introduces blob metadata - Adds ability to specify arbitrary user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during blob creation
Posted by Michal Fojtik <mf...@redhat.com>.
On Feb 1, 2011, at 6:16 PM, marios@redhat.com wrote:
> still playing with .inject :) ...
>
> (for the azure driver) the azure server responds with a metadata hash like:
>
> {:x_ms_blob_type=>"BlockBlob", :date=>"Tue, 01 Feb 2011 17:07:07 GMT", :content_type=>"text/plain", :x_ms_lease_status=>"unlocked", :x_ms_version=>"2009-09-19", :x_ms_meta_v=>"2.1", :etag=>"0x8CD8CFB422A88C4", :x_ms_meta_foo=>"bar", :content_length=>"86", :server=>"Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", :last_modified=>"Fri, 28 Jan 2011 12:08:50 GMT", :x_ms_request_id=>"d757adb5-dcf9-46be-8463-357e3f88c368"}
>
>
> The above hash is 'waz_blob.metadata'. I needed to extract a new hash which I call 'blob_metadata', containing all the key:value where key was 'x_ms_meta_' (blob metadata key). So, especially for Michal:
>
> waz_blob.metadata.inject({}) { |result_hash, (k,v)| blob_metadata[k]=v if k.to_s.match(/x_ms_meta/i)}
Like that very much! :-) I didn't know that you can do something like |result_hash, (k,v)|, that could
make code much more readable :-) (as using v[1]/v[2]), thanks!.
-- Michal
> :) thanks
>
> marios
>
>
>
>
>
> On 01/02/11 17:08, Michal Fojtik wrote:
>> On 31/01/11 22:52 +0200, marios@redhat.com wrote:
>>
>>>> Yes, actually it's http://rubybestpractices.com ;-) There is an article
>>>> about functional programming, which describes this.
>>>
>>> This site is a ruby book (pdf) - by article i assume you meant
>>> chapter... i tried searching but only thing I could find was a
>>> comparison of using .inject vs using recursion. However its a pretty
>>> big book so I may well have missed the relevant bit.
>>
>> Maybe it's not this book, but meta-programming one but I saw some
>> comparison between looping vs inject in some of them. I'll do the job today
>> and do detailed research in those book.
>>
>>> That got me curious; what *actually* is the advantage of using
>>> .inject. Don't get me wrong, I still think its pretty cool and elegant
>>> in its concise expressiveness, but most search results seem to suggest
>>> that .inject is actually slower (for this case, it makes no
>>> difference, since the input will never really be a HUGE structure; we
>>> are talking about key:value pairs for blob metadata). The way I
>>> understood it, if you are handling large data structures, the
>>> resultant object is returned at each 'loop' of the inner code block,
>>> making it expensive.
>>
>> Did some research and I found:
>>
>> http://stackoverflow.com/questions/318854/inject-and-slowness
>>
>> So yes, it seems like I was wrong and it's slow. But if I understand it
>> correctly
>> it's just implementation issue, so in meantime (eg in Ruby 1.9) it should
>> be fast.
>> The reason why I like .inject() is that it's one of the Ruby fundamental
>> features and it's more 'ruby-ish' ;-) So besides that current
>> implementation is slow, it can be improved by programmers and it's
>> always good to use it now than refactor later (from my point of view).
>> But you're right that in this particular case (blob) it absolutely doesn't
>> matter what approach we choose.
>>
>>>> Anyway, it's not a blocker, just small hint (I found myself .inject very
>>>> useful when I was doing some meta programming in client library. It
>>>> saves
>>>> variables and lines of code (and it's better for garbage collecting)).
>>>
>>> In this particular case it doesn't save variables ( I assume you're
>>> referring to the instantiation of the resulting structure, 'user_meta'
>>> in this case). Do you have a reference/article/explanation for the
>>> 'better for garbage collecting' (aka, please give me a reason to use
>>> .inject rather than .each, since the former is clearly much cooler),
>>
>> I did some research about that and I found this 'hint':
>>
>> When is each useful? Simple: when you want to create side-effects, like
>> saving to the database, printing a result, or sending a web service call.
>> In these cases, you’re not concerned with the return value; you want to
>> change state on the screen, the disk, the database, or something else.
>>
>> But don’t use each if you want to extract some new value from an array.
>> That’s not what it’s for. Instead, take a look at three other powerful
>> functions: map, inject, or select.
>>
>> http://railspikes.com/2008/7/29/functional-loops-in-ruby-each-map-inject-select-and-for
>>
>>
>> -- Michal
>>
>>
>>>
>>> marios
>>>
>>
>
Michal Fojtik
Software Engineer, Deltacloud API project
http://www.deltacloud.org
mfojtik@redhat.com
Re: [PATCH] Introduces blob metadata - Adds ability to specify arbitrary
user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during blob creation
Posted by "marios@redhat.com" <ma...@redhat.com>.
still playing with .inject :) ...
(for the azure driver) the azure server responds with a metadata hash like:
{:x_ms_blob_type=>"BlockBlob", :date=>"Tue, 01 Feb 2011 17:07:07 GMT",
:content_type=>"text/plain", :x_ms_lease_status=>"unlocked",
:x_ms_version=>"2009-09-19", :x_ms_meta_v=>"2.1",
:etag=>"0x8CD8CFB422A88C4", :x_ms_meta_foo=>"bar",
:content_length=>"86", :server=>"Windows-Azure-Blob/1.0
Microsoft-HTTPAPI/2.0", :last_modified=>"Fri, 28 Jan 2011 12:08:50 GMT",
:x_ms_request_id=>"d757adb5-dcf9-46be-8463-357e3f88c368"}
The above hash is 'waz_blob.metadata'. I needed to extract a new hash
which I call 'blob_metadata', containing all the key:value where key was
'x_ms_meta_' (blob metadata key). So, especially for Michal:
waz_blob.metadata.inject({}) { |result_hash, (k,v)| blob_metadata[k]=v
if k.to_s.match(/x_ms_meta/i)}
:) thanks
marios
On 01/02/11 17:08, Michal Fojtik wrote:
> On 31/01/11 22:52 +0200, marios@redhat.com wrote:
>
>>> Yes, actually it's http://rubybestpractices.com ;-) There is an article
>>> about functional programming, which describes this.
>>
>> This site is a ruby book (pdf) - by article i assume you meant
>> chapter... i tried searching but only thing I could find was a
>> comparison of using .inject vs using recursion. However its a pretty
>> big book so I may well have missed the relevant bit.
>
> Maybe it's not this book, but meta-programming one but I saw some
> comparison between looping vs inject in some of them. I'll do the job today
> and do detailed research in those book.
>
>> That got me curious; what *actually* is the advantage of using
>> .inject. Don't get me wrong, I still think its pretty cool and elegant
>> in its concise expressiveness, but most search results seem to suggest
>> that .inject is actually slower (for this case, it makes no
>> difference, since the input will never really be a HUGE structure; we
>> are talking about key:value pairs for blob metadata). The way I
>> understood it, if you are handling large data structures, the
>> resultant object is returned at each 'loop' of the inner code block,
>> making it expensive.
>
> Did some research and I found:
>
> http://stackoverflow.com/questions/318854/inject-and-slowness
>
> So yes, it seems like I was wrong and it's slow. But if I understand it
> correctly
> it's just implementation issue, so in meantime (eg in Ruby 1.9) it should
> be fast.
> The reason why I like .inject() is that it's one of the Ruby fundamental
> features and it's more 'ruby-ish' ;-) So besides that current
> implementation is slow, it can be improved by programmers and it's
> always good to use it now than refactor later (from my point of view).
> But you're right that in this particular case (blob) it absolutely doesn't
> matter what approach we choose.
>
>>> Anyway, it's not a blocker, just small hint (I found myself .inject very
>>> useful when I was doing some meta programming in client library. It
>>> saves
>>> variables and lines of code (and it's better for garbage collecting)).
>>
>> In this particular case it doesn't save variables ( I assume you're
>> referring to the instantiation of the resulting structure, 'user_meta'
>> in this case). Do you have a reference/article/explanation for the
>> 'better for garbage collecting' (aka, please give me a reason to use
>> .inject rather than .each, since the former is clearly much cooler),
>
> I did some research about that and I found this 'hint':
>
> When is each useful? Simple: when you want to create side-effects, like
> saving to the database, printing a result, or sending a web service call.
> In these cases, you’re not concerned with the return value; you want to
> change state on the screen, the disk, the database, or something else.
>
> But don’t use each if you want to extract some new value from an array.
> That’s not what it’s for. Instead, take a look at three other powerful
> functions: map, inject, or select.
>
> http://railspikes.com/2008/7/29/functional-loops-in-ruby-each-map-inject-select-and-for
>
>
> -- Michal
>
>
>>
>> marios
>>
>
Re: [PATCH] Introduces blob metadata - Adds ability to specify
arbitrary user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during
blob creation
Posted by Michal Fojtik <mf...@redhat.com>.
On 31/01/11 22:52 +0200, marios@redhat.com wrote:
>>Yes, actually it's http://rubybestpractices.com ;-) There is an article
>>about functional programming, which describes this.
>
>This site is a ruby book (pdf) - by article i assume you meant
>chapter... i tried searching but only thing I could find was a
>comparison of using .inject vs using recursion. However its a pretty
>big book so I may well have missed the relevant bit.
Maybe it's not this book, but meta-programming one but I saw some
comparison between looping vs inject in some of them. I'll do the job today
and do detailed research in those book.
>That got me curious; what *actually* is the advantage of using
>.inject. Don't get me wrong, I still think its pretty cool and
>elegant in its concise expressiveness, but most search results seem
>to suggest that .inject is actually slower (for this case, it makes
>no difference, since the input will never really be a HUGE structure;
>we are talking about key:value pairs for blob metadata). The way I
>understood it, if you are handling large data structures, the
>resultant object is returned at each 'loop' of the inner code block,
>making it expensive.
Did some research and I found:
http://stackoverflow.com/questions/318854/inject-and-slowness
So yes, it seems like I was wrong and it's slow. But if I understand it correctly
it's just implementation issue, so in meantime (eg in Ruby 1.9) it should
be fast.
The reason why I like .inject() is that it's one of the Ruby fundamental
features and it's more 'ruby-ish' ;-)
So besides that current implementation is slow, it can be improved by programmers
and it's always good to use it now than refactor later (from my point of view).
But you're right that in this particular case (blob) it absolutely doesn't
matter what approach we choose.
>>Anyway, it's not a blocker, just small hint (I found myself .inject very
>>useful when I was doing some meta programming in client library. It saves
>>variables and lines of code (and it's better for garbage collecting)).
>
>In this particular case it doesn't save variables ( I assume you're
>referring to the instantiation of the resulting structure,
>'user_meta' in this case). Do you have a
>reference/article/explanation for the 'better for garbage collecting'
>(aka, please give me a reason to use .inject rather than .each, since
>the former is clearly much cooler),
I did some research about that and I found this 'hint':
When is each useful? Simple: when you want to create side-effects, like
saving to the database, printing a result, or sending a web service call.
In these cases, you’re not concerned with the return value; you want to
change state on the screen, the disk, the database, or something else.
But don’t use each if you want to extract some new value from an array.
That’s not what it’s for. Instead, take a look at three other powerful
functions: map, inject, or select.
http://railspikes.com/2008/7/29/functional-loops-in-ruby-each-map-inject-select-and-for
-- Michal
>
>marios
>
--
--------------------------------------------------------
Michal Fojtik, mfojtik@redhat.com
Deltacloud API: http://deltacloud.org
--------------------------------------------------------
Re: [PATCH] Introduces blob metadata - Adds ability to specify arbitrary
user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during blob creation
Posted by "marios@redhat.com" <ma...@redhat.com>.
Ok, I have changed the code to use .inject - its a new toy and its
pretty cool. I will let this sit till tomorrow and if there are no other
objections I will commit this code (aka I *aint* sendin out another
patch unless someone asks me nicely).
> Very good example of how to use .inject can be found here:
>
> http://blog.jayfields.com/2008/03/ruby-inject.html
^^ This site does a *really* good job of explaining .inject ^^
>
> Yes, actually it's http://rubybestpractices.com ;-) There is an article
> about functional programming, which describes this.
This site is a ruby book (pdf) - by article i assume you meant
chapter... i tried searching but only thing I could find was a
comparison of using .inject vs using recursion. However its a pretty big
book so I may well have missed the relevant bit.
That got me curious; what *actually* is the advantage of using .inject.
Don't get me wrong, I still think its pretty cool and elegant in its
concise expressiveness, but most search results seem to suggest that
.inject is actually slower (for this case, it makes no difference, since
the input will never really be a HUGE structure; we are talking about
key:value pairs for blob metadata). The way I understood it, if you are
handling large data structures, the resultant object is returned at each
'loop' of the inner code block, making it expensive.
>
> Anyway, it's not a blocker, just small hint (I found myself .inject very
> useful when I was doing some meta programming in client library. It saves
> variables and lines of code (and it's better for garbage collecting)).
In this particular case it doesn't save variables ( I assume you're
referring to the instantiation of the resulting structure, 'user_meta'
in this case). Do you have a reference/article/explanation for the
'better for garbage collecting' (aka, please give me a reason to use
.inject rather than .each, since the former is clearly much cooler),
marios
Re: [PATCH] Introduces blob metadata - Adds ability to specify
arbitrary user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during
blob creation
Posted by Michal Fojtik <mf...@redhat.com>.
On 31/01/11 18:18 +0200, marios@redhat.com wrote:
>Hi Michal, thanks for taking the time.
>
>I am confused by what the code fragment below does exactly. I *think*
>i understand what .inject does, but the code below doesn't make sense
>to me (or to irb it seems :) ) ; where does r come from? (I tried irb
>and ruby debugger cos I thought I was going crazy but the code
>doesn't work - i even thought it was a ruby version issue so i tried
>1.8.6 AND 1.9.2 as I have come across a few 1.8/1.9 issues).
>
>Can you please explain in words what:
>
>meta_array.inject({}) { |user_meta, v| user_meta[v.first]=v.last ; r }
This will create a new Hash ({} is an alias for Hash.new), then it will
start to iterate through meta_array.
Then user_meta is variable which holds that Hash and in 'v' is stored actual
item from 'meta_array'.
Then user_meta[v.first]=v.last create a new key using first item in array
and store value from second item.
Then I made a *typo*, 'r' variable should be 'user_meta', so you return this
Hash and iterate to next item...
I'm very sorry for that typo, it makes all this very confusing.
Very good example of how to use .inject can be found here:
http://blog.jayfields.com/2008/03/ruby-inject.html
>does? (e.g. take the meta_array two dimensional array and ... iterate
>over the user_meta array etc etc). Out of curiosity where are you
>getting 'ruby best practice' from (e.g. can you recommend a good
>book/site/blog etc?)
Yes, actually it's http://rubybestpractices.com ;-) There is an article
about functional programming, which describes this.
Anyway, it's not a blocker, just small hint (I found myself .inject very
useful when I was doing some meta programming in client library. It saves
variables and lines of code (and it's better for garbage collecting)).
-- Michal
>On 31/01/11 11:28, Michal Fojtik wrote:
>>On 28/01/11 20:15 +0200, marios@redhat.com wrote:
>>
>>Code looks good, applied cleanly, Cucumber is happy, ACK.
>>A small, minor inline comment bellow.
>>
>>-- Michal
>>
>>>From: marios <ma...@redhat.com>
>
>>>> max.to_i.times do |i| could be used here as well (but do the same job ;-)
>>
>>>+ key = params[:"meta_name#{i}"]
>>>+ key = "HTTP_X_Deltacloud_Blobmeta_#{key}"
>>>+ value = params[:"meta_value#{i}"]
>>>+ user_meta[key] = value
>>>+ end #max.each do
>>>+ else #can try to get blob_metadata from http headers
>>>+ meta_array = request.env.select{|k,v|
>>>k.match(/^HTTP[-_]X[-_]Deltacloud[-_]Blobmeta[-_]/i)}
>>>+ meta_array.each do |k,v|
>>>+ user_meta[k] = v
>>>+ end
>>
>>meta_array.inject({}) { |user_meta, v| user_meta[v.first]=v.last ; r }
>>
>>(same as above ;-) But it's a good Ruby practive to use inject if it's
>>possible.
>>
>>
--
--------------------------------------------------------
Michal Fojtik, mfojtik@redhat.com
Deltacloud API: http://deltacloud.org
--------------------------------------------------------
Re: [PATCH] Introduces blob metadata - Adds ability to specify arbitrary
user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during blob creation
Posted by "marios@redhat.com" <ma...@redhat.com>.
Hi Michal, thanks for taking the time.
I am confused by what the code fragment below does exactly. I *think* i
understand what .inject does, but the code below doesn't make sense to
me (or to irb it seems :) ) ; where does r come from? (I tried irb and
ruby debugger cos I thought I was going crazy but the code doesn't work
- i even thought it was a ruby version issue so i tried 1.8.6 AND 1.9.2
as I have come across a few 1.8/1.9 issues).
Can you please explain in words what:
meta_array.inject({}) { |user_meta, v| user_meta[v.first]=v.last ; r }
does? (e.g. take the meta_array two dimensional array and ... iterate
over the user_meta array etc etc). Out of curiosity where are you
getting 'ruby best practice' from (e.g. can you recommend a good
book/site/blog etc?)
many thanks, marios
On 31/01/11 11:28, Michal Fojtik wrote:
> On 28/01/11 20:15 +0200, marios@redhat.com wrote:
>
> Code looks good, applied cleanly, Cucumber is happy, ACK.
> A small, minor inline comment bellow.
>
> -- Michal
>
>> From: marios <ma...@redhat.com>
>> > max.to_i.times do |i| could be used here as well (but do the same job ;-)
>
>> + key = params[:"meta_name#{i}"]
>> + key = "HTTP_X_Deltacloud_Blobmeta_#{key}"
>> + value = params[:"meta_value#{i}"]
>> + user_meta[key] = value
>> + end #max.each do
>> + else #can try to get blob_metadata from http headers
>> + meta_array = request.env.select{|k,v|
>> k.match(/^HTTP[-_]X[-_]Deltacloud[-_]Blobmeta[-_]/i)}
>> + meta_array.each do |k,v|
>> + user_meta[k] = v
>> + end
>
> meta_array.inject({}) { |user_meta, v| user_meta[v.first]=v.last ; r }
>
> (same as above ;-) But it's a good Ruby practive to use inject if it's
> possible.
>
>
Re: [PATCH] Introduces blob metadata - Adds ability to specify
arbitrary user metadata (KEY-VALUE) for S3, Cloudfiles and Azure, during
blob creation
Posted by Michal Fojtik <mf...@redhat.com>.
On 28/01/11 20:15 +0200, marios@redhat.com wrote:
Code looks good, applied cleanly, Cucumber is happy, ACK.
A small, minor inline comment bellow.
-- Michal
>From: marios <ma...@redhat.com>
>
>---
> server/lib/deltacloud/base_driver/base_driver.rb | 4 +-
> .../lib/deltacloud/drivers/azure/azure_driver.rb | 47 +++++++++++++-------
> server/lib/deltacloud/drivers/ec2/ec2_driver.rb | 23 ++++++----
> server/lib/deltacloud/drivers/mock/mock_driver.rb | 10 ++--
> .../drivers/rackspace/rackspace_driver.rb | 37 ++++++++++------
> server/lib/deltacloud/helpers/blob_stream.rb | 17 +++++++
> server/lib/deltacloud/models/blob.rb | 1 +
> server/public/javascripts/application.js | 35 +++++++++++++++
> server/server.rb | 18 +++++++-
> server/views/blobs/new.html.haml | 18 +++++++-
> server/views/blobs/show.html.haml | 6 +++
> server/views/blobs/show.xml.haml | 6 ++-
> 12 files changed, 173 insertions(+), 49 deletions(-)
>
>diff --git a/server/lib/deltacloud/base_driver/base_driver.rb b/server/lib/deltacloud/base_driver/base_driver.rb
>index bf06e56..2442385 100644
>--- a/server/lib/deltacloud/base_driver/base_driver.rb
>+++ b/server/lib/deltacloud/base_driver/base_driver.rb
>@@ -186,12 +186,12 @@ module Deltacloud
> storage_snapshots(credentials, opts).first if has_capability?(:storage_snapshots)
> end
>
>- def bucket(credentials, opts = nil)
>+ def bucket(credentials, opts = {})
> #list of objects within bucket
> buckets(credentials, opts).first if has_capability?(:buckets)
> end
>
>- def blob(credentials, opts = nil)
>+ def blob(credentials, opts = {})
> blobs(credentials, opts).first if has_capability?(:blobs)
> end
>
>diff --git a/server/lib/deltacloud/drivers/azure/azure_driver.rb b/server/lib/deltacloud/drivers/azure/azure_driver.rb
>index abbd353..dce63dd 100644
>--- a/server/lib/deltacloud/drivers/azure/azure_driver.rb
>+++ b/server/lib/deltacloud/drivers/azure/azure_driver.rb
>@@ -32,7 +32,7 @@ class AzureDriver < Deltacloud::BaseDriver
> #--
> # Buckets
> #--
>- def buckets(credentials, opts)
>+ def buckets(credentials, opts={})
> buckets = []
> azure_connect(credentials)
> safely do
>@@ -46,7 +46,7 @@ class AzureDriver < Deltacloud::BaseDriver
> #--
> # Create bucket
> #--
>- def create_bucket(credentials, name, opts)
>+ def create_bucket(credentials, name, opts={})
> #for whatever reason, bucket names MUST be lowercase...
> #http://msdn.microsoft.com/en-us/library/dd135715.aspx
> name.downcase!
>@@ -62,7 +62,7 @@ class AzureDriver < Deltacloud::BaseDriver
> #--
> # Delete bucket
> #--
>- def delete_bucket(credentials, name, opts)
>+ def delete_bucket(credentials, name, opts={})
> azure_connect(credentials)
> safely do
> WAZ::Blobs::Container.find(name).destroy!
>@@ -72,20 +72,25 @@ class AzureDriver < Deltacloud::BaseDriver
> #--
> # Blobs
> #--
>- def blobs(credentials, opts)
>+ def blobs(credentials, opts={})
> blob_list = []
> azure_connect(credentials)
> safely do
> the_bucket = WAZ::Blobs::Container.find(opts['bucket'])
>- the_bucket.blobs.each do |waz_blob|
>- blob_list << convert_blob(waz_blob)
>- end
>- end
>+ if(opts[:id])
>+ the_blob = the_bucket[opts[:id]]
>+ blob_list << convert_blob(the_blob) unless the_blob.nil?
>+ else
>+ the_bucket.blobs.each do |waz_blob|
>+ blob_list << convert_blob(waz_blob)
>+ end #each.do
>+ end #if
>+ end #safely do
> blob_list = filter_on(blob_list, :id, opts)
> blob_list
> end
>
>- def blob_data(credentials, bucket_id, blob_id, opts)
>+ def blob_data(credentials, bucket_id, blob_id, opts={})
> azure_connect(credentials)
> # WAZ get blob data methods cant accept blocks for 'streaming'... FIXME
> yield WAZ::Blobs::Container.find(bucket_id)[blob_id].value
>@@ -94,23 +99,28 @@ class AzureDriver < Deltacloud::BaseDriver
> #--
> # Create Blob
> #--
>- def create_blob(credentials, bucket_id, blob_id, blob_data, opts=nil)
>+ def create_blob(credentials, bucket_id, blob_id, blob_data, opts={})
> azure_connect(credentials)
>- #get a handle to the bucket in order to put there
>- the_bucket = WAZ::Blobs::Container.find(bucket_id)
>- the_bucket.store(blob_id, blob_data[:tempfile], blob_data[:type])
>+ #insert azure-specific header for user metadata ... x-ms-meta-kEY = VALUE
>+ opts.gsub_keys("HTTP_X_Deltacloud_Blobmeta_", "x-ms-meta-")
>+ safely do
>+ #get a handle to the bucket in order to put there
>+ the_bucket = WAZ::Blobs::Container.find(bucket_id)
>+ the_bucket.store(blob_id, blob_data[:tempfile], blob_data[:type], opts)
>+ end
> Blob.new( { :id => blob_id,
> :bucket => bucket_id,
> :content_lengh => blob_data[:tempfile].length,
> :content_type => blob_data[:type],
>- :last_modified => ''
>+ :last_modified => '',
>+ :user_metadata => opts.gsub_keys('x-ms-meta-','')
> } )
> end
>
> #--
> # Delete Blob
> #--
>- def delete_blob(credentials, bucket_id, blob_id, opts=nil)
>+ def delete_blob(credentials, bucket_id, blob_id, opts={})
> azure_connect(credentials)
> #get a handle to bucket and blob, and destroy!
> the_bucket = WAZ::Blobs::Container.find(bucket_id)
>@@ -142,11 +152,16 @@ class AzureDriver < Deltacloud::BaseDriver
> def convert_blob(waz_blob)
> url = waz_blob.url.split('/')
> bucket = url[url.length-2] #FIXME
>+ #get only user defined metadata
>+ blob_metadata = waz_blob.metadata.reject{|k,v| (!(k.to_s.match('x_ms_meta')))}
>+ #strip off the x_ms_meta_ from each key
>+ blob_metadata.gsub_keys('x_ms_meta_', '')
> Blob.new({ :id => waz_blob.name,
> :bucket => bucket,
> :content_length => waz_blob.metadata[:content_length],
> :content_type => waz_blob.metadata[:content_type],
>- :last_modified => waz_blob.metadata[:last_modified]
>+ :last_modified => waz_blob.metadata[:last_modified],
>+ :user_metadata => blob_metadata
> })
> end
>
>diff --git a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
>index bcd848e..2d0194f 100644
>--- a/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
>+++ b/server/lib/deltacloud/drivers/ec2/ec2_driver.rb
>@@ -324,7 +324,7 @@ module Deltacloud
> end
>
>
>- def buckets(credentials, opts)
>+ def buckets(credentials, opts={})
> buckets = []
> safely do
> s3_client = new_client(credentials, :s3)
>@@ -357,7 +357,7 @@ module Deltacloud
> end
> end
>
>- def blobs(credentials, opts = nil)
>+ def blobs(credentials, opts = {})
> s3_client = new_client(credentials, :s3)
> blobs = []
> safely do
>@@ -373,25 +373,29 @@ module Deltacloud
> #--
> # Create Blob
> #--
>- def create_blob(credentials, bucket_id, blob_id, data = nil, opts = nil)
>+ def create_blob(credentials, bucket_id, blob_id, data = nil, opts = {})
> s3_client = new_client(credentials, :s3)
> #data is a construct with the temporary file created by server @.tempfile
> #also file[:type] will give us the content-type
> res = nil
> # File stream needs to be reopened in binary mode for whatever reason
> file = File::open(data[:tempfile].path, 'rb')
>+ #insert ec2-specific header for user metadata ... x-amz-meta-KEY = VALUE
>+ opts.gsub_keys('HTTP_X_Deltacloud_Blobmeta_', 'x-amz-meta-')
>+ opts["Content-Type"] = data[:type]
> safely do
> res = s3_client.interface.put(bucket_id,
> blob_id,
> file,
>- {"Content-Type" => data[:type]})
>+ opts)
> end
> #create a new Blob object and return that
> Blob.new( { :id => blob_id,
> :bucket => bucket_id,
> :content_length => data[:tempfile].length,
> :content_type => data[:type],
>- :last_modified => ''
>+ :last_modified => '',
>+ :user_metadata => opts.select{|k,v| k.match(/^x-amz-meta-/)}
> }
> )
> end
>@@ -399,7 +403,7 @@ module Deltacloud
> #--
> # Delete Blob
> #--
>- def delete_blob(credentials, bucket_id, blob_id, opts=nil)
>+ def delete_blob(credentials, bucket_id, blob_id, opts={})
> s3_client = new_client(credentials, :s3)
> safely do
> s3_client.interface.delete(bucket_id, blob_id)
>@@ -407,7 +411,7 @@ module Deltacloud
> end
>
>
>- def blob_data(credentials, bucket_id, blob_id, opts)
>+ def blob_data(credentials, bucket_id, blob_id, opts={})
> s3_client = new_client(credentials, :s3)
> safely do
> s3_client.interface.get(bucket_id, blob_id) do |chunk|
>@@ -522,7 +526,7 @@ module Deltacloud
> 'us-east-1' => 'elasticloadbalancing.us-east-1.amazonaws.com',
> 'us-west-1' => 'elasticloadbalancing.us-west-1.amazonaws.com'
> },
>-
>+
> 's3' => {
> 'us-east-1' => 's3.amazonaws.com',
> 'us-west-1' => 's3-us-west-1.amazonaws.com',
>@@ -587,7 +591,8 @@ module Deltacloud
> :bucket => s3_object.bucket.name.to_s,
> :content_length => s3_object.headers['content-length'],
> :content_type => s3_object.headers['content-type'],
>- :last_modified => s3_object.last_modified
>+ :last_modified => s3_object.last_modified,
>+ :user_metadata => s3_object.meta_headers
> )
> end
>
>diff --git a/server/lib/deltacloud/drivers/mock/mock_driver.rb b/server/lib/deltacloud/drivers/mock/mock_driver.rb
>index da2931b..a36d569 100644
>--- a/server/lib/deltacloud/drivers/mock/mock_driver.rb
>+++ b/server/lib/deltacloud/drivers/mock/mock_driver.rb
>@@ -318,7 +318,7 @@ class MockDriver < Deltacloud::BaseDriver
> #--
> # Buckets
> #--
>- def buckets(credentials, opts=nil)
>+ def buckets(credentials, opts={})
> check_credentials(credentials)
> buckets=[]
> Dir[ "#{@storage_root}/buckets/*.yml" ].each do |bucket_file|
>@@ -334,7 +334,7 @@ class MockDriver < Deltacloud::BaseDriver
> #--
> # Create bucket
> #--
>- def create_bucket(credentials, name, opts=nil)
>+ def create_bucket(credentials, name, opts={})
> check_credentials(credentials)
> bucket = {
> :id => name,
>@@ -349,7 +349,7 @@ class MockDriver < Deltacloud::BaseDriver
> #--
> # Delete bucket
> #--
>- def delete_bucket(credentials, name, opts=nil)
>+ def delete_bucket(credentials, name, opts={})
> bucket = bucket(credentials, {:id => name})
> unless (bucket.size == "0")
> raise Deltacloud::BackendError.new(403, self.class.to_s, "bucket-not-empty", "delete operation not valid for non-empty bucket")
>@@ -362,7 +362,7 @@ class MockDriver < Deltacloud::BaseDriver
> #--
> # Blobs
> #--
>- def blobs(credentials, opts = nil)
>+ def blobs(credentials, opts = {})
> check_credentials(credentials)
> blobs=[]
> Dir[ "#{@storage_root}/buckets/blobs/*.yml" ].each do |blob_file|
>@@ -378,7 +378,7 @@ class MockDriver < Deltacloud::BaseDriver
> #--
> # Blob content
> #--
>- def blob_data(credentials, bucket_id, blob_id, opts = nil)
>+ def blob_data(credentials, bucket_id, blob_id, opts = {})
> check_credentials(credentials)
> blob=nil
> Dir[ "#{@storage_root}/buckets/blobs/*.yml" ].each do |blob_file|
>diff --git a/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb b/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
>index e40fb9c..8e6612b 100644
>--- a/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
>+++ b/server/lib/deltacloud/drivers/rackspace/rackspace_driver.rb
>@@ -166,7 +166,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
> #--
> # Buckets
> #--
>- def buckets(credentials, opts)
>+ def buckets(credentials, opts = {})
> bucket_list = []
> cf = cloudfiles_client(credentials)
> safely do
>@@ -182,7 +182,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
> #--
> # Create Bucket
> #--
>- def create_bucket(credentials, name, opts)
>+ def create_bucket(credentials, name, opts = {})
> bucket = nil
> cf = cloudfiles_client(credentials)
> safely do
>@@ -195,7 +195,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
> #--
> # Delete Bucket
> #--
>- def delete_bucket(credentials, name, opts)
>+ def delete_bucket(credentials, name, opts = {})
> cf = cloudfiles_client(credentials)
> safely do
> cf.delete_container(name)
>@@ -205,7 +205,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
> #--
> # Blobs
> #--
>- def blobs(credentials, opts)
>+ def blobs(credentials, opts = {})
> cf = cloudfiles_client(credentials)
> blobs = []
> safely do
>@@ -221,7 +221,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
> #-
> # Blob data
> #-
>- def blob_data(credentials, bucket_id, blob_id, opts)
>+ def blob_data(credentials, bucket_id, blob_id, opts = {})
> cf = cloudfiles_client(credentials)
> cf.container(bucket_id).object(blob_id).data_stream do |chunk|
> yield chunk
>@@ -231,18 +231,26 @@ class RackspaceDriver < Deltacloud::BaseDriver
> #--
> # Create Blob
> #--
>- def create_blob(credentials, bucket_id, blob_id, blob_data, opts=nil)
>+ def create_blob(credentials, bucket_id, blob_id, blob_data, opts={})
> cf = cloudfiles_client(credentials)
>- #must first create the object using cloudfiles_client.create_object
>- #then can write using object.write(data)
>- object = cf.container(bucket_id).create_object(blob_id)
>- #blob_data is a construct with data in .tempfile and content-type in {:type}
>- res = object.write(blob_data[:tempfile], {'Content-Type' => blob_data[:type]})
>+ user_meta = opts.select{|k,v| k.match(/^META-/)}
>+ #insert ec2-specific header for user metadata ... X-Object-Meta-KEY = VALUE
>+ opts.gsub_keys("HTTP_X_Deltacloud_Blobmeta_", "X-Object-Meta-")
>+ opts['Content-Type'] = blob_data[:type]
>+ object = nil
>+ safely do
>+ #must first create the object using cloudfiles_client.create_object
>+ #then can write using object.write(data)
>+ object = cf.container(bucket_id).create_object(blob_id)
>+ #blob_data is a construct with data in .tempfile and content-type in {:type}
>+ res = object.write(blob_data[:tempfile], opts)
>+ end
> Blob.new( { :id => object.name,
> :bucket => object.container.name,
> :content_length => blob_data[:tempfile].length,
> :content_type => blob_data[:type],
>- :last_modified => ''
>+ :last_modified => '',
>+ :user_metadata => opts.select{|k,v| k.match(/^X-Object-Meta-/)}
> }
> )
> end
>@@ -250,7 +258,7 @@ class RackspaceDriver < Deltacloud::BaseDriver
> #--
> # Delete Blob
> #--
>- def delete_blob(credentials, bucket_id, blob_id, opts=nil)
>+ def delete_blob(credentials, bucket_id, blob_id, opts={})
> cf = cloudfiles_client(credentials)
> cf.container(bucket_id).delete_object(blob_id)
> end
>@@ -292,7 +300,8 @@ private
> :bucket => cf_object.container.name,
> :content_length => cf_object.bytes,
> :content_type => cf_object.content_type,
>- :last_modified => cf_object.last_modified
>+ :last_modified => cf_object.last_modified,
>+ :user_metadata => cf_object.metadata
> })
> end
>
>diff --git a/server/lib/deltacloud/helpers/blob_stream.rb b/server/lib/deltacloud/helpers/blob_stream.rb
>index a99e6c2..6a8a2c5 100644
>--- a/server/lib/deltacloud/helpers/blob_stream.rb
>+++ b/server/lib/deltacloud/helpers/blob_stream.rb
>@@ -62,3 +62,20 @@ rescue LoadError => e
> end
> end
> end
>+
>+class Hash
>+
>+ def gsub_keys(pattern, replacement)
>+ remove = []
>+ self.each_key do |key|
>+ if key.to_s.match(pattern)
>+ new_key = key.to_s.gsub(pattern, replacement)
>+ self[new_key] = self[key]
>+ remove << key
>+ end #key.match
>+ end # each_key do
>+ #remove the original keys
>+ self.delete_if{|k,v| remove.include?(k)}
>+ end #def
>+
>+end #class
>diff --git a/server/lib/deltacloud/models/blob.rb b/server/lib/deltacloud/models/blob.rb
>index dfa67fe..0bf473b 100644
>--- a/server/lib/deltacloud/models/blob.rb
>+++ b/server/lib/deltacloud/models/blob.rb
>@@ -24,5 +24,6 @@ class Blob < BaseModel
> attr_accessor :content_type
> attr_accessor :last_modified
> attr_accessor :content
>+ attr_accessor :user_metadata
>
> end
>diff --git a/server/public/javascripts/application.js b/server/public/javascripts/application.js
>index 5ee0f7f..95c9bc2 100644
>--- a/server/public/javascripts/application.js
>+++ b/server/public/javascripts/application.js
>@@ -16,3 +16,38 @@ $(document).ready(function() {
> }
>
> })
>+
>+function more_fields()
>+{
>+ //increment the hidden input that captures how many meta_data are passed
>+ var meta_params = document.getElementsByName('meta_params')
>+ current_number_params = eval(meta_params[0].value)+1
>+ meta_params[0].value = current_number_params
>+ var new_meta = document.getElementById('metadata_holder').cloneNode(true);
>+ new_meta.id = 'metadata_holder' + current_number_params;
>+ new_meta.style.display = 'block';
>+ var nodes = new_meta.childNodes;
>+ for (var i=0;i < nodes.length;i++) {
>+ var theName = nodes[i].name;
>+ if (theName)
>+ nodes[i].name = theName + current_number_params;
>+ }
>+ var insertHere = document.getElementById('metadata_holder');
>+ insertHere.parentNode.insertBefore(new_meta,insertHere);
>+}
>+
>+function less_fields()
>+{
>+ var meta_params = document.getElementsByName('meta_params')
>+ current_val = eval(meta_params[0].value)
>+ if (current_val == 0)
>+ {
>+ return;
>+ }
>+ else
>+ {
>+ var theDiv = document.getElementById('metadata_holder'+current_val)
>+ theDiv.parentNode.removeChild(theDiv)
>+ meta_params[0].value = eval(current_val)-1
>+ }
>+}
>diff --git a/server/server.rb b/server/server.rb
>index 043cbee..8cdabc6 100644
>--- a/server/server.rb
>+++ b/server/server.rb
>@@ -626,7 +626,23 @@ post '/api/buckets/:bucket' do
> bucket_id = params[:bucket]
> blob_id = params['blob_id']
> blob_data = params['blob_data']
>- @blob = driver.create_blob(credentials, bucket_id, blob_id, blob_data )
>+ user_meta = {}
>+#first try get blob_metadata from params (i.e., passed by http form post, e.g. browser)
>+ max = params[:meta_params]
>+ if(max)
>+ (1..max.to_i).each do |i|
max.to_i.times do |i| could be used here as well (but do the same job ;-)
>+ key = params[:"meta_name#{i}"]
>+ key = "HTTP_X_Deltacloud_Blobmeta_#{key}"
>+ value = params[:"meta_value#{i}"]
>+ user_meta[key] = value
>+ end #max.each do
>+ else #can try to get blob_metadata from http headers
>+ meta_array = request.env.select{|k,v| k.match(/^HTTP[-_]X[-_]Deltacloud[-_]Blobmeta[-_]/i)}
>+ meta_array.each do |k,v|
>+ user_meta[k] = v
>+ end
meta_array.inject({}) { |user_meta, v| user_meta[v.first]=v.last ; r }
(same as above ;-) But it's a good Ruby practive to use inject if it's
possible.
>+ end #end if
>+ @blob = driver.create_blob(credentials, bucket_id, blob_id, blob_data, user_meta)
> respond_to do |format|
> format.html { haml :"blobs/show"}
> format.xml { haml :"blobs/show" }
>diff --git a/server/views/blobs/new.html.haml b/server/views/blobs/new.html.haml
>index feb94e5..a075f0a 100644
>--- a/server/views/blobs/new.html.haml
>+++ b/server/views/blobs/new.html.haml
>@@ -4,7 +4,23 @@
> %label
> Blob Name:
> %input{ :name => 'blob_id', :size => 512}/
>+ %label
> Blob Data:
>- %input{ :type => "file", :name => 'blob_data', :size => 50}/
> %br
>+ %input{ :type => "file", :name => 'blob_data', :size => 50}/
>+ %br
>+ %br
>+ %input{ :type => "hidden", :name => "meta_params", :value => "0"}
>+ %a{ :href => "javascript:;", :onclick => "more_fields();"} Add Metadata
>+ %div{ :id => "metadata_holder", :style => "display: none;"}
>+ %label
>+ Metadata Key:
>+ %input{ :type => "text", :name => "meta_name", :value => ""}/
>+ %label
>+ Metadata Value:
>+ %input{ :type => "text", :name => "meta_value", :value => ""}/
>+ %br
>+ %a{ :href => "javascript:;", :onclick => "less_fields();"} Less Metadata
>+ %br
>+ %br
> %input{ :type => :submit, :name => "commit", :value => "create"}/
>diff --git a/server/views/blobs/show.html.haml b/server/views/blobs/show.html.haml
>index e5c2cee..99c090a 100644
>--- a/server/views/blobs/show.html.haml
>+++ b/server/views/blobs/show.html.haml
>@@ -19,6 +19,12 @@
> %dt Content
> %dd
> =link_to 'Blob content', bucket_url(@blob.bucket) + '/' + @blob.id + '/content'
>+ %dt User_Metadata
>+ %dd
>+ -@blob.user_metadata.each do |k,v|
>+ %b #{k} :
>+ #{v}
>+ %br
> %dt Delete this Blob
> %dd
> %form{ :action => bucket_url(@blob.bucket) + '/' + @blob.id , :method => :post }
>diff --git a/server/views/blobs/show.xml.haml b/server/views/blobs/show.xml.haml
>index bc499ca..34b9cca 100644
>--- a/server/views/blobs/show.xml.haml
>+++ b/server/views/blobs/show.xml.haml
>@@ -1,7 +1,11 @@
> !!! XML
> %blob{:href => bucket_url(@blob.bucket) + '/' + @blob.id, :id => @blob.id}
>- - @blob.attributes.select{ |attr| attr!=:id}.each do |attribute|
>+ - @blob.attributes.select{ |attr| (attr!=:id && attr!=:user_metadata) }.each do |attribute|
> - unless attribute == :content
> - haml_tag(attribute, :<) do
> - haml_concat @blob.send(attribute)
>+ %user_metadata
>+ - @blob.user_metadata.each do |k, v|
>+ %entry{:key => k}
>+ #{cdata v}
> %content{:href => bucket_url(@blob.bucket) + '/' + @blob.id + '/content'}
>--
>1.7.3.4
>
--
--------------------------------------------------------
Michal Fojtik, mfojtik@redhat.com
Deltacloud API: http://deltacloud.org
--------------------------------------------------------