You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@deltacloud.apache.org by ma...@redhat.com on 2012/06/11 12:09:56 UTC
[PATCH 2/2] Fixes for blob streaming support - bugs/nits brought over with modular deltacloud
From: marios <ma...@redhat.com>
Signed-off-by: marios <ma...@redhat.com>
---
server/lib/deltacloud/collections/buckets.rb | 38 +++++++++++++++++++-
.../lib/deltacloud/helpers/blob_stream_helper.rb | 3 +-
2 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/server/lib/deltacloud/collections/buckets.rb b/server/lib/deltacloud/collections/buckets.rb
index 355d4d0..9a8698f 100644
--- a/server/lib/deltacloud/collections/buckets.rb
+++ b/server/lib/deltacloud/collections/buckets.rb
@@ -48,6 +48,40 @@ module Deltacloud::Collections
end
end
+ put route_for ("/buckets/:bucket/:blob") do
+ if(env["BLOB_SUCCESS"]) #ie got a 200ok after putting blob
+ content_type = env["CONTENT_TYPE"]
+ content_type ||= ""
+ @blob = driver.blob(credentials, {:id => params[:blob],
+ 'bucket' => params[:bucket]})
+ respond_to do |format|
+ format.xml { haml :"blobs/show" }
+ format.html { haml :"blobs/show" }
+ format.json { convert_to_json(:blob, @blob) }
+ end
+ elsif(env["BLOB_FAIL"])
+ report_error(500) #OK?
+ else # small blobs - < 112kb dont hit the streaming monkey patch - use 'normal' create_blob
+ # also, if running under webrick don't hit the streaming patch (Thin specific)
+ bucket_id = params[:bucket]
+ blob_id = params[:blob]
+ temp_file = Tempfile.new("temp_blob_file")
+ temp_file.write(env['rack.input'].read)
+ temp_file.flush
+ content_type = env['CONTENT_TYPE'] || ""
+ blob_data = {:tempfile => temp_file, :type => content_type}
+ user_meta = BlobHelper::extract_blob_metadata_hash(request.env)
+ @blob = driver.create_blob(credentials, bucket_id, blob_id, blob_data, user_meta)
+ temp_file.delete
+ respond_to do |format|
+ format.xml { haml :"blobs/show" }
+ format.html { haml :"blobs/show" }
+ format.json { convert_to_json(:blob, @blob) }
+ end
+ end
+ end
+
+
collection :buckets do
standard_show_operation
@@ -142,9 +176,11 @@ module Deltacloud::Collections
end
end
+require 'ruby-debug'
action :stream, :http_method => :put, :with_capability => :create_blob do
description "Stream new blob data into the blob"
control do
+debugger
if(env["BLOB_SUCCESS"]) #ie got a 200ok after putting blob
content_type = env["CONTENT_TYPE"]
content_type ||= ""
@@ -201,7 +237,7 @@ module Deltacloud::Collections
control do
meta_hash = BlobHelper::extract_blob_metadata_hash(request.env)
success = driver.update_blob_metadata(credentials, {'bucket'=>params[:id], :id =>params[:blob_id], 'meta_hash' => meta_hash})
- if(success)
+ if(success)
meta_hash.each do |k,v|
headers["X-Deltacloud-Blobmeta-#{k}"] = v
end
diff --git a/server/lib/deltacloud/helpers/blob_stream_helper.rb b/server/lib/deltacloud/helpers/blob_stream_helper.rb
index 08cafe2..8567543 100644
--- a/server/lib/deltacloud/helpers/blob_stream_helper.rb
+++ b/server/lib/deltacloud/helpers/blob_stream_helper.rb
@@ -149,7 +149,6 @@ require 'base64'
class BlobStreamIO
attr_accessor :size, :provider, :sock
-
def initialize(request)
@client_request = request
@size = 0
@@ -159,7 +158,7 @@ class BlobStreamIO
#deal with blob_metadata: (X-Deltacloud-Blobmeta-name: value)
user_meta = BlobHelper::extract_blob_metadata_hash(request.env)
@content_length = request.env['CONTENT_LENGTH']
- @http, provider_request = driver.blob_stream_connection({:user=>user,
+ @http, provider_request = Deltacloud::API.driver.blob_stream_connection({:user=>user,
:password=>password, :bucket=>bucket, :blob=>blob, :metadata=> user_meta,
:content_type=>content_type, :content_length=>@content_length })
@content_length = @content_length.to_i #for comparison of size in '<< (data)'
--
1.7.6.5
Re: [PATCH 2/2] Fixes for blob streaming support - bugs/nits brought over with modular deltacloud
Posted by Ronelle Landy <rl...@redhat.com>.
QE ACK ... works for ec2 provider
Applied patch over deltacloud git commit a6876a9675847af85ba0290c081479903fbb8367.
Test output below (using curl for demo):
>> curl -H 'content-type: text/html' -H 'X-Deltacloud-Blobmeta-Name:myblob' -H 'X-Deltacloud-Blobmeta-Version:2.4' -H 'X-Deltacloud-Blobmeta-Author:rlandy' -iv --upload-file "/home/rlandy/Temp/rhel62rhevm.xml" --user "AKIAJCBB265A57CGKR2Q:mRqPo4C566epOauIxO4faFsOdeTyus7oOVdHKgw5" "http://qe-blade-02.idm.lab.bos.redhat.com:3006/api/buckets/bucket-rlandy/newblob?format=xml"
* About to connect() to qe-blade-02.idm.lab.bos.redhat.com port 3006 (#0)
* Trying 10.16.76.33... Connection refused
* couldn't connect to host
* Closing connection #0
curl: (7) couldn't connect to host
[rlandy@localhost /]$ curl -H 'content-type: text/html' -H 'X-Deltacloud-Blobmeta-Name:myblob' -H 'X-Deltacloud-Blobmeta-Version:2.4' -H 'X-Deltacloud-Blobmeta-Author:rlandy' -iv --upload-file "/home/rlandy/Temp/rhel62rhevm.xml" --user "un:pw" "http://localhost:3008/api/buckets/bucket-rlandy/newblob?format=xml"
* About to connect() to localhost port 3008 (#0)
* Trying 127.0.0.1... connected
* Connected to localhost (127.0.0.1) port 3008 (#0)
* Server auth using Basic with user 'x'
> PUT /api/buckets/bucket-rlandy/newblob?format=xml HTTP/1.1
> Authorization: Basic x==
> User-Agent: curl/7.21.7 (x86_64-redhat-linux-gnu) libcurl/7.21.7 NSS/3.13.1.0 zlib/1.2.5 libidn/1.22 libssh2/1.2.7
> Host: localhost:3008
> Accept: */*
> content-type: text/html
> X-Deltacloud-Blobmeta-Name:myblob
> X-Deltacloud-Blobmeta-Version:2.4
> X-Deltacloud-Blobmeta-Author:rlandy
> Content-Length: 931
> Expect: 100-continue
>
* Done waiting for 100-continue
* We are completely uploaded and fine
< HTTP/1.1 200 OK
HTTP/1.1 200 OK
< Server: Apache-Deltacloud/0.5.0
Server: Apache-Deltacloud/0.5.0
< Cache-Control: max-age=0, private, must-revalidate
Cache-Control: max-age=0, private, must-revalidate
< Content-Length: 645
Content-Length: 645
< Date: Tue, 12 Jun 2012 13:58:55 GMT
Date: Tue, 12 Jun 2012 13:58:55 GMT
< X-XSS-Protection: 1; mode=block
X-XSS-Protection: 1; mode=block
< Content-Type: application/xml
Content-Type: application/xml
< X-Frame-Options: sameorigin
X-Frame-Options: sameorigin
< ETag: "x"
ETag: "x"
< Connection: keep-alive
Connection: keep-alive
<
<?xml version='1.0' encoding='utf-8' ?>
<blob href='http://localhost:3008/api/buckets/bucket-rlandy/newblob' id='newblob'>
<bucket>bucket-rlandy</bucket>
<content_length>931</content_length>
<content_type>text/html</content_type>
<last_modified></last_modified>
<user_metadata>
<entry key='x-amz-meta-name'>
<![CDATA[myblob]]>
</entry>
<entry key='x-amz-meta-version'>
<![CDATA[2.4]]>
</entry>
<entry key='x-amz-meta-author'>
<![CDATA[rlandy]]>
</entry>
</user_metadata>
<content href='http://localhost:3008/api/buckets/bucket-rlandy/newblob/content' rel='blob_content'></content>
</blob>
* Connection #0 to host localhost left intact
* Closing connection #0
----- Original Message -----
> From: marios@redhat.com
> To: dev@deltacloud.apache.org
> Sent: Monday, June 11, 2012 6:09:56 AM
> Subject: [PATCH 2/2] Fixes for blob streaming support - bugs/nits brought over with modular deltacloud
>
> From: marios <ma...@redhat.com>
>
>
> Signed-off-by: marios <ma...@redhat.com>
> ---
> server/lib/deltacloud/collections/buckets.rb | 38
> +++++++++++++++++++-
> .../lib/deltacloud/helpers/blob_stream_helper.rb | 3 +-
> 2 files changed, 38 insertions(+), 3 deletions(-)
>
> diff --git a/server/lib/deltacloud/collections/buckets.rb
> b/server/lib/deltacloud/collections/buckets.rb
> index 355d4d0..9a8698f 100644
> --- a/server/lib/deltacloud/collections/buckets.rb
> +++ b/server/lib/deltacloud/collections/buckets.rb
> @@ -48,6 +48,40 @@ module Deltacloud::Collections
> end
> end
>
> + put route_for ("/buckets/:bucket/:blob") do
> + if(env["BLOB_SUCCESS"]) #ie got a 200ok after putting blob
> + content_type = env["CONTENT_TYPE"]
> + content_type ||= ""
> + @blob = driver.blob(credentials, {:id => params[:blob],
> + 'bucket' =>
> params[:bucket]})
> + respond_to do |format|
> + format.xml { haml :"blobs/show" }
> + format.html { haml :"blobs/show" }
> + format.json { convert_to_json(:blob, @blob) }
> + end
> + elsif(env["BLOB_FAIL"])
> + report_error(500) #OK?
> + else # small blobs - < 112kb dont hit the streaming monkey
> patch - use 'normal' create_blob
> + # also, if running under webrick don't hit the streaming
> patch (Thin specific)
> + bucket_id = params[:bucket]
> + blob_id = params[:blob]
> + temp_file = Tempfile.new("temp_blob_file")
> + temp_file.write(env['rack.input'].read)
> + temp_file.flush
> + content_type = env['CONTENT_TYPE'] || ""
> + blob_data = {:tempfile => temp_file, :type => content_type}
> + user_meta =
> BlobHelper::extract_blob_metadata_hash(request.env)
> + @blob = driver.create_blob(credentials, bucket_id, blob_id,
> blob_data, user_meta)
> + temp_file.delete
> + respond_to do |format|
> + format.xml { haml :"blobs/show" }
> + format.html { haml :"blobs/show" }
> + format.json { convert_to_json(:blob, @blob) }
> + end
> + end
> + end
> +
> +
> collection :buckets do
>
> standard_show_operation
> @@ -142,9 +176,11 @@ module Deltacloud::Collections
> end
> end
>
> +require 'ruby-debug'
> action :stream, :http_method => :put, :with_capability =>
> :create_blob do
> description "Stream new blob data into the blob"
> control do
> +debugger
> if(env["BLOB_SUCCESS"]) #ie got a 200ok after putting
> blob
> content_type = env["CONTENT_TYPE"]
> content_type ||= ""
> @@ -201,7 +237,7 @@ module Deltacloud::Collections
> control do
> meta_hash =
> BlobHelper::extract_blob_metadata_hash(request.env)
> success = driver.update_blob_metadata(credentials,
> {'bucket'=>params[:id], :id =>params[:blob_id],
> 'meta_hash' => meta_hash})
> - if(success)
> + if(success)
> meta_hash.each do |k,v|
> headers["X-Deltacloud-Blobmeta-#{k}"] = v
> end
> diff --git a/server/lib/deltacloud/helpers/blob_stream_helper.rb
> b/server/lib/deltacloud/helpers/blob_stream_helper.rb
> index 08cafe2..8567543 100644
> --- a/server/lib/deltacloud/helpers/blob_stream_helper.rb
> +++ b/server/lib/deltacloud/helpers/blob_stream_helper.rb
> @@ -149,7 +149,6 @@ require 'base64'
> class BlobStreamIO
>
> attr_accessor :size, :provider, :sock
> -
> def initialize(request)
> @client_request = request
> @size = 0
> @@ -159,7 +158,7 @@ class BlobStreamIO
> #deal with blob_metadata: (X-Deltacloud-Blobmeta-name: value)
> user_meta = BlobHelper::extract_blob_metadata_hash(request.env)
> @content_length = request.env['CONTENT_LENGTH']
> - @http, provider_request =
> driver.blob_stream_connection({:user=>user,
> + @http, provider_request =
> Deltacloud::API.driver.blob_stream_connection({:user=>user,
> :password=>password, :bucket=>bucket, :blob=>blob,
> :metadata=> user_meta,
> :content_type=>content_type, :content_length=>@content_length
> })
> @content_length = @content_length.to_i #for comparison of size
> in '<< (data)'
> --
> 1.7.6.5
>
>