You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@deltacloud.apache.org by mf...@redhat.com on 2013/01/15 16:08:52 UTC
Replace datamapper with sequel
Hi,
This is an initial attempt to replace DataMapper with more ligher
ORM (Sequel).
The main reason of doing this is lack of DataMapper RPM's in distributions
we want to support.
Pros:
- Less dependencies (sequel + sqlite3)
- More performance tunning (sequel allows to modify SQL queries to get less
queries)
- Caching (not interesting right now, but might be in future)
Cons:
- No automigration (you will need to run fixture reset rake task)
Let me know what you think :)
If you want try this patch, please reset the 'mock' fixtures and DB using:
'rake mock:fixtures:reset'
Also please run 'bundle update' to get rid of DataMapper and install Sequel.
Let me know what you think :)
-- Michal
[PATCH core 1/3] CIMI: Replaced DataMapper with Sequel ORM
Posted by mf...@redhat.com.
From: Michal Fojtik <mf...@redhat.com>
Signed-off-by: Michal fojtik <mf...@redhat.com>
---
server/deltacloud-core.gemspec | 5 +-
server/lib/cimi/helpers/database_helper.rb | 19 +++++--
server/lib/db.rb | 81 +++++++++++++++++++++++-------
3 files changed, 80 insertions(+), 25 deletions(-)
diff --git a/server/deltacloud-core.gemspec b/server/deltacloud-core.gemspec
index 7fad5be..977cc26 100644
--- a/server/deltacloud-core.gemspec
+++ b/server/deltacloud-core.gemspec
@@ -78,9 +78,8 @@ Gem::Specification.new do |s|
s.add_dependency('nokogiri', '>= 1.4.3')
s.add_dependency('require_relative') if RUBY_VERSION < '1.9'
- s.add_dependency('data_mapper')
- s.add_dependency('dm-sqlite-adapter')
- s.add_dependency('dm-timestamps')
+ s.add_dependency('sequel', '<= 3.42.0')
+ s.add_dependency('sqlite3')
# dependencies for various cloud providers:
diff --git a/server/lib/cimi/helpers/database_helper.rb b/server/lib/cimi/helpers/database_helper.rb
index ac6ae4f..d432f69 100644
--- a/server/lib/cimi/helpers/database_helper.rb
+++ b/server/lib/cimi/helpers/database_helper.rb
@@ -27,11 +27,14 @@ module Deltacloud
def delete_attributes_for(model)
return if test_environment?
entity = get_entity(model)
- !entity.nil? && entity.destroy!
+ !entity.nil? && entity.destroy
end
def get_entity(model)
- Deltacloud::Database::Entity.first(:be_kind => model.to_entity, :be_id => model.id, 'provider.driver' => driver_symbol.to_s, 'provider.url' => current_provider)
+ current_db.entities_dataset.first(
+ :be_kind => model.to_entity,
+ :be_id => model.id,
+ )
end
def current_provider
@@ -43,23 +46,29 @@ module Deltacloud
#
def current_db
- Deltacloud::Database::Provider.first_or_create(:driver => driver_symbol.to_s, :url => current_provider)
+ Deltacloud::Database::Provider.find_or_create(:driver => driver_symbol.to_s, :url => current_provider)
end
def store_attributes_for(model, attrs={})
return if test_environment? or model.nil? or attrs.empty?
return if model.id.nil?
- entity = get_entity(model) || current_db.entities.new(:be_kind => model.to_entity, :be_id => model.id)
+
+ unless entity = get_entity(model)
+ entity = Deltacloud::Database::Entity.new(:provider_id => current_db.id)
+ end
entity.description = extract_attribute_value('description', attrs) if attrs.has_key? 'description'
entity.name = extract_attribute_value('name', attrs) if attrs.has_key? 'name'
+
if attrs.has_key? 'properties'
entity.ent_properties = extract_attribute_value('properties', attrs).to_json
elsif attrs.has_key? 'property'
entity.ent_properties = extract_attribute_value('property', attrs).to_json
end
- entity.save && entity
+ entity.exists? ? entity.save_changes : entity.save
+
+ entity
end
# In XML serialization the values stored in attrs are arrays, dues to
diff --git a/server/lib/db.rb b/server/lib/db.rb
index 567b356..64cc283 100644
--- a/server/lib/db.rb
+++ b/server/lib/db.rb
@@ -4,27 +4,74 @@ module Deltacloud
ENV['RACK_ENV'] == 'test' || ENV['DELTACLOUD_NO_DATABASE']
end
- unless test_environment?
- require 'data_mapper'
- require_relative './db/provider'
- require_relative './db/entity'
- require_relative './db/machine_template'
- require_relative './db/address_template'
- require_relative './db/volume_configuration'
- require_relative './db/volume_template'
- end
+ require 'sequel' unless test_environment?
+ require 'logger'
+
+ DATABASE_LOCATION = ENV['DATABASE_LOCATION'] ||
+ 'sqlite://'+File.join('/', 'var', 'tmp', "deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
- DATABASE_LOCATION = ENV['DATABASE_LOCATION'] || File.join('/', 'var', 'tmp', "deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
+ def self.database(opts={})
+ opts[:logger] = ::Logger.new($stdout) if ENV['API_VERBOSE']
+ @db ||= Sequel.connect(DATABASE_LOCATION, opts)
+ end
def self.initialize_database
- DataMapper::Logger.new($stdout, :debug) if ENV['API_VERBOSE']
- dbdir = File::dirname(DATABASE_LOCATION)
- FileUtils::mkdir(dbdir) unless File::directory?(dbdir)
- DataMapper::setup(:default, "sqlite://#{DATABASE_LOCATION}")
- DataMapper::finalize
- DataMapper::auto_upgrade!
+
+ database.create_table?(:providers) {
+ primary_key :id
+
+ column :driver, :string, { :null => false }
+ column :url, :string
+ index [ :url, :driver ]
+ }
+
+ database.create_table?(:entities) {
+ primary_key :id
+ foreign_key :provider_id, :providers, { :index => true, :null => false }
+ column :created_at, :timestamp
+
+ # Base
+ column :model, :string, { :index => true, :null => false, :default => 'entity' }
+
+ # Map Entity to Deltacloud model
+ # (like: Machine => Instance)
+ column :be_kind, :string
+ column :be_id, :string
+
+ # Entity
+ column :name, :string
+ column :description, :string
+ column :ent_properties, :string, { :text => true }
+
+
+ column :machine_config, :string
+ column :machine_image, :string
+
+ column :network, :string
+ column :ip, :string
+ column :hostname, :string
+ column :allocation, :string
+ column :default_gateway, :string
+ column :dns, :string
+ column :protocol, :string
+ column :mask, :string
+
+ column :format, :string
+ column :capacity, :string
+
+ column :volume_config, :string
+ column :volume_image, :string
+ }
end
end
-Deltacloud::initialize_database unless Deltacloud.test_environment?
+unless Deltacloud.test_environment?
+ Deltacloud::initialize_database
+ require_relative './db/provider'
+ require_relative './db/entity'
+ require_relative './db/machine_template'
+ require_relative './db/address_template'
+ require_relative './db/volume_configuration'
+ require_relative './db/volume_template'
+end
--
1.8.0.2
[PATCH core 3/3] CIMI: Updated CIMI models to use Sequel methods and helpers
Posted by mf...@redhat.com.
From: Michal Fojtik <mf...@redhat.com>
Signed-off-by: Michal fojtik <mf...@redhat.com>
---
server/lib/cimi/models/address_template.rb | 25 ++++------------
server/lib/cimi/models/machine.rb | 2 +-
server/lib/cimi/models/machine_template.rb | 25 ++++------------
server/lib/cimi/models/volume_configuration.rb | 25 ++++------------
server/lib/cimi/models/volume_template.rb | 40 ++++++++++----------------
5 files changed, 34 insertions(+), 83 deletions(-)
diff --git a/server/lib/cimi/models/address_template.rb b/server/lib/cimi/models/address_template.rb
index 2d81f23..1e28602 100644
--- a/server/lib/cimi/models/address_template.rb
+++ b/server/lib/cimi/models/address_template.rb
@@ -42,20 +42,13 @@ class CIMI::Model::AddressTemplate < CIMI::Model::Base
if context.driver.respond_to? :address_templates
context.driver.address_templates(context.credentials, {:env=>context})
else
- Deltacloud::Database::AddressTemplate.all(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider
- ).map { |t| from_db(t, context) }
+ current_db.address_teplates.map { |t| from_db(t, context) }
end
else
if context.driver.respond_to? :address_template
context.driver.address_template(context.credentials, id, :env=>context)
else
- template = Deltacloud::Database::AddressTemplate.first(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider,
- :id => id
- )
+ template = current_db.address_templates_dataset.first(:id => id)
raise CIMI::Model::NotFound unless template
from_db(template, context)
end
@@ -64,7 +57,7 @@ class CIMI::Model::AddressTemplate < CIMI::Model::Base
def self.create_from_json(body, context)
json = JSON.parse(body)
- new_template = current_db.address_templates.new(
+ new_template = current_db.add_address_template(
:name => json['name'],
:description => json['description'],
:hostname => json['hostname'],
@@ -75,17 +68,14 @@ class CIMI::Model::AddressTemplate < CIMI::Model::Base
:protocol => json['protocol'],
:mask => json['mask'],
:ent_properties => json['properties'].to_json,
- :be_kind => 'address_template',
- :be_id => ''
)
- new_template.save
from_db(new_template, context)
end
def self.create_from_xml(body, context)
xml = XmlSimple.xml_in(body)
xml['property'] ||= []
- new_template = current_db.address_templates.new(
+ new_template = current_db.add_address_template(
:name => xml['name'].first,
:description => xml['description'].first,
:ip => xml['ip'].first,
@@ -95,11 +85,8 @@ class CIMI::Model::AddressTemplate < CIMI::Model::Base
:dns => xml['dns'].first,
:protocol => xml['protocol'].nil? ? nil : xml['protocol'].first,
:mask => xml['mask'].first,
- :ent_properties => xml['property'].inject({}) { |r, p| r[p['key']]=p['content']; r },
- :be_kind => 'machine_template',
- :be_id => ''
+ :ent_properties => JSON::dump(xml['property'].inject({}) { |r, p| r[p['key']]=p['content']; r }),
)
- new_template.save
from_db(new_template, context)
end
@@ -121,7 +108,7 @@ class CIMI::Model::AddressTemplate < CIMI::Model::Base
:dns => model.dns,
:protocol => model.protocol,
:mask => model.mask,
- :property => model.ent_properties,
+ :property => JSON::parse(model.ent_properties),
:operations => [
{ :href => context.destroy_address_template_url(model.id), :rel => 'http://schemas.dmtf.org/cimi/1/action/delete' }
]
diff --git a/server/lib/cimi/models/machine.rb b/server/lib/cimi/models/machine.rb
index 7ac8f83..67e3eef 100644
--- a/server/lib/cimi/models/machine.rb
+++ b/server/lib/cimi/models/machine.rb
@@ -81,7 +81,7 @@ class CIMI::Model::Machine < CIMI::Model::Base
def self.create_from_xml(body, context)
xml = XmlSimple.xml_in(body)
if xml['machineTemplate'][0]['href']
- template = current_db.machine_templates.first(:id => xml['machineTemplate'][0]['href'].split('/').last)
+ template = current_db.machine_templates_dataset.first(:id => xml['machineTemplate'][0]['href'].split('/').last)
hardware_profile_id = template.machine_config.split('/').last
image_id = template.machine_image.split('/').last
else
diff --git a/server/lib/cimi/models/machine_template.rb b/server/lib/cimi/models/machine_template.rb
index eb660d2..675f709 100644
--- a/server/lib/cimi/models/machine_template.rb
+++ b/server/lib/cimi/models/machine_template.rb
@@ -44,16 +44,9 @@ class CIMI::Model::MachineTemplate < CIMI::Model::Base
class << self
def find(id, context)
if id == :all
- Deltacloud::Database::MachineTemplate.all(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider
- ).map { |t| from_db(t, context) }
+ current_db.machine_templates.map { |t| from_db(t, context) }
else
- template = Deltacloud::Database::MachineTemplate.first(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider,
- :id => id
- )
+ template = current_db.machine_templates_dataset.first(:id => id)
raise CIMI::Model::NotFound unless template
from_db(template, context)
end
@@ -61,31 +54,25 @@ class CIMI::Model::MachineTemplate < CIMI::Model::Base
def create_from_json(body, context)
json = JSON.parse(body)
- new_template = current_db.machine_templates.new(
+ new_template = current_db.add_machine_template(
:name => json['name'],
:description => json['description'],
:machine_config => json['machineConfig']['href'],
:machine_image => json['machineImage']['href'],
:ent_properties => json['properties'].to_json,
- :be_kind => 'machine_template',
- :be_id => ''
)
- new_template.save
from_db(new_template, context)
end
def create_from_xml(body, context)
xml = XmlSimple.xml_in(body)
- new_template = current_db.machine_templates.new(
+ new_template = current_db.add_machine_template(
:name => xml['name'].first,
:description => xml['description'].first,
:machine_config => xml['machineConfig'].first['href'],
:machine_image => xml['machineImage'].first['href'],
- :ent_properties => xml['property'].inject({}) { |r, p| r[p['key']]=p['content']; r },
- :be_kind => 'machine_template',
- :be_id => ''
+ :ent_properties => JSON::dump(xml['property'].inject({}) { |r, p| r[p['key']]=p['content']; r }),
)
- new_template.save
from_db(new_template, context)
end
@@ -102,7 +89,7 @@ class CIMI::Model::MachineTemplate < CIMI::Model::Base
:description => model.description,
:machine_config => { :href => model.machine_config },
:machine_image => { :href => model.machine_image },
- :property => model.ent_properties,
+ :property => JSON::parse(model.ent_properties),
:created => Time.parse(model.created_at.to_s).xmlschema,
:operations => [
{ :href => context.destroy_machine_template_url(model.id), :rel => 'http://schemas.dmtf.org/cimi/1/action/delete' }
diff --git a/server/lib/cimi/models/volume_configuration.rb b/server/lib/cimi/models/volume_configuration.rb
index b0ba85b..12a3198 100644
--- a/server/lib/cimi/models/volume_configuration.rb
+++ b/server/lib/cimi/models/volume_configuration.rb
@@ -27,32 +27,26 @@ class CIMI::Model::VolumeConfiguration < CIMI::Model::Base
def self.create_from_json(body, context)
json = JSON.parse(body)
- new_config = current_db.volume_configurations.new(
+ new_config = current_db.add_volume_configuration(
:name => json['name'],
:description => json['description'],
:format => json['format'],
:capacity => json['capacity'],
:ent_properties => json['properties'].to_json,
- :be_kind => 'volume_configuration',
- :be_id => ''
)
- new_config.save
from_db(new_config, context)
end
def self.create_from_xml(body, context)
xml = XmlSimple.xml_in(body)
xml['property'] ||= []
- new_config = current_db.volume_configurations.new(
+ new_config = current_db.add_volume_configuration(
:name => xml['name'].first,
:description => xml['description'].first,
:format => xml['format'].first,
:capacity => xml['capacity'].first,
- :ent_properties => xml['property'].inject({}) { |r, p| r[p['key']]=p['content']; r },
- :be_kind => 'volume_configuration',
- :be_id => ''
+ :ent_properties => JSON::dump(xml['property'].inject({}) { |r, p| r[p['key']]=p['content']; r }),
)
- new_config.save
from_db(new_config, context)
end
@@ -65,20 +59,13 @@ class CIMI::Model::VolumeConfiguration < CIMI::Model::Base
if context.driver.respond_to? :volume_configurations
context.driver.volume_configurations(context.credentials, {:env=>context})
else
- Deltacloud::Database::VolumeConfiguration.all(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider
- ).map { |t| from_db(t, context) }
+ current_db.volume_configurations.map { |t| from_db(t, context) }
end
else
if context.driver.respond_to? :volume_configuration
context.driver.volume_configuration(context.credentials, id, :env=>context)
else
- config = Deltacloud::Database::VolumeConfiguration.first(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider,
- :id => id
- )
+ config = current_db.volume_configurations_dataset.first(:id => id)
raise CIMI::Model::NotFound unless config
from_db(config, context)
end
@@ -94,7 +81,7 @@ class CIMI::Model::VolumeConfiguration < CIMI::Model::Base
:description => model.description,
:format => model.format,
:capacity => context.to_kibibyte(model.capacity, "GB"),
- :property => model.ent_properties,
+ :property => JSON::parse(model.ent_properties),
:operations => [
{ :href => context.destroy_volume_configuration_url(model.id), :rel => 'http://schemas.dmtf.org/cimi/1/action/delete' }
]
diff --git a/server/lib/cimi/models/volume_template.rb b/server/lib/cimi/models/volume_template.rb
index a2b8eb7..47bbb3f 100644
--- a/server/lib/cimi/models/volume_template.rb
+++ b/server/lib/cimi/models/volume_template.rb
@@ -30,27 +30,20 @@ class CIMI::Model::VolumeTemplate < CIMI::Model::Base
end
def self.find(id, context)
- if id==:all
- if context.driver.respond_to? :volume_templates
- context.driver.volume_templates(context.credentials, {:env=>context})
- else
- Deltacloud::Database::VolumeTemplate.all(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider
- ).map { |t| from_db(t, context) }
+ if id==:all
+ if context.driver.respond_to? :volume_templates
+ context.driver.volume_templates(context.credentials, {:env=>context})
+ else
+ current_db.volume_templates.map { |t| from_db(t, context) }
end
else
- if context.driver.respond_to? :volume_template
- context.driver.volume_template(context.credentials, id, :env=>context)
- else
- template = Deltacloud::Database::VolumeTemplate.first(
- 'provider.driver' => driver_symbol.to_s,
- 'provider.url' => current_provider,
- :id => id
- )
- raise CIMI::Model::NotFound unless template
- from_db(template, context)
- end
+ if context.driver.respond_to? :volume_template
+ context.driver.volume_template(context.credentials, id, :env=>context)
+ else
+ template = current_db.volume_templates_dataset.first(:id => id)
+ raise CIMI::Model::NotFound unless template
+ from_db(template, context)
+ end
end
end
@@ -58,16 +51,13 @@ class CIMI::Model::VolumeTemplate < CIMI::Model::Base
input = (type == :xml)? XmlSimple.xml_in(body, {"ForceArray"=>false,"NormaliseSpace"=>2}) : JSON.parse(body)
input['property'] ||= []
vol_image = input['volumeImage']['href'] if input['volumeImage']
- new_template = current_db.volume_templates.new(
+ new_template = current_db.add_volume_template(
:name => input['name'],
:description => input['description'],
:volume_config => input['volumeConfig']['href'],
:volume_image => vol_image,
- :ent_properties => input['property'].inject({}) { |r, p| r[p['key']]=p['content']; r },
- :be_kind => 'volume_template',
- :be_id => ''
+ :ent_properties => JSON::dump(input['property'].inject({}) { |r, p| r[p['key']]=p['content']; r }),
)
- new_template.save
from_db(new_template, context)
end
@@ -84,7 +74,7 @@ private
:description => model.description,
:volume_config => {:href => model.volume_config},
:volume_image => {:href => model.volume_image},
- :property => model.ent_properties,
+ :property => JSON::parse(model.ent_properties),
:operations => [
{ :href => context.destroy_volume_template_url(model.id), :rel => 'http://schemas.dmtf.org/cimi/1/action/delete' }
]
--
1.8.0.2
Re: Replace datamapper with sequel
Posted by jvlcek <jv...@redhat.com>.
On 01/16/2013 04:41 AM, Michal Fojtik wrote:
> On 01/15, jvlcek wrote:
>
> Hi Joe,
>
> Thanks for trying and looking at this patch!
>
>> I did some minimal testing and response time seems faster.
>>
>> I tried creating a macine and machine_templates. I was not able
>> to create the machine_templates but that is very likely because I
>> was not doing it correctly. Can you please pass along an example
>> of how to create a machine_template?
> Sure, some sample MachineCreate XML/JSON files are already included
> in Deltacloud GIT repository (server/support/cimi), you can use them
> with the "curl" command:
>
> $ curl -d @support/cimi/machine_template.xml -H 'Content-Type: text/xml' \
> --user 'mockuser:mockpassword' -X POST \
> http://localhost:3001/cimi/machine_templates
>
> This should result into MachineTemplate creation. If you want to tweak
> attributes for new MachineTemplate, just edit 'machine_template.xml' file.
>
> That directory contains many other examples for entities we support.
> I mostly added them there while I was working on database stuff (for
> testing).
>
>> + DATABASE_LOCATION = ENV['DATABASE_LOCATION'] ||
>> + 'sqlite://'+File.join('/', 'var', 'tmp',
>> "deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
>>
>> - DATABASE_LOCATION = ENV['DATABASE_LOCATION'] || File.join('/', 'var',
>> 'tmp', "deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
>> + def self.database(opts={})
>> + opts[:logger] = ::Logger.new($stdout) if ENV['API_VERBOSE']
>> + @db ||= Sequel.connect(DATABASE_LOCATION, opts)
>> + end
>>
>>
>> JoeV: Question/nit:
>> JoeV:
>> JoeV: Would it make more sense to allow DATABASE_LOCATION to specify
>> JoeV: the path to db.sqlite but still use "db.sqlite" at the end of the
>> path?
> The benefit for using 'full' path is that in some case you don't want to
> use sqlite as a database backend, but some other database (like postgres).
> In that case all you need to do is to set 'DATABASE_LOCATION' system env
> variable to something like: "postgres://...".
>
>> JoeV: Question/nit:
>> JoeV:
>> JoeV: Does sequel require the other files which "rake mock:fixtures:reset"
>> JoeV: populates in DATABASE_LOCATION or just "db.sqlite" ? If so:
>> JoeV: doesn't the Rakefile need to be updated to only populate
>> JoeV: DATABASE_LOCATION or just "db.sqlite" ?
> In some cases, like when you create a new Machine using the mock driver,
> Mock driver create YAML files and also add record to the database.
> To reset all this, you need to remove both mock YAML files and also the
> database file.
>
> The only complication could be situlation when you are using DB backend
> other than sqlite. In that case this rake task will of course don't rip
> off the database.
>
>
>> + database.create_table?(:providers) {
>> + primary_key :id
>> +
>> + column :driver, :string, { :null => false }
>> + column :url, :string
>> + index [ :url, :driver ]
>> + }
>> +
>> + database.create_table?(:entities) {
>> + primary_key :id
>> + foreign_key :provider_id, :providers, { :index => true, :null =>
>> false }
>> + column :created_at, :timestamp
>> +
>>
>> JoeV: Question/nit:
>> JoeV: Why the "?" postfix for create_table"?" ?
>> JoeV: Does it return true on success?
> According to Sequel rdoc, the 'create_table?' method will create the table
> only if the table does not already exists. Without '?' suffix Sequel will
> try (and fail) to create the table structure every time you start
> Deltacloud. If you have 'verbose' mode turned on (-V deltacloudd parameter)
> you will in that case see very nasty sqlite error :-)
>
> Now you will see something like 'CREATE TABLE IF DOES NOT EXISTS' or
> something similar :-)
>
> -- Michal
>
>>
Thanks for the reply and explanations!
The curl for creating machine_templates
works like a charm!
Thanks,
Joe
Re: Replace datamapper with sequel
Posted by Michal Fojtik <mf...@redhat.com>.
On 01/15, jvlcek wrote:
Hi Joe,
Thanks for trying and looking at this patch!
> I did some minimal testing and response time seems faster.
>
> I tried creating a macine and machine_templates. I was not able
> to create the machine_templates but that is very likely because I
> was not doing it correctly. Can you please pass along an example
> of how to create a machine_template?
Sure, some sample MachineCreate XML/JSON files are already included
in Deltacloud GIT repository (server/support/cimi), you can use them
with the "curl" command:
$ curl -d @support/cimi/machine_template.xml -H 'Content-Type: text/xml' \
--user 'mockuser:mockpassword' -X POST \
http://localhost:3001/cimi/machine_templates
This should result into MachineTemplate creation. If you want to tweak
attributes for new MachineTemplate, just edit 'machine_template.xml' file.
That directory contains many other examples for entities we support.
I mostly added them there while I was working on database stuff (for
testing).
> + DATABASE_LOCATION = ENV['DATABASE_LOCATION'] ||
> + 'sqlite://'+File.join('/', 'var', 'tmp',
> "deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
>
> - DATABASE_LOCATION = ENV['DATABASE_LOCATION'] || File.join('/', 'var',
> 'tmp', "deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
> + def self.database(opts={})
> + opts[:logger] = ::Logger.new($stdout) if ENV['API_VERBOSE']
> + @db ||= Sequel.connect(DATABASE_LOCATION, opts)
> + end
>
>
> JoeV: Question/nit:
> JoeV:
> JoeV: Would it make more sense to allow DATABASE_LOCATION to specify
> JoeV: the path to db.sqlite but still use "db.sqlite" at the end of the
> path?
The benefit for using 'full' path is that in some case you don't want to
use sqlite as a database backend, but some other database (like postgres).
In that case all you need to do is to set 'DATABASE_LOCATION' system env
variable to something like: "postgres://...".
> JoeV: Question/nit:
> JoeV:
> JoeV: Does sequel require the other files which "rake mock:fixtures:reset"
> JoeV: populates in DATABASE_LOCATION or just "db.sqlite" ? If so:
> JoeV: doesn't the Rakefile need to be updated to only populate
> JoeV: DATABASE_LOCATION or just "db.sqlite" ?
In some cases, like when you create a new Machine using the mock driver,
Mock driver create YAML files and also add record to the database.
To reset all this, you need to remove both mock YAML files and also the
database file.
The only complication could be situlation when you are using DB backend
other than sqlite. In that case this rake task will of course don't rip
off the database.
> + database.create_table?(:providers) {
> + primary_key :id
> +
> + column :driver, :string, { :null => false }
> + column :url, :string
> + index [ :url, :driver ]
> + }
> +
> + database.create_table?(:entities) {
> + primary_key :id
> + foreign_key :provider_id, :providers, { :index => true, :null =>
> false }
> + column :created_at, :timestamp
> +
>
> JoeV: Question/nit:
> JoeV: Why the "?" postfix for create_table"?" ?
> JoeV: Does it return true on success?
According to Sequel rdoc, the 'create_table?' method will create the table
only if the table does not already exists. Without '?' suffix Sequel will
try (and fail) to create the table structure every time you start
Deltacloud. If you have 'verbose' mode turned on (-V deltacloudd parameter)
you will in that case see very nasty sqlite error :-)
Now you will see something like 'CREATE TABLE IF DOES NOT EXISTS' or
something similar :-)
-- Michal
>
>
--
Michal Fojtik <mf...@redhat.com>
Deltacloud API, CloudForms
Re: Replace datamapper with sequel
Posted by jvlcek <jv...@redhat.com>.
On 01/15/2013 10:08 AM, mfojtik@redhat.com wrote:
> Hi,
>
> This is an initial attempt to replace DataMapper with more ligher
> ORM (Sequel).
> The main reason of doing this is lack of DataMapper RPM's in distributions
> we want to support.
>
> Pros:
>
> - Less dependencies (sequel + sqlite3)
> - More performance tunning (sequel allows to modify SQL queries to get less
> queries)
> - Caching (not interesting right now, but might be in future)
>
> Cons:
>
> - No automigration (you will need to run fixture reset rake task)
>
> Let me know what you think :)
>
> If you want try this patch, please reset the 'mock' fixtures and DB using:
>
> 'rake mock:fixtures:reset'
>
> Also please run 'bundle update' to get rid of DataMapper and install Sequel.
>
> Let me know what you think :)
>
> -- Michal
>
Michal,
This looks great.
I did some minimal testing and response time seems faster.
I tried creating a macine and machine_templates. I was not able
to create the machine_templates but that is very likely because I
was not doing it correctly. Can you please pass along an example
of how to create a machine_template?
I just have a couple minor nits/questions about the code in:
[PATCH core 1/3] CIMI: Replaced DataMapper with Sequel ORM
Please see "JoeV:" below for my comments.
Hope this helps!
Joe
- - -
diff --git a/server/lib/db.rb b/server/lib/db.rb
index 567b356..64cc283 100644
--- a/server/lib/db.rb
+++ b/server/lib/db.rb
@@ -4,27 +4,74 @@ module Deltacloud
ENV['RACK_ENV'] == 'test' || ENV['DELTACLOUD_NO_DATABASE']
end
- unless test_environment?
- require 'data_mapper'
- require_relative './db/provider'
- require_relative './db/entity'
- require_relative './db/machine_template'
- require_relative './db/address_template'
- require_relative './db/volume_configuration'
- require_relative './db/volume_template'
- end
+ require 'sequel' unless test_environment?
+ require 'logger'
+
+ DATABASE_LOCATION = ENV['DATABASE_LOCATION'] ||
+ 'sqlite://'+File.join('/', 'var', 'tmp',
"deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
- DATABASE_LOCATION = ENV['DATABASE_LOCATION'] || File.join('/', 'var',
'tmp', "deltacloud-mock-#{ENV['USER']}", 'db.sqlite')
+ def self.database(opts={})
+ opts[:logger] = ::Logger.new($stdout) if ENV['API_VERBOSE']
+ @db ||= Sequel.connect(DATABASE_LOCATION, opts)
+ end
JoeV: Question/nit:
JoeV:
JoeV: Would it make more sense to allow DATABASE_LOCATION to specify
JoeV: the path to db.sqlite but still use "db.sqlite" at the end of the
path?
JoeV: Question/nit:
JoeV:
JoeV: Does sequel require the other files which "rake mock:fixtures:reset"
JoeV: populates in DATABASE_LOCATION or just "db.sqlite" ? If so:
JoeV: doesn't the Rakefile need to be updated to only populate
JoeV: DATABASE_LOCATION or just "db.sqlite" ?
def self.initialize_database
- DataMapper::Logger.new($stdout, :debug) if ENV['API_VERBOSE']
- dbdir = File::dirname(DATABASE_LOCATION)
- FileUtils::mkdir(dbdir) unless File::directory?(dbdir)
- DataMapper::setup(:default, "sqlite://#{DATABASE_LOCATION}")
- DataMapper::finalize
- DataMapper::auto_upgrade!
+
+ database.create_table?(:providers) {
+ primary_key :id
+
+ column :driver, :string, { :null => false }
+ column :url, :string
+ index [ :url, :driver ]
+ }
+
+ database.create_table?(:entities) {
+ primary_key :id
+ foreign_key :provider_id, :providers, { :index => true, :null =>
false }
+ column :created_at, :timestamp
+
JoeV: Question/nit:
JoeV: Why the "?" postfix for create_table"?" ?
JoeV: Does it return true on success?
[PATCH core 2/3] CIMI: Migrated database models to use Sequel
Posted by mf...@redhat.com.
From: Michal Fojtik <mf...@redhat.com>
Signed-off-by: Michal fojtik <mf...@redhat.com>
---
server/lib/db/address_template.rb | 10 ---------
server/lib/db/entity.rb | 25 +++++----------------
server/lib/db/machine_template.rb | 5 -----
server/lib/db/provider.rb | 41 +++++------------------------------
server/lib/db/volume_configuration.rb | 4 ----
server/lib/db/volume_template.rb | 4 ----
6 files changed, 11 insertions(+), 78 deletions(-)
diff --git a/server/lib/db/address_template.rb b/server/lib/db/address_template.rb
index 5a08346..8586255 100644
--- a/server/lib/db/address_template.rb
+++ b/server/lib/db/address_template.rb
@@ -2,16 +2,6 @@ module Deltacloud
module Database
class AddressTemplate < Entity
- belongs_to :provider
-
- property :ip, String
- property :hostname, String
- property :allocation, String, :default => 'dynamic'
- property :default_gateway, String, :default => 'unknown'
- property :dns, String, :default => 'unknown'
- property :protocol, String, :default => 'ipv4'
- property :mask, String, :default => 'unknown'
- property :network, String
end
end
diff --git a/server/lib/db/entity.rb b/server/lib/db/entity.rb
index f9b34ff..963a26b 100644
--- a/server/lib/db/entity.rb
+++ b/server/lib/db/entity.rb
@@ -1,33 +1,18 @@
module Deltacloud
module Database
- class BaseEntity
- include DataMapper::Resource
+ class Entity < Sequel::Model
- belongs_to :provider
+ many_to_one :provider
- property :id, Serial
- property :type, Discriminator
-
- property :be_kind, String, :required => true # => Machine, MachineImage, ...
- property :be_id, String # => Original Machine 'id'
-
- timestamps :created_at, :updated_on
- end
-
- class Entity < BaseEntity
-
- belongs_to :provider
-
- property :name, String
- property :description, String
- property :ent_properties, Json
+ plugin :single_table_inheritance, :model
+ plugin :timestamps, :create => :created_at
def to_hash
retval = {}
retval.merge!(:name => self.name) if !self.name.nil?
retval.merge!(:description => self.description) if !self.description.nil?
- retval.merge!(:property => self.ent_properties) if !self.ent_properties.nil?
+ retval.merge!(:property => JSON::parse(self.ent_properties)) if !self.ent_properties.nil?
retval
end
diff --git a/server/lib/db/machine_template.rb b/server/lib/db/machine_template.rb
index 134eb23..1b34161 100644
--- a/server/lib/db/machine_template.rb
+++ b/server/lib/db/machine_template.rb
@@ -2,11 +2,6 @@ module Deltacloud
module Database
class MachineTemplate < Entity
- belongs_to :provider
-
- property :machine_config, String, :length => 255
- property :machine_image, String, :length => 255
-
end
end
diff --git a/server/lib/db/provider.rb b/server/lib/db/provider.rb
index e8ca4f5..c0b8485 100644
--- a/server/lib/db/provider.rb
+++ b/server/lib/db/provider.rb
@@ -1,41 +1,12 @@
module Deltacloud
module Database
- class Provider
- include DataMapper::Resource
-
- property :id, Serial
- property :driver, String, :required => true
- property :url, Text
-
- has n, :entities
- has n, :machine_templates
- has n, :address_templates
- has n, :volume_configurations
- has n, :volume_templates
-
- # This is a workaround for strange bug in Fedora MRI:
- #
- def machine_templates
- MachineTemplate.all(:provider_id => self.id)
- end
-
- def address_templates
- AddressTemplate.all(:provider_id => self.id)
- end
-
- def volume_configurations
- VolumeConfiguration.all(:provider_id => self.id)
- end
-
- def volume_templates
- VolumeTemplate.all(:provider_id => self.id)
- end
-
- def entities
- Entity.all(:provider_id => self.id)
- end
-
+ class Provider < Sequel::Model
+ one_to_many :entities
+ one_to_many :machine_templates
+ one_to_many :address_templates
+ one_to_many :volume_templates
+ one_to_many :volume_configurations
end
end
diff --git a/server/lib/db/volume_configuration.rb b/server/lib/db/volume_configuration.rb
index 58aa624..aa956ce 100644
--- a/server/lib/db/volume_configuration.rb
+++ b/server/lib/db/volume_configuration.rb
@@ -2,10 +2,6 @@ module Deltacloud
module Database
class VolumeConfiguration < Entity
- belongs_to :provider
-
- property :format, String
- property :capacity, String
end
end
diff --git a/server/lib/db/volume_template.rb b/server/lib/db/volume_template.rb
index 8658b8b..0d876bf 100644
--- a/server/lib/db/volume_template.rb
+++ b/server/lib/db/volume_template.rb
@@ -2,10 +2,6 @@ module Deltacloud
module Database
class VolumeTemplate < Entity
- belongs_to :provider
-
- property :volume_config, String
- property :volume_image, String
end
end
--
1.8.0.2