You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by er...@apache.org on 2015/11/09 16:34:39 UTC
libcloud git commit: [google compute] add pricing data update script
Repository: libcloud
Updated Branches:
refs/heads/trunk 23127e387 -> c07dbe8c3
[google compute] add pricing data update script
Also used the script to update `pricing.json` with current prices on a
per-region basis. Added numeric sorting to the JSON output; as a result, all
other pricing is now sorted numerically, e.g., "12GB" sorts after "3GB" and
"150" sorts after "20" whereas previously, it would be the opposite.
Closes #464
Signed-off-by: Eric Johnson <er...@google.com>
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/c07dbe8c
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/c07dbe8c
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/c07dbe8c
Branch: refs/heads/trunk
Commit: c07dbe8c359b8a1ac760c2b168ffbd4f00dca952
Parents: 23127e3
Author: Misha Brukman <mb...@google.com>
Authored: Tue Feb 17 12:00:16 2015 -0500
Committer: Eric Johnson <er...@google.com>
Committed: Mon Nov 9 15:34:11 2015 +0000
----------------------------------------------------------------------
CHANGES.rst | 4 +
contrib/update_google_prices.py | 89 ++++++++++++++++++++
contrib/utils.py | 48 +++++++++++
contrib/utils_test.py | 68 +++++++++++++++
libcloud/data/pricing.json | 158 ++++++++++++++++++++++++++++++-----
5 files changed, 345 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/CHANGES.rst
----------------------------------------------------------------------
diff --git a/CHANGES.rst b/CHANGES.rst
index ebc76e3..39db15a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -26,6 +26,10 @@ General
Compute
~~~~~~~
+- [google compute] add pricing data update script
+ (GITHUB-464)
+ [Misha Brukman]
+
- Fix a bug in the ``list_volumes`` method in the CloudStack driver so it
returns an empty list if no volumes are found.
(GITHUB-617)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/contrib/update_google_prices.py
----------------------------------------------------------------------
diff --git a/contrib/update_google_prices.py b/contrib/update_google_prices.py
new file mode 100755
index 0000000..e69ffe1
--- /dev/null
+++ b/contrib/update_google_prices.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Loads Google Cloud Platform prices and updates the `pricing.json` data file.
+"""
+
+import os
+try:
+ import simplejson as json
+except ImportError:
+ import json
+import sys
+import time
+import urllib2
+import utils
+
+BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+PRICING_FILE_PATH = os.path.join(BASE_PATH, '../libcloud/data/pricing.json')
+PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH)
+
+GOOGLE_CLOUD_PRICES = 'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json'
+
+
+def main(argv):
+ # Read the current pricing data.
+ libcloud_data = {}
+ with open(PRICING_FILE_PATH, 'r') as libcloud_in:
+ libcloud_data = json.loads(libcloud_in.read())
+
+ # Download the current Google Cloud Platform pricing.
+ req = urllib2.Request(GOOGLE_CLOUD_PRICES, '')
+ google_ext_prices = json.loads(urllib2.urlopen(req).read())
+ if 'gcp_price_list' not in google_ext_prices:
+ sys.stderr.write('Google Cloud pricing data missing "gcp_price_list" node\n')
+ sys.exit(1)
+
+ # This is a map from regions used in the pricing JSON file to the regions as
+ # reflected in the Google Cloud Platform documentation and APIs.
+ pricing_to_region = {
+ 'us': 'us',
+ 'eu': 'europe', # alias for 'europe'
+ 'europe': 'europe',
+ 'apac': 'asia', # alias for 'asia'
+ 'asia': 'asia'
+ }
+
+ # Initialize Google Cloud Platform regions.
+ for _, region in pricing_to_region.iteritems():
+ libcloud_data['compute']['google_%s' % region] = {}
+
+ # Update Google Compute Engine pricing.
+ gcp_price_list = google_ext_prices['gcp_price_list']
+ gce_vm_prefix = 'CP-COMPUTEENGINE-VMIMAGE-'
+ for name, prices in gcp_price_list.iteritems():
+ if not name.startswith(gce_vm_prefix):
+ continue
+ short_name = name[len(gce_vm_prefix):]
+ machine_type = short_name.lower()
+ for key, price in prices.iteritems():
+ if key in pricing_to_region:
+ region = pricing_to_region[key]
+ libcloud_data['compute']['google_%s' % region][machine_type] = price
+
+ # Update last-modified timestamp.
+ libcloud_data['updated'] = int(time.time())
+
+ # Write updated price list.
+ with open(PRICING_FILE_PATH, 'w') as libcloud_out:
+ json_str = json.dumps(libcloud_data, indent=4 * ' ',
+ item_sort_key=utils.sortKeysNumerically)
+ libcloud_out.write(json_str)
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/contrib/utils.py
----------------------------------------------------------------------
diff --git a/contrib/utils.py b/contrib/utils.py
new file mode 100644
index 0000000..2028721
--- /dev/null
+++ b/contrib/utils.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+#
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+import re
+
+
+def splitStringWithNumbers(string):
+ """Splits input string into a list of items, numeric and otherwise.
+
+ Returns a list of values, each either an interpreted number, or a substring
+ of the original input.
+
+ E.g., 'abc-123-def' => ['abc-', 123, '-def']
+ """
+ rawParts = re.split(r'(\d+)', string)
+
+ # Filter out empty strings.
+ nonEmptyParts = filter(None, rawParts)
+
+ # Convert any numeric strings to numbers.
+ def splitHelper(nonEmptyParts):
+ for part in nonEmptyParts:
+ if re.match(r'\d+', part):
+ yield int(part)
+ else:
+ yield part
+
+ return list(splitHelper(nonEmptyParts))
+
+
+def sortKeysNumerically(key_value):
+ key, value = key_value
+ return splitStringWithNumbers(key)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/contrib/utils_test.py
----------------------------------------------------------------------
diff --git a/contrib/utils_test.py b/contrib/utils_test.py
new file mode 100644
index 0000000..98016a4
--- /dev/null
+++ b/contrib/utils_test.py
@@ -0,0 +1,68 @@
+#!/usr/bin/python
+#
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+try:
+ import simplejson as json
+except ImportError:
+ import json
+import unittest
+import utils
+
+
+class SplitStringToAlphaNumTest(unittest.TestCase):
+
+ def testInitial(self):
+ self.assertEqual(utils.splitStringWithNumbers('12-abc'),
+ [12, '-abc'])
+
+ def testMiddle(self):
+ self.assertEqual(utils.splitStringWithNumbers('abc-345-def'),
+ ['abc-', 345, '-def'])
+
+ def testFinal(self):
+ self.assertEqual(utils.splitStringWithNumbers('xyz-42'),
+ ['xyz-', 42])
+
+ def testMultiple(self):
+ self.assertEqual(utils.splitStringWithNumbers('Aaa-123-Bbb-456-Ccc'),
+ ['Aaa-', 123, '-Bbb-', 456, '-Ccc'])
+
+
+class SortKeysNumericallyTest(unittest.TestCase):
+
+ def testSimple(self):
+ input = {
+ 'a-1': 1,
+ 'a-12': 12,
+ 'a-2': 2,
+ }
+ output = """\
+{
+ "a-1": 1,
+ "a-2": 2,
+ "a-12": 12
+}\
+"""
+ self.assertEqual(
+ json.dumps(input, indent=4 * ' ',
+ item_sort_key=utils.sortKeysNumerically),
+ output)
+
+
+if __name__ == '__main__':
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/libcloud/blob/c07dbe8c/libcloud/data/pricing.json
----------------------------------------------------------------------
diff --git a/libcloud/data/pricing.json b/libcloud/data/pricing.json
index ab85715..5cb17e3 100644
--- a/libcloud/data/pricing.json
+++ b/libcloud/data/pricing.json
@@ -467,28 +467,142 @@
"x-large": 0.12
},
"gogrid": {
- "16GB": 3.04,
"1GB": 0.19,
- "24GB": 4.56,
"2GB": 0.38,
"4GB": 0.76,
- "512MB": 0.095,
- "8GB": 1.52
+ "8GB": 1.52,
+ "16GB": 3.04,
+ "24GB": 4.56,
+ "512MB": 0.095
+ },
+ "google_asia": {
+ "f1-micro": 0.009,
+ "f1-micro-preemptible": 0.005,
+ "g1-small": 0.03,
+ "g1-small-preemptible": 0.01,
+ "n1-highcpu-2": 0.084,
+ "n1-highcpu-2-preemptible": 0.022,
+ "n1-highcpu-4": 0.168,
+ "n1-highcpu-4-preemptible": 0.044,
+ "n1-highcpu-8": 0.336,
+ "n1-highcpu-8-preemptible": 0.088,
+ "n1-highcpu-16": 0.672,
+ "n1-highcpu-16-preemptible": 0.176,
+ "n1-highcpu-32": 1.344,
+ "n1-highcpu-32-preemptible": 0.352,
+ "n1-highmem-2": 0.139,
+ "n1-highmem-2-preemptible": 0.0385,
+ "n1-highmem-4": 0.278,
+ "n1-highmem-4-preemptible": 0.077,
+ "n1-highmem-8": 0.556,
+ "n1-highmem-8-preemptible": 0.154,
+ "n1-highmem-16": 1.112,
+ "n1-highmem-16-preemptible": 0.308,
+ "n1-highmem-32": 2.224,
+ "n1-highmem-32-preemptible": 0.616,
+ "n1-standard-1": 0.055,
+ "n1-standard-1-preemptible": 0.0165,
+ "n1-standard-2": 0.11,
+ "n1-standard-2-preemptible": 0.033,
+ "n1-standard-4": 0.22,
+ "n1-standard-4-preemptible": 0.066,
+ "n1-standard-8": 0.44,
+ "n1-standard-8-preemptible": 0.132,
+ "n1-standard-16": 0.88,
+ "n1-standard-16-preemptible": 0.264,
+ "n1-standard-32": 1.76,
+ "n1-standard-32-preemptible": 0.528
+ },
+ "google_europe": {
+ "f1-micro": 0.009,
+ "f1-micro-preemptible": 0.005,
+ "g1-small": 0.03,
+ "g1-small-preemptible": 0.01,
+ "n1-highcpu-2": 0.084,
+ "n1-highcpu-2-preemptible": 0.022,
+ "n1-highcpu-4": 0.168,
+ "n1-highcpu-4-preemptible": 0.044,
+ "n1-highcpu-8": 0.336,
+ "n1-highcpu-8-preemptible": 0.088,
+ "n1-highcpu-16": 0.672,
+ "n1-highcpu-16-preemptible": 0.176,
+ "n1-highcpu-32": 1.344,
+ "n1-highcpu-32-preemptible": 0.352,
+ "n1-highmem-2": 0.139,
+ "n1-highmem-2-preemptible": 0.0385,
+ "n1-highmem-4": 0.278,
+ "n1-highmem-4-preemptible": 0.077,
+ "n1-highmem-8": 0.556,
+ "n1-highmem-8-preemptible": 0.154,
+ "n1-highmem-16": 1.112,
+ "n1-highmem-16-preemptible": 0.308,
+ "n1-highmem-32": 2.224,
+ "n1-highmem-32-preemptible": 0.616,
+ "n1-standard-1": 0.055,
+ "n1-standard-1-preemptible": 0.0165,
+ "n1-standard-2": 0.11,
+ "n1-standard-2-preemptible": 0.033,
+ "n1-standard-4": 0.22,
+ "n1-standard-4-preemptible": 0.066,
+ "n1-standard-8": 0.44,
+ "n1-standard-8-preemptible": 0.132,
+ "n1-standard-16": 0.88,
+ "n1-standard-16-preemptible": 0.264,
+ "n1-standard-32": 1.76,
+ "n1-standard-32-preemptible": 0.528
+ },
+ "google_us": {
+ "f1-micro": 0.008,
+ "f1-micro-preemptible": 0.005,
+ "g1-small": 0.027,
+ "g1-small-preemptible": 0.01,
+ "n1-highcpu-2": 0.076,
+ "n1-highcpu-2-preemptible": 0.02,
+ "n1-highcpu-4": 0.152,
+ "n1-highcpu-4-preemptible": 0.04,
+ "n1-highcpu-8": 0.304,
+ "n1-highcpu-8-preemptible": 0.08,
+ "n1-highcpu-16": 0.608,
+ "n1-highcpu-16-preemptible": 0.16,
+ "n1-highcpu-32": 1.216,
+ "n1-highcpu-32-preemptible": 0.32,
+ "n1-highmem-2": 0.126,
+ "n1-highmem-2-preemptible": 0.035,
+ "n1-highmem-4": 0.252,
+ "n1-highmem-4-preemptible": 0.07,
+ "n1-highmem-8": 0.504,
+ "n1-highmem-8-preemptible": 0.14,
+ "n1-highmem-16": 1.008,
+ "n1-highmem-16-preemptible": 0.28,
+ "n1-highmem-32": 2.016,
+ "n1-highmem-32-preemptible": 0.56,
+ "n1-standard-1": 0.05,
+ "n1-standard-1-preemptible": 0.015,
+ "n1-standard-2": 0.1,
+ "n1-standard-2-preemptible": 0.03,
+ "n1-standard-4": 0.2,
+ "n1-standard-4-preemptible": 0.06,
+ "n1-standard-8": 0.4,
+ "n1-standard-8-preemptible": 0.12,
+ "n1-standard-16": 0.8,
+ "n1-standard-16-preemptible": 0.24,
+ "n1-standard-32": 1.6,
+ "n1-standard-32-preemptible": 0.48
},
"nephoscale": {
"1": 0.6,
+ "3": 0.063,
+ "5": 0.031,
+ "7": 0.125,
+ "9": 0.188,
"11": 0.35,
"27": 0.0,
- "3": 0.063,
"46": 0.1,
"48": 0.15,
- "5": 0.031,
"50": 0.28,
"52": 0.48,
"54": 0.938,
- "56": 0.75,
- "7": 0.125,
- "9": 0.188
+ "56": 0.75
},
"nimbus": {
"m1.large": 0.0,
@@ -640,11 +754,11 @@
"performance1-2": 0.08,
"performance1-4": 0.16,
"performance1-8": 0.32,
- "performance2-120": 5.44,
"performance2-15": 0.68,
"performance2-30": 1.36,
"performance2-60": 2.72,
- "performance2-90": 4.08
+ "performance2-90": 4.08,
+ "performance2-120": 5.44
},
"rackspacenovalon": {
"2": 0.032,
@@ -658,11 +772,11 @@
"performance1-2": 0.08,
"performance1-4": 0.16,
"performance1-8": 0.32,
- "performance2-120": 5.44,
"performance2-15": 0.68,
"performance2-30": 1.36,
"performance2-60": 2.72,
- "performance2-90": 4.08
+ "performance2-90": 4.08,
+ "performance2-120": 5.44
},
"rackspacenovasyd": {
"2": 0.026,
@@ -676,11 +790,11 @@
"performance1-2": 0.08,
"performance1-4": 0.16,
"performance1-8": 0.32,
- "performance2-120": 5.44,
"performance2-15": 0.68,
"performance2-30": 1.36,
"performance2-60": 2.72,
- "performance2-90": 4.08
+ "performance2-90": 4.08,
+ "performance2-120": 5.44
},
"rackspacenovaus": {
"2": 0.022,
@@ -694,11 +808,11 @@
"performance1-2": 0.08,
"performance1-4": 0.16,
"performance1-8": 0.32,
- "performance2-120": 5.44,
"performance2-15": 0.68,
"performance2-30": 1.36,
"performance2-60": 2.72,
- "performance2-90": 4.08
+ "performance2-90": 4.08,
+ "performance2-120": 5.44
},
"serverlove": {
"extra-large": 0.615,
@@ -724,10 +838,10 @@
"4": 0.045,
"5": 0.045,
"6": 0.045,
- "7": 0.090,
- "8": 0.090,
- "9": 0.090,
- "10": 0.090,
+ "7": 0.09,
+ "8": 0.09,
+ "9": 0.09,
+ "10": 0.09,
"11": 0.205,
"12": 0.205
},
@@ -736,5 +850,5 @@
}
},
"storage": {},
- "updated": 1427829652
+ "updated": 1447082902
}
\ No newline at end of file