You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kibble.apache.org by tu...@apache.org on 2020/11/02 18:29:18 UTC
[kibble] branch main updated: Use KibbleConfigParser in setup
script (#83)
This is an automated email from the ASF dual-hosted git repository.
turbaszek pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/kibble.git
The following commit(s) were added to refs/heads/main by this push:
new 6959f3c Use KibbleConfigParser in setup script (#83)
6959f3c is described below
commit 6959f3c51a594941abc08face115cf3057aaed88
Author: Tomek Urbaszek <tu...@gmail.com>
AuthorDate: Mon Nov 2 19:21:44 2020 +0100
Use KibbleConfigParser in setup script (#83)
* Use KibbleConfigParser in setup script
* fixup! Use KibbleConfigParser in setup script
---
docker-compose-dev.yaml | 2 +-
kibble.ini | 26 +++++
kibble/configuration.py | 20 ++--
kibble/setup/kibble.ini | 16 ---
kibble/setup/setup.py | 111 ++++-----------------
setup.py | 2 +-
kibble/configuration.py => tests/__init__.py | 19 ----
.../test_configuration.py | 36 ++++---
8 files changed, 81 insertions(+), 151 deletions(-)
diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml
index 55c81c6..7744b99 100644
--- a/docker-compose-dev.yaml
+++ b/docker-compose-dev.yaml
@@ -7,7 +7,7 @@ services:
build:
context: .
dockerfile: Dockerfile.dev
- command: bash -c "python kibble/setup/setup.py -e elasticsearch -a -k"
+ command: bash -c "python kibble/setup/setup.py --autoadmin --skiponexist"
volumes:
- .:/kibble/
depends_on:
diff --git a/kibble.ini b/kibble.ini
new file mode 100644
index 0000000..0089f4d
--- /dev/null
+++ b/kibble.ini
@@ -0,0 +1,26 @@
+[accounts]
+allowSignup = True
+verify = True
+
+[api]
+# Kibble elasticsearch database revision
+database = 2
+# Version f the API
+version = 0.1.0
+
+[elasticsearch]
+# Elasticsearch database name
+dbname = kibble
+# Connection uri used to determine host and port of elasticsearch instance
+conn_uri = elasticsearch:9200
+# Number of shards in es cluster
+shards = 5
+# Number of replicase in es cluster
+replicas = 1
+ssl = False
+uri =
+auth =
+
+[mail]
+mailhost = localhost:25
+sender = Kibble <no...@kibble.kibble>
diff --git a/kibble/configuration.py b/kibble/configuration.py
index 9f8ca99..26619d0 100644
--- a/kibble/configuration.py
+++ b/kibble/configuration.py
@@ -14,22 +14,20 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+import os
from configparser import ConfigParser
+DEFAULT_KIBBLE_CONFIG_LOCATION = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)), os.pardir, "kibble.ini"
+)
+
class KibbleConfigParser(ConfigParser):
+ """Custom Kibble config parser"""
+
def __init__(self):
super().__init__()
- def get_int(self, section: str, key: str) -> int:
- try:
- return int(self.get(section, key))
- except Exception:
- raise TypeError("Unable to convert value to int")
- def get_bool(self, section: str, key: str) -> bool:
- try:
- return bool(self.get(section, key))
- except Exception:
- raise TypeError("Unable to convert value to bool")
+conf = KibbleConfigParser()
+conf.read(DEFAULT_KIBBLE_CONFIG_LOCATION)
diff --git a/kibble/setup/kibble.ini b/kibble/setup/kibble.ini
deleted file mode 100644
index e57c16d..0000000
--- a/kibble/setup/kibble.ini
+++ /dev/null
@@ -1,16 +0,0 @@
-[accounts]
-allowSignup = True
-verify = True
-
-[api]
-database = 2
-version = 0.1.0
-
-[elasticsearch]
-dbname = kibble
-conn_uri = elasticsearch:9200
-ss = False
-
-[mail]
-mailhost = localhost:25
-sender = Kibble <no...@kibble.kibble>
diff --git a/kibble/setup/setup.py b/kibble/setup/setup.py
index bded26d..79d67a9 100644
--- a/kibble/setup/setup.py
+++ b/kibble/setup/setup.py
@@ -22,15 +22,15 @@ import logging
from getpass import getpass
import tenacity
-import yaml
import bcrypt
import json
from elasticsearch import Elasticsearch
-from kibble.settings import KIBBLE_YAML
+from kibble.configuration import conf
-KIBBLE_VERSION = "0.1.0" # ABI/API compat demarcation.
-KIBBLE_DB_VERSION = 2 # Second database revision
+
+KIBBLE_VERSION = conf.get("api", "version")
+KIBBLE_DB_VERSION = conf.get("api", "database") # database revision
if sys.version_info <= (3, 3):
print("This script requires Python 3.4 or higher")
@@ -42,60 +42,53 @@ def get_parser():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
"-e",
- "--hostname",
- help="Pre-defined hostname for ElasticSearch (docker setups). Default: localhost",
- default="localhost",
- )
- arg_parser.add_argument(
- "-p",
- "--port",
- help="Pre-defined port for ES (docker setups). Default: 9200",
- default=9200,
+ "--conn-uri",
+ help="Pre-defined connection uri for ElasticSearch.",
+ default=conf.get("elasticsearch", "conn_uri"),
)
arg_parser.add_argument(
"-d",
"--dbname",
- help="Pre-defined Database prefix (docker setups). Default: kibble",
- default="kibble",
+ help="Pre-defined Database prefix. Default: kibble",
+ default=conf.get("elasticsearch", "dbname"),
)
arg_parser.add_argument(
"-s",
"--shards",
- help="Predefined number of ES shards (docker setups), Default: 5",
- default=5,
+ help="Predefined number of ES shards, Default: 5",
+ default=conf.get("elasticsearch", "shards"),
)
arg_parser.add_argument(
"-r",
"--replicas",
- help="Predefined number of replicas for ES (docker setups). Default: 1",
- default=1,
+ help="Predefined number of replicas for ES. Default: 1",
+ default=conf.get("elasticsearch", "replicas"),
)
arg_parser.add_argument(
"-m",
"--mailhost",
- help="Pre-defined mail server host (docker setups). Default: localhost:25",
- default="localhost:25",
+ help="Pre-defined mail server host. Default: localhost:25",
+ default=conf.get("mail", "mailhost"),
)
arg_parser.add_argument(
"-a",
"--autoadmin",
action="store_true",
- help="Generate generic admin account (docker setups). Default: False",
+ help="Generate generic admin account. Default: False",
default=False,
)
arg_parser.add_argument(
"-k",
"--skiponexist",
action="store_true",
- help="Skip DB creation if DBs exist (docker setups). Defaul: True",
+ help="Skip DB creation if DBs exist. Defaul: True",
default=True,
)
return arg_parser
def create_es_index(
- hostname: str,
- port: int,
+ conn_uri: str,
dbname: str,
shards: int,
replicas: int,
@@ -114,11 +107,7 @@ def create_es_index(
with open(mappings_json, "r") as f:
mappings = json.load(f)
- es = Elasticsearch(
- [{"host": hostname, "port": port, "use_ssl": False, "url_prefix": ""}],
- max_retries=5,
- retry_on_timeout=True,
- )
+ es = Elasticsearch([conn_uri], max_retries=5, retry_on_timeout=True)
es_version = es.info()["version"]["number"]
es6 = int(es_version.split(".")[0]) >= 6
@@ -223,51 +212,6 @@ def create_es_index(
print("Account created!")
-def get_kibble_yaml() -> str:
- """Resolve path to kibble config yaml"""
- kibble_yaml = KIBBLE_YAML
- if os.path.exists(kibble_yaml):
- print(f"{kibble_yaml} already exists! Writing to {kibble_yaml}.tmp instead")
- kibble_yaml = kibble_yaml + ".tmp"
- return kibble_yaml
-
-
-def save_config(mlserver: str, hostname: str, port: int, dbname: str):
- """Save kibble config to yaml file"""
- if ":" in mlserver:
- try:
- mailhost, mailport = mlserver.split(":")
- except ValueError:
- raise ValueError(
- "mailhost argument must be in form of `host:port` or `host`"
- )
- else:
- mailhost = mlserver
- mailport = 25
-
- config = {
- "api": {"version": KIBBLE_VERSION, "database": KIBBLE_DB_VERSION},
- "elasticsearch": {
- "host": hostname,
- "port": port,
- "ssl": False,
- "dbname": dbname,
- },
- "mail": {
- "mailhost": mailhost,
- "mailport": int(mailport),
- "sender": "Kibble <no...@kibble.kibble>",
- },
- "accounts": {"allowSignup": True, "verify": True},
- }
-
- kibble_yaml = get_kibble_yaml()
- print(f"Writing Kibble config to {kibble_yaml}")
- with open(kibble_yaml, "w") as f:
- f.write(yaml.dump(config, default_flow_style=False))
- f.close()
-
-
def get_user_input(msg: str, secure: bool = False):
value = None
while not value:
@@ -279,8 +223,7 @@ def print_configuration(args):
print(
"Configuring Apache Kibble elasticsearch instance with the following arguments:"
)
- print(f"- hostname: {args.hostname}")
- print(f"- port: {int(args.port)}")
+ print(f"- conn_uri: {args.conn_uri}")
print(f"- dbname: {args.dbname}")
print(f"- shards: {int(args.shards)}")
print(f"- replicas: {int(args.replicas)}")
@@ -311,7 +254,7 @@ def main():
# Create Elasticsearch index
# Retry in case ES is not yet up
- print(f"Elasticsearch: {args.hostname}:{args.port}")
+ print(f"Elasticsearch: {args.conn_uri}")
for attempt in tenacity.Retrying(
retry=tenacity.retry_if_exception_type(exception_types=Exception),
wait=tenacity.wait_fixed(10),
@@ -321,8 +264,7 @@ def main():
with attempt:
print("Trying to create ES index...")
create_es_index(
- hostname=args.hostname,
- port=int(args.port),
+ conn_uri=args.conn_uri,
dbname=args.dbname,
shards=int(args.shards),
replicas=int(args.replicas),
@@ -331,15 +273,6 @@ def main():
skiponexist=args.skiponexist,
)
print()
-
- # Create Kibble configuration file
- save_config(
- mlserver=args.mailhost,
- hostname=args.hostname,
- port=int(args.port),
- dbname=args.dbname,
- )
- print()
print("All done, Kibble should...work now :)")
diff --git a/setup.py b/setup.py
index abfe996..6c3de19 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ mod = util.module_from_spec(spec)
spec.loader.exec_module(mod) # type: ignore
version = mod.version # type: ignore
-DEVEL_REQUIREMENTS = ["pre-commit==2.7.1", "black==20.8b1"]
+DEVEL_REQUIREMENTS = ["black==20.8b1", "pre-commit==2.7.1", "pytest==6.1.1"]
INSTALL_REQUIREMENTS = [
"bcrypt==3.2.0",
diff --git a/kibble/configuration.py b/tests/__init__.py
similarity index 59%
copy from kibble/configuration.py
copy to tests/__init__.py
index 9f8ca99..13a8339 100644
--- a/kibble/configuration.py
+++ b/tests/__init__.py
@@ -14,22 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from configparser import ConfigParser
-
-
-class KibbleConfigParser(ConfigParser):
- def __init__(self):
- super().__init__()
-
- def get_int(self, section: str, key: str) -> int:
- try:
- return int(self.get(section, key))
- except Exception:
- raise TypeError("Unable to convert value to int")
-
- def get_bool(self, section: str, key: str) -> bool:
- try:
- return bool(self.get(section, key))
- except Exception:
- raise TypeError("Unable to convert value to bool")
diff --git a/kibble/configuration.py b/tests/test_configuration.py
similarity index 50%
copy from kibble/configuration.py
copy to tests/test_configuration.py
index 9f8ca99..32c13f1 100644
--- a/kibble/configuration.py
+++ b/tests/test_configuration.py
@@ -15,21 +15,29 @@
# specific language governing permissions and limitations
# under the License.
-from configparser import ConfigParser
+import pytest
+from kibble.configuration import conf
-class KibbleConfigParser(ConfigParser):
- def __init__(self):
- super().__init__()
- def get_int(self, section: str, key: str) -> int:
- try:
- return int(self.get(section, key))
- except Exception:
- raise TypeError("Unable to convert value to int")
+class TestDefaultConfig:
+ @pytest.mark.parametrize(
+ "section, key, value",
+ [
+ ("accounts", "allowSignup", True),
+ ("accounts", "verify", True),
+ ("api", "database", 2),
+ ("api", "version", "0.1.0"),
+ ("elasticsearch", "conn_uri", "elasticsearch:9200"),
+ ("mail", "mailhost", "localhost:25"),
+ ],
+ )
+ def test_default_values(self, section, key, value):
+ if isinstance(value, bool):
+ config_value = conf.getboolean(section, key)
+ elif isinstance(value, int):
+ config_value = conf.getint(section, key)
+ else:
+ config_value = conf.get(section, key)
- def get_bool(self, section: str, key: str) -> bool:
- try:
- return bool(self.get(section, key))
- except Exception:
- raise TypeError("Unable to convert value to bool")
+ assert config_value == value