You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by an...@apache.org on 2019/09/04 16:32:06 UTC
[metron] branch master updated: METRON-2221 Notebook import fails
through Zeppelin REST API (anandsubbu) closes apache/metron#1489
This is an automated email from the ASF dual-hosted git repository.
anandsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 5eed666 METRON-2221 Notebook import fails through Zeppelin REST API (anandsubbu) closes apache/metron#1489
5eed666 is described below
commit 5eed6663f2689e160be0a525c8e423961cd2269f
Author: anandsubbu <an...@gmail.com>
AuthorDate: Wed Sep 4 22:01:45 2019 +0530
METRON-2221 Notebook import fails through Zeppelin REST API (anandsubbu) closes apache/metron#1489
---
.../CURRENT/package/scripts/indexing_commands.py | 71 +++++++++++-----------
.../CURRENT/package/scripts/indexing_master.py | 17 +++---
2 files changed, 44 insertions(+), 44 deletions(-)
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 69e8dff..87397f4 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -18,7 +18,6 @@ limitations under the License.
import os
import re
-import requests
import time
import metron_security
@@ -28,6 +27,8 @@ from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute
from resource_management.libraries.functions import format as ambari_format
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.get_user_call_output import \
+ get_user_call_output
# Wrap major operations and functionality in this class
@@ -435,47 +436,43 @@ class IndexingCommands:
Logger.info("Indexing service check completed successfully")
- def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env):
+ def get_zeppelin_auth_details(self, zeppelin_server_url, env):
"""
- With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password
- for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details).
+ With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication by default, which requires
+ user/password for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html
+ for details).
+
+ This method uses the Shiro authentication credentials on the Zeppelin server to authenticate and returns the
+ cookie information to be used for importing Zeppelin notebooks.
- This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the
- session connection details to be used for importing Zeppelin notebooks.
- :param ses: Session handle
:param zeppelin_server_url: Zeppelin Server URL
- :return: ses
+ :return: session_id - the cookie handle to be used for subsequent interaction
"""
from params import params
env.set_params(params)
- # Check if authentication is enabled on the Zeppelin server
+ session_id = None
try:
- ses.get(ambari_format('http://{zeppelin_server_url}/api/login'))
-
- # Establish connection if authentication is enabled
- try:
- Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.")
- # Read the Shiro admin user credentials from Zeppelin config in Ambari
- seen_users = False
- username = None
- password = None
- if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE):
- seen_users = True
- tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group()
- userpassword = tokens.split(',')[0].strip()
- username = userpassword.split('=')[0].strip()
- password = userpassword.split('=')[1].strip()
- else:
- Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.")
-
- zeppelin_payload = {'userName': username, 'password' : password}
- ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload)
- except:
- pass
-
- # If authentication is not enabled, fall back to default method of imporing notebooks
- except requests.exceptions.RequestException:
- ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook'))
-
- return ses
+ Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.")
+ # Read the Shiro admin user credentials from Zeppelin config in Ambari
+ username = None
+ password = None
+ if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE):
+ tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group()
+ userpassword = tokens.split(',')[0].strip()
+ username = userpassword.split('=')[0].strip()
+ password = userpassword.split('=')[1].strip()
+ else:
+ Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.")
+
+ zeppelin_creds = "userName=%s&password=%s" % (username, password)
+ cmd = 'curl -i --data \'{0}\' -X POST \"http://{1}/api/login\" | grep JSESSIONID | grep -v deleteMe | tail -1'
+ cmd = cmd.format(zeppelin_creds, params.zeppelin_server_url)
+ return_code, stdout, stderr = get_user_call_output(cmd, user=params.metron_user)
+ session_id = stdout.replace("Set-Cookie: ",'').strip()
+
+ except Exception as e1:
+ msg = "Unable to get Shiro authentication details: Error={0}"
+ Logger.error(msg.format(e1))
+
+ return session_id
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 87064bc..57423ca 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -17,7 +17,6 @@ limitations under the License.
import os
import errno
-import requests
import metron_service
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -32,6 +31,8 @@ from resource_management.core.source import Template
from resource_management.libraries.functions import format as ambari_format
from resource_management.libraries.functions.format import format
from resource_management.libraries.script import Script
+from resource_management.libraries.functions.get_user_call_output import \
+ get_user_call_output
class Indexing(Script):
@@ -228,16 +229,18 @@ class Indexing(Script):
Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
# Check if authentication is configured on Zeppelin server, and fetch details if enabled.
- ses = requests.session()
- ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env)
+ session_id = commands.get_zeppelin_auth_details(params.zeppelin_server_url, env)
for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
for fileName in files:
if fileName.endswith(".json"):
Logger.info("Importing notebook: " + fileName)
- zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import')
- zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')}
- res = ses.post(zeppelin_import_url, files=zeppelin_notebook)
- Logger.info("Result: " + res.text)
+ zeppelin_notebook = os.path.join(dirName, fileName)
+ zeppelin_import_url = 'curl -i -b \"{0}\" http://{1}/api/notebook/import -d @\'{2}\''
+ zeppelin_import_url = zeppelin_import_url.format(session_id, params.zeppelin_server_url, zeppelin_notebook)
+ return_code, import_result, stderr = get_user_call_output(zeppelin_import_url, user=params.metron_user)
+ Logger.info("Status of importing notebook: " + import_result)
+ if return_code != 0:
+ Logger.error("Error importing notebook: " + fileName + " Error Message: " + stderr)
if __name__ == "__main__":
Indexing().execute()