I am attempting to connect to an Amazon EMR cluster using Livy 0.7 and Spark from an Amazon Sagemaker Notebook running Amazon Linux 2. Can anyone help me understand this error and how I might go about fixing it?
When I go to run the following command on a SageMaker Notebook running Amazon Linux 2:
%reload_ext sparkmagic.magics %spark add -s sparksession -l python -u $LIVY_ENDPOINT -t None
This results in the following error message:
AttributeError Traceback (most recent call last) in 1 get_ipython().run_line_magic('reload_ext', 'sparkmagic.magics') 2 get_ipython().run_line_magic('spark', 'add -s jsmith -l python -u $LIVY_ENDPOINT -t None')
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/interactiveshell.py in run_line_magic(self, magic_name, line, _stack_depth) 2415 kwargs['local_ns'] = self.get_local_scope(stack_depth) 2416 with self.builtin_trap: 2417 result = fn(*args, **kwargs) 2418 return result 2419
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/decorator.py in fun(*args, **kw) 230 if not kwsyntax: 231 args, kw = fix(args, kw, sig) 232 return caller(func, *(extras + args), **kw) 233 fun.name = func.name 234 fun.doc = func.doc
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/magic.py in (f, *a, **k) 185 # but it's overkill for just that one bit of state. 186 def magic_deco(arg): 187 call = lambda f, *a, **k: f(*a, **k) 188 189 if callable(arg):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/magics/extension.py in reload_ext(self, module_str) 61 if not module_str: 62 raise UsageError('Missing module name.') 63 self.shell.extension_manager.reload_extension(module_str)
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/extensions.py in reload_extension(self, module_str) 128 self.loaded.add(module_str) 129 else: 130 self.load_extension(module_str) 131 132 def _call_load_ipython_extension(self, mod):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/extensions.py in load_extension(self, module_str) 85 dir=compress_user(self.ipython_extension_dir))) 86 mod = sys.modules[module_str] 87 if self._call_load_ipython_extension(mod): 88 self.loaded.add(module_str) 89 else:
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/extensions.py in _call_load_ipython_extension(self, mod) 132 def _call_load_ipython_extension(self, mod): 133 if hasattr(mod, 'load_ipython_extension'): 134 mod.load_ipython_extension(self.shell) 135 return True 136
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/magics/remotesparkmagics.py in load_ipython_extension(ip) 322 323 def load_ipython_extension(ip): 324 ip.register_magics(RemoteSparkMagics)
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/IPython/core/magic.py in register(self, *magic_objects) 449 if isinstance(m, type): 450 # If we're given an uninstantiated class 451 m = m(shell=self.shell) 452 453 # Now that we have an instance, we can register it and update the
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/magics/remotesparkmagics.py in init(self, shell, data, widget) 40 if widget is None: 41 widget = MagicsControllerWidget( 42 self.spark_controller, IpyWidgetFactory(), self.ipython_display 43 ) 44 self.manage_widget = widget
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/controllerwidget/magicscontrollerwidget.py in init(self, spark_controller, ipywidget_factory, ipython_display, endpoints) 26 self.endpoints = endpoints 27 28 self._refresh() 29 30 def run(self):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/controllerwidget/magicscontrollerwidget.py in _refresh(self) 85 self.endpoints, 86 self.endpoints_dropdown_widget, 87 self._refresh, 88 ) 89 self.manage_endpoint = ManageEndpointWidget(
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/controllerwidget/addendpointwidget.py in init(self, spark_controller, ipywidget_factory, ipython_display, endpoints, endpoints_dropdown_widget, refresh_method) 32 events_handler_module = importlib.import_module(module) 33 auth_class = getattr(events_handler_module, class_name) 34 self.auth_instances[auth] = auth_class() 35 36 self.auth_type = self.ipywidget_factory.get_dropdown(
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/auth/basic.py in init(self, parsed_attributes) 33 self.password = "password" 34 HTTPBasicAuth.init(self, self.username, self.password) 35 Authenticator.init(self, parsed_attributes) 36 37 def get_widgets(self, widget_width):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/auth/customauth.py in init(self, parsed_attributes) 22 else: 23 self.url = "http://example.com/livy" 24 self.widgets = self.get_widgets(WIDGET_WIDTH) 25 26 def get_widgets(self, widget_width):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/sparkmagic/auth/basic.py in get_widgets(self, widget_width) 50 ) 51 52 self.password_widget = ipywidget_factory.get_password( 53 description="Password:", value=self.password, width=widget_width 54 )
AttributeError: 'IpyWidgetFactory' object has no attribute 'get_password'
