I have an mlflow server running locally and being exposed at port 80. I also have a model in the mlflow registry and I want to deploy it using the mlflow sagemaker run-local
because after testing this locally, I am going to deploy everything to AWS and Sagemaker. My problem is that when I run:
export MODEL_PATH=models:/churn-lgb-test/2
export LOCAL_PORT=8000
mlflow sagemaker run-local -m $MODEL_PATH -p $LOCAL_PORT -f python_function -i splicemachine/mlflow-pyfunc:1.6.0
it starts the container and I immediately get this error:
2020-07-27 13:02:13 +0000] [827] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/ggevent.py", line 162, in init_process
super().init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/miniconda/envs/custom_env/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/models/container/scoring_server/wsgi.py", line 3, in <module>
app = scoring_server.init(pyfunc.load_model("/opt/ml/model/"))
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 292, in load_model
return importlib.import_module(conf[MAIN])._load_pyfunc(data_path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 219, in _load_pyfunc
return _load_model_from_local_file(path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 206, in _load_model_from_local_file
with open(path, "rb") as f:
IsADirectoryError: [Errno 21] Is a directory: '/opt/ml/model'
[2020-07-27 13:02:13 +0000] [828] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/ggevent.py", line 162, in init_process
super().init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/miniconda/envs/custom_env/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/models/container/scoring_server/wsgi.py", line 3, in <module>
app = scoring_server.init(pyfunc.load_model("/opt/ml/model/"))
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 292, in load_model
return importlib.import_module(conf[MAIN])._load_pyfunc(data_path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 219, in _load_pyfunc
return _load_model_from_local_file(path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 206, in _load_model_from_local_file
with open(path, "rb") as f:
IsADirectoryError: [Errno 21] Is a directory: '/opt/ml/model'
[2020-07-27 13:02:13 +0000] [828] [INFO] Worker exiting (pid: 828)
[2020-07-27 13:02:13 +0000] [827] [INFO] Worker exiting (pid: 827)
[2020-07-27 13:02:13 +0000] [829] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/ggevent.py", line 162, in init_process
super().init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/miniconda/envs/custom_env/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/models/container/scoring_server/wsgi.py", line 3, in <module>
app = scoring_server.init(pyfunc.load_model("/opt/ml/model/"))
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 292, in load_model
return importlib.import_module(conf[MAIN])._load_pyfunc(data_path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 219, in _load_pyfunc
return _load_model_from_local_file(path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 206, in _load_model_from_local_file
with open(path, "rb") as f:
IsADirectoryError: [Errno 21] Is a directory: '/opt/ml/model'
[2020-07-27 13:02:13 +0000] [829] [INFO] Worker exiting (pid: 829)
[2020-07-27 13:02:13 +0000] [830] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/ggevent.py", line 162, in init_process
super().init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/miniconda/envs/custom_env/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/models/container/scoring_server/wsgi.py", line 3, in <module>
app = scoring_server.init(pyfunc.load_model("/opt/ml/model/"))
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 292, in load_model
return importlib.import_module(conf[MAIN])._load_pyfunc(data_path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 219, in _load_pyfunc
return _load_model_from_local_file(path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 206, in _load_model_from_local_file
with open(path, "rb") as f:
IsADirectoryError: [Errno 21] Is a directory: '/opt/ml/model'
[2020-07-27 13:02:13 +0000] [830] [INFO] Worker exiting (pid: 830)
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 209, in run
self.sleep()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 357, in sleep
ready = select.select([self.PIPE[0]], [], [], 1.0)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 242, in handle_chld
self.reap_workers()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 525, in reap_workers
raise HaltServer(reason, self.WORKER_BOOT_ERROR)
gunicorn.errors.HaltServer: <HaltServer 'Worker failed to boot.' 3>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/miniconda/envs/custom_env/bin/gunicorn", line 8, in <module>
sys.exit(run())
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 58, in run
WSGIApplication("%(prog)s [OPTIONS] [APP_MODULE]").run()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 228, in run
super().run()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 72, in run
Arbiter(self).run()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 229, in run
self.halt(reason=inst.reason, exit_status=inst.exit_status)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 342, in halt
self.stop()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 393, in stop
time.sleep(0.1)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 242, in handle_chld
self.reap_workers()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 525, in reap_workers
raise HaltServer(reason, self.WORKER_BOOT_ERROR)
gunicorn.errors.HaltServer: <HaltServer 'Worker failed to boot.' 3>
creating and activating custom environment
Got sigterm signal, exiting.
[2020-07-27 13:02:13 +0000] [831] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/ggevent.py", line 162, in init_process
super().init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/miniconda/envs/custom_env/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/models/container/scoring_server/wsgi.py", line 3, in <module>
app = scoring_server.init(pyfunc.load_model("/opt/ml/model/"))
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 292, in load_model
return importlib.import_module(conf[MAIN])._load_pyfunc(data_path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 219, in _load_pyfunc
return _load_model_from_local_file(path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 206, in _load_model_from_local_file
with open(path, "rb") as f:
IsADirectoryError: [Errno 21] Is a directory: '/opt/ml/model'
[2020-07-27 13:02:13 +0000] [831] [INFO] Worker exiting (pid: 831)
[2020-07-27 13:02:14 +0000] [833] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/ggevent.py", line 162, in init_process
super().init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/miniconda/envs/custom_env/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/models/container/scoring_server/wsgi.py", line 3, in <module>
app = scoring_server.init(pyfunc.load_model("/opt/ml/model/"))
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 292, in load_model
return importlib.import_module(conf[MAIN])._load_pyfunc(data_path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 219, in _load_pyfunc
return _load_model_from_local_file(path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 206, in _load_model_from_local_file
with open(path, "rb") as f:
IsADirectoryError: [Errno 21] Is a directory: '/opt/ml/model'
[2020-07-27 13:02:14 +0000] [833] [INFO] Worker exiting (pid: 833)
[2020-07-27 13:02:14 +0000] [832] [ERROR] Exception in worker process
Traceback (most recent call last):
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
worker.init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/ggevent.py", line 162, in init_process
super().init_process()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 119, in init_process
self.load_wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/workers/base.py", line 144, in load_wsgi
self.wsgi = self.app.wsgi()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/base.py", line 67, in wsgi
self.callable = self.load()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 49, in load
return self.load_wsgiapp()
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 39, in load_wsgiapp
return util.import_app(self.app_uri)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/gunicorn/util.py", line 358, in import_app
mod = importlib.import_module(module)
File "/miniconda/envs/custom_env/lib/python3.7/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
File "<frozen importlib._bootstrap>", line 983, in _find_and_load
File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 728, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/models/container/scoring_server/wsgi.py", line 3, in <module>
app = scoring_server.init(pyfunc.load_model("/opt/ml/model/"))
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 292, in load_model
return importlib.import_module(conf[MAIN])._load_pyfunc(data_path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 219, in _load_pyfunc
return _load_model_from_local_file(path)
File "/miniconda/envs/custom_env/lib/python3.7/site-packages/mlflow/sklearn.py", line 206, in _load_model_from_local_file
with open(path, "rb") as f:
IsADirectoryError: [Errno 21] Is a directory: '/opt/ml/model'
[2020-07-27 13:02:14 +0000] [832] [INFO] Worker exiting (pid: 832)