I'm new one in Dagster. Could you help me, please? I want to understand how to set up an etl process error notification through a telegram bot
My code:
import pygsheets
import os
import telegram
from dagster import resource
from typing import Callable, Optional
from dagster._core.definitions import success_hook
from dagster._core.execution.context.hook import HookContext
create a resource:
@resource
def send_message(message):
botid = os.environ['telegram_bot']
chat = os.environ['telegram_chat']
bot = telegram.Bot(token=botid)
bot.sendMessage(chat_id=chat, text=message, parse_mode='HTML')
create ops:
@op (required_resource_keys={"telegram"})
def load_df(merged_df):
# путь до credentials
PATH = os.getenv(PATH_TO_FILE)
gc = pygsheets.authorize(service_file=PATH)
gc = gc.open_by_key(os.getenv('OPEN_BY_KEY'))
sheet = gc.worksheet('title', 'Список для ОЦЕНКИ NEW')
sheet.set_dataframe(merged_df, start=(1,1))
@op(required_resource_keys={'telegram'})
def telegram_op(context):
context.resources.telegram.send_message(text=':wave: hey there!')
create a job:
@job (resource_defs={"telegram": send_message}, hooks={telegram_on_success}, op_retry_policy = default_policy)
def job_text_for_pictures():
load_df(merged_df)
create a hook:
def _default_status_message(context: HookContext, status: str) -> str:
return "Op {op_name} on job {pipeline_name} {status}!\nRun ID: {run_id}".format(
op_name=context.op.name,
pipeline_name=context.pipeline_name,
run_id=context.run_id,
status=status,
)
def _default_success_message(context: HookContext) -> str:
return _default_status_message(context, status="succeeded")
def telegram_on_success(
message_fn: Callable[[HookContext], str] = _default_success_message,
dagit_base_url: Optional[str] = "http://localhost:3000",
):
@success_hook(required_resource_keys={"telegram"})
def _hook(context: HookContext):
text = message_fn(context)
if dagit_base_url:
text += "\n<{base_url}/instance/runs/{run_id}|View in Dagit>".format(
base_url=dagit_base_url, run_id=context.run_id
)
context.resources.telegram.send_message(text=text) # type: ignore
return _hook
And after that I get an error: dagster._core.errors.DagsterSubprocessError: During multiprocess execution errors occurred in child processes: In process 2984: dagster._core.errors.DagsterResourceFunctionError: Error executing resource_fn on ResourceDefinition telegram
