I have multiple queries I want to define as individual Luigi tasks depending on each other. Since the dependencies (sub-tasks) of BDX_Query_0XX are dynamically defined based on a dictionary (cmdList), I'm using "yield [Task_Class (klass)]" statement. For example, BDX_Query_0XX has 3 dependencies, BDX010, BDX020, BDX030 where BDX010 is the dependency of BDX020, and BDX020 is the dependency of BDX030.
The issue is that when BDX020 is yielded and Python looks for BDX010 as the dependency, Python complains BDX010 is not defined.
Things work fine if I use "yield BDX_Task" instead of "yield klass", but it makes the Luigi visualizer to show all dynamically created tasks as "BDX_Task", instead of showing them as BDX010, ..020, ..030.
cmdList = {
'BDX010': (f'"{bdx_sql}BDX_001_NI_DM 010.sql" -S LWVPDBSQLC070 ',''),
'BDX020': (f'"{bdx_sql}BDX_001_NI_DM 020.sql" ','BDX010'),
'BDX030': (f'"{bdx_sql}BDX_001_NI_DM 030.sql" ','BDX020') }
class BDX_Task(SQLTask):
acctDate = luigi.Parameter()
ssisDate = luigi.Parameter(default=None)
queryKey = luigi.Parameter()
queryCmd = luigi.Parameter()
runDesc = luigi.Parameter()
dependQry = luigi.Parameter()
def __init__(self, *args, **kwargs):
super(BDX_Task, self).__init__(*args, **kwargs)
logger.debug(f'BDX_Task.__init__ called for queryKey ="{self.queryKey}"')
self.trans_id = f"00903_BDX_Query_{self.queryKey}__{self.runDesc}"
def requires(self):
cmdListComb = dict(cmdList)
cmdListComb.update(cmdList2)
if self.dependQry != '' and self.dependQry in cmdListComb:
dep_cmd, dep_dep_key = cmdListComb[self.dependQry]
return [self.__class__(
acctDate = self.acctDate,
ssisDate = self.ssisDate,
queryKey = self.dependQry,
queryCmd = dep_cmd,
runDesc = self.runDesc,
dependQry = dep_dep_key
)]
else:
return []
def run(self):
strQuery_and_args = f""" -i {self.queryCmd} """
print(strQuery_and_args)
helpers.call_sqlcmd(self.queryKey,strQuery_and_args )
self.get_target().touch()
class BDX_Query_0XX(SQLTask):
acctDate = luigi.Parameter()
ssisDate = luigi.Parameter()
runDesc = luigi.Parameter()
def __init__(self, *args, **kwargs):
super(BDX_Query_0XX, self).__init__(*args, **kwargs)
self.trans_id = "00902_BDX_Query_0XX" + "__" + self.runDesc # static.
def requires(self):
for queryKey, (queryCmd, dependQry) in cmdList.items():
klass = type(queryKey, (BDX_Task,),{})
# I can avoid the error, if I do "yield BDX_Task" below instead.
# but it causes Luigi Visualizer to show all BDX0?X tasks as "BDX_Task".
yield klass(
acctDate = self.acctDate,
ssisDate = self.ssisDate,
queryKey = queryKey,
queryCmd = queryCmd,
runDesc = self.runDesc,
dependQry = dependQry
)
def run(self):
self.get_target().touch()