Why do logging and futures not work together?

Question

I tried the following script but the log does not work.

import time  
import concurrent.futures
import logging
import os

######## Logging ################
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
file_handler = logging.FileHandler(os.path.join('logs','pipeline2.log'))
#In case we have also a StreamHandler with level of INFO
file_handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s:%(message)s')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)

def f(x):
    time.sleep(0.001)  # to visualize the progress
    logger.info(x**2)

def run(f, my_iter):
    """
    https://stackoverflow.com/questions/51601756/use-tqdm-with-concurrent-futures
    """
    import concurrent.futures
    my_iter = list(my_iter)
    l = len(my_iter)
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        futures = {executor.submit(f, arg): arg for arg in my_iter}
        results = {}
        for future in concurrent.futures.as_completed(futures):
            arg = futures[future]
            results[arg] = future.result()

my_iter = range(10000)
run(f, my_iter)

When running the script: I do not see any file pipeline2.log which appears in the logs. How can I solve this issue?

Edit: as suggested i retrieve tqdm (i do not think it can interfer)n in the code so it could be easyly reproductible.

If your question is about futures and logging, why does your example include tdqm? That precludes others from testing your code if they don't have it installed. See [mre]. — mkrieger1, Oct 23 '20 at 12:48
@mkrieger: i edited the question to explian what i mean .Moreover i wanted to test it with tqdm since process a lot of file (i put a toy fun and want to see the progress — amous, Oct 23 '20 at 12:50

score 0 · Answer 1 · answered Oct 23 '20 at 15:39

How do you run your code? I can think of the following situation the log does not work.

import time  
import concurrent.futures
import logging
import os
logger = logging.getLogger(__name__)
log_path = os.path.join('logs','pipeline2.log')

def add_file_handler(log_path):
    logger.setLevel(logging.INFO)
    file_handler = logging.FileHandler(log_path)
    file_handler.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s:%(message)s')
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    print(f"handler fileno {file_handler.stream.fileno()} added")

def remove_file(filepath):
    print(f"{filepath} exists: {os.path.exists(filepath)}")
    try:
        print(f"{filepath} remove: {os.remove(filepath)}")
    except FileNotFoundError:
        pass

def f(x):
    time.sleep(0.001)  # to visualize the progress
    logger.info(x**2)

def run(f, my_iter):
    """
    https://stackoverflow.com/questions/51601756/use-tqdm-with-concurrent-futures
    """
    import concurrent.futures
    my_iter = list(my_iter)
    l = len(my_iter)
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        futures = {executor.submit(f, arg): arg for arg in my_iter}
        results = {}
        for future in concurrent.futures.as_completed(futures):
            arg = futures[future]
            results[arg] = future.result()

run

remove_file(log_path)
add_file_handler(log_path)
remove_file(log_path)
my_iter = range(10)
run(f, my_iter)    

try:
    with open(log_path) as f:
        print(f.read())
except FileNotFoundError:
    print(f"No such file or directory {log_path}")

print

logs/pipeline2.log exists: True
logs/pipeline2.log remove: None
handler fileno 58 added
logs/pipeline2.log exists: True
logs/pipeline2.log remove: None
No such file or directory logs/pipeline2.log

then run

add_file_handler(log_path)
my_iter = range(10)
run(f, my_iter)    

for handler in logger.handlers:
    print(handler.stream.fileno())
    print(handler.stream.write(f"fileno: {handler.stream.fileno()}\n"))
    print(handler.stream.flush())
try:
    with open(log_path) as f:
        print(f.read())
except FileNotFoundError:
    print(f"No such file or directory {log_path}")

output

handler fileno 61 added
58
11
None
61
11
None
2020-10-23 23:31:27,149:INFO:__main__:0
2020-10-23 23:31:27,149:INFO:__main__:1
2020-10-23 23:31:27,157:INFO:__main__:36
2020-10-23 23:31:27,155:INFO:__main__:16
2020-10-23 23:31:27,155:INFO:__main__:25
2020-10-23 23:31:27,156:INFO:__main__:9
2020-10-23 23:31:27,150:INFO:__main__:4
2020-10-23 23:31:27,159:INFO:__main__:64
2020-10-23 23:31:27,159:INFO:__main__:49
2020-10-23 2

Why do logging and futures not work together?

1 Answers1