I'm trying to have a progress bar for each json I translate but I'm getting random duplication errors using tqdm. I'm guessing race conditions but I can't figure out why.
main():
# Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for filename in os.listdir("files"):
if filename.endswith('json'):
executor.submit(handle, filename)
handle():
def handle(filename):
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile:
with open('files/' + filename, 'r', encoding='UTF-8') as f:
data = json.load(f)
# Start Timer
start = time.time()
# Start Translation
translatedData = parseMap(data, filename)
end = time.time()
json.dump(translatedData[0], outFile, ensure_ascii=False)
parseMap() (Problem is probably here somewhere)
def parseMap(data, filename):
totalTokens = 0
totalLines = 0
events = data['events']
# Get total for progress bar
for event in events:
if event is not None:
for page in event['pages']:
totalLines += len(page['list'])
with tqdm(total = totalLines, leave=False, desc=filename, bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}', position=0) as pbar:
for event in events:
if event is not None:
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for page in event['pages']:
future = executor.submit(searchCodes, page, pbar)
# Verify if an exception was thrown
try:
totalTokens += future.result()
except Exception as e:
return [data, totalTokens, e]
searchCodes():
try:
for i in range(len(page['list'])):
pbar.update(1)
# Translating Code: 401
if page['list'][i]['code'] == 401:
currentGroup.append(page['list'][i]['parameters'][0])
while (page['list'][i+1]['code'] == 401):
del page['list'][i]
currentGroup.append(page['list'][i]['parameters'][0])
blah blah
return tokens
This is what I'm getting. Unsure whats causing the issue.
files/Map011.json: [0 Tokens/$0.0000][0.1s] ✓
files/Map008.json: [0 Tokens/$0.0000][0.1s] ✓
files/Map020.json: [0 Tokens/$0.0000][0.1s] ✓
files/Map002.json: [0 Tokens/$0.0000][0.2s] ✓
files/Map001.json: [0 Tokens/$0.0000][0.2s] ✓
files/Map013.json: [0 Tokens/$0.0000][0.2s] ✓
Map023.json: 8%|▊ | 140/1698 [00:00<00:01, 1331.62it/s] files/Map004.json: [0 Tokens/$0.0000][0.3s] ✓
Map017.json: 40%|████ | 17/42 [00:00<00:00, 73.99it/s] files/Map003.json: [0 Tokens/$0.0000][0.3s] ✓
files/Map019.json: [0 Tokens/$0.0000][0.3s] ✓
Map015.json: 75%|███████▍ | 1252/1675 [00:00<00:00, 4364.80it/s] files/Map014.json: [0 Tokens/$0.0000][0.3s] ✓
Map026.json: 43%|████▎ | 136/313 [00:00<00:00, 1307.69it/s] files/Map023.json: [0 Tokens/$0.0000][0.2s] ✓
Map028.json: 7%|▋ | 15/212 [00:00<00:01, 146.15it/s] files/Map006.json: [0 Tokens/$0.0000][0.4s] ✓
Map030.json: 63%|██████▎ | 186/296 [00:00<00:00, 1509.57it/s] files/Map018.json: [0 Tokens/$0.0000][0.4s] ✓
Map017.json: 100%|██████████| 42/42 [00:00<00:00, 89.30it/s] files/Map005.json: [0 Tokens/$0.0000][0.5s] ✓
Map012.json: 73%|███████▎ | 1304/1779 [00:00<00:00, 3480.51it/s]
Map024.json: 9%|▊ | 37/431 [00:00<00:03, 100.03it/s] files/Map017.json: [0 Tokens/$0.0000][0.5s] ✓
Map016.json: 1%|▏ | 89/7080 [00:00<00:38, 181.16it/s]
Map025.json: 14%|█▍ | 148/1052 [00:00<00:02, 377.28it/s] files/Map027.json: [0 Tokens/$0.0000][0.4s] ✓
Map021.json: 0%| | 43/10984 [00:00<02:14, 81.05it/s] fMap036.json: 7%|▋ | 71/951 [00:00<00:01, 604.80it/s]
files/Map030.json: [0 Tokens/$0.0000][0.4s] ✓
files/Map038.json: [0 Tokens/$0.0000][0.1s] ✓
Map024.json: 21%|██ | 91/431 [00:00<00:01, 180.67it/s] files/Map040.json: [0 Tokens/$0.0000][0.0s] ✓
files/Map032.json: [0 Tokens/$0.0000][0.4s] ✓
Map037.json: 24%|██▎ | 92/390 [00:00<00:00, 472.02it/s] files/Map026.json: [0 Tokens/$0.0000][0.6s] ✓
iles/Map021.json: [0 Tokens/$0.0000][0.9s] ✓
Map031.json: 88%|████████▊ | 580/659 [00:00<00:00, 655.48it/s] files/Map010.json: [0 Tokens/$0.0000][1.2s] ✓
files/Map033.json: [0 Tokens/$0.0000][0.8s] ✓
files/Map031.json: [0 Tokens/$0.0000][1.0s] ✓
files/Map035.json: [0 Tokens/$0.0000][0.9s] ✓
files/Map024.json: [0 Tokens/$0.0000][1.3s] ✓
files/Map037.json: [0 Tokens/$0.0000][0.9s] ✓
files/Map036.json: [0 Tokens/$0.0000][1.0s] ✓
Map029.json: 23%|██▎ | 609/2617 [00:01<00:03, 662.47it/s] files/Map022.json: [0 Tokens/$0.0000][1.5s] ✓
files/Map016.json: [0 Tokens/$0.0000][1.5s] ✓
files/Map029.json: [0 Tokens/$0.0000][1.4s] ✓
