I'm working on a tool for monitoring the jobs currently running on a cluster (19 nodes, 40 cores). Is there any way to determine which specific cpus each job in the slurm queue is using? I'm getting data using 'pidstat', 'mpstat', and 'ps -eFj', that tells me what processes are running on a particular core, but have no way to relate those process IDs to the Job IDs that Slurm uses. 'scontrol show job' gives a lot of information, but not specific cpu allocation. Is there any way to do this?
Heres the code that collects the data:
#!/usr/bin/env python
import subprocess
import threading
import time
def scan():
data = [[None, None, None] for i in range(19)]
def mpstat(node):
if(node == 1):
output = subprocess.check_output(['mpstat', '-P', 'ALL', '1', '1'])
else:
output = subprocess.check_output(['ssh', 'node' + str(node), 'mpstat', '-P', 'ALL', '1', '1'])
data[node - 1][0] = output
def pidstat(node):
if(node == 1):
output = subprocess.check_output(['pidstat', '1', '1'])
else:
output = subprocess.check_output(['ssh', 'node' + str(node), 'pidstat', '1', '1'])
data[node - 1][1] = output
def ps(node):
if(node == 1):
output = subprocess.check_output(['ps', '-eFj'])
else:
output = subprocess.check_output(['ssh', 'node' + str(node), 'ps', '-eFj'])
data[node - 1][2] = output
threads = [[None, None, None] for i in range(19)]
for node in range(1, 19 + 1):
threads[node - 1][0] = threading.Thread(target=mpstat, args=(node,))
threads[node - 1][0].start()
threads[node - 1][1] = threading.Thread(target=pidstat, args=(node,))
threads[node - 1][1].start()
threads[node - 1][2] = threading.Thread(target=ps, args=(node,))
threads[node - 1][2].start()
while True:
alive = [[not t.isAlive() for t in n] for n in threads]
alive = [t for n in alive for t in n]
if(all(alive)):
break
time.sleep(1.0)
return(data)
