Create a configureable, multithreaded log file accumulator in python using SSH (paramiko?)

Question

I am about to code something and thought I'd put my idea out here first to see if anyone has comments, etc.

I want to create a class in python for simultaneously monitoring (and merging) multiple log files that will be used as part of an automated test. I am hoping to simply execute 'tail -f' (over SSH with paramiko, maybe) on each file using a separate thread. Then, every few seconds, get the stdout from each thread and merge it into one file with a suffix added to each line that identifies the source. This way I can write tests of distributed systems and monitor the logs of maybe a dozen machines at once (numerous of which have the same purpose and are behind a load balancer, etc)

Startup:
    for machine, logfile in config_list:
        create thread running tail -f on logfile on machine
    create accumulator thread that:
        wakes up each second and 
        gets all config_list stdout and merges it into one in-memory list

Test_API:
    method to get/query data from the in memory accumulator.  
    in memory list would be the only data item needed to be synchronized

So, I am wondering: is paramiko the correct choice? any caveats, etc about handling the threading (have never done anything with threading in python)? any additional ideas that come to mind?

thanks in advance!

feel free to post code snippets. i will update this post with a working solution once it is finished. i anticipate it will be pretty small

Just found this: Creating multiple SSH connections at a time using Paramiko

---

EDIT

From looking at a couple other posts, I have this so far. It is just doing a tail, not a tail -f and does not have the polling I need.

from someplace import TestLogger
import threading
import paramiko


def start_watching():

    logger = TestLogger().get()
    logs_to_watch = [('somemachine1', '/var/log/foo'),
                     ('somemachine2', '/var/log/bar')]

    threads = []
    for machine, filename in logs_to_watch:
        logger.info(machine)
        logger.info(filename)
        t = threading.Thread(target=workon, args=(machine, filename))
        t.start()
        threads.append(t)

    for t in threads:
        t.join()

    for merge_line in merged_log:
        logger.info(merge_line.dump())

outlock = threading.Lock()
merged_log = []

def workon(host, logfile):
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(host, username='yourusername', allow_agent=True, look_for_keys=True)
    stdin, stdout, stderr = ssh.exec_command('sudo tail ' + logfile)

    stdin.flush()

    with outlock:
        line = stdout.readline()
        while line:
            line = stdout.readline()
            merged_log.append(MergeLogLine(line, host, logfile))


class MergeLogLine():
    def __init__(self, line, host, logfile):
        self._line = line
        self._host = host
        self._logfile = logfile

    def line(self):
        return self._line

    def host(self):
        return self._host

    def logfile(self):
        return self._logfile

    def dump(self):
        return self._line + '(from host = ' + self._host + ', log = ' + self._logfile + ')'

Answer 1

This turned out to be pretty hard. Here is a working sample:

Sample 'client code':

import sys
import traceback
import tail_accumulate as ta
import time


def main(argv):

    user = 'cmead'
    logs_to_watch = [('somemachine1', '/var/log/bar/sample.log'),
                     ('somemachine2', '/var/log/foo')]

    tac = ta.TailAccumulateConfig(logs_to_watch, user)

    try:
        ta.start_watching(tac)

        time.sleep(10)

        for merge_line in ta.get_merged_log():
            print merge_line.dump()

    except Exception as e:
        print traceback.format_exc()

    ta.stop()


if __name__ == "__main__":
    main(sys.argv[1:])

Tail accumulate package:

import threading
import paramiko
import select
import time

threads = []
stopFlag = None


class TailAccumulateConfig():
    def __init__(self, log_list, user):
        self._log_list = log_list
        self._user = user

    def user(self):
        return self._user

    def log_list(self):
        return self._log_list


def start_watching(tail_accumulate_config):
    global stopFlag
    stopFlag = threading.Event()
    for machine, filename in tail_accumulate_config.log_list():
        t = LogListenWorker(stopFlag, machine, filename, tail_accumulate_config.user())
        t.start()
        global threads
        threads.append(t)


def stop():
    global stopFlag
    stopFlag.set()


def get_merged_log():
    with outlock:
        global merged_log
        temp = merged_log[:]
        del merged_log[:]
        return temp

outlock = threading.Lock()
merged_log = []


class LogListenWorker(threading.Thread):
    def __init__(self, event, host, logfile, username):
        threading.Thread.__init__(self)
        self.stopped = event
        self.host = host
        self.logfile = logfile
        self.username = username

    def run(self):
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(self.host, username=self.username)
        transport = ssh.get_transport()
        channel = transport.open_session()
        channel.exec_command('sudo tail -f ' + self.logfile)

        while not self.stopped.isSet():
            try:
                rl, wl, xl = select.select([channel],[],[],3.0)
                if len(rl) > 0:
                    # Must be stdout
                    line = channel.recv(1024)
                else:
                    time.sleep(1.0)
                    continue

            except Exception as e:
                break
            if line:
                with outlock:
                    sublines = line.split('\n')
                    for subline in sublines:
                        merged_log.append(MergeLogLine(subline, self.host, self.logfile))

        ssh.close()


class MergeLogLine():
    def __init__(self, line, host, logfile):
        self._line = line
        self._host = host
        self._logfile = logfile

    def line(self):
        return self._line

    def host(self):
        return self._host

    def logfile(self):
        return self._logfile

    def dump(self):
        return self._line + ' ---> (from host = ' + self._host + ', log = ' + self._logfile + ')'