subprocess collect output while external program is running

For this to work properly the external program might need to set the output to unbuffered. In Python by default prining to STDERR is unbuffered, but we had to pass flush=True to the print function to make it unbuffered for STDOUT as well.

examples/process/run_command_collect_while_running.py

import subprocess
import sys
import time

def run_process(command, timeout):
    print("Before Popen")
    proc = subprocess.Popen(command,
        stdout = subprocess.PIPE,
        stderr = subprocess.PIPE,
        universal_newlines=True,
        bufsize=0,
    )
    print("After Popen")
    out = ""
    err = ""

    while True:
        exit_code = proc.poll()
        print(f"poll: {exit_code} {time.time()}")
        this_out = proc.stdout.readline()
        this_err = proc.stderr.readline()
        print(f"out: {this_out}", end="")
        print(f"err: {this_err}", end="")
        out += this_out
        err += this_err
        time.sleep(0.5)  # here we could actually do something useful
        timeout -= 0.5
        if timeout <= 0:
            break
        if exit_code is not None:
            break

    print(f"Final: {exit_code}")
    if exit_code is None:
        raise Exception("Timeout")

    return exit_code, out, err

exit_code, out, err = run_process([sys.executable, 'process.py', '4', '3'], 20)
#exit_code, out, err = run_process(['docker-compose', 'up', '-d'], 20)

print("-----")
print(f"exit_code: {exit_code}")
print("OUT")
print(out)
print("ERR")
print(err)

Before Popen
After Popen
poll: None 1637589106.083494
out: OUT 0
err: ERR 0
poll: None 1637589106.6035957
out: OUT 1
err: ERR 1
poll: None 1637589107.6047328
out: OUT 2
err: ERR 2
poll: None 1637589108.6051855
out: OUT 3
err: ERR 3
poll: None 1637589109.6066446
out: err: poll: 0 1637589110.6227856
out: err: Final: 0
-----
exit_code: 0
OUT

ERR