Mocking a subprocess call in Python

The testfixtures library (docs, github) can mock the subprocess package.

Here’s an example on using the mock subprocess.Popen:

from unittest import TestCase

from testfixtures.mock import call
from testfixtures import Replacer, ShouldRaise, compare
from testfixtures.popen import MockPopen, PopenBehaviour


class TestMyFunc(TestCase):
    def setUp(self):
        self.Popen = MockPopen()
        self.r = Replacer()
        self.r.replace(dotted_path, self.Popen)
        self.addCleanup(self.r.restore)

    def test_example(self):
        # set up
        self.Popen.set_command("svn ls -R foo", stdout=b"o", stderr=b"e")

        # testing of results
        compare(my_func(), b"o")

        # testing calls were in the right order and with the correct parameters:
        process = call.Popen(["svn", "ls", "-R", "foo"], stderr=PIPE, stdout=PIPE)
        compare(Popen.all_calls, expected=[process, process.communicate()])

    def test_example_bad_returncode(self):
        # set up
        Popen.set_command("svn ls -R foo", stdout=b"o", stderr=b"e", returncode=1)

        # testing of error
        with ShouldRaise(RuntimeError("something bad happened")):
            my_func()

There is no need for anything complex. You can simply move the call to subprocess to a function and pass a mock function in your test.

import subprocess
import unittest
import logging

from unittest.mock import Mock


def _exec(cmd):
    p = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    stdout, stderr = p.communicate()
    return p, stdout, stderr


def uses_exec(execute=_exec):
    cmd = ['ls']
    p, stdout, stderr = execute(cmd)
    if p.returncode != 0:
        logging.error(
            '%s returned Error Code: %s',
            cmd,
            p.returncode
        )
        logging.error(stderr)
    else:
        logging.info(stdout)


class TestUsesExecute(unittest.TestCase):

    def test_get_aws_creds_from_namespace(self):
        p = lambda:None
        p.returncode = 0
        stdout = 'file1  file2'
        mock_execute = Mock(return_value=(p, stdout, ''))
        uses_exec(execute=mock_execute)
        mock_execute.assert_called_with(['ls'])


if __name__ == '__main__':
    unittest.main()

This pattern works in all programming languages that allow functions to be passed as parameters. It can be used with other python system calls for file and network reads just as easily.

I used this for a test suite where there were many subprocess.run calls to check.

import contextlib
import re
import subprocess
from typing import NamedTuple
from unittest.mock import MagicMock


class CmdMatch(NamedTuple):
    cmd: str
    match: str = ".*"
    result: str = ""
    side_effect: Callable = None


@contextlib.contextmanager
def mock_run(*cmd_match: Union[str, CmdMatch], **kws):
    sub_run = subprocess.run
    mock = MagicMock()
    if isinstance(cmd_match[0], str):
        cmd_match = [CmdMatch(*cmd_match, **kws)]

    def new_run(cmd, **_kws):
        check_cmd = " ".join(cmd[1:])
        mock(*cmd[1:])
        for m in cmd_match:
            if m.cmd in cmd[0].lower() and re.match(m.match, check_cmd):
                if m.side_effect:
                    m.side_effect()
                return subprocess.CompletedProcess(cmd, 0, m.result, "")
        assert False, "No matching call for %s" % check_cmd

    subprocess.run = new_run
    yield mock
    subprocess.run = sub_run

So now you can write a test like this:

def test_call_git():
    with mock_run("git", "describe.*", "v2.5") as cmd:
        do_something()
        cmd.assert_called_once()


def test_other_calls():
    with mock_run(
        CmdMatch("git", "describe.*", "v2.5"),
        CmdMatch("aws", "s3.*links.*", side_effect=subprocess.CalledProcessError),
    ) as cmd:
        do_something()

Couple funky things you might want to change, but I liked them:

• the resulting mock calls ignore the first argument (so which/full-paths aren’t part of the tests)
• if nothing matches any calls, it asserts (you have to have a match)
• the first positional is considered the "name of the command"
• kwargs to run are ignored (_call assertions are easy but loose)

If you want to check that the mocked object was called with a certain parameter, you can add the side_effect argument to the mock.patch decorator.

The return value of the side_effect function determines the return value of subprocess.Popen. If the side_effect_func returns DEFAULT, subprocess.Popen will be called in a normal way.

from unittest import mock, TestCase
from unittest.mock import DEFAULT
import subprocess


def run_script(script_path, my_arg):
    process = subprocess.Popen([script_path, my_arg])
    return process


def side_effect_func(*args, **kwargs):

    # Print the arguments
    print(args)
    
    # If 'bar' is contained within the arguments, return 'foo'
    if any(['bar' in arg for arg in args]):
        return 'foo'
    
    # If 'bar' is not contained within the arguments, run subprocess.Popen
    else:
        return DEFAULT


class TestRunScriptClass(TestCase):

    @mock.patch("subprocess.Popen", side_effect=side_effect_func)
    def test_run_script(self, mock):
        # Run the function
        process = run_script(script_path='my_script.py', my_arg='bar')
        
        # Assert if the mock object was called
        self.assertTrue(mock.called)
        
        # Assert if the mock object returned 'foo' when providing 'bar'
        self.assertEqual(process, 'foo')