How To catch python stdout in c++ code

Here is a C++ friendly solution I have developed lately.

I explain a few details of it on my blog: Python sys.stdout redirection in C++ where I also point to repository at my GitHub where most recent version can be found.
Here is complete example based on the current code at the time of posting this answer:

#include <functional>
#include <iostream>
#include <string>
#include <Python.h>

namespace emb
{

typedef std::function<void(std::string)> stdout_write_type;

struct Stdout
{
    PyObject_HEAD
    stdout_write_type write;
};

PyObject* Stdout_write(PyObject* self, PyObject* args)
{
    std::size_t written(0);
    Stdout* selfimpl = reinterpret_cast<Stdout*>(self);
    if (selfimpl->write)
    {
        char* data;
        if (!PyArg_ParseTuple(args, "s", &data))
            return 0;

        std::string str(data);
        selfimpl->write(str);
        written = str.size();
    }
    return PyLong_FromSize_t(written);
}

PyObject* Stdout_flush(PyObject* self, PyObject* args)
{
    // no-op
    return Py_BuildValue("");
}

PyMethodDef Stdout_methods[] =
{
    {"write", Stdout_write, METH_VARARGS, "sys.stdout.write"},
    {"flush", Stdout_flush, METH_VARARGS, "sys.stdout.flush"},
    {0, 0, 0, 0} // sentinel
};

PyTypeObject StdoutType =
{
    PyVarObject_HEAD_INIT(0, 0)
    "emb.StdoutType",     /* tp_name */
    sizeof(Stdout),       /* tp_basicsize */
    0,                    /* tp_itemsize */
    0,                    /* tp_dealloc */
    0,                    /* tp_print */
    0,                    /* tp_getattr */
    0,                    /* tp_setattr */
    0,                    /* tp_reserved */
    0,                    /* tp_repr */
    0,                    /* tp_as_number */
    0,                    /* tp_as_sequence */
    0,                    /* tp_as_mapping */
    0,                    /* tp_hash  */
    0,                    /* tp_call */
    0,                    /* tp_str */
    0,                    /* tp_getattro */
    0,                    /* tp_setattro */
    0,                    /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT,   /* tp_flags */
    "emb.Stdout objects", /* tp_doc */
    0,                    /* tp_traverse */
    0,                    /* tp_clear */
    0,                    /* tp_richcompare */
    0,                    /* tp_weaklistoffset */
    0,                    /* tp_iter */
    0,                    /* tp_iternext */
    Stdout_methods,       /* tp_methods */
    0,                    /* tp_members */
    0,                    /* tp_getset */
    0,                    /* tp_base */
    0,                    /* tp_dict */
    0,                    /* tp_descr_get */
    0,                    /* tp_descr_set */
    0,                    /* tp_dictoffset */
    0,                    /* tp_init */
    0,                    /* tp_alloc */
    0,                    /* tp_new */
};

PyModuleDef embmodule =
{
    PyModuleDef_HEAD_INIT,
    "emb", 0, -1, 0,
};

// Internal state
PyObject* g_stdout;
PyObject* g_stdout_saved;

PyMODINIT_FUNC PyInit_emb(void) 
{
    g_stdout = 0;
    g_stdout_saved = 0;

    StdoutType.tp_new = PyType_GenericNew;
    if (PyType_Ready(&StdoutType) < 0)
        return 0;

    PyObject* m = PyModule_Create(&embmodule);
    if (m)
    {
        Py_INCREF(&StdoutType);
        PyModule_AddObject(m, "Stdout", reinterpret_cast<PyObject*>(&StdoutType));
    }
    return m;
}

void set_stdout(stdout_write_type write)
{
    if (!g_stdout)
    {
        g_stdout_saved = PySys_GetObject("stdout"); // borrowed
        g_stdout = StdoutType.tp_new(&StdoutType, 0, 0);
    }

    Stdout* impl = reinterpret_cast<Stdout*>(g_stdout);
    impl->write = write;
    PySys_SetObject("stdout", g_stdout);    
}

void reset_stdout()
{
    if (g_stdout_saved)
        PySys_SetObject("stdout", g_stdout_saved);

    Py_XDECREF(g_stdout);
    g_stdout = 0;
}

} // namespace emb

int main()
{
    PyImport_AppendInittab("emb", emb::PyInit_emb);
    Py_Initialize();
    PyImport_ImportModule("emb");

    PyRun_SimpleString("print('hello to console')");

    // here comes the ***magic***
    std::string buffer;
    {
        // switch sys.stdout to custom handler
        emb::stdout_write_type write = [&buffer] (std::string s) { buffer += s; };
        emb::set_stdout(write);
        PyRun_SimpleString("print('hello to buffer')");
        PyRun_SimpleString("print(3.14)");
        PyRun_SimpleString("print('still talking to buffer')");
        emb::reset_stdout();
    }

    PyRun_SimpleString("print('hello to console again')");
    Py_Finalize();

    // output what was written to buffer object
    std::clog << buffer << std::endl;
}

This allows to intercept sys.stdout.write output with any kind of callable C++ entity: free function, class member function, named function objects or even anonymous functions as in the example above where I use C++11 lambda.

Note, this is a minimal example to present the essential concept. In production-ready code, it certainly needs more attention around reference counting of PyObject, getting rid of global state, and so on.

I know this question is old, but one part of the question has not been answered yet:

“How to catch output of commands that don’t directly write to the stdout of Python, like: 1+1 ?”

Here are the steps (for Python 3.4):

1. Redirect stdout/stderr into a Python variable using Mark’s solution: https://stackoverflow.com/a/4307737/1046299

2. Copy function PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) from Python source code. It is located in file pythonrun.c

3. Modify the PyRun_InteractiveOneObject function name and signature so that the new function takes a const char* (your command) as first parameter instead of a FILE*. Then you will need to use PyParser_ASTFromStringObject instead of PyParser_ASTFromFileObject in the function implementation. Note that you will need to copy the function run_mod as is from Python since it is called within the function.

4. Call the new function with your command, for example 1+1. Stdout should now receive the output 2.