How can I profile a SQLAlchemy powered application?

I have had some success in using cprofile and looking at the results in runsnakerun. This at least told me what functions and calls where taking a long time and if the database was the issue.
The documentation is here. You need wxpython. The presentation on it is good to get you started.
Its as easy as

import cProfile
command = """foo.run()"""
cProfile.runctx( command, globals(), locals(), filename="output.profile" )

Then

python runsnake.py output.profile

If you are looking to optimise your queries you will need postgrsql profiling.

It is also worth putting logging on to record the queries, but there is no parser for this that I know of to get the long running queries (and it wont be useful for concurrent requests).

sqlhandler = logging.FileHandler("sql.log")
sqllogger = logging.getLogger('sqlalchemy.engine')
sqllogger.setLevel(logging.INFO)
sqllogger.addHandler(sqlhandler)

and making sure your create engine statement has echo = True.

When I did it is was actually my code that was the main issue, so the cprofile thing helped.

There’s an extremely useful profiling recipe on the SQLAlchemy wiki

With a couple of minor modifications,

from sqlalchemy import event
from sqlalchemy.engine import Engine
import time
import logging

logging.basicConfig()
logger = logging.getLogger("myapp.sqltime")
logger.setLevel(logging.DEBUG)

@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement, 
                        parameters, context, executemany):
    context._query_start_time = time.time()
    logger.debug("Start Query:n%s" % statement)
    # Modification for StackOverflow answer:
    # Show parameters, which might be too verbose, depending on usage..
    logger.debug("Parameters:n%r" % (parameters,))


@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement, 
                        parameters, context, executemany):
    total = time.time() - context._query_start_time
    logger.debug("Query Complete!")

    # Modification for StackOverflow: times in milliseconds
    logger.debug("Total Time: %.02fms" % (total*1000))

if __name__ == '__main__':
    from sqlalchemy import *

    engine = create_engine('sqlite://')

    m1 = MetaData(engine)
    t1 = Table("sometable", m1, 
            Column("id", Integer, primary_key=True),
            Column("data", String(255), nullable=False),
        )

    conn = engine.connect()
    m1.create_all(conn)

    conn.execute(
        t1.insert(), 
        [{"data":"entry %d" % x} for x in xrange(100000)]
    )

    conn.execute(
        t1.select().where(t1.c.data.between("entry 25", "entry 7800")).order_by(desc(t1.c.data))
    )

Output is something like:

DEBUG:myapp.sqltime:Start Query:
SELECT sometable.id, sometable.data 
FROM sometable 
WHERE sometable.data BETWEEN ? AND ? ORDER BY sometable.data DESC
DEBUG:myapp.sqltime:Parameters:
('entry 25', 'entry 7800')
DEBUG:myapp.sqltime:Query Complete!
DEBUG:myapp.sqltime:Total Time: 410.46ms

Then if you find an oddly slow query, you could take the query string, format in the parameters (can be done the % string-formatting operator, for psycopg2 at least), prefix it with “EXPLAIN ANALYZE” and shove the query plan output into http://explain.depesz.com/ (found via this good article on PostgreSQL performance)

I’ve just discovered the library sqltap (https://github.com/inconshreveable/sqltap). It produces nicely styled HTML pages that help with inspecting and profiling SQL queries that are generated by SQLAlchemy.

An example usage:

profiler = sqltap.start()
run_some_queries()
statistics = profiler.collect()
sqltap.report(statistics, "report.html")

The library hasn’t been updated in 2 years, however, it seemed to be working just fine when I tested it with my application earlier today.

If it is only the query times you want to profile you can use a context manager to log all queries executed in a certain context:

"""SQLAlchemy Query profiler and logger."""
import logging
import time
import traceback

import sqlalchemy


class QueryProfiler:
    """Log query duration and SQL as a context manager."""

    def __init__(self,
                engine: sqlalchemy.engine.Engine,
                logger: logging.Logger,
                path: str):
        """
        Initialize for an engine and logger and filepath.
        engine: The sqlalchemy engine for which events should be logged.
                You can pass the class `sqlalchemy.engine.Engine` to capture all engines
        logger: The logger that should capture the query
        path: Only log the stacktrace for files in this path, use `'/'` to log all files
        """
        self.engine = engine
        self.logger = logger
        self.path = path

    def _before_cursor_execute(self, conn, cursor, statement, parameters, context, executemany):
        """Set the time on the connection to measure query duration."""
        conn._sqla_query_start_time = time.time()

    def _after_cursor_execute(self, conn, cursor, statement, parameters, context, executemany):
        """Listen for the 'after_cursor_execute' event and log sqlstatement and time."""
        end_time = time.time()
        start_time = getattr(conn, '_sqla_query_start_time', end_time)
        elapsed_time = round((end_time-start_time) * 1000)
        # only include the files in self.path in the stacktrace to reduce the noise
        stack = [frame for frame in traceback.extract_stack()[:-1] if frame.filename.startswith(self.path)]
        self.logger.debug('Query `%s` took %s ms. Stack: %s', statement, elapsed_time, traceback.format_list(stack))

    def __enter__(self, *args, **kwargs):
        """Context manager."""
        if isinstance(self.engine, sqlalchemy.engine.Engine):
            sqlalchemy.event.listen(self.engine, "before_cursor_execute", self._before_cursor_execute)
            sqlalchemy.event.listen(self.engine, "after_cursor_execute", self._after_cursor_execute)
        return self

    def __exit__(self, *args, **kwargs) -> None:
        """Context manager."""
        if isinstance(self.engine, sqlalchemy.engine.Engine):
            sqlalchemy.event.remove(self.engine, "before_cursor_execute", self._before_cursor_execute)
            sqlalchemy.event.remove(self.engine, "after_cursor_execute", self._after_cursor_execute)

Usage and test:

"""Test SQLAlchemy Query profiler and logger."""
import logging
import os

import sqlalchemy

from .sqlaprofiler import QueryProfiler

def test_sqlite_query(caplog):
    """Create logger and sqllite engine and profile the queries."""
    logging.basicConfig()
    logger = logging.getLogger(f'{__name__}')
    logger.setLevel(logging.DEBUG)
    caplog.set_level(logging.DEBUG, logger=f'{__name__}')
    path = os.path.dirname(os.path.realpath(__file__))
    engine = sqlalchemy.create_engine('sqlite://')
    metadata = sqlalchemy.MetaData(engine)
    table1 = sqlalchemy.Table(
            "sometable", metadata,
            sqlalchemy.Column("id", sqlalchemy.Integer, primary_key=True),
            sqlalchemy.Column("data", sqlalchemy.String(255), nullable=False),
        )
    conn = engine.connect()
    metadata.create_all(conn)

    with QueryProfiler(engine, logger, path):
        conn.execute(
            table1.insert(),
            [{"data": f"entry {i}"} for i in range(100000)]
        )

        conn.execute(
            table1.select()
            .where(table1.c.data.between("entry 25", "entry 7800"))
            .order_by(sqlalchemy.desc(table1.c.data))
        )

    assert caplog.messages[0].startswith('Query `INSERT INTO sometable (data) VALUES (?)` took')
    assert caplog.messages[1].startswith('Query `SELECT sometable.id, sometable.data n'
                                        'FROM sometable n'
                                        'WHERE sometable.data BETWEEN ? AND ? '
                                        'ORDER BY sometable.data DESC` took ')