hunt for root cause of flaky test in python

I exclude sys.base_exec_prefix, sys.base_prefix to not trace Python library modules.

import pytest

def f(x):
   import random

choice = random.choice((True, False))

if choice:
   raise ValueError
else:
   return x * 2

def trace(f, * args, ** kwargs):
   import trace
import sys

tracer = trace.Trace(
   ignoredirs = (sys.base_exec_prefix, sys.base_prefix), timing = True,
)
ret = tracer.runfunc(f, * args, ** kwargs)
r = tracer.results()
r.write_results(coverdir = "/tmp/xx-trace")
return ret

def test_f():
   with pytest.raises(ValueError):
   trace(f, 3)

Coverage report;
Lines marked with >>>>>> were not executed i.e. not traced and the numbers with colon are execution counts.

>>> >>>
import pytest

   >>>
   >>> def f(x):
   1: import random

1: choice = random.choice((True, False))

1: if choice:
   >>>
   >>> raise ValueError
else:
   1: return x * 2

      >>>
      >>> def trace(f, * args, ** kwargs):
      >>>
      >>>
      import trace >>>
      >>>
      import sys

      >>>
      >>> tracer = trace.Trace( >>>
         >>> ignoredirs = (sys.base_exec_prefix, sys.base_prefix), timing = True,
      ) >>>
      >>> tracer.runfunc(f, * args, ** kwargs) >>>
      >>> r = tracer.results() >>>
      >>> r.write_results(coverdir = "/tmp/xx-trace")

      >>>
      >>> def test_f():
      >>>
      >>> with pytest.raises(ValueError):
      >>>
      >>> trace(f, 3)

A sample to give you an idea looks like this:

import sys

def tracer(frame, event, arg):
   print(frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno, event, arg)

def bad_function(param: int):
   if param == 20:
   raise RuntimeError(f 'that failed for {param}')

sys.settrace(tracer)

bad_function(1)
bad_function(20)
bad_function(2)

Suggestion : 2

The trace module gives programmatic access via trace.Trace class. On test failures the Trace class' console output is visible. And it has coverage report to write on chosen path.,The trace library does not help, as it does not enable writing to a string or a StringIO object, it only writes to real files.,What you can do is to use sys.settrace() and define a simple function, which is called for every execution. You will find the documentation here: https://docs.python.org/3/library/sys.html#sys.settrace. The basic magic is to get the details out of the frameobject, which is documented here: https://docs.python.org/3/library/inspect.html.,Python test explorer not recognizing test in VS Code when test files split into multiple files

I exclude sys.base_exec_prefix, sys.base_prefix to not trace Python library modules.

import pytest

def f(x):
   import random

choice = random.choice((True, False))

if choice:
   raise ValueError
else:
   return x * 2

def trace(f, * args, ** kwargs):
   import trace
import sys

tracer = trace.Trace(
   ignoredirs = (sys.base_exec_prefix, sys.base_prefix), timing = True,
)
ret = tracer.runfunc(f, * args, ** kwargs)
r = tracer.results()
r.write_results(coverdir = "/tmp/xx-trace")
return ret

def test_f():
   with pytest.raises(ValueError):
   trace(f, 3)

Coverage report;
Lines marked with >>>>>> were not executed i.e. not traced and the numbers with colon are execution counts.

>>> >>>
import pytest

   >>>
   >>> def f(x):
   1: import random

1: choice = random.choice((True, False))

1: if choice:
   >>>
   >>> raise ValueError
else:
   1: return x * 2

      >>>
      >>> def trace(f, * args, ** kwargs):
      >>>
      >>>
      import trace >>>
      >>>
      import sys

      >>>
      >>> tracer = trace.Trace( >>>
         >>> ignoredirs = (sys.base_exec_prefix, sys.base_prefix), timing = True,
      ) >>>
      >>> tracer.runfunc(f, * args, ** kwargs) >>>
      >>> r = tracer.results() >>>
      >>> r.write_results(coverdir = "/tmp/xx-trace")

      >>>
      >>> def test_f():
      >>>
      >>> with pytest.raises(ValueError):
      >>>
      >>> trace(f, 3)

A sample to give you an idea looks like this:

import sys

def tracer(frame, event, arg):
   print(frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno, event, arg)

def bad_function(param: int):
   if param == 20:
   raise RuntimeError(f 'that failed for {param}')

sys.settrace(tracer)

bad_function(1)
bad_function(20)
bad_function(2)

Suggestion : 3

The trace module gives programmatic access via trace.Trace class. On test failures the Trace class console output is visible. And it has coverage report to write on chosen path.,It should be easy to store that information into a string for further investigation, or to handle the exception, if it is raised.,Additionally, if youre not already using pytest-html it may be worth integrating to get a nice report of what went on (via log lines) and what failed or just using,The trace library does not help, as it does not enable writing to a string or a StringIO object, it only writes to real files.

I exclude sys.base_exec_prefix, sys.base_prefix to not trace Python library modules.

import pytest

def f(x):
   import random

choice = random.choice((True, False))

if choice:
   raise ValueError
else:
   return x * 2

def trace(f, * args, ** kwargs):
   import trace
import sys

tracer = trace.Trace(
   ignoredirs = (sys.base_exec_prefix, sys.base_prefix), timing = True,
)
ret = tracer.runfunc(f, * args, ** kwargs)
r = tracer.results()
r.write_results(coverdir = /tmp/xx - trace)
return ret

def test_f():
   with pytest.raises(ValueError):
   trace(f, 3)

Coverage report;
Lines marked with >>>>>> were not executed i.e. not traced and the numbers with colon are execution counts.

>>> >>>
import pytest

   >>>
   >>> def f(x):
   1: import random

1: choice = random.choice((True, False))

1: if choice:
   >>>
   >>> raise ValueError
else:
   1: return x * 2

      >>>
      >>> def trace(f, * args, ** kwargs):
      >>>
      >>>
      import trace >>>
      >>>
      import sys

      >>>
      >>> tracer = trace.Trace( >>>
         >>> ignoredirs = (sys.base_exec_prefix, sys.base_prefix), timing = True,
      ) >>>
      >>> tracer.runfunc(f, * args, ** kwargs) >>>
      >>> r = tracer.results() >>>
      >>> r.write_results(coverdir = /tmp/xx - trace)

      >>>
      >>> def test_f():
      >>>
      >>> with pytest.raises(ValueError):
      >>>
      >>> trace(f, 3)

A sample to give you an idea looks like this:

import sys

def tracer(frame, event, arg):
   print(frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno, event, arg)

def bad_function(param: int):
   if param == 20:
   raise RuntimeError(fthat failed
      for {
         param
      })

sys.settrace(tracer)

bad_function(1)
bad_function(20)
bad_function(2)