TransactionManagementError "You can't execute queries until the end of the 'atomic' block" while using signals, but only during Unit Testing
Question:
I am getting TransactionManagementError when trying to save a Django User model instance and in its post_save signal, I’m saving some models that have the user as the foreign key.
The context and error is pretty similar to this question
django TransactionManagementError when using signals
However, in this case, the error occurs only while unit testing.
It works well in manual testing, but unit tests fails.
Is there anything that I’m missing?
Here are the code snippets:
views.py
@csrf_exempt
def mobileRegister(request):
if request.method == 'GET':
response = {"error": "GET request not accepted!!"}
return HttpResponse(json.dumps(response), content_type="application/json",status=500)
elif request.method == 'POST':
postdata = json.loads(request.body)
try:
# Get POST data which is to be used to save the user
username = postdata.get('phone')
password = postdata.get('password')
email = postdata.get('email',"")
first_name = postdata.get('first_name',"")
last_name = postdata.get('last_name',"")
user = User(username=username, email=email,
first_name=first_name, last_name=last_name)
user._company = postdata.get('company',None)
user._country_code = postdata.get('country_code',"+91")
user.is_verified=True
user._gcm_reg_id = postdata.get('reg_id',None)
user._gcm_device_id = postdata.get('device_id',None)
# Set Password for the user
user.set_password(password)
# Save the user
user.save()
signal.py
def create_user_profile(sender, instance, created, **kwargs):
if created:
company = None
companycontact = None
try: # Try to make userprofile with company and country code provided
user = User.objects.get(id=instance.id)
rand_pass = random.randint(1000, 9999)
company = Company.objects.get_or_create(name=instance._company,user=user)
companycontact = CompanyContact.objects.get_or_create(contact_type="Owner",company=company,contact_number=instance.username)
profile = UserProfile.objects.get_or_create(user=instance,phone=instance.username,verification_code=rand_pass,company=company,country_code=instance._country_code)
gcmDevice = GCMDevice.objects.create(registration_id=instance._gcm_reg_id,device_id=instance._gcm_reg_id,user=instance)
except Exception, e:
pass
tests.py
class AuthTestCase(TestCase):
fixtures = ['nextgencatalogs/fixtures.json']
def setUp(self):
self.user_data={
"phone":"0000000000",
"password":"123",
"first_name":"Gaurav",
"last_name":"Toshniwal"
}
def test_registration_api_get(self):
response = self.client.get("/mobileRegister/")
self.assertEqual(response.status_code,500)
def test_registration_api_post(self):
response = self.client.post(path="/mobileRegister/",
data=json.dumps(self.user_data),
content_type="application/json")
self.assertEqual(response.status_code,201)
self.user_data['username']=self.user_data['phone']
user = User.objects.get(username=self.user_data['username'])
# Check if the company was created
company = Company.objects.get(user__username=self.user_data['phone'])
self.assertIsInstance(company,Company)
# Check if the owner's contact is the same as the user's phone number
company_contact = CompanyContact.objects.get(company=company,contact_type="owner")
self.assertEqual(user.username,company_contact[0].contact_number)
Traceback:
======================================================================
ERROR: test_registration_api_post (nextgencatalogs.apps.catalogsapp.tests.AuthTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/nextgencatalogs/apps/catalogsapp/tests.py", line 29, in test_registration_api_post
user = User.objects.get(username=self.user_data['username'])
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/manager.py", line 151, in get
return self.get_queryset().get(*args, **kwargs)
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 301, in get
num = len(clone)
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 77, in __len__
self._fetch_all()
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 854, in _fetch_all
self._result_cache = list(self.iterator())
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 220, in iterator
for row in compiler.results_iter():
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 710, in results_iter
for rows in self.execute_sql(MULTI):
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 781, in execute_sql
cursor.execute(sql, params)
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/backends/util.py", line 47, in execute
self.db.validate_no_broken_transaction()
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/backends/__init__.py", line 365, in validate_no_broken_transaction
"An error occurred in the current transaction. You can't "
TransactionManagementError: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.
----------------------------------------------------------------------
Answers:
I ran into this same problem myself. This is caused by a quirk in how transactions are handled in the newer versions of Django coupled with a unittest that intentionally triggers an exception.
I had a unittest that checked to make sure a unique column constraint was enforced by purposefully triggering an IntegrityError exception:
def test_constraint(self):
try:
# Duplicates should be prevented.
models.Question.objects.create(domain=self.domain, slug='barks')
self.fail('Duplicate question allowed.')
except IntegrityError:
pass
do_more_model_stuff()
In Django 1.4, this works fine. However, in Django 1.5/1.6, each test is wrapped in a transaction, so if an exception occurs, it breaks the transaction until you explicitly roll it back. Therefore, any further ORM operations in that transaction, such as my do_more_model_stuff(), will fail with that django.db.transaction.TransactionManagementError exception.
Like caio mentioned in the comments, the solution is to capture your exception with transaction.atomic like:
from django.db import transaction
def test_constraint(self):
try:
# Duplicates should be prevented.
with transaction.atomic():
models.Question.objects.create(domain=self.domain, slug='barks')
self.fail('Duplicate question allowed.')
except IntegrityError:
pass
That will prevent the purposefully-thrown exception from breaking the entire unittest’s transaction.
Since @mkoistinen never made his comment an answer, I’ll post his suggestion so people won’t have to dig through comments.
consider just declaring your test class as a TransactionTestCase rather than just TestCase.
From the Django docs: A TransactionTestCase may call commit and rollback and observe the effects of these calls on the database.
I had the same issue.
In My Case I was doing this
author.tasks.add(tasks)
so converting it to
author.tasks.add(*tasks)
Removed that error.
I was getting this error on running unit tests in my create_test_data function using django 1.9.7. It worked in earlier versions of django.
It looked like this:
cls.localauth,_ = Organisation.objects.get_or_create(organisation_type=cls.orgtypeLA, name='LA for test', email_general='[email protected]', address='test', postcode='test', telephone='test')
cls.chamber,_ = Organisation.objects.get_or_create(organisation_type=cls.orgtypeC, name='chamber for test', email_general='[email protected]', address='test', postcode='test', telephone='test')
cls.lawfirm,_ = Organisation.objects.get_or_create(organisation_type=cls.orgtypeL, name='lawfirm for test', email_general='[email protected]', address='test', postcode='test', telephone='test')
cls.chamber.active = True
cls.chamber.save()
cls.localauth.active = True
cls.localauth.save() <---- error here
cls.lawfirm.active = True
cls.lawfirm.save()
My solution was to use update_or_create instead:
cls.localauth,_ = Organisation.objects.update_or_create(organisation_type=cls.orgtypeLA, name='LA for test', email_general='[email protected]', address='test', postcode='test', telephone='test', defaults={'active': True})
cls.chamber,_ = Organisation.objects.update_or_create(organisation_type=cls.orgtypeC, name='chamber for test', email_general='[email protected]', address='test', postcode='test', telephone='test', defaults={'active': True})
cls.lawfirm,_ = Organisation.objects.update_or_create(organisation_type=cls.orgtypeL, name='lawfirm for test', email_general='[email protected]', address='test', postcode='test', telephone='test', defaults={'active': True})
I have the same issue, but with transaction.atomic() and TransactionTestCase didn’t work for me.
python manage.py test -r instead of python manage.py test is ok for me, maybe the order of execution is crucial
then i find a doc about Order in which tests are executed, It mentions which test will run first.
So, i use TestCase for database interaction, unittest.TestCase for other simple test, it works now!
For me, the proposed fixes did not work. In my tests, I open some subprocesses with Popen to analyze/lint migrations (e.g. one test checks if there are no model changes).
For me, subclassing from SimpleTestCase instead of TestCase did do the trick.
Note that SimpleTestCase doesn’t allow to use the database.
While this does not answer the original question, I hope this helps some people anyway.
The answer of @kdazzle is correct. I didnt try it because people said that ‘Django’s TestCase class is a more commonly used subclass of TransactionTestCase’ so I thought it was the same use one or another. But the blog of Jahongir Rahmonov explained it better:
the TestCase class wraps the tests within two nested atomic() blocks:
one for the whole class and one for each test. This is where
TransactionTestCase should be used. It does not wrap the tests with
atomic() block and thus you can test your special methods that require
a transaction without any problem.
EDIT: It didn’t work, I thought yes, but NO.
In 4 years they could fixed this…………………………………
If using pytest-django you can pass transaction=True to the django_db decorator to avoid this error.
See https://pytest-django.readthedocs.io/en/latest/database.html#testing-transactions
Django itself has the TransactionTestCase which allows you to test
transactions and will flush the database between tests to isolate
them. The downside of this is that these tests are much slower to set up due to the required flushing of the database. pytest-django also supports this style of tests, which you can select using an argument to the django_db mark:
@pytest.mark.django_db(transaction=True)
def test_spam():
pass # test relying on transactions
Here is another way to do it, based on the answer to this question:
with transaction.atomic():
self.assertRaises(IntegrityError, models.Question.objects.create, **{'domain':self.domain, 'slug':'barks'})
def test_wrong_user_country_db_constraint(self):
"""
Check whether or not DB constraint doesnt allow to save wrong country code in DB.
"""
self.test_user_data['user_country'] = 'XX'
expected_constraint_name = "country_code_within_list_of_countries_check"
with transaction.atomic():
with self.assertRaisesRegex(IntegrityError, expected_constraint_name) as cm:
get_user_model().objects.create_user(**self.test_user_data)
self.assertFalse(
get_user_model().objects.filter(email=self.test_user_data['email']).exists()
)
with transaction.atomic() seems do the job correct
In my case it was caused but not calling super().tearDownClass()
class TnsFileViewSetTestCase(APITestCase):
@classmethod
def tearDownClass(self):
super().tearDownClass() # without this line we will get TransactionManagementError
for tnsfile in TnsFile.objects.all():
tnsfile.file.delete()
I am getting TransactionManagementError when trying to save a Django User model instance and in its post_save signal, I’m saving some models that have the user as the foreign key.
The context and error is pretty similar to this question
django TransactionManagementError when using signals
However, in this case, the error occurs only while unit testing.
It works well in manual testing, but unit tests fails.
Is there anything that I’m missing?
Here are the code snippets:
views.py
@csrf_exempt
def mobileRegister(request):
if request.method == 'GET':
response = {"error": "GET request not accepted!!"}
return HttpResponse(json.dumps(response), content_type="application/json",status=500)
elif request.method == 'POST':
postdata = json.loads(request.body)
try:
# Get POST data which is to be used to save the user
username = postdata.get('phone')
password = postdata.get('password')
email = postdata.get('email',"")
first_name = postdata.get('first_name',"")
last_name = postdata.get('last_name',"")
user = User(username=username, email=email,
first_name=first_name, last_name=last_name)
user._company = postdata.get('company',None)
user._country_code = postdata.get('country_code',"+91")
user.is_verified=True
user._gcm_reg_id = postdata.get('reg_id',None)
user._gcm_device_id = postdata.get('device_id',None)
# Set Password for the user
user.set_password(password)
# Save the user
user.save()
signal.py
def create_user_profile(sender, instance, created, **kwargs):
if created:
company = None
companycontact = None
try: # Try to make userprofile with company and country code provided
user = User.objects.get(id=instance.id)
rand_pass = random.randint(1000, 9999)
company = Company.objects.get_or_create(name=instance._company,user=user)
companycontact = CompanyContact.objects.get_or_create(contact_type="Owner",company=company,contact_number=instance.username)
profile = UserProfile.objects.get_or_create(user=instance,phone=instance.username,verification_code=rand_pass,company=company,country_code=instance._country_code)
gcmDevice = GCMDevice.objects.create(registration_id=instance._gcm_reg_id,device_id=instance._gcm_reg_id,user=instance)
except Exception, e:
pass
tests.py
class AuthTestCase(TestCase):
fixtures = ['nextgencatalogs/fixtures.json']
def setUp(self):
self.user_data={
"phone":"0000000000",
"password":"123",
"first_name":"Gaurav",
"last_name":"Toshniwal"
}
def test_registration_api_get(self):
response = self.client.get("/mobileRegister/")
self.assertEqual(response.status_code,500)
def test_registration_api_post(self):
response = self.client.post(path="/mobileRegister/",
data=json.dumps(self.user_data),
content_type="application/json")
self.assertEqual(response.status_code,201)
self.user_data['username']=self.user_data['phone']
user = User.objects.get(username=self.user_data['username'])
# Check if the company was created
company = Company.objects.get(user__username=self.user_data['phone'])
self.assertIsInstance(company,Company)
# Check if the owner's contact is the same as the user's phone number
company_contact = CompanyContact.objects.get(company=company,contact_type="owner")
self.assertEqual(user.username,company_contact[0].contact_number)
Traceback:
======================================================================
ERROR: test_registration_api_post (nextgencatalogs.apps.catalogsapp.tests.AuthTestCase)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/nextgencatalogs/apps/catalogsapp/tests.py", line 29, in test_registration_api_post
user = User.objects.get(username=self.user_data['username'])
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/manager.py", line 151, in get
return self.get_queryset().get(*args, **kwargs)
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 301, in get
num = len(clone)
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 77, in __len__
self._fetch_all()
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 854, in _fetch_all
self._result_cache = list(self.iterator())
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/query.py", line 220, in iterator
for row in compiler.results_iter():
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 710, in results_iter
for rows in self.execute_sql(MULTI):
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 781, in execute_sql
cursor.execute(sql, params)
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/backends/util.py", line 47, in execute
self.db.validate_no_broken_transaction()
File "/Users/gauravtoshniwal1989/Developer/Web/Server/ngc/ngcvenv/lib/python2.7/site-packages/django/db/backends/__init__.py", line 365, in validate_no_broken_transaction
"An error occurred in the current transaction. You can't "
TransactionManagementError: An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.
----------------------------------------------------------------------
I ran into this same problem myself. This is caused by a quirk in how transactions are handled in the newer versions of Django coupled with a unittest that intentionally triggers an exception.
I had a unittest that checked to make sure a unique column constraint was enforced by purposefully triggering an IntegrityError exception:
def test_constraint(self):
try:
# Duplicates should be prevented.
models.Question.objects.create(domain=self.domain, slug='barks')
self.fail('Duplicate question allowed.')
except IntegrityError:
pass
do_more_model_stuff()
In Django 1.4, this works fine. However, in Django 1.5/1.6, each test is wrapped in a transaction, so if an exception occurs, it breaks the transaction until you explicitly roll it back. Therefore, any further ORM operations in that transaction, such as my do_more_model_stuff(), will fail with that django.db.transaction.TransactionManagementError exception.
Like caio mentioned in the comments, the solution is to capture your exception with transaction.atomic like:
from django.db import transaction
def test_constraint(self):
try:
# Duplicates should be prevented.
with transaction.atomic():
models.Question.objects.create(domain=self.domain, slug='barks')
self.fail('Duplicate question allowed.')
except IntegrityError:
pass
That will prevent the purposefully-thrown exception from breaking the entire unittest’s transaction.
Since @mkoistinen never made his comment an answer, I’ll post his suggestion so people won’t have to dig through comments.
consider just declaring your test class as a TransactionTestCase rather than just TestCase.
From the Django docs: A TransactionTestCase may call commit and rollback and observe the effects of these calls on the database.
I had the same issue.
In My Case I was doing this
author.tasks.add(tasks)
so converting it to
author.tasks.add(*tasks)
Removed that error.
I was getting this error on running unit tests in my create_test_data function using django 1.9.7. It worked in earlier versions of django.
It looked like this:
cls.localauth,_ = Organisation.objects.get_or_create(organisation_type=cls.orgtypeLA, name='LA for test', email_general='[email protected]', address='test', postcode='test', telephone='test')
cls.chamber,_ = Organisation.objects.get_or_create(organisation_type=cls.orgtypeC, name='chamber for test', email_general='[email protected]', address='test', postcode='test', telephone='test')
cls.lawfirm,_ = Organisation.objects.get_or_create(organisation_type=cls.orgtypeL, name='lawfirm for test', email_general='[email protected]', address='test', postcode='test', telephone='test')
cls.chamber.active = True
cls.chamber.save()
cls.localauth.active = True
cls.localauth.save() <---- error here
cls.lawfirm.active = True
cls.lawfirm.save()
My solution was to use update_or_create instead:
cls.localauth,_ = Organisation.objects.update_or_create(organisation_type=cls.orgtypeLA, name='LA for test', email_general='[email protected]', address='test', postcode='test', telephone='test', defaults={'active': True})
cls.chamber,_ = Organisation.objects.update_or_create(organisation_type=cls.orgtypeC, name='chamber for test', email_general='[email protected]', address='test', postcode='test', telephone='test', defaults={'active': True})
cls.lawfirm,_ = Organisation.objects.update_or_create(organisation_type=cls.orgtypeL, name='lawfirm for test', email_general='[email protected]', address='test', postcode='test', telephone='test', defaults={'active': True})
I have the same issue, but with transaction.atomic() and TransactionTestCase didn’t work for me.
python manage.py test -r instead of python manage.py test is ok for me, maybe the order of execution is crucial
then i find a doc about Order in which tests are executed, It mentions which test will run first.
So, i use TestCase for database interaction, unittest.TestCase for other simple test, it works now!
For me, the proposed fixes did not work. In my tests, I open some subprocesses with Popen to analyze/lint migrations (e.g. one test checks if there are no model changes).
For me, subclassing from SimpleTestCase instead of TestCase did do the trick.
Note that SimpleTestCase doesn’t allow to use the database.
While this does not answer the original question, I hope this helps some people anyway.
The answer of @kdazzle is correct. I didnt try it because people said that ‘Django’s TestCase class is a more commonly used subclass of TransactionTestCase’ so I thought it was the same use one or another. But the blog of Jahongir Rahmonov explained it better:
the TestCase class wraps the tests within two nested atomic() blocks:
one for the whole class and one for each test. This is where
TransactionTestCase should be used. It does not wrap the tests with
atomic() block and thus you can test your special methods that require
a transaction without any problem.
EDIT: It didn’t work, I thought yes, but NO.
In 4 years they could fixed this…………………………………
If using pytest-django you can pass transaction=True to the django_db decorator to avoid this error.
See https://pytest-django.readthedocs.io/en/latest/database.html#testing-transactions
Django itself has the TransactionTestCase which allows you to test
transactions and will flush the database between tests to isolate
them. The downside of this is that these tests are much slower to set up due to the required flushing of the database. pytest-django also supports this style of tests, which you can select using an argument to the django_db mark:
@pytest.mark.django_db(transaction=True)
def test_spam():
pass # test relying on transactions
Here is another way to do it, based on the answer to this question:
with transaction.atomic():
self.assertRaises(IntegrityError, models.Question.objects.create, **{'domain':self.domain, 'slug':'barks'})
def test_wrong_user_country_db_constraint(self):
"""
Check whether or not DB constraint doesnt allow to save wrong country code in DB.
"""
self.test_user_data['user_country'] = 'XX'
expected_constraint_name = "country_code_within_list_of_countries_check"
with transaction.atomic():
with self.assertRaisesRegex(IntegrityError, expected_constraint_name) as cm:
get_user_model().objects.create_user(**self.test_user_data)
self.assertFalse(
get_user_model().objects.filter(email=self.test_user_data['email']).exists()
)
with transaction.atomic() seems do the job correct
In my case it was caused but not calling super().tearDownClass()
class TnsFileViewSetTestCase(APITestCase):
@classmethod
def tearDownClass(self):
super().tearDownClass() # without this line we will get TransactionManagementError
for tnsfile in TnsFile.objects.all():
tnsfile.file.delete()