How to avoid IntegrityError issue with update_or_create?
Question:
I would appreciate it if someone could explain why I’m having IntegrityError
exception being thrown here, and how to avoid it.
When an object already exists, isn’t the update_or_create
method supposed to update it ?
models.py
class OHLCV(TimestampedModel):
market = models.ForeignKey(Market, on_delete=models.CASCADE, related_name='candles', null=True)
index = models.ForeignKey(Index, on_delete=models.CASCADE, related_name='candles', null=True)
timeframe = models.CharField(max_length=10)
open, high, low, close, volume = [models.FloatField(null=True) for i in range(5)]
datetime = models.DateTimeField(null=True)
class Meta:
verbose_name_plural = "OHLCV"
unique_together = [['market', 'timeframe', 'datetime'],
['index', 'timeframe', 'datetime']]
tasks.py
@app.task
def bulk_update_ohlcv(timeframe):
for obj in Market.objects.filter(active=True):
if obj.exchange.is_status_ok():
update_ohlcv.delay('market', obj.pk, timeframe)
@app.task
def update_ohlcv(self, type, pk, timeframe):
[code here]
if obj.__class__ == Market:
ohlcv, created = OHLCV.objects.update_or_create(market=obj,
timeframe=timeframe,
datetime=dt,
open=candle[1],
defaults=defaults
)
elif obj.__class__ == Index:
ohlcv, created = OHLCV.objects.update_or_create(index=obj,
timeframe=timeframe,
datetime=dt,
open=candle[1],
defaults=defaults
)
Error:
-celery_worker-1 | 2023-04-01T06:55:47.710298043Z IntegrityError: duplicate key value violates unique constraint
-celery_worker-1 | 2023-04-01T06:55:47.710302260Z "market_ohlcv_market_id_timeframe_datetime_8ffd84de_uniq"
-celery_worker-1 | 2023-04-01T06:55:47.710306227Z DETAIL: Key (market_id, timeframe, datetime)=(84, 5m, 2023-03-31 21:20:00+00)
-celery_worker-1 | 2023-04-01T06:55:47.710310646Z already exists.
Answers:
The issue lies with the open
field being in the kwargs of the update_or_create
.
Django will try to get an OHLCV
matching the four fields market
, timeframe
, datetime
and open
. If there is a instance matching the first three and not open
, Django will create a new instance clashing with your constraint.
You’ll need to either move the open
field to the default dictionnary or include it in the unique together constraint
I would appreciate it if someone could explain why I’m having IntegrityError
exception being thrown here, and how to avoid it.
When an object already exists, isn’t the update_or_create
method supposed to update it ?
models.py
class OHLCV(TimestampedModel):
market = models.ForeignKey(Market, on_delete=models.CASCADE, related_name='candles', null=True)
index = models.ForeignKey(Index, on_delete=models.CASCADE, related_name='candles', null=True)
timeframe = models.CharField(max_length=10)
open, high, low, close, volume = [models.FloatField(null=True) for i in range(5)]
datetime = models.DateTimeField(null=True)
class Meta:
verbose_name_plural = "OHLCV"
unique_together = [['market', 'timeframe', 'datetime'],
['index', 'timeframe', 'datetime']]
tasks.py
@app.task
def bulk_update_ohlcv(timeframe):
for obj in Market.objects.filter(active=True):
if obj.exchange.is_status_ok():
update_ohlcv.delay('market', obj.pk, timeframe)
@app.task
def update_ohlcv(self, type, pk, timeframe):
[code here]
if obj.__class__ == Market:
ohlcv, created = OHLCV.objects.update_or_create(market=obj,
timeframe=timeframe,
datetime=dt,
open=candle[1],
defaults=defaults
)
elif obj.__class__ == Index:
ohlcv, created = OHLCV.objects.update_or_create(index=obj,
timeframe=timeframe,
datetime=dt,
open=candle[1],
defaults=defaults
)
Error:
-celery_worker-1 | 2023-04-01T06:55:47.710298043Z IntegrityError: duplicate key value violates unique constraint
-celery_worker-1 | 2023-04-01T06:55:47.710302260Z "market_ohlcv_market_id_timeframe_datetime_8ffd84de_uniq"
-celery_worker-1 | 2023-04-01T06:55:47.710306227Z DETAIL: Key (market_id, timeframe, datetime)=(84, 5m, 2023-03-31 21:20:00+00)
-celery_worker-1 | 2023-04-01T06:55:47.710310646Z already exists.
The issue lies with the open
field being in the kwargs of the update_or_create
.
Django will try to get an OHLCV
matching the four fields market
, timeframe
, datetime
and open
. If there is a instance matching the first three and not open
, Django will create a new instance clashing with your constraint.
You’ll need to either move the open
field to the default dictionnary or include it in the unique together constraint