Creating an SQLAlchemy column to dynamically generate a list of models with an expression
Question:
I want to create a relationship column on my model, where it will be built with an expression so that it can be queried.
Here’s a brief example of my setup:
I have each application (eg. Python) stored in the App table. Each version of the application (eg. Python 3.7) is stored under the AppVersion table.
My items (in the Item table) have a minimum and maximum supported version per application. This is done with the ItemVersion table, with ItemVersion.version_min and ItemVersion.version_max, for example:
min_version=None, max_version=None: Compatible with all versionsmin_version=None, max_version=27: Compatible with Python 2 and belowmin_version=37, max_version=None: Compatible with Python 3 and abovemin_version=37, max_version=39: Compatible with Python 3.7 to 3.9
In this case, I want to generate an expression to return a list of AppVersion records compatible with my item.
Below I have used @hybrid_property as an example to mock up how ItemVersion.versions and Item.versions should work. I need it to be compatible with queries though, which this is not (eg. Item.versions.any(AppVersion.id == 1)).
from sqlalchemy import select, create_engine, Column, Integer, ForeignKey, String, case, and_, or_
from sqlalchemy.orm import relationship, sessionmaker, column_property
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.ext.associationproxy import association_proxy
Engine = create_engine('sqlite://')
Base = declarative_base(Engine)
session = sessionmaker(Engine)()
class App(Base):
__tablename__ = 'app'
id = Column(Integer, primary_key=True)
name = Column(String(64))
versions = relationship('AppVersion', back_populates='app')
def __repr__(self):
return self.name
class AppVersion(Base):
__tablename__ = 'app_version'
id = Column(Integer, primary_key=True)
app_id = Column(Integer, ForeignKey('app.id'), nullable=False)
value = Column(Integer, nullable=False)
app = relationship('App', foreign_keys=app_id, back_populates='versions', innerjoin=True)
def __repr__(self):
return f'{self.app.name}:{self.value}'
class ItemVersion(Base):
__tablename__ = 'item_version'
id = Column(Integer, primary_key=True)
item_id = Column(Integer, ForeignKey('item.id'))
app_id = Column(Integer, ForeignKey('app.id'))
version_min_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
version_max_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
item = relationship('Item', foreign_keys=item_id)
app = relationship('App', foreign_keys=app_id)
version_min = relationship('AppVersion', foreign_keys=version_min_id)
version_max = relationship('AppVersion', foreign_keys=version_max_id)
@hybrid_property
def versions(self):
# All versions
if self.version_min is None and self.version_max is None:
return self.app.versions
# Single version
elif self.version_min == self.version_max:
return [self.version_min]
# Max version and below
elif self.version_min is None:
return [version for version in self.app.versions
if version.value <= self.version_max.value]
# Min version and above
elif self.version_max is None:
return [version for version in self.app.versions
if self.version_min.value <= version.value]
# Custom range
return [version for version in self.app.versions
if self.version_min.value <= version.value <= self.version_max.value]
class Item(Base):
__tablename__ = 'item'
id = Column(Integer, primary_key=True)
item_versions = relationship('ItemVersion', back_populates='item')
def __repr__(self):
return f'Item {self.id}'
@hybrid_property
def versions(self):
versions = []
for item_version in self.item_versions:
versions.extend(item_version.versions)
return versions
Base.metadata.create_all()
py = App(name='Python')
session.add(py)
py27 = AppVersion(app=py, value=27)
py37 = AppVersion(app=py, value=37)
py38 = AppVersion(app=py, value=38)
py39 = AppVersion(app=py, value=39)
session.add(Item(item_versions=[ItemVersion(app=py)])) # [Python:27, Python:37, Python:38, Python:39]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py37)])) # [Python:37, Python:38, Python:39]
session.add(Item(item_versions=[ItemVersion(app=py, version_max=py37)])) # [Python:27, Python:37]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py27, version_max=py27)])) # [Python:27]
session.commit()
for item in session.execute(select(Item)).scalars():
print(f'{item}: {item.versions}')
My attempts so far have hit issues before I’ve got to writing the actual query.
With relationships they don’t apply any filter on value:
class ItemVersion(Base):
...
versions = relationship(
AppVersion,
primaryjoin=and_(AppVersion.app_id == App.id, AppVersion.value == 0),
secondaryjoin=app_id == App.id,
secondary=App.__table__,
viewonly=True, uselist=True,
)
# sqlalchemy.exc.ArgumentError: Could not locate any relevant foreign key columns for primary join condition 'app_version.app_id = item_version.app_id' on relationship ItemVersion.versions. Ensure that referencing columns are associated with a ForeignKey or ForeignKeyConstraint, or are annotated in the join condition with the foreign() annotation.
With column_property (which I could link with a relationship) it doesn’t like more than 1 result:
class ItemVersion(Base):
...
version_ids = column_property(
select(AppVersion.id).where(AppVersion.app_id == app_id)
)
# sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) sub-select returns 3 columns - expected 1
This would be my ideal result:
class ItemVersion(Base):
versions = # generate expression
class Item(Base):
...
item_versions = relationship('ItemVersion', back_populates='item')
versions = association_proxy('item_versions', 'versions')
If anyone has a particular section of documentation to point to that would also be appreciated, I’m just struggling a lot with this one.
Answers:
It’s possible via a relationship but it took a lot of trial and error with joins to get there. Below is what was needed to get it working, although I wouldn’t be surprised if there’s a more optimal way.
class ItemVersion(Base):
...
version_min_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_min_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
version_max_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_max_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
versions = relationship(
AppVersion,
primaryjoin=and_(
app_id == AppVersion.app_id,
case(
[and_(version_min_id == None, version_max_id == None), literal(True)],
[and_(version_min_id == None, version_max_id != None), AppVersion.value <= version_max_val],
[and_(version_min_id != None, version_max_id == None), version_min_val <= AppVersion.value],
else_=and_(version_min_val <= AppVersion.value, AppVersion.value <= version_max_val),
)
),
viewonly=True, uselist=True,
)
class Item(Base):
...
versions = relationship(
AppVersion,
primaryjoin=and_(
id == ItemVersion.item_id,
case(
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id == None), literal(True)],
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id != None), AppVersion.value <= ItemVersion.version_max_val],
[and_(ItemVersion.version_min_id != None, ItemVersion.version_max_id == None), ItemVersion.version_min_val <= AppVersion.value],
else_=and_(ItemVersion.version_min_val <= AppVersion.value, AppVersion.value <= ItemVersion.version_max_val),
)
),
secondaryjoin=ItemVersion.app_id == AppVersion.app_id,
secondary=ItemVersion.__table__,
viewonly=True, uselist=True,
)
Full code I used for testing:
from sqlalchemy import select, create_engine, Column, Integer, ForeignKey, String, case, and_, literal
from sqlalchemy.orm import relationship, sessionmaker, column_property
from sqlalchemy.ext.declarative import declarative_base
Engine = create_engine('sqlite://')
Base = declarative_base(Engine)
session = sessionmaker(Engine)()
class App(Base):
"""Each App has name and list of versions."""
__tablename__ = 'app'
id = Column(Integer, primary_key=True)
name = Column(String(64))
versions = relationship('AppVersion', back_populates='app')
def __repr__(self):
return self.name
class AppVersion(Base):
"""Each App version has a particular value."""
__tablename__ = 'app_version'
id = Column(Integer, primary_key=True)
app_id = Column(Integer, ForeignKey('app.id'), nullable=False)
value = Column(Integer, nullable=False)
app = relationship('App', foreign_keys=app_id, back_populates='versions', innerjoin=True)
def __repr__(self):
return f'{self.app.name}:{self.value}'
class ItemVersion(Base):
"""The item version links a particular item to an App and it's min/max versions.
Using the min and max versions, a range of compatible versions can be generated.
"""
__tablename__ = 'item_version'
id = Column(Integer, primary_key=True)
item_id = Column(Integer, ForeignKey('item.id'))
app_id = Column(Integer, ForeignKey('app.id'))
version_min_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
version_max_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
item = relationship('Item', foreign_keys=item_id)
app = relationship('App', foreign_keys=app_id)
version_min = relationship('AppVersion', foreign_keys=version_min_id)
version_max = relationship('AppVersion', foreign_keys=version_max_id)
version_min_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_min_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
version_max_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_max_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
versions = relationship(
AppVersion,
primaryjoin=and_(
app_id == AppVersion.app_id,
case(
[and_(version_min_id == None, version_max_id == None), literal(True)],
[and_(version_min_id == None, version_max_id != None), AppVersion.value <= version_max_val],
[and_(version_min_id != None, version_max_id == None), version_min_val <= AppVersion.value],
else_=and_(version_min_val <= AppVersion.value, AppVersion.value <= version_max_val),
)
),
viewonly=True, uselist=True,
)
class Item(Base):
"""Each item may have multiple compatible applications with specific versions."""
__tablename__ = 'item'
id = Column(Integer, primary_key=True)
item_versions = relationship('ItemVersion', back_populates='item')
def __repr__(self):
return f'Item {self.id}'
versions = relationship(
AppVersion,
primaryjoin=and_(
id == ItemVersion.item_id,
case(
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id == None), literal(True)],
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id != None), AppVersion.value <= ItemVersion.version_max_val],
[and_(ItemVersion.version_min_id != None, ItemVersion.version_max_id == None), ItemVersion.version_min_val <= AppVersion.value],
else_=and_(ItemVersion.version_min_val <= AppVersion.value, AppVersion.value <= ItemVersion.version_max_val),
)
),
secondaryjoin=ItemVersion.app_id == AppVersion.app_id,
secondary=ItemVersion.__table__,
viewonly=True, uselist=True,
)
Base.metadata.create_all()
py = App(name='Python')
py27 = AppVersion(app=py, value=27)
py37 = AppVersion(app=py, value=37)
py38 = AppVersion(app=py, value=38)
py39 = AppVersion(app=py, value=39)
maya = App(name='Maya')
m22 = AppVersion(app=maya, value=2022)
m23 = AppVersion(app=maya, value=2023)
# [Python:27, Python:37, Python:38, Python:39, Maya:2022, Maya:2023]
session.add(Item(item_versions=[ItemVersion(app=py), ItemVersion(app=maya)]))
# [Python:37, Python:38, Python:39]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py37)]))
# [Python:27, Python:37]
session.add(Item(item_versions=[ItemVersion(app=py, version_max=py37)]))
# [Python:27]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py27, version_max=py27)]))
# [Python:27, Python:37, Python:38]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py27, version_max=py38)]))
# [Python:27, Python:39, Maya:2022]
session.add(Item(item_versions=[ItemVersion(app=py, version_max=py27), ItemVersion(app=py, version_min=py39), ItemVersion(app=maya, version_min=m22, version_max=m22)]))
session.commit()
stmt = select(Item).where(Item.versions.any(AppVersion.app.has(App.name == 'Maya')))
for item in session.execute(stmt).scalars():
print(f'{item}: {item.versions}')
I want to create a relationship column on my model, where it will be built with an expression so that it can be queried.
Here’s a brief example of my setup:
I have each application (eg. Python) stored in the App table. Each version of the application (eg. Python 3.7) is stored under the AppVersion table.
My items (in the Item table) have a minimum and maximum supported version per application. This is done with the ItemVersion table, with ItemVersion.version_min and ItemVersion.version_max, for example:
min_version=None, max_version=None: Compatible with all versionsmin_version=None, max_version=27: Compatible with Python 2 and belowmin_version=37, max_version=None: Compatible with Python 3 and abovemin_version=37, max_version=39: Compatible with Python 3.7 to 3.9
In this case, I want to generate an expression to return a list of AppVersion records compatible with my item.
Below I have used @hybrid_property as an example to mock up how ItemVersion.versions and Item.versions should work. I need it to be compatible with queries though, which this is not (eg. Item.versions.any(AppVersion.id == 1)).
from sqlalchemy import select, create_engine, Column, Integer, ForeignKey, String, case, and_, or_
from sqlalchemy.orm import relationship, sessionmaker, column_property
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.ext.associationproxy import association_proxy
Engine = create_engine('sqlite://')
Base = declarative_base(Engine)
session = sessionmaker(Engine)()
class App(Base):
__tablename__ = 'app'
id = Column(Integer, primary_key=True)
name = Column(String(64))
versions = relationship('AppVersion', back_populates='app')
def __repr__(self):
return self.name
class AppVersion(Base):
__tablename__ = 'app_version'
id = Column(Integer, primary_key=True)
app_id = Column(Integer, ForeignKey('app.id'), nullable=False)
value = Column(Integer, nullable=False)
app = relationship('App', foreign_keys=app_id, back_populates='versions', innerjoin=True)
def __repr__(self):
return f'{self.app.name}:{self.value}'
class ItemVersion(Base):
__tablename__ = 'item_version'
id = Column(Integer, primary_key=True)
item_id = Column(Integer, ForeignKey('item.id'))
app_id = Column(Integer, ForeignKey('app.id'))
version_min_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
version_max_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
item = relationship('Item', foreign_keys=item_id)
app = relationship('App', foreign_keys=app_id)
version_min = relationship('AppVersion', foreign_keys=version_min_id)
version_max = relationship('AppVersion', foreign_keys=version_max_id)
@hybrid_property
def versions(self):
# All versions
if self.version_min is None and self.version_max is None:
return self.app.versions
# Single version
elif self.version_min == self.version_max:
return [self.version_min]
# Max version and below
elif self.version_min is None:
return [version for version in self.app.versions
if version.value <= self.version_max.value]
# Min version and above
elif self.version_max is None:
return [version for version in self.app.versions
if self.version_min.value <= version.value]
# Custom range
return [version for version in self.app.versions
if self.version_min.value <= version.value <= self.version_max.value]
class Item(Base):
__tablename__ = 'item'
id = Column(Integer, primary_key=True)
item_versions = relationship('ItemVersion', back_populates='item')
def __repr__(self):
return f'Item {self.id}'
@hybrid_property
def versions(self):
versions = []
for item_version in self.item_versions:
versions.extend(item_version.versions)
return versions
Base.metadata.create_all()
py = App(name='Python')
session.add(py)
py27 = AppVersion(app=py, value=27)
py37 = AppVersion(app=py, value=37)
py38 = AppVersion(app=py, value=38)
py39 = AppVersion(app=py, value=39)
session.add(Item(item_versions=[ItemVersion(app=py)])) # [Python:27, Python:37, Python:38, Python:39]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py37)])) # [Python:37, Python:38, Python:39]
session.add(Item(item_versions=[ItemVersion(app=py, version_max=py37)])) # [Python:27, Python:37]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py27, version_max=py27)])) # [Python:27]
session.commit()
for item in session.execute(select(Item)).scalars():
print(f'{item}: {item.versions}')
My attempts so far have hit issues before I’ve got to writing the actual query.
With relationships they don’t apply any filter on value:
class ItemVersion(Base):
...
versions = relationship(
AppVersion,
primaryjoin=and_(AppVersion.app_id == App.id, AppVersion.value == 0),
secondaryjoin=app_id == App.id,
secondary=App.__table__,
viewonly=True, uselist=True,
)
# sqlalchemy.exc.ArgumentError: Could not locate any relevant foreign key columns for primary join condition 'app_version.app_id = item_version.app_id' on relationship ItemVersion.versions. Ensure that referencing columns are associated with a ForeignKey or ForeignKeyConstraint, or are annotated in the join condition with the foreign() annotation.
With column_property (which I could link with a relationship) it doesn’t like more than 1 result:
class ItemVersion(Base):
...
version_ids = column_property(
select(AppVersion.id).where(AppVersion.app_id == app_id)
)
# sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) sub-select returns 3 columns - expected 1
This would be my ideal result:
class ItemVersion(Base):
versions = # generate expression
class Item(Base):
...
item_versions = relationship('ItemVersion', back_populates='item')
versions = association_proxy('item_versions', 'versions')
If anyone has a particular section of documentation to point to that would also be appreciated, I’m just struggling a lot with this one.
It’s possible via a relationship but it took a lot of trial and error with joins to get there. Below is what was needed to get it working, although I wouldn’t be surprised if there’s a more optimal way.
class ItemVersion(Base):
...
version_min_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_min_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
version_max_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_max_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
versions = relationship(
AppVersion,
primaryjoin=and_(
app_id == AppVersion.app_id,
case(
[and_(version_min_id == None, version_max_id == None), literal(True)],
[and_(version_min_id == None, version_max_id != None), AppVersion.value <= version_max_val],
[and_(version_min_id != None, version_max_id == None), version_min_val <= AppVersion.value],
else_=and_(version_min_val <= AppVersion.value, AppVersion.value <= version_max_val),
)
),
viewonly=True, uselist=True,
)
class Item(Base):
...
versions = relationship(
AppVersion,
primaryjoin=and_(
id == ItemVersion.item_id,
case(
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id == None), literal(True)],
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id != None), AppVersion.value <= ItemVersion.version_max_val],
[and_(ItemVersion.version_min_id != None, ItemVersion.version_max_id == None), ItemVersion.version_min_val <= AppVersion.value],
else_=and_(ItemVersion.version_min_val <= AppVersion.value, AppVersion.value <= ItemVersion.version_max_val),
)
),
secondaryjoin=ItemVersion.app_id == AppVersion.app_id,
secondary=ItemVersion.__table__,
viewonly=True, uselist=True,
)
Full code I used for testing:
from sqlalchemy import select, create_engine, Column, Integer, ForeignKey, String, case, and_, literal
from sqlalchemy.orm import relationship, sessionmaker, column_property
from sqlalchemy.ext.declarative import declarative_base
Engine = create_engine('sqlite://')
Base = declarative_base(Engine)
session = sessionmaker(Engine)()
class App(Base):
"""Each App has name and list of versions."""
__tablename__ = 'app'
id = Column(Integer, primary_key=True)
name = Column(String(64))
versions = relationship('AppVersion', back_populates='app')
def __repr__(self):
return self.name
class AppVersion(Base):
"""Each App version has a particular value."""
__tablename__ = 'app_version'
id = Column(Integer, primary_key=True)
app_id = Column(Integer, ForeignKey('app.id'), nullable=False)
value = Column(Integer, nullable=False)
app = relationship('App', foreign_keys=app_id, back_populates='versions', innerjoin=True)
def __repr__(self):
return f'{self.app.name}:{self.value}'
class ItemVersion(Base):
"""The item version links a particular item to an App and it's min/max versions.
Using the min and max versions, a range of compatible versions can be generated.
"""
__tablename__ = 'item_version'
id = Column(Integer, primary_key=True)
item_id = Column(Integer, ForeignKey('item.id'))
app_id = Column(Integer, ForeignKey('app.id'))
version_min_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
version_max_id = Column(Integer, ForeignKey('app_version.id'), nullable=True)
item = relationship('Item', foreign_keys=item_id)
app = relationship('App', foreign_keys=app_id)
version_min = relationship('AppVersion', foreign_keys=version_min_id)
version_max = relationship('AppVersion', foreign_keys=version_max_id)
version_min_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_min_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
version_max_val = column_property(
select(AppVersion.value)
.where(AppVersion.id == version_max_id)
.correlate_except(AppVersion)
.scalar_subquery(),
)
versions = relationship(
AppVersion,
primaryjoin=and_(
app_id == AppVersion.app_id,
case(
[and_(version_min_id == None, version_max_id == None), literal(True)],
[and_(version_min_id == None, version_max_id != None), AppVersion.value <= version_max_val],
[and_(version_min_id != None, version_max_id == None), version_min_val <= AppVersion.value],
else_=and_(version_min_val <= AppVersion.value, AppVersion.value <= version_max_val),
)
),
viewonly=True, uselist=True,
)
class Item(Base):
"""Each item may have multiple compatible applications with specific versions."""
__tablename__ = 'item'
id = Column(Integer, primary_key=True)
item_versions = relationship('ItemVersion', back_populates='item')
def __repr__(self):
return f'Item {self.id}'
versions = relationship(
AppVersion,
primaryjoin=and_(
id == ItemVersion.item_id,
case(
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id == None), literal(True)],
[and_(ItemVersion.version_min_id == None, ItemVersion.version_max_id != None), AppVersion.value <= ItemVersion.version_max_val],
[and_(ItemVersion.version_min_id != None, ItemVersion.version_max_id == None), ItemVersion.version_min_val <= AppVersion.value],
else_=and_(ItemVersion.version_min_val <= AppVersion.value, AppVersion.value <= ItemVersion.version_max_val),
)
),
secondaryjoin=ItemVersion.app_id == AppVersion.app_id,
secondary=ItemVersion.__table__,
viewonly=True, uselist=True,
)
Base.metadata.create_all()
py = App(name='Python')
py27 = AppVersion(app=py, value=27)
py37 = AppVersion(app=py, value=37)
py38 = AppVersion(app=py, value=38)
py39 = AppVersion(app=py, value=39)
maya = App(name='Maya')
m22 = AppVersion(app=maya, value=2022)
m23 = AppVersion(app=maya, value=2023)
# [Python:27, Python:37, Python:38, Python:39, Maya:2022, Maya:2023]
session.add(Item(item_versions=[ItemVersion(app=py), ItemVersion(app=maya)]))
# [Python:37, Python:38, Python:39]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py37)]))
# [Python:27, Python:37]
session.add(Item(item_versions=[ItemVersion(app=py, version_max=py37)]))
# [Python:27]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py27, version_max=py27)]))
# [Python:27, Python:37, Python:38]
session.add(Item(item_versions=[ItemVersion(app=py, version_min=py27, version_max=py38)]))
# [Python:27, Python:39, Maya:2022]
session.add(Item(item_versions=[ItemVersion(app=py, version_max=py27), ItemVersion(app=py, version_min=py39), ItemVersion(app=maya, version_min=m22, version_max=m22)]))
session.commit()
stmt = select(Item).where(Item.versions.any(AppVersion.app.has(App.name == 'Maya')))
for item in session.execute(stmt).scalars():
print(f'{item}: {item.versions}')