FTS5 on sqlite through SQLalchemy

Question

My goal is to use SQLAlchemy to access a FTS5 table in sqlite3, with the virtual tables being updated at each insert. To this end, after doing some research (see this answer), I have coded the following:

class CreateFtsTable(DDLElement):
    """Represents a CREATE VIRTUAL TABLE ... USING fts5 statement, for indexing
    a given table.
    """

    def __init__(self, table, version=5):
        self.table = table
        self.version = version


@compiles(CreateFtsTable)
def compile_create_fts_table(element, compiler, **kw):
    tbl = element.table
    version = element.version
    preparer = compiler.preparer

    vtbl_name = preparer.quote(tbl.__table__.name + "_idx")

    columns = [x.name for x in tbl.__mapper__.columns]
    columns.append('tokenize="porter unicode61"')
    columns = ', '.join(columns)

    return f"CREATE VIRTUAL TABLE IF NOT EXISTS {vtbl_name} USING FTS{version} ({columns})"

class WorkItem(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    type = db.Column(db.String, nullable=False)
    state = db.Column(db.String, nullable=False)
    title = db.Column(db.String, nullable=False)
    description = db.Column(db.String, nullable=False)

update_fts = DDL('''CREATE TRIGGER work_item_update AFTER INSERT ON work_item
  BEGIN
    INSERT INTO work_item_idx (id, type, state, title, description) 
    VALUES (new.id, new.type, new.state, new.title, new.description);
  END;''')
db.event.listen(WorkItem.__table__, 'after_create', CreateFtsTable(WorkItem))
db.event.listen(WorkItem.__table__, 'after_create', update_fts)

With the SQLAlchemy echo enabled, I can see that the table gets properly created, as well as the virtual table and the trigger… but I do not see how to create a SQLAlchemy object that represents the FTS virtual table. On that answer there is a reference to using aliases, declaring a new table with a specific key… but I do not see why a new table would have to be created if the table is created through the CreateFtsTable.

How can I map an existing virtual table to an SQLAlchemy object?

Asked By: chronos

||

Source

Answer 1

After the comment from Ilja Everilä, this is a version that works… together with some notes:

flask_sqlalchemy creates all the tables at create_all
the FTS tables can only be referenced after the tables have been created

This results in having a global module variable initialized to None, until the get_fts_references method gets called (which will set them)

therefore, in main.py I got:

    db.init_app(app)
    with app.app_context():
        db.create_all()

then, in the fts module, I have:

class CreateFtsTable(DDLElement):
    """Represents a CREATE VIRTUAL TABLE ... USING fts5 statement, for indexing
    a given table.
    """

    def __init__(self, table, version=5):
        self.table = table
        self.version = version


@compiles(CreateFtsTable)
def compile_create_fts_table(element, compiler, **kw):
    tbl = element.table
    version = element.version
    preparer = compiler.preparer

    vtbl_name = preparer.quote(tbl.__table__.name + "_idx")

    columns = [x.name for x in tbl.__mapper__.columns]
    columns.append('tokenize="porter unicode61"')
    columns = ', '.join(columns)

    return f"CREATE VIRTUAL TABLE IF NOT EXISTS {vtbl_name} USING FTS{version} ({columns})"

WorkItemFts = None
def get_fts_references(target: db.Table, connection, **kwargs):
    base_model = next(c.entity for c in db.Model.registry.mappers if c.mapped_table.name == target.name)
    _temp_fts = db.Table(target.name+'_fts', db.metadata,
                         db.Column('id', db.Integer(), key='id', primary_key=True),
                         db.Column('type', db.Text()),
                         db.Column('state', db.Text()),
                         db.Column('title', db.Text()),
                         db.Column('description', db.Text()))
    globals()[base_model.__name__+'Fts'] = db.aliased(globals()[base_model.__name__], _temp_fts, adapt_on_names=True)

class WorkItem(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    type = db.Column(db.String, nullable=False)
    state = db.Column(db.String, nullable=False)
    title = db.Column(db.String, nullable=False)
    description = db.Column(db.String, nullable=False)

update_fts = DDL('''CREATE TRIGGER work_item_update AFTER INSERT ON work_item
  BEGIN
    INSERT INTO work_item_idx (id, type, state, title, description) 
    VALUES (new.id, new.type, new.state, new.title, new.description);
  END;''')
db.event.listen(WorkItem.__table__, 'after_create', CreateFtsTable(WorkItem))
db.event.listen(WorkItem.__table__, 'after_create', update_fts)
db.event.listen(WorkItem.__table__, 'after_create', get_fts_references)

Answered By: chronos

FTS5 on sqlite through SQLalchemy

Question:

Answers: