dump csv from sqlalchemy
Question:
For some reason, I want to dump a table from a database (sqlite3) in the form of a csv file. I’m using a python script with elixir (based on sqlalchemy) to modify the database. I was wondering if there is any way to dump the table I use to csv.
I’ve seen sqlalchemy serializer but it doesn’t seem to be what I want. Am I doing it wrong? Should I call the sqlite3 python module after closing my sqlalchemy session to dump to a file instead? Or should I use something homemade?
Answers:
There are numerous ways to achieve this, including a simple os.system() call to the sqlite3 utility if you have that installed, but here’s roughly what I’d do from Python:
import sqlite3
import csv
con = sqlite3.connect('mydatabase.db')
outfile = open('mydump.csv', 'wb')
outcsv = csv.writer(outfile)
cursor = con.execute('select * from mytable')
# dump column titles (optional)
outcsv.writerow(x[0] for x in cursor.description)
# dump rows
outcsv.writerows(cursor.fetchall())
outfile.close()
Modifying Peter Hansen’s answer here a bit, to use SQLAlchemy instead of raw db access
import csv
outfile = open('mydump.csv', 'wb')
outcsv = csv.writer(outfile)
records = session.query(MyModel).all()
[outcsv.writerow([getattr(curr, column.name) for column in MyTable.__mapper__.columns]) for curr in records]
# or maybe use outcsv.writerows(records)
outfile.close()
I adapted the above examples to my sqlalchemy based code like this:
import csv
import sqlalchemy as sqAl
metadata = sqAl.MetaData()
engine = sqAl.create_engine('sqlite:///%s' % 'data.db')
metadata.bind = engine
mytable = sqAl.Table('sometable', metadata, autoload=True)
db_connection = engine.connect()
select = sqAl.sql.select([mytable])
result = db_connection.execute(select)
fh = open('data.csv', 'wb')
outcsv = csv.writer(fh)
outcsv.writerow(result.keys())
outcsv.writerows(result)
fh.close
This works for me with sqlalchemy 0.7.9. I suppose that this would work with all sqlalchemy table and result objects.
with open('dump.csv', 'wb') as f:
out = csv.writer(f)
out.writerow(['id', 'description'])
for item in session.query(Queue).all():
out.writerow([item.id, item.description])
I found this to be useful if you don’t mind hand-crafting your column labels.
import csv
f = open('ratings.csv', 'w')
out = csv.writer(f)
out.writerow(['id', 'user_id', 'movie_id', 'rating'])
for item in db.query.all():
out.writerow([item.username, item.username, item.movie_name, item.rating])
f.close()
In a modular way: an example using slqalchemy with automap and mysql.
database.py:
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine
Base = automap_base()
engine = create_engine('mysql://user:pass@localhost:3306/database_name', echo=True)
Base.prepare(engine, reflect=True)
# Map the tables
State = Base.classes.states
session = Session(engine, autoflush=False)
export_to_csv.py:
from databases import *
import csv
def export():
q = session.query(State)
file = './data/states.csv'
with open(file, 'w') as csvfile:
outcsv = csv.writer(csvfile, delimiter=',',quotechar='"', quoting = csv.QUOTE_MINIMAL)
header = State.__table__.columns.keys()
outcsv.writerow(header)
for record in q.all():
outcsv.writerow([getattr(record, c) for c in header ])
if __name__ == "__main__":
export()
Results:
name,abv,country,is_state,is_lower48,slug,latitude,longitude,population,area
Alaska,AK,US,y,n,alaska,61.370716,-152.404419,710231,571951.25
Alabama,AL,US,y,y,alabama,32.806671,-86.79113,4779736,50744.0
Arkansas,AR,US,y,y,arkansas,34.969704,-92.373123,2915918,52068.17
Arizona,AZ,US,y,y,arizona,33.729759,-111.431221,6392017,113634.57
California,CA,US,y,y,california,36.116203,-119.681564,37253956,155939.52
Colorado,CO,US,y,y,colorado,39.059811,-105.311104,5029196,103717.53
Connecticut,CT,US,y,y,connecticut,41.597782,-72.755371,3574097,4844.8
District of Columbia,DC,US,n,n,district-of-columbia,38.897438,-77.026817,601723,68.34
Delaware,DE,US,y,y,delaware,39.318523,-75.507141,897934,1953.56
Florida,FL,US,y,y,florida,27.766279,-81.686783,18801310,53926.82
Georgia,GA,US,y,y,georgia,33.040619,-83.643074,9687653,57906.14
I know this is old, but i just had this problem and this is how i solved it
import pandas as pd
from sqlalchemy import create_engine
basedir = os.path.abspath(os.path.dirname(__file__))
sql_engine = create_engine(os.path.join('sqlite:///' + os.path.join(basedir, 'single_file_app.db')), echo=False)
results = pd.read_sql_query('select * from users',sql_engine)
results.to_csv(os.path.join(basedir, 'mydump2.csv'),index=False,sep=";")
I spent a lot of time searching for a solution to this problem and finally created something like this:
from sqlalchemy import inspect
with open(file_to_write, 'w') as file:
out_csv = csv.writer(file, lineterminator='n')
columns = [column.name for column in inspect(Movies).columns][1:]
out_csv.writerow(columns)
session_3 = session_maker()
extract_query = [getattr(Movies, col) for col in columns]
for mov in session_3.query(*extract_query):
out_csv.writerow(mov)
session_3.close()
It creates a CSV file with column names and a dump of the entire "movies" table without "id" primary column.
For some reason, I want to dump a table from a database (sqlite3) in the form of a csv file. I’m using a python script with elixir (based on sqlalchemy) to modify the database. I was wondering if there is any way to dump the table I use to csv.
I’ve seen sqlalchemy serializer but it doesn’t seem to be what I want. Am I doing it wrong? Should I call the sqlite3 python module after closing my sqlalchemy session to dump to a file instead? Or should I use something homemade?
There are numerous ways to achieve this, including a simple os.system() call to the sqlite3 utility if you have that installed, but here’s roughly what I’d do from Python:
import sqlite3
import csv
con = sqlite3.connect('mydatabase.db')
outfile = open('mydump.csv', 'wb')
outcsv = csv.writer(outfile)
cursor = con.execute('select * from mytable')
# dump column titles (optional)
outcsv.writerow(x[0] for x in cursor.description)
# dump rows
outcsv.writerows(cursor.fetchall())
outfile.close()
Modifying Peter Hansen’s answer here a bit, to use SQLAlchemy instead of raw db access
import csv
outfile = open('mydump.csv', 'wb')
outcsv = csv.writer(outfile)
records = session.query(MyModel).all()
[outcsv.writerow([getattr(curr, column.name) for column in MyTable.__mapper__.columns]) for curr in records]
# or maybe use outcsv.writerows(records)
outfile.close()
I adapted the above examples to my sqlalchemy based code like this:
import csv
import sqlalchemy as sqAl
metadata = sqAl.MetaData()
engine = sqAl.create_engine('sqlite:///%s' % 'data.db')
metadata.bind = engine
mytable = sqAl.Table('sometable', metadata, autoload=True)
db_connection = engine.connect()
select = sqAl.sql.select([mytable])
result = db_connection.execute(select)
fh = open('data.csv', 'wb')
outcsv = csv.writer(fh)
outcsv.writerow(result.keys())
outcsv.writerows(result)
fh.close
This works for me with sqlalchemy 0.7.9. I suppose that this would work with all sqlalchemy table and result objects.
with open('dump.csv', 'wb') as f:
out = csv.writer(f)
out.writerow(['id', 'description'])
for item in session.query(Queue).all():
out.writerow([item.id, item.description])
I found this to be useful if you don’t mind hand-crafting your column labels.
import csv
f = open('ratings.csv', 'w')
out = csv.writer(f)
out.writerow(['id', 'user_id', 'movie_id', 'rating'])
for item in db.query.all():
out.writerow([item.username, item.username, item.movie_name, item.rating])
f.close()
In a modular way: an example using slqalchemy with automap and mysql.
database.py:
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine
Base = automap_base()
engine = create_engine('mysql://user:pass@localhost:3306/database_name', echo=True)
Base.prepare(engine, reflect=True)
# Map the tables
State = Base.classes.states
session = Session(engine, autoflush=False)
export_to_csv.py:
from databases import *
import csv
def export():
q = session.query(State)
file = './data/states.csv'
with open(file, 'w') as csvfile:
outcsv = csv.writer(csvfile, delimiter=',',quotechar='"', quoting = csv.QUOTE_MINIMAL)
header = State.__table__.columns.keys()
outcsv.writerow(header)
for record in q.all():
outcsv.writerow([getattr(record, c) for c in header ])
if __name__ == "__main__":
export()
Results:
name,abv,country,is_state,is_lower48,slug,latitude,longitude,population,area
Alaska,AK,US,y,n,alaska,61.370716,-152.404419,710231,571951.25
Alabama,AL,US,y,y,alabama,32.806671,-86.79113,4779736,50744.0
Arkansas,AR,US,y,y,arkansas,34.969704,-92.373123,2915918,52068.17
Arizona,AZ,US,y,y,arizona,33.729759,-111.431221,6392017,113634.57
California,CA,US,y,y,california,36.116203,-119.681564,37253956,155939.52
Colorado,CO,US,y,y,colorado,39.059811,-105.311104,5029196,103717.53
Connecticut,CT,US,y,y,connecticut,41.597782,-72.755371,3574097,4844.8
District of Columbia,DC,US,n,n,district-of-columbia,38.897438,-77.026817,601723,68.34
Delaware,DE,US,y,y,delaware,39.318523,-75.507141,897934,1953.56
Florida,FL,US,y,y,florida,27.766279,-81.686783,18801310,53926.82
Georgia,GA,US,y,y,georgia,33.040619,-83.643074,9687653,57906.14
I know this is old, but i just had this problem and this is how i solved it
import pandas as pd
from sqlalchemy import create_engine
basedir = os.path.abspath(os.path.dirname(__file__))
sql_engine = create_engine(os.path.join('sqlite:///' + os.path.join(basedir, 'single_file_app.db')), echo=False)
results = pd.read_sql_query('select * from users',sql_engine)
results.to_csv(os.path.join(basedir, 'mydump2.csv'),index=False,sep=";")
I spent a lot of time searching for a solution to this problem and finally created something like this:
from sqlalchemy import inspect
with open(file_to_write, 'w') as file:
out_csv = csv.writer(file, lineterminator='n')
columns = [column.name for column in inspect(Movies).columns][1:]
out_csv.writerow(columns)
session_3 = session_maker()
extract_query = [getattr(Movies, col) for col in columns]
for mov in session_3.query(*extract_query):
out_csv.writerow(mov)
session_3.close()
It creates a CSV file with column names and a dump of the entire "movies" table without "id" primary column.