Database Magic with SQLAlchemy

PyConUK 2007

Paul Johnston
http://pajhome.org.uk/

SQL Alchemy

Database Toolkit
Object-Relational Mapper
Version 0.4

Creating a Table

>>> from sqlalchemy import *

>>> metadata = MetaData('mssql://./test')

>>> customer = Table('customer', metadata,
...         Column('id', Integer, primary_key=True),
...         Column('name', String),
...         Column('priority', Boolean, default=False)
...     )

>>> customer.create()

Creating a Mapped Class

>>> from sqlalchemy.orm import *
>>> session = scoped_session(sessionmaker(autoflush=False, transactional=False))

>>> class Customer(object):
...     pass            

>>> session.mapper(Customer, customer)

Creating a Customer

>>> c = Customer()
>>> c.name = 'PyConUK'

>>> session.flush()

Selecting Customers

>>> Customer.query.all()
[<Customer object at 0x00D8F130>]

>>> Customer.query.filter(Customer.priority == True).all()
[]

>>> Customer.query.filter(Customer.name.startswith('Py')).all()
[<Customer object at 0x00D8F130>]

>>> Customer.query.get(1)
<Customer object at 0x00D8F130>

Updating and Deleting

>>> c = Customer.query.get(1)
>>> c.priority = True
>>> session.flush()

>>> session.delete(c)
>>> session.flush()

Refining the Class

class Customer(object):
    def __init__(self, **kw):
        self.__dict__.update(kw)

    def __str__(self):
        return self.name

    @property
    def initials(self):
        return ''.join(n[0] for n in self.name.split())

Refining the Class

>>> c = Customer(name='National Aeronautics& Space Agency', priority=True)

>>> print c
National Aeronautics& Space Agency

>>> print c.initials
NASA

Defining a Relation

pentest = Table('pentest', metadata,
        Column('id', Integer, primary_key=True),
        Column('customerid', Integer, ForeignKey('customer.id')),
        Column('name', String),
    )

class PenTest(object):
    def __init__(self, **kw):
        self.__dict__.update(kw)
    def __str__(self):
        return self.name

session.mapper(PenTest, pentest)

session.mapper(Customer, customer, properties={
        'pentest': relation(PenTest, backref='customer'),
    })

Using Relations

>>> c = Customer.query.get(1)

>>> p = PenTest(name='Web site test')

>>> c.pentest.append(p)

>>> print p.customer
PyConUK

>>> c.pentest.append(PenTest(name='Wireless site survey'))

>>> c.pentest
[<PenTest object at 0x0172C5F0>, <PenTest object at 0x01724A70>]

>>> Customer.query.get(2).pentest.append(PenTest(name='Perimeter scan'))

Joins

>>> PenTest.query.join('customer').filter(Customer.priority == True)

SQLAlchemy in a Web Framework

@expose(template='satalk.templates.customers')
def customers(self):
    return dict(customers=Customer.query.all())
    
    
<html xmlns:py="http://genshi.edgewall.org/">
    <body>
        <h1>Customers</h1>
        <ul>
            <li py:for="c in customers">$c</li>
        </ul>
    </body>
</html>

Using a Relation

@expose(template='satalk.templates.customers')
def customers(self):
    return dict(customers=Customer.query.all())
    
    
<html xmlns:py="http://genshi.edgewall.org/">
    <body>
        <h1>Customers</h1>
        <ul>
            <li py:for="c in customers">
                $c
                <ul><li py:for="t in c.pentest">$t</li></ul>
            </li>
        </ul>
    </body>
</html>

Lazy Loading

>>> c = customer.query.all()
SELECT  customer.priority AS customer_priority, 
        customer.name AS customer_name, 
        customer.id AS customer_id
FROM    customer 
ORDER BY customer.id

>>> c[0].pentest
SELECT  pentest.name AS pentest_name, 
        pentest.row_type AS pentest_row_type, 
        pentest.customerid AS pentest_customerid, 
        pentest.id AS pentest_id, 
FROM    pentest 
WHERE   pentest.customerid = ?

Eager Loading

>>> c = Customer.query.options(eagerload('pentest')).all()
SELECT  customer.priority AS customer_priority, 
        anon_d683.name AS anon_d683_name, 
        anon_d683.customerid AS anon_d683_customerid, 
        anon_d683.id AS anon_d683_id, 
        customer.name AS customer_name, 
        customer.id AS customer_id
FROM    customer 
LEFT OUTER JOIN pentest AS anon_d683 ON anon_d683.customerid = customer.id 
ORDER BY customer.id, 
        anon_d683.id

>>> c[0].pentest
[no query issued]

Data Mapper Pattern

customer = Table('customer', metadata,
        Column('id', Integer, primary_key=True),
        Column('name', String),
        Column('priority', Boolean, default=False)
    )
    
class Customer(object):
    def __init__(self, **kw):
        self.__dict__.update(kw)        
    def __str__(self):
        return self.name
    @property
    def initials(self):
        return ''.join(n[0] for n in self.name.split())

session.mapper(Customer, customer, properties={
        'pentest': relation(PenTest, backref='customer'),
    })

Elixir - Active Record Pattern

from elixir import *

class Customer(Entity):
    has_field('name', String)
    has_field('priority', Boolean, default=False)

    def __str__(self):
        return self.name
    @property
    def initials(self):
        return ''.join(n[0] for n in self.name.split())

Elixir - Active Record Pattern

from elixir import *

class Customer(Entity):
    has_field('name', String)
    has_field('priority', Boolean, default=False)

    def __str__(self):
        return self.name
    @property
    def initials(self):
        return ''.join(n[0] for n in self.name.split())

class PenTest(Entity):
    belongs_to('customer', of_kind='Customer', 
            colname='customerid', backref='pentest')
    has_field('name', String)

    def __str__(self):
        return self.name

Inheritence

Specialise PenTest into:

WebAppTest
VulnScanTest

Inheritence

class PenTest(Entity):
    using_options(inheritance='multi', polymorphic=True)
    belongs_to('customer', of_kind='Customer', 
            colname='customerid', backref='pentest')
    has_field('name', String)
    def __str__(self):
        return self.name

class WebAppTest(PenTest):
    using_options(inheritance='multi', polymorphic=True)
    has_field('url', String)
    def __str__(self):
        return '%s (%s)' % (self.name, self.url)
    
class VulnScanTest(PenTest):
    using_options(inheritance='multi', polymorphic=True)
    has_field('ipaddress', String)
    def __str__(self):
        return '%s (%s)' % (self.name, self.ip)

Using Inheritence

>>> c = Customer.query.get(1)
>>> c.pentest.append(WebAppTest(name='Web test', url='example.com'))
>>> c.pentest.append(VulnScanTest(name='Scan', ipaddress='1.2.3.4'))

>>> WebAppTest.query.all()
[<WebAppTest object at 0x00D8DAB0>]

>>> PenTest.query.all()
[<WebAppTest object at 0x00D8DAB0>, <VulnScanTest object at 0x00DC86F0>,
<PenTest object at 0x00D8D4B0>, <PenTest object at 0x00D8D2B0>,
<PenTest object at 0x00D8D0B0>]

>>> c.pentest
[<PenTest object at 0x00D8D4B0>, <PenTest object at 0x00D8D2B0>,
<WebAppTest object at 0x00D8DAB0>, <VulnScanTest object at 0x00DC86F0>]

Polymorphic Query

SELECT  pentest.name AS pentest_name, 
        webapptest.url AS webapptest_url, 
        pentest.row_type AS pentest_row_type, 
        pentest.customerid AS pentest_customerid, 
        pentest.id AS pentest_id, 
        vulnscantest.ipaddress AS vulnscantest_ipaddress, 
        webapptest.pentest_id AS webapptest_pentest_id, 
        vulnscantest.pentest_id AS vulnscantest_pentest_id
FROM    pentest 
LEFT OUTER JOIN webapptest ON pentest.id = webapptest.pentest_id 
LEFT OUTER JOIN vulnscantest ON pentest.id = vulnscantest.pentest_id
WHERE   pentest.customerid = ? 
ORDER BY pentest.id

Polymorphic Relation

@expose(template='satalk.templates.customers')
def customers(self):
    return dict(customers=Customer.query.all())
    
    
<html xmlns:py="http://genshi.edgewall.org/">
    <body>
        <h1>Customers</h1>
        <ul>
            <li py:for="c in customers">
                $c
                <ul><li py:for="t in c.pentest">$t</li></ul>
            </li>
        </ul>
    </body>
</html>

Many-to-Many Relations

Secondary table
Similar to one-to-many
Association proxy

Maintaining Database Structure

Two approaches:

Reflection - load structure from database
Syncronisation - update database to match Python definitions

Reflection

Table reflection

customer = Table('customer', metadata, autoload=True)

Metadata reflection

metadata = MetaData('mssql://./test', reflect=True)

SQLSoup

from sqlalchemy.ext.sqlsoup import SqlSoup
soup = SqlSoup('mssql://./test')

Syncronisation

ModelUpdate
AutoCode
Migrate

Other Features

Custom relations
Unit-of-work pattern & transactional systems
Migrating databases
Optimistic concurrent locking
Versioning

Database Magic with SQLAlchemy

PyConUK 2007

SQL Alchemy

Creating a Table

Creating a Mapped Class

Creating a Customer

Selecting Customers

Updating and Deleting

Refining the Class

Refining the Class

Defining a Relation

Using Relations

Joins

SQLAlchemy in a Web Framework

Using a Relation

Lazy Loading

Eager Loading

Data Mapper Pattern

Elixir - Active Record Pattern

Elixir - Active Record Pattern

Inheritence

Inheritence

Using Inheritence

Polymorphic Query

Polymorphic Relation

Many-to-Many Relations

Maintaining Database Structure

Reflection

Syncronisation

Other Features

Benefits of SQLAlchemy

Database Magic with SQLAlchemy

PyConUK 2007