# Source code for sqlobject.sqlbuilder

    """
    sqlobject.sqlbuilder
    --------------------

    :author: Ian Bicking <ianb@colorstudy.com>

    Builds SQL expressions from normal Python expressions.

    Disclaimer
    ----------

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License as
    published by the Free Software Foundation; either version 2.1 of the
    License, or (at your option any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this program; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301,
    USA.

    Instructions
    ------------

    To begin a SQL expression, you must use some sort of SQL object -- a
    field, table, or SQL statement (``SELECT``, ``INSERT``, etc.)  You can
    then use normal operators, with the exception of: `and`, `or`, `not`,
    and `in`.  You can use the `AND`, `OR`, `NOT`, and `IN` functions
    instead, or you can also use `&`, `|`, and `~` for `and`, `or`, and
    `not` respectively (however -- the precidence for these operators
    doesn't work as you would want, so you must use many parenthesis).

    To create a sql field, table, or constant/function, use the namespaces
    `table`, `const`, and `func`.  For instance, ``table.address`` refers
    to the ``address`` table, and ``table.address.state`` refers to the
    ``state`` field in the address table.  ``const.NULL`` is the ``NULL``
    SQL constant, and ``func.NOW()`` is the ``NOW()`` function call
    (`const` and `func` are actually identicle, but the two names are
    provided for clarity).  Once you create this object, expressions
    formed with it will produce SQL statements.

    The ``sqlrepr(obj)`` function gets the SQL representation of these
    objects, as well as the proper SQL representation of basic Python
    types (None==NULL).

    There are a number of DB-specific SQL features that this does not
    implement.  There are a bunch of normal ANSI features also not present.

    See the bottom of this module for some examples, and run it (i.e.
    ``python sql.py``) to see the results of those examples.

    """

    ########################################
    # Constants
    ########################################

    import fnmatch
    import operator
    import re
    import threading
    import types
    import weakref

    from . import classregistry
    from .converters import registerConverter, sqlrepr, quote_str, unquote_str
    from .compat import PY2, string_type

    class VersionError(Exception):
        pass

    class NoDefault:
        pass

    class SQLObjectState(object):
        def __init__(self, soObject, connection=None):
            self.soObject = weakref.proxy(soObject)
            self.connection = connection

    safeSQLRE = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_\.]*$')

    def sqlIdentifier(obj):
        # some db drivers return unicode column names
        return isinstance(obj, string_type) and bool(safeSQLRE.search(obj.strip()))

    def execute(expr, executor):
        if hasattr(expr, 'execute'):
            return expr.execute(executor)
        else:
            return expr

    def _str_or_sqlrepr(expr, db):
        if isinstance(expr, string_type):
            return expr
        return sqlrepr(expr, db)

    ########################################
    # Expression generation
    ########################################

    class SQLExpression:
        def __add__(self, other):
            return SQLOp("+", self, other)

        def __radd__(self, other):
            return SQLOp("+", other, self)

        def __sub__(self, other):
            return SQLOp("-", self, other)

        def __rsub__(self, other):
            return SQLOp("-", other, self)

        def __mul__(self, other):
            return SQLOp("*", self, other)

        def __rmul__(self, other):
            return SQLOp("*", other, self)

        def __div__(self, other):
            return SQLOp("/", self, other)

        def __rdiv__(self, other):
            return SQLOp("/", other, self)

        def __truediv__(self, other):
            return SQLOp("/", self, other)

        def __rtruediv__(self, other):
            return SQLOp("/", other, self)

        def __floordiv__(self, other):
            return SQLConstant("FLOOR")(SQLOp("/", self, other))

        def __rfloordiv__(self, other):
            return SQLConstant("FLOOR")(SQLOp("/", other, self))

        def __pos__(self):
            return SQLPrefix("+", self)

        def __neg__(self):
            return SQLPrefix("-", self)

        def __pow__(self, other):
            return SQLConstant("POW")(self, other)

        def __rpow__(self, other):
            return SQLConstant("POW")(other, self)

        def __abs__(self):
            return SQLConstant("ABS")(self)

        def __mod__(self, other):
            return SQLModulo(self, other)

        def __rmod__(self, other):
            return SQLConstant("MOD")(other, self)

        def __lt__(self, other):
            return SQLOp("<", self, other)

        def __le__(self, other):
            return SQLOp("<=", self, other)

        def __gt__(self, other):
            return SQLOp(">", self, other)

        def __ge__(self, other):
            return SQLOp(">=", self, other)

        def __eq__(self, other):
            if other is None:
                return ISNULL(self)
            else:
                return SQLOp("=", self, other)

        def __ne__(self, other):
            if other is None:
                return ISNOTNULL(self)
            else:
                return SQLOp("<>", self, other)

        def __and__(self, other):
            return SQLOp("AND", self, other)

        def __rand__(self, other):
            return SQLOp("AND", other, self)

        def __or__(self, other):
            return SQLOp("OR", self, other)

        def __ror__(self, other):
            return SQLOp("OR", other, self)

        def __invert__(self):
            return SQLPrefix("NOT", self)

        def __call__(self, *args):
            return SQLCall(self, args)

        def __repr__(self):
            try:
                return self.__sqlrepr__(None)
            except AssertionError:
                return '<%s %s>' % (
                    self.__class__.__name__, hex(id(self))[2:])

        def __str__(self):
            return repr(self)

        def __cmp__(self, other):
            raise VersionError("Python 2.1+ required")

        def __rcmp__(self, other):
            raise VersionError("Python 2.1+ required")

        def startswith(self, s):
            return STARTSWITH(self, s)

        def endswith(self, s):
            return ENDSWITH(self, s)

        def contains(self, s):
            return CONTAINSSTRING(self, s)

        def components(self):
            return []

        def tablesUsed(self, db):
            return self.tablesUsedSet(db)

        def tablesUsedSet(self, db):
            tables = set()
            for table in self.tablesUsedImmediate():
                if hasattr(table, '__sqlrepr__'):
                    table = sqlrepr(table, db)
                tables.add(table)
            for component in self.components():
                tables.update(tablesUsedSet(component, db))
            return tables

        def tablesUsedImmediate(self):
            return []

    #######################################
    # Converter for SQLExpression instances
    #######################################

    def SQLExprConverter(value, db):
        return value.__sqlrepr__()

    registerConverter(SQLExpression, SQLExprConverter)

    def tablesUsedSet(obj, db):
        if hasattr(obj, "tablesUsedSet"):
            return obj.tablesUsedSet(db)
        else:
            return {}

    if PY2:
        div = operator.div
    else:
        div = operator.truediv

    operatorMap = {
        "+": operator.add,
        "/": div,
        "-": operator.sub,
        "*": operator.mul,
        "<": operator.lt,
        "<=": operator.le,
        "=": operator.eq,
        "!=": operator.ne,
        ">=": operator.ge,
        ">": operator.gt,
        "IN": operator.contains,
        "IS": operator.eq,
    }

    class SQLOp(SQLExpression):
        def __init__(self, op, expr1, expr2):
            self.op = op.upper()
            if isinstance(expr1, Subquery):
                expr1, expr2 = expr2, expr1
            self.expr1 = expr1
            self.expr2 = expr2

        def __sqlrepr__(self, db):
            s1 = sqlrepr(self.expr1, db)
            s2 = sqlrepr(self.expr2, db)
            if s1[0] != '(' and s1 != 'NULL':
                s1 = '(' + s1 + ')'
            if s2[0] != '(' and s2 != 'NULL' and \
                    not isinstance(self.expr2, Subquery):
                s2 = '(' + s2 + ')'
            return "(%s %s %s)" % (s1, self.op, s2)

        def components(self):
            return [self.expr1, self.expr2]

        def execute(self, executor):
            if self.op == "AND":
                return execute(self.expr1, executor) \
                    and execute(self.expr2, executor)
            elif self.op == "OR":
                return execute(self.expr1, executor) \
                    or execute(self.expr2, executor)
            else:
                return operatorMap[self.op.upper()](execute(self.expr1, executor),
                                                    execute(self.expr2, executor))

    registerConverter(SQLOp, SQLExprConverter)

    class SQLModulo(SQLOp):
        def __init__(self, expr1, expr2):
            SQLOp.__init__(self, '%', expr1, expr2)

        def __sqlrepr__(self, db):
            if db == 'sqlite':
                return SQLOp.__sqlrepr__(self, db)
            s1 = sqlrepr(self.expr1, db)
            s2 = sqlrepr(self.expr2, db)
            return "MOD(%s, %s)" % (s1, s2)

    registerConverter(SQLModulo, SQLExprConverter)

    class SQLCall(SQLExpression):
        def __init__(self, expr, args):
            self.expr = expr
            self.args = args

        def __sqlrepr__(self, db):
            return "%s%s" % (sqlrepr(self.expr, db), sqlrepr(self.args, db))

        def components(self):
            return [self.expr] + list(self.args)

        def execute(self, executor):
            raise ValueError("I don't yet know how to locally execute functions")

    registerConverter(SQLCall, SQLExprConverter)

    class SQLPrefix(SQLExpression):
        def __init__(self, prefix, expr):
            self.prefix = prefix
            self.expr = expr

        def __sqlrepr__(self, db):
            return "%s %s" % (self.prefix, sqlrepr(self.expr, db))

        def components(self):
            return [self.expr]

        def execute(self, executor):
            prefix = self.prefix
            expr = execute(self.expr, executor)
            if prefix == "+":
                return expr
            elif prefix == "-":
                return -expr
            elif prefix.upper() == "NOT":
                return not expr

    registerConverter(SQLPrefix, SQLExprConverter)

    class SQLConstant(SQLExpression):
        def __init__(self, const):
            self.const = const

        def __sqlrepr__(self, db):
            return self.const

        def execute(self, executor):
            raise ValueError("I don't yet know how to execute SQL constants")

    registerConverter(SQLConstant, SQLExprConverter)

    class SQLTrueClauseClass(SQLExpression):
        def __sqlrepr__(self, db):
            return "1 = 1"

        def execute(self, executor):
            return 1

    SQLTrueClause = SQLTrueClauseClass()

    registerConverter(SQLTrueClauseClass, SQLExprConverter)

    ########################################
    # Namespaces
    ########################################

    class Field(SQLExpression):
        def __init__(self, tableName, fieldName):
            self.tableName = tableName
            self.fieldName = fieldName

        def __sqlrepr__(self, db):
            return self.tableName + "." + self.fieldName

        def tablesUsedImmediate(self):
            return [self.tableName]

        def execute(self, executor):
            return executor.field(self.tableName, self.fieldName)

    class SQLObjectField(Field):
        def __init__(self, tableName, fieldName, original, soClass, column):
            Field.__init__(self, tableName, fieldName)
            self.original = original
            self.soClass = soClass
            self.column = column

        def _from_python(self, value):
            column = self.column
            if not isinstance(value, SQLExpression) and \
                    column and column.from_python:
                value = column.from_python(value, SQLObjectState(self.soClass))
            return value

        def __eq__(self, other):
            if other is None:
                return ISNULL(self)
            other = self._from_python(other)
            return SQLOp('=', self, other)

        def __ne__(self, other):
            if other is None:
                return ISNOTNULL(self)
            other = self._from_python(other)
            return SQLOp('<>', self, other)

        def startswith(self, s):
            s = self._from_python(s)
            return STARTSWITH(self, s)

        def endswith(self, s):
            s = self._from_python(s)
            return ENDSWITH(self, s)

        def contains(self, s):
            s = self._from_python(s)
            return CONTAINSSTRING(self, s)

    registerConverter(SQLObjectField, SQLExprConverter)

    class Table(SQLExpression):
        FieldClass = Field

        def __init__(self, tableName):
            self.tableName = tableName

        def __getattr__(self, attr):
            if attr.startswith('__'):
                raise AttributeError("Attribute '%s' is forbidden in '%s'" % (
                    attr, self.__class__.__name__))
            return self.FieldClass(self.tableName, attr)

        def __sqlrepr__(self, db):
            return _str_or_sqlrepr(self.tableName, db)

        def execute(self, executor):
            raise ValueError("Tables don't have values")

    class SQLObjectTable(Table):
        FieldClass = SQLObjectField

        def __init__(self, soClass):
            self.soClass = soClass
            assert soClass.sqlmeta.table, (
                "Bad table name in class %r: %r"
                % (soClass, soClass.sqlmeta.table))
            Table.__init__(self, soClass.sqlmeta.table)

        def __getattr__(self, attr):
            if attr.startswith('__'):
                raise AttributeError("Attribute '%s' is forbidden in '%s'" % (
                    attr, self.__class__.__name__))
            if attr == 'id':
                return self._getattrFromID(attr)
            elif attr in self.soClass.sqlmeta.columns:
                column = self.soClass.sqlmeta.columns[attr]
                return self._getattrFromColumn(column, attr)
            elif attr + 'ID' in \
                [k for (k, v) in self.soClass.sqlmeta.columns.items()
                    if v.foreignKey]:
                attr += 'ID'
                column = self.soClass.sqlmeta.columns[attr]
                return self._getattrFromColumn(column, attr)
            else:
                raise AttributeError(
                    "%s instance has no attribute '%s'" % (self.soClass.__name__,
                                                           attr))

        def _getattrFromID(self, attr):
            return self.FieldClass(self.tableName, self.soClass.sqlmeta.idName,
                                   attr, self.soClass, None)

        def _getattrFromColumn(self, column, attr):
            return self.FieldClass(self.tableName, column.dbName, attr,
                                   self.soClass, column)

    class SQLObjectTableWithJoins(SQLObjectTable):

        def __getattr__(self, attr):
            if attr + 'ID' in \
                [k for (k, v) in self.soClass.sqlmeta.columns.items()
                    if v.foreignKey]:
                column = self.soClass.sqlmeta.columns[attr + 'ID']
                return self._getattrFromForeignKey(column, attr)
            elif attr in [x.joinMethodName for x in self.soClass.sqlmeta.joins]:
                join = [x for x in self.soClass.sqlmeta.joins
                        if x.joinMethodName == attr][0]
                return self._getattrFromJoin(join, attr)
            else:
                return SQLObjectTable.__getattr__(self, attr)

        def _getattrFromForeignKey(self, column, attr):
            ret = getattr(self, column.name) == \
                getattr(self.soClass, '_SO_class_' + column.foreignKey).q.id
            return ret

        def _getattrFromJoin(self, join, attr):
            if hasattr(join, 'otherColumn'):
                return AND(
                    join.otherClass.q.id == Field(join.intermediateTable,
                                                  join.otherColumn),
                    Field(join.intermediateTable,
                          join.joinColumn) == self.soClass.q.id)
            else:
                return getattr(join.otherClass.q, join.joinColumn) == \
                    self.soClass.q.id

    class TableSpace:
        TableClass = Table

        def __getattr__(self, attr):
            if attr.startswith('__'):
                raise AttributeError("Attribute '%s' is forbidden in '%s'" % (
                    attr, self.__class__.__name__))
            return self.TableClass(attr)

    class ConstantSpace:
        def __getattr__(self, attr):
            if attr.startswith('__'):
                raise AttributeError("Attribute '%s' is forbidden in '%s'" % (
                    attr, self.__class__.__name__))
            return SQLConstant(attr)

    ########################################
    # Table aliases
    ########################################

    class AliasField(Field):
        def __init__(self, tableName, fieldName, alias, aliasTable):
            Field.__init__(self, tableName, fieldName)
            self.alias = alias
            self.aliasTable = aliasTable

        def __sqlrepr__(self, db):
            fieldName = self.fieldName
            if isinstance(fieldName, SQLExpression):
                fieldName = sqlrepr(fieldName, db)
            return self.alias + "." + fieldName

        def tablesUsedImmediate(self):
            return [self.aliasTable]

    class AliasTable(Table):
        as_string = ''  # set it to "AS" if your database requires it
        FieldClass = AliasField

        _alias_lock = threading.Lock()
        _alias_counter = 0

        def __init__(self, table, alias=None):
            if hasattr(table, "sqlmeta"):
                tableName = SQLConstant(table.sqlmeta.table)
            elif isinstance(table, (Select, Union)):
                assert alias is not None, \
                    "Alias name cannot be constructed from Select instances, " \
                    "please provide an 'alias' keyword."
                tableName = Subquery('', table)
                table = None
            else:
                tableName = SQLConstant(table)
                table = None
            Table.__init__(self, tableName)
            self.table = table
            if alias is None:
                self._alias_lock.acquire()
                try:
                    AliasTable._alias_counter += 1
                    alias = "%s_alias%d" % (tableName, AliasTable._alias_counter)
                finally:
                    self._alias_lock.release()
            self.alias = alias

        def __getattr__(self, attr):
            if attr.startswith('__'):
                raise AttributeError("Attribute '%s' is forbidden in '%s'" % (
                    attr, self.__class__.__name__))
            if self.table:
                attr = getattr(self.table.q, attr).fieldName
            return self.FieldClass(self.tableName, attr, self.alias, self)

        def __sqlrepr__(self, db):
            return "%s %s %s" % (sqlrepr(self.tableName, db), self.as_string,
                                 self.alias)

    class AliasSQLMeta():
        def __init__(self, table, alias):
            self.__table = table
            self.__alias = alias

        def __getattr__(self, attr):
            if attr.startswith('__'):
                raise AttributeError("Attribute '%s' is forbidden in '%s'" % (
                    attr, self.__class__.__name__))
            table = self.__table
            if (attr == "table"):
                return '%s  %s' % (table.sqlmeta.table, self.__alias)
            return getattr(table.sqlmeta, attr)

    class Alias(SQLExpression):
        def __init__(self, table, alias=None):
            self.q = AliasTable(table, alias)

        def __getattr__(self, attr):
            table = self.q.table
            if (attr == "sqlmeta") and hasattr(table, "sqlmeta"):
                alias = self.q.alias
                return AliasSQLMeta(table, alias)
            return getattr(table, attr)

        def __sqlrepr__(self, db):
            return sqlrepr(self.q, db)

        def components(self):
            return [self.q]

        def select(self, clause=None, clauseTables=None,
                   orderBy=NoDefault, limit=None,
                   lazyColumns=False, reversed=False,
                   distinct=False, connection=None,
                   join=None, forUpdate=False):
            # Import here to avoid circular import
            from .sresults import SelectResults
            return SelectResults(self, clause,
                                 clauseTables=clauseTables,
                                 orderBy=orderBy,
                                 limit=limit,
                                 lazyColumns=lazyColumns,
                                 reversed=reversed,
                                 distinct=distinct,
                                 connection=connection,
                                 join=join, forUpdate=forUpdate)

    class Union(SQLExpression):
        def __init__(self, *tables):
            tabs = []
            for t in tables:
                if not isinstance(t, SQLExpression) and hasattr(t, 'sqlmeta'):
                    t = t.sqlmeta.table
                    if isinstance(t, Alias):
                        t = t.q
                    if isinstance(t, Table):
                        t = t.tableName
                    if not isinstance(t, SQLExpression):
                        t = SQLConstant(t)
                tabs.append(t)
            self.tables = tabs

        def __sqlrepr__(self, db):
            return " UNION ".join([str(sqlrepr(t, db)) for t in self.tables])

    ########################################
    # SQL Statements
    ########################################

    class Select(SQLExpression):
        def __init__(self, items=NoDefault, where=NoDefault, groupBy=NoDefault,
                     having=NoDefault, orderBy=NoDefault, limit=NoDefault,
                     join=NoDefault, lazyColumns=False, distinct=False,
                     start=0, end=None, reversed=False, forUpdate=False,
                     clause=NoDefault, staticTables=NoDefault,
                     distinctOn=NoDefault):
            self.ops = {}
            if not isinstance(items, (list, tuple, types.GeneratorType)):
                items = [items]
            if clause is NoDefault and where is not NoDefault:
                clause = where
            if staticTables is NoDefault:
                staticTables = []
            self.ops['items'] = items
            self.ops['clause'] = clause
            self.ops['groupBy'] = groupBy
            self.ops['having'] = having
            self.ops['orderBy'] = orderBy
            self.ops['limit'] = limit
            self.ops['join'] = join
            self.ops['lazyColumns'] = lazyColumns
            self.ops['distinct'] = distinct
            self.ops['distinctOn'] = distinctOn
            self.ops['start'] = start
            self.ops['end'] = end
            self.ops['reversed'] = reversed
            self.ops['forUpdate'] = forUpdate
            self.ops['staticTables'] = staticTables

        def clone(self, **newOps):
            ops = self.ops.copy()
            ops.update(newOps)
            return self.__class__(**ops)

        def newItems(self, items):
            return self.clone(items=items)

        def newClause(self, new_clause):
            return self.clone(clause=new_clause)

        def orderBy(self, orderBy):
            return self.clone(orderBy=orderBy)

        def unlimited(self):
            return self.clone(limit=NoDefault, start=0, end=None)

        def limit(self, limit):
            self.clone(limit=limit)

        def lazyColumns(self, value):
            return self.clone(lazyColumns=value)

        def reversed(self):
            return self.clone(reversed=not self.ops.get('reversed', False))

        def distinct(self):
            return self.clone(distinct=True)

        def filter(self, filter_clause):
            if filter_clause is None:
                # None doesn't filter anything, it's just a no-op:
                return self
            clause = self.ops['clause']
            if isinstance(clause, string_type):
                clause = SQLConstant('(%s)' % clause)
            return self.newClause(AND(clause, filter_clause))

        def __sqlrepr__(self, db):

            select = "SELECT"
            if self.ops['distinct']:
                select += " DISTINCT"
                if self.ops['distinctOn'] is not NoDefault:
                    select += " ON(%s)" % _str_or_sqlrepr(
                        self.ops['distinctOn'], db)
            if not self.ops['lazyColumns']:
                select += " %s" % ", ".join(
                    [str(_str_or_sqlrepr(v, db)) for v in self.ops['items']])
            else:
                select += " %s" % _str_or_sqlrepr(self.ops['items'][0], db)

            join = []
            join_str = ''
            if self.ops['join'] is not NoDefault and self.ops['join'] is not None:
                _join = self.ops['join']
                if isinstance(_join, str):
                    join_str = " " + _join
                elif isinstance(_join, SQLJoin):
                    join.append(_join)
                else:
                    join.extend(_join)
            tables = set()
            for x in self.ops['staticTables']:
                if isinstance(x, SQLExpression):
                    x = sqlrepr(x, db)
                tables.add(x)
            things = list(self.ops['items']) + join
            if self.ops['clause'] is not NoDefault:
                things.append(self.ops['clause'])
            for thing in things:
                if isinstance(thing, SQLExpression):
                    tables.update(tablesUsedSet(thing, db))
            for j in join:
                t1 = _str_or_sqlrepr(j.table1, db)
                if t1 in tables:
                    tables.remove(t1)
                t2 = _str_or_sqlrepr(j.table2, db)
                if t2 in tables:
                    tables.remove(t2)
            if tables:
                select += " FROM %s" % ", ".join(sorted(tables))
            elif join:
                select += " FROM"
            tablesYet = tables
            for j in join:
                if tablesYet and j.table1:
                    sep = ", "
                else:
                    sep = " "
                select += sep + sqlrepr(j, db)
                tablesYet = True

            if join_str:
                select += join_str

            if self.ops['clause'] is not NoDefault:
                select += " WHERE %s" % _str_or_sqlrepr(self.ops['clause'], db)
            if self.ops['groupBy'] is not NoDefault:
                groupBy = _str_or_sqlrepr(self.ops['groupBy'], db)
                if isinstance(self.ops['groupBy'], (list, tuple)):
                    groupBy = groupBy[1:-1]  # Remove parens
                select += " GROUP BY %s" % groupBy
            if self.ops['having'] is not NoDefault:
                select += " HAVING %s" % _str_or_sqlrepr(self.ops['having'], db)
            if self.ops['orderBy'] is not NoDefault and \
               self.ops['orderBy'] is not None:
                orderBy = self.ops['orderBy']
                if self.ops['reversed']:
                    reverser = DESC
                else:
                    def reverser(x):
                        return x
                if isinstance(orderBy, (list, tuple)):
                    select += " ORDER BY %s" % ", ".join(
                        [_str_or_sqlrepr(reverser(_x), db) for _x in orderBy])
                else:
                    select += " ORDER BY %s" % _str_or_sqlrepr(
                        reverser(orderBy), db)
            start, end = self.ops['start'], self.ops['end']
            if self.ops['limit'] is not NoDefault:
                end = start + self.ops['limit']
            if start or end:
                from .dbconnection import dbConnectionForScheme
                select = dbConnectionForScheme(db)._queryAddLimitOffset(select,
                                                                        start, end)
            if self.ops['forUpdate']:
                select += " FOR UPDATE"
            return select

    registerConverter(Select, SQLExprConverter)

    class Insert(SQLExpression):
        def __init__(self, table, valueList=None, values=None, template=NoDefault):
            self.template = template
            self.table = table
            if valueList:
                if values:
                    raise TypeError("You may only give valueList *or* values")
                self.valueList = valueList
            else:
                self.valueList = [values]

        def __sqlrepr__(self, db):
            if not self.valueList:
                return ''
            insert = "INSERT INTO %s" % self.table
            allowNonDict = True
            template = self.template
            if (template is NoDefault) and isinstance(self.valueList[0], dict):
                template = list(sorted(self.valueList[0].keys()))
                allowNonDict = False
            if template is not NoDefault:
                insert += " (%s)" % ", ".join(template)
            insert += " VALUES "
            listToJoin = []
            listToJoin_app = listToJoin.append
            for value in self.valueList:
                if isinstance(value, dict):
                    if template is NoDefault:
                        raise TypeError(
                            "You can't mix non-dictionaries with dictionaries "
                            "in an INSERT if you don't provide a template (%s)" %
                            repr(value))
                    value = dictToList(template, value)
                elif not allowNonDict:
                    raise TypeError(
                        "You can't mix non-dictionaries with dictionaries "
                        "in an INSERT if you don't provide a template (%s)" %
                        repr(value))
                listToJoin_app("(%s)" % ", ".join([sqlrepr(v, db) for v in value]))
            insert = "%s%s" % (insert, ", ".join(listToJoin))
            return insert

    registerConverter(Insert, SQLExprConverter)

    def dictToList(template, dict):
        list = []
        for key in template:
            list.append(dict[key])
        if len(dict.keys()) > len(template):
            raise TypeError(
                "Extra entries in dictionary that aren't asked for in template "
                "(template=%s, dict=%s)" % (repr(template), repr(dict)))
        return list

    class Update(SQLExpression):
        def __init__(self, table, values, template=NoDefault, where=NoDefault):
            self.table = table
            self.values = values
            self.template = template
            self.whereClause = where

        def __sqlrepr__(self, db):
            update = "%s %s" % (self.sqlName(), self.table)
            update += " SET"
            first = True
            if self.template is not NoDefault:
                for i in range(len(self.template)):
                    if first:
                        first = False
                    else:
                        update += ","
                    update += " %s=%s" % (self.template[i],
                                          sqlrepr(self.values[i], db))
            else:
                for key, value in sorted(self.values.items()):
                    if first:
                        first = False
                    else:
                        update += ","
                    update += " %s=%s" % (key, sqlrepr(value, db))
            if self.whereClause is not NoDefault:
                update += " WHERE %s" % _str_or_sqlrepr(self.whereClause, db)
            return update

        def sqlName(self):
            return "UPDATE"

    registerConverter(Update, SQLExprConverter)

    class Delete(SQLExpression):
        """To be safe, this will signal an error if there is no where clause,
        unless you pass in where=None to the constructor."""
        def __init__(self, table, where=NoDefault):
            self.table = table
            if where is NoDefault:
                raise TypeError(
                    "You must give a where clause or pass in None "
                    "to indicate no where clause")
            self.whereClause = where

        def __sqlrepr__(self, db):
            whereClause = self.whereClause
            if whereClause is None:
                return "DELETE FROM %s" % self.table
            whereClause = _str_or_sqlrepr(whereClause, db)
            return "DELETE FROM %s WHERE %s" % (self.table, whereClause)

    registerConverter(Delete, SQLExprConverter)

    class Replace(Update):

        def sqlName(self):
            return "REPLACE"

    registerConverter(Replace, SQLExprConverter)

    ########################################
    # SQL Builtins
    ########################################

    class DESC(SQLExpression):

        def __init__(self, expr):
            self.expr = expr

        def __sqlrepr__(self, db):
            if isinstance(self.expr, DESC):
                return sqlrepr(self.expr.expr, db)
            return '%s DESC' % sqlrepr(self.expr, db)

    def AND(*ops):
        if not ops:
            return None
        op1 = ops[0]
        ops = ops[1:]
        if ops:
            return SQLOp("AND", op1, AND(*ops))
        else:
            return op1

    def OR(*ops):
        if not ops:
            return None
        op1 = ops[0]
        ops = ops[1:]
        if ops:
            return SQLOp("OR", op1, OR(*ops))
        else:
            return op1

    def NOT(op):
        return SQLPrefix("NOT", op)

    def _IN(item, list):
        return SQLOp("IN", item, list)

    def IN(item, list):
        from .sresults import SelectResults  # Import here to avoid circular import
        if isinstance(list, SelectResults):
            query = list.queryForSelect()
            query.ops['items'] = [list.sourceClass.q.id]
            list = query
        if isinstance(list, Select):
            return INSubquery(item, list)
        else:
            return _IN(item, list)

    def NOTIN(item, list):
        if isinstance(list, Select):
            return NOTINSubquery(item, list)
        else:
            return NOT(_IN(item, list))

    def STARTSWITH(expr, pattern):
        return LIKE(expr, _LikeQuoted(pattern) + '%', escape='\\')

    def ENDSWITH(expr, pattern):
        return LIKE(expr, '%' + _LikeQuoted(pattern), escape='\\')

    def CONTAINSSTRING(expr, pattern):
        return LIKE(expr, '%' + _LikeQuoted(pattern) + '%', escape='\\')

    def ISNULL(expr):
        return SQLOp("IS", expr, None)

    def ISNOTNULL(expr):
        return SQLOp("IS NOT", expr, None)

    class ColumnAS(SQLOp):
        ''' Just like SQLOp('AS', expr, name) except without the parentheses '''
        def __init__(self, expr, name):
            if isinstance(name, string_type):
                name = SQLConstant(name)
            SQLOp.__init__(self, 'AS', expr, name)

        def __sqlrepr__(self, db):
            return "%s %s %s" % (sqlrepr(self.expr1, db), self.op,
                                 sqlrepr(self.expr2, db))

    class _LikeQuoted:
        # It assumes prefix and postfix are strings; usually just a percent sign.

        # @@: I'm not sure what the quoting rules really are for all the
        # databases

        def __init__(self, expr):
            self.expr = expr
            self.prefix = ''
            self.postfix = ''

        def __radd__(self, s):
            self.prefix = s + self.prefix
            return self

        def __add__(self, s):
            self.postfix += s
            return self

        def __sqlrepr__(self, db):
            s = self.expr
            if isinstance(s, SQLExpression):
                values = []
                if self.prefix:
                    values.append(quote_str(self.prefix, db))
                s = _quote_like_special(sqlrepr(s, db), db)
                values.append(s)
                if self.postfix:
                    values.append(quote_str(self.postfix, db))
                if db == "mysql":
                    return "CONCAT(%s)" % ", ".join(values)
                elif db in ("mssql", "sybase"):
                    return " + ".join(values)
                else:
                    return " || ".join(values)
            elif isinstance(s, string_type):
                s = _quote_like_special(unquote_str(sqlrepr(s, db)), db)
                return quote_str("%s%s%s" % (self.prefix, s, self.postfix), db)
            else:
                raise TypeError(
                    "expected str, unicode or SQLExpression, got %s" % type(s))

    def _quote_like_special(s, db):
        if db == 'postgres':
            escape = r'\\'
        else:
            escape = '\\'
        s = s.replace('\\', r'\\').\
            replace('%', escape + '%').\
            replace('_', escape + '_')
        return s

    class CONCAT(SQLExpression):
        def __init__(self, *expressions):
            self.expressions = expressions

        def __sqlrepr__(self, db):
            values = [sqlrepr(expr, db) for expr in self.expressions]
            if db == "mysql":
                return "CONCAT(%s)" % ", ".join(values)
            elif db in ("mssql", "sybase"):
                return " + ".join(values)
            else:
                return " || ".join(values)

    ########################################
    # SQL JOINs
    ########################################

    class SQLJoin(SQLExpression):
        def __init__(self, table1, table2, op=','):
            if hasattr(table1, 'sqlmeta'):
                table1 = table1.sqlmeta.table
            if hasattr(table2, 'sqlmeta'):
                table2 = table2.sqlmeta.table
            if isinstance(table1, str):
                table1 = SQLConstant(table1)
            if isinstance(table2, str):
                table2 = SQLConstant(table2)
            self.table1 = table1
            self.table2 = table2
            self.op = op

        def __sqlrepr__(self, db):
            if self.table1:
                return "%s%s %s" % (sqlrepr(self.table1, db), self.op,
                                    sqlrepr(self.table2, db))
            else:
                return "%s %s" % (self.op, sqlrepr(self.table2, db))

    registerConverter(SQLJoin, SQLExprConverter)

    def JOIN(table1, table2):
        return SQLJoin(table1, table2, " JOIN")

    def INNERJOIN(table1, table2):
        return SQLJoin(table1, table2, " INNER JOIN")

    def CROSSJOIN(table1, table2):
        return SQLJoin(table1, table2, " CROSS JOIN")

    def STRAIGHTJOIN(table1, table2):
        return SQLJoin(table1, table2, " STRAIGHT JOIN")

    def LEFTJOIN(table1, table2):
        return SQLJoin(table1, table2, " LEFT JOIN")

    def LEFTOUTERJOIN(table1, table2):
        return SQLJoin(table1, table2, " LEFT OUTER JOIN")

    def NATURALJOIN(table1, table2):
        return SQLJoin(table1, table2, " NATURAL JOIN")

    def NATURALLEFTJOIN(table1, table2):
        return SQLJoin(table1, table2, " NATURAL LEFT JOIN")

    def NATURALLEFTOUTERJOIN(table1, table2):
        return SQLJoin(table1, table2, " NATURAL LEFT OUTER JOIN")

    def RIGHTJOIN(table1, table2):
        return SQLJoin(table1, table2, " RIGHT JOIN")

    def RIGHTOUTERJOIN(table1, table2):
        return SQLJoin(table1, table2, " RIGHT OUTER JOIN")

    def NATURALRIGHTJOIN(table1, table2):
        return SQLJoin(table1, table2, " NATURAL RIGHT JOIN")

    def NATURALRIGHTOUTERJOIN(table1, table2):
        return SQLJoin(table1, table2, " NATURAL RIGHT OUTER JOIN")

    def FULLJOIN(table1, table2):
        return SQLJoin(table1, table2, " FULL JOIN")

    def FULLOUTERJOIN(table1, table2):
        return SQLJoin(table1, table2, " FULL OUTER JOIN")

    def NATURALFULLJOIN(table1, table2):
        return SQLJoin(table1, table2, " NATURAL FULL JOIN")

    def NATURALFULLOUTERJOIN(table1, table2):
        return SQLJoin(table1, table2, " NATURAL FULL OUTER JOIN")

    class SQLJoinConditional(SQLJoin):
        """Conditional JOIN"""
        def __init__(self, table1, table2, op,
                     on_condition=None, using_columns=None):
            """For condition you must give on_condition or using_columns
            but not both

            on_condition can be a string or SQLExpression, for example
                Table1.q.col1 == Table2.q.col2
            using_columns can be a string or a list of columns, e.g.
                (Table1.q.col1, Table2.q.col2)
            """
            if not on_condition and not using_columns:
                raise TypeError("You must give ON condition or USING columns")
            if on_condition and using_columns:
                raise TypeError(
                    "You must give ON condition or USING columns but not both")
            SQLJoin.__init__(self, table1, table2, op)
            self.on_condition = on_condition
            self.using_columns = using_columns

        def __sqlrepr__(self, db):
            if self.on_condition:
                on_condition = self.on_condition
                if hasattr(on_condition, "__sqlrepr__"):
                    on_condition = sqlrepr(on_condition, db)
                join = "%s %s ON %s" % (self.op, sqlrepr(self.table2, db),
                                        on_condition)
                if self.table1:
                    join = "%s %s" % (sqlrepr(self.table1, db), join)
                return join
            elif self.using_columns:
                using_columns = []
                for col in self.using_columns:
                    if hasattr(col, "__sqlrepr__"):
                        col = sqlrepr(col, db)
                    using_columns.append(col)
                using_columns = ", ".join(using_columns)
                join = "%s %s USING (%s)" % (self.op, sqlrepr(self.table2, db),
                                             using_columns)
                if self.table1:
                    join = "%s %s" % (sqlrepr(self.table1, db), join)
                return join
            else:
                RuntimeError, "Impossible error"

    registerConverter(SQLJoinConditional, SQLExprConverter)

    def INNERJOINConditional(table1, table2,
                             on_condition=None, using_columns=None):
        return SQLJoinConditional(table1, table2, "INNER JOIN",
                                  on_condition, using_columns)

    def LEFTJOINConditional(table1, table2, on_condition=None, using_columns=None):
        return SQLJoinConditional(table1, table2, "LEFT JOIN",
                                  on_condition, using_columns)

    def LEFTOUTERJOINConditional(table1, table2,
                                 on_condition=None, using_columns=None):
        return SQLJoinConditional(table1, table2, "LEFT OUTER JOIN",
                                  on_condition, using_columns)

    def RIGHTJOINConditional(table1, table2,
                             on_condition=None, using_columns=None):
        return SQLJoinConditional(table1, table2, "RIGHT JOIN",
                                  on_condition, using_columns)

    def RIGHTOUTERJOINConditional(table1, table2,
                                  on_condition=None, using_columns=None):
        return SQLJoinConditional(table1, table2, "RIGHT OUTER JOIN",
                                  on_condition, using_columns)

    def FULLJOINConditional(table1, table2, on_condition=None, using_columns=None):
        return SQLJoinConditional(table1, table2, "FULL JOIN",
                                  on_condition, using_columns)

    def FULLOUTERJOINConditional(table1, table2,
                                 on_condition=None, using_columns=None):
        return SQLJoinConditional(table1, table2, "FULL OUTER JOIN",
                                  on_condition, using_columns)

    class SQLJoinOn(SQLJoinConditional):
        """Conditional JOIN ON"""
        def __init__(self, table1, table2, op, on_condition):
            SQLJoinConditional.__init__(self, table1, table2, op, on_condition)

    registerConverter(SQLJoinOn, SQLExprConverter)

    class SQLJoinUsing(SQLJoinConditional):
        """Conditional JOIN USING"""
        def __init__(self, table1, table2, op, using_columns):
            SQLJoinConditional.__init__(self, table1, table2,
                                        op, None, using_columns)

    registerConverter(SQLJoinUsing, SQLExprConverter)

    def INNERJOINOn(table1, table2, on_condition):
        return SQLJoinOn(table1, table2, "INNER JOIN", on_condition)

    def LEFTJOINOn(table1, table2, on_condition):
        return SQLJoinOn(table1, table2, "LEFT JOIN", on_condition)

    def LEFTOUTERJOINOn(table1, table2, on_condition):
        return SQLJoinOn(table1, table2, "LEFT OUTER JOIN", on_condition)

    def RIGHTJOINOn(table1, table2, on_condition):
        return SQLJoinOn(table1, table2, "RIGHT JOIN", on_condition)

    def RIGHTOUTERJOINOn(table1, table2, on_condition):
        return SQLJoinOn(table1, table2, "RIGHT OUTER JOIN", on_condition)

    def FULLJOINOn(table1, table2, on_condition):
        return SQLJoinOn(table1, table2, "FULL JOIN", on_condition)

    def FULLOUTERJOINOn(table1, table2, on_condition):
        return SQLJoinOn(table1, table2, "FULL OUTER JOIN", on_condition)

    def INNERJOINUsing(table1, table2, using_columns):
        return SQLJoinUsing(table1, table2, "INNER JOIN", using_columns)

    def LEFTJOINUsing(table1, table2, using_columns):
        return SQLJoinUsing(table1, table2, "LEFT JOIN", using_columns)

    def LEFTOUTERJOINUsing(table1, table2, using_columns):
        return SQLJoinUsing(table1, table2, "LEFT OUTER JOIN", using_columns)

    def RIGHTJOINUsing(table1, table2, using_columns):
        return SQLJoinUsing(table1, table2, "RIGHT JOIN", using_columns)

    def RIGHTOUTERJOINUsing(table1, table2, using_columns):
        return SQLJoinUsing(table1, table2, "RIGHT OUTER JOIN", using_columns)

    def FULLJOINUsing(table1, table2, using_columns):
        return SQLJoinUsing(table1, table2, "FULL JOIN", using_columns)

    def FULLOUTERJOINUsing(table1, table2, using_columns):
        return SQLJoinUsing(table1, table2, "FULL OUTER JOIN", using_columns)

    ########################################
    # Subqueries (subselects)
    ########################################

    class OuterField(SQLObjectField):

        def tablesUsedImmediate(self):
            return []

    class OuterTable(SQLObjectTable):
        FieldClass = OuterField

    class Outer:
        def __init__(self, table):
            self.q = OuterTable(table)

    class LIKE(SQLExpression):
        op = "LIKE"

        def __init__(self, expr, string, escape=None):
            self.expr = expr
            self.string = string
            self.escape = escape

        def __sqlrepr__(self, db):
            escape = self.escape
            like = "%s %s (%s)" % (sqlrepr(self.expr, db),
                                   self.op, sqlrepr(self.string, db))
            if escape is None:
                return "(%s)" % like
            else:
                return "(%s ESCAPE %s)" % (like, sqlrepr(escape, db))

        def components(self):
            return [self.expr, self.string]

        def execute(self, executor):
            if not hasattr(self, '_regex'):
                # @@: Crude, not entirely accurate
                dest = self.string
                dest = dest.replace("%%", "\001")
                dest = dest.replace("*", "\002")
                dest = dest.replace("%", "*")
                dest = dest.replace("\001", "%")
                dest = dest.replace("\002", "[*]")
                self._regex = re.compile(fnmatch.translate(dest), re.I)
            return self._regex.search(execute(self.expr, executor))

    class RLIKE(LIKE):
        op = "RLIKE"

        op_db = {
            'firebird': 'RLIKE',
            'maxdb': 'RLIKE',
            'mysql': 'RLIKE',
            'postgres': '~',
            'sqlite': 'REGEXP'
        }

        def _get_op(self, db):
            return self.op_db.get(db, 'LIKE')

        def __sqlrepr__(self, db):
            return "(%s %s (%s))" % (
                sqlrepr(self.expr, db), self._get_op(db), sqlrepr(self.string, db)
            )

        def execute(self, executor):
            self.op = self._get_op(self.db)
            return LIKE.execute(self, executor)

    class INSubquery(SQLExpression):
        op = "IN"

        def __init__(self, item, subquery):
            self.item = item
            self.subquery = subquery

        def components(self):
            return [self.item]

        def __sqlrepr__(self, db):
            return "%s %s (%s)" % (sqlrepr(self.item, db),
                                   self.op, sqlrepr(self.subquery, db))

    class NOTINSubquery(INSubquery):
        op = "NOT IN"

    class Subquery(SQLExpression):
        def __init__(self, op, subquery):
            self.op = op
            self.subquery = subquery

        def __sqlrepr__(self, db):
            return "%s (%s)" % (self.op, sqlrepr(self.subquery, db))

    def EXISTS(subquery):
        return Subquery("EXISTS", subquery)

    def NOTEXISTS(subquery):
        return Subquery("NOT EXISTS", subquery)

    def SOME(subquery):
        return Subquery("SOME", subquery)

    def ANY(subquery):
        return Subquery("ANY", subquery)

    def ALL(subquery):
        return Subquery("ALL", subquery)

    ####

    class ImportProxyField(SQLObjectField):

        def tablesUsedImmediate(self):
            return [str(self.tableName)]

    class ImportProxy(SQLExpression):
        '''
        Class to be used in column definitions that rely on other tables that might
        not yet be in a classregistry
        '''
        FieldClass = ImportProxyField

        def __init__(self, clsName, registry=None):
            self.tableName = _DelayClass(self, clsName)
            self.sqlmeta = _Delay_proxy(table=_DelayClass(self, clsName))
            self.q = self
            self.soClass = None
            classregistry.registry(registry).addClassCallback(
                clsName, lambda foreign, me: setattr(me, 'soClass', foreign), self)

        def __nonzero__(self):
            return True
        __bool__ = __nonzero__

        def __getattr__(self, attr):
            if self.soClass is None:
                return _Delay(self, attr)
            return getattr(self.soClass.q, attr)

    class _Delay(SQLExpression):
        def __init__(self, proxy, attr):
            self.attr = attr
            self.proxy = proxy

        def __sqlrepr__(self, db):
            if self.proxy.soClass is None:
                return '_DELAYED_' + self.attr
            val = self._resolve()
            if isinstance(val, SQLExpression):
                val = sqlrepr(val, db)
            return val

        def tablesUsedImmediate(self):
            return getattr(self._resolve(), 'tablesUsedImmediate', lambda: [])()

        def components(self):
            return getattr(self._resolve(), 'components', lambda: [])()

        def _resolve(self):
            return getattr(self.proxy, self.attr)

        # For AliasTable etc
        def fieldName(self):
            class _aliasFieldName(SQLExpression):
                def __init__(self, proxy):
                    self.proxy = proxy

                def __sqlrepr__(self, db):
                    return self.proxy._resolve().fieldName
            return _aliasFieldName(self)
        fieldName = property(fieldName)

    class _DelayClass(_Delay):
        def _resolve(self):
            return self.proxy.soClass.sqlmeta.table

    class _Delay_proxy(object):
        def __init__(self, **kw):
            self.__dict__.update(kw)

    ######

    ########################################
    # Global initializations
    ########################################

    table = TableSpace()
    const = ConstantSpace()
    func = const

    ########################################
    # Testing
    ########################################

    if __name__ == "__main__":
        tests = """
    >>> AND(table.address.name == "Ian Bicking", table.address.zip > 30000)
    >>> table.address.name
    >>> AND(LIKE(table.address.name, "this"), IN(table.address.zip, [100, 200, 300]))
    >>> Select([table.address.name, table.address.state], where=LIKE(table.address.name, "%ian%"))
    >>> Select([table.user.name], where=AND(table.user.state == table.states.abbrev))
    >>> Insert(table.address, [{"name": "BOB", "address": "3049 N. 18th St."}, {"name": "TIM", "address": "409 S. 10th St."}])
    >>> Insert(table.address, [("BOB", "3049 N. 18th St."), ("TIM", "409 S. 10th St.")], template=('name', 'address'))
    >>> Delete(table.address, where="BOB"==table.address.name)
    >>> Update(table.address, {"lastModified": const.NOW()})
    >>> Replace(table.address, [("BOB", "3049 N. 18th St."), ("TIM", "409 S. 10th St.")], template=('name', 'address'))
    """  # noqa: allow long (> 79) lines
        for expr in tests.split('\n'):
            if not expr.strip():
                continue
            if expr.startswith('>>> '):
                expr = expr[4:]