openlp/openlp/core/lib/db.py

# -*- coding: utf-8 -*-
# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4

###############################################################################
# OpenLP - Open Source Lyrics Projection                                      #
# --------------------------------------------------------------------------- #
# Copyright (c) 2008-2013 Raoul Snyman                                        #
# Portions copyright (c) 2008-2013 Tim Bentley, Gerald Britton, Jonathan      #
# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub,      #
# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer.   #
# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru,          #
# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith,             #
# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock,              #
# Frode Woldsund, Martin Zibricky, Patrick Zimmermann                         #
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it     #
# under the terms of the GNU General Public License as published by the Free  #
# Software Foundation; version 2 of the License.                              #
#                                                                             #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
# more details.                                                               #
#                                                                             #
# You should have received a copy of the GNU General Public License along     #
# with this program; if not, write to the Free Software Foundation, Inc., 59  #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
###############################################################################

"""
The :mod:`db` module provides the core database functionality for OpenLP
"""
import logging
import os
from urllib import quote_plus as urlquote

from sqlalchemy import Table, MetaData, Column, types, create_engine
from sqlalchemy.exc import SQLAlchemyError, InvalidRequestError, DBAPIError, OperationalError
from sqlalchemy.orm import scoped_session, sessionmaker, mapper
from sqlalchemy.pool import NullPool

from openlp.core.lib import translate, Settings
from openlp.core.lib.ui import critical_error_message_box
from openlp.core.utils import AppLocation, delete_file

log = logging.getLogger(__name__)


def init_db(url, auto_flush=True, auto_commit=False):
    """
    Initialise and return the session and metadata for a database

    ``url``
        The database to initialise connection with

    ``auto_flush``
        Sets the flushing behaviour of the session

    ``auto_commit``
        Sets the commit behaviour of the session
    """
    engine = create_engine(url, poolclass=NullPool)
    metadata = MetaData(bind=engine)
    session = scoped_session(sessionmaker(autoflush=auto_flush, autocommit=auto_commit, bind=engine))
    return session, metadata


def upgrade_db(url, upgrade):
    """
    Upgrade a database.

    ``url``
        The url of the database to upgrade.

    ``upgrade``
        The python module that contains the upgrade instructions.
    """
    session, metadata = init_db(url)

    class Metadata(BaseModel):
        """
        Provides a class for the metadata table.
        """
        pass
    load_changes = False
    tables = []
    try:
        tables = upgrade.upgrade_setup(metadata)
        load_changes = True
    except (SQLAlchemyError, DBAPIError):
        pass
    metadata_table = Table(u'metadata', metadata,
        Column(u'key', types.Unicode(64), primary_key=True),
        Column(u'value', types.UnicodeText(), default=None)
    )
    metadata_table.create(checkfirst=True)
    mapper(Metadata, metadata_table)
    version_meta = session.query(Metadata).get(u'version')
    if version_meta is None:
        version_meta = Metadata.populate(key=u'version', value=u'0')
        session.add(version_meta)
        version = 0
    else:
        version = int(version_meta.value)
    if version > upgrade.__version__:
        return version, upgrade.__version__
    version += 1
    if load_changes:
        while hasattr(upgrade, u'upgrade_%d' % version):
            log.debug(u'Running upgrade_%d', version)
            try:
                upgrade_func = getattr(upgrade, u'upgrade_%d' % version)
                upgrade_func(session, metadata, tables)
                session.commit()
                # Update the version number AFTER a commit so that we are sure the previous transaction happened
                version_meta.value = unicode(version)
                session.commit()
                version += 1
            except (SQLAlchemyError, DBAPIError):
                log.exception(u'Could not run database upgrade script '
                    '"upgrade_%s", upgrade process has been halted.', version)
                break
    else:
        version_meta = Metadata.populate(key=u'version', value=int(upgrade.__version__))
        session.commit()
    return int(version_meta.value), upgrade.__version__


def delete_database(plugin_name, db_file_name=None):
    """
    Remove a database file from the system.

    ``plugin_name``
        The name of the plugin to remove the database for

    ``db_file_name``
        The database file name. Defaults to None resulting in the plugin_name being used.
    """
    db_file_path = None
    if db_file_name:
        db_file_path = os.path.join(AppLocation.get_section_data_path(plugin_name), db_file_name)
    else:
        db_file_path = os.path.join(AppLocation.get_section_data_path(plugin_name), plugin_name)
    return delete_file(db_file_path)


class BaseModel(object):
    """
    BaseModel provides a base object with a set of generic functions
    """
    @classmethod
    def populate(cls, **kwargs):
        """
        Creates an instance of a class and populates it, returning the instance
        """
        instance = cls()
        for key, value in kwargs.iteritems():
            instance.__setattr__(key, value)
        return instance


class Manager(object):
    """
    Provide generic object persistence management
    """
    def __init__(self, plugin_name, init_schema, db_file_name=None, upgrade_mod=None):
        """
        Runs the initialisation process that includes creating the connection to the database and the tables if they do
        not exist.

        ``plugin_name``
            The name to setup paths and settings section names

        ``init_schema``
            The init_schema function for this database

        ``upgrade_schema``
            The upgrade_schema function for this database

        ``db_file_name``
            The file name to use for this database. Defaults to None resulting in the plugin_name being used.
        """
        settings = Settings()
        settings.beginGroup(plugin_name)
        self.db_url = u''
        self.is_dirty = False
        self.session = None
        db_type = settings.value(u'db type')
        if db_type == u'sqlite':
            if db_file_name:
                self.db_url = u'sqlite:///%s/%s' % (AppLocation.get_section_data_path(plugin_name), db_file_name)
            else:
                self.db_url = u'sqlite:///%s/%s.sqlite' % (AppLocation.get_section_data_path(plugin_name), plugin_name)
        else:
            self.db_url = u'%s://%s:%s@%s/%s' % (db_type,
                urlquote(settings.value(u'db username')),
                urlquote(settings.value(u'db password')),
                urlquote(settings.value(u'db hostname')),
                urlquote(settings.value(u'db database')))
            if db_type == u'mysql':
                db_encoding = settings.value(u'db encoding')
                self.db_url += u'?charset=%s' % urlquote(db_encoding)
        settings.endGroup()
        if upgrade_mod:
            db_ver, up_ver = upgrade_db(self.db_url, upgrade_mod)
            if db_ver > up_ver:
                critical_error_message_box(
                    translate('OpenLP.Manager', 'Database Error'),
                    translate('OpenLP.Manager', 'The database being loaded was created in a more recent version of '
                        'OpenLP. The database is version %d, while OpenLP expects version %d. The database will not '
                        'be loaded.\n\nDatabase: %s') % (db_ver, up_ver, self.db_url)
                )
                return
        try:
            self.session = init_schema(self.db_url)
        except (SQLAlchemyError, DBAPIError):
            log.exception(u'Error loading database: %s', self.db_url)
            critical_error_message_box(
                translate('OpenLP.Manager', 'Database Error'),
                translate('OpenLP.Manager', 'OpenLP cannot load your database.\n\nDatabase: %s') % self.db_url
            )

    def save_object(self, object_instance, commit=True):
        """
        Save an object to the database

        ``object_instance``
            The object to save

        ``commit``
            Commit the session with this object
        """
        for try_count in range(3):
            try:
                self.session.add(object_instance)
                if commit:
                    self.session.commit()
                self.is_dirty = True
                return True
            except OperationalError:
                # This exception clause is for users running MySQL which likes to terminate connections on its own
                # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                # non-recoverable way. So we only retry 3 times.
                log.exception(u'Probably a MySQL issue - "MySQL has gone away"')
                self.session.rollback()
                if try_count >= 2:
                    raise
            except InvalidRequestError:
                self.session.rollback()
                log.exception(u'Object list save failed')
                return False
            except:
                self.session.rollback()
                raise

    def save_objects(self, object_list, commit=True):
        """
        Save a list of objects to the database

        ``object_list``
            The list of objects to save

        ``commit``
            Commit the session with this object
        """
        for try_count in range(3):
            try:
                self.session.add_all(object_list)
                if commit:
                    self.session.commit()
                self.is_dirty = True
                return True
            except OperationalError:
                # This exception clause is for users running MySQL which likes to terminate connections on its own
                # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                # non-recoverable way. So we only retry 3 times.
                log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
                self.session.rollback()
                if try_count >= 2:
                    raise
            except InvalidRequestError:
                self.session.rollback()
                log.exception(u'Object list save failed')
                return False
            except:
                self.session.rollback()
                raise

    def get_object(self, object_class, key=None):
        """
        Return the details of an object

        ``object_class``
            The type of object to return

        ``key``
            The unique reference or primary key for the instance to return
        """
        if not key:
            return object_class()
        else:
            for try_count in range(3):
                try:
                    return self.session.query(object_class).get(key)
                except OperationalError:
                    # This exception clause is for users running MySQL which likes to terminate connections on its own
                    # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                    # non-recoverable way. So we only retry 3 times.
                    log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
                    if try_count >= 2:
                        raise

    def get_object_filtered(self, object_class, filter_clause):
        """
        Returns an object matching specified criteria

        ``object_class``
            The type of object to return

        ``filter_clause``
            The criteria to select the object by
        """
        for try_count in range(3):
            try:
                return self.session.query(object_class).filter(filter_clause).first()
            except OperationalError:
                # This exception clause is for users running MySQL which likes to terminate connections on its own
                # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                # non-recoverable way. So we only retry 3 times.
                log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
                if try_count >= 2:
                    raise

    def get_all_objects(self, object_class, filter_clause=None, order_by_ref=None):
        """
        Returns all the objects from the database

        ``object_class``
            The type of objects to return

        ``filter_clause``
            The filter governing selection of objects to return. Defaults to None.

        ``order_by_ref``
            Any parameters to order the returned objects by. Defaults to None.
        """
        query = self.session.query(object_class)
        if filter_clause is not None:
            query = query.filter(filter_clause)
        if isinstance(order_by_ref, list):
            query = query.order_by(*order_by_ref)
        elif order_by_ref is not None:
            query = query.order_by(order_by_ref)
        for try_count in range(3):
            try:
                return query.all()
            except OperationalError:
                # This exception clause is for users running MySQL which likes to terminate connections on its own
                # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                # non-recoverable way. So we only retry 3 times.
                log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
                if try_count >= 2:
                    raise

    def get_object_count(self, object_class, filter_clause=None):
        """
        Returns a count of the number of objects in the database.

        ``object_class``
            The type of objects to return.

        ``filter_clause``
            The filter governing selection of objects to return. Defaults to None.
        """
        query = self.session.query(object_class)
        if filter_clause is not None:
            query = query.filter(filter_clause)
        for try_count in range(3):
            try:
                return query.count()
            except OperationalError:
                # This exception clause is for users running MySQL which likes to terminate connections on its own
                # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                # non-recoverable way. So we only retry 3 times.
                log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
                if try_count >= 2:
                    raise

    def delete_object(self, object_class, key):
        """
        Delete an object from the database

        ``object_class``
            The type of object to delete

        ``key``
            The unique reference or primary key for the instance to be deleted
        """
        if key != 0:
            object_instance = self.get_object(object_class, key)
            for try_count in range(3):
                try:
                    self.session.delete(object_instance)
                    self.session.commit()
                    self.is_dirty = True
                    return True
                except OperationalError:
                    # This exception clause is for users running MySQL which likes to terminate connections on its own
                    # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                    # non-recoverable way. So we only retry 3 times.
                    log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
                    self.session.rollback()
                    if try_count >= 2:
                        raise
                except InvalidRequestError:
                    self.session.rollback()
                    log.exception(u'Failed to delete object')
                    return False
                except:
                    self.session.rollback()
                    raise
        else:
            return True

    def delete_all_objects(self, object_class, filter_clause=None):
        """
        Delete all object records. This method should only be used for simple tables and **not** ones with
        relationships. The relationships are not deleted from the database and this will lead to database corruptions.

        ``object_class``
            The type of object to delete

        ``filter_clause``
            The filter governing selection of objects to return. Defaults to None.
        """
        for try_count in range(3):
            try:
                query = self.session.query(object_class)
                if filter_clause is not None:
                    query = query.filter(filter_clause)
                query.delete(synchronize_session=False)
                self.session.commit()
                self.is_dirty = True
                return True
            except OperationalError:
                # This exception clause is for users running MySQL which likes to terminate connections on its own
                # without telling anyone. See bug #927473. However, other dbms can raise it, usually in a
                # non-recoverable way. So we only retry 3 times.
                log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
                self.session.rollback()
                if try_count >= 2:
                    raise
            except InvalidRequestError:
                self.session.rollback()
                log.exception(u'Failed to delete %s records', object_class.__name__)
                return False
            except:
                self.session.rollback()
                raise

    def finalise(self):
        """
        VACUUM the database on exit.
        """
        if self.is_dirty:
            engine = create_engine(self.db_url)
            if self.db_url.startswith(u'sqlite'):
                engine.execute("vacuum")