openlp/openlp/core/lib/db.py

494 lines
19 KiB
Python
Raw Normal View History

2010-05-28 00:26:49 +00:00
# -*- coding: utf-8 -*-
2012-12-28 22:06:43 +00:00
# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
2010-05-28 00:26:49 +00:00
###############################################################################
# OpenLP - Open Source Lyrics Projection #
# --------------------------------------------------------------------------- #
2012-12-29 20:56:56 +00:00
# Copyright (c) 2008-2013 Raoul Snyman #
# Portions copyright (c) 2008-2013 Tim Bentley, Gerald Britton, Jonathan #
# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub, #
2012-11-11 21:16:14 +00:00
# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer. #
2012-10-21 13:16:22 +00:00
# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru, #
# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith, #
# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock, #
# Frode Woldsund, Martin Zibricky, Patrick Zimmermann #
2010-05-28 00:26:49 +00:00
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it #
# under the terms of the GNU General Public License as published by the Free #
# Software Foundation; version 2 of the License. #
# #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
# more details. #
# #
# You should have received a copy of the GNU General Public License along #
# with this program; if not, write to the Free Software Foundation, Inc., 59 #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
###############################################################################
2012-10-21 13:16:22 +00:00
"""
The :mod:`db` module provides the core database functionality for OpenLP
"""
2010-06-12 23:00:14 +00:00
import logging
2010-06-18 01:26:01 +00:00
import os
2011-11-08 06:22:35 +00:00
from urllib import quote_plus as urlquote
2010-06-12 23:00:14 +00:00
from PyQt4 import QtCore
2011-08-25 09:02:59 +00:00
from sqlalchemy import Table, MetaData, Column, types, create_engine
2012-12-28 22:06:43 +00:00
from sqlalchemy.exc import SQLAlchemyError, InvalidRequestError, DBAPIError, OperationalError
2011-08-25 09:02:59 +00:00
from sqlalchemy.orm import scoped_session, sessionmaker, mapper
2011-05-27 07:54:49 +00:00
from sqlalchemy.pool import NullPool
2012-05-17 15:13:09 +00:00
from openlp.core.lib import translate, Settings
from openlp.core.lib.ui import critical_error_message_box
2011-01-14 18:58:47 +00:00
from openlp.core.utils import AppLocation, delete_file
2010-06-12 23:00:14 +00:00
log = logging.getLogger(__name__)
def init_db(url, auto_flush=True, auto_commit=False):
"""
Initialise and return the session and metadata for a database
``url``
The database to initialise connection with
``auto_flush``
Sets the flushing behaviour of the session
``auto_commit``
Sets the commit behaviour of the session
"""
2011-05-27 07:54:49 +00:00
engine = create_engine(url, poolclass=NullPool)
metadata = MetaData(bind=engine)
session = scoped_session(sessionmaker(autoflush=auto_flush,
autocommit=auto_commit, bind=engine))
return session, metadata
2010-05-28 00:26:49 +00:00
2011-08-25 09:02:59 +00:00
def upgrade_db(url, upgrade):
"""
Upgrade a database.
``url``
The url of the database to upgrade.
``upgrade``
The python module that contains the upgrade instructions.
"""
session, metadata = init_db(url)
2011-08-26 19:09:25 +00:00
class Metadata(BaseModel):
"""
Provides a class for the metadata table.
"""
pass
load_changes = False
tables = []
2011-08-27 18:43:05 +00:00
try:
tables = upgrade.upgrade_setup(metadata)
load_changes = True
except (SQLAlchemyError, DBAPIError):
pass
2011-08-25 09:02:59 +00:00
metadata_table = Table(u'metadata', metadata,
Column(u'key', types.Unicode(64), primary_key=True),
Column(u'value', types.UnicodeText(), default=None)
)
metadata_table.create(checkfirst=True)
mapper(Metadata, metadata_table)
version_meta = session.query(Metadata).get(u'version')
if version_meta is None:
version_meta = Metadata.populate(key=u'version', value=u'0')
session.add(version_meta)
2011-08-25 09:02:59 +00:00
version = 0
else:
version = int(version_meta.value)
if version > upgrade.__version__:
return version, upgrade.__version__
2011-08-25 09:02:59 +00:00
version += 1
2011-08-27 18:43:05 +00:00
if load_changes:
while hasattr(upgrade, u'upgrade_%d' % version):
log.debug(u'Running upgrade_%d', version)
try:
2012-12-28 22:06:43 +00:00
getattr(upgrade, u'upgrade_%d' % version) (session, metadata, tables)
except (SQLAlchemyError, DBAPIError):
2011-08-27 18:43:05 +00:00
log.exception(u'Could not run database upgrade script '
'"upgrade_%s", upgrade process has been halted.', version)
break
version_meta.value = unicode(version)
session.commit()
2011-08-27 18:43:05 +00:00
version += 1
else:
version_meta = Metadata.populate(key=u'version',
value=int(upgrade.__version__))
session.commit()
return int(version_meta.value), upgrade.__version__
2011-08-25 09:02:59 +00:00
2011-08-26 19:09:25 +00:00
2010-06-18 01:26:01 +00:00
def delete_database(plugin_name, db_file_name=None):
"""
Remove a database file from the system.
``plugin_name``
The name of the plugin to remove the database for
``db_file_name``
2011-02-25 17:05:01 +00:00
The database file name. Defaults to None resulting in the
2010-06-18 01:26:01 +00:00
plugin_name being used.
"""
db_file_path = None
if db_file_name:
2012-12-28 22:06:43 +00:00
db_file_path = os.path.join(AppLocation.get_section_data_path(plugin_name), db_file_name)
2010-06-18 01:26:01 +00:00
else:
2012-12-28 22:06:43 +00:00
db_file_path = os.path.join(AppLocation.get_section_data_path(plugin_name), plugin_name)
2011-01-14 18:58:47 +00:00
return delete_file(db_file_path)
2010-06-18 01:26:01 +00:00
2011-08-25 09:02:59 +00:00
2010-05-28 00:26:49 +00:00
class BaseModel(object):
"""
BaseModel provides a base object with a set of generic functions
"""
@classmethod
def populate(cls, **kwargs):
"""
Creates an instance of a class and populates it, returning the instance
"""
2011-02-07 22:07:48 +00:00
instance = cls()
2011-05-07 14:14:34 +00:00
for key, value in kwargs.iteritems():
instance.__setattr__(key, value)
2011-02-07 22:07:48 +00:00
return instance
2010-06-12 23:00:14 +00:00
class Manager(object):
"""
Provide generic object persistence management
"""
def __init__(self, plugin_name, init_schema, db_file_name=None,
upgrade_mod=None):
2010-06-12 23:00:14 +00:00
"""
Runs the initialisation process that includes creating the connection
to the database and the tables if they don't exist.
``plugin_name``
The name to setup paths and settings section names
2010-06-15 00:44:06 +00:00
``init_schema``
The init_schema function for this database
2011-08-25 09:02:59 +00:00
``upgrade_schema``
The upgrade_schema function for this database
2010-06-15 00:44:06 +00:00
``db_file_name``
2011-02-25 17:05:01 +00:00
The file name to use for this database. Defaults to None resulting
2010-06-15 00:44:06 +00:00
in the plugin_name being used.
2010-06-12 23:00:14 +00:00
"""
2012-05-17 15:13:09 +00:00
settings = Settings()
2010-06-12 23:00:14 +00:00
settings.beginGroup(plugin_name)
self.db_url = u''
2010-10-28 17:02:28 +00:00
self.is_dirty = False
2011-12-03 12:51:40 +00:00
self.session = None
2012-05-17 16:19:06 +00:00
db_type = settings.value(u'db type', u'sqlite')
2010-06-12 23:00:14 +00:00
if db_type == u'sqlite':
2010-06-15 00:44:06 +00:00
if db_file_name:
2012-12-28 22:06:43 +00:00
self.db_url = u'sqlite:///%s/%s' % (AppLocation.get_section_data_path(plugin_name), db_file_name)
2010-06-15 00:44:06 +00:00
else:
2012-12-28 22:06:43 +00:00
self.db_url = u'sqlite:///%s/%s.sqlite' % (AppLocation.get_section_data_path(plugin_name), plugin_name)
2010-06-12 23:00:14 +00:00
else:
self.db_url = u'%s://%s:%s@%s/%s' % (db_type,
2012-05-17 16:19:06 +00:00
urlquote(settings.value(u'db username', u'')),
urlquote(settings.value(u'db password', u'')),
urlquote(settings.value(u'db hostname', u'')),
urlquote(settings.value(u'db database', u'')))
if db_type == u'mysql':
2012-05-17 16:19:06 +00:00
db_encoding = settings.value(u'db encoding', u'utf8')
self.db_url += u'?charset=%s' % urlquote(db_encoding)
2010-06-12 23:00:14 +00:00
settings.endGroup()
if upgrade_mod:
db_ver, up_ver = upgrade_db(self.db_url, upgrade_mod)
if db_ver > up_ver:
critical_error_message_box(
translate('OpenLP.Manager', 'Database Error'),
2012-05-17 18:57:01 +00:00
translate('OpenLP.Manager', 'The database being '
'loaded was created in a more recent version of '
'OpenLP. The database is version %d, while OpenLP '
'expects version %d. The database will not be loaded.'
2012-05-17 18:57:01 +00:00
'\n\nDatabase: %s') % \
(db_ver, up_ver, self.db_url)
)
return
try:
self.session = init_schema(self.db_url)
except (SQLAlchemyError, DBAPIError):
log.exception(u'Error loading database: %s', self.db_url)
2012-12-28 22:06:43 +00:00
critical_error_message_box(translate('OpenLP.Manager', 'Database Error'),
translate('OpenLP.Manager', 'OpenLP cannot load your database.\n\nDatabase: %s') % self.db_url
)
2010-06-12 23:00:14 +00:00
2010-07-20 15:22:05 +00:00
def save_object(self, object_instance, commit=True):
2010-06-12 23:00:14 +00:00
"""
Save an object to the database
``object_instance``
The object to save
2010-07-20 15:22:05 +00:00
``commit``
Commit the session with this object
2010-06-12 23:00:14 +00:00
"""
for try_count in range(3):
try:
self.session.add(object_instance)
if commit:
self.session.commit()
self.is_dirty = True
return True
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue - "MySQL has gone away"')
self.session.rollback()
if try_count >= 2:
raise
except InvalidRequestError:
self.session.rollback()
log.exception(u'Object list save failed')
return False
except:
self.session.rollback()
raise
2010-06-12 23:00:14 +00:00
2010-11-16 06:19:23 +00:00
def save_objects(self, object_list, commit=True):
"""
Save a list of objects to the database
``object_list``
The list of objects to save
``commit``
Commit the session with this object
"""
for try_count in range(3):
try:
self.session.add_all(object_list)
if commit:
self.session.commit()
self.is_dirty = True
return True
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
self.session.rollback()
if try_count >= 2:
raise
except InvalidRequestError:
self.session.rollback()
log.exception(u'Object list save failed')
return False
except:
self.session.rollback()
raise
2010-11-16 06:19:23 +00:00
2010-06-18 01:26:01 +00:00
def get_object(self, object_class, key=None):
2010-06-12 23:00:14 +00:00
"""
Return the details of an object
``object_class``
The type of object to return
2010-06-18 01:26:01 +00:00
``key``
The unique reference or primary key for the instance to return
2010-06-12 23:00:14 +00:00
"""
2010-06-18 01:26:01 +00:00
if not key:
2010-06-12 23:00:14 +00:00
return object_class()
else:
for try_count in range(3):
try:
return self.session.query(object_class).get(key)
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
if try_count >= 2:
raise
2010-06-12 23:00:14 +00:00
2010-06-30 22:05:51 +00:00
def get_object_filtered(self, object_class, filter_clause):
2010-06-15 18:08:02 +00:00
"""
Returns an object matching specified criteria
``object_class``
The type of object to return
2010-06-30 22:05:51 +00:00
``filter_clause``
2010-06-15 18:08:02 +00:00
The criteria to select the object by
"""
for try_count in range(3):
try:
return self.session.query(object_class).filter(filter_clause).first()
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
if try_count >= 2:
raise
2010-06-15 18:08:02 +00:00
2010-07-18 23:37:24 +00:00
def get_all_objects(self, object_class, filter_clause=None,
2010-06-30 22:05:51 +00:00
order_by_ref=None):
2010-06-15 02:08:22 +00:00
"""
2010-07-18 23:37:24 +00:00
Returns all the objects from the database
2010-06-15 02:08:22 +00:00
``object_class``
The type of objects to return
2010-06-30 22:05:51 +00:00
``filter_clause``
2011-02-25 17:05:01 +00:00
The filter governing selection of objects to return. Defaults to
2010-07-18 23:37:24 +00:00
None.
2010-06-30 22:05:51 +00:00
``order_by_ref``
2011-02-25 17:05:01 +00:00
Any parameters to order the returned objects by. Defaults to None.
2010-06-15 02:08:22 +00:00
"""
2010-07-18 23:37:24 +00:00
query = self.session.query(object_class)
2010-08-02 02:53:14 +00:00
if filter_clause is not None:
2010-07-18 23:37:24 +00:00
query = query.filter(filter_clause)
if isinstance(order_by_ref, list):
query = query.order_by(*order_by_ref)
elif order_by_ref is not None:
query = query.order_by(order_by_ref)
for try_count in range(3):
try:
return query.all()
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
if try_count >= 2:
raise
2010-06-15 02:08:22 +00:00
2010-11-16 06:19:23 +00:00
def get_object_count(self, object_class, filter_clause=None):
"""
Returns a count of the number of objects in the database.
``object_class``
The type of objects to return.
``filter_clause``
2011-02-25 17:05:01 +00:00
The filter governing selection of objects to return. Defaults to
2010-11-16 06:19:23 +00:00
None.
"""
query = self.session.query(object_class)
if filter_clause is not None:
query = query.filter(filter_clause)
for try_count in range(3):
try:
return query.count()
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
if try_count >= 2:
raise
2010-11-16 06:19:23 +00:00
2010-06-18 01:26:01 +00:00
def delete_object(self, object_class, key):
2010-06-12 23:00:14 +00:00
"""
Delete an object from the database
``object_class``
The type of object to delete
2010-06-18 01:26:01 +00:00
``key``
The unique reference or primary key for the instance to be deleted
2010-06-12 23:00:14 +00:00
"""
2010-06-18 01:26:01 +00:00
if key != 0:
object_instance = self.get_object(object_class, key)
for try_count in range(3):
try:
self.session.delete(object_instance)
self.session.commit()
self.is_dirty = True
return True
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
self.session.rollback()
if try_count >= 2:
raise
except InvalidRequestError:
self.session.rollback()
log.exception(u'Failed to delete object')
return False
except:
self.session.rollback()
raise
2010-06-12 23:00:14 +00:00
else:
return True
2010-07-18 23:37:24 +00:00
def delete_all_objects(self, object_class, filter_clause=None):
2010-06-12 23:00:14 +00:00
"""
2011-12-04 16:36:18 +00:00
Delete all object records.
This method should only be used for simple tables and not ones with
relationships. The relationships are not deleted from the database and
this will lead to database corruptions.
2010-06-12 23:00:14 +00:00
``object_class``
The type of object to delete
2011-12-03 17:01:57 +00:00
2011-12-03 09:05:01 +00:00
``filter_clause``
The filter governing selection of objects to return. Defaults to
2011-12-03 17:01:57 +00:00
None.
2010-06-12 23:00:14 +00:00
"""
for try_count in range(3):
try:
query = self.session.query(object_class)
if filter_clause is not None:
query = query.filter(filter_clause)
query.delete(synchronize_session=False)
self.session.commit()
self.is_dirty = True
return True
except OperationalError:
# This exception clause is for users running MySQL which likes
# to terminate connections on its own without telling anyone.
# See bug #927473
# However, other dbms can raise it, usually in a non-recoverable
# way. So we only retry 3 times.
log.exception(u'Probably a MySQL issue, "MySQL has gone away"')
self.session.rollback()
if try_count >= 2:
raise
except InvalidRequestError:
self.session.rollback()
log.exception(u'Failed to delete %s records', object_class.__name__)
return False
except:
self.session.rollback()
raise
def finalise(self):
"""
VACUUM the database on exit.
"""
2010-10-28 17:02:28 +00:00
if self.is_dirty:
engine = create_engine(self.db_url)
2010-11-28 15:20:53 +00:00
if self.db_url.startswith(u'sqlite'):
2011-01-14 18:58:47 +00:00
engine.execute("vacuum")