merge w/ trunk

openlp/core/__init__.py

@@ -27,26 +27,26 @@ All the core functions of the OpenLP application including the GUI, settings,
 logging and a plugin framework are contained within the openlp.core module.
 """
 
-import os
-import sys
-import logging
 import argparse
-from traceback import format_exception
+import logging
+import os
 import shutil
+import sys
 import time
+from traceback import format_exception
+
 from PyQt5 import QtCore, QtGui, QtWidgets
 
 from openlp.core.common import Registry, OpenLPMixin, AppLocation, LanguageManager, Settings, UiStrings, \
     check_directory_exists, is_macosx, is_win, translate
+from openlp.core.common.versionchecker import VersionThread, get_application_version
 from openlp.core.lib import ScreenList
 from openlp.core.resources import qInitResources
-from openlp.core.ui.mainwindow import MainWindow
-from openlp.core.ui.firsttimelanguageform import FirstTimeLanguageForm
-from openlp.core.ui.firsttimeform import FirstTimeForm
-from openlp.core.ui.exceptionform import ExceptionForm
 from openlp.core.ui import SplashScreen
-from openlp.core.utils import VersionThread, get_application_version
+from openlp.core.ui.exceptionform import ExceptionForm
-
+from openlp.core.ui.firsttimeform import FirstTimeForm
+from openlp.core.ui.firsttimelanguageform import FirstTimeLanguageForm
+from openlp.core.ui.mainwindow import MainWindow
 
 __all__ = ['OpenLP', 'main']
 

openlp/core/common/__init__.py

@@ -30,8 +30,9 @@ import re
 import sys
 import traceback
 from ipaddress import IPv4Address, IPv6Address, AddressValueError
+from shutil import which
 
-from PyQt5 import QtCore
+from PyQt5 import QtCore, QtGui
 from PyQt5.QtCore import QCryptographicHash as QHash
 
 log = logging.getLogger(__name__ + '.__init__')
@@ -39,6 +40,9 @@ log = logging.getLogger(__name__ + '.__init__')
 
 FIRST_CAMEL_REGEX = re.compile('(.)([A-Z][a-z]+)')
 SECOND_CAMEL_REGEX = re.compile('([a-z0-9])([A-Z])')
+CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]', re.UNICODE)
+INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]', re.UNICODE)
+IMAGES_FILTER = None
 
 
 def trace_error_handler(logger):
@@ -257,3 +261,113 @@ def add_actions(target, actions):
             target.addSeparator()
         else:
             target.addAction(action)
+
+
+def get_uno_command(connection_type='pipe'):
+    """
+    Returns the UNO command to launch an libreoffice.org instance.
+    """
+    for command in ['libreoffice', 'soffice']:
+        if which(command):
+            break
+    else:
+        raise FileNotFoundError('Command not found')
+
+    OPTIONS = '--nologo --norestore --minimized --nodefault --nofirststartwizard'
+    if connection_type == 'pipe':
+        CONNECTION = '"--accept=pipe,name=openlp_pipe;urp;"'
+    else:
+        CONNECTION = '"--accept=socket,host=localhost,port=2002;urp;"'
+    return '%s %s %s' % (command, OPTIONS, CONNECTION)
+
+
+def get_uno_instance(resolver, connection_type='pipe'):
+    """
+    Returns a running libreoffice.org instance.
+
+    :param resolver: The UNO resolver to use to find a running instance.
+    """
+    log.debug('get UNO Desktop Openoffice - resolve')
+    if connection_type == 'pipe':
+        return resolver.resolve('uno:pipe,name=openlp_pipe;urp;StarOffice.ComponentContext')
+    else:
+        return resolver.resolve('uno:socket,host=localhost,port=2002;urp;StarOffice.ComponentContext')
+
+
+def get_filesystem_encoding():
+    """
+    Returns the name of the encoding used to convert Unicode filenames into system file names.
+    """
+    encoding = sys.getfilesystemencoding()
+    if encoding is None:
+        encoding = sys.getdefaultencoding()
+    return encoding
+
+
+def split_filename(path):
+    """
+    Return a list of the parts in a given path.
+    """
+    path = os.path.abspath(path)
+    if not os.path.isfile(path):
+        return path, ''
+    else:
+        return os.path.split(path)
+
+
+def delete_file(file_path_name):
+    """
+    Deletes a file from the system.
+
+    :param file_path_name: The file, including path, to delete.
+    """
+    if not file_path_name:
+        return False
+    try:
+        if os.path.exists(file_path_name):
+            os.remove(file_path_name)
+        return True
+    except (IOError, OSError):
+        log.exception("Unable to delete file %s" % file_path_name)
+        return False
+
+
+def get_images_filter():
+    """
+    Returns a filter string for a file dialog containing all the supported image formats.
+    """
+    global IMAGES_FILTER
+    if not IMAGES_FILTER:
+        log.debug('Generating images filter.')
+        formats = list(map(bytes.decode, list(map(bytes, QtGui.QImageReader.supportedImageFormats()))))
+        visible_formats = '(*.%s)' % '; *.'.join(formats)
+        actual_formats = '(*.%s)' % ' *.'.join(formats)
+        IMAGES_FILTER = '%s %s %s' % (translate('OpenLP', 'Image Files'), visible_formats, actual_formats)
+    return IMAGES_FILTER
+
+
+def is_not_image_file(file_name):
+    """
+    Validate that the file is not an image file.
+
+    :param file_name: File name to be checked.
+    """
+    if not file_name:
+        return True
+    else:
+        formats = [bytes(fmt).decode().lower() for fmt in QtGui.QImageReader.supportedImageFormats()]
+        file_part, file_extension = os.path.splitext(str(file_name))
+        if file_extension[1:].lower() in formats and os.path.exists(file_name):
+            return False
+        return True
+
+
+def clean_filename(filename):
+    """
+    Removes invalid characters from the given ``filename``.
+
+    :param filename:  The "dirty" file name to clean.
+    """
+    if not isinstance(filename, str):
+        filename = str(filename, 'utf-8')
+    return INVALID_FILE_CHARS.sub('_', CONTROL_CHARS.sub('', filename))

openlp/core/common/languagemanager.py

@@ -22,9 +22,9 @@
 """
 The :mod:`languagemanager` module provides all the translation settings and language file loading for OpenLP.
 """
+import locale
 import logging
 import re
-import sys
 
 from PyQt5 import QtCore, QtWidgets
 
@@ -33,6 +33,9 @@ from openlp.core.common import AppLocation, Settings, translate, is_win, is_maco
 
 log = logging.getLogger(__name__)
 
+ICU_COLLATOR = None
+DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+', re.UNICODE)
+
 
 class LanguageManager(object):
     """
@@ -144,3 +147,60 @@ class LanguageManager(object):
         if not LanguageManager.__qm_list__:
             LanguageManager.init_qm_list()
         return LanguageManager.__qm_list__
+
+
+def format_time(text, local_time):
+    """
+    Workaround for Python built-in time formatting function time.strftime().
+
+    time.strftime() accepts only ascii characters. This function accepts
+    unicode string and passes individual % placeholders to time.strftime().
+    This ensures only ascii characters are passed to time.strftime().
+
+    :param text:  The text to be processed.
+    :param local_time: The time to be used to add to the string.  This is a time object
+    """
+
+    def match_formatting(match):
+        """
+        Format the match
+        """
+        return local_time.strftime(match.group())
+
+    return re.sub('\%[a-zA-Z]', match_formatting, text)
+
+
+def get_locale_key(string):
+    """
+    Creates a key for case insensitive, locale aware string sorting.
+
+    :param string: The corresponding string.
+    """
+    string = string.lower()
+    # ICU is the prefered way to handle locale sort key, we fallback to locale.strxfrm which will work in most cases.
+    global ICU_COLLATOR
+    try:
+        if ICU_COLLATOR is None:
+            import icu
+            language = LanguageManager.get_language()
+            icu_locale = icu.Locale(language)
+            ICU_COLLATOR = icu.Collator.createInstance(icu_locale)
+        return ICU_COLLATOR.getSortKey(string)
+    except:
+        return locale.strxfrm(string).encode()
+
+
+def get_natural_key(string):
+    """
+    Generate a key for locale aware natural string sorting.
+
+    :param string: string to be sorted by
+    Returns a list of string compare keys and integers.
+    """
+    key = DIGITS_OR_NONDIGITS.findall(string)
+    key = [int(part) if part.isdigit() else get_locale_key(part) for part in key]
+    # Python 3 does not support comparison of different types anymore. So make sure, that we do not compare str
+    # and int.
+    if string and string[0].isdigit():
+        return [b''] + key
+    return key

openlp/core/common/versionchecker.py

@@ -0,0 +1,170 @@
+import logging
+import os
+import platform
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from datetime import datetime
+from distutils.version import LooseVersion
+from subprocess import Popen, PIPE
+
+from openlp.core.common import AppLocation, Settings
+
+from PyQt5 import QtCore
+
+log = logging.getLogger(__name__)
+
+APPLICATION_VERSION = {}
+CONNECTION_TIMEOUT = 30
+CONNECTION_RETRIES = 2
+
+
+class VersionThread(QtCore.QThread):
+    """
+    A special Qt thread class to fetch the version of OpenLP from the website.
+    This is threaded so that it doesn't affect the loading time of OpenLP.
+    """
+    def __init__(self, main_window):
+        """
+        Constructor for the thread class.
+
+        :param main_window: The main window Object.
+        """
+        log.debug("VersionThread - Initialise")
+        super(VersionThread, self).__init__(None)
+        self.main_window = main_window
+
+    def run(self):
+        """
+        Run the thread.
+        """
+        self.sleep(1)
+        log.debug('Version thread - run')
+        app_version = get_application_version()
+        version = check_latest_version(app_version)
+        log.debug("Versions %s and %s " % (LooseVersion(str(version)), LooseVersion(str(app_version['full']))))
+        if LooseVersion(str(version)) > LooseVersion(str(app_version['full'])):
+            self.main_window.openlp_version_check.emit('%s' % version)
+
+
+def get_application_version():
+    """
+    Returns the application version of the running instance of OpenLP::
+
+        {'full': '1.9.4-bzr1249', 'version': '1.9.4', 'build': 'bzr1249'}
+    """
+    global APPLICATION_VERSION
+    if APPLICATION_VERSION:
+        return APPLICATION_VERSION
+    if '--dev-version' in sys.argv or '-d' in sys.argv:
+        # NOTE: The following code is a duplicate of the code in setup.py. Any fix applied here should also be applied
+        # there.
+
+        # Get the revision of this tree.
+        bzr = Popen(('bzr', 'revno'), stdout=PIPE)
+        tree_revision, error = bzr.communicate()
+        tree_revision = tree_revision.decode()
+        code = bzr.wait()
+        if code != 0:
+            raise Exception('Error running bzr log')
+
+        # Get all tags.
+        bzr = Popen(('bzr', 'tags'), stdout=PIPE)
+        output, error = bzr.communicate()
+        code = bzr.wait()
+        if code != 0:
+            raise Exception('Error running bzr tags')
+        tags = list(map(bytes.decode, output.splitlines()))
+        if not tags:
+            tag_version = '0.0.0'
+            tag_revision = '0'
+        else:
+            # Remove any tag that has "?" as revision number. A "?" as revision number indicates, that this tag is from
+            # another series.
+            tags = [tag for tag in tags if tag.split()[-1].strip() != '?']
+            # Get the last tag and split it in a revision and tag name.
+            tag_version, tag_revision = tags[-1].split()
+        # If they are equal, then this tree is tarball with the source for the release. We do not want the revision
+        # number in the full version.
+        if tree_revision == tag_revision:
+            full_version = tag_version.strip()
+        else:
+            full_version = '%s-bzr%s' % (tag_version.strip(), tree_revision.strip())
+    else:
+        # We're not running the development version, let's use the file.
+        file_path = AppLocation.get_directory(AppLocation.VersionDir)
+        file_path = os.path.join(file_path, '.version')
+        version_file = None
+        try:
+            version_file = open(file_path, 'r')
+            full_version = str(version_file.read()).rstrip()
+        except IOError:
+            log.exception('Error in version file.')
+            full_version = '0.0.0-bzr000'
+        finally:
+            if version_file:
+                version_file.close()
+    bits = full_version.split('-')
+    APPLICATION_VERSION = {
+        'full': full_version,
+        'version': bits[0],
+        'build': bits[1] if len(bits) > 1 else None
+    }
+    if APPLICATION_VERSION['build']:
+        log.info('Openlp version %s build %s', APPLICATION_VERSION['version'], APPLICATION_VERSION['build'])
+    else:
+        log.info('Openlp version %s' % APPLICATION_VERSION['version'])
+    return APPLICATION_VERSION
+
+
+def check_latest_version(current_version):
+    """
+    Check the latest version of OpenLP against the version file on the OpenLP
+    site.
+
+    **Rules around versions and version files:**
+
+    * If a version number has a build (i.e. -bzr1234), then it is a nightly.
+    * If a version number's minor version is an odd number, it is a development release.
+    * If a version number's minor version is an even number, it is a stable release.
+
+    :param current_version: The current version of OpenLP.
+    """
+    version_string = current_version['full']
+    # set to prod in the distribution config file.
+    settings = Settings()
+    settings.beginGroup('core')
+    last_test = settings.value('last version test')
+    this_test = str(datetime.now().date())
+    settings.setValue('last version test', this_test)
+    settings.endGroup()
+    if last_test != this_test:
+        if current_version['build']:
+            req = urllib.request.Request('http://www.openlp.org/files/nightly_version.txt')
+        else:
+            version_parts = current_version['version'].split('.')
+            if int(version_parts[1]) % 2 != 0:
+                req = urllib.request.Request('http://www.openlp.org/files/dev_version.txt')
+            else:
+                req = urllib.request.Request('http://www.openlp.org/files/version.txt')
+        req.add_header('User-Agent', 'OpenLP/%s %s/%s; ' % (current_version['full'], platform.system(),
+                                                            platform.release()))
+        remote_version = None
+        retries = 0
+        while True:
+            try:
+                remote_version = str(urllib.request.urlopen(req, None,
+                                                            timeout=CONNECTION_TIMEOUT).read().decode()).strip()
+            except (urllib.error.URLError, ConnectionError):
+                if retries > CONNECTION_RETRIES:
+                    log.exception('Failed to download the latest OpenLP version file')
+                else:
+                    retries += 1
+                    time.sleep(0.1)
+                    continue
+            break
+        if remote_version:
+            version_string = remote_version
+    return version_string

openlp/core/lib/db.py

@@ -34,9 +34,8 @@ from sqlalchemy.pool import NullPool
 from alembic.migration import MigrationContext
 from alembic.operations import Operations
 
-from openlp.core.common import AppLocation, Settings, translate
+from openlp.core.common import AppLocation, Settings, translate, delete_file
 from openlp.core.lib.ui import critical_error_message_box
-from openlp.core.utils import delete_file
 
 log = logging.getLogger(__name__)
 

openlp/core/lib/plugin.py

@@ -24,11 +24,10 @@ Provide the generic plugin functionality for OpenLP plugins.
 """
 import logging
 
-
 from PyQt5 import QtCore
 
 from openlp.core.common import Registry, RegistryProperties, Settings, UiStrings
-from openlp.core.utils import get_application_version
+from openlp.core.common.versionchecker import get_application_version
 
 log = logging.getLogger(__name__)
 

openlp/core/lib/webpagereader.py

@@ -0,0 +1,183 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection                                      #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2016 OpenLP Developers                                   #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; version 2 of the License.                              #
+#                                                                             #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with this program; if not, write to the Free Software Foundation, Inc., 59  #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
+###############################################################################
+"""
+The :mod:`openlp.core.utils` module provides the utility libraries for OpenLP.
+"""
+import logging
+import socket
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from http.client import HTTPException
+from random import randint
+
+from openlp.core.common import Registry
+
+log = logging.getLogger(__name__ + '.__init__')
+
+USER_AGENTS = {
+    'win32': [
+        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36'
+    ],
+    'darwin': [
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) '
+        'Chrome/26.0.1410.43 Safari/537.31',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.11 (KHTML, like Gecko) '
+        'Chrome/20.0.1132.57 Safari/536.11',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/536.11 (KHTML, like Gecko) '
+        'Chrome/20.0.1132.47 Safari/536.11',
+    ],
+    'linux2': [
+        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 '
+        'Chrome/25.0.1364.160 Safari/537.22',
+        'Mozilla/5.0 (X11; CrOS armv7l 2913.260.0) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.99 '
+        'Safari/537.11',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.27 (KHTML, like Gecko) Chrome/26.0.1389.0 Safari/537.27'
+    ],
+    'default': [
+        'Mozilla/5.0 (X11; NetBSD amd64; rv:18.0) Gecko/20130120 Firefox/18.0'
+    ]
+}
+CONNECTION_TIMEOUT = 30
+CONNECTION_RETRIES = 2
+
+
+class HTTPRedirectHandlerFixed(urllib.request.HTTPRedirectHandler):
+    """
+    Special HTTPRedirectHandler used to work around http://bugs.python.org/issue22248
+    (Redirecting to urls with special chars)
+    """
+    def redirect_request(self, req, fp, code, msg, headers, new_url):
+        #
+        """
+        Test if the new_url can be decoded to ascii
+
+        :param req:
+        :param fp:
+        :param code:
+        :param msg:
+        :param headers:
+        :param new_url:
+        :return:
+        """
+        try:
+            new_url.encode('latin1').decode('ascii')
+            fixed_url = new_url
+        except Exception:
+            # The url could not be decoded to ascii, so we do some url encoding
+            fixed_url = urllib.parse.quote(new_url.encode('latin1').decode('utf-8', 'replace'), safe='/:')
+        return super(HTTPRedirectHandlerFixed, self).redirect_request(req, fp, code, msg, headers, fixed_url)
+
+
+def _get_user_agent():
+    """
+    Return a user agent customised for the platform the user is on.
+    """
+    browser_list = USER_AGENTS.get(sys.platform, None)
+    if not browser_list:
+        browser_list = USER_AGENTS['default']
+    random_index = randint(0, len(browser_list) - 1)
+    return browser_list[random_index]
+
+
+def get_web_page(url, header=None, update_openlp=False):
+    """
+    Attempts to download the webpage at url and returns that page or None.
+
+    :param url: The URL to be downloaded.
+    :param header:  An optional HTTP header to pass in the request to the web server.
+    :param update_openlp: Tells OpenLP to update itself if the page is successfully downloaded.
+        Defaults to False.
+    """
+    # TODO: Add proxy usage. Get proxy info from OpenLP settings, add to a
+    # proxy_handler, build into an opener and install the opener into urllib2.
+    # http://docs.python.org/library/urllib2.html
+    if not url:
+        return None
+    # This is needed to work around http://bugs.python.org/issue22248 and https://bugs.launchpad.net/openlp/+bug/1251437
+    opener = urllib.request.build_opener(HTTPRedirectHandlerFixed())
+    urllib.request.install_opener(opener)
+    req = urllib.request.Request(url)
+    if not header or header[0].lower() != 'user-agent':
+        user_agent = _get_user_agent()
+        req.add_header('User-Agent', user_agent)
+    if header:
+        req.add_header(header[0], header[1])
+    log.debug('Downloading URL = %s' % url)
+    retries = 0
+    while retries <= CONNECTION_RETRIES:
+        retries += 1
+        time.sleep(0.1)
+        try:
+            page = urllib.request.urlopen(req, timeout=CONNECTION_TIMEOUT)
+            log.debug('Downloaded page {}'.format(page.geturl()))
+            break
+        except urllib.error.URLError as err:
+            log.exception('URLError on {}'.format(url))
+            log.exception('URLError: {}'.format(err.reason))
+            page = None
+            if retries > CONNECTION_RETRIES:
+                raise
+        except socket.timeout:
+            log.exception('Socket timeout: {}'.format(url))
+            page = None
+            if retries > CONNECTION_RETRIES:
+                raise
+        except socket.gaierror:
+            log.exception('Socket gaierror: {}'.format(url))
+            page = None
+            if retries > CONNECTION_RETRIES:
+                raise
+        except ConnectionRefusedError:
+            log.exception('ConnectionRefused: {}'.format(url))
+            page = None
+            if retries > CONNECTION_RETRIES:
+                raise
+            break
+        except ConnectionError:
+            log.exception('Connection error: {}'.format(url))
+            page = None
+            if retries > CONNECTION_RETRIES:
+                raise
+        except HTTPException:
+            log.exception('HTTPException error: {}'.format(url))
+            page = None
+            if retries > CONNECTION_RETRIES:
+                raise
+        except:
+            # Don't know what's happening, so reraise the original
+            raise
+    if update_openlp:
+        Registry().get('application').process_events()
+    if not page:
+        log.exception('{} could not be downloaded'.format(url))
+        return None
+    log.debug(page)
+    return page
+
+
+__all__ = ['get_application_version', 'check_latest_version',
+           'get_web_page']

openlp/core/ui/aboutform.py

@@ -26,8 +26,8 @@ import webbrowser
 
 from PyQt5 import QtCore, QtWidgets
 
+from openlp.core.common.versionchecker import get_application_version
 from openlp.core.lib import translate
-from openlp.core.utils import get_application_version
 from .aboutdialog import UiAboutDialog
 
 

openlp/core/ui/advancedtab.py

@@ -29,9 +29,9 @@ import sys
 
 from PyQt5 import QtCore, QtGui, QtWidgets
 
-from openlp.core.common import AppLocation, Settings, SlideLimits, UiStrings, translate
+from openlp.core.common import AppLocation, Settings, SlideLimits, UiStrings, translate, get_images_filter
 from openlp.core.lib import ColorButton, SettingsTab, build_icon
-from openlp.core.utils import format_time, get_images_filter
+from openlp.core.common.languagemanager import format_time
 
 log = logging.getLogger(__name__)
 

openlp/core/ui/exceptionform.py

@@ -23,18 +23,17 @@
 The actual exception dialog form.
 """
 import logging
-import re
 import os
 import platform
+import re
 
 import bs4
 import sqlalchemy
+from PyQt5 import Qt, QtCore, QtGui, QtWebKit, QtWidgets
 from lxml import etree
 
 from openlp.core.common import RegistryProperties, is_linux
 
-from PyQt5 import Qt, QtCore, QtGui, QtWebKit, QtWidgets
-
 try:
     import migrate
     MIGRATE_VERSION = getattr(migrate, '__version__', '< 0.7')
@@ -74,7 +73,7 @@ except ImportError:
     VLC_VERSION = '-'
 
 from openlp.core.common import Settings, UiStrings, translate
-from openlp.core.utils import get_application_version
+from openlp.core.common.versionchecker import get_application_version
 
 from .exceptiondialog import Ui_ExceptionDialog
 

openlp/core/ui/firsttimeform.py

@@ -39,7 +39,7 @@ from openlp.core.common import Registry, RegistryProperties, AppLocation, Settin
     translate, clean_button_text, trace_error_handler
 from openlp.core.lib import PluginStatus, build_icon
 from openlp.core.lib.ui import critical_error_message_box
-from openlp.core.utils import get_web_page, CONNECTION_RETRIES, CONNECTION_TIMEOUT
+from openlp.core.lib.webpagereader import get_web_page, CONNECTION_RETRIES, CONNECTION_TIMEOUT
 from .firsttimewizard import UiFirstTimeWizard, FirstTimePage
 
 log = logging.getLogger(__name__)

openlp/core/ui/mainwindow.py

@@ -37,6 +37,7 @@ from PyQt5 import QtCore, QtGui, QtWidgets
 from openlp.core.common import Registry, RegistryProperties, AppLocation, LanguageManager, Settings, \
     check_directory_exists, translate, is_win, is_macosx, add_actions
 from openlp.core.common.actions import ActionList, CategoryOrder
+from openlp.core.common.versionchecker import get_application_version
 from openlp.core.lib import Renderer, OpenLPDockWidget, PluginManager, ImageManager, PluginStatus, ScreenList, \
     build_icon
 from openlp.core.lib.ui import UiStrings, create_action
@@ -46,7 +47,6 @@ from openlp.core.ui.firsttimeform import FirstTimeForm
 from openlp.core.ui.media import MediaController
 from openlp.core.ui.printserviceform import PrintServiceForm
 from openlp.core.ui.projector.manager import ProjectorManager
-from openlp.core.utils import get_application_version
 
 log = logging.getLogger(__name__)
 

openlp/core/ui/servicemanager.py

@@ -33,12 +33,12 @@ from tempfile import mkstemp
 from PyQt5 import QtCore, QtGui, QtWidgets
 
 from openlp.core.common import Registry, RegistryProperties, AppLocation, Settings, ThemeLevel, OpenLPMixin, \
-    RegistryMixin, check_directory_exists, UiStrings, translate
+    RegistryMixin, check_directory_exists, UiStrings, translate, split_filename, delete_file
 from openlp.core.common.actions import ActionList, CategoryOrder
 from openlp.core.lib import OpenLPToolbar, ServiceItem, ItemCapabilities, PluginStatus, build_icon
 from openlp.core.lib.ui import critical_error_message_box, create_widget_action, find_and_set_in_combo_box
 from openlp.core.ui import ServiceNoteForm, ServiceItemEditForm, StartTimeForm
-from openlp.core.utils import delete_file, split_filename, format_time
+from openlp.core.common.languagemanager import format_time
 
 
 class ServiceManagerList(QtWidgets.QTreeWidget):

openlp/core/ui/themeform.py

@@ -27,11 +27,10 @@ import os
 
 from PyQt5 import QtCore, QtGui, QtWidgets
 
-from openlp.core.common import Registry, RegistryProperties, UiStrings, translate
+from openlp.core.common import Registry, RegistryProperties, UiStrings, translate, get_images_filter, is_not_image_file
 from openlp.core.lib.theme import BackgroundType, BackgroundGradientType
 from openlp.core.lib.ui import critical_error_message_box
 from openlp.core.ui import ThemeLayoutForm
-from openlp.core.utils import get_images_filter, is_not_image_file
 from .themewizard import Ui_ThemeWizard
 
 log = logging.getLogger(__name__)

openlp/core/ui/thememanager.py

@@ -30,13 +30,13 @@ from xml.etree.ElementTree import ElementTree, XML
 from PyQt5 import QtCore, QtGui, QtWidgets
 
 from openlp.core.common import Registry, RegistryProperties, AppLocation, Settings, OpenLPMixin, RegistryMixin, \
-    check_directory_exists, UiStrings, translate, is_win
+    check_directory_exists, UiStrings, translate, is_win, get_filesystem_encoding, delete_file
 from openlp.core.lib import FileDialog, ImageSource, OpenLPToolbar, ValidationError, get_text_file_string, build_icon, \
     check_item_selected, create_thumb, validate_thumb
 from openlp.core.lib.theme import ThemeXML, BackgroundType
 from openlp.core.lib.ui import critical_error_message_box, create_widget_action
 from openlp.core.ui import FileRenameForm, ThemeForm
-from openlp.core.utils import delete_file, get_locale_key, get_filesystem_encoding
+from openlp.core.common.languagemanager import get_locale_key
 
 
 class Ui_ThemeManager(object):

openlp/core/utils/__init__.py

@@ -1,524 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
-
-###############################################################################
-# OpenLP - Open Source Lyrics Projection                                      #
-# --------------------------------------------------------------------------- #
-# Copyright (c) 2008-2016 OpenLP Developers                                   #
-# --------------------------------------------------------------------------- #
-# This program is free software; you can redistribute it and/or modify it     #
-# under the terms of the GNU General Public License as published by the Free  #
-# Software Foundation; version 2 of the License.                              #
-#                                                                             #
-# This program is distributed in the hope that it will be useful, but WITHOUT #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
-# more details.                                                               #
-#                                                                             #
-# You should have received a copy of the GNU General Public License along     #
-# with this program; if not, write to the Free Software Foundation, Inc., 59  #
-# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
-###############################################################################
-"""
-The :mod:`openlp.core.utils` module provides the utility libraries for OpenLP.
-"""
-import locale
-import logging
-import os
-import platform
-import re
-import socket
-import sys
-import time
-import urllib.error
-import urllib.parse
-import urllib.request
-from datetime import datetime
-from distutils.version import LooseVersion
-from http.client import HTTPException
-from random import randint
-from shutil import which
-from subprocess import Popen, PIPE
-
-from PyQt5 import QtGui, QtCore
-
-from openlp.core.common import Registry, AppLocation, Settings, is_win, is_macosx
-
-if not is_win() and not is_macosx():
-    try:
-        from xdg import BaseDirectory
-        XDG_BASE_AVAILABLE = True
-    except ImportError:
-        BaseDirectory = None
-        XDG_BASE_AVAILABLE = False
-
-from openlp.core.common import translate
-
-log = logging.getLogger(__name__ + '.__init__')
-
-APPLICATION_VERSION = {}
-IMAGES_FILTER = None
-ICU_COLLATOR = None
-CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]', re.UNICODE)
-INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]', re.UNICODE)
-DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+', re.UNICODE)
-USER_AGENTS = {
-    'win32': [
-        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
-        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
-        'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36'
-    ],
-    'darwin': [
-        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) '
-        'Chrome/26.0.1410.43 Safari/537.31',
-        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.11 (KHTML, like Gecko) '
-        'Chrome/20.0.1132.57 Safari/536.11',
-        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/536.11 (KHTML, like Gecko) '
-        'Chrome/20.0.1132.47 Safari/536.11',
-    ],
-    'linux2': [
-        'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 '
-        'Chrome/25.0.1364.160 Safari/537.22',
-        'Mozilla/5.0 (X11; CrOS armv7l 2913.260.0) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.99 '
-        'Safari/537.11',
-        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.27 (KHTML, like Gecko) Chrome/26.0.1389.0 Safari/537.27'
-    ],
-    'default': [
-        'Mozilla/5.0 (X11; NetBSD amd64; rv:18.0) Gecko/20130120 Firefox/18.0'
-    ]
-}
-CONNECTION_TIMEOUT = 30
-CONNECTION_RETRIES = 2
-
-
-class VersionThread(QtCore.QThread):
-    """
-    A special Qt thread class to fetch the version of OpenLP from the website.
-    This is threaded so that it doesn't affect the loading time of OpenLP.
-    """
-    def __init__(self, main_window):
-        """
-        Constructor for the thread class.
-
-        :param main_window: The main window Object.
-        """
-        log.debug("VersionThread - Initialise")
-        super(VersionThread, self).__init__(None)
-        self.main_window = main_window
-
-    def run(self):
-        """
-        Run the thread.
-        """
-        self.sleep(1)
-        log.debug('Version thread - run')
-        app_version = get_application_version()
-        version = check_latest_version(app_version)
-        log.debug("Versions %s and %s " % (LooseVersion(str(version)), LooseVersion(str(app_version['full']))))
-        if LooseVersion(str(version)) > LooseVersion(str(app_version['full'])):
-            self.main_window.openlp_version_check.emit('%s' % version)
-
-
-class HTTPRedirectHandlerFixed(urllib.request.HTTPRedirectHandler):
-    """
-    Special HTTPRedirectHandler used to work around http://bugs.python.org/issue22248
-    (Redirecting to urls with special chars)
-    """
-    def redirect_request(self, req, fp, code, msg, headers, new_url):
-        #
-        """
-        Test if the new_url can be decoded to ascii
-
-        :param req:
-        :param fp:
-        :param code:
-        :param msg:
-        :param headers:
-        :param new_url:
-        :return:
-        """
-        try:
-            new_url.encode('latin1').decode('ascii')
-            fixed_url = new_url
-        except Exception:
-            # The url could not be decoded to ascii, so we do some url encoding
-            fixed_url = urllib.parse.quote(new_url.encode('latin1').decode('utf-8', 'replace'), safe='/:')
-        return super(HTTPRedirectHandlerFixed, self).redirect_request(req, fp, code, msg, headers, fixed_url)
-
-
-def get_application_version():
-    """
-    Returns the application version of the running instance of OpenLP::
-
-        {'full': '1.9.4-bzr1249', 'version': '1.9.4', 'build': 'bzr1249'}
-    """
-    global APPLICATION_VERSION
-    if APPLICATION_VERSION:
-        return APPLICATION_VERSION
-    if '--dev-version' in sys.argv or '-d' in sys.argv:
-        # NOTE: The following code is a duplicate of the code in setup.py. Any fix applied here should also be applied
-        # there.
-
-        # Get the revision of this tree.
-        bzr = Popen(('bzr', 'revno'), stdout=PIPE)
-        tree_revision, error = bzr.communicate()
-        tree_revision = tree_revision.decode()
-        code = bzr.wait()
-        if code != 0:
-            raise Exception('Error running bzr log')
-
-        # Get all tags.
-        bzr = Popen(('bzr', 'tags'), stdout=PIPE)
-        output, error = bzr.communicate()
-        code = bzr.wait()
-        if code != 0:
-            raise Exception('Error running bzr tags')
-        tags = list(map(bytes.decode, output.splitlines()))
-        if not tags:
-            tag_version = '0.0.0'
-            tag_revision = '0'
-        else:
-            # Remove any tag that has "?" as revision number. A "?" as revision number indicates, that this tag is from
-            # another series.
-            tags = [tag for tag in tags if tag.split()[-1].strip() != '?']
-            # Get the last tag and split it in a revision and tag name.
-            tag_version, tag_revision = tags[-1].split()
-        # If they are equal, then this tree is tarball with the source for the release. We do not want the revision
-        # number in the full version.
-        if tree_revision == tag_revision:
-            full_version = tag_version.strip()
-        else:
-            full_version = '%s-bzr%s' % (tag_version.strip(), tree_revision.strip())
-    else:
-        # We're not running the development version, let's use the file.
-        file_path = AppLocation.get_directory(AppLocation.VersionDir)
-        file_path = os.path.join(file_path, '.version')
-        version_file = None
-        try:
-            version_file = open(file_path, 'r')
-            full_version = str(version_file.read()).rstrip()
-        except IOError:
-            log.exception('Error in version file.')
-            full_version = '0.0.0-bzr000'
-        finally:
-            if version_file:
-                version_file.close()
-    bits = full_version.split('-')
-    APPLICATION_VERSION = {
-        'full': full_version,
-        'version': bits[0],
-        'build': bits[1] if len(bits) > 1 else None
-    }
-    if APPLICATION_VERSION['build']:
-        log.info('Openlp version %s build %s', APPLICATION_VERSION['version'], APPLICATION_VERSION['build'])
-    else:
-        log.info('Openlp version %s' % APPLICATION_VERSION['version'])
-    return APPLICATION_VERSION
-
-
-def check_latest_version(current_version):
-    """
-    Check the latest version of OpenLP against the version file on the OpenLP
-    site.
-
-    **Rules around versions and version files:**
-
-    * If a version number has a build (i.e. -bzr1234), then it is a nightly.
-    * If a version number's minor version is an odd number, it is a development release.
-    * If a version number's minor version is an even number, it is a stable release.
-
-    :param current_version: The current version of OpenLP.
-    """
-    version_string = current_version['full']
-    # set to prod in the distribution config file.
-    settings = Settings()
-    settings.beginGroup('core')
-    last_test = settings.value('last version test')
-    this_test = str(datetime.now().date())
-    settings.setValue('last version test', this_test)
-    settings.endGroup()
-    if last_test != this_test:
-        if current_version['build']:
-            req = urllib.request.Request('http://www.openlp.org/files/nightly_version.txt')
-        else:
-            version_parts = current_version['version'].split('.')
-            if int(version_parts[1]) % 2 != 0:
-                req = urllib.request.Request('http://www.openlp.org/files/dev_version.txt')
-            else:
-                req = urllib.request.Request('http://www.openlp.org/files/version.txt')
-        req.add_header('User-Agent', 'OpenLP/%s %s/%s; ' % (current_version['full'], platform.system(),
-                                                            platform.release()))
-        remote_version = None
-        retries = 0
-        while True:
-            try:
-                remote_version = str(urllib.request.urlopen(req, None,
-                                                            timeout=CONNECTION_TIMEOUT).read().decode()).strip()
-            except (urllib.error.URLError, ConnectionError):
-                if retries > CONNECTION_RETRIES:
-                    log.exception('Failed to download the latest OpenLP version file')
-                else:
-                    retries += 1
-                    time.sleep(0.1)
-                    continue
-            break
-        if remote_version:
-            version_string = remote_version
-    return version_string
-
-
-def get_filesystem_encoding():
-    """
-    Returns the name of the encoding used to convert Unicode filenames into system file names.
-    """
-    encoding = sys.getfilesystemencoding()
-    if encoding is None:
-        encoding = sys.getdefaultencoding()
-    return encoding
-
-
-def get_images_filter():
-    """
-    Returns a filter string for a file dialog containing all the supported image formats.
-    """
-    global IMAGES_FILTER
-    if not IMAGES_FILTER:
-        log.debug('Generating images filter.')
-        formats = list(map(bytes.decode, list(map(bytes, QtGui.QImageReader.supportedImageFormats()))))
-        visible_formats = '(*.%s)' % '; *.'.join(formats)
-        actual_formats = '(*.%s)' % ' *.'.join(formats)
-        IMAGES_FILTER = '%s %s %s' % (translate('OpenLP', 'Image Files'), visible_formats, actual_formats)
-    return IMAGES_FILTER
-
-
-def is_not_image_file(file_name):
-    """
-    Validate that the file is not an image file.
-
-    :param file_name: File name to be checked.
-    """
-    if not file_name:
-        return True
-    else:
-        formats = [bytes(fmt).decode().lower() for fmt in QtGui.QImageReader.supportedImageFormats()]
-        file_part, file_extension = os.path.splitext(str(file_name))
-        if file_extension[1:].lower() in formats and os.path.exists(file_name):
-            return False
-        return True
-
-
-def split_filename(path):
-    """
-    Return a list of the parts in a given path.
-    """
-    path = os.path.abspath(path)
-    if not os.path.isfile(path):
-        return path, ''
-    else:
-        return os.path.split(path)
-
-
-def clean_filename(filename):
-    """
-    Removes invalid characters from the given ``filename``.
-
-    :param filename:  The "dirty" file name to clean.
-    """
-    if not isinstance(filename, str):
-        filename = str(filename, 'utf-8')
-    return INVALID_FILE_CHARS.sub('_', CONTROL_CHARS.sub('', filename))
-
-
-def delete_file(file_path_name):
-    """
-    Deletes a file from the system.
-
-    :param file_path_name: The file, including path, to delete.
-    """
-    if not file_path_name:
-        return False
-    try:
-        if os.path.exists(file_path_name):
-            os.remove(file_path_name)
-        return True
-    except (IOError, OSError):
-        log.exception("Unable to delete file %s" % file_path_name)
-        return False
-
-
-def _get_user_agent():
-    """
-    Return a user agent customised for the platform the user is on.
-    """
-    browser_list = USER_AGENTS.get(sys.platform, None)
-    if not browser_list:
-        browser_list = USER_AGENTS['default']
-    random_index = randint(0, len(browser_list) - 1)
-    return browser_list[random_index]
-
-
-def get_web_page(url, header=None, update_openlp=False):
-    """
-    Attempts to download the webpage at url and returns that page or None.
-
-    :param url: The URL to be downloaded.
-    :param header:  An optional HTTP header to pass in the request to the web server.
-    :param update_openlp: Tells OpenLP to update itself if the page is successfully downloaded.
-        Defaults to False.
-    """
-    # TODO: Add proxy usage. Get proxy info from OpenLP settings, add to a
-    # proxy_handler, build into an opener and install the opener into urllib2.
-    # http://docs.python.org/library/urllib2.html
-    if not url:
-        return None
-    # This is needed to work around http://bugs.python.org/issue22248 and https://bugs.launchpad.net/openlp/+bug/1251437
-    opener = urllib.request.build_opener(HTTPRedirectHandlerFixed())
-    urllib.request.install_opener(opener)
-    req = urllib.request.Request(url)
-    if not header or header[0].lower() != 'user-agent':
-        user_agent = _get_user_agent()
-        req.add_header('User-Agent', user_agent)
-    if header:
-        req.add_header(header[0], header[1])
-    log.debug('Downloading URL = %s' % url)
-    retries = 0
-    while retries <= CONNECTION_RETRIES:
-        retries += 1
-        time.sleep(0.1)
-        try:
-            page = urllib.request.urlopen(req, timeout=CONNECTION_TIMEOUT)
-            log.debug('Downloaded page {}'.format(page.geturl()))
-            break
-        except urllib.error.URLError as err:
-            log.exception('URLError on {}'.format(url))
-            log.exception('URLError: {}'.format(err.reason))
-            page = None
-            if retries > CONNECTION_RETRIES:
-                raise
-        except socket.timeout:
-            log.exception('Socket timeout: {}'.format(url))
-            page = None
-            if retries > CONNECTION_RETRIES:
-                raise
-        except socket.gaierror:
-            log.exception('Socket gaierror: {}'.format(url))
-            page = None
-            if retries > CONNECTION_RETRIES:
-                raise
-        except ConnectionRefusedError:
-            log.exception('ConnectionRefused: {}'.format(url))
-            page = None
-            if retries > CONNECTION_RETRIES:
-                raise
-            break
-        except ConnectionError:
-            log.exception('Connection error: {}'.format(url))
-            page = None
-            if retries > CONNECTION_RETRIES:
-                raise
-        except HTTPException:
-            log.exception('HTTPException error: {}'.format(url))
-            page = None
-            if retries > CONNECTION_RETRIES:
-                raise
-        except:
-            # Don't know what's happening, so reraise the original
-            raise
-    if update_openlp:
-        Registry().get('application').process_events()
-    if not page:
-        log.exception('{} could not be downloaded'.format(url))
-        return None
-    log.debug(page)
-    return page
-
-
-def get_uno_command(connection_type='pipe'):
-    """
-    Returns the UNO command to launch an libreoffice.org instance.
-    """
-    for command in ['libreoffice', 'soffice']:
-        if which(command):
-            break
-    else:
-        raise FileNotFoundError('Command not found')
-
-    OPTIONS = '--nologo --norestore --minimized --nodefault --nofirststartwizard'
-    if connection_type == 'pipe':
-        CONNECTION = '"--accept=pipe,name=openlp_pipe;urp;"'
-    else:
-        CONNECTION = '"--accept=socket,host=localhost,port=2002;urp;"'
-    return '%s %s %s' % (command, OPTIONS, CONNECTION)
-
-
-def get_uno_instance(resolver, connection_type='pipe'):
-    """
-    Returns a running libreoffice.org instance.
-
-    :param resolver: The UNO resolver to use to find a running instance.
-    """
-    log.debug('get UNO Desktop Openoffice - resolve')
-    if connection_type == 'pipe':
-        return resolver.resolve('uno:pipe,name=openlp_pipe;urp;StarOffice.ComponentContext')
-    else:
-        return resolver.resolve('uno:socket,host=localhost,port=2002;urp;StarOffice.ComponentContext')
-
-
-def format_time(text, local_time):
-    """
-    Workaround for Python built-in time formatting function time.strftime().
-
-    time.strftime() accepts only ascii characters. This function accepts
-    unicode string and passes individual % placeholders to time.strftime().
-    This ensures only ascii characters are passed to time.strftime().
-
-    :param text:  The text to be processed.
-    :param local_time: The time to be used to add to the string.  This is a time object
-    """
-    def match_formatting(match):
-        """
-        Format the match
-        """
-        return local_time.strftime(match.group())
-    return re.sub('\%[a-zA-Z]', match_formatting, text)
-
-
-def get_locale_key(string):
-    """
-    Creates a key for case insensitive, locale aware string sorting.
-
-    :param string: The corresponding string.
-    """
-    string = string.lower()
-    # ICU is the prefered way to handle locale sort key, we fallback to locale.strxfrm which will work in most cases.
-    global ICU_COLLATOR
-    try:
-        if ICU_COLLATOR is None:
-            import icu
-            from openlp.core.common.languagemanager import LanguageManager
-            language = LanguageManager.get_language()
-            icu_locale = icu.Locale(language)
-            ICU_COLLATOR = icu.Collator.createInstance(icu_locale)
-        return ICU_COLLATOR.getSortKey(string)
-    except:
-        return locale.strxfrm(string).encode()
-
-
-def get_natural_key(string):
-    """
-    Generate a key for locale aware natural string sorting.
-
-    :param string: string to be sorted by
-    Returns a list of string compare keys and integers.
-    """
-    key = DIGITS_OR_NONDIGITS.findall(string)
-    key = [int(part) if part.isdigit() else get_locale_key(part) for part in key]
-    # Python 3 does not support comparison of different types anymore. So make sure, that we do not compare str
-    # and int.
-    if string and string[0].isdigit():
-        return [b''] + key
-    return key
-
-__all__ = ['get_application_version', 'check_latest_version',
-           'get_filesystem_encoding', 'get_web_page', 'get_uno_command', 'get_uno_instance',
-           'delete_file', 'clean_filename', 'format_time', 'get_locale_key', 'get_natural_key']

openlp/plugins/bibles/forms/bibleimportform.py

@@ -28,11 +28,11 @@ import urllib.error
 
 from PyQt5 import QtWidgets
 
-from openlp.core.common import AppLocation, Settings, UiStrings, translate
+from openlp.core.common import AppLocation, Settings, UiStrings, translate, clean_filename
 from openlp.core.lib.db import delete_database
 from openlp.core.lib.ui import critical_error_message_box
 from openlp.core.ui.wizard import OpenLPWizard, WizardStrings
-from openlp.core.utils import get_locale_key
+from openlp.core.common.languagemanager import get_locale_key
 from openlp.plugins.bibles.lib.manager import BibleFormat
 from openlp.plugins.bibles.lib.db import BiblesResourcesDB, clean_filename
 from openlp.plugins.bibles.lib.http import CWExtract, BGExtract, BSExtract

openlp/plugins/bibles/forms/bibleupgradeform.py

@@ -29,10 +29,10 @@ from tempfile import gettempdir
 
 from PyQt5 import QtCore, QtWidgets
 
-from openlp.core.common import Registry, AppLocation, UiStrings, Settings, check_directory_exists, translate
+from openlp.core.common import Registry, AppLocation, UiStrings, Settings, check_directory_exists, translate, \
+    delete_file
 from openlp.core.lib.ui import critical_error_message_box
 from openlp.core.ui.wizard import OpenLPWizard, WizardStrings
-from openlp.core.utils import delete_file
 from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta, OldBibleDB, BiblesResourcesDB
 from openlp.plugins.bibles.lib.http import BSExtract, BGExtract, CWExtract
 

openlp/plugins/bibles/lib/db.py

@@ -33,10 +33,9 @@ from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm import class_mapper, mapper, relation
 from sqlalchemy.orm.exc import UnmappedClassError
 
-from openlp.core.common import Registry, RegistryProperties, AppLocation, translate
+from openlp.core.common import Registry, RegistryProperties, AppLocation, translate, clean_filename
 from openlp.core.lib.db import BaseModel, init_db, Manager
 from openlp.core.lib.ui import critical_error_message_box
-from openlp.core.utils import clean_filename
 from openlp.plugins.bibles.lib import upgrade
 
 log = logging.getLogger(__name__)

openlp/plugins/bibles/lib/http.py

@@ -32,7 +32,7 @@ from bs4 import BeautifulSoup, NavigableString, Tag
 
 from openlp.core.common import Registry, RegistryProperties, translate
 from openlp.core.lib.ui import critical_error_message_box
-from openlp.core.utils import get_web_page
+from openlp.core.lib.webpagereader import get_web_page
 from openlp.plugins.bibles.lib import SearchResults
 from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB, Book
 

openlp/plugins/bibles/lib/manager.py

@@ -23,8 +23,7 @@
 import logging
 import os
 
-from openlp.core.common import RegistryProperties, AppLocation, Settings, translate
+from openlp.core.common import RegistryProperties, AppLocation, Settings, translate, delete_file
-from openlp.core.utils import delete_file
 from openlp.plugins.bibles.lib import parse_reference, get_reference_separator, LanguageSelection
 from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta
 from .csvbible import CSVBible

openlp/plugins/bibles/lib/mediaitem.py

@@ -29,7 +29,7 @@ from openlp.core.lib import MediaManagerItem, ItemCapabilities, ServiceItemConte
 from openlp.core.lib.searchedit import SearchEdit
 from openlp.core.lib.ui import set_case_insensitive_completer, create_horizontal_adjusting_combo_box, \
     critical_error_message_box, find_and_set_in_combo_box, build_icon
-from openlp.core.utils import get_locale_key
+from openlp.core.common.languagemanager import get_locale_key
 from openlp.plugins.bibles.forms.bibleimportform import BibleImportForm
 from openlp.plugins.bibles.forms.editbibleform import EditBibleForm
 from openlp.plugins.bibles.lib import LayoutStyle, DisplayStyle, VerseReferenceList, get_reference_separator, \

openlp/plugins/custom/lib/db.py

@@ -28,7 +28,7 @@ from sqlalchemy import Column, Table, types
 from sqlalchemy.orm import mapper
 
 from openlp.core.lib.db import BaseModel, init_db
-from openlp.core.utils import get_locale_key
+from openlp.core.common.languagemanager import get_locale_key
 
 
 class CustomSlide(BaseModel):

openlp/plugins/images/lib/mediaitem.py

@@ -25,11 +25,12 @@ import os
 
 from PyQt5 import QtCore, QtGui, QtWidgets
 
-from openlp.core.common import Registry, AppLocation, Settings, UiStrings, check_directory_exists, translate
+from openlp.core.common import Registry, AppLocation, Settings, UiStrings, check_directory_exists, translate, \
+    delete_file, get_images_filter
 from openlp.core.lib import ItemCapabilities, MediaManagerItem, ServiceItemContext, StringContent, TreeWidgetWithDnD,\
     build_icon, check_item_selected, create_thumb, validate_thumb
 from openlp.core.lib.ui import create_widget_action, critical_error_message_box
-from openlp.core.utils import delete_file, get_locale_key, get_images_filter
+from openlp.core.common.languagemanager import get_locale_key
 from openlp.plugins.images.forms import AddGroupForm, ChooseGroupForm
 from openlp.plugins.images.lib.db import ImageFilenames, ImageGroups
 

openlp/plugins/media/lib/mediaitem.py

@@ -32,7 +32,7 @@ from openlp.core.lib import ItemCapabilities, MediaManagerItem, MediaType, Servi
 from openlp.core.lib.ui import critical_error_message_box, create_horizontal_adjusting_combo_box
 from openlp.core.ui import DisplayController, Display, DisplayControllerType
 from openlp.core.ui.media import get_media_players, set_media_players, parse_optical_path, format_milliseconds
-from openlp.core.utils import get_locale_key
+from openlp.core.common.languagemanager import get_locale_key
 from openlp.core.ui.media.vlcplayer import get_vlc
 
 if get_vlc() is not None:

openlp/plugins/presentations/lib/impresscontroller.py

@@ -35,7 +35,7 @@ import logging
 import os
 import time
 
-from openlp.core.common import is_win, Registry
+from openlp.core.common import is_win, Registry, get_uno_command, get_uno_instance, delete_file
 
 if is_win():
     from win32com.client import Dispatch
@@ -57,7 +57,7 @@ else:
 from PyQt5 import QtCore
 
 from openlp.core.lib import ScreenList
-from openlp.core.utils import delete_file, get_uno_command, get_uno_instance
+from openlp.core.common import get_uno_command, get_uno_instance
 from .presentationcontroller import PresentationController, PresentationDocument, TextType
 
 

openlp/plugins/presentations/lib/mediaitem.py

@@ -29,7 +29,7 @@ from openlp.core.common import Registry, Settings, UiStrings, translate
 from openlp.core.lib import MediaManagerItem, ItemCapabilities, ServiceItemContext,\
     build_icon, check_item_selected, create_thumb, validate_thumb
 from openlp.core.lib.ui import critical_error_message_box, create_horizontal_adjusting_combo_box
-from openlp.core.utils import get_locale_key
+from openlp.core.common.languagemanager import get_locale_key
 from openlp.plugins.presentations.lib import MessageListener
 from openlp.plugins.presentations.lib.pdfcontroller import PDF_CONTROLLER_FILETYPES
 

openlp/plugins/presentations/lib/pdfcontroller.py

@@ -27,7 +27,7 @@ import re
 from shutil import which
 from subprocess import check_output, CalledProcessError, STDOUT
 
-from openlp.core.utils import AppLocation
+from openlp.core.common import AppLocation
 from openlp.core.common import Settings, is_win, trace_error_handler
 from openlp.core.lib import ScreenList
 from .presentationcontroller import PresentationController, PresentationDocument

openlp/plugins/presentations/lib/pptviewcontroller.py

@@ -20,7 +20,6 @@
 # Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
 ###############################################################################
 
-import logging
 import os
 import logging
 import zipfile
@@ -34,7 +33,7 @@ if is_win():
     from ctypes import cdll
     from ctypes.wintypes import RECT
 
-from openlp.core.utils import AppLocation
+from openlp.core.common import AppLocation
 from openlp.core.lib import ScreenList
 from .presentationcontroller import PresentationController, PresentationDocument
 

openlp/plugins/songs/forms/editsongform.py

@@ -34,7 +34,7 @@ from PyQt5 import QtCore, QtWidgets
 from openlp.core.common import Registry, RegistryProperties, AppLocation, UiStrings, check_directory_exists, translate
 from openlp.core.lib import FileDialog, PluginStatus, MediaType, create_separated_list
 from openlp.core.lib.ui import set_case_insensitive_completer, critical_error_message_box, find_and_set_in_combo_box
-from openlp.core.utils import get_natural_key
+from openlp.core.common.languagemanager import get_natural_key
 from openlp.plugins.songs.lib import VerseType, clean_song
 from openlp.plugins.songs.lib.db import Book, Song, Author, AuthorType, Topic, MediaFile, SongBookEntry
 from openlp.plugins.songs.lib.ui import SongStrings

openlp/plugins/songs/forms/songmaintenanceform.py

@@ -27,7 +27,7 @@ from sqlalchemy.sql import and_
 
 from openlp.core.common import Registry, RegistryProperties, UiStrings, translate
 from openlp.core.lib.ui import critical_error_message_box
-from openlp.core.utils import get_natural_key
+from openlp.core.common.languagemanager import get_natural_key
 from openlp.plugins.songs.forms.authorsform import AuthorsForm
 from openlp.plugins.songs.forms.topicsform import TopicsForm
 from openlp.plugins.songs.forms.songbookform import SongBookForm

openlp/plugins/songs/lib/__init__.py

@@ -29,9 +29,8 @@ import re
 
 from PyQt5 import QtWidgets
 
-from openlp.core.common import AppLocation
+from openlp.core.common import AppLocation, CONTROL_CHARS
 from openlp.core.lib import translate
-from openlp.core.utils import CONTROL_CHARS
 from openlp.plugins.songs.lib.db import MediaFile, Song
 from .db import Author
 from .ui import SongStrings

openlp/plugins/songs/lib/db.py

@@ -29,7 +29,7 @@ from sqlalchemy.orm import mapper, relation, reconstructor
 from sqlalchemy.sql.expression import func, text
 
 from openlp.core.lib.db import BaseModel, init_db
-from openlp.core.utils import get_natural_key
+from openlp.core.common.languagemanager import get_natural_key
 from openlp.core.lib import translate
 
 

openlp/plugins/songs/lib/importers/openoffice.py

@@ -25,8 +25,7 @@ import time
 
 from PyQt5 import QtCore
 
-from openlp.core.common import is_win
+from openlp.core.common import is_win, get_uno_command, get_uno_instance
-from openlp.core.utils import get_uno_command, get_uno_instance
 from openlp.core.lib import translate
 from .songimport import SongImport
 

openlp/plugins/songs/lib/mediaitem.py

@@ -32,7 +32,7 @@ from openlp.core.common import Registry, AppLocation, Settings, check_directory_
 from openlp.core.lib import MediaManagerItem, ItemCapabilities, PluginStatus, ServiceItemContext, \
     check_item_selected, create_separated_list
 from openlp.core.lib.ui import create_widget_action
-from openlp.core.utils import get_natural_key
+from openlp.core.common.languagemanager import get_natural_key
 from openlp.plugins.songs.forms.editsongform import EditSongForm
 from openlp.plugins.songs.forms.songmaintenanceform import SongMaintenanceForm
 from openlp.plugins.songs.forms.songimportform import SongImportForm

openlp/plugins/songs/lib/openlyricsexport.py

@@ -28,8 +28,7 @@ import os
 
 from lxml import etree
 
-from openlp.core.common import RegistryProperties, check_directory_exists, translate
+from openlp.core.common import RegistryProperties, check_directory_exists, translate, clean_filename
-from openlp.core.utils import clean_filename
 from openlp.plugins.songs.lib.openlyricsxml import OpenLyrics
 
 log = logging.getLogger(__name__)

openlp/plugins/songs/lib/openlyricsxml.py

@@ -62,10 +62,10 @@ import re
 from lxml import etree, objectify
 
 from openlp.core.common import translate
+from openlp.core.common.versionchecker import get_application_version
 from openlp.core.lib import FormattingTags
 from openlp.plugins.songs.lib import VerseType, clean_song
 from openlp.plugins.songs.lib.db import Author, AuthorType, Book, Song, Topic
-from openlp.core.utils import get_application_version
 
 log = logging.getLogger(__name__)
 

setup.py

@@ -65,8 +65,8 @@ def natural_sort(seq):
     return temp
 
 
-# NOTE: The following code is a duplicate of the code in openlp/core/utils/__init__.py. Any fix applied here should also
+# NOTE: The following code is a duplicate of the code in openlp/core/common/checkversion.py.
-# be applied there.
+# Any fix applied here should also be applied there.
 ver_file = None
 try:
     # Get the revision of this tree.

tests/functional/openlp_core_common/test_applocation.py

@@ -171,7 +171,7 @@ class TestAppLocation(TestCase):
         """
         Test the _get_frozen_path() function when the application is not frozen (compiled by PyInstaller)
         """
-        with patch('openlp.core.utils.sys') as mocked_sys:
+        with patch('openlp.core.common.sys') as mocked_sys:
             # GIVEN: The sys module "without" a "frozen" attribute
             mocked_sys.frozen = None
 

tests/functional/openlp_core_common/test_init.py

@@ -25,15 +25,29 @@ Functional tests to test the AppLocation class and related methods.
 import os
 from unittest import TestCase
 
-from openlp.core.common import add_actions
+from openlp.core.common import add_actions, get_uno_instance, get_uno_command, delete_file, get_filesystem_encoding, \
+    split_filename, clean_filename
 from tests.functional import MagicMock, patch
+from tests.helpers.testmixin import TestMixin
 
 
-class TestInit(TestCase):
+class TestInit(TestCase, TestMixin):
     """
     A test suite to test out various methods around the common __init__ class.
     """
 
+    def setUp(self):
+        """
+        Create an instance and a few example actions.
+        """
+        self.build_settings()
+
+    def tearDown(self):
+        """
+        Clean up
+        """
+        self.destroy_settings()
+
     def add_actions_empty_list_test(self):
         """
         Test that no actions are added when the list is empty
@@ -93,3 +107,236 @@ class TestInit(TestCase):
         # THEN: The addSeparator method is called, and the addAction method is called
         mocked_target.addSeparator.assert_called_with()
         mocked_target.addAction.assert_called_with('action')
+
+    def get_uno_instance_pipe_test(self):
+        """
+        Test that when the UNO connection type is "pipe" the resolver is given the "pipe" URI
+        """
+        # GIVEN: A mock resolver object and UNO_CONNECTION_TYPE is "pipe"
+        mock_resolver = MagicMock()
+
+        # WHEN: get_uno_instance() is called
+        get_uno_instance(mock_resolver)
+
+        # THEN: the resolve method is called with the correct argument
+        mock_resolver.resolve.assert_called_with('uno:pipe,name=openlp_pipe;urp;StarOffice.ComponentContext')
+
+    def get_uno_instance_socket_test(self):
+        """
+        Test that when the UNO connection type is other than "pipe" the resolver is given the "socket" URI
+        """
+        # GIVEN: A mock resolver object and UNO_CONNECTION_TYPE is "socket"
+        mock_resolver = MagicMock()
+
+        # WHEN: get_uno_instance() is called
+        get_uno_instance(mock_resolver, 'socket')
+
+        # THEN: the resolve method is called with the correct argument
+        mock_resolver.resolve.assert_called_with('uno:socket,host=localhost,port=2002;urp;StarOffice.ComponentContext')
+
+    def get_uno_command_libreoffice_command_exists_test(self):
+        """
+        Test the ``get_uno_command`` function uses the libreoffice command when available.
+        :return:
+        """
+
+        # GIVEN: A patched 'which' method which returns a path when called with 'libreoffice'
+        with patch('openlp.core.common.which',
+                   **{'side_effect': lambda command: {'libreoffice': '/usr/bin/libreoffice'}[command]}):
+            # WHEN: Calling get_uno_command
+            result = get_uno_command()
+
+            # THEN: The command 'libreoffice' should be called with the appropriate parameters
+            self.assertEquals(result,
+                              'libreoffice --nologo --norestore --minimized --nodefault --nofirststartwizard'
+                              ' "--accept=pipe,name=openlp_pipe;urp;"')
+
+    def get_uno_command_only_soffice_command_exists_test(self):
+        """
+        Test the ``get_uno_command`` function uses the soffice command when the libreoffice command is not available.
+        :return:
+        """
+
+        # GIVEN: A patched 'which' method which returns None when called with 'libreoffice' and a path when called with
+        #        'soffice'
+        with patch('openlp.core.common.which',
+                   **{'side_effect': lambda command: {'libreoffice': None, 'soffice': '/usr/bin/soffice'}[
+                       command]}):
+            # WHEN: Calling get_uno_command
+            result = get_uno_command()
+
+            # THEN: The command 'soffice' should be called with the appropriate parameters
+            self.assertEquals(result, 'soffice --nologo --norestore --minimized --nodefault --nofirststartwizard'
+                                      ' "--accept=pipe,name=openlp_pipe;urp;"')
+
+    def get_uno_command_when_no_command_exists_test(self):
+        """
+        Test the ``get_uno_command`` function raises an FileNotFoundError when neither the libreoffice or soffice
+        commands are available.
+        :return:
+        """
+
+        # GIVEN: A patched 'which' method which returns None
+        with patch('openlp.core.common.which', **{'return_value': None}):
+            # WHEN: Calling get_uno_command
+
+            # THEN: a FileNotFoundError exception should be raised
+            self.assertRaises(FileNotFoundError, get_uno_command)
+
+    def get_uno_command_connection_type_test(self):
+        """
+        Test the ``get_uno_command`` function when the connection type is anything other than pipe.
+        :return:
+        """
+
+        # GIVEN: A patched 'which' method which returns 'libreoffice'
+        with patch('openlp.core.common.which', **{'return_value': 'libreoffice'}):
+            # WHEN: Calling get_uno_command with a connection type other than pipe
+            result = get_uno_command('socket')
+
+            # THEN: The connection parameters should be set for socket
+            self.assertEqual(result, 'libreoffice --nologo --norestore --minimized --nodefault --nofirststartwizard'
+                                     ' "--accept=socket,host=localhost,port=2002;urp;"')
+
+    def get_filesystem_encoding_sys_function_not_called_test(self):
+        """
+        Test the get_filesystem_encoding() function does not call the sys.getdefaultencoding() function
+        """
+        # GIVEN: sys.getfilesystemencoding returns "cp1252"
+        with patch('openlp.core.common.sys.getfilesystemencoding') as mocked_getfilesystemencoding, \
+                patch('openlp.core.common.sys.getdefaultencoding') as mocked_getdefaultencoding:
+            mocked_getfilesystemencoding.return_value = 'cp1252'
+
+            # WHEN: get_filesystem_encoding() is called
+            result = get_filesystem_encoding()
+
+            # THEN: getdefaultencoding should have been called
+            mocked_getfilesystemencoding.assert_called_with()
+            self.assertEqual(0, mocked_getdefaultencoding.called, 'getdefaultencoding should not have been called')
+            self.assertEqual('cp1252', result, 'The result should be "cp1252"')
+
+    def get_filesystem_encoding_sys_function_is_called_test(self):
+        """
+        Test the get_filesystem_encoding() function calls the sys.getdefaultencoding() function
+        """
+        # GIVEN: sys.getfilesystemencoding returns None and sys.getdefaultencoding returns "utf-8"
+        with patch('openlp.core.common.sys.getfilesystemencoding') as mocked_getfilesystemencoding, \
+                patch('openlp.core.common.sys.getdefaultencoding') as mocked_getdefaultencoding:
+            mocked_getfilesystemencoding.return_value = None
+            mocked_getdefaultencoding.return_value = 'utf-8'
+
+            # WHEN: get_filesystem_encoding() is called
+            result = get_filesystem_encoding()
+
+            # THEN: getdefaultencoding should have been called
+            mocked_getfilesystemencoding.assert_called_with()
+            mocked_getdefaultencoding.assert_called_with()
+            self.assertEqual('utf-8', result, 'The result should be "utf-8"')
+
+    def split_filename_with_file_path_test(self):
+        """
+        Test the split_filename() function with a path to a file
+        """
+        # GIVEN: A path to a file.
+        if os.name == 'nt':
+            file_path = 'C:\\home\\user\\myfile.txt'
+            wanted_result = ('C:\\home\\user', 'myfile.txt')
+        else:
+            file_path = '/home/user/myfile.txt'
+            wanted_result = ('/home/user', 'myfile.txt')
+        with patch('openlp.core.common.os.path.isfile') as mocked_is_file:
+            mocked_is_file.return_value = True
+
+            # WHEN: Split the file name.
+            result = split_filename(file_path)
+
+            # THEN: A tuple should be returned.
+            self.assertEqual(wanted_result, result, 'A tuple with the dir and file name should have been returned')
+
+    def split_filename_with_dir_path_test(self):
+        """
+        Test the split_filename() function with a path to a directory
+        """
+        # GIVEN: A path to a dir.
+        if os.name == 'nt':
+            file_path = 'C:\\home\\user\\mydir'
+            wanted_result = ('C:\\home\\user\\mydir', '')
+        else:
+            file_path = '/home/user/mydir'
+            wanted_result = ('/home/user/mydir', '')
+        with patch('openlp.core.common.os.path.isfile') as mocked_is_file:
+            mocked_is_file.return_value = False
+
+            # WHEN: Split the file name.
+            result = split_filename(file_path)
+
+            # THEN: A tuple should be returned.
+            self.assertEqual(wanted_result, result,
+                             'A two-entry tuple with the directory and file name (empty) should have been returned.')
+
+    def clean_filename_test(self):
+        """
+        Test the clean_filename() function
+        """
+        # GIVEN: A invalid file name and the valid file name.
+        invalid_name = 'A_file_with_invalid_characters_[\\/:\*\?"<>\|\+\[\]%].py'
+        wanted_name = 'A_file_with_invalid_characters______________________.py'
+
+        # WHEN: Clean the name.
+        result = clean_filename(invalid_name)
+
+        # THEN: The file name should be cleaned.
+        self.assertEqual(wanted_name, result, 'The file name should not contain any special characters.')
+
+    def delete_file_no_path_test(self):
+        """
+        Test the delete_file function when called with out a valid path
+        """
+        # GIVEN: A blank path
+        # WEHN: Calling delete_file
+        result = delete_file('')
+
+        # THEN: delete_file should return False
+        self.assertFalse(result, "delete_file should return False when called with ''")
+
+    def delete_file_path_success_test(self):
+        """
+        Test the delete_file function when it successfully deletes a file
+        """
+        # GIVEN: A mocked os which returns True when os.path.exists is called
+        with patch('openlp.core.common.os', **{'path.exists.return_value': False}):
+
+            # WHEN: Calling delete_file with a file path
+            result = delete_file('path/file.ext')
+
+            # THEN: delete_file should return True
+            self.assertTrue(result, 'delete_file should return True when it successfully deletes a file')
+
+    def delete_file_path_no_file_exists_test(self):
+        """
+        Test the delete_file function when the file to remove does not exist
+        """
+        # GIVEN: A mocked os which returns False when os.path.exists is called
+        with patch('openlp.core.common.os', **{'path.exists.return_value': False}):
+
+            # WHEN: Calling delete_file with a file path
+            result = delete_file('path/file.ext')
+
+            # THEN: delete_file should return True
+            self.assertTrue(result, 'delete_file should return True when the file doesnt exist')
+
+    def delete_file_path_exception_test(self):
+        """
+        Test the delete_file function when os.remove raises an exception
+        """
+        # GIVEN: A mocked os which returns True when os.path.exists is called and raises an OSError when os.remove is
+        #       called.
+        with patch('openlp.core.common.os', **{'path.exists.return_value': True, 'path.exists.side_effect': OSError}), \
+                patch('openlp.core.common.log') as mocked_log:
+
+            # WHEN: Calling delete_file with a file path
+            result = delete_file('path/file.ext')
+
+            # THEN: delete_file should log and exception and return False
+            self.assertEqual(mocked_log.exception.call_count, 1)
+            self.assertFalse(result, 'delete_file should return False when os.remove raises an OSError')

tests/functional/openlp_core_common/test_languagemanager.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection                                      #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2016 OpenLP Developers                                   #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; version 2 of the License.                              #
+#                                                                             #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with this program; if not, write to the Free Software Foundation, Inc., 59  #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
+###############################################################################
+"""
+Functional tests to test the AppLocation class and related methods.
+"""
+from unittest import TestCase
+
+from tests.functional import patch
+from openlp.core.common.languagemanager import get_locale_key, get_natural_key
+
+
+class TestLanguageManager(TestCase):
+    """
+    A test suite to test out various methods around the common __init__ class.
+    """
+
+    def get_locale_key_test(self):
+        """
+        Test the get_locale_key(string) function
+        """
+        with patch('openlp.core.common.languagemanager.LanguageManager.get_language') as mocked_get_language:
+            # GIVEN: The language is German
+            # 0x00C3 (A with diaresis) should be sorted as "A". 0x00DF (sharp s) should be sorted as "ss".
+            mocked_get_language.return_value = 'de'
+            unsorted_list = ['Auszug', 'Aushang', '\u00C4u\u00DFerung']
+
+            # WHEN: We sort the list and use get_locale_key() to generate the sorting keys
+            sorted_list = sorted(unsorted_list, key=get_locale_key)
+
+            # THEN: We get a properly sorted list
+            self.assertEqual(['Aushang', '\u00C4u\u00DFerung', 'Auszug'], sorted_list,
+                             'Strings should be sorted properly')
+
+    def get_natural_key_test(self):
+        """
+        Test the get_natural_key(string) function
+        """
+        with patch('openlp.core.common.languagemanager.LanguageManager.get_language') as mocked_get_language:
+            # GIVEN: The language is English (a language, which sorts digits before letters)
+            mocked_get_language.return_value = 'en'
+            unsorted_list = ['item 10a', 'item 3b', '1st item']
+
+            # WHEN: We sort the list and use get_natural_key() to generate the sorting keys
+            sorted_list = sorted(unsorted_list, key=get_natural_key)
+
+            # THEN: We get a properly sorted list
+            self.assertEqual(['1st item', 'item 3b', 'item 10a'], sorted_list, 'Numbers should be sorted naturally')

tests/functional/openlp_core_common/test_versionchecker.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection                                      #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2016 OpenLP Developers                                   #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; version 2 of the License.                              #
+#                                                                             #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with this program; if not, write to the Free Software Foundation, Inc., 59  #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
+###############################################################################
+"""
+Package to test the openlp.core.common.versionchecker package.
+"""
+from unittest import TestCase
+
+from openlp.core.common.settings import Settings
+from openlp.core.common.versionchecker import VersionThread
+from tests.functional import MagicMock, patch
+from tests.helpers.testmixin import TestMixin
+
+
+class TestVersionchecker(TestMixin, TestCase):
+
+    def setUp(self):
+        """
+        Create an instance and a few example actions.
+        """
+        self.build_settings()
+
+    def tearDown(self):
+        """
+        Clean up
+        """
+        self.destroy_settings()
+
+    def version_thread_triggered_test(self):
+        """
+        Test the version thread call does not trigger UI
+        :return:
+        """
+        # GIVEN: a equal version setup and the data is not today.
+        mocked_main_window = MagicMock()
+        Settings().setValue('core/last version test', '1950-04-01')
+        # WHEN: We check to see if the version is different .
+        with patch('PyQt5.QtCore.QThread'),\
+                patch('openlp.core.common.versionchecker.get_application_version') as mocked_get_application_version:
+            mocked_get_application_version.return_value = {'version': '1.0.0', 'build': '', 'full': '2.0.4'}
+            version_thread = VersionThread(mocked_main_window)
+            version_thread.run()
+        # THEN: If the version has changed the main window is notified
+        self.assertTrue(mocked_main_window.openlp_version_check.emit.called,
+                        'The main windows should have been notified')

tests/functional/openlp_core_lib/test_webpagereader.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection                                      #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2016 OpenLP Developers                                   #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; version 2 of the License.                              #
+#                                                                             #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with this program; if not, write to the Free Software Foundation, Inc., 59  #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
+###############################################################################
+"""
+Functional tests to test the AppLocation class and related methods.
+"""
+from unittest import TestCase
+
+from openlp.core.lib.webpagereader import _get_user_agent, get_web_page
+
+from tests.functional import MagicMock, patch
+
+
+class TestUtils(TestCase):
+    """
+    A test suite to test out various methods around the AppLocation class.
+    """
+    def get_user_agent_linux_test(self):
+        """
+        Test that getting a user agent on Linux returns a user agent suitable for Linux
+        """
+        with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
+
+            # GIVEN: The system is Linux
+            mocked_sys.platform = 'linux2'
+
+            # WHEN: We call _get_user_agent()
+            user_agent = _get_user_agent()
+
+            # THEN: The user agent is a Linux (or ChromeOS) user agent
+            result = 'Linux' in user_agent or 'CrOS' in user_agent
+            self.assertTrue(result, 'The user agent should be a valid Linux user agent')
+
+    def get_user_agent_windows_test(self):
+        """
+        Test that getting a user agent on Windows returns a user agent suitable for Windows
+        """
+        with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
+
+            # GIVEN: The system is Linux
+            mocked_sys.platform = 'win32'
+
+            # WHEN: We call _get_user_agent()
+            user_agent = _get_user_agent()
+
+            # THEN: The user agent is a Linux (or ChromeOS) user agent
+            self.assertIn('Windows', user_agent, 'The user agent should be a valid Windows user agent')
+
+    def get_user_agent_macos_test(self):
+        """
+        Test that getting a user agent on OS X returns a user agent suitable for OS X
+        """
+        with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
+
+            # GIVEN: The system is Linux
+            mocked_sys.platform = 'darwin'
+
+            # WHEN: We call _get_user_agent()
+            user_agent = _get_user_agent()
+
+            # THEN: The user agent is a Linux (or ChromeOS) user agent
+            self.assertIn('Mac OS X', user_agent, 'The user agent should be a valid OS X user agent')
+
+    def get_user_agent_default_test(self):
+        """
+        Test that getting a user agent on a non-Linux/Windows/OS X platform returns the default user agent
+        """
+        with patch('openlp.core.lib.webpagereader.sys') as mocked_sys:
+
+            # GIVEN: The system is Linux
+            mocked_sys.platform = 'freebsd'
+
+            # WHEN: We call _get_user_agent()
+            user_agent = _get_user_agent()
+
+            # THEN: The user agent is a Linux (or ChromeOS) user agent
+            self.assertIn('NetBSD', user_agent, 'The user agent should be the default user agent')
+
+    def get_web_page_no_url_test(self):
+        """
+        Test that sending a URL of None to the get_web_page method returns None
+        """
+        # GIVEN: A None url
+        test_url = None
+
+        # WHEN: We try to get the test URL
+        result = get_web_page(test_url)
+
+        # THEN: None should be returned
+        self.assertIsNone(result, 'The return value of get_web_page should be None')
+
+    def get_web_page_test(self):
+        """
+        Test that the get_web_page method works correctly
+        """
+        with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
+                patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
+                patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \
+                patch('openlp.core.common.Registry') as MockRegistry:
+            # GIVEN: Mocked out objects and a fake URL
+            mocked_request_object = MagicMock()
+            MockRequest.return_value = mocked_request_object
+            mocked_page_object = MagicMock()
+            mock_urlopen.return_value = mocked_page_object
+            mock_get_user_agent.return_value = 'user_agent'
+            fake_url = 'this://is.a.fake/url'
+
+            # WHEN: The get_web_page() method is called
+            returned_page = get_web_page(fake_url)
+
+            # THEN: The correct methods are called with the correct arguments and a web page is returned
+            MockRequest.assert_called_with(fake_url)
+            mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
+            self.assertEqual(1, mocked_request_object.add_header.call_count,
+                             'There should only be 1 call to add_header')
+            mock_get_user_agent.assert_called_with()
+            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
+            mocked_page_object.geturl.assert_called_with()
+            self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called')
+            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
+
+    def get_web_page_with_header_test(self):
+        """
+        Test that adding a header to the call to get_web_page() adds the header to the request
+        """
+        with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
+                patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
+                patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent:
+            # GIVEN: Mocked out objects, a fake URL and a fake header
+            mocked_request_object = MagicMock()
+            MockRequest.return_value = mocked_request_object
+            mocked_page_object = MagicMock()
+            mock_urlopen.return_value = mocked_page_object
+            mock_get_user_agent.return_value = 'user_agent'
+            fake_url = 'this://is.a.fake/url'
+            fake_header = ('Fake-Header', 'fake value')
+
+            # WHEN: The get_web_page() method is called
+            returned_page = get_web_page(fake_url, header=fake_header)
+
+            # THEN: The correct methods are called with the correct arguments and a web page is returned
+            MockRequest.assert_called_with(fake_url)
+            mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1])
+            self.assertEqual(2, mocked_request_object.add_header.call_count,
+                             'There should only be 2 calls to add_header')
+            mock_get_user_agent.assert_called_with()
+            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
+            mocked_page_object.geturl.assert_called_with()
+            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
+
+    def get_web_page_with_user_agent_in_headers_test(self):
+        """
+        Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request
+        """
+        with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
+                patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
+                patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent:
+            # GIVEN: Mocked out objects, a fake URL and a fake header
+            mocked_request_object = MagicMock()
+            MockRequest.return_value = mocked_request_object
+            mocked_page_object = MagicMock()
+            mock_urlopen.return_value = mocked_page_object
+            fake_url = 'this://is.a.fake/url'
+            user_agent_header = ('User-Agent', 'OpenLP/2.2.0')
+
+            # WHEN: The get_web_page() method is called
+            returned_page = get_web_page(fake_url, header=user_agent_header)
+
+            # THEN: The correct methods are called with the correct arguments and a web page is returned
+            MockRequest.assert_called_with(fake_url)
+            mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1])
+            self.assertEqual(1, mocked_request_object.add_header.call_count,
+                             'There should only be 1 call to add_header')
+            self.assertEqual(0, mock_get_user_agent.call_count, '_get_user_agent should not have been called')
+            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
+            mocked_page_object.geturl.assert_called_with()
+            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
+
+    def get_web_page_update_openlp_test(self):
+        """
+        Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
+        """
+        with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
+                patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
+                patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \
+                patch('openlp.core.lib.webpagereader.Registry') as MockRegistry:
+            # GIVEN: Mocked out objects, a fake URL
+            mocked_request_object = MagicMock()
+            MockRequest.return_value = mocked_request_object
+            mocked_page_object = MagicMock()
+            mock_urlopen.return_value = mocked_page_object
+            mock_get_user_agent.return_value = 'user_agent'
+            mocked_registry_object = MagicMock()
+            mocked_application_object = MagicMock()
+            mocked_registry_object.get.return_value = mocked_application_object
+            MockRegistry.return_value = mocked_registry_object
+            fake_url = 'this://is.a.fake/url'
+
+            # WHEN: The get_web_page() method is called
+            returned_page = get_web_page(fake_url, update_openlp=True)
+
+            # THEN: The correct methods are called with the correct arguments and a web page is returned
+            MockRequest.assert_called_with(fake_url)
+            mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
+            self.assertEqual(1, mocked_request_object.add_header.call_count,
+                             'There should only be 1 call to add_header')
+            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
+            mocked_page_object.geturl.assert_called_with()
+            mocked_registry_object.get.assert_called_with('application')
+            mocked_application_object.process_events.assert_called_with()
+            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')

tests/functional/openlp_core_ui/test_first_time.py

@@ -28,10 +28,10 @@ import urllib.request
 import urllib.error
 import urllib.parse
 
-from tests.functional import MagicMock, patch
+from tests.functional import patch
 from tests.helpers.testmixin import TestMixin
 
-from openlp.core.utils import CONNECTION_TIMEOUT, CONNECTION_RETRIES, get_web_page
+from openlp.core.lib.webpagereader import CONNECTION_RETRIES, get_web_page
 
 
 class TestFirstTimeWizard(TestMixin, TestCase):

tests/functional/openlp_core_utils/__init__.py

@@ -1,21 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
-
-###############################################################################
-# OpenLP - Open Source Lyrics Projection                                      #
-# --------------------------------------------------------------------------- #
-# Copyright (c) 2008-2016 OpenLP Developers                                   #
-# --------------------------------------------------------------------------- #
-# This program is free software; you can redistribute it and/or modify it     #
-# under the terms of the GNU General Public License as published by the Free  #
-# Software Foundation; version 2 of the License.                              #
-#                                                                             #
-# This program is distributed in the hope that it will be useful, but WITHOUT #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
-# more details.                                                               #
-#                                                                             #
-# You should have received a copy of the GNU General Public License along     #
-# with this program; if not, write to the Free Software Foundation, Inc., 59  #
-# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
-###############################################################################

tests/functional/openlp_core_utils/test_init.py

@@ -1,129 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
-
-###############################################################################
-# OpenLP - Open Source Lyrics Projection                                      #
-# --------------------------------------------------------------------------- #
-# Copyright (c) 2008-2016 OpenLP Developers                                   #
-# --------------------------------------------------------------------------- #
-# This program is free software; you can redistribute it and/or modify it     #
-# under the terms of the GNU General Public License as published by the Free  #
-# Software Foundation; version 2 of the License.                              #
-#                                                                             #
-# This program is distributed in the hope that it will be useful, but WITHOUT #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
-# more details.                                                               #
-#                                                                             #
-# You should have received a copy of the GNU General Public License along     #
-# with this program; if not, write to the Free Software Foundation, Inc., 59  #
-# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
-###############################################################################
-"""
-Package to test the openlp.core.utils.actions package.
-"""
-from unittest import TestCase
-
-from openlp.core.common.settings import Settings
-from openlp.core.utils import VersionThread, get_uno_command
-from tests.functional import MagicMock, patch
-from tests.helpers.testmixin import TestMixin
-
-
-class TestInitFunctions(TestMixin, TestCase):
-
-    def setUp(self):
-        """
-        Create an instance and a few example actions.
-        """
-        self.build_settings()
-
-    def tearDown(self):
-        """
-        Clean up
-        """
-        self.destroy_settings()
-
-    def version_thread_triggered_test(self):
-        """
-        Test the version thread call does not trigger UI
-        :return:
-        """
-        # GIVEN: a equal version setup and the data is not today.
-        mocked_main_window = MagicMock()
-        Settings().setValue('core/last version test', '1950-04-01')
-        # WHEN: We check to see if the version is different .
-        with patch('PyQt5.QtCore.QThread'),\
-                patch('openlp.core.utils.get_application_version') as mocked_get_application_version:
-            mocked_get_application_version.return_value = {'version': '1.0.0', 'build': '', 'full': '2.0.4'}
-            version_thread = VersionThread(mocked_main_window)
-            version_thread.run()
-        # THEN: If the version has changed the main window is notified
-        self.assertTrue(mocked_main_window.openlp_version_check.emit.called,
-                        'The main windows should have been notified')
-
-    def get_uno_command_libreoffice_command_exists_test(self):
-        """
-        Test the ``get_uno_command`` function uses the libreoffice command when available.
-        :return:
-        """
-
-        # GIVEN: A patched 'which' method which returns a path when called with 'libreoffice'
-        with patch('openlp.core.utils.which',
-                   **{'side_effect': lambda command: {'libreoffice': '/usr/bin/libreoffice'}[command]}):
-
-            # WHEN: Calling get_uno_command
-            result = get_uno_command()
-
-            # THEN: The command 'libreoffice' should be called with the appropriate parameters
-            self.assertEquals(result, 'libreoffice --nologo --norestore --minimized --nodefault --nofirststartwizard'
-                                      ' "--accept=pipe,name=openlp_pipe;urp;"')
-
-    def get_uno_command_only_soffice_command_exists_test(self):
-        """
-        Test the ``get_uno_command`` function uses the soffice command when the libreoffice command is not available.
-        :return:
-        """
-
-        # GIVEN: A patched 'which' method which returns None when called with 'libreoffice' and a path when called with
-        #        'soffice'
-        with patch('openlp.core.utils.which',
-                   **{'side_effect': lambda command: {'libreoffice': None, 'soffice': '/usr/bin/soffice'}[command]}):
-
-            # WHEN: Calling get_uno_command
-            result = get_uno_command()
-
-            # THEN: The command 'soffice' should be called with the appropriate parameters
-            self.assertEquals(result, 'soffice --nologo --norestore --minimized --nodefault --nofirststartwizard'
-                                      ' "--accept=pipe,name=openlp_pipe;urp;"')
-
-    def get_uno_command_when_no_command_exists_test(self):
-        """
-        Test the ``get_uno_command`` function raises an FileNotFoundError when neither the libreoffice or soffice
-        commands are available.
-        :return:
-        """
-
-        # GIVEN: A patched 'which' method which returns None
-        with patch('openlp.core.utils.which', **{'return_value': None}):
-
-            # WHEN: Calling get_uno_command
-
-            # THEN: a FileNotFoundError exception should be raised
-            self.assertRaises(FileNotFoundError, get_uno_command)
-
-    def get_uno_command_connection_type_test(self):
-        """
-        Test the ``get_uno_command`` function when the connection type is anything other than pipe.
-        :return:
-        """
-
-        # GIVEN: A patched 'which' method which returns 'libreoffice'
-        with patch('openlp.core.utils.which', **{'return_value': 'libreoffice'}):
-
-            # WHEN: Calling get_uno_command with a connection type other than pipe
-            result = get_uno_command('socket')
-
-            # THEN: The connection parameters should be set for socket
-            self.assertEqual(result, 'libreoffice --nologo --norestore --minimized --nodefault --nofirststartwizard'
-                                     ' "--accept=socket,host=localhost,port=2002;urp;"')

tests/functional/openlp_core_utils/test_utils.py

@@ -1,433 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
-
-###############################################################################
-# OpenLP - Open Source Lyrics Projection                                      #
-# --------------------------------------------------------------------------- #
-# Copyright (c) 2008-2016 OpenLP Developers                                   #
-# --------------------------------------------------------------------------- #
-# This program is free software; you can redistribute it and/or modify it     #
-# under the terms of the GNU General Public License as published by the Free  #
-# Software Foundation; version 2 of the License.                              #
-#                                                                             #
-# This program is distributed in the hope that it will be useful, but WITHOUT #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
-# more details.                                                               #
-#                                                                             #
-# You should have received a copy of the GNU General Public License along     #
-# with this program; if not, write to the Free Software Foundation, Inc., 59  #
-# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
-###############################################################################
-"""
-Functional tests to test the AppLocation class and related methods.
-"""
-import os
-from unittest import TestCase
-
-from openlp.core.utils import clean_filename, delete_file, get_filesystem_encoding, get_locale_key, \
-    get_natural_key, split_filename, _get_user_agent, get_web_page, get_uno_instance
-
-from tests.functional import MagicMock, patch
-
-
-class TestUtils(TestCase):
-    """
-    A test suite to test out various methods around the AppLocation class.
-    """
-
-    def get_filesystem_encoding_sys_function_not_called_test(self):
-        """
-        Test the get_filesystem_encoding() function does not call the sys.getdefaultencoding() function
-        """
-        # GIVEN: sys.getfilesystemencoding returns "cp1252"
-        with patch('openlp.core.utils.sys.getfilesystemencoding') as mocked_getfilesystemencoding, \
-                patch('openlp.core.utils.sys.getdefaultencoding') as mocked_getdefaultencoding:
-            mocked_getfilesystemencoding.return_value = 'cp1252'
-
-            # WHEN: get_filesystem_encoding() is called
-            result = get_filesystem_encoding()
-
-            # THEN: getdefaultencoding should have been called
-            mocked_getfilesystemencoding.assert_called_with()
-            self.assertEqual(0, mocked_getdefaultencoding.called, 'getdefaultencoding should not have been called')
-            self.assertEqual('cp1252', result, 'The result should be "cp1252"')
-
-    def get_filesystem_encoding_sys_function_is_called_test(self):
-        """
-        Test the get_filesystem_encoding() function calls the sys.getdefaultencoding() function
-        """
-        # GIVEN: sys.getfilesystemencoding returns None and sys.getdefaultencoding returns "utf-8"
-        with patch('openlp.core.utils.sys.getfilesystemencoding') as mocked_getfilesystemencoding, \
-                patch('openlp.core.utils.sys.getdefaultencoding') as mocked_getdefaultencoding:
-            mocked_getfilesystemencoding.return_value = None
-            mocked_getdefaultencoding.return_value = 'utf-8'
-
-            # WHEN: get_filesystem_encoding() is called
-            result = get_filesystem_encoding()
-
-            # THEN: getdefaultencoding should have been called
-            mocked_getfilesystemencoding.assert_called_with()
-            mocked_getdefaultencoding.assert_called_with()
-            self.assertEqual('utf-8', result, 'The result should be "utf-8"')
-
-    def split_filename_with_file_path_test(self):
-        """
-        Test the split_filename() function with a path to a file
-        """
-        # GIVEN: A path to a file.
-        if os.name == 'nt':
-            file_path = 'C:\\home\\user\\myfile.txt'
-            wanted_result = ('C:\\home\\user', 'myfile.txt')
-        else:
-            file_path = '/home/user/myfile.txt'
-            wanted_result = ('/home/user', 'myfile.txt')
-        with patch('openlp.core.utils.os.path.isfile') as mocked_is_file:
-            mocked_is_file.return_value = True
-
-            # WHEN: Split the file name.
-            result = split_filename(file_path)
-
-            # THEN: A tuple should be returned.
-            self.assertEqual(wanted_result, result, 'A tuple with the dir and file name should have been returned')
-
-    def split_filename_with_dir_path_test(self):
-        """
-        Test the split_filename() function with a path to a directory
-        """
-        # GIVEN: A path to a dir.
-        if os.name == 'nt':
-            file_path = 'C:\\home\\user\\mydir'
-            wanted_result = ('C:\\home\\user\\mydir', '')
-        else:
-            file_path = '/home/user/mydir'
-            wanted_result = ('/home/user/mydir', '')
-        with patch('openlp.core.utils.os.path.isfile') as mocked_is_file:
-            mocked_is_file.return_value = False
-
-            # WHEN: Split the file name.
-            result = split_filename(file_path)
-
-            # THEN: A tuple should be returned.
-            self.assertEqual(wanted_result, result,
-                             'A two-entry tuple with the directory and file name (empty) should have been returned.')
-
-    def clean_filename_test(self):
-        """
-        Test the clean_filename() function
-        """
-        # GIVEN: A invalid file name and the valid file name.
-        invalid_name = 'A_file_with_invalid_characters_[\\/:\*\?"<>\|\+\[\]%].py'
-        wanted_name = 'A_file_with_invalid_characters______________________.py'
-
-        # WHEN: Clean the name.
-        result = clean_filename(invalid_name)
-
-        # THEN: The file name should be cleaned.
-        self.assertEqual(wanted_name, result, 'The file name should not contain any special characters.')
-
-    def delete_file_no_path_test(self):
-        """
-        Test the delete_file function when called with out a valid path
-        """
-        # GIVEN: A blank path
-        # WEHN: Calling delete_file
-        result = delete_file('')
-
-        # THEN: delete_file should return False
-        self.assertFalse(result, "delete_file should return False when called with ''")
-
-    def delete_file_path_success_test(self):
-        """
-        Test the delete_file function when it successfully deletes a file
-        """
-        # GIVEN: A mocked os which returns True when os.path.exists is called
-        with patch('openlp.core.utils.os', **{'path.exists.return_value': False}):
-
-            # WHEN: Calling delete_file with a file path
-            result = delete_file('path/file.ext')
-
-            # THEN: delete_file should return True
-            self.assertTrue(result, 'delete_file should return True when it successfully deletes a file')
-
-    def delete_file_path_no_file_exists_test(self):
-        """
-        Test the delete_file function when the file to remove does not exist
-        """
-        # GIVEN: A mocked os which returns False when os.path.exists is called
-        with patch('openlp.core.utils.os', **{'path.exists.return_value': False}):
-
-            # WHEN: Calling delete_file with a file path
-            result = delete_file('path/file.ext')
-
-            # THEN: delete_file should return True
-            self.assertTrue(result, 'delete_file should return True when the file doesnt exist')
-
-    def delete_file_path_exception_test(self):
-        """
-        Test the delete_file function when os.remove raises an exception
-        """
-        # GIVEN: A mocked os which returns True when os.path.exists is called and raises an OSError when os.remove is
-        #       called.
-        with patch('openlp.core.utils.os', **{'path.exists.return_value': True, 'path.exists.side_effect': OSError}), \
-                patch('openlp.core.utils.log') as mocked_log:
-
-            # WHEN: Calling delete_file with a file path
-            result = delete_file('path/file.ext')
-
-            # THEN: delete_file should log and exception and return False
-            self.assertEqual(mocked_log.exception.call_count, 1)
-            self.assertFalse(result, 'delete_file should return False when os.remove raises an OSError')
-
-    def get_locale_key_test(self):
-        """
-        Test the get_locale_key(string) function
-        """
-        with patch('openlp.core.common.languagemanager.LanguageManager.get_language') as mocked_get_language:
-            # GIVEN: The language is German
-            # 0x00C3 (A with diaresis) should be sorted as "A". 0x00DF (sharp s) should be sorted as "ss".
-            mocked_get_language.return_value = 'de'
-            unsorted_list = ['Auszug', 'Aushang', '\u00C4u\u00DFerung']
-
-            # WHEN: We sort the list and use get_locale_key() to generate the sorting keys
-            sorted_list = sorted(unsorted_list, key=get_locale_key)
-
-            # THEN: We get a properly sorted list
-            self.assertEqual(['Aushang', '\u00C4u\u00DFerung', 'Auszug'], sorted_list,
-                             'Strings should be sorted properly')
-
-    def get_natural_key_test(self):
-        """
-        Test the get_natural_key(string) function
-        """
-        with patch('openlp.core.common.languagemanager.LanguageManager.get_language') as mocked_get_language:
-            # GIVEN: The language is English (a language, which sorts digits before letters)
-            mocked_get_language.return_value = 'en'
-            unsorted_list = ['item 10a', 'item 3b', '1st item']
-
-            # WHEN: We sort the list and use get_natural_key() to generate the sorting keys
-            sorted_list = sorted(unsorted_list, key=get_natural_key)
-
-            # THEN: We get a properly sorted list
-            self.assertEqual(['1st item', 'item 3b', 'item 10a'], sorted_list, 'Numbers should be sorted naturally')
-
-    def get_uno_instance_pipe_test(self):
-        """
-        Test that when the UNO connection type is "pipe" the resolver is given the "pipe" URI
-        """
-        # GIVEN: A mock resolver object and UNO_CONNECTION_TYPE is "pipe"
-        mock_resolver = MagicMock()
-
-        # WHEN: get_uno_instance() is called
-        get_uno_instance(mock_resolver)
-
-        # THEN: the resolve method is called with the correct argument
-        mock_resolver.resolve.assert_called_with('uno:pipe,name=openlp_pipe;urp;StarOffice.ComponentContext')
-
-    def get_uno_instance_socket_test(self):
-        """
-        Test that when the UNO connection type is other than "pipe" the resolver is given the "socket" URI
-        """
-        # GIVEN: A mock resolver object and UNO_CONNECTION_TYPE is "socket"
-        mock_resolver = MagicMock()
-
-        # WHEN: get_uno_instance() is called
-        get_uno_instance(mock_resolver, 'socket')
-
-        # THEN: the resolve method is called with the correct argument
-        mock_resolver.resolve.assert_called_with('uno:socket,host=localhost,port=2002;urp;StarOffice.ComponentContext')
-
-    def get_user_agent_linux_test(self):
-        """
-        Test that getting a user agent on Linux returns a user agent suitable for Linux
-        """
-        with patch('openlp.core.utils.sys') as mocked_sys:
-
-            # GIVEN: The system is Linux
-            mocked_sys.platform = 'linux2'
-
-            # WHEN: We call _get_user_agent()
-            user_agent = _get_user_agent()
-
-            # THEN: The user agent is a Linux (or ChromeOS) user agent
-            result = 'Linux' in user_agent or 'CrOS' in user_agent
-            self.assertTrue(result, 'The user agent should be a valid Linux user agent')
-
-    def get_user_agent_windows_test(self):
-        """
-        Test that getting a user agent on Windows returns a user agent suitable for Windows
-        """
-        with patch('openlp.core.utils.sys') as mocked_sys:
-
-            # GIVEN: The system is Linux
-            mocked_sys.platform = 'win32'
-
-            # WHEN: We call _get_user_agent()
-            user_agent = _get_user_agent()
-
-            # THEN: The user agent is a Linux (or ChromeOS) user agent
-            self.assertIn('Windows', user_agent, 'The user agent should be a valid Windows user agent')
-
-    def get_user_agent_macos_test(self):
-        """
-        Test that getting a user agent on OS X returns a user agent suitable for OS X
-        """
-        with patch('openlp.core.utils.sys') as mocked_sys:
-
-            # GIVEN: The system is Linux
-            mocked_sys.platform = 'darwin'
-
-            # WHEN: We call _get_user_agent()
-            user_agent = _get_user_agent()
-
-            # THEN: The user agent is a Linux (or ChromeOS) user agent
-            self.assertIn('Mac OS X', user_agent, 'The user agent should be a valid OS X user agent')
-
-    def get_user_agent_default_test(self):
-        """
-        Test that getting a user agent on a non-Linux/Windows/OS X platform returns the default user agent
-        """
-        with patch('openlp.core.utils.sys') as mocked_sys:
-
-            # GIVEN: The system is Linux
-            mocked_sys.platform = 'freebsd'
-
-            # WHEN: We call _get_user_agent()
-            user_agent = _get_user_agent()
-
-            # THEN: The user agent is a Linux (or ChromeOS) user agent
-            self.assertIn('NetBSD', user_agent, 'The user agent should be the default user agent')
-
-    def get_web_page_no_url_test(self):
-        """
-        Test that sending a URL of None to the get_web_page method returns None
-        """
-        # GIVEN: A None url
-        test_url = None
-
-        # WHEN: We try to get the test URL
-        result = get_web_page(test_url)
-
-        # THEN: None should be returned
-        self.assertIsNone(result, 'The return value of get_web_page should be None')
-
-    def get_web_page_test(self):
-        """
-        Test that the get_web_page method works correctly
-        """
-        with patch('openlp.core.utils.urllib.request.Request') as MockRequest, \
-                patch('openlp.core.utils.urllib.request.urlopen') as mock_urlopen, \
-                patch('openlp.core.utils._get_user_agent') as mock_get_user_agent, \
-                patch('openlp.core.utils.Registry') as MockRegistry:
-            # GIVEN: Mocked out objects and a fake URL
-            mocked_request_object = MagicMock()
-            MockRequest.return_value = mocked_request_object
-            mocked_page_object = MagicMock()
-            mock_urlopen.return_value = mocked_page_object
-            mock_get_user_agent.return_value = 'user_agent'
-            fake_url = 'this://is.a.fake/url'
-
-            # WHEN: The get_web_page() method is called
-            returned_page = get_web_page(fake_url)
-
-            # THEN: The correct methods are called with the correct arguments and a web page is returned
-            MockRequest.assert_called_with(fake_url)
-            mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
-            self.assertEqual(1, mocked_request_object.add_header.call_count,
-                             'There should only be 1 call to add_header')
-            mock_get_user_agent.assert_called_with()
-            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
-            mocked_page_object.geturl.assert_called_with()
-            self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called')
-            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
-
-    def get_web_page_with_header_test(self):
-        """
-        Test that adding a header to the call to get_web_page() adds the header to the request
-        """
-        with patch('openlp.core.utils.urllib.request.Request') as MockRequest, \
-                patch('openlp.core.utils.urllib.request.urlopen') as mock_urlopen, \
-                patch('openlp.core.utils._get_user_agent') as mock_get_user_agent:
-            # GIVEN: Mocked out objects, a fake URL and a fake header
-            mocked_request_object = MagicMock()
-            MockRequest.return_value = mocked_request_object
-            mocked_page_object = MagicMock()
-            mock_urlopen.return_value = mocked_page_object
-            mock_get_user_agent.return_value = 'user_agent'
-            fake_url = 'this://is.a.fake/url'
-            fake_header = ('Fake-Header', 'fake value')
-
-            # WHEN: The get_web_page() method is called
-            returned_page = get_web_page(fake_url, header=fake_header)
-
-            # THEN: The correct methods are called with the correct arguments and a web page is returned
-            MockRequest.assert_called_with(fake_url)
-            mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1])
-            self.assertEqual(2, mocked_request_object.add_header.call_count,
-                             'There should only be 2 calls to add_header')
-            mock_get_user_agent.assert_called_with()
-            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
-            mocked_page_object.geturl.assert_called_with()
-            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
-
-    def get_web_page_with_user_agent_in_headers_test(self):
-        """
-        Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request
-        """
-        with patch('openlp.core.utils.urllib.request.Request') as MockRequest, \
-                patch('openlp.core.utils.urllib.request.urlopen') as mock_urlopen, \
-                patch('openlp.core.utils._get_user_agent') as mock_get_user_agent:
-            # GIVEN: Mocked out objects, a fake URL and a fake header
-            mocked_request_object = MagicMock()
-            MockRequest.return_value = mocked_request_object
-            mocked_page_object = MagicMock()
-            mock_urlopen.return_value = mocked_page_object
-            fake_url = 'this://is.a.fake/url'
-            user_agent_header = ('User-Agent', 'OpenLP/2.2.0')
-
-            # WHEN: The get_web_page() method is called
-            returned_page = get_web_page(fake_url, header=user_agent_header)
-
-            # THEN: The correct methods are called with the correct arguments and a web page is returned
-            MockRequest.assert_called_with(fake_url)
-            mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1])
-            self.assertEqual(1, mocked_request_object.add_header.call_count,
-                             'There should only be 1 call to add_header')
-            self.assertEqual(0, mock_get_user_agent.call_count, '_get_user_agent should not have been called')
-            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
-            mocked_page_object.geturl.assert_called_with()
-            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
-
-    def get_web_page_update_openlp_test(self):
-        """
-        Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
-        """
-        with patch('openlp.core.utils.urllib.request.Request') as MockRequest, \
-                patch('openlp.core.utils.urllib.request.urlopen') as mock_urlopen, \
-                patch('openlp.core.utils._get_user_agent') as mock_get_user_agent, \
-                patch('openlp.core.utils.Registry') as MockRegistry:
-            # GIVEN: Mocked out objects, a fake URL
-            mocked_request_object = MagicMock()
-            MockRequest.return_value = mocked_request_object
-            mocked_page_object = MagicMock()
-            mock_urlopen.return_value = mocked_page_object
-            mock_get_user_agent.return_value = 'user_agent'
-            mocked_registry_object = MagicMock()
-            mocked_application_object = MagicMock()
-            mocked_registry_object.get.return_value = mocked_application_object
-            MockRegistry.return_value = mocked_registry_object
-            fake_url = 'this://is.a.fake/url'
-
-            # WHEN: The get_web_page() method is called
-            returned_page = get_web_page(fake_url, update_openlp=True)
-
-            # THEN: The correct methods are called with the correct arguments and a web page is returned
-            MockRequest.assert_called_with(fake_url)
-            mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
-            self.assertEqual(1, mocked_request_object.add_header.call_count,
-                             'There should only be 1 call to add_header')
-            mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
-            mocked_page_object.geturl.assert_called_with()
-            mocked_registry_object.get.assert_called_with('application')
-            mocked_application_object.process_events.assert_called_with()
-            self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')

tests/functional/openlp_plugins/remotes/test_remotetab.py

@@ -99,7 +99,7 @@ class TestRemoteTab(TestCase, TestMixin):
         """
         # GIVEN: A mocked location
         with patch('openlp.core.common.Settings') as mocked_class, \
-                patch('openlp.core.utils.AppLocation.get_directory') as mocked_get_directory, \
+                patch('openlp.core.common.applocation.AppLocation.get_directory') as mocked_get_directory, \
                 patch('openlp.core.common.check_directory_exists') as mocked_check_directory_exists, \
                 patch('openlp.core.common.applocation.os') as mocked_os:
             # GIVEN: A mocked out Settings class and a mocked out AppLocation.get_directory()
@@ -127,7 +127,7 @@ class TestRemoteTab(TestCase, TestMixin):
         """
         # GIVEN: A mocked location
         with patch('openlp.core.common.Settings') as mocked_class, \
-                patch('openlp.core.utils.AppLocation.get_directory') as mocked_get_directory, \
+                patch('openlp.core.common.applocation.AppLocation.get_directory') as mocked_get_directory, \
                 patch('openlp.core.common.check_directory_exists') as mocked_check_directory_exists, \
                 patch('openlp.core.common.applocation.os') as mocked_os:
             # GIVEN: A mocked out Settings class and a mocked out AppLocation.get_directory()

tests/interfaces/openlp_core_common/test_utils.py

@@ -25,7 +25,7 @@ Functional tests to test the AppLocation class and related methods.
 import os
 from unittest import TestCase
 
-from openlp.core.utils import is_not_image_file
+from openlp.core.common import is_not_image_file
 from tests.utils.constants import TEST_RESOURCES_PATH
 from tests.helpers.testmixin import TestMixin
 

tests/interfaces/openlp_core_utils/__init__.py

@@ -1,21 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
-
-###############################################################################
-# OpenLP - Open Source Lyrics Projection                                      #
-# --------------------------------------------------------------------------- #
-# Copyright (c) 2008-2016 OpenLP Developers                                   #
-# --------------------------------------------------------------------------- #
-# This program is free software; you can redistribute it and/or modify it     #
-# under the terms of the GNU General Public License as published by the Free  #
-# Software Foundation; version 2 of the License.                              #
-#                                                                             #
-# This program is distributed in the hope that it will be useful, but WITHOUT #
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
-# more details.                                                               #
-#                                                                             #
-# You should have received a copy of the GNU General Public License along     #
-# with this program; if not, write to the Free Software Foundation, Inc., 59  #
-# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
-###############################################################################