Merge trunk.

openlp/core/ui/exceptionform.py

@@ -69,6 +69,14 @@ try:
     MAKO_VERSION = mako.__version__
 except ImportError:
     MAKO_VERSION = u'-'
+try:
+    import icu
+    try:
+       ICU_VERSION = icu.VERSION
+    except AttributeError:
+       ICU_VERSION = u'OK'
+except ImportError:
+    ICU_VERSION = u'-'
 try:
     import uno
     arg = uno.createUnoStruct(u'com.sun.star.beans.PropertyValue')
@@ -143,6 +151,7 @@ class ExceptionForm(QtGui.QDialog, Ui_ExceptionDialog):
             u'PyEnchant: %s\n' % ENCHANT_VERSION + \
             u'PySQLite: %s\n' % SQLITE_VERSION + \
             u'Mako: %s\n' % MAKO_VERSION + \
+            u'pyICU: %s\n' % ICU_VERSION + \
             u'pyUNO bridge: %s\n' % UNO_VERSION + \
             u'VLC: %s\n' % VLC_VERSION
         if platform.system() == u'Linux':

openlp/core/ui/thememanager.py

@@ -44,7 +44,7 @@ from openlp.core.lib.theme import ThemeXML, BackgroundType, VerticalType, Backgr
 from openlp.core.lib.ui import critical_error_message_box, create_widget_action
 from openlp.core.theme import Theme
 from openlp.core.ui import FileRenameForm, ThemeForm
-from openlp.core.utils import AppLocation, delete_file, locale_compare, get_filesystem_encoding
+from openlp.core.utils import AppLocation, delete_file, get_locale_key, get_filesystem_encoding
 
 log = logging.getLogger(__name__)
 
@@ -418,7 +418,7 @@ class ThemeManager(QtGui.QWidget):
         self.theme_list_widget.clear()
         files = AppLocation.get_files(self.settings_section, u'.png')
         # Sort the themes by its name considering language specific
-        files.sort(key=lambda file_name: unicode(file_name), cmp=locale_compare)
+        files.sort(key=lambda file_name: get_locale_key(unicode(file_name)))
         # now process the file list of png files
         for name in files:
             # check to see file is in theme root directory

openlp/core/utils/__init__.py

@@ -38,6 +38,7 @@ import re
 from subprocess import Popen, PIPE
 import sys
 import urllib2
+import icu
 
 from PyQt4 import QtGui, QtCore
 
@@ -56,10 +57,12 @@ from openlp.core.lib import translate
 log = logging.getLogger(__name__)
 APPLICATION_VERSION = {}
 IMAGES_FILTER = None
+ICU_COLLATOR = None
 UNO_CONNECTION_TYPE = u'pipe'
 #UNO_CONNECTION_TYPE = u'socket'
 CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]', re.UNICODE)
 INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]', re.UNICODE)
+DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+', re.UNICODE)
 
 
 class VersionThread(QtCore.QThread):
@@ -379,21 +382,32 @@ def format_time(text, local_time):
     return re.sub('\%[a-zA-Z]', match_formatting, text)
 
 
-def locale_compare(string1, string2):
+def get_locale_key(string):
     """
-    Compares two strings according to the current locale settings.
+    Creates a key for case insensitive, locale aware string sorting.
-
-    As any other compare function, returns a negative, or a positive value,
-    or 0, depending on whether string1 collates before or after string2 or
-    is equal to it. Comparison is case insensitive.
     """
-    # Function locale.strcoll() from standard Python library does not work properly on Windows.
+    string = string.lower()
-    return locale.strcoll(string1.lower(), string2.lower())
+    # For Python 3 on platforms other than Windows ICU is not necessary. In those cases locale.strxfrm(str) can be used.
+    global ICU_COLLATOR
+    if ICU_COLLATOR is None:
+        from languagemanager import LanguageManager
+        locale = LanguageManager.get_language()
+        icu_locale = icu.Locale(locale)
+        ICU_COLLATOR = icu.Collator.createInstance(icu_locale)
+    return ICU_COLLATOR.getSortKey(string)
 
 
-# For performance reasons provide direct reference to compare function without wrapping it in another function making
+def get_natural_key(string):
-# the string lowercase. This is needed for sorting songs.
+    """
-locale_direct_compare = locale.strcoll
+    Generate a key for locale aware natural string sorting.
+    Returns a list of string compare keys and integers.
+    """
+    key = DIGITS_OR_NONDIGITS.findall(string)
+    key = [int(part) if part.isdigit() else get_locale_key(part) for part in key]
+    # Python 3 does not support comparision of different types anymore. So make sure, that we do not compare str and int.
+    #if string[0].isdigit():
+    #    return [''] + key 
+    return key
 
 
 from applocation import AppLocation
@@ -403,4 +417,4 @@ from actions import ActionList
 
 __all__ = [u'AppLocation', u'ActionList', u'LanguageManager', u'get_application_version', u'check_latest_version',
     u'add_actions', u'get_filesystem_encoding', u'get_web_page', u'get_uno_command', u'get_uno_instance',
-    u'delete_file', u'clean_filename', u'format_time', u'locale_compare', u'locale_direct_compare']
+    u'delete_file', u'clean_filename', u'format_time', u'get_locale_key', u'get_natural_key']

openlp/plugins/bibles/forms/bibleimportform.py

@@ -38,7 +38,7 @@ from openlp.core.lib import Settings, UiStrings, translate
 from openlp.core.lib.db import delete_database
 from openlp.core.lib.ui import critical_error_message_box
 from openlp.core.ui.wizard import OpenLPWizard, WizardStrings
-from openlp.core.utils import AppLocation, locale_compare
+from openlp.core.utils import AppLocation, get_locale_key
 from openlp.plugins.bibles.lib.manager import BibleFormat
 from openlp.plugins.bibles.lib.db import BiblesResourcesDB, clean_filename
 
@@ -455,7 +455,7 @@ class BibleImportForm(OpenLPWizard):
         """
         self.webTranslationComboBox.clear()
         bibles = self.web_bible_list[index].keys()
-        bibles.sort(cmp=locale_compare)
+        bibles.sort(key=get_locale_key)
         self.webTranslationComboBox.addItems(bibles)
 
     def onOsisBrowseButtonClicked(self):

openlp/plugins/bibles/lib/mediaitem.py

@@ -36,7 +36,7 @@ from openlp.core.lib import Registry, MediaManagerItem, ItemCapabilities, Servic
 from openlp.core.lib.searchedit import SearchEdit
 from openlp.core.lib.ui import set_case_insensitive_completer, create_horizontal_adjusting_combo_box, \
     critical_error_message_box, find_and_set_in_combo_box, build_icon
-from openlp.core.utils import locale_compare
+from openlp.core.utils import get_locale_key
 from openlp.plugins.bibles.forms import BibleImportForm, EditBibleForm
 from openlp.plugins.bibles.lib import LayoutStyle, DisplayStyle, VerseReferenceList, get_reference_separator, \
     LanguageSelection, BibleStrings
@@ -325,7 +325,7 @@ class BibleMediaItem(MediaManagerItem):
         # Get all bibles and sort the list.
         bibles = self.plugin.manager.get_bibles().keys()
         bibles = filter(None, bibles)
-        bibles.sort(cmp=locale_compare)
+        bibles.sort(key=get_locale_key)
         # Load the bibles into the combo boxes.
         self.quickVersionComboBox.addItems(bibles)
         self.quickSecondComboBox.addItems(bibles)
@@ -461,7 +461,7 @@ class BibleMediaItem(MediaManagerItem):
                     for book in book_data:
                         data = BiblesResourcesDB.get_book_by_id(book.book_reference_id)
                         books.append(data[u'name'] + u' ')
-                books.sort(cmp=locale_compare)
+                books.sort(key=get_locale_key)
         set_case_insensitive_completer(books, self.quickSearchEdit)
 
     def on_import_click(self):

openlp/plugins/custom/lib/db.py

@@ -35,7 +35,7 @@ from sqlalchemy import Column, Table, types
 from sqlalchemy.orm import mapper
 
 from openlp.core.lib.db import BaseModel, init_db
-from openlp.core.utils import locale_compare
+from openlp.core.utils import get_locale_key
 
 class CustomSlide(BaseModel):
     """
@@ -44,11 +44,10 @@ class CustomSlide(BaseModel):
     # By default sort the customs by its title considering language specific
     # characters.
     def __lt__(self, other):
-        r = locale_compare(self.title, other.title)
+        return get_locale_key(self.title) < get_locale_key(other.title)
-        return True if r < 0 else False
 
     def __eq__(self, other):
-        return 0 == locale_compare(self.title, other.title)
+        return get_locale_key(self.title) == get_locale_key(other.title)
 
 
 def init_schema(url):

openlp/plugins/images/lib/mediaitem.py

@@ -36,7 +36,7 @@ from openlp.core.lib import ItemCapabilities, MediaManagerItem, Registry, Servic
     StringContent, TreeWidgetWithDnD, UiStrings, build_icon, check_directory_exists, check_item_selected, \
     create_thumb, translate, validate_thumb
 from openlp.core.lib.ui import create_widget_action, critical_error_message_box
-from openlp.core.utils import AppLocation, delete_file, locale_compare, get_images_filter
+from openlp.core.utils import AppLocation, delete_file, get_locale_key, get_images_filter
 from openlp.plugins.images.forms import AddGroupForm, ChooseGroupForm
 from openlp.plugins.images.lib.db import ImageFilenames, ImageGroups
 
@@ -255,7 +255,7 @@ class ImageMediaItem(MediaManagerItem):
             The ID of the group that will be added recursively
         """
         image_groups = self.manager.get_all_objects(ImageGroups, ImageGroups.parent_id == parent_group_id)
-        image_groups.sort(cmp=locale_compare, key=lambda group_object: group_object.group_name)
+        image_groups.sort(key=lambda group_object: get_locale_key(group_object.group_name))
         folder_icon = build_icon(u':/images/image_group.png')
         for image_group in image_groups:
             group = QtGui.QTreeWidgetItem()
@@ -286,7 +286,7 @@ class ImageMediaItem(MediaManagerItem):
             combobox.clear()
             combobox.top_level_group_added = False
         image_groups = self.manager.get_all_objects(ImageGroups, ImageGroups.parent_id == parent_group_id)
-        image_groups.sort(cmp=locale_compare, key=lambda group_object: group_object.group_name)
+        image_groups.sort(key=lambda group_object: get_locale_key(group_object.group_name))
         for image_group in image_groups:
             combobox.addItem(prefix + image_group.group_name, image_group.id)
             self.fill_groups_combobox(combobox, image_group.id, prefix + '   ')
@@ -338,7 +338,7 @@ class ImageMediaItem(MediaManagerItem):
             self.expand_group(open_group.id)
         # Sort the images by its filename considering language specific
         # characters.
-        images.sort(cmp=locale_compare, key=lambda image_object: os.path.split(unicode(image_object.filename))[1])
+        images.sort(key=lambda image_object: get_locale_key(os.path.split(unicode(image_object.filename))[1]))
         for imageFile in images:
             log.debug(u'Loading image: %s', imageFile.filename)
             filename = os.path.split(imageFile.filename)[1]
@@ -525,9 +525,9 @@ class ImageMediaItem(MediaManagerItem):
                 group_items.append(item)
             if isinstance(item.data(0, QtCore.Qt.UserRole), ImageFilenames):
                 image_items.append(item)
-        group_items.sort(cmp=locale_compare, key=lambda item: item.text(0))
+        group_items.sort(key=lambda item: get_locale_key(item.text(0)))
         target_group.addChildren(group_items)
-        image_items.sort(cmp=locale_compare, key=lambda item: item.text(0))
+        image_items.sort(key=lambda item: get_locale_key(item.text(0)))
         target_group.addChildren(image_items)
 
     def generate_slide_data(self, service_item, item=None, xmlVersion=False,

openlp/plugins/media/lib/mediaitem.py

@@ -37,7 +37,7 @@ from openlp.core.lib import ItemCapabilities, MediaManagerItem,MediaType, Regist
 from openlp.core.lib.ui import critical_error_message_box, create_horizontal_adjusting_combo_box
 from openlp.core.ui import DisplayController, Display, DisplayControllerType
 from openlp.core.ui.media import get_media_players, set_media_players
-from openlp.core.utils import AppLocation, locale_compare
+from openlp.core.utils import AppLocation, get_locale_key
 
 log = logging.getLogger(__name__)
 
@@ -261,7 +261,7 @@ class MediaMediaItem(MediaManagerItem):
     def load_list(self, media, target_group=None):
         # Sort the media by its filename considering language specific
         # characters.
-        media.sort(cmp=locale_compare, key=lambda filename: os.path.split(unicode(filename))[1])
+        media.sort(key=lambda filename: get_locale_key(os.path.split(unicode(filename))[1]))
         for track in media:
             track_info = QtCore.QFileInfo(track)
             if not os.path.exists(track):
@@ -287,7 +287,7 @@ class MediaMediaItem(MediaManagerItem):
 
     def getList(self, type=MediaType.Audio):
         media = Settings().value(self.settings_section + u'/media files')
-        media.sort(cmp=locale_compare, key=lambda filename: os.path.split(unicode(filename))[1])
+        media.sort(key=lambda filename: get_locale_key(os.path.split(unicode(filename))[1]))
         ext = []
         if type == MediaType.Audio:
             ext = self.media_controller.audio_extensions_list

openlp/plugins/presentations/lib/mediaitem.py

@@ -35,7 +35,7 @@ from PyQt4 import QtCore, QtGui
 from openlp.core.lib import MediaManagerItem, Registry, ItemCapabilities, ServiceItemContext, Settings, UiStrings, \
     build_icon, check_item_selected, create_thumb, translate, validate_thumb
 from openlp.core.lib.ui import critical_error_message_box, create_horizontal_adjusting_combo_box
-from openlp.core.utils import locale_compare
+from openlp.core.utils import get_locale_key
 from openlp.plugins.presentations.lib import MessageListener
 
 log = logging.getLogger(__name__)
@@ -153,8 +153,7 @@ class PresentationMediaItem(MediaManagerItem):
         if not initialLoad:
             self.main_window.display_progress_bar(len(files))
         # Sort the presentations by its filename considering language specific characters.
-        files.sort(cmp=locale_compare,
+        files.sort(key=lambda filename: get_locale_key(os.path.split(unicode(filename))[1]))
-            key=lambda filename: os.path.split(unicode(filename))[1])
         for file in files:
             if not initialLoad:
                 self.main_window.increment_progress_bar()

openlp/plugins/songs/forms/songexportform.py

@@ -37,7 +37,6 @@ from PyQt4 import QtCore, QtGui
 from openlp.core.lib import Registry, UiStrings, create_separated_list, build_icon, translate
 from openlp.core.lib.ui import critical_error_message_box
 from openlp.core.ui.wizard import OpenLPWizard, WizardStrings
-from openlp.plugins.songs.lib import natcmp
 from openlp.plugins.songs.lib.db import Song
 from openlp.plugins.songs.lib.openlyricsexport import OpenLyricsExport
 
@@ -222,7 +221,7 @@ class SongExportForm(OpenLPWizard):
         # Load the list of songs.
         self.application.set_busy_cursor()
         songs = self.plugin.manager.get_all_objects(Song)
-        songs.sort(cmp=natcmp, key=lambda song: song.sort_key)
+        songs.sort(key=lambda song: song.sort_key)
         for song in songs:
             # No need to export temporary songs.
             if song.temporary:

openlp/plugins/songs/lib/__init__.py

@@ -34,7 +34,7 @@ import re
 from PyQt4 import QtGui
 
 from openlp.core.lib import translate
-from openlp.core.utils import CONTROL_CHARS, locale_direct_compare
+from openlp.core.utils import CONTROL_CHARS
 from db import Author
 from ui import SongStrings
 
@@ -168,6 +168,7 @@ class VerseType(object):
         translate('SongsPlugin.VerseType', 'Intro'),
         translate('SongsPlugin.VerseType', 'Ending'),
         translate('SongsPlugin.VerseType', 'Other')]
+
     translated_tags = [name[0].lower() for name in translated_names]
 
     @staticmethod
@@ -592,37 +593,3 @@ def strip_rtf(text, default_encoding=None):
     text = u''.join(out)
     return text, default_encoding
 
-
-def natcmp(a, b):
-    """
-    Natural string comparison which mimics the behaviour of Python's internal cmp function.
-    """
-    if len(a) <= len(b):
-        for i, key in enumerate(a):
-            if isinstance(key, int) and isinstance(b[i], int):
-                result = cmp(key, b[i])
-            elif isinstance(key, int) and not isinstance(b[i], int):
-                result = locale_direct_compare(str(key), b[i])
-            elif not isinstance(key, int) and isinstance(b[i], int):
-                result = locale_direct_compare(key, str(b[i]))
-            else:
-                result = locale_direct_compare(key, b[i])
-            if result != 0:
-                return result
-        if len(a) == len(b):
-            return 0
-        else:
-            return -1
-    else:
-        for i, key in enumerate(b):
-            if isinstance(a[i], int) and isinstance(key, int):
-                result = cmp(a[i], key)
-            elif isinstance(a[i], int) and not isinstance(key, int):
-                result = locale_direct_compare(str(a[i]), key)
-            elif not isinstance(a[i], int) and isinstance(key, int):
-                result = locale_direct_compare(a[i], str(key))
-            else:
-                result = locale_direct_compare(a[i], key)
-            if result != 0:
-                return result
-        return 1

openlp/plugins/songs/lib/db.py

@@ -38,6 +38,7 @@ from sqlalchemy.orm import mapper, relation, reconstructor
 from sqlalchemy.sql.expression import func
 
 from openlp.core.lib.db import BaseModel, init_db
+from openlp.core.utils import get_natural_key
 
 
 class Author(BaseModel):
@@ -69,36 +70,15 @@ class Song(BaseModel):
     def __init__(self):
         self.sort_key = ()
 
-    def _try_int(self, s):
-        """
-        Convert to integer if possible.
-        """
-        try:
-            return int(s)
-        except:
-            return s.lower()
-
-    def _natsort_key(self, s):
-        """
-        Used internally to get a tuple by which s is sorted.
-        """
-        return map(self._try_int, re.findall(r'(\d+|\D+)', s))
-
-    # This decorator tells sqlalchemy to call this method everytime
-    # any data on this object is updated.
-
     @reconstructor
     def init_on_load(self):
         """
-        Precompute a tuple to be used for sorting.
+        Precompute a natural sorting, locale aware sorting key.
 
         Song sorting is performance sensitive operation.
-        To get maximum speed lets precompute the string
+        To get maximum speed lets precompute the sorting key.
-        used for comparison.
         """
-        # Avoid the overhead of converting string to lowercase and to QString
+        self.sort_key = get_natural_key(self.title)
-        # with every call to sort().
-        self.sort_key = self._natsort_key(self.title)
 
 
 class Topic(BaseModel):

openlp/plugins/songs/lib/mediaitem.py

@@ -43,7 +43,7 @@ from openlp.plugins.songs.forms.editsongform import EditSongForm
 from openlp.plugins.songs.forms.songmaintenanceform import SongMaintenanceForm
 from openlp.plugins.songs.forms.songimportform import SongImportForm
 from openlp.plugins.songs.forms.songexportform import SongExportForm
-from openlp.plugins.songs.lib import VerseType, clean_string, natcmp
+from openlp.plugins.songs.lib import VerseType, clean_string
 from openlp.plugins.songs.lib.db import Author, Song, Book, MediaFile
 from openlp.plugins.songs.lib.ui import SongStrings
 from openlp.plugins.songs.lib.xml import OpenLyrics, SongXML
@@ -225,7 +225,7 @@ class SongMediaItem(MediaManagerItem):
         log.debug(u'display results Song')
         self.save_auto_select_id()
         self.list_view.clear()
-        searchresults.sort(cmp=natcmp, key=lambda song: song.sort_key)
+        searchresults.sort(key=lambda song: song.sort_key)
         for song in searchresults:
             # Do not display temporary songs
             if song.temporary:

openlp/plugins/songs/lib/songimport.py

@@ -260,7 +260,10 @@ class SongImport(QtCore.QObject):
         elif int(verse_def[1:]) > self.verseCounts[verse_def[0]]:
             self.verseCounts[verse_def[0]] = int(verse_def[1:])
         self.verses.append([verse_def, verse_text.rstrip(), lang])
-        self.verseOrderListGenerated.append(verse_def)
+        # A verse_def refers to all verses with that name, adding it once adds every instance, so do not add if already
+        # used.
+        if verse_def not in self.verseOrderListGenerated:
+            self.verseOrderListGenerated.append(verse_def)
 
     def repeatVerse(self):
         """

openlp/plugins/songs/lib/songshowplusimport.py

@@ -32,6 +32,7 @@ SongShow Plus songs into the OpenLP database.
 """
 import os
 import logging
+import re
 import struct
 
 from openlp.core.ui.wizard import WizardStrings
@@ -44,43 +45,36 @@ COPYRIGHT = 3
 CCLI_NO = 5
 VERSE = 12
 CHORUS = 20
+BRIDGE = 24
 TOPIC = 29
 COMMENTS = 30
 VERSE_ORDER = 31
 SONG_BOOK = 35
 SONG_NUMBER = 36
 CUSTOM_VERSE = 37
-BRIDGE = 24
 
 log = logging.getLogger(__name__)
 
 class SongShowPlusImport(SongImport):
     """
-    The :class:`SongShowPlusImport` class provides the ability to import song
+    The :class:`SongShowPlusImport` class provides the ability to import song files from SongShow Plus.
-    files from SongShow Plus.
 
     **SongShow Plus Song File Format:**
 
     The SongShow Plus song file format is as follows:
 
-    * Each piece of data in the song file has some information that precedes
+    * Each piece of data in the song file has some information that precedes it.
-    it.
     * The general format of this data is as follows:
-    4 Bytes, forming a 32 bit number, a key if you will, this describes what
+    4 Bytes, forming a 32 bit number, a key if you will, this describes what the data is (see blockKey below)
-    the data is (see blockKey below)
+    4 Bytes, forming a 32 bit number, which is the number of bytes until the next block starts
-    4 Bytes, forming a 32 bit number, which is the number of bytes until the
-    next block starts
     1 Byte, which tells how many bytes follows
-    1 or 4 Bytes, describes how long the string is, if its 1 byte, the string
+    1 or 4 Bytes, describes how long the string is, if its 1 byte, the string is less than 255
-    is less than 255
     The next bytes are the actual data.
     The next block of data follows on.
 
-    This description does differ for verses. Which includes extra bytes
+    This description does differ for verses. Which includes extra bytes stating the verse type or number. In some cases
-    stating the verse type or number. In some cases a "custom" verse is used,
+    a "custom" verse is used, in that case, this block will in include 2 strings, with the associated string length
-    in that case, this block will in include 2 strings, with the associated
+    descriptors. The first string is the name of the verse, the second is the verse content.
-    string length descriptors. The first string is the name of the verse, the
-    second is the verse content.
 
     The file is ended with four null bytes.
 
@@ -88,8 +82,9 @@ class SongShowPlusImport(SongImport):
 
     * .sbsong
     """
-    otherList = {}
+
-    otherCount = 0
+    other_count = 0
+    other_list = {}
 
     def __init__(self, manager, **kwargs):
         """
@@ -107,9 +102,9 @@ class SongShowPlusImport(SongImport):
         for file in self.import_source:
             if self.stop_import_flag:
                 return
-            self.sspVerseOrderList = []
+            self.ssp_verse_order_list = []
-            other_count = 0
+            self.other_count = 0
-            other_list = {}
+            self.other_list = {}
             file_name = os.path.split(file)[1]
             self.import_wizard.increment_progress_bar(WizardStrings.ImportingType % file_name, 0)
             song_data = open(file, 'rb')
@@ -162,34 +157,37 @@ class SongShowPlusImport(SongImport):
                 elif block_key == COMMENTS:
                     self.comments = unicode(data, u'cp1252')
                 elif block_key == VERSE_ORDER:
-                    verse_tag = self.toOpenLPVerseTag(data, True)
+                    verse_tag = self.to_openlp_verse_tag(data, True)
                     if verse_tag:
                         if not isinstance(verse_tag, unicode):
                             verse_tag = unicode(verse_tag, u'cp1252')
-                        self.sspVerseOrderList.append(verse_tag)
+                        self.ssp_verse_order_list.append(verse_tag)
                 elif block_key == SONG_BOOK:
                     self.songBookName = unicode(data, u'cp1252')
                 elif block_key == SONG_NUMBER:
                     self.songNumber = ord(data)
                 elif block_key == CUSTOM_VERSE:
-                    verse_tag = self.toOpenLPVerseTag(verse_name)
+                    verse_tag = self.to_openlp_verse_tag(verse_name)
                     self.addVerse(unicode(data, u'cp1252'), verse_tag)
                 else:
                     log.debug("Unrecognised blockKey: %s, data: %s" % (block_key, data))
                     song_data.seek(next_block_starts)
-            self.verseOrderList = self.sspVerseOrderList
+            self.verseOrderList = self.ssp_verse_order_list
             song_data.close()
             if not self.finish():
                 self.logError(file)
 
-    def toOpenLPVerseTag(self, verse_name, ignore_unique=False):
+    def to_openlp_verse_tag(self, verse_name, ignore_unique=False):
-        if verse_name.find(" ") != -1:
+        # Have we got any digits? If so, verse number is everything from the digits to the end (OpenLP does not have
-            verse_parts = verse_name.split(" ")
+        # concept of part verses, so just ignore any non integers on the end (including floats))
-            verse_type = verse_parts[0]
+        match = re.match(r'(\D*)(\d+)', verse_name)
-            verse_number = verse_parts[1]
+        if match:
+            verse_type = match.group(1).strip()
+            verse_number = match.group(2)
         else:
+            # otherwise we assume number 1 and take the whole prefix as the verse tag
             verse_type = verse_name
-            verse_number = "1"
+            verse_number = u'1'
         verse_type = verse_type.lower()
         if verse_type == "verse":
             verse_tag = VerseType.tags[VerseType.Verse]
@@ -200,11 +198,11 @@ class SongShowPlusImport(SongImport):
         elif verse_type == "pre-chorus":
             verse_tag = VerseType.tags[VerseType.PreChorus]
         else:
-            if verse_name not in self.otherList:
+            if verse_name not in self.other_list:
                 if ignore_unique:
                     return None
-                self.otherCount += 1
+                self.other_count += 1
-                self.otherList[verse_name] = str(self.otherCount)
+                self.other_list[verse_name] = str(self.other_count)
             verse_tag = VerseType.tags[VerseType.Other]
-            verse_number = self.otherList[verse_name]
+            verse_number = self.other_list[verse_name]
         return verse_tag + verse_number

scripts/check_dependencies.py

@@ -83,6 +83,7 @@ MODULES = [
     'mako',
     'migrate',
     'uno',
+    'icu',
 ]
 
 

tests/functional/openlp_core_utils/test_utils.py

@@ -5,7 +5,7 @@ from unittest import TestCase
 
 from mock import patch
 
-from openlp.core.utils import get_filesystem_encoding, _get_frozen_path
+from openlp.core.utils import get_filesystem_encoding, _get_frozen_path, get_locale_key, get_natural_key
 
 class TestUtils(TestCase):
     """
@@ -56,3 +56,30 @@ class TestUtils(TestCase):
             # THEN: The frozen parameter is returned
             assert _get_frozen_path(u'frozen', u'not frozen') == u'frozen', u'Should return "frozen"'
 
+    def get_locale_key_test(self):
+        """
+        Test the get_locale_key(string) function
+        """
+        with patch(u'openlp.core.utils.languagemanager.LanguageManager.get_language') as mocked_get_language:
+            # GIVEN: The language is German
+            # 0x00C3 (A with diaresis) should be sorted as "A". 0x00DF (sharp s) should be sorted as "ss".
+            mocked_get_language.return_value = u'de'
+            unsorted_list = [u'Auszug', u'Aushang', u'\u00C4u\u00DFerung']
+            # WHEN: We sort the list and use get_locale_key() to generate the sorting keys
+            # THEN: We get a properly sorted list
+            test_passes = sorted(unsorted_list, key=get_locale_key) == [u'Aushang', u'\u00C4u\u00DFerung', u'Auszug']
+            assert test_passes, u'Strings should be sorted properly'
+
+    def get_natural_key_test(self):
+        """
+        Test the get_natural_key(string) function
+        """
+        with patch(u'openlp.core.utils.languagemanager.LanguageManager.get_language') as mocked_get_language:
+            # GIVEN: The language is English (a language, which sorts digits before letters)
+            mocked_get_language.return_value = u'en'
+            unsorted_list = [u'item 10a', u'item 3b', u'1st item']
+            # WHEN: We sort the list and use get_natural_key() to generate the sorting keys
+            # THEN: We get a properly sorted list
+            test_passes = sorted(unsorted_list, key=get_natural_key) == [u'1st item', u'item 3b', u'item 10a']
+            assert test_passes, u'Numbers should be sorted naturally'
+

tests/functional/openlp_plugins/songs/test_songshowplusimport.py

@@ -0,0 +1,235 @@
+"""
+This module contains tests for the SongShow Plus song importer.
+"""
+
+import os
+from unittest import TestCase
+from mock import patch, MagicMock
+
+from openlp.plugins.songs.lib import VerseType
+from openlp.plugins.songs.lib.songshowplusimport import SongShowPlusImport
+
+TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), u'../../../resources/songshowplussongs'))
+SONG_TEST_DATA = {u'Amazing Grace.sbsong':
+        {u'title': u'Amazing Grace (Demonstration)',
+         u'authors': [u'John Newton', u'Edwin Excell', u'John P. Rees'],
+         u'copyright': u'Public Domain ',
+         u'ccli_number': 22025,
+         u'verses':
+             [(u'Amazing grace! How sweet the sound!\r\nThat saved a wretch like me!\r\n'
+               u'I once was lost, but now am found;\r\nWas blind, but now I see.', u'v1'),
+              (u'\'Twas grace that taught my heart to fear,\r\nAnd grace my fears relieved.\r\n'
+               u'How precious did that grace appear,\r\nThe hour I first believed.', u'v2'),
+              (u'The Lord has promised good to me,\r\nHis Word my hope secures.\r\n'
+               u'He will my shield and portion be\r\nAs long as life endures.', u'v3'),
+              (u'Thro\' many dangers, toils and snares\r\nI have already come.\r\n'
+               u'\'Tis grace that brought me safe thus far,\r\nAnd grace will lead me home.', u'v4'),
+              (u'When we\'ve been there ten thousand years,\r\nBright shining as the sun,\r\n'
+               u'We\'ve no less days to sing God\'s praise,\r\nThan when we first begun.', u'v5')],
+         u'topics': [u'Assurance', u'Grace', u'Praise', u'Salvation'],
+         u'comments': u'\n\n\n',
+         u'song_book_name': u'Demonstration Songs',
+         u'song_number': 0,
+         u'verse_order_list': []},
+    u'Beautiful Garden Of Prayer.sbsong':
+        {u'title': u'Beautiful Garden Of Prayer (Demonstration)',
+        u'authors': [u'Eleanor Allen Schroll', u'James H. Fillmore'],
+        u'copyright': u'Public Domain ',
+        u'ccli_number': 60252,
+        u'verses':
+           [(u'There\'s a garden where Jesus is waiting,\r\nThere\'s a place that is wondrously fair.\r\n'
+             u'For it glows with the light of His presence,\r\n\'Tis the beautiful garden of prayer.', u'v1'),
+            (u'There\'s a garden where Jesus is waiting,\r\nAnd I go with my burden and care.\r\n'
+             u'Just to learn from His lips, words of comfort,\r\nIn the beautiful garden of prayer.', u'v2'),
+            (u'There\'s a garden where Jesus is waiting,\r\nAnd He bids you to come meet Him there,\r\n'
+             u'Just to bow and receive a new blessing,\r\nIn the beautiful garden of prayer.', u'v3'),
+            (u'O the beautiful garden, the garden of prayer,\r\nO the beautiful garden of prayer.\r\n'
+             u'There my Savior awaits, and He opens the gates\r\nTo the beautiful garden of prayer.', u'c1')],
+        u'topics': [u'Devotion', u'Prayer'],
+        u'comments': u'',
+        u'song_book_name': u'',
+        u'song_number': 0,
+        u'verse_order_list': []}}
+
+
+class TestSongShowPlusImport(TestCase):
+    """
+    Test the functions in the :mod:`songshowplusimport` module.
+    """
+    def create_importer_test(self):
+        """
+        Test creating an instance of the SongShow Plus file importer
+        """
+        # GIVEN: A mocked out SongImport class, and a mocked out "manager"
+        with patch(u'openlp.plugins.songs.lib.songshowplusimport.SongImport'):
+            mocked_manager = MagicMock()
+
+            # WHEN: An importer object is created
+            importer = SongShowPlusImport(mocked_manager)
+
+            # THEN: The importer object should not be None
+            self.assertIsNotNone(importer, u'Import should not be none')
+
+    def invalid_import_source_test(self):
+        """
+        Test SongShowPlusImport.doImport handles different invalid import_source values
+        """
+        # GIVEN: A mocked out SongImport class, and a mocked out "manager"
+        with patch(u'openlp.plugins.songs.lib.songshowplusimport.SongImport'):
+            mocked_manager = MagicMock()
+            mocked_import_wizard = MagicMock()
+            importer = SongShowPlusImport(mocked_manager)
+            importer.import_wizard = mocked_import_wizard
+            importer.stop_import_flag = True
+
+            # WHEN: Import source is not a list
+            for source in [u'not a list', 0]:
+                importer.import_source = source
+
+                # THEN: doImport should return none and the progress bar maximum should not be set.
+                self.assertIsNone(importer.doImport(), u'doImport should return None when import_source is not a list')
+                self.assertEquals(mocked_import_wizard.progress_bar.setMaximum.called, False,
+                                  u'setMaxium on import_wizard.progress_bar should not have been called')
+
+    def valid_import_source_test(self):
+        """
+        Test SongShowPlusImport.doImport handles different invalid import_source values
+        """
+        # GIVEN: A mocked out SongImport class, and a mocked out "manager"
+        with patch(u'openlp.plugins.songs.lib.songshowplusimport.SongImport'):
+            mocked_manager = MagicMock()
+            mocked_import_wizard = MagicMock()
+            importer = SongShowPlusImport(mocked_manager)
+            importer.import_wizard = mocked_import_wizard
+            importer.stop_import_flag = True
+
+            # WHEN: Import source is a list
+            importer.import_source = [u'List', u'of', u'files']
+
+            # THEN: doImport should return none and the progress bar setMaximum should be called with the length of
+            #       import_source.
+            self.assertIsNone(importer.doImport(),
+                u'doImport should return None when import_source is a list and stop_import_flag is True')
+            mocked_import_wizard.progress_bar.setMaximum.assert_called_with(len(importer.import_source))
+
+    def to_openlp_verse_tag_test(self):
+        """
+        Test to_openlp_verse_tag method by simulating adding a verse
+        """
+        # GIVEN: A mocked out SongImport class, and a mocked out "manager"
+        with patch(u'openlp.plugins.songs.lib.songshowplusimport.SongImport'):
+            mocked_manager = MagicMock()
+            importer = SongShowPlusImport(mocked_manager)
+
+            # WHEN: Supplied with the following arguments replicating verses being added
+            test_values = [(u'Verse 1', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'Verse 2', VerseType.tags[VerseType.Verse] + u'2'),
+                (u'verse1', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'Verse', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'Verse1', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'chorus 1', VerseType.tags[VerseType.Chorus] + u'1'),
+                (u'bridge 1', VerseType.tags[VerseType.Bridge] + u'1'),
+                (u'pre-chorus 1', VerseType.tags[VerseType.PreChorus] + u'1'),
+                (u'different 1', VerseType.tags[VerseType.Other] + u'1'),
+                (u'random 1', VerseType.tags[VerseType.Other] + u'2')]
+
+            # THEN: The returned value should should correlate with the input arguments
+            for original_tag, openlp_tag in test_values:
+                self.assertEquals(importer.to_openlp_verse_tag(original_tag), openlp_tag,
+                    u'SongShowPlusImport.to_openlp_verse_tag should return "%s" when called with "%s"'
+                    % (openlp_tag, original_tag))
+
+    def to_openlp_verse_tag_verse_order_test(self):
+        """
+        Test to_openlp_verse_tag method by simulating adding a verse to the verse order
+        """
+        # GIVEN: A mocked out SongImport class, and a mocked out "manager"
+        with patch(u'openlp.plugins.songs.lib.songshowplusimport.SongImport'):
+            mocked_manager = MagicMock()
+            importer = SongShowPlusImport(mocked_manager)
+
+            # WHEN: Supplied with the following arguments replicating a verse order being added
+            test_values = [(u'Verse 1', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'Verse 2', VerseType.tags[VerseType.Verse] + u'2'),
+                (u'verse1', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'Verse', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'Verse1', VerseType.tags[VerseType.Verse] + u'1'),
+                (u'chorus 1', VerseType.tags[VerseType.Chorus] + u'1'),
+                (u'bridge 1', VerseType.tags[VerseType.Bridge] + u'1'),
+                (u'pre-chorus 1', VerseType.tags[VerseType.PreChorus] + u'1'),
+                (u'different 1', VerseType.tags[VerseType.Other] + u'1'),
+                (u'random 1', VerseType.tags[VerseType.Other] + u'2'),
+                (u'unused 2', None)]
+
+            # THEN: The returned value should should correlate with the input arguments
+            for original_tag, openlp_tag in test_values:
+                self.assertEquals(importer.to_openlp_verse_tag(original_tag, ignore_unique=True), openlp_tag,
+                    u'SongShowPlusImport.to_openlp_verse_tag should return "%s" when called with "%s"'
+                    % (openlp_tag, original_tag))
+
+    def file_import_test(self):
+        """
+        Test the actual import of real song files and check that the imported data is correct.
+        """
+
+        # GIVEN: Test files with a mocked out SongImport class, a mocked out "manager", a mocked out "import_wizard",
+        #       and mocked out "author", "add_copyright", "add_verse", "finish" methods.
+        with patch(u'openlp.plugins.songs.lib.songshowplusimport.SongImport'):
+            for song_file in SONG_TEST_DATA:
+                mocked_manager = MagicMock()
+                mocked_import_wizard = MagicMock()
+                mocked_parse_author = MagicMock()
+                mocked_add_copyright = MagicMock()
+                mocked_add_verse = MagicMock()
+                mocked_finish = MagicMock()
+                mocked_finish.return_value = True
+                importer = SongShowPlusImport(mocked_manager)
+                importer.import_wizard = mocked_import_wizard
+                importer.stop_import_flag = False
+                importer.parse_author = mocked_parse_author
+                importer.addCopyright = mocked_add_copyright
+                importer.addVerse = mocked_add_verse
+                importer.finish = mocked_finish
+                importer.topics = []
+
+                # WHEN: Importing each file
+                importer.import_source = [os.path.join(TEST_PATH, song_file)]
+                title = SONG_TEST_DATA[song_file][u'title']
+                author_calls = SONG_TEST_DATA[song_file][u'authors']
+                song_copyright = SONG_TEST_DATA[song_file][u'copyright']
+                ccli_number = SONG_TEST_DATA[song_file][u'ccli_number']
+                add_verse_calls = SONG_TEST_DATA[song_file][u'verses']
+                topics = SONG_TEST_DATA[song_file][u'topics']
+                comments = SONG_TEST_DATA[song_file][u'comments']
+                song_book_name = SONG_TEST_DATA[song_file][u'song_book_name']
+                song_number = SONG_TEST_DATA[song_file][u'song_number']
+                verse_order_list = SONG_TEST_DATA[song_file][u'verse_order_list']
+
+                # THEN: doImport should return none, the song data should be as expected, and finish should have been
+                #       called.
+                self.assertIsNone(importer.doImport(), u'doImport should return None when it has completed')
+                self.assertEquals(importer.title, title, u'title for %s should be "%s"' % (song_file, title))
+                for author in author_calls:
+                    mocked_parse_author.assert_any_call(author)
+                if song_copyright:
+                    mocked_add_copyright.assert_called_with(song_copyright)
+                if ccli_number:
+                    self.assertEquals(importer.ccliNumber, ccli_number, u'ccliNumber for %s should be %s'
+                        % (song_file, ccli_number))
+                for verse_text, verse_tag in add_verse_calls:
+                    mocked_add_verse.assert_any_call(verse_text, verse_tag)
+                if topics:
+                    self.assertEquals(importer.topics, topics, u'topics for %s should be %s' % (song_file, topics))
+                if comments:
+                    self.assertEquals(importer.comments, comments, u'comments for %s should be "%s"'
+                        % (song_file, comments))
+                if song_book_name:
+                    self.assertEquals(importer.songBookName, song_book_name, u'songBookName for %s should be "%s"'
+                        % (song_file, song_book_name))
+                if song_number:
+                    self.assertEquals(importer.songNumber, song_number, u'songNumber for %s should be %s'
+                        % (song_file, song_number))
+                if verse_order_list:
+                    self.assertEquals(importer.verseOrderList, [], u'verseOrderList for %s should be %s'
+                        % (song_file, verse_order_list))
+                mocked_finish.assert_called_with()