Replace PyICU with PyQt's QCollator

Use QCollator as new collator to get rid of the PyICU dependency.
Simplify the natural sorting with its numeric mode.
Simplify one test that is heavily dependent on implementation.
Run one sorting test on macOS which was disabled.

openlp/core/common/i18n.py

@@ -52,8 +52,7 @@ def translate(context, text, comment=None, qt_translate=QtCore.QCoreApplication.
 
 
 Language = namedtuple('Language', ['id', 'name', 'code'])
-ICU_COLLATOR = None
+COLLATOR = None
-DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+')
 LANGUAGES = sorted([
     Language(1, translate('common.languages', '(Afan) Oromo', 'Language code: om'), 'om'),
     Language(2, translate('common.languages', 'Abkhazian', 'Language code: ab'), 'ab'),
@@ -506,24 +505,19 @@ def format_time(text, local_time):
     return re.sub(r'\%[a-zA-Z]', match_formatting, text)
 
 
-def get_locale_key(string):
+def get_locale_key(string, numeric=False):
     """
     Creates a key for case insensitive, locale aware string sorting.
 
     :param string: The corresponding string.
     """
     string = string.lower()
-    # ICU is the prefered way to handle locale sort key, we fallback to locale.strxfrm which will work in most cases.
+    global COLLATOR
-    global ICU_COLLATOR
+    if COLLATOR is None:
-    try:
+        language = LanguageManager.get_language()
-        if ICU_COLLATOR is None:
+        COLLATOR = QtCore.QCollator(QtCore.QLocale(language))
-            import icu
+    COLLATOR.setNumericMode(numeric)
-            language = LanguageManager.get_language()
+    return COLLATOR.sortKey(string)
-            icu_locale = icu.Locale(language)
-            ICU_COLLATOR = icu.Collator.createInstance(icu_locale)
-        return ICU_COLLATOR.getSortKey(string)
-    except Exception:
-        return locale.strxfrm(string).encode()
 
 
 def get_natural_key(string):
@@ -533,13 +527,7 @@ def get_natural_key(string):
     :param string: string to be sorted by
     Returns a list of string compare keys and integers.
     """
-    key = DIGITS_OR_NONDIGITS.findall(string)
+    return get_locale_key(string, True)
-    key = [int(part) if part.isdigit() else get_locale_key(part) for part in key]
-    # Python 3 does not support comparison of different types anymore. So make sure, that we do not compare str
-    # and int.
-    if string and string[0].isdigit():
-        return [b''] + key
-    return key
 
 
 def get_language(name):

openlp/core/ui/exceptionform.py

@@ -52,14 +52,6 @@ try:
     MAKO_VERSION = mako.__version__
 except ImportError:
     MAKO_VERSION = '-'
-try:
-    import icu
-    try:
-        ICU_VERSION = icu.VERSION
-    except AttributeError:
-        ICU_VERSION = 'OK'
-except ImportError:
-    ICU_VERSION = '-'
 try:
     WEBKIT_VERSION = QtWebKit.qWebKitVersion()
 except AttributeError:
@@ -119,12 +111,12 @@ class ExceptionForm(QtWidgets.QDialog, Ui_ExceptionDialog, RegistryProperties):
         system = translate('OpenLP.ExceptionForm', 'Platform: {platform}\n').format(platform=platform.platform())
         libraries = ('Python: {python}\nQt5: {qt5}\nPyQt5: {pyqt5}\nQtWebkit: {qtwebkit}\nSQLAlchemy: {sqalchemy}\n'
                      'SQLAlchemy Migrate: {migrate}\nBeautifulSoup: {soup}\nlxml: {etree}\nChardet: {chardet}\n'
-                     'PyEnchant: {enchant}\nMako: {mako}\npyICU: {icu}\npyUNO bridge: {uno}\n'
+                     'PyEnchant: {enchant}\nMako: {mako}\npyUNO bridge: {uno}\n'
                      'VLC: {vlc}\n').format(python=platform.python_version(), qt5=Qt.qVersion(),
                                             pyqt5=Qt.PYQT_VERSION_STR, qtwebkit=WEBKIT_VERSION,
                                             sqalchemy=sqlalchemy.__version__, migrate=MIGRATE_VERSION,
                                             soup=bs4.__version__, etree=etree.__version__, chardet=CHARDET_VERSION,
-                                            enchant=ENCHANT_VERSION, mako=MAKO_VERSION, icu=ICU_VERSION,
+                                            enchant=ENCHANT_VERSION, mako=MAKO_VERSION,
                                             uno=self._pyuno_import(), vlc=VLC_VERSION)
 
         if is_linux():

openlp/plugins/songs/lib/mediaitem.py

@@ -324,12 +324,12 @@ class SongMediaItem(MediaManagerItem):
         :param search_results: A tuple containing (songbook entry, book name, song title, song id)
         :return: None
         """
-        def get_songbook_key(text_array):
+        def get_songbook_key(text):
             """
             Get the key to sort by
-            :param text_array: the result text to be processed.
+            :param text: the text tuple to be processed.
             """
-            return get_natural_key(text_array[1]), get_natural_key(text_array[0]), get_natural_key(text_array[2])
+            return get_natural_key('{0} {1} {2}'.format(text[1], text[0], text[2]))
 
         log.debug('display results Book')
         self.list_view.clear()

scripts/appveyor.yml

@@ -12,19 +12,17 @@ environment:
 install:
   # Install dependencies from pypi
   - "%PYTHON%\\python.exe -m pip install sqlalchemy alembic appdirs chardet beautifulsoup4 lxml Mako mysql-connector-python nose mock pyodbc==4.0.8 psycopg2 pypiwin32==219 pyenchant pymediainfo websockets asyncio waitress six webob requests QtAwesome"
-  # Download and install pyicu (originally from http://www.lfd.uci.edu/~gohlke/pythonlibs/)
-  - "%PYTHON%\\python.exe -m pip install https://get.openlp.org/win-sdk/PyICU-1.9.5-cp34-cp34m-win32.whl"
   # Download and install PyQt5
-  - appveyor DownloadFile http://downloads.sourceforge.net/project/pyqt/PyQt5/PyQt-5.5.1/PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe
+  - appveyor DownloadFile https://downloads.sourceforge.net/project/pyqt/PyQt5/PyQt-5.5.1/PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe
   - PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe /S
   # Download and unpack mupdf
-  - appveyor DownloadFile http://mupdf.com/downloads/archive/mupdf-1.9a-windows.zip
+  - appveyor DownloadFile https://mupdf.com/downloads/archive/mupdf-1.14.0-windows.zip
-  - 7z x mupdf-1.9a-windows.zip
+  - 7z x mupdf-1.14.0-windows.zip
-  - cp mupdf-1.9a-windows/mupdf.exe openlp-branch/mupdf.exe
+  - cp mupdf-1.14.0-windows/mupdf.exe openlp-branch/mupdf.exe
   # Download and unpack mediainfo
-  - appveyor DownloadFile https://mediaarea.net/download/binary/mediainfo/0.7.90/MediaInfo_CLI_0.7.90_Windows_i386.zip
+  - appveyor DownloadFile https://mediaarea.net/download/binary/mediainfo/18.08.1/MediaInfo_CLI_18.08.1_Windows_i386.zip
   - mkdir MediaInfo
-  - 7z x -oMediaInfo MediaInfo_CLI_0.7.90_Windows_i386.zip
+  - 7z x -oMediaInfo MediaInfo_CLI_18.08.1_Windows_i386.zip
   - cp MediaInfo\\MediaInfo.exe openlp-branch\\MediaInfo.exe
 
 build: off

scripts/check_dependencies.py

@@ -40,8 +40,8 @@ IS_MAC = sys.platform.startswith('dar')
 
 VERS = {
     'Python': '3.6',
-    'PyQt5': '5.0',
+    'PyQt5': '5.5',
-    'Qt5': '5.0',
+    'Qt5': '5.5',
     'pymediainfo': '2.2',
     'sqlalchemy': '0.5',
     'enchant': '1.6'
@@ -52,7 +52,6 @@ WIN32_MODULES = [
     'win32com',
     'win32ui',
     'pywintypes',
-    'icu',
 ]
 
 LINUX_MODULES = [

setup.py

@@ -119,7 +119,7 @@ requires = [
     'lxml',
     'Mako',
     'pymediainfo >= 2.2',
-    'PyQt5',
+    'PyQt5 >= 5.5',
     'QtAwesome',
     'requests',
     'SQLAlchemy >= 0.5',
@@ -128,10 +128,7 @@ requires = [
     'websockets'
 ]
 if sys.platform.startswith('win'):
-    requires.extend([
+    requires.append('pywin32')
-        'PyICU',
-        'pywin32'
-    ])
 elif sys.platform.startswith('darwin'):
     requires.extend([
         'pyobjc',
@@ -204,7 +201,7 @@ using a computer and a data projector.""",
         'jenkins': ['python-jenkins'],
         'launchpad': ['launchpadlib']
     },
-    tests_require=['nose2', 'PyICU', 'pylint', 'pyodbc', 'pysword'],
+    tests_require=['nose2', 'pylint', 'pyodbc', 'pysword'],
     test_suite='nose2.collector.collector',
     entry_points={'gui_scripts': ['openlp = run_openlp:start']}
 )

tests/functional/openlp_core/common/test_i18n.py

@@ -113,7 +113,6 @@ def test_get_language_invalid_with_none():
     assert language is None
 
 
-@skipIf(is_macosx(), 'This test doesn\'t work on macOS currently')
 def test_get_locale_key():
     """
     Test the get_locale_key(string) function

tests/functional/openlp_core/ui/test_exceptionform.py

@@ -37,7 +37,6 @@ exceptionform.MIGRATE_VERSION = 'Migrate Test'
 exceptionform.CHARDET_VERSION = 'CHARDET Test'
 exceptionform.ENCHANT_VERSION = 'Enchant Test'
 exceptionform.MAKO_VERSION = 'Mako Test'
-exceptionform.ICU_VERSION = 'ICU Test'
 exceptionform.VLC_VERSION = 'VLC Test'
 
 MAIL_ITEM_TEXT = ('**OpenLP Bug Report**\nVersion: Trunk Test\n\n--- Details of the Exception. ---\n\n'
@@ -46,7 +45,7 @@ MAIL_ITEM_TEXT = ('**OpenLP Bug Report**\nVersion: Trunk Test\n\n--- Details of
                   'Python: Python Test\nQt5: Qt5 test\nPyQt5: PyQt5 Test\nQtWebkit: Webkit Test\n'
                   'SQLAlchemy: SqlAlchemy Test\nSQLAlchemy Migrate: Migrate Test\nBeautifulSoup: BeautifulSoup Test\n'
                   'lxml: ETree Test\nChardet: CHARDET Test\nPyEnchant: Enchant Test\nMako: Mako Test\n'
-                  'pyICU: ICU Test\npyUNO bridge: UNO Bridge Test\nVLC: VLC Test\n\n')
+                  'pyUNO bridge: UNO Bridge Test\nVLC: VLC Test\n\n')
 
 
 @patch("openlp.core.ui.exceptionform.Qt.qVersion")

tests/functional/openlp_plugins/songs/test_mediaitem.py

@@ -170,27 +170,23 @@ class TestMediaItem(TestCase, TestMixin):
         """
         Test that songbooks are sorted naturally
         """
-        # GIVEN: Search results grouped by book and entry, plus a mocked QtListWidgetItem
+        # GIVEN: Search results grouped by book and entry
-        with patch('openlp.core.lib.QtWidgets.QListWidgetItem') as MockedQListWidgetItem:
+        search_results = [('2', 'Thy Book', 'Thy Song', 50),
-            mock_search_results = [('2', 'Thy Book', 'Thy Song', 50),
+                          ('2', 'My Book', 'Your Song', 7),
-                                   ('2', 'My Book', 'Your Song', 7),
+                          ('10', 'My Book', 'Our Song', 12),
-                                   ('10', 'My Book', 'Our Song', 12),
+                          ('1', 'My Book', 'My Song', 1),
-                                   ('1', 'My Book', 'My Song', 1),
+                          ('2', 'Thy Book', 'A Song', 8)]
-                                   ('2', 'Thy Book', 'A Song', 8)]
-            mock_qlist_widget = MagicMock()
-            MockedQListWidgetItem.return_value = mock_qlist_widget
 
-            # WHEN: I display song search results grouped by book
+        # WHEN: I display song search results grouped by book
-            self.media_item.display_results_book(mock_search_results)
+        self.media_item.display_results_book(search_results)
 
-            # THEN: The songbooks are inserted in the right (natural) order,
+        # THEN: The songbooks are sorted inplace in the right (natural) order,
-            #       grouped first by book, then by number, then by song title
+        #       grouped first by book, then by number, then by song title
-            calls = [call('My Book #1: My Song'), call().setData(QtCore.Qt.UserRole, 1),
+        assert search_results == [('1', 'My Book', 'My Song', 1),
-                     call('My Book #2: Your Song'), call().setData(QtCore.Qt.UserRole, 7),
+                                  ('2', 'My Book', 'Your Song', 7),
-                     call('My Book #10: Our Song'), call().setData(QtCore.Qt.UserRole, 12),
+                                  ('10', 'My Book', 'Our Song', 12),
-                     call('Thy Book #2: A Song'), call().setData(QtCore.Qt.UserRole, 8),
+                                  ('2', 'Thy Book', 'A Song', 8),
-                     call('Thy Book #2: Thy Song'), call().setData(QtCore.Qt.UserRole, 50)]
+                                  ('2', 'Thy Book', 'Thy Song', 50)]
-            MockedQListWidgetItem.assert_has_calls(calls)
 
     def test_display_results_topic(self):
         """