Replace PyICU with PyQt's QCollator

bzr-revno: 2839

openlp/core/common/i18n.py

@@ -23,7 +23,6 @@
 The :mod:`languages` module provides a list of language names with utility functions.
 """
 import itertools
-import locale
 import logging
 import re
 from collections import namedtuple
@@ -52,8 +51,7 @@ def translate(context, text, comment=None, qt_translate=QtCore.QCoreApplication.
 
 
 Language = namedtuple('Language', ['id', 'name', 'code'])
-ICU_COLLATOR = None
+COLLATOR = None
-DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+')
 LANGUAGES = sorted([
     Language(1, translate('common.languages', '(Afan) Oromo', 'Language code: om'), 'om'),
     Language(2, translate('common.languages', 'Abkhazian', 'Language code: ab'), 'ab'),
@@ -506,24 +504,19 @@ def format_time(text, local_time):
     return re.sub(r'\%[a-zA-Z]', match_formatting, text)
 
 
-def get_locale_key(string):
+def get_locale_key(string, numeric=False):
     """
     Creates a key for case insensitive, locale aware string sorting.
 
     :param string: The corresponding string.
     """
     string = string.lower()
-    # ICU is the prefered way to handle locale sort key, we fallback to locale.strxfrm which will work in most cases.
+    global COLLATOR
-    global ICU_COLLATOR
+    if COLLATOR is None:
-    try:
-        if ICU_COLLATOR is None:
-            import icu
         language = LanguageManager.get_language()
-            icu_locale = icu.Locale(language)
+        COLLATOR = QtCore.QCollator(QtCore.QLocale(language))
-            ICU_COLLATOR = icu.Collator.createInstance(icu_locale)
+    COLLATOR.setNumericMode(numeric)
-        return ICU_COLLATOR.getSortKey(string)
+    return COLLATOR.sortKey(string)
-    except Exception:
-        return locale.strxfrm(string).encode()
 
 
 def get_natural_key(string):
@@ -533,13 +526,7 @@ def get_natural_key(string):
     :param string: string to be sorted by
     Returns a list of string compare keys and integers.
     """
-    key = DIGITS_OR_NONDIGITS.findall(string)
+    return get_locale_key(string, True)
-    key = [int(part) if part.isdigit() else get_locale_key(part) for part in key]
-    # Python 3 does not support comparison of different types anymore. So make sure, that we do not compare str
-    # and int.
-    if string and string[0].isdigit():
-        return [b''] + key
-    return key
 
 
 def get_language(name):

openlp/core/ui/exceptionform.py

@@ -52,14 +52,6 @@ try:
     MAKO_VERSION = mako.__version__
 except ImportError:
     MAKO_VERSION = '-'
-try:
-    import icu
-    try:
-        ICU_VERSION = icu.VERSION
-    except AttributeError:
-        ICU_VERSION = 'OK'
-except ImportError:
-    ICU_VERSION = '-'
 try:
     WEBKIT_VERSION = QtWebKit.qWebKitVersion()
 except AttributeError:
@@ -119,12 +111,12 @@ class ExceptionForm(QtWidgets.QDialog, Ui_ExceptionDialog, RegistryProperties):
         system = translate('OpenLP.ExceptionForm', 'Platform: {platform}\n').format(platform=platform.platform())
         libraries = ('Python: {python}\nQt5: {qt5}\nPyQt5: {pyqt5}\nQtWebkit: {qtwebkit}\nSQLAlchemy: {sqalchemy}\n'
                      'SQLAlchemy Migrate: {migrate}\nBeautifulSoup: {soup}\nlxml: {etree}\nChardet: {chardet}\n'
-                     'PyEnchant: {enchant}\nMako: {mako}\npyICU: {icu}\npyUNO bridge: {uno}\n'
+                     'PyEnchant: {enchant}\nMako: {mako}\npyUNO bridge: {uno}\n'
                      'VLC: {vlc}\n').format(python=platform.python_version(), qt5=Qt.qVersion(),
                                             pyqt5=Qt.PYQT_VERSION_STR, qtwebkit=WEBKIT_VERSION,
                                             sqalchemy=sqlalchemy.__version__, migrate=MIGRATE_VERSION,
                                             soup=bs4.__version__, etree=etree.__version__, chardet=CHARDET_VERSION,
-                                            enchant=ENCHANT_VERSION, mako=MAKO_VERSION, icu=ICU_VERSION,
+                                            enchant=ENCHANT_VERSION, mako=MAKO_VERSION,
                                             uno=self._pyuno_import(), vlc=VLC_VERSION)
 
         if is_linux():

openlp/plugins/songs/lib/mediaitem.py

@@ -324,12 +324,12 @@ class SongMediaItem(MediaManagerItem):
         :param search_results: A tuple containing (songbook entry, book name, song title, song id)
         :return: None
         """
-        def get_songbook_key(text_array):
+        def get_songbook_key(text):
             """
             Get the key to sort by
-            :param text_array: the result text to be processed.
+            :param text: the text tuple to be processed.
             """
-            return get_natural_key(text_array[1]), get_natural_key(text_array[0]), get_natural_key(text_array[2])
+            return get_natural_key('{0} {1} {2}'.format(text[1], text[0], text[2]))
 
         log.debug('display results Book')
         self.list_view.clear()

scripts/appveyor.yml

@@ -12,19 +12,17 @@ environment:
 install:
   # Install dependencies from pypi
   - "%PYTHON%\\python.exe -m pip install sqlalchemy alembic appdirs chardet beautifulsoup4 lxml Mako mysql-connector-python nose mock pyodbc==4.0.8 psycopg2 pypiwin32==219 pyenchant pymediainfo websockets asyncio waitress six webob requests QtAwesome"
-  # Download and install pyicu (originally from http://www.lfd.uci.edu/~gohlke/pythonlibs/)
-  - "%PYTHON%\\python.exe -m pip install https://get.openlp.org/win-sdk/PyICU-1.9.5-cp34-cp34m-win32.whl"
   # Download and install PyQt5
-  - appveyor DownloadFile http://downloads.sourceforge.net/project/pyqt/PyQt5/PyQt-5.5.1/PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe
+  - appveyor DownloadFile https://downloads.sourceforge.net/project/pyqt/PyQt5/PyQt-5.5.1/PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe
   - PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe /S
   # Download and unpack mupdf
-  - appveyor DownloadFile http://mupdf.com/downloads/archive/mupdf-1.9a-windows.zip
+  - appveyor DownloadFile https://mupdf.com/downloads/archive/mupdf-1.14.0-windows.zip
-  - 7z x mupdf-1.9a-windows.zip
+  - 7z x mupdf-1.14.0-windows.zip
-  - cp mupdf-1.9a-windows/mupdf.exe openlp-branch/mupdf.exe
+  - cp mupdf-1.14.0-windows/mupdf.exe openlp-branch/mupdf.exe
   # Download and unpack mediainfo
-  - appveyor DownloadFile https://mediaarea.net/download/binary/mediainfo/0.7.90/MediaInfo_CLI_0.7.90_Windows_i386.zip
+  - appveyor DownloadFile https://mediaarea.net/download/binary/mediainfo/18.08.1/MediaInfo_CLI_18.08.1_Windows_i386.zip
   - mkdir MediaInfo
-  - 7z x -oMediaInfo MediaInfo_CLI_0.7.90_Windows_i386.zip
+  - 7z x -oMediaInfo MediaInfo_CLI_18.08.1_Windows_i386.zip
   - cp MediaInfo\\MediaInfo.exe openlp-branch\\MediaInfo.exe
 
 build: off

scripts/check_dependencies.py

@@ -40,8 +40,8 @@ IS_MAC = sys.platform.startswith('dar')
 
 VERS = {
     'Python': '3.6',
-    'PyQt5': '5.0',
+    'PyQt5': '5.5',
-    'Qt5': '5.0',
+    'Qt5': '5.5',
     'pymediainfo': '2.2',
     'sqlalchemy': '0.5',
     'enchant': '1.6'
@@ -52,7 +52,6 @@ WIN32_MODULES = [
     'win32com',
     'win32ui',
     'pywintypes',
-    'icu',
 ]
 
 LINUX_MODULES = [

setup.py

@@ -119,7 +119,7 @@ requires = [
     'lxml',
     'Mako',
     'pymediainfo >= 2.2',
-    'PyQt5',
+    'PyQt5 >= 5.5',
     'QtAwesome',
     'requests',
     'SQLAlchemy >= 0.5',
@@ -128,10 +128,7 @@ requires = [
     'websockets'
 ]
 if sys.platform.startswith('win'):
-    requires.extend([
+    requires.append('pywin32')
-        'PyICU',
-        'pywin32'
-    ])
 elif sys.platform.startswith('darwin'):
     requires.extend([
         'pyobjc',
@@ -204,7 +201,7 @@ using a computer and a data projector.""",
         'jenkins': ['python-jenkins'],
         'launchpad': ['launchpadlib']
     },
-    tests_require=['nose2', 'PyICU', 'pylint', 'pyodbc', 'pysword'],
+    tests_require=['nose2', 'pylint', 'pyodbc', 'pysword'],
     test_suite='nose2.collector.collector',
     entry_points={'gui_scripts': ['openlp = run_openlp:start']}
 )

tests/functional/openlp_core/common/test_i18n.py

@@ -22,10 +22,8 @@
 """
 Package to test the openlp.core.lib.languages package.
 """
-from unittest import skipIf
 from unittest.mock import MagicMock, patch
 
-from openlp.core.common import is_macosx
 from openlp.core.common.i18n import LANGUAGES, Language, UiStrings, get_language, get_locale_key, get_natural_key, \
     translate, LanguageManager
 from openlp.core.common.settings import Settings
@@ -113,7 +111,6 @@ def test_get_language_invalid_with_none():
     assert language is None
 
 
-@skipIf(is_macosx(), 'This test doesn\'t work on macOS currently')
 def test_get_locale_key():
     """
     Test the get_locale_key(string) function

tests/functional/openlp_core/ui/test_exceptionform.py

@@ -37,7 +37,6 @@ exceptionform.MIGRATE_VERSION = 'Migrate Test'
 exceptionform.CHARDET_VERSION = 'CHARDET Test'
 exceptionform.ENCHANT_VERSION = 'Enchant Test'
 exceptionform.MAKO_VERSION = 'Mako Test'
-exceptionform.ICU_VERSION = 'ICU Test'
 exceptionform.VLC_VERSION = 'VLC Test'
 
 MAIL_ITEM_TEXT = ('**OpenLP Bug Report**\nVersion: Trunk Test\n\n--- Details of the Exception. ---\n\n'
@@ -46,7 +45,7 @@ MAIL_ITEM_TEXT = ('**OpenLP Bug Report**\nVersion: Trunk Test\n\n--- Details of
                   'Python: Python Test\nQt5: Qt5 test\nPyQt5: PyQt5 Test\nQtWebkit: Webkit Test\n'
                   'SQLAlchemy: SqlAlchemy Test\nSQLAlchemy Migrate: Migrate Test\nBeautifulSoup: BeautifulSoup Test\n'
                   'lxml: ETree Test\nChardet: CHARDET Test\nPyEnchant: Enchant Test\nMako: Mako Test\n'
-                  'pyICU: ICU Test\npyUNO bridge: UNO Bridge Test\nVLC: VLC Test\n\n')
+                  'pyUNO bridge: UNO Bridge Test\nVLC: VLC Test\n\n')
 
 
 @patch("openlp.core.ui.exceptionform.Qt.qVersion")

tests/functional/openlp_plugins/songs/test_mediaitem.py

@@ -23,7 +23,7 @@
 This module contains tests for the lib submodule of the Songs plugin.
 """
 from unittest import TestCase
-from unittest.mock import MagicMock, patch, call
+from unittest.mock import MagicMock, patch
 
 from PyQt5 import QtCore
 
@@ -170,27 +170,23 @@ class TestMediaItem(TestCase, TestMixin):
         """
         Test that songbooks are sorted naturally
         """
-        # GIVEN: Search results grouped by book and entry, plus a mocked QtListWidgetItem
+        # GIVEN: Search results grouped by book and entry
-        with patch('openlp.core.lib.QtWidgets.QListWidgetItem') as MockedQListWidgetItem:
+        search_results = [('2', 'Thy Book', 'Thy Song', 50),
-            mock_search_results = [('2', 'Thy Book', 'Thy Song', 50),
                           ('2', 'My Book', 'Your Song', 7),
                           ('10', 'My Book', 'Our Song', 12),
                           ('1', 'My Book', 'My Song', 1),
                           ('2', 'Thy Book', 'A Song', 8)]
-            mock_qlist_widget = MagicMock()
-            MockedQListWidgetItem.return_value = mock_qlist_widget
 
         # WHEN: I display song search results grouped by book
-            self.media_item.display_results_book(mock_search_results)
+        self.media_item.display_results_book(search_results)
 
-            # THEN: The songbooks are inserted in the right (natural) order,
+        # THEN: The songbooks are sorted inplace in the right (natural) order,
         #       grouped first by book, then by number, then by song title
-            calls = [call('My Book #1: My Song'), call().setData(QtCore.Qt.UserRole, 1),
+        assert search_results == [('1', 'My Book', 'My Song', 1),
-                     call('My Book #2: Your Song'), call().setData(QtCore.Qt.UserRole, 7),
+                                  ('2', 'My Book', 'Your Song', 7),
-                     call('My Book #10: Our Song'), call().setData(QtCore.Qt.UserRole, 12),
+                                  ('10', 'My Book', 'Our Song', 12),
-                     call('Thy Book #2: A Song'), call().setData(QtCore.Qt.UserRole, 8),
+                                  ('2', 'Thy Book', 'A Song', 8),
-                     call('Thy Book #2: Thy Song'), call().setData(QtCore.Qt.UserRole, 50)]
+                                  ('2', 'Thy Book', 'Thy Song', 50)]
-            MockedQListWidgetItem.assert_has_calls(calls)
 
     def test_display_results_topic(self):
         """