First attempt at re-introducing the use of ICU, based on simply reverting the original patch

appveyor.yml

@@ -13,11 +13,13 @@ environment:
       CHOCO_VLC_ARG:
       FORCE_PACKAGING: 0
       FORCE_PACKAGING_MANUAL: 0
+      PYICU_PACK: PyICU-2.9-cp38-cp38-win_amd64.whl
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
       PY_DIR: C:\\Python38
       CHOCO_VLC_ARG: --forcex86
       FORCE_PACKAGING: 0
       FORCE_PACKAGING_MANUAL: 0
+      PYICU_PACK: PyICU-2.9-cp38-cp38-win32.whl
     - APPVEYOR_BUILD_WORKER_IMAGE: macos-catalina
       QT_QPA_PLATFORM: offscreen
       FORCE_PACKAGING: 0
@@ -36,9 +38,13 @@ install:
   # Install Windows only dependencies
   - cmd: python -m pip install pyodbc pypiwin32
   - cmd: choco install vlc %CHOCO_VLC_ARG% --no-progress --limit-output
+  # Download and install pyicu for windows (originally from http://www.lfd.uci.edu/~gohlke/pythonlibs/)
+  - cmd: python -m pip install https://get.openlp.org/win-sdk/%PYICU_PACK%
   # Mac only dependencies
-  - sh: python -m pip install Pyro4 'pyobjc-core<8.2' 'pyobjc-framework-Cocoa<8.2' py-applescript
   - sh: brew install --cask vlc
+  - sh: brew install pkg-config icu4c
+  - sh: PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH" PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig" python -m pip install pyicu
+  - sh: python -m pip install Pyro4 'pyobjc-core<8.2' 'pyobjc-framework-Cocoa<8.2' py-applescript
 
 build: off
 

openlp/core/common/i18n.py

@@ -22,6 +22,7 @@
 The :mod:`languages` module provides a list of language names with utility functions.
 """
 import itertools
+import locale
 import logging
 import re
 from collections import namedtuple
@@ -51,7 +52,8 @@ def translate(context, text, comment=None, qt_translate=QtCore.QCoreApplication.
 
 
 Language = namedtuple('Language', ['id', 'name', 'code'])
-COLLATOR = None
+ICU_COLLATOR = None
+DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+')
 LANGUAGES = sorted([
     Language(1, translate('common.languages', '(Afan) Oromo', 'Language code: om'), 'om'),
     Language(2, translate('common.languages', 'Abkhazian', 'Language code: ab'), 'ab'),
@@ -503,19 +505,25 @@ def format_time(text, local_time):
     return re.sub(r'%[a-zA-Z]', match_formatting, text)
 
 
-def get_locale_key(string, numeric=False):
+def get_locale_key(string):
     """
     Creates a key for case insensitive, locale aware string sorting.
 
     :param string: The corresponding string.
     """
     string = string.lower()
-    global COLLATOR
-    if COLLATOR is None:
-        language = LanguageManager.get_language()
-        COLLATOR = QtCore.QCollator(QtCore.QLocale(language))
-    COLLATOR.setNumericMode(numeric)
-    return COLLATOR.sortKey(string)
+    # ICU is the prefered way to handle locale sort key, we fallback to locale.strxfrm which will work in most cases.
+    global ICU_COLLATOR
+    try:
+        if ICU_COLLATOR is None:
+            import icu
+            language = LanguageManager.get_language()
+            icu_locale = icu.Locale(language)
+            ICU_COLLATOR = icu.Collator.createInstance(icu_locale)
+        return ICU_COLLATOR.getSortKey(string)
+    except Exception:
+        log.warning('ICU not found! Fallback to strxfrm')
+        return locale.strxfrm(string).encode()
 
 
 def get_natural_key(string):
@@ -525,7 +533,13 @@ def get_natural_key(string):
     :param string: string to be sorted by
     Returns a list of string compare keys and integers.
     """
-    return get_locale_key(string, True)
+    key = DIGITS_OR_NONDIGITS.findall(string)
+    key = [int(part) if part.isdigit() else get_locale_key(part) for part in key]
+    # Python 3 does not support comparison of different types anymore. So make sure, that we do not compare str
+    # and int.
+    if string and string[0].isdigit():
+        return [b''] + key
+    return key
 
 
 def get_language(name):

scripts/check_dependencies.py

@@ -52,6 +52,7 @@ WIN32_MODULES = [
     'win32com',
     'win32ui',
     'pywintypes',
+    'icu',
 ]
 
 LINUX_MODULES = [

setup.py

@@ -107,6 +107,7 @@ using a computer and a display/projector.""",
         'lxml',
         'Mako',
         "pillow",
+        'PyICU',
         'pymediainfo >= 2.2',
         'pyobjc; platform_system=="Darwin"',
         'pyobjc-framework-Cocoa; platform_system=="Darwin"',

tests/openlp_core/ui/test_exceptionform.py

@@ -38,6 +38,7 @@ exceptionform.MIGRATE_VERSION = 'Migrate Test'
 exceptionform.CHARDET_VERSION = 'CHARDET Test'
 exceptionform.ENCHANT_VERSION = 'Enchant Test'
 exceptionform.MAKO_VERSION = 'Mako Test'
+exceptionform.ICU_VERSION = 'ICU Test'
 exceptionform.VLC_VERSION = 'VLC Test'
 
 MAIL_ITEM_TEXT = ('**OpenLP Bug Report**\nVersion: Trunk Test\n\n--- Details of the Exception. ---\n\n'