From fbb6ecd2df629de4a46977118cdcf4687c2063e5 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 10 Jan 2023 08:16:02 +0000 Subject: [PATCH] First attempt at re-introducing the use of ICU, based on simply reverting the original patch --- appveyor.yml | 8 +++++- openlp/core/common/i18n.py | 32 ++++++++++++++++------ scripts/check_dependencies.py | 1 + setup.py | 1 + tests/openlp_core/ui/test_exceptionform.py | 1 + 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 84d21d041..5ab2be0ad 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,11 +13,13 @@ environment: CHOCO_VLC_ARG: FORCE_PACKAGING: 0 FORCE_PACKAGING_MANUAL: 0 + PYICU_PACK: PyICU-2.9-cp38-cp38-win_amd64.whl - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 PY_DIR: C:\\Python38 CHOCO_VLC_ARG: --forcex86 FORCE_PACKAGING: 0 FORCE_PACKAGING_MANUAL: 0 + PYICU_PACK: PyICU-2.9-cp38-cp38-win32.whl - APPVEYOR_BUILD_WORKER_IMAGE: macos-catalina QT_QPA_PLATFORM: offscreen FORCE_PACKAGING: 0 @@ -36,9 +38,13 @@ install: # Install Windows only dependencies - cmd: python -m pip install pyodbc pypiwin32 - cmd: choco install vlc %CHOCO_VLC_ARG% --no-progress --limit-output + # Download and install pyicu for windows (originally from http://www.lfd.uci.edu/~gohlke/pythonlibs/) + - cmd: python -m pip install https://get.openlp.org/win-sdk/%PYICU_PACK% # Mac only dependencies - - sh: python -m pip install Pyro4 'pyobjc-core<8.2' 'pyobjc-framework-Cocoa<8.2' py-applescript - sh: brew install --cask vlc + - sh: brew install pkg-config icu4c + - sh: PATH="/usr/local/opt/icu4c/bin:/usr/local/opt/icu4c/sbin:$PATH" PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig" python -m pip install pyicu + - sh: python -m pip install Pyro4 'pyobjc-core<8.2' 'pyobjc-framework-Cocoa<8.2' py-applescript build: off diff --git a/openlp/core/common/i18n.py b/openlp/core/common/i18n.py index 43b37170e..733450d8c 100644 --- a/openlp/core/common/i18n.py +++ b/openlp/core/common/i18n.py @@ -22,6 +22,7 @@ The :mod:`languages` module provides a list of language names with utility functions. """ import itertools +import locale import logging import re from collections import namedtuple @@ -51,7 +52,8 @@ def translate(context, text, comment=None, qt_translate=QtCore.QCoreApplication. Language = namedtuple('Language', ['id', 'name', 'code']) -COLLATOR = None +ICU_COLLATOR = None +DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+') LANGUAGES = sorted([ Language(1, translate('common.languages', '(Afan) Oromo', 'Language code: om'), 'om'), Language(2, translate('common.languages', 'Abkhazian', 'Language code: ab'), 'ab'), @@ -503,19 +505,25 @@ def format_time(text, local_time): return re.sub(r'%[a-zA-Z]', match_formatting, text) -def get_locale_key(string, numeric=False): +def get_locale_key(string): """ Creates a key for case insensitive, locale aware string sorting. :param string: The corresponding string. """ string = string.lower() - global COLLATOR - if COLLATOR is None: - language = LanguageManager.get_language() - COLLATOR = QtCore.QCollator(QtCore.QLocale(language)) - COLLATOR.setNumericMode(numeric) - return COLLATOR.sortKey(string) + # ICU is the prefered way to handle locale sort key, we fallback to locale.strxfrm which will work in most cases. + global ICU_COLLATOR + try: + if ICU_COLLATOR is None: + import icu + language = LanguageManager.get_language() + icu_locale = icu.Locale(language) + ICU_COLLATOR = icu.Collator.createInstance(icu_locale) + return ICU_COLLATOR.getSortKey(string) + except Exception: + log.warning('ICU not found! Fallback to strxfrm') + return locale.strxfrm(string).encode() def get_natural_key(string): @@ -525,7 +533,13 @@ def get_natural_key(string): :param string: string to be sorted by Returns a list of string compare keys and integers. """ - return get_locale_key(string, True) + key = DIGITS_OR_NONDIGITS.findall(string) + key = [int(part) if part.isdigit() else get_locale_key(part) for part in key] + # Python 3 does not support comparison of different types anymore. So make sure, that we do not compare str + # and int. + if string and string[0].isdigit(): + return [b''] + key + return key def get_language(name): diff --git a/scripts/check_dependencies.py b/scripts/check_dependencies.py index c335d3d05..2c55662f4 100755 --- a/scripts/check_dependencies.py +++ b/scripts/check_dependencies.py @@ -52,6 +52,7 @@ WIN32_MODULES = [ 'win32com', 'win32ui', 'pywintypes', + 'icu', ] LINUX_MODULES = [ diff --git a/setup.py b/setup.py index b93741af1..098d93466 100644 --- a/setup.py +++ b/setup.py @@ -107,6 +107,7 @@ using a computer and a display/projector.""", 'lxml', 'Mako', "pillow", + 'PyICU', 'pymediainfo >= 2.2', 'pyobjc; platform_system=="Darwin"', 'pyobjc-framework-Cocoa; platform_system=="Darwin"', diff --git a/tests/openlp_core/ui/test_exceptionform.py b/tests/openlp_core/ui/test_exceptionform.py index fbdb7a2d5..689543be6 100644 --- a/tests/openlp_core/ui/test_exceptionform.py +++ b/tests/openlp_core/ui/test_exceptionform.py @@ -38,6 +38,7 @@ exceptionform.MIGRATE_VERSION = 'Migrate Test' exceptionform.CHARDET_VERSION = 'CHARDET Test' exceptionform.ENCHANT_VERSION = 'Enchant Test' exceptionform.MAKO_VERSION = 'Mako Test' +exceptionform.ICU_VERSION = 'ICU Test' exceptionform.VLC_VERSION = 'VLC Test' MAIL_ITEM_TEXT = ('**OpenLP Bug Report**\nVersion: Trunk Test\n\n--- Details of the Exception. ---\n\n'