From 76ea8126294d088d5ae4e002cd860b3f1faf7764 Mon Sep 17 00:00:00 2001 From: M2j Date: Sat, 30 Mar 2013 22:54:42 +0100 Subject: [PATCH] - switch to Python3 style sorting - use ICU for string sorting --- openlp/core/ui/exceptionform.py | 6 +++ openlp/core/ui/thememanager.py | 4 +- openlp/core/utils/__init__.py | 47 ++++++++++++++----- .../plugins/bibles/forms/bibleimportform.py | 4 +- openlp/plugins/bibles/lib/mediaitem.py | 6 +-- openlp/plugins/custom/lib/db.py | 7 ++- openlp/plugins/images/lib/mediaitem.py | 12 ++--- openlp/plugins/media/lib/mediaitem.py | 6 +-- openlp/plugins/presentations/lib/mediaitem.py | 5 +- openlp/plugins/songs/forms/songexportform.py | 4 +- openlp/plugins/songs/lib/__init__.py | 36 +------------- openlp/plugins/songs/lib/mediaitem.py | 6 +-- scripts/check_dependencies.py | 1 + 13 files changed, 69 insertions(+), 75 deletions(-) diff --git a/openlp/core/ui/exceptionform.py b/openlp/core/ui/exceptionform.py index 25f8201b1..e4854d0c0 100644 --- a/openlp/core/ui/exceptionform.py +++ b/openlp/core/ui/exceptionform.py @@ -69,6 +69,11 @@ try: MAKO_VERSION = mako.__version__ except ImportError: MAKO_VERSION = u'-' +try: + import icu + ICU_VERSION = u'OK' +except ImportError: + ICU_VERSION = u'-' try: import uno arg = uno.createUnoStruct(u'com.sun.star.beans.PropertyValue') @@ -143,6 +148,7 @@ class ExceptionForm(QtGui.QDialog, Ui_ExceptionDialog): u'PyEnchant: %s\n' % ENCHANT_VERSION + \ u'PySQLite: %s\n' % SQLITE_VERSION + \ u'Mako: %s\n' % MAKO_VERSION + \ + u'pyICU: %s\n' % ICU_VERSION + \ u'pyUNO bridge: %s\n' % UNO_VERSION + \ u'VLC: %s\n' % VLC_VERSION if platform.system() == u'Linux': diff --git a/openlp/core/ui/thememanager.py b/openlp/core/ui/thememanager.py index 89bbe86f8..53d4a513b 100644 --- a/openlp/core/ui/thememanager.py +++ b/openlp/core/ui/thememanager.py @@ -44,7 +44,7 @@ from openlp.core.lib.theme import ThemeXML, BackgroundType, VerticalType, Backgr from openlp.core.lib.ui import critical_error_message_box, create_widget_action from openlp.core.theme import Theme from openlp.core.ui import FileRenameForm, ThemeForm -from openlp.core.utils import AppLocation, delete_file, locale_compare, get_filesystem_encoding +from openlp.core.utils import AppLocation, delete_file, get_local_key, get_filesystem_encoding log = logging.getLogger(__name__) @@ -418,7 +418,7 @@ class ThemeManager(QtGui.QWidget): self.theme_list_widget.clear() files = AppLocation.get_files(self.settings_section, u'.png') # Sort the themes by its name considering language specific - files.sort(key=lambda file_name: unicode(file_name), cmp=locale_compare) + files.sort(key=lambda file_name: get_local_key(unicode(file_name))) # now process the file list of png files for name in files: # check to see file is in theme root directory diff --git a/openlp/core/utils/__init__.py b/openlp/core/utils/__init__.py index d32729699..625bd25b5 100644 --- a/openlp/core/utils/__init__.py +++ b/openlp/core/utils/__init__.py @@ -38,6 +38,7 @@ import re from subprocess import Popen, PIPE import sys import urllib2 +import icu from PyQt4 import QtGui, QtCore @@ -56,6 +57,7 @@ from openlp.core.lib import translate log = logging.getLogger(__name__) APPLICATION_VERSION = {} IMAGES_FILTER = None +ICU_COLLATOR = None UNO_CONNECTION_TYPE = u'pipe' #UNO_CONNECTION_TYPE = u'socket' CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]', re.UNICODE) @@ -379,21 +381,42 @@ def format_time(text, local_time): return re.sub('\%[a-zA-Z]', match_formatting, text) -def locale_compare(string1, string2): +def get_local_key(string): """ - Compares two strings according to the current locale settings. - - As any other compare function, returns a negative, or a positive value, - or 0, depending on whether string1 collates before or after string2 or - is equal to it. Comparison is case insensitive. + Creates a key for case insensitive, locale aware string sorting. """ - # Function locale.strcoll() from standard Python library does not work properly on Windows. - return locale.strcoll(string1.lower(), string2.lower()) + string = string.lower() + # For Python 3 on platforms other than Windows ICU is not necessary. In those cases locale.strxfrm(str) can be used. + global ICU_COLLATOR + if ICU_COLLATOR is None: + from languagemanager import LanguageManager + locale = LanguageManager.get_language() + icu_locale = icu.Locale(locale) + ICU_COLLATOR = icu.Collator.createInstance(icu_locale) + return ICU_COLLATOR.getSortKey(string) -# For performance reasons provide direct reference to compare function without wrapping it in another function making -# the string lowercase. This is needed for sorting songs. -locale_direct_compare = locale.strcoll +def get_natural_key(string): + """ + Generate a key for locale aware natural string sorting. Returns a list of strings and integers. + + ``string`` + A string, list or tuple which represents the item string. + """ + if isinstance(string, basestring): + string = re.findall(r'(\d+|\D+)', string) + if len(string) == 1: + return list(get_local_key(string[0])) + elif isinstance(string, tuple): + string = list(string) + if isinstance(string, list): + for index, part in enumerate(string): + if isinstance(part, basestring): + if part.isdigit(): + string[index] = int(part) + else: + string[index] = get_local_key(part) + return string from applocation import AppLocation @@ -403,4 +426,4 @@ from actions import ActionList __all__ = [u'AppLocation', u'ActionList', u'LanguageManager', u'get_application_version', u'check_latest_version', u'add_actions', u'get_filesystem_encoding', u'get_web_page', u'get_uno_command', u'get_uno_instance', - u'delete_file', u'clean_filename', u'format_time', u'locale_compare', u'locale_direct_compare'] + u'delete_file', u'clean_filename', u'format_time', u'get_local_key', u'get_natural_key'] diff --git a/openlp/plugins/bibles/forms/bibleimportform.py b/openlp/plugins/bibles/forms/bibleimportform.py index e360cd4a1..319989433 100644 --- a/openlp/plugins/bibles/forms/bibleimportform.py +++ b/openlp/plugins/bibles/forms/bibleimportform.py @@ -38,7 +38,7 @@ from openlp.core.lib import Settings, UiStrings, translate from openlp.core.lib.db import delete_database from openlp.core.lib.ui import critical_error_message_box from openlp.core.ui.wizard import OpenLPWizard, WizardStrings -from openlp.core.utils import AppLocation, locale_compare +from openlp.core.utils import AppLocation, get_local_key from openlp.plugins.bibles.lib.manager import BibleFormat from openlp.plugins.bibles.lib.db import BiblesResourcesDB, clean_filename @@ -455,7 +455,7 @@ class BibleImportForm(OpenLPWizard): """ self.webTranslationComboBox.clear() bibles = self.web_bible_list[index].keys() - bibles.sort(cmp=locale_compare) + bibles.sort(key=get_local_key) self.webTranslationComboBox.addItems(bibles) def onOsisBrowseButtonClicked(self): diff --git a/openlp/plugins/bibles/lib/mediaitem.py b/openlp/plugins/bibles/lib/mediaitem.py index abe3cc45a..4ae7e76b5 100644 --- a/openlp/plugins/bibles/lib/mediaitem.py +++ b/openlp/plugins/bibles/lib/mediaitem.py @@ -36,7 +36,7 @@ from openlp.core.lib import Registry, MediaManagerItem, ItemCapabilities, Servic from openlp.core.lib.searchedit import SearchEdit from openlp.core.lib.ui import set_case_insensitive_completer, create_horizontal_adjusting_combo_box, \ critical_error_message_box, find_and_set_in_combo_box, build_icon -from openlp.core.utils import locale_compare +from openlp.core.utils import get_local_key from openlp.plugins.bibles.forms import BibleImportForm, EditBibleForm from openlp.plugins.bibles.lib import LayoutStyle, DisplayStyle, VerseReferenceList, get_reference_separator, \ LanguageSelection, BibleStrings @@ -325,7 +325,7 @@ class BibleMediaItem(MediaManagerItem): # Get all bibles and sort the list. bibles = self.plugin.manager.get_bibles().keys() bibles = filter(None, bibles) - bibles.sort(cmp=locale_compare) + bibles.sort(key=get_local_key) # Load the bibles into the combo boxes. self.quickVersionComboBox.addItems(bibles) self.quickSecondComboBox.addItems(bibles) @@ -461,7 +461,7 @@ class BibleMediaItem(MediaManagerItem): for book in book_data: data = BiblesResourcesDB.get_book_by_id(book.book_reference_id) books.append(data[u'name'] + u' ') - books.sort(cmp=locale_compare) + books.sort(key=get_local_key) set_case_insensitive_completer(books, self.quickSearchEdit) def on_import_click(self): diff --git a/openlp/plugins/custom/lib/db.py b/openlp/plugins/custom/lib/db.py index cc6e45742..ad876b6b6 100644 --- a/openlp/plugins/custom/lib/db.py +++ b/openlp/plugins/custom/lib/db.py @@ -35,7 +35,7 @@ from sqlalchemy import Column, Table, types from sqlalchemy.orm import mapper from openlp.core.lib.db import BaseModel, init_db -from openlp.core.utils import locale_compare +from openlp.core.utils import get_local_key class CustomSlide(BaseModel): """ @@ -44,11 +44,10 @@ class CustomSlide(BaseModel): # By default sort the customs by its title considering language specific # characters. def __lt__(self, other): - r = locale_compare(self.title, other.title) - return True if r < 0 else False + return get_local_key(self.title) < get_local_key(other.title) def __eq__(self, other): - return 0 == locale_compare(self.title, other.title) + return get_local_key(self.title) == get_local_key(other.title) def init_schema(url): diff --git a/openlp/plugins/images/lib/mediaitem.py b/openlp/plugins/images/lib/mediaitem.py index d74b1ccab..8c6bc8d9f 100644 --- a/openlp/plugins/images/lib/mediaitem.py +++ b/openlp/plugins/images/lib/mediaitem.py @@ -36,7 +36,7 @@ from openlp.core.lib import ItemCapabilities, MediaManagerItem, Registry, Servic StringContent, TreeWidgetWithDnD, UiStrings, build_icon, check_directory_exists, check_item_selected, \ create_thumb, translate, validate_thumb from openlp.core.lib.ui import create_widget_action, critical_error_message_box -from openlp.core.utils import AppLocation, delete_file, locale_compare, get_images_filter +from openlp.core.utils import AppLocation, delete_file, get_local_key, get_images_filter from openlp.plugins.images.forms import AddGroupForm, ChooseGroupForm from openlp.plugins.images.lib.db import ImageFilenames, ImageGroups @@ -255,7 +255,7 @@ class ImageMediaItem(MediaManagerItem): The ID of the group that will be added recursively """ image_groups = self.manager.get_all_objects(ImageGroups, ImageGroups.parent_id == parent_group_id) - image_groups.sort(cmp=locale_compare, key=lambda group_object: group_object.group_name) + image_groups.sort(key=lambda group_object: get_local_key(group_object.group_name)) folder_icon = build_icon(u':/images/image_group.png') for image_group in image_groups: group = QtGui.QTreeWidgetItem() @@ -286,7 +286,7 @@ class ImageMediaItem(MediaManagerItem): combobox.clear() combobox.top_level_group_added = False image_groups = self.manager.get_all_objects(ImageGroups, ImageGroups.parent_id == parent_group_id) - image_groups.sort(cmp=locale_compare, key=lambda group_object: group_object.group_name) + image_groups.sort(key=lambda group_object: get_local_key(group_object.group_name)) for image_group in image_groups: combobox.addItem(prefix + image_group.group_name, image_group.id) self.fill_groups_combobox(combobox, image_group.id, prefix + ' ') @@ -338,7 +338,7 @@ class ImageMediaItem(MediaManagerItem): self.expand_group(open_group.id) # Sort the images by its filename considering language specific # characters. - images.sort(cmp=locale_compare, key=lambda image_object: os.path.split(unicode(image_object.filename))[1]) + images.sort(key=lambda image_object: get_local_key(os.path.split(unicode(image_object.filename))[1])) for imageFile in images: log.debug(u'Loading image: %s', imageFile.filename) filename = os.path.split(imageFile.filename)[1] @@ -525,9 +525,9 @@ class ImageMediaItem(MediaManagerItem): group_items.append(item) if isinstance(item.data(0, QtCore.Qt.UserRole), ImageFilenames): image_items.append(item) - group_items.sort(cmp=locale_compare, key=lambda item: item.text(0)) + group_items.sort(key=lambda item: get_local_key(item.text(0))) target_group.addChildren(group_items) - image_items.sort(cmp=locale_compare, key=lambda item: item.text(0)) + image_items.sort(key=lambda item: get_local_key(item.text(0))) target_group.addChildren(image_items) def generate_slide_data(self, service_item, item=None, xmlVersion=False, diff --git a/openlp/plugins/media/lib/mediaitem.py b/openlp/plugins/media/lib/mediaitem.py index 57bc6947b..26cb35cd2 100644 --- a/openlp/plugins/media/lib/mediaitem.py +++ b/openlp/plugins/media/lib/mediaitem.py @@ -37,7 +37,7 @@ from openlp.core.lib import ItemCapabilities, MediaManagerItem,MediaType, Regist from openlp.core.lib.ui import critical_error_message_box, create_horizontal_adjusting_combo_box from openlp.core.ui import DisplayController, Display, DisplayControllerType from openlp.core.ui.media import get_media_players, set_media_players -from openlp.core.utils import AppLocation, locale_compare +from openlp.core.utils import AppLocation, get_local_key log = logging.getLogger(__name__) @@ -261,7 +261,7 @@ class MediaMediaItem(MediaManagerItem): def load_list(self, media, target_group=None): # Sort the media by its filename considering language specific # characters. - media.sort(cmp=locale_compare, key=lambda filename: os.path.split(unicode(filename))[1]) + media.sort(key=lambda filename: get_local_key(os.path.split(unicode(filename))[1])) for track in media: track_info = QtCore.QFileInfo(track) if not os.path.exists(track): @@ -287,7 +287,7 @@ class MediaMediaItem(MediaManagerItem): def getList(self, type=MediaType.Audio): media = Settings().value(self.settings_section + u'/media files') - media.sort(cmp=locale_compare, key=lambda filename: os.path.split(unicode(filename))[1]) + media.sort(key=lambda filename: get_local_key(os.path.split(unicode(filename))[1])) ext = [] if type == MediaType.Audio: ext = self.media_controller.audio_extensions_list diff --git a/openlp/plugins/presentations/lib/mediaitem.py b/openlp/plugins/presentations/lib/mediaitem.py index f92562541..a87821e8c 100644 --- a/openlp/plugins/presentations/lib/mediaitem.py +++ b/openlp/plugins/presentations/lib/mediaitem.py @@ -35,7 +35,7 @@ from PyQt4 import QtCore, QtGui from openlp.core.lib import MediaManagerItem, Registry, ItemCapabilities, ServiceItemContext, Settings, UiStrings, \ build_icon, check_item_selected, create_thumb, translate, validate_thumb from openlp.core.lib.ui import critical_error_message_box, create_horizontal_adjusting_combo_box -from openlp.core.utils import locale_compare +from openlp.core.utils import get_local_key from openlp.plugins.presentations.lib import MessageListener log = logging.getLogger(__name__) @@ -153,8 +153,7 @@ class PresentationMediaItem(MediaManagerItem): if not initialLoad: self.main_window.display_progress_bar(len(files)) # Sort the presentations by its filename considering language specific characters. - files.sort(cmp=locale_compare, - key=lambda filename: os.path.split(unicode(filename))[1]) + files.sort(key=lambda filename: get_local_key(os.path.split(unicode(filename))[1])) for file in files: if not initialLoad: self.main_window.increment_progress_bar() diff --git a/openlp/plugins/songs/forms/songexportform.py b/openlp/plugins/songs/forms/songexportform.py index 79f21a454..94748d3e4 100644 --- a/openlp/plugins/songs/forms/songexportform.py +++ b/openlp/plugins/songs/forms/songexportform.py @@ -37,7 +37,7 @@ from PyQt4 import QtCore, QtGui from openlp.core.lib import Registry, UiStrings, create_separated_list, build_icon, translate from openlp.core.lib.ui import critical_error_message_box from openlp.core.ui.wizard import OpenLPWizard, WizardStrings -from openlp.plugins.songs.lib import natcmp +from openlp.core.utils import get_natural_key from openlp.plugins.songs.lib.db import Song from openlp.plugins.songs.lib.openlyricsexport import OpenLyricsExport @@ -222,7 +222,7 @@ class SongExportForm(OpenLPWizard): # Load the list of songs. self.application.set_busy_cursor() songs = self.plugin.manager.get_all_objects(Song) - songs.sort(cmp=natcmp, key=lambda song: song.sort_key) + songs.sort(key=lambda song: get_natural_key(song.sort_key)) for song in songs: # No need to export temporary songs. if song.temporary: diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index 5c1485b9e..abeec1253 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -34,7 +34,7 @@ import re from PyQt4 import QtGui from openlp.core.lib import translate -from openlp.core.utils import CONTROL_CHARS, locale_direct_compare +from openlp.core.utils import CONTROL_CHARS from db import Author from ui import SongStrings @@ -592,37 +592,3 @@ def strip_rtf(text, default_encoding=None): text = u''.join(out) return text, default_encoding - -def natcmp(a, b): - """ - Natural string comparison which mimics the behaviour of Python's internal cmp function. - """ - if len(a) <= len(b): - for i, key in enumerate(a): - if isinstance(key, int) and isinstance(b[i], int): - result = cmp(key, b[i]) - elif isinstance(key, int) and not isinstance(b[i], int): - result = locale_direct_compare(str(key), b[i]) - elif not isinstance(key, int) and isinstance(b[i], int): - result = locale_direct_compare(key, str(b[i])) - else: - result = locale_direct_compare(key, b[i]) - if result != 0: - return result - if len(a) == len(b): - return 0 - else: - return -1 - else: - for i, key in enumerate(b): - if isinstance(a[i], int) and isinstance(key, int): - result = cmp(a[i], key) - elif isinstance(a[i], int) and not isinstance(key, int): - result = locale_direct_compare(str(a[i]), key) - elif not isinstance(a[i], int) and isinstance(key, int): - result = locale_direct_compare(a[i], str(key)) - else: - result = locale_direct_compare(a[i], key) - if result != 0: - return result - return 1 diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index 0c4898fd9..81aab4d0f 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -38,12 +38,12 @@ from sqlalchemy.sql import or_ from openlp.core.lib import Registry, MediaManagerItem, ItemCapabilities, PluginStatus, ServiceItemContext, Settings, \ UiStrings, translate, check_item_selected, create_separated_list, check_directory_exists from openlp.core.lib.ui import create_widget_action -from openlp.core.utils import AppLocation +from openlp.core.utils import AppLocation, get_natural_key from openlp.plugins.songs.forms.editsongform import EditSongForm from openlp.plugins.songs.forms.songmaintenanceform import SongMaintenanceForm from openlp.plugins.songs.forms.songimportform import SongImportForm from openlp.plugins.songs.forms.songexportform import SongExportForm -from openlp.plugins.songs.lib import VerseType, clean_string, natcmp +from openlp.plugins.songs.lib import VerseType, clean_string from openlp.plugins.songs.lib.db import Author, Song, Book, MediaFile from openlp.plugins.songs.lib.ui import SongStrings from openlp.plugins.songs.lib.xml import OpenLyrics, SongXML @@ -225,7 +225,7 @@ class SongMediaItem(MediaManagerItem): log.debug(u'display results Song') self.save_auto_select_id() self.list_view.clear() - searchresults.sort(cmp=natcmp, key=lambda song: song.sort_key) + searchresults.sort(key=lambda song: get_natural_key(song.sort_key)) for song in searchresults: # Do not display temporary songs if song.temporary: diff --git a/scripts/check_dependencies.py b/scripts/check_dependencies.py index 4c0f69b91..a6e075db4 100755 --- a/scripts/check_dependencies.py +++ b/scripts/check_dependencies.py @@ -83,6 +83,7 @@ MODULES = [ 'mako', 'migrate', 'uno', + 'icu', ]