From 47e7937baf9e0e5db08161b248543d203ba2cf30 Mon Sep 17 00:00:00 2001 From: M2j Date: Mon, 1 Nov 2010 21:48:30 +0100 Subject: [PATCH 01/10] - recognize openlp.org 1.x import char encoding by title, lyrics and copyright - prefere utf-8 encoding for CCLI imports --- openlp/plugins/songs/lib/cclifileimport.py | 7 ++++++- openlp/plugins/songs/lib/olp1import.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) mode change 100755 => 100644 openlp/plugins/songs/lib/cclifileimport.py diff --git a/openlp/plugins/songs/lib/cclifileimport.py b/openlp/plugins/songs/lib/cclifileimport.py old mode 100755 new mode 100644 index 669201325..c52343844 --- a/openlp/plugins/songs/lib/cclifileimport.py +++ b/openlp/plugins/songs/lib/cclifileimport.py @@ -76,7 +76,12 @@ class CCLIFileImport(SongImport): lines = [] if os.path.isfile(filename): detect_file = open(filename, u'r') - details = chardet.detect(detect_file.read(2048)) + detect_content = detect_file.read(2048) + try: + unicode(detect_content, u'utf-8') + details = {'confidence': 1, 'encoding': 'utf-8'} + except UnicodeDecodeError: + details = chardet.detect(detect_content) detect_file.close() infile = codecs.open(filename, u'r', details['encoding']) lines = infile.readlines() diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index de750cb24..0d981b87f 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -118,7 +118,7 @@ class OpenLP1SongImport(SongImport): success = False break song_id = song[0] - guess = chardet.detect(song[2]) + guess = chardet.detect(song[1] + song[2] + song[3]) title = self.decode_string(song[1], guess) lyrics = self.decode_string(song[2], guess).replace(u'\r', u'') copyright = self.decode_string(song[3], guess) From 4294ae83b8f158a1ac33424cd60079fd509aa7e1 Mon Sep 17 00:00:00 2001 From: M2j Date: Thu, 4 Nov 2010 20:32:43 +0100 Subject: [PATCH 02/10] split song verse order at consecutive whitespaces and not just u' ' --- openlp/plugins/songs/forms/editsongform.py | 2 +- openlp/plugins/songs/lib/mediaitem.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index 67ff6f32f..005ca0192 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -527,7 +527,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): return False if self.song.verse_order: order = [] - order_names = self.song.verse_order.split(u' ') + order_names = self.song.verse_order.split(None) for item in order_names: if len(item) == 1: order.append(item.lower() + u'1') diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index a211344c2..03e4e909c 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -358,7 +358,7 @@ class SongMediaItem(MediaManagerItem): verse[1][:30], unicode(verse[1]), verseTag) else: #Loop through the verse list and expand the song accordingly. - for order in song.verse_order.upper().split(u' '): + for order in song.verse_order.upper().split(None): if len(order) == 0: break for verse in verseList: From fa8997177143e52a94fd0398db3c80240ea82ee3 Mon Sep 17 00:00:00 2001 From: M2j Date: Thu, 4 Nov 2010 21:45:49 +0100 Subject: [PATCH 03/10] openlp.org 1.x importer graped authors out of the lyrics --- openlp/plugins/songs/lib/olp1import.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index 0d981b87f..9d584ab0d 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -126,7 +126,10 @@ class OpenLP1SongImport(SongImport): unicode(translate('SongsPlugin.ImportWizardForm', 'Importing "%s"...')) % title) self.title = title - self.process_song_text(lyrics) + verses = lyrics.split(u'\n\n') + for verse in verses: + if verse.strip() != u'': + self.add_verse(verse.strip()) self.add_copyright(copyright) cursor.execute(u'SELECT authorid FROM songauthors ' u'WHERE songid = %s' % song_id) From 0853e52de7c5d1482d801b173d8575fc45e991b6 Mon Sep 17 00:00:00 2001 From: M2j Date: Thu, 4 Nov 2010 23:25:42 +0100 Subject: [PATCH 04/10] single chardet for the whole olp1import database --- openlp/plugins/songs/lib/olp1import.py | 35 +++++++++++++++----------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index 9d584ab0d..904250c52 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -28,7 +28,7 @@ The :mod:`olp1import` module provides the functionality for importing openlp.org 1.x song databases into the current installation database. """ import logging -import chardet +from chardet.universaldetector import UniversalDetector import sqlite from openlp.core.lib import translate @@ -66,22 +66,11 @@ class OpenLP1SongImport(SongImport): ``guess`` What chardet guessed the encoding to be. """ - if guess[u'confidence'] < 0.8: - codec = u'windows-1252' - else: - codec = guess[u'encoding'] try: - decoded = unicode(raw, codec) - self.last_encoding = codec + decoded = unicode(raw, guess[u'encoding']) except UnicodeDecodeError: log.exception(u'Error in detecting openlp.org 1.x database encoding.') - try: - decoded = unicode(raw, self.last_encoding) - except UnicodeDecodeError: - # possibly show an error form - #self.import_wizard.showError(u'There was a problem ' - # u'detecting the encoding of a string') - decoded = raw + decoded = raw return decoded def do_import(self): @@ -112,13 +101,29 @@ class OpenLP1SongImport(SongImport): cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, ' u'copyrightinfo FROM songs') songs = cursor.fetchall() + detector = UniversalDetector() + for author in authors: + detector.feed(author[1]) + if detector.done: + break + for index in [1, 3, 2]: + for song in songs: + detector.feed(song[index]) + if detector.done: + break + if new_db: + for track in tracks: + detector.feed(track[1]) + if detector.done: + break + detector.close() + guess = detector.result for song in songs: self.set_defaults() if self.stop_import_flag: success = False break song_id = song[0] - guess = chardet.detect(song[1] + song[2] + song[3]) title = self.decode_string(song[1], guess) lyrics = self.decode_string(song[2], guess).replace(u'\r', u'') copyright = self.decode_string(song[3], guess) From 5e9ff18927306d5788cf4c0f0be60eb8d538a9cf Mon Sep 17 00:00:00 2001 From: M2j Date: Sun, 7 Nov 2010 23:57:30 +0100 Subject: [PATCH 05/10] caracter encoding comboBox for openlp.org 1.x import --- openlp/plugins/songs/lib/olp1import.py | 148 ++++++++++++++++++++----- 1 file changed, 121 insertions(+), 27 deletions(-) diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index 904250c52..73fc0284b 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -27,6 +27,9 @@ The :mod:`olp1import` module provides the functionality for importing openlp.org 1.x song databases into the current installation database. """ + +from PyQt4 import QtGui, QtCore + import logging from chardet.universaldetector import UniversalDetector import sqlite @@ -56,20 +59,21 @@ class OpenLP1SongImport(SongImport): SongImport.__init__(self, manager) self.import_source = kwargs[u'filename'] - def decode_string(self, raw, guess): + def decode_string(self, raw, encoding): """ Use chardet to detect the encoding of the raw string, and convert it to unicode. ``raw`` The raw bytestring to decode. - ``guess`` - What chardet guessed the encoding to be. + ``encoding`` + The bytestring character encoding. """ try: - decoded = unicode(raw, guess[u'encoding']) + decoded = unicode(raw, encoding) except UnicodeDecodeError: - log.exception(u'Error in detecting openlp.org 1.x database encoding.') + log.exception(u'The openlp.org 1.x database is not %s encoded.' % \ + encoding) decoded = raw return decoded @@ -101,32 +105,19 @@ class OpenLP1SongImport(SongImport): cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, ' u'copyrightinfo FROM songs') songs = cursor.fetchall() - detector = UniversalDetector() - for author in authors: - detector.feed(author[1]) - if detector.done: - break - for index in [1, 3, 2]: - for song in songs: - detector.feed(song[index]) - if detector.done: - break - if new_db: - for track in tracks: - detector.feed(track[1]) - if detector.done: - break - detector.close() - guess = detector.result + encoding = self.get_encoding() + if not encoding: + self.stop_import_flag = True + return False for song in songs: self.set_defaults() if self.stop_import_flag: success = False break song_id = song[0] - title = self.decode_string(song[1], guess) - lyrics = self.decode_string(song[2], guess).replace(u'\r', u'') - copyright = self.decode_string(song[3], guess) + title = self.decode_string(song[1], encoding) + lyrics = self.decode_string(song[2], encoding).replace(u'\r', u'') + copyright = self.decode_string(song[3], encoding) self.import_wizard.incrementProgressBar( unicode(translate('SongsPlugin.ImportWizardForm', 'Importing "%s"...')) % title) @@ -145,7 +136,7 @@ class OpenLP1SongImport(SongImport): break for author in authors: if author[0] == author_id[0]: - self.parse_author(self.decode_string(author[1], guess)) + self.parse_author(self.decode_string(author[1], encoding)) break if self.stop_import_flag: success = False @@ -160,7 +151,7 @@ class OpenLP1SongImport(SongImport): break for track in tracks: if track[0] == track_id[0]: - self.add_media_file(self.decode_string(track[1], guess)) + self.add_media_file(self.decode_string(track[1], encoding)) break if self.stop_import_flag: success = False @@ -168,3 +159,106 @@ class OpenLP1SongImport(SongImport): self.finish() return success + def get_encoding(self): + """ + Detect character encoding of an openlp.org 1.x song database. + """ + # Connect to the database + connection = sqlite.connect(self.import_source) + cursor = connection.cursor() + + detector = UniversalDetector() + # detect charset by authors + cursor.execute(u'SELECT authorname FROM authors') + authors = cursor.fetchall() + for author in authors: + detector.feed(author[0]) + if detector.done: + detector.close() + return detector.result[u'encoding'] + # detect charset by songs + cursor.execute(u'SELECT songtitle, copyrightinfo, ' + u'lyrics || \'\' AS lyrics FROM songs') + songs = cursor.fetchall() + for index in [0, 1, 2]: + for song in songs: + detector.feed(song[index]) + if detector.done: + detector.close() + return detector.result[u'encoding'] + # detect charset by songs + cursor.execute(u'SELECT name FROM sqlite_master ' + u'WHERE type = \'table\' AND name = \'tracks\'') + if len(cursor.fetchall()) > 0: + cursor.execute(u'SELECT fulltrackname FROM tracks') + tracks = cursor.fetchall() + for track in tracks: + detector.feed(track[0]) + if detector.done: + detector.close() + return detector.result[u'encoding'] + detector.close() + guess = detector.result[u'encoding'] + + # map chardet result to compatible windows standard code page + codepage_mapping = {'IBM866': u'cp866', 'TIS-620': u'cp874', + 'SHIFT_JIS': u'cp932', 'GB2312': u'cp936', 'HZ-GB-2312': u'cp936', + 'EUC-KR': u'cp949', 'Big5': u'cp950', 'ISO-8859-2': u'cp1250', + 'windows-1250': u'cp1250', 'windows-1251': u'cp1251', + 'windows-1252': u'cp1252', 'ISO-8859-7': u'cp1253', + 'windows-1253': u'cp1253', 'ISO-8859-8': u'cp1255', + 'windows-1255': u'cp1255'} + if guess in codepage_mapping: + guess = codepage_mapping[guess] + else: + guess = u'cp1252' + + encodings = {u'cp874': translate('SongsPlugin.OpenLP1SongImport', + 'CP-874 (Thai)'), + u'cp932': translate('SongsPlugin.OpenLP1SongImport', + 'CP-932 (Japanese)'), + u'cp936': translate('SongsPlugin.OpenLP1SongImport', + 'CP-936 (Simplified Chinese)'), + u'cp949': translate('SongsPlugin.OpenLP1SongImport', + 'CP-949 (Korean)'), + u'cp950': translate('SongsPlugin.OpenLP1SongImport', + 'CP-950 (Traditional Chinese)'), + u'cp1250': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1250 (Central European)'), + u'cp1251': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1251 (Cyrillic)'), + u'cp1252': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1252 (Western European)'), + u'cp1253': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1253 (Greek)'), + u'cp1254': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1254 (Turkish)'), + u'cp1255': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1255 (Hebrew)'), + u'cp1256': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1256 (Arabic)'), + u'cp1257': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1257 (Baltic)'), + u'cp1258': translate('SongsPlugin.OpenLP1SongImport', + 'CP-1258 (Vietnam)')} + encoding_list = encodings.keys() + encoding_index = 0 + for encoding in encoding_list: + if encoding == guess: + break + else: + encoding_index = encoding_index + 1 + ok_applied = False + chosen_encoding = QtGui.QInputDialog.getItem(None, + translate('SongsPlugin.OpenLP1SongImport', + 'Database Character Encoding'), + translate('SongsPlugin.OpenLP1SongImport', + 'The codepage setting is responsible\n' + 'for the correct character representation.\n' + 'Usually you are fine with the preselected choise.'), + encodings.values(), encoding_index, False) + if not chosen_encoding[1]: + return None + for encoding in encodings.items(): + if encoding[1] == unicode(chosen_encoding[0]): + return encoding[0] From ef91b24d8a41ac138302f6619927733123dbf82c Mon Sep 17 00:00:00 2001 From: M2j Date: Fri, 12 Nov 2010 13:23:44 +0100 Subject: [PATCH 06/10] using PySQLite type conversion --- openlp/plugins/songs/lib/olp1import.py | 47 ++++++++++---------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index 73fc0284b..94a84bc0d 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -59,65 +59,50 @@ class OpenLP1SongImport(SongImport): SongImport.__init__(self, manager) self.import_source = kwargs[u'filename'] - def decode_string(self, raw, encoding): - """ - Use chardet to detect the encoding of the raw string, and convert it - to unicode. - - ``raw`` - The raw bytestring to decode. - ``encoding`` - The bytestring character encoding. - """ - try: - decoded = unicode(raw, encoding) - except UnicodeDecodeError: - log.exception(u'The openlp.org 1.x database is not %s encoded.' % \ - encoding) - decoded = raw - return decoded - def do_import(self): """ Run the import for an openlp.org 1.x song database. """ # Connect to the database - connection = sqlite.connect(self.import_source) + encoding = self.get_encoding() + if not encoding: + return False + connection = sqlite.connect(self.import_source, mode=0555, + encoding=(encoding, 'replace')) cursor = connection.cursor() # Determine if we're using a new or an old DB cursor.execute(u'SELECT name FROM sqlite_master ' u'WHERE type = \'table\' AND name = \'tracks\'') - table_list = cursor.fetchall() - new_db = len(table_list) > 0 + new_db = len(cursor.fetchall()) > 0 # Count the number of records we need to import, for the progress bar + cursor.execute(u'-- types int') cursor.execute(u'SELECT COUNT(songid) FROM songs') count = int(cursor.fetchone()[0]) success = True self.import_wizard.importProgressBar.setMaximum(count) # "cache" our list of authors + cursor.execute(u'-- types int, unicode') cursor.execute(u'SELECT authorid, authorname FROM authors') authors = cursor.fetchall() if new_db: # "cache" our list of tracks + cursor.execute(u'-- types int, unicode') cursor.execute(u'SELECT trackid, fulltrackname FROM tracks') tracks = cursor.fetchall() # Import the songs + cursor.execute(u'-- types int, unicode, unicode, unicode') cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, ' u'copyrightinfo FROM songs') songs = cursor.fetchall() - encoding = self.get_encoding() - if not encoding: - self.stop_import_flag = True - return False for song in songs: self.set_defaults() if self.stop_import_flag: success = False break song_id = song[0] - title = self.decode_string(song[1], encoding) - lyrics = self.decode_string(song[2], encoding).replace(u'\r', u'') - copyright = self.decode_string(song[3], encoding) + title = song[1] + lyrics = song[2].replace(u'\r\n', u'\n') + copyright = song[3] self.import_wizard.incrementProgressBar( unicode(translate('SongsPlugin.ImportWizardForm', 'Importing "%s"...')) % title) @@ -127,6 +112,7 @@ class OpenLP1SongImport(SongImport): if verse.strip() != u'': self.add_verse(verse.strip()) self.add_copyright(copyright) + cursor.execute(u'-- types int') cursor.execute(u'SELECT authorid FROM songauthors ' u'WHERE songid = %s' % song_id) author_ids = cursor.fetchall() @@ -136,12 +122,13 @@ class OpenLP1SongImport(SongImport): break for author in authors: if author[0] == author_id[0]: - self.parse_author(self.decode_string(author[1], encoding)) + self.parse_author(author[1]) break if self.stop_import_flag: success = False break if new_db: + cursor.execute(u'-- types int') cursor.execute(u'SELECT trackid FROM songtracks ' u'WHERE songid = %s ORDER BY listindex' % song_id) track_ids = cursor.fetchall() @@ -151,7 +138,7 @@ class OpenLP1SongImport(SongImport): break for track in tracks: if track[0] == track_id[0]: - self.add_media_file(self.decode_string(track[1], encoding)) + self.add_media_file(track[1]) break if self.stop_import_flag: success = False From 682fcc240540694f2a04c85fa17f174e53b7e138 Mon Sep 17 00:00:00 2001 From: M2j Date: Fri, 12 Nov 2010 14:20:45 +0100 Subject: [PATCH 07/10] use a list in the OLP1 import to get sorted entries in the encoding comboBox --- openlp/plugins/songs/lib/olp1import.py | 81 +++++++++++++------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index 94a84bc0d..c91dd3bc4 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -67,7 +67,7 @@ class OpenLP1SongImport(SongImport): encoding = self.get_encoding() if not encoding: return False - connection = sqlite.connect(self.import_source, mode=0555, + connection = sqlite.connect(self.import_source, mode=0444, encoding=(encoding, 'replace')) cursor = connection.cursor() # Determine if we're using a new or an old DB @@ -151,7 +151,7 @@ class OpenLP1SongImport(SongImport): Detect character encoding of an openlp.org 1.x song database. """ # Connect to the database - connection = sqlite.connect(self.import_source) + connection = sqlite.connect(self.import_source, mode=0444) cursor = connection.cursor() detector = UniversalDetector() @@ -200,42 +200,43 @@ class OpenLP1SongImport(SongImport): else: guess = u'cp1252' - encodings = {u'cp874': translate('SongsPlugin.OpenLP1SongImport', - 'CP-874 (Thai)'), - u'cp932': translate('SongsPlugin.OpenLP1SongImport', - 'CP-932 (Japanese)'), - u'cp936': translate('SongsPlugin.OpenLP1SongImport', - 'CP-936 (Simplified Chinese)'), - u'cp949': translate('SongsPlugin.OpenLP1SongImport', - 'CP-949 (Korean)'), - u'cp950': translate('SongsPlugin.OpenLP1SongImport', - 'CP-950 (Traditional Chinese)'), - u'cp1250': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1250 (Central European)'), - u'cp1251': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1251 (Cyrillic)'), - u'cp1252': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1252 (Western European)'), - u'cp1253': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1253 (Greek)'), - u'cp1254': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1254 (Turkish)'), - u'cp1255': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1255 (Hebrew)'), - u'cp1256': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1256 (Arabic)'), - u'cp1257': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1257 (Baltic)'), - u'cp1258': translate('SongsPlugin.OpenLP1SongImport', - 'CP-1258 (Vietnam)')} - encoding_list = encodings.keys() + # Show dialog for encoding selection + encodings = [[u'cp874', u'cp932', u'cp936', u'cp949', u'cp950', + u'cp1250', u'cp1251', u'cp1252', u'cp1253', u'cp1254', + u'cp1255', u'cp1256', u'cp1257', u'cp1258'], + [translate('SongsPlugin.OpenLP1SongImport', + 'CP-874 (Thai)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-932 (Japanese)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-936 (Simplified Chinese)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-949 (Korean)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-950 (Traditional Chinese)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1250 (Central European)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1251 (Cyrillic)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1252 (Western European)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1253 (Greek)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1254 (Turkish)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1255 (Hebrew)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1256 (Arabic)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1257 (Baltic)'), + translate('SongsPlugin.OpenLP1SongImport', + 'CP-1258 (Vietnam)')]] encoding_index = 0 - for encoding in encoding_list: - if encoding == guess: + for index in range(len(encodings[0])): + if guess == encodings[0][index]: + encoding_index = index break - else: - encoding_index = encoding_index + 1 - ok_applied = False chosen_encoding = QtGui.QInputDialog.getItem(None, translate('SongsPlugin.OpenLP1SongImport', 'Database Character Encoding'), @@ -243,9 +244,9 @@ class OpenLP1SongImport(SongImport): 'The codepage setting is responsible\n' 'for the correct character representation.\n' 'Usually you are fine with the preselected choise.'), - encodings.values(), encoding_index, False) + encodings[1], encoding_index, False) if not chosen_encoding[1]: return None - for encoding in encodings.items(): - if encoding[1] == unicode(chosen_encoding[0]): - return encoding[0] + for index in range(len(encodings[1])): + if unicode(chosen_encoding[0]) == encodings[1][index]: + return encodings[0][index] From e81bd1c3fc7c3211d9012d7e7b0ebe377ceddec4 Mon Sep 17 00:00:00 2001 From: M2j Date: Sat, 13 Nov 2010 09:01:47 +0100 Subject: [PATCH 08/10] remove QtCore from header --- openlp/plugins/songs/lib/olp1import.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index c91dd3bc4..b58eaeff7 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -28,7 +28,7 @@ The :mod:`olp1import` module provides the functionality for importing openlp.org 1.x song databases into the current installation database. """ -from PyQt4 import QtGui, QtCore +from PyQt4 import QtGui import logging from chardet.universaldetector import UniversalDetector @@ -77,7 +77,7 @@ class OpenLP1SongImport(SongImport): # Count the number of records we need to import, for the progress bar cursor.execute(u'-- types int') cursor.execute(u'SELECT COUNT(songid) FROM songs') - count = int(cursor.fetchone()[0]) + count = cursor.fetchone()[0] success = True self.import_wizard.importProgressBar.setMaximum(count) # "cache" our list of authors @@ -248,5 +248,5 @@ class OpenLP1SongImport(SongImport): if not chosen_encoding[1]: return None for index in range(len(encodings[1])): - if unicode(chosen_encoding[0]) == encodings[1][index]: + if chosen_encoding[0] == encodings[1][index]: return encodings[0][index] From bb571b0df583498f8ce355314268706bbb77980a Mon Sep 17 00:00:00 2001 From: M2j Date: Sat, 13 Nov 2010 22:04:11 +0100 Subject: [PATCH 09/10] using list of tuples --- openlp/plugins/songs/lib/olp1import.py | 70 ++++++++++++-------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index b58eaeff7..6b93195db 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -201,40 +201,37 @@ class OpenLP1SongImport(SongImport): guess = u'cp1252' # Show dialog for encoding selection - encodings = [[u'cp874', u'cp932', u'cp936', u'cp949', u'cp950', - u'cp1250', u'cp1251', u'cp1252', u'cp1253', u'cp1254', - u'cp1255', u'cp1256', u'cp1257', u'cp1258'], - [translate('SongsPlugin.OpenLP1SongImport', - 'CP-874 (Thai)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-932 (Japanese)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-936 (Simplified Chinese)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-949 (Korean)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-950 (Traditional Chinese)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1250 (Central European)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1251 (Cyrillic)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1252 (Western European)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1253 (Greek)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1254 (Turkish)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1255 (Hebrew)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1256 (Arabic)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1257 (Baltic)'), - translate('SongsPlugin.OpenLP1SongImport', - 'CP-1258 (Vietnam)')]] + encodings = [(u'cp874', translate('SongsPlugin.OpenLP1SongImport', + 'CP-874 (Thai)')), + (u'cp932', translate('SongsPlugin.OpenLP1SongImport', + 'CP-932 (Japanese)')), + (u'cp936', translate('SongsPlugin.OpenLP1SongImport', + 'CP-936 (Simplified Chinese)')), + (u'cp949', translate('SongsPlugin.OpenLP1SongImport', + 'CP-949 (Korean)')), + (u'cp950', translate('SongsPlugin.OpenLP1SongImport', + 'CP-950 (Traditional Chinese)')), + (u'cp1250', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1250 (Central European)')), + (u'cp1251', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1251 (Cyrillic)')), + (u'cp1252', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1252 (Western European)')), + (u'cp1253', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1253 (Greek)')), + (u'cp1254', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1254 (Turkish)')), + (u'cp1255', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1255 (Hebrew)')), + (u'cp1256', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1256 (Arabic)')), + (u'cp1257', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1257 (Baltic)')), + (u'cp1258', translate('SongsPlugin.OpenLP1SongImport', + 'CP-1258 (Vietnam)'))] encoding_index = 0 - for index in range(len(encodings[0])): - if guess == encodings[0][index]: + for index in range(len(encodings)): + if guess == encodings[index][0]: encoding_index = index break chosen_encoding = QtGui.QInputDialog.getItem(None, @@ -244,9 +241,8 @@ class OpenLP1SongImport(SongImport): 'The codepage setting is responsible\n' 'for the correct character representation.\n' 'Usually you are fine with the preselected choise.'), - encodings[1], encoding_index, False) + [pair[1] for pair in encodings], encoding_index, False) if not chosen_encoding[1]: return None - for index in range(len(encodings[1])): - if chosen_encoding[0] == encodings[1][index]: - return encodings[0][index] + return filter(lambda item: item[1] == chosen_encoding[0], + encodings)[0][0] From 62766269f13a3397da67e289d9f64f132ff828ed Mon Sep 17 00:00:00 2001 From: M2j Date: Wed, 8 Dec 2010 09:21:17 +0100 Subject: [PATCH 10/10] - remove None in .split() method - change translatable codepage selection strings --- openlp/plugins/songs/forms/editsongform.py | 2 +- openlp/plugins/songs/lib/mediaitem.py | 2 +- openlp/plugins/songs/lib/olp1import.py | 52 +++++++++++----------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index 1d7b5f1c2..2846a98df 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -525,7 +525,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): return False if self.song.verse_order: order = [] - order_names = self.song.verse_order.split(None) + order_names = self.song.verse_order.split() for item in order_names: if len(item) == 1: order.append(item.lower() + u'1') diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index cd921b9ef..4ca391f20 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -358,7 +358,7 @@ class SongMediaItem(MediaManagerItem): verse[1][:30], unicode(verse[1]), verseTag) else: #Loop through the verse list and expand the song accordingly. - for order in song.verse_order.upper().split(None): + for order in song.verse_order.upper().split(): if len(order) == 0: break for verse in verseList: diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py index 6b93195db..7c01a4c19 100644 --- a/openlp/plugins/songs/lib/olp1import.py +++ b/openlp/plugins/songs/lib/olp1import.py @@ -201,34 +201,34 @@ class OpenLP1SongImport(SongImport): guess = u'cp1252' # Show dialog for encoding selection - encodings = [(u'cp874', translate('SongsPlugin.OpenLP1SongImport', - 'CP-874 (Thai)')), - (u'cp932', translate('SongsPlugin.OpenLP1SongImport', - 'CP-932 (Japanese)')), - (u'cp936', translate('SongsPlugin.OpenLP1SongImport', - 'CP-936 (Simplified Chinese)')), - (u'cp949', translate('SongsPlugin.OpenLP1SongImport', - 'CP-949 (Korean)')), - (u'cp950', translate('SongsPlugin.OpenLP1SongImport', - 'CP-950 (Traditional Chinese)')), - (u'cp1250', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1250 (Central European)')), - (u'cp1251', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1251 (Cyrillic)')), - (u'cp1252', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1252 (Western European)')), - (u'cp1253', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1253 (Greek)')), - (u'cp1254', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1254 (Turkish)')), - (u'cp1255', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1255 (Hebrew)')), - (u'cp1256', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1256 (Arabic)')), + encodings = [(u'cp1256', translate('SongsPlugin.OpenLP1SongImport', + 'Arabic (CP-1256)')), (u'cp1257', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1257 (Baltic)')), + 'Baltic (CP-1257)')), + (u'cp1250', translate('SongsPlugin.OpenLP1SongImport', + 'Central European (CP-1250)')), + (u'cp1251', translate('SongsPlugin.OpenLP1SongImport', + 'Cyrillic (CP-1251)')), + (u'cp1253', translate('SongsPlugin.OpenLP1SongImport', + 'Greek (CP-1253)')), + (u'cp1255', translate('SongsPlugin.OpenLP1SongImport', + 'Hebrew (CP-1255)')), + (u'cp932', translate('SongsPlugin.OpenLP1SongImport', + 'Japanese (CP-932)')), + (u'cp949', translate('SongsPlugin.OpenLP1SongImport', + 'Korean (CP-949)')), + (u'cp936', translate('SongsPlugin.OpenLP1SongImport', + 'Simplified Chinese (CP-936)')), + (u'cp874', translate('SongsPlugin.OpenLP1SongImport', + 'Thai (CP-874)')), + (u'cp950', translate('SongsPlugin.OpenLP1SongImport', + 'Traditional Chinese (CP-950)')), + (u'cp1254', translate('SongsPlugin.OpenLP1SongImport', + 'Turkish (CP-1254)')), (u'cp1258', translate('SongsPlugin.OpenLP1SongImport', - 'CP-1258 (Vietnam)'))] + 'Vietnam (CP-1258)')), + (u'cp1252', translate('SongsPlugin.OpenLP1SongImport', + 'Western European (CP-1252)'))] encoding_index = 0 for index in range(len(encodings)): if guess == encodings[index][0]: