From 74e5bcf500b943d62c8ba6b4656e779cfe8e9c8d Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Fri, 6 Sep 2013 20:40:48 -0500 Subject: [PATCH 01/16] Update helper scripts to Python 3 --- scripts/check_dependencies.py | 6 +++--- scripts/translation_utils.py | 8 ++++---- setup.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/check_dependencies.py b/scripts/check_dependencies.py index 348a809eb..c37bc5821 100755 --- a/scripts/check_dependencies.py +++ b/scripts/check_dependencies.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 @@ -49,7 +49,7 @@ except ImportError: IS_WIN = sys.platform.startswith('win') VERS = { - 'Python': '2.6', + 'Python': '3.0', 'PyQt4': '4.6', 'Qt4': '4.6', 'sqlalchemy': '0.5', @@ -76,6 +76,7 @@ MODULES = [ 'PyQt4.QtWebKit', 'PyQt4.phonon', 'sqlalchemy', + 'alembic', 'sqlite3', 'lxml', 'chardet', @@ -85,7 +86,6 @@ MODULES = [ 'cherrypy', 'uno', 'icu', - 'bs4', ] diff --git a/scripts/translation_utils.py b/scripts/translation_utils.py index a9fa86db9..32f54a950 100755 --- a/scripts/translation_utils.py +++ b/scripts/translation_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 @@ -102,7 +102,7 @@ class CommandStack(object): def __iter__(self): return self - def next(self): + def __next__(self): if self.current_index == len(self.data): raise StopIteration else: @@ -145,9 +145,9 @@ def print_quiet(text, linefeed=True): global quiet_mode if not quiet_mode: if linefeed: - print text + print (text) else: - print text, + print (text, end=' ') def print_verbose(text): """ diff --git a/setup.py b/setup.py index 0e8f02384..4e97683a9 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 From a23022d84f31012327c817d49e4d2cc8f426203e Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Sat, 7 Sep 2013 12:33:36 -0500 Subject: [PATCH 02/16] Fix EasyWorship importer and its tests for py3 --- openlp/plugins/songs/lib/ewimport.py | 45 ++++++++++--------- .../openlp_plugins/songs/test_ewimport.py | 30 ++++++------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/openlp/plugins/songs/lib/ewimport.py b/openlp/plugins/songs/lib/ewimport.py index 380083aa2..622b7c39d 100644 --- a/openlp/plugins/songs/lib/ewimport.py +++ b/openlp/plugins/songs/lib/ewimport.py @@ -122,7 +122,7 @@ class EasyWorshipSongImport(SongImport): db_file.seek(120) field_info = db_file.read(num_fields * 2) db_file.seek(4 + (num_fields * 4) + 261, os.SEEK_CUR) - field_names = db_file.read(header_size - db_file.tell()).split('\0', num_fields) + field_names = db_file.read(header_size - db_file.tell()).split(b'\0', num_fields) field_names.pop() field_descs = [] for i, field_name in enumerate(field_names): @@ -132,12 +132,12 @@ class EasyWorshipSongImport(SongImport): # Pick out the field description indexes we will need try: success = True - fi_title = self.findField('Title') - fi_author = self.findField('Author') - fi_copy = self.findField('Copyright') - fi_admin = self.findField('Administrator') - fi_words = self.findField('Words') - fi_ccli = self.findField('Song Number') + fi_title = self.findField(b'Title') + fi_author = self.findField(b'Author') + fi_copy = self.findField(b'Copyright') + fi_admin = self.findField(b'Administrator') + fi_words = self.findField(b'Words') + fi_ccli = self.findField(b'Song Number') except IndexError: # This is the wrong table success = False @@ -150,7 +150,7 @@ class EasyWorshipSongImport(SongImport): cur_block_pos = header_size + ((cur_block - 1) * 1024 * block_size) db_file.seek(cur_block_pos) cur_block, rec_count = struct.unpack(' 63: - return '' + return b'' self.memoFile.seek(11 + (5 * sub_block), os.SEEK_CUR) sub_block_start, = struct.unpack('B', self.memoFile.read(1)) self.memoFile.seek(block_start + (sub_block_start * 16)) else: - return '' + return b'' return self.memoFile.read(blob_size) else: return 0 diff --git a/tests/functional/openlp_plugins/songs/test_ewimport.py b/tests/functional/openlp_plugins/songs/test_ewimport.py index 57959c8e2..68e934b6c 100644 --- a/tests/functional/openlp_plugins/songs/test_ewimport.py +++ b/tests/functional/openlp_plugins/songs/test_ewimport.py @@ -74,18 +74,18 @@ TEST_FIELD_DESCS = [TestFieldDesc('Title', FieldType.String, 50), TestFieldDesc('Default Background', FieldType.Logical, 1), TestFieldDesc('Words', FieldType.Memo, 250), TestFieldDesc('Words', FieldType.Memo, 250), TestFieldDesc('BK Bitmap', FieldType.Blob, 10), TestFieldDesc('Last Modified', FieldType.Timestamp, 10)] -TEST_FIELDS = ['A Heart Like Thine\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0', 32868, 2147483750, - 129, '{\\rtf1\\ansi\\deff0\\deftab254{\\fonttbl{\\f0\\fnil\\fcharset0 Arial;}{\\f1\\fnil\\fcharset0 Verdana;}}' - '{\\colortbl\\red0\\green0\\blue0;\\red255\\green0\\blue0;\\red0\\green128\\blue0;\\red0\\green0\\blue255;' - '\\red255\\green255\\blue0;\\red255\\green0\\blue255;\\red128\\g��\7\0f\r\0\0\1\0', - '{\\rtf1\\ansi\\deff0\\deftab254{\\fonttbl{\\f0\\fnil\\fcharset0 Arial;}{\\f1\\fnil\\fcharset0 Verdana;}}' - '{\\colortbl\\red0\\green0\\blue0;\\red255\\green0\\blue0;\\red0\\green128\\blue0;\\red0\\green0\\blue255;\\red255' - '\\green255\\blue0;\\red255\\green0\\blue255;\\red128\\g>�\6\0�\6\0\0\1\0', '\0\0\0\0\0\0\0\0\0\0', 0] +TEST_FIELDS = [b'A Heart Like Thine\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0', 32868, 2147483750, + 129, b'{\\rtf1\\ansi\\deff0\\deftab254{\\fonttbl{\\f0\\fnil\\fcharset0 Arial;}{\\f1\\fnil\\fcharset0 Verdana;}}' + b'{\\colortbl\\red0\\green0\\blue0;\\red255\\green0\\blue0;\\red0\\green128\\blue0;\\red0\\green0\\blue255;' + b'\\red255\\green255\\blue0;\\red255\\green0\\blue255;\\red128\\g\xBF\xBD\7\0f\r\0\0\1\0', + b'{\\rtf1\\ansi\\deff0\\deftab254{\\fonttbl{\\f0\\fnil\\fcharset0 Arial;}{\\f1\\fnil\\fcharset0 Verdana;}}' + b'{\\colortbl\\red0\\green0\\blue0;\\red255\\green0\\blue0;\\red0\\green128\\blue0;\\red0\\green0\\blue255;\\red255' + b'\\green255\\blue0;\\red255\\green0\\blue255;\\red128\\g\6\0\xEF\xBF\xBD\6\0\0\1\0', b'\0\0\0\0\0\0\0\0\0\0', 0] GET_MEMO_FIELD_TEST_RESULTS = [ - (4, '\2', {'return': '\2','read': (1, 3430), 'seek': (507136, (8, os.SEEK_CUR))}), - (4, '\3', {'return': '', 'read': (1, ), 'seek': (507136, )}), - (5, '\3', {'return': '\3', 'read': (1, 1725), 'seek': (3220111360, (41, os.SEEK_CUR), 3220111408)}), - (5, '\4', {'return': '', 'read': (), 'seek': ()})] + (4, b'\2', {'return': b'\2','read': (1, 3430), 'seek': (507136, (8, os.SEEK_CUR))}), + (4, b'\3', {'return': b'', 'read': (1, ), 'seek': (507136, )}), + (5, b'\3', {'return': b'\3', 'read': (1, 1725), 'seek': (3220111360, (41, os.SEEK_CUR), 3220111408)}), + (5, b'\4', {'return': b'', 'read': (), 'seek': ()})] class TestEasyWorshipSongImport(TestCase): """ @@ -189,7 +189,7 @@ class TestEasyWorshipSongImport(TestCase): importer.encoding = TEST_DATA_ENCODING importer.fields = TEST_FIELDS importer.fieldDescs = TEST_FIELD_DESCS - field_results = [(0, 'A Heart Like Thine'), (1, 100), (2, 102), (3, True), (6, None), (7, None)] + field_results = [(0, b'A Heart Like Thine'), (1, 100), (2, 102), (3, True), (6, None), (7, None)] # WHEN: Called with test data for field_index, result in field_results: @@ -276,7 +276,7 @@ class TestEasyWorshipSongImport(TestCase): # GIVEN: A mocked out SongImport class, a mocked out "manager" with patch('openlp.plugins.songs.lib.ewimport.SongImport'), \ patch('openlp.plugins.songs.lib.ewimport.os.path') as mocked_os_path, \ - patch('__builtin__.open') as mocked_open, \ + patch('builtins.open') as mocked_open, \ patch('openlp.plugins.songs.lib.ewimport.struct') as mocked_struct: mocked_manager = MagicMock() importer = EasyWorshipSongImport(mocked_manager) @@ -303,7 +303,7 @@ class TestEasyWorshipSongImport(TestCase): # GIVEN: A mocked out SongImport class, a mocked out "manager" with patch('openlp.plugins.songs.lib.ewimport.SongImport'), \ patch('openlp.plugins.songs.lib.ewimport.os.path') as mocked_os_path, \ - patch('__builtin__.open'), patch('openlp.plugins.songs.lib.ewimport.struct') as mocked_struct, \ + patch('builtins.open'), patch('openlp.plugins.songs.lib.ewimport.struct') as mocked_struct, \ patch('openlp.plugins.songs.lib.ewimport.retrieve_windows_encoding') as mocked_retrieve_windows_encoding: mocked_manager = MagicMock() importer = EasyWorshipSongImport(mocked_manager) @@ -354,7 +354,7 @@ class TestEasyWorshipSongImport(TestCase): # called. self.assertIsNone(importer.doImport(), 'doImport should return None when it has completed') for song_data in SONG_TEST_DATA: - print mocked_title.mocked_calls() + print (mocked_title.mocked_calls()) title = song_data['title'] author_calls = song_data['authors'] song_copyright = song_data['copyright'] From e0592a76e42930588e46c3f0f62349b2c956b900 Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Sat, 7 Sep 2013 13:36:33 -0500 Subject: [PATCH 03/16] Remove worthless print statement in EasyWorhip song import test --- tests/functional/openlp_plugins/songs/test_ewimport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/openlp_plugins/songs/test_ewimport.py b/tests/functional/openlp_plugins/songs/test_ewimport.py index 68e934b6c..3f1735a3c 100644 --- a/tests/functional/openlp_plugins/songs/test_ewimport.py +++ b/tests/functional/openlp_plugins/songs/test_ewimport.py @@ -354,7 +354,6 @@ class TestEasyWorshipSongImport(TestCase): # called. self.assertIsNone(importer.doImport(), 'doImport should return None when it has completed') for song_data in SONG_TEST_DATA: - print (mocked_title.mocked_calls()) title = song_data['title'] author_calls = song_data['authors'] song_copyright = song_data['copyright'] From c45d225b9dc453d829b5971a8eae22fb9be020b2 Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Sat, 7 Sep 2013 16:29:31 -0500 Subject: [PATCH 04/16] PyIcu is really a Windows-only dependency --- scripts/check_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_dependencies.py b/scripts/check_dependencies.py index c37bc5821..07c84b76f 100755 --- a/scripts/check_dependencies.py +++ b/scripts/check_dependencies.py @@ -63,6 +63,7 @@ WIN32_MODULES = [ 'win32ui', 'pywintypes', 'pyodbc', + 'icu', ] MODULES = [ @@ -85,7 +86,6 @@ MODULES = [ 'mako', 'cherrypy', 'uno', - 'icu', ] From 53ac150337c10a4a65d3820bc612a6b117c7a9bd Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Sat, 7 Sep 2013 22:50:51 -0500 Subject: [PATCH 05/16] Fix strip_rtf to handle CJK encodings --- openlp/plugins/songs/lib/__init__.py | 115 ++++++++++-------- .../openlp_plugins/songs/test_lib.py | 34 +++++- 2 files changed, 99 insertions(+), 50 deletions(-) diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index 271a94710..ccbe35fd9 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -46,7 +46,7 @@ log = logging.getLogger(__name__) WHITESPACE = re.compile(r'[\W_]+', re.UNICODE) APOSTROPHE = re.compile('[\'`’ʻ′]', re.UNICODE) -PATTERN = re.compile(r"\\([a-z]{1,32})(-?\d{1,10})?[ ]?|\\'([0-9a-f]{2})|\\([^a-z])|([{}])|[\r\n]+|(.)", re.I) +PATTERN = re.compile(r"(\\\*)?\\([a-z]{1,32})(-?\d{1,10})?[ ]?|\\'([0-9a-f]{2})|\\([^a-z*])|([{}])|[\r\n]+|([^\\{}\r\n]+)", re.I) # RTF control words which specify a "destination" to be ignored. DESTINATIONS = frozenset(( 'aftncn', 'aftnsep', 'aftnsepc', 'annotation', 'atnauthor', @@ -57,8 +57,8 @@ DESTINATIONS = frozenset(( 'datafield', 'datastore', 'defchp', 'defpap', 'do', 'doccomm', 'docvar', 'dptxbxtext', 'ebcend', 'ebcstart', 'factoidname', 'falt', 'fchars', 'ffdeftext', 'ffentrymcr', 'ffexitmcr', - 'ffformat', 'ffhelptext', 'ffl', 'ffname', 'ffstattext', 'field', - 'file', 'filetbl', 'fldinst', 'fldrslt', 'fldtype', 'fname', + 'ffformat', 'ffhelptext', 'ffl', 'ffname', 'ffstattext', + 'file', 'filetbl', 'fldinst', 'fldtype', 'fname', 'fontemb', 'fontfile', 'footer', 'footerf', 'footerl', 'footerr', 'footnote', 'formfield', 'ftncn', 'ftnsep', 'ftnsepc', 'g', 'generator', 'gridtbl', 'header', 'headerf', 'headerl', @@ -106,6 +106,11 @@ DESTINATIONS = frozenset(( 'xmlclose', 'xmlname', 'xmlnstbl', 'xmlopen')) # Translation of some special characters. SPECIAL_CHARS = { + '\n': '\n', + '\r': '\n', + '~': '\u00A0', + '-': '\u00AD', + '_': '\u2011', 'par': '\n', 'sect': '\n\n', # Required page and column break. @@ -132,16 +137,19 @@ SPECIAL_CHARS = { 'zwj': '\u200D', 'zwnj': '\u200C'} CHARSET_MAPPING = { - 'fcharset0': 'cp1252', - 'fcharset161': 'cp1253', - 'fcharset162': 'cp1254', - 'fcharset163': 'cp1258', - 'fcharset177': 'cp1255', - 'fcharset178': 'cp1256', - 'fcharset186': 'cp1257', - 'fcharset204': 'cp1251', - 'fcharset222': 'cp874', - 'fcharset238': 'cp1250'} + '0': 'cp1252', + '128': 'cp932', + '129': 'cp949', + '134': 'cp936', + '161': 'cp1253', + '162': 'cp1254', + '163': 'cp1258', + '177': 'cp1255', + '178': 'cp1256', + '186': 'cp1257', + '204': 'cp1251', + '222': 'cp874', + '238': 'cp1250'} class VerseType(object): @@ -351,7 +359,7 @@ def retrieve_windows_encoding(recommendation=None): if recommendation == encodings[index][0]: recommended_index = index break - if recommended_index > 0: + if recommended_index > -1: choice = QtGui.QInputDialog.getItem(None, translate('SongsPlugin', 'Character Encoding'), translate('SongsPlugin', 'The codepage setting is responsible\n' @@ -365,7 +373,7 @@ def retrieve_windows_encoding(recommendation=None): [pair[1] for pair in encodings], 0, False) if not choice[1]: return None - return filter(lambda item: item[1] == choice[0], encodings)[0][0] + return next(filter(lambda item: item[1] == choice[0], encodings))[0] def clean_string(string): @@ -521,43 +529,59 @@ def strip_rtf(text, default_encoding=None): curskip = 0 # Output buffer. out = [] + # Encoded buffer. + ebytes = bytearray() for match in PATTERN.finditer(text): - word, arg, hex, char, brace, tchar = match.groups() + iinu, word, arg, hex, char, brace, tchar = match.groups() + # \x (non-alpha character) + if char: + if char in '\\{}': + tchar = char + else: + word = char + # Flush encoded buffer to output buffer + if ebytes and not hex and not tchar: + failed = False + while True: + try: + encoding, default_encoding = get_encoding(font, font_table, default_encoding, failed=failed) + if not encoding: + return None + dbytes = ebytes.decode(encoding) + # Code 5C is a peculiar case with Windows Codepage 932 + if encoding == 'cp932' and '\\' in dbytes: + dbytes = dbytes.replace('\\', '\u00A5') + out.append(dbytes) + ebytes.clear() + except UnicodeDecodeError: + failed = True + else: + break + # {} if brace: curskip = 0 if brace == '{': # Push state stack.append((ucskip, ignorable, font)) - elif brace == '}': + elif brace == '}' and len(stack) > 0: # Pop state ucskip, ignorable, font = stack.pop() - # \x (not a letter) - elif char: - curskip = 0 - if char == '~' and not ignorable: - out.append('\xA0') - elif char in '{}\\' and not ignorable: - out.append(char) - elif char == '-' and not ignorable: - out.append('\u00AD') - elif char == '_' and not ignorable: - out.append('\u2011') - elif char == '*': - ignorable = True # \command elif word: curskip = 0 if word in DESTINATIONS: ignorable = True elif word in SPECIAL_CHARS: - out.append(SPECIAL_CHARS[word]) + if not ignorable: + out.append(SPECIAL_CHARS[word]) elif word == 'uc': ucskip = int(arg) - elif word == ' ': + elif word == 'u': c = int(arg) if c < 0: c += 0x10000 - out.append(chr(c)) + if not ignorable: + out.append(chr(c)) curskip = ucskip elif word == 'fonttbl': ignorable = True @@ -565,31 +589,24 @@ def strip_rtf(text, default_encoding=None): font = arg elif word == 'ansicpg': font_table[font] = 'cp' + arg - elif word == 'fcharset' and font not in font_table and word + arg in CHARSET_MAPPING: - # \ansicpg overrides \fcharset, if present. - font_table[font] = CHARSET_MAPPING[word + arg] + elif word == 'fcharset' and font not in font_table and arg in CHARSET_MAPPING: + font_table[font] = CHARSET_MAPPING[arg] + elif word == 'fldrslt': + pass + # \* 'Ignore if not understood' marker + elif iinu: + ignorable = True # \'xx elif hex: if curskip > 0: curskip -= 1 elif not ignorable: - charcode = int(hex, 16) - failed = False - while True: - try: - encoding, default_encoding = get_encoding(font, font_table, default_encoding, failed=failed) - if not encoding: - return None - out.append(chr(charcode).decode(encoding)) - except UnicodeDecodeError: - failed = True - else: - break + ebytes.append(int(hex, 16)) elif tchar: if curskip > 0: curskip -= 1 elif not ignorable: - out.append(tchar) + ebytes += tchar.encode() text = ''.join(out) return text, default_encoding diff --git a/tests/functional/openlp_plugins/songs/test_lib.py b/tests/functional/openlp_plugins/songs/test_lib.py index ac22ae1ef..a9e64b5c9 100644 --- a/tests/functional/openlp_plugins/songs/test_lib.py +++ b/tests/functional/openlp_plugins/songs/test_lib.py @@ -6,7 +6,7 @@ from unittest import TestCase from mock import patch, MagicMock -from openlp.plugins.songs.lib import VerseType, clean_string, clean_title +from openlp.plugins.songs.lib import VerseType, clean_string, clean_title, strip_rtf from openlp.plugins.songs.lib.songcompare import songs_probably_equal, _remove_typos, _op_length @@ -215,6 +215,38 @@ class TestLib(TestCase): # THEN: The maximum length should be returned. assert result == 10, 'The length should be 10.' + def strip_rtf_charsets_test(self): + """ + Test that the strip_rtf() method properly decodes the supported charsets. + """ + test_charset_table = [ + ('0', 'weor\\\'F0-myndum \\\'FEah\\par ', 'weorð-myndum þah\n'), + ('128', '\\\'83C\\\'83G\\\'83X\\\'A5\\\'83L\\\'83\\\'8A\\\'83X\\\'83g\\\'A1 ' + '\\\\ \\\'95\\\\ \\\'8E\\} \\\'8E\\{ \\\'A1\\par ', 'イエス･キリスト｡ ¥ 表 枝 施 ｡\n'), + ('129', '\\\'BF\\\'B9\\\'BC\\\'F6 \\\'B1\\\'D7\\\'B8\\\'AE\\\'BD\\\'BA\\\'B5\\\'B5\\par ', '예수 그리스도\n'), + ('134', '\\\'D2\\\'AE\\\'F6\\\'D5\\\'BB\\\'F9\\\'B6\\\'BD\\\'CA\\\'C7\\\'D6\\\'F7\\par ', '耶稣基督是主\n'), + ('161', '\\\'D7\\\'F1\\\'E9\\\'F3\\\'F4\\\'FC\\\'F2\\par ', 'Χριστός\n'), + ('162', 'Hazreti \\\'DDsa\\par ', 'Hazreti İsa\n'), + ('163', 'ph\\\'FD\\\'F5ng\\par ', 'phương\n'), + ('177', '\\\'E1\\\'F8\\\'E0\\\'F9\\\'E9\\\'FA\\par ', 'בראשית\n'), + ('178', '\\\'ED\\\'D3\\\'E6\\\'DA \\\'C7\\\'E1\\\'E3\\\'D3\\\'ED\\\'CD\\par ', 'يسوع المسيح\n'), + ('186', 'J\\\'EBzus Kristus yra Vie\\\'F0pats\\par ', 'Jėzus Kristus yra Viešpats\n'), + ('204', '\\\'D0\\\'EE\\\'F1\\\'F1\\\'E8\\\'FF\\par ', 'Россия\n'), + ('222', '\\\'A4\\\'C3\\\'D4\\\'CA\\\'B5\\\'EC\\par ', 'คริสต์\n'), + ('238', 'Z\\\'E1v\\\'ECre\\\'E8n\\\'E1 zkou\\\'9Aka\\par ', 'Závěrečná zkouška\n') + ] + + # GIVEN: For each character set and input + for charset, input, exp_result in test_charset_table: + + # WHEN: We call strip_rtf on the input RTF + result, result_enc = strip_rtf( + '{\\rtf1 \\ansi \\ansicpg1252 {\\fonttbl \\f0 \\fswiss \\fcharset%s Helvetica;}' \ + '{\\colortbl ;\\red0 \\green0 \\blue0 ;}\\pard \\f0 %s}' % (charset, input)) + + # THEN: The stripped text matches thed expected result + assert result == exp_result, 'The result should be %s' % exp_result + class TestVerseType(TestCase): """ From 5406a44d5977f45647bd02f01e736793b62acdef Mon Sep 17 00:00:00 2001 From: Oliver Wieland Date: Sun, 8 Sep 2013 11:29:36 +0200 Subject: [PATCH 06/16] fixes wrong url on the remotes tab --- openlp/plugins/remotes/lib/remotetab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/remotes/lib/remotetab.py b/openlp/plugins/remotes/lib/remotetab.py index 338abc4a0..b9c71f338 100644 --- a/openlp/plugins/remotes/lib/remotetab.py +++ b/openlp/plugins/remotes/lib/remotetab.py @@ -209,7 +209,7 @@ class RemoteTab(SettingsTab): for address in interface.addressEntries(): ip = address.ip() if ip.protocol() == 0 and ip != QtNetwork.QHostAddress.LocalHost: - ip_address = ip + ip_address = ip.toString() break else: ip_address = self.address_edit.text() From afb8f088a6484b51c5e120f927ea28ff50b4ad70 Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Sun, 8 Sep 2013 08:17:14 -0500 Subject: [PATCH 07/16] Remove extra whitespace --- scripts/translation_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/translation_utils.py b/scripts/translation_utils.py index 32f54a950..0069d411a 100755 --- a/scripts/translation_utils.py +++ b/scripts/translation_utils.py @@ -145,9 +145,9 @@ def print_quiet(text, linefeed=True): global quiet_mode if not quiet_mode: if linefeed: - print (text) + print(text) else: - print (text, end=' ') + print(text, end=' ') def print_verbose(text): """ From 2aeaa24871416c0df9772ba8f245c6add38ff977 Mon Sep 17 00:00:00 2001 From: Oliver Wieland Date: Sun, 8 Sep 2013 18:16:18 +0200 Subject: [PATCH 08/16] remotetab.py: separate get ip address for metter testing test_remotetab.py:tests for get_ip_address --- openlp/plugins/remotes/lib/remotetab.py | 30 ++++++++--------- .../openlp_plugins/remotes/test_remotetab.py | 33 ++++++++++++++++++- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/openlp/plugins/remotes/lib/remotetab.py b/openlp/plugins/remotes/lib/remotetab.py index b9c71f338..b84faf7bd 100644 --- a/openlp/plugins/remotes/lib/remotetab.py +++ b/openlp/plugins/remotes/lib/remotetab.py @@ -198,21 +198,7 @@ class RemoteTab(SettingsTab): """ Update the display based on the data input on the screen """ - ip_address = 'localhost' - if self.address_edit.text() == ZERO_URL: - interfaces = QtNetwork.QNetworkInterface.allInterfaces() - for interface in interfaces: - if not interface.isValid(): - continue - if not (interface.flags() & (QtNetwork.QNetworkInterface.IsUp | QtNetwork.QNetworkInterface.IsRunning)): - continue - for address in interface.addressEntries(): - ip = address.ip() - if ip.protocol() == 0 and ip != QtNetwork.QHostAddress.LocalHost: - ip_address = ip.toString() - break - else: - ip_address = self.address_edit.text() + ip_address = self.get_ip_address(self.address_edit.text()) http_url = 'http://%s:%s/' % (ip_address, self.port_spin_box.value()) https_url = 'https://%s:%s/' % (ip_address, self.https_port_spin_box.value()) self.remote_url.setText('%s' % (http_url, http_url)) @@ -226,6 +212,20 @@ class RemoteTab(SettingsTab): self.live_url.setText('%s' % (http_url_temp, http_url_temp)) self.live_https_url.setText('%s' % (https_url_temp, https_url_temp)) + def get_ip_address(self, ip): + if ip == ZERO_URL: + interfaces = QtNetwork.QNetworkInterface.allInterfaces() + for interface in interfaces: + if not interface.isValid(): + continue + if not (interface.flags() & (QtNetwork.QNetworkInterface.IsUp | QtNetwork.QNetworkInterface.IsRunning)): + continue + for address in interface.addressEntries(): + ip = address.ip() + if ip.protocol() == 0 and ip != QtNetwork.QHostAddress.LocalHost: + return ip.toString() + return ip + def load(self): """ Load the configuration and update the server configuration if necessary diff --git a/tests/functional/openlp_plugins/remotes/test_remotetab.py b/tests/functional/openlp_plugins/remotes/test_remotetab.py index 9ff795e73..81b67dbea 100644 --- a/tests/functional/openlp_plugins/remotes/test_remotetab.py +++ b/tests/functional/openlp_plugins/remotes/test_remotetab.py @@ -2,7 +2,7 @@ This module contains tests for the lib submodule of the Remotes plugin. """ import os - +import re from unittest import TestCase from tempfile import mkstemp from mock import patch @@ -52,6 +52,37 @@ class TestRemoteTab(TestCase): del self.form os.unlink(self.ini_file) + def get_ip_address_default_test(self): + """ + Test the get_ip_address function with ZERO_URL + """ + # GIVEN: A mocked location + with patch('openlp.core.utils.applocation.Settings') as mocked_class, \ + patch('openlp.core.utils.AppLocation.get_directory') as mocked_get_directory, \ + patch('openlp.core.utils.applocation.check_directory_exists') as mocked_check_directory_exists, \ + patch('openlp.core.utils.applocation.os') as mocked_os: + # GIVEN: A mocked out Settings class and a mocked out AppLocation.get_directory() + # WHEN: the default ip address is given + ip_address = self.form.get_ip_address(ZERO_URL) + # THEN: the default ip address will be returned + self.assertTrue(re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip_address), 'The return value should be a valid ip address') + + def get_ip_address_with_ip_test(self): + """ + Test the get_ip_address function with given ip address + """ + # GIVEN: A mocked location + with patch('openlp.core.utils.applocation.Settings') as mocked_class, \ + patch('openlp.core.utils.AppLocation.get_directory') as mocked_get_directory, \ + patch('openlp.core.utils.applocation.check_directory_exists') as mocked_check_directory_exists, \ + patch('openlp.core.utils.applocation.os') as mocked_os: + # GIVEN: An ip address + given_ip = '192.168.1.1' + # WHEN: the default ip address is given + ip_address = self.form.get_ip_address(given_ip) + # THEN: the default ip address will be returned + self.assertEqual(ip_address, given_ip, 'The return value should be %s' % given_ip) + def set_basic_urls_test(self): """ Test the set_urls function with standard defaults From 7d4701030492143e6bc94629ddbd30ff226cf39b Mon Sep 17 00:00:00 2001 From: Oliver Wieland Date: Sun, 8 Sep 2013 18:24:21 +0200 Subject: [PATCH 09/16] cleaning tests --- .../openlp_plugins/remotes/test_remotetab.py | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/tests/functional/openlp_plugins/remotes/test_remotetab.py b/tests/functional/openlp_plugins/remotes/test_remotetab.py index 81b67dbea..1ee9a3695 100644 --- a/tests/functional/openlp_plugins/remotes/test_remotetab.py +++ b/tests/functional/openlp_plugins/remotes/test_remotetab.py @@ -56,32 +56,22 @@ class TestRemoteTab(TestCase): """ Test the get_ip_address function with ZERO_URL """ - # GIVEN: A mocked location - with patch('openlp.core.utils.applocation.Settings') as mocked_class, \ - patch('openlp.core.utils.AppLocation.get_directory') as mocked_get_directory, \ - patch('openlp.core.utils.applocation.check_directory_exists') as mocked_check_directory_exists, \ - patch('openlp.core.utils.applocation.os') as mocked_os: - # GIVEN: A mocked out Settings class and a mocked out AppLocation.get_directory() - # WHEN: the default ip address is given - ip_address = self.form.get_ip_address(ZERO_URL) - # THEN: the default ip address will be returned - self.assertTrue(re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip_address), 'The return value should be a valid ip address') + # WHEN: the default ip address is given + ip_address = self.form.get_ip_address(ZERO_URL) + # THEN: the default ip address will be returned + self.assertTrue(re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ip_address), 'The return value should be a valid ip address') def get_ip_address_with_ip_test(self): """ Test the get_ip_address function with given ip address """ # GIVEN: A mocked location - with patch('openlp.core.utils.applocation.Settings') as mocked_class, \ - patch('openlp.core.utils.AppLocation.get_directory') as mocked_get_directory, \ - patch('openlp.core.utils.applocation.check_directory_exists') as mocked_check_directory_exists, \ - patch('openlp.core.utils.applocation.os') as mocked_os: - # GIVEN: An ip address - given_ip = '192.168.1.1' - # WHEN: the default ip address is given - ip_address = self.form.get_ip_address(given_ip) - # THEN: the default ip address will be returned - self.assertEqual(ip_address, given_ip, 'The return value should be %s' % given_ip) + # GIVEN: An ip address + given_ip = '192.168.1.1' + # WHEN: the default ip address is given + ip_address = self.form.get_ip_address(given_ip) + # THEN: the default ip address will be returned + self.assertEqual(ip_address, given_ip, 'The return value should be %s' % given_ip) def set_basic_urls_test(self): """ From c0d19ec840fed8d666f76be1101b4a1321855063 Mon Sep 17 00:00:00 2001 From: Oliver Wieland Date: Sun, 8 Sep 2013 18:40:48 +0200 Subject: [PATCH 10/16] rename variable --- openlp/plugins/remotes/lib/remotetab.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/remotes/lib/remotetab.py b/openlp/plugins/remotes/lib/remotetab.py index b84faf7bd..e30ad3ea8 100644 --- a/openlp/plugins/remotes/lib/remotetab.py +++ b/openlp/plugins/remotes/lib/remotetab.py @@ -212,8 +212,8 @@ class RemoteTab(SettingsTab): self.live_url.setText('%s' % (http_url_temp, http_url_temp)) self.live_https_url.setText('%s' % (https_url_temp, https_url_temp)) - def get_ip_address(self, ip): - if ip == ZERO_URL: + def get_ip_address(self, ip_address): + if ip_address == ZERO_URL: interfaces = QtNetwork.QNetworkInterface.allInterfaces() for interface in interfaces: if not interface.isValid(): @@ -224,7 +224,7 @@ class RemoteTab(SettingsTab): ip = address.ip() if ip.protocol() == 0 and ip != QtNetwork.QHostAddress.LocalHost: return ip.toString() - return ip + return ip_address def load(self): """ From 86c37481ab4a750207e2de58950e4489532f6158 Mon Sep 17 00:00:00 2001 From: Oliver Wieland Date: Sun, 8 Sep 2013 21:24:53 +0200 Subject: [PATCH 11/16] doc string added --- openlp/plugins/remotes/lib/remotetab.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openlp/plugins/remotes/lib/remotetab.py b/openlp/plugins/remotes/lib/remotetab.py index e30ad3ea8..b1cf4a803 100644 --- a/openlp/plugins/remotes/lib/remotetab.py +++ b/openlp/plugins/remotes/lib/remotetab.py @@ -213,6 +213,11 @@ class RemoteTab(SettingsTab): self.live_https_url.setText('%s' % (https_url_temp, https_url_temp)) def get_ip_address(self, ip_address): + """ + returns the IP address in dependency of the passed address + ip_address == 0.0.0.0: return the IP address of the first valid interface + else: return ip_address + """ if ip_address == ZERO_URL: interfaces = QtNetwork.QNetworkInterface.allInterfaces() for interface in interfaces: From e5849077188110b2c6d1112ad3277aa2a94658d2 Mon Sep 17 00:00:00 2001 From: Oliver Wieland Date: Mon, 9 Sep 2013 18:32:13 +0200 Subject: [PATCH 12/16] replaced constant value with enum --- openlp/plugins/remotes/lib/remotetab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/remotes/lib/remotetab.py b/openlp/plugins/remotes/lib/remotetab.py index b1cf4a803..c06d71ee9 100644 --- a/openlp/plugins/remotes/lib/remotetab.py +++ b/openlp/plugins/remotes/lib/remotetab.py @@ -227,7 +227,7 @@ class RemoteTab(SettingsTab): continue for address in interface.addressEntries(): ip = address.ip() - if ip.protocol() == 0 and ip != QtNetwork.QHostAddress.LocalHost: + if ip.protocol() == QtNetwork.QAbstractSocket.IPv4Protocol and ip != QtNetwork.QHostAddress.LocalHost: return ip.toString() return ip_address From 334cef478da3d57fef89323919a7ec932ba20c3c Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Mon, 9 Sep 2013 16:10:40 -0500 Subject: [PATCH 13/16] Fix various tests and broken code revealed by tests --- openlp/core/ui/starttimeform.py | 4 ++-- openlp/core/utils/__init__.py | 2 +- openlp/plugins/bibles/lib/http.py | 2 +- tests/functional/openlp_core_lib/test_lib.py | 4 ++-- tests/functional/openlp_core_utils/test_utils.py | 4 ++++ tests/interfaces/openlp_plugins/bibles/test_lib_http.py | 4 ++-- tests/interfaces/openlp_plugins/remotes/test_server.py | 2 +- 7 files changed, 13 insertions(+), 9 deletions(-) diff --git a/openlp/core/ui/starttimeform.py b/openlp/core/ui/starttimeform.py index 4fddb2a90..0a0867b3f 100644 --- a/openlp/core/ui/starttimeform.py +++ b/openlp/core/ui/starttimeform.py @@ -88,9 +88,9 @@ class StartTimeForm(QtGui.QDialog, Ui_StartTimeDialog): """ Split time up into hours minutes and seconds from secongs """ - hours = seconds / 3600 + hours = seconds // 3600 seconds -= 3600 * hours - minutes = seconds / 60 + minutes = seconds // 60 seconds -= 60 * minutes return hours, minutes, seconds diff --git a/openlp/core/utils/__init__.py b/openlp/core/utils/__init__.py index 3bd052a0e..d2e664e75 100644 --- a/openlp/core/utils/__init__.py +++ b/openlp/core/utils/__init__.py @@ -256,7 +256,7 @@ def is_not_image_file(file_name): if not file_name: return True else: - formats = [str(fmt).lower() for fmt in QtGui.QImageReader.supportedImageFormats()] + formats = [bytes(fmt).decode().lower() for fmt in QtGui.QImageReader.supportedImageFormats()] file_part, file_extension = os.path.splitext(str(file_name)) if file_extension[1:].lower() in formats and os.path.exists(file_name): return False diff --git a/openlp/plugins/bibles/lib/http.py b/openlp/plugins/bibles/lib/http.py index 5d9e51c15..0fee09265 100644 --- a/openlp/plugins/bibles/lib/http.py +++ b/openlp/plugins/bibles/lib/http.py @@ -717,7 +717,7 @@ def get_soup_for_bible_ref(reference_url, header=None, pre_parse_regex=None, pre return None page_source = page.read() if pre_parse_regex and pre_parse_substitute is not None: - page_source = re.sub(pre_parse_regex, pre_parse_substitute, page_source) + page_source = re.sub(pre_parse_regex, pre_parse_substitute, page_source.decode()) soup = None try: soup = BeautifulSoup(page_source) diff --git a/tests/functional/openlp_core_lib/test_lib.py b/tests/functional/openlp_core_lib/test_lib.py index a7d7d5d88..fa7fe92e9 100644 --- a/tests/functional/openlp_core_lib/test_lib.py +++ b/tests/functional/openlp_core_lib/test_lib.py @@ -187,7 +187,7 @@ class TestLib(TestCase): """ Test the get_text_file_string() method when a read error happens """ - with patch('openlp.core.lib.os.path.isfile') as mocked_isfile, patch('builtins.open') as mocked_open: + with patch('openlp.core.lib.os.path.isfile') as mocked_isfile, patch('openlp.core.lib.open', create=True) as mocked_open: # GIVEN: A mocked-out open() which raises an exception and isfile returns True filename = 'testfile.txt' mocked_isfile.return_value = True @@ -252,7 +252,7 @@ class TestLib(TestCase): # GIVEN: A set of mocked-out Qt classes mocked_byte_array = MagicMock() MockedQtCore.QByteArray.return_value = mocked_byte_array - mocked_byte_array.toBase64.return_value = 'base64mock' + mocked_byte_array.toBase64.return_value = QtCore.QByteArray('base64mock') mocked_buffer = MagicMock() MockedQtCore.QBuffer.return_value = mocked_buffer MockedQtCore.QIODevice.WriteOnly = 'writeonly' diff --git a/tests/functional/openlp_core_utils/test_utils.py b/tests/functional/openlp_core_utils/test_utils.py index 6eacb2e48..2aa1f9611 100644 --- a/tests/functional/openlp_core_utils/test_utils.py +++ b/tests/functional/openlp_core_utils/test_utils.py @@ -8,6 +8,8 @@ from mock import patch from openlp.core.utils import clean_filename, get_filesystem_encoding, _get_frozen_path, get_locale_key, \ get_natural_key, split_filename +import os + class TestUtils(TestCase): """ @@ -120,6 +122,8 @@ class TestUtils(TestCase): # THEN: We get a properly sorted list test_passes = sorted(unsorted_list, key=get_locale_key) == ['Aushang', '\u00C4u\u00DFerung', 'Auszug'] assert test_passes, 'Strings should be sorted properly' + if os.name != 'nt': + del get_locale_key_windows_test def get_locale_key_linux_test(self): diff --git a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py index f760996b5..bd645f1ff 100644 --- a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py +++ b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py @@ -43,7 +43,7 @@ class TestBibleHTTP(TestCase): results = handler.get_bible_chapter('NIV', 'John', 3) # THEN: We should get back a valid service item - assert len(results.verselist) == 36, 'The book of John should not have had any verses added or removed' + assert len(results.verse_list) == 36, 'The book of John should not have had any verses added or removed' def crosswalk_extract_books_test(self): """ @@ -69,5 +69,5 @@ class TestBibleHTTP(TestCase): results = handler.get_bible_chapter('niv', 'john', 3) # THEN: We should get back a valid service item - assert len(results.verselist) == 36, 'The book of John should not have had any verses added or removed' + assert len(results.verse_list) == 36, 'The book of John should not have had any verses added or removed' diff --git a/tests/interfaces/openlp_plugins/remotes/test_server.py b/tests/interfaces/openlp_plugins/remotes/test_server.py index 63975370f..6cb44a933 100644 --- a/tests/interfaces/openlp_plugins/remotes/test_server.py +++ b/tests/interfaces/openlp_plugins/remotes/test_server.py @@ -9,7 +9,7 @@ from mock import MagicMock import urllib.request, urllib.error, urllib.parse import cherrypy -from BeautifulSoup import BeautifulSoup +from bs4 import BeautifulSoup from openlp.core.lib import Settings from openlp.plugins.remotes.lib.httpserver import HttpServer From 1106bca319efca63027fcf07afd0d009535cec98 Mon Sep 17 00:00:00 2001 From: Oliver Wieland Date: Tue, 10 Sep 2013 20:18:53 +0200 Subject: [PATCH 14/16] added blank line --- tests/functional/openlp_plugins/remotes/test_remotetab.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/openlp_plugins/remotes/test_remotetab.py b/tests/functional/openlp_plugins/remotes/test_remotetab.py index 1ee9a3695..e683699cd 100644 --- a/tests/functional/openlp_plugins/remotes/test_remotetab.py +++ b/tests/functional/openlp_plugins/remotes/test_remotetab.py @@ -3,6 +3,7 @@ This module contains tests for the lib submodule of the Remotes plugin. """ import os import re + from unittest import TestCase from tempfile import mkstemp from mock import patch From 30618ad60c44b2bf0c36d2c79622c91c7437900e Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Tue, 10 Sep 2013 15:36:46 -0500 Subject: [PATCH 15/16] Explain a complex regex used by strip_rtf --- openlp/plugins/songs/lib/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index ccbe35fd9..12874aa89 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -46,6 +46,12 @@ log = logging.getLogger(__name__) WHITESPACE = re.compile(r'[\W_]+', re.UNICODE) APOSTROPHE = re.compile('[\'`’ʻ′]', re.UNICODE) +# PATTERN will look for the next occurence of one of these symbols: +# \controlword - optionally preceded by \*, optionally followed by a number +# \'## - where ## is a pair of hex digits, representing a single character +# \# - where # is a single non-alpha character, representing a special symbol +# { or } - marking the beginning/end of a group +# a run of characters without any \ { } or end-of-line PATTERN = re.compile(r"(\\\*)?\\([a-z]{1,32})(-?\d{1,10})?[ ]?|\\'([0-9a-f]{2})|\\([^a-z*])|([{}])|[\r\n]+|([^\\{}\r\n]+)", re.I) # RTF control words which specify a "destination" to be ignored. DESTINATIONS = frozenset(( From cb62bf593621fdc8d11360a6488f584b8fbc3131 Mon Sep 17 00:00:00 2001 From: "Jeffrey S. Smith" Date: Wed, 11 Sep 2013 10:09:36 -0500 Subject: [PATCH 16/16] Need a different approach for platform-specific tests --- tests/functional/openlp_core_utils/test_utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/functional/openlp_core_utils/test_utils.py b/tests/functional/openlp_core_utils/test_utils.py index 2aa1f9611..6eacb2e48 100644 --- a/tests/functional/openlp_core_utils/test_utils.py +++ b/tests/functional/openlp_core_utils/test_utils.py @@ -8,8 +8,6 @@ from mock import patch from openlp.core.utils import clean_filename, get_filesystem_encoding, _get_frozen_path, get_locale_key, \ get_natural_key, split_filename -import os - class TestUtils(TestCase): """ @@ -122,8 +120,6 @@ class TestUtils(TestCase): # THEN: We get a properly sorted list test_passes = sorted(unsorted_list, key=get_locale_key) == ['Aushang', '\u00C4u\u00DFerung', 'Auszug'] assert test_passes, 'Strings should be sorted properly' - if os.name != 'nt': - del get_locale_key_windows_test def get_locale_key_linux_test(self):