From 90f927ae7ff93baf6bb5b57968467dfbea122416 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 24 May 2016 13:40:01 +0200 Subject: [PATCH 1/6] Fix traceback while handling traceback in videopsalm import --- openlp/plugins/songs/lib/importers/videopsalm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/songs/lib/importers/videopsalm.py b/openlp/plugins/songs/lib/importers/videopsalm.py index 0bc581239..6723fb4c1 100644 --- a/openlp/plugins/songs/lib/importers/videopsalm.py +++ b/openlp/plugins/songs/lib/importers/videopsalm.py @@ -117,6 +117,6 @@ class VideoPsalmImport(SongImport): if not self.finish(): self.log_error('Could not import %s' % self.title) except Exception as e: - self.log_error(translate('SongsPlugin.VideoPsalmImport', 'File %s' % file.name), + self.log_error(translate('SongsPlugin.VideoPsalmImport', 'File %s') % song_file.name, translate('SongsPlugin.VideoPsalmImport', 'Error: %s') % e) song_file.close() From 72120d8b2fd29167cc156598df6277ae26769ca9 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 24 May 2016 13:49:08 +0200 Subject: [PATCH 2/6] Skip PresentationManager files we do not support. --- openlp/plugins/songs/lib/importers/presentationmanager.py | 8 +++++++- openlp/plugins/songs/lib/importers/videopsalm.py | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/presentationmanager.py b/openlp/plugins/songs/lib/importers/presentationmanager.py index c26f11312..5d8db4d26 100644 --- a/openlp/plugins/songs/lib/importers/presentationmanager.py +++ b/openlp/plugins/songs/lib/importers/presentationmanager.py @@ -55,7 +55,13 @@ class PresentationManagerImport(SongImport): # Open file with detected encoding and remove encoding declaration text = open(file_path, mode='r', encoding=encoding).read() text = re.sub('.+\?>\n', '', text) - tree = etree.fromstring(text, parser=etree.XMLParser(recover=True)) + try: + tree = etree.fromstring(text, parser=etree.XMLParser(recover=True)) + except ValueError: + self.log_error(file_path, + translate('SongsPlugin.PresentationManagerImport', + 'File is not in XML-format, which is the only format supported.')) + continue root = objectify.fromstring(etree.tostring(tree)) self.process_song(root) diff --git a/openlp/plugins/songs/lib/importers/videopsalm.py b/openlp/plugins/songs/lib/importers/videopsalm.py index 6723fb4c1..edf5e89a8 100644 --- a/openlp/plugins/songs/lib/importers/videopsalm.py +++ b/openlp/plugins/songs/lib/importers/videopsalm.py @@ -117,6 +117,5 @@ class VideoPsalmImport(SongImport): if not self.finish(): self.log_error('Could not import %s' % self.title) except Exception as e: - self.log_error(translate('SongsPlugin.VideoPsalmImport', 'File %s') % song_file.name, - translate('SongsPlugin.VideoPsalmImport', 'Error: %s') % e) + self.log_error(song_file.name, translate('SongsPlugin.VideoPsalmImport', 'Error: %s') % e) song_file.close() From fd4cfd1eaa4819bd67498158e366e7854c792981 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Wed, 25 May 2016 09:04:41 +0200 Subject: [PATCH 3/6] Fix traceback during songshowplus import. Fixes bug 1585489. Fixes: https://launchpad.net/bugs/1585489 --- .../songs/lib/importers/songshowplus.py | 14 +++++-- .../songs/test_songshowplusimport.py | 2 + .../songshowplussongs/cleanse-me.json | 38 ++++++++++++++++++ .../songshowplussongs/cleanse-me.sbsong | Bin 0 -> 1093 bytes 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 tests/resources/songshowplussongs/cleanse-me.json create mode 100644 tests/resources/songshowplussongs/cleanse-me.sbsong diff --git a/openlp/plugins/songs/lib/importers/songshowplus.py b/openlp/plugins/songs/lib/importers/songshowplus.py index d9a205e22..23aa5173b 100644 --- a/openlp/plugins/songs/lib/importers/songshowplus.py +++ b/openlp/plugins/songs/lib/importers/songshowplus.py @@ -105,6 +105,7 @@ class SongShowPlusImport(SongImport): song_data = open(file, 'rb') while True: block_key, = struct.unpack("I", song_data.read(4)) + log.debug('block_key: %d' % block_key) # The file ends with 4 NULL's if block_key == 0: break @@ -116,7 +117,13 @@ class SongShowPlusImport(SongImport): null, verse_name_length, = struct.unpack("BB", song_data.read(2)) verse_name = self.decode(song_data.read(verse_name_length)) length_descriptor_size, = struct.unpack("B", song_data.read(1)) - log.debug(length_descriptor_size) + log.debug('length_descriptor_size: %d' % length_descriptor_size) + # In the case of song_numbers the number is in the data from the + # current position to the next block starts + if block_key == SONG_NUMBER: + sn_bytes = song_data.read(length_descriptor_size - 1) + self.song_number = int.from_bytes(sn_bytes, byteorder='little') + continue # Detect if/how long the length descriptor is if length_descriptor_size == 12 or length_descriptor_size == 20: length_descriptor, = struct.unpack("I", song_data.read(4)) @@ -126,8 +133,9 @@ class SongShowPlusImport(SongImport): length_descriptor = 0 else: length_descriptor, = struct.unpack("B", song_data.read(1)) - log.debug(length_descriptor_size) + log.debug('length_descriptor: %d' % length_descriptor) data = song_data.read(length_descriptor) + log.debug(data) if block_key == TITLE: self.title = self.decode(data) elif block_key == AUTHOR: @@ -164,8 +172,6 @@ class SongShowPlusImport(SongImport): self.ssp_verse_order_list.append(verse_tag) elif block_key == SONG_BOOK: self.song_book_name = self.decode(data) - elif block_key == SONG_NUMBER: - self.song_number = ord(data) elif block_key == CUSTOM_VERSE: verse_tag = self.to_openlp_verse_tag(verse_name) self.add_verse(self.decode(data), verse_tag) diff --git a/tests/functional/openlp_plugins/songs/test_songshowplusimport.py b/tests/functional/openlp_plugins/songs/test_songshowplusimport.py index ec86eca07..a96f21a47 100644 --- a/tests/functional/openlp_plugins/songs/test_songshowplusimport.py +++ b/tests/functional/openlp_plugins/songs/test_songshowplusimport.py @@ -52,6 +52,8 @@ class TestSongShowPlusFileImport(SongImportTestHelper): self.load_external_result_data(os.path.join(TEST_PATH, 'Beautiful Garden Of Prayer.json'))) self.file_import([os.path.join(TEST_PATH, 'a mighty fortress is our god.sbsong')], self.load_external_result_data(os.path.join(TEST_PATH, 'a mighty fortress is our god.json'))) + self.file_import([os.path.join(TEST_PATH, 'cleanse-me.sbsong')], + self.load_external_result_data(os.path.join(TEST_PATH, 'cleanse-me.json'))) class TestSongShowPlusImport(TestCase): diff --git a/tests/resources/songshowplussongs/cleanse-me.json b/tests/resources/songshowplussongs/cleanse-me.json new file mode 100644 index 000000000..c88b434f9 --- /dev/null +++ b/tests/resources/songshowplussongs/cleanse-me.json @@ -0,0 +1,38 @@ +{ + "authors": [ + "J. Edwin Orr" + ], + "ccli_number": 56307, + "comments": "", + "copyright": "Public Domain ", + "song_book_name": "", + "song_number": 438, + "title": "Cleanse Me [438]", + "topics": [ + "Cleansing", + "Communion", + "Consecration", + "Holiness", + "Holy Spirit", + "Revival" + ], + "verse_order_list": [], + "verses": [ + [ + "Search me, O God,\r\nAnd know my heart today;\r\nTry me, O Savior,\r\nKnow my thoughts, I pray.\r\nSee if there be\r\nSome wicked way in me;\r\nCleanse me from every sin\r\nAnd set me free.", + "v1" + ], + [ + "I praise Thee, Lord,\r\nFor cleansing me from sin;\r\nFulfill Thy Word,\r\nAnd make me pure within.\r\nFill me with fire\r\nWhere once I burned with shame;\r\nGrant my desire\r\nTo magnify Thy name.", + "v2" + ], + [ + "Lord, take my life,\r\nAnd make it wholly Thine;\r\nFill my poor heart\r\nWith Thy great love divine.\r\nTake all my will,\r\nMy passion, self and pride;\r\nI now surrender, Lord\r\nIn me abide.", + "v3" + ], + [ + "O Holy Ghost,\r\nRevival comes from Thee;\r\nSend a revival,\r\nStart the work in me.\r\nThy Word declares\r\nThou wilt supply our need;\r\nFor blessings now,\r\nO Lord, I humbly plead.", + "v4" + ] + ] +} diff --git a/tests/resources/songshowplussongs/cleanse-me.sbsong b/tests/resources/songshowplussongs/cleanse-me.sbsong new file mode 100644 index 0000000000000000000000000000000000000000..aa9915f8ecb4b40487d288e4e86c8926104cc294 GIT binary patch literal 1093 zcmYjRO>Yx15KUW}wkd&%JJKBZIz&}V3$4Tnl_nIZf=EOUh=a2`$y)K+E8ClBe-j5T zs6t#g@W*(w8&D3b*w1ff-kY)Wq}6J*<-2oqc6`2p)dSfbTo_h1FkLf!IXyZ5x(W2Y zoOFlY_vqarU8YNIw*VaoeD7m9F*>0)E?3&pBVcm2b-S^RpBag1xF_WHB%-Azc7=X)}mO7bp zN=sD{yyc9v|N4W|sdqW?f>9`F+h_?K!NU>rp*Z-n=HPkzXI)Rj%{XJY_~5*l=sQnI z-FIzgO*k?mC+hV}Gu6f*prV_GE}nBWXJHm4^e%PGw1tblFl*g0qp9}raZ@{THer~Z zl-`OT@F`@fHZ<_cLUTnahdN^HkbP$Lw5p3*&}u8c*Q}hhf7IG3);cOOddjPD)Y5dM zW#){L9NJ3b8f_I74sPpFL7WH?XEV<#l5q>BR4)(!Gh<1u#83sr#vuJQ!c_>`*&YQp zQ&MO};dLqnu1LlkO7GdGjJqld0n6Y>O+cz`+^*R;ZGRimTL+bc%!P;wpLn4c%20yw zhi1YuDx@DFD=G2~0n|~fuUm%xJ3ntOh{#?I3jIus@*D(mrC5kiR}`q`N>7$KmA0T8 z6T>iNXF(hw^RT%X7+6;36YvXMj`Z*$lsAv0xrB&VgIu1M7M&63o_M@_;qZ_Xui^^r p2)YZq=x+$Z>k6`8H(*p~ucLr_0`9CD@e~_*$nxjVdbYk1;4gFPC9D7d literal 0 HcmV?d00001 From 3163d335439691499cd2dd4d1c0a0c1800333a35 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Thu, 26 May 2016 15:03:00 +0200 Subject: [PATCH 4/6] Fix of tracback during SongPro import. Fixes bug 1582152. Fixes: https://launchpad.net/bugs/1582152 --- openlp/plugins/songs/lib/importers/songpro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/songs/lib/importers/songpro.py b/openlp/plugins/songs/lib/importers/songpro.py index b1c8e7fe8..90fe34492 100644 --- a/openlp/plugins/songs/lib/importers/songpro.py +++ b/openlp/plugins/songs/lib/importers/songpro.py @@ -72,7 +72,7 @@ class SongProImport(SongImport): Receive a single file or a list of files to import. """ self.encoding = None - with open(self.import_source, 'rt') as songs_file: + with open(self.import_source, 'rt', errors='ignore') as songs_file: self.import_wizard.progress_bar.setMaximum(0) tag = '' text = '' From 41c0d3fcf9c01430ab36d8c42f01021f5d8bdf88 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 14 Jun 2016 21:11:57 +0200 Subject: [PATCH 5/6] Fix various pyodbc related issues. Fixes bug 1590657. Fixes: https://launchpad.net/bugs/1590657 --- .../plugins/songs/lib/importers/mediashout.py | 28 +++++++++++-------- openlp/plugins/songs/lib/importers/opspro.py | 6 ++-- .../songs/lib/importers/worshipcenterpro.py | 2 +- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/mediashout.py b/openlp/plugins/songs/lib/importers/mediashout.py index cb3a19640..af9b855a0 100644 --- a/openlp/plugins/songs/lib/importers/mediashout.py +++ b/openlp/plugins/songs/lib/importers/mediashout.py @@ -24,15 +24,17 @@ The :mod:`mediashout` module provides the functionality for importing a MediaShout database into the OpenLP database. """ -# WARNING: See https://docs.python.org/2/library/sqlite3.html for value substitution +# WARNING: See https://docs.python.org/3/library/sqlite3.html for value substitution # in SQL statements import pyodbc +import logging from openlp.core.lib import translate from openlp.plugins.songs.lib.importers.songimport import SongImport VERSE_TAGS = ['V', 'C', 'B', 'O', 'P', 'I', 'E'] +log = logging.getLogger(__name__) class MediaShoutImport(SongImport): @@ -44,17 +46,19 @@ class MediaShoutImport(SongImport): """ Initialise the MediaShout importer. """ - SongImport.__init__(self, manager, **kwargs) + super(MediaShoutImport, self).__init__(manager, **kwargs) + #SongImport.__init__(self, manager, **kwargs) def do_import(self): """ Receive a single file to import. """ try: - conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ={source};' - 'PWD=6NOZ4eHK7k'.format(sorce=self.import_source)) - except: + conn = pyodbc.connect('DRIVER={{Microsoft Access Driver (*.mdb)}};DBQ={source};' + 'PWD=6NOZ4eHK7k'.format(source=self.import_source)) + except Exception as e: # Unfortunately no specific exception type + log.exception(e) self.log_error(self.import_source, translate('SongsPlugin.MediaShoutImport', 'Unable to open the MediaShout database.')) return @@ -63,17 +67,19 @@ class MediaShoutImport(SongImport): songs = cursor.fetchall() self.import_wizard.progress_bar.setMaximum(len(songs)) for song in songs: + topics = [] if self.stop_import_flag: break - cursor.execute('SELECT Type, Number, Text FROM Verses WHERE Record = ? ORDER BY Type, Number', song.Record) + cursor.execute('SELECT Type, Number, Text FROM Verses WHERE Record = ? ORDER BY Type, Number', float(song.Record)) verses = cursor.fetchall() - cursor.execute('SELECT Type, Number, POrder FROM PlayOrder WHERE Record = ? ORDER BY POrder', song.Record) + cursor.execute('SELECT Type, Number, POrder FROM PlayOrder WHERE Record = ? ORDER BY POrder', float(song.Record)) verse_order = cursor.fetchall() - cursor.execute('SELECT Name FROM Themes INNER JOIN SongThemes ON SongThemes.ThemeId = Themes.ThemeId ' - 'WHERE SongThemes.Record = ?', song.Record) - topics = cursor.fetchall() + if cursor.tables(table='TableName', tableType='TABLE').fetchone(): + cursor.execute('SELECT Name FROM Themes INNER JOIN SongThemes ON SongThemes.ThemeId = Themes.ThemeId ' + 'WHERE SongThemes.Record = ?', float(song.Record)) + topics = cursor.fetchall() cursor.execute('SELECT Name FROM Groups INNER JOIN SongGroups ON SongGroups.GroupId = Groups.GroupId ' - 'WHERE SongGroups.Record = ?', song.Record) + 'WHERE SongGroups.Record = ?', float(song.Record)) topics += cursor.fetchall() self.process_song(song, verses, verse_order, topics) diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 8f9674deb..03c5001c6 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -55,7 +55,7 @@ class OPSProImport(SongImport): """ password = self.extract_mdb_password() try: - conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ={source};' + conn = pyodbc.connect('DRIVER={{Microsoft Access Driver (*.mdb)}};DBQ={source};' 'PWD={password}'.format(source=self.import_source, password=password)) except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e: log.warning('Unable to connect the OPS Pro database {source}. {error}'.format(source=self.import_source, @@ -74,11 +74,11 @@ class OPSProImport(SongImport): break # Type means: 0=Original, 1=Projection, 2=Own cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = ? AND Type < 2 ' - 'ORDER BY Type DESC', song.ID) + 'ORDER BY Type DESC', float(song.ID)) lyrics = cursor.fetchone() cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ' 'ON Category.ID = SongCategory.CategoryID WHERE SongCategory.SongID = ? ' - 'ORDER BY CategoryName', song.ID) + 'ORDER BY CategoryName', float(song.ID)) topics = cursor.fetchall() try: self.process_song(song, lyrics, topics) diff --git a/openlp/plugins/songs/lib/importers/worshipcenterpro.py b/openlp/plugins/songs/lib/importers/worshipcenterpro.py index df04823e8..3d5cbe9ba 100644 --- a/openlp/plugins/songs/lib/importers/worshipcenterpro.py +++ b/openlp/plugins/songs/lib/importers/worshipcenterpro.py @@ -49,7 +49,7 @@ class WorshipCenterProImport(SongImport): Receive a single file to import. """ try: - conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};' + conn = pyodbc.connect('DRIVER={{Microsoft Access Driver (*.mdb)}};' 'DBQ={source}'.format(source=self.import_source)) except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e: log.warning('Unable to connect the WorshipCenter Pro ' From 4fc4fa896959b4b0f0e73ee669aa05e81604b747 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 14 Jun 2016 22:36:51 +0200 Subject: [PATCH 6/6] pep8 fixes --- openlp/plugins/songs/lib/importers/mediashout.py | 7 ++++--- openlp/plugins/songs/lib/importers/videopsalm.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/mediashout.py b/openlp/plugins/songs/lib/importers/mediashout.py index af9b855a0..a3bd7bbbc 100644 --- a/openlp/plugins/songs/lib/importers/mediashout.py +++ b/openlp/plugins/songs/lib/importers/mediashout.py @@ -47,7 +47,6 @@ class MediaShoutImport(SongImport): Initialise the MediaShout importer. """ super(MediaShoutImport, self).__init__(manager, **kwargs) - #SongImport.__init__(self, manager, **kwargs) def do_import(self): """ @@ -70,9 +69,11 @@ class MediaShoutImport(SongImport): topics = [] if self.stop_import_flag: break - cursor.execute('SELECT Type, Number, Text FROM Verses WHERE Record = ? ORDER BY Type, Number', float(song.Record)) + cursor.execute('SELECT Type, Number, Text FROM Verses WHERE Record = ? ORDER BY Type, Number', + float(song.Record)) verses = cursor.fetchall() - cursor.execute('SELECT Type, Number, POrder FROM PlayOrder WHERE Record = ? ORDER BY POrder', float(song.Record)) + cursor.execute('SELECT Type, Number, POrder FROM PlayOrder WHERE Record = ? ORDER BY POrder', + float(song.Record)) verse_order = cursor.fetchall() if cursor.tables(table='TableName', tableType='TABLE').fetchone(): cursor.execute('SELECT Name FROM Themes INNER JOIN SongThemes ON SongThemes.ThemeId = Themes.ThemeId ' diff --git a/openlp/plugins/songs/lib/importers/videopsalm.py b/openlp/plugins/songs/lib/importers/videopsalm.py index 781abfeaf..25fd4d8eb 100644 --- a/openlp/plugins/songs/lib/importers/videopsalm.py +++ b/openlp/plugins/songs/lib/importers/videopsalm.py @@ -118,4 +118,3 @@ class VideoPsalmImport(SongImport): self.log_error('Could not import {title}'.format(title=self.title)) except Exception as e: self.log_error(song_file.name, translate('SongsPlugin.VideoPsalmImport', 'Error: {error}').format(error=e)) -