From d28ca7500e682f93a47d7edc126b405835a26242 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Mon, 29 Feb 2016 22:35:53 +0100 Subject: [PATCH 01/13] Beginning of an OPS Pro importer --- openlp/plugins/songs/lib/importer.py | 13 +++ openlp/plugins/songs/lib/importers/opspro.py | 87 ++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 openlp/plugins/songs/lib/importers/opspro.py diff --git a/openlp/plugins/songs/lib/importer.py b/openlp/plugins/songs/lib/importer.py index 5e099dde9..409bc897a 100644 --- a/openlp/plugins/songs/lib/importer.py +++ b/openlp/plugins/songs/lib/importer.py @@ -170,6 +170,7 @@ class SongFormat(object): WorshipAssistant = 23 WorshipCenterPro = 24 ZionWorx = 25 + OPSPro = 26 # Set optional attribute defaults __defaults__ = { @@ -382,6 +383,17 @@ class SongFormat(object): 'First convert your ZionWorx database to a CSV text file, as ' 'explained in the User Manual.') + }, + OPSPro: { + 'name': 'OPS Pro', + 'prefix': 'OPSPro', + 'canDisable': True, + 'selectMode': SongFormatSelect.SingleFile, + 'filter': '%s (*.mdb)' % translate('SongsPlugin.ImportWizardForm', 'OPS Pro database'), + 'disabledLabelText': translate('SongsPlugin.ImportWizardForm', + 'The OPS Pro importer is only supported on Windows. It has been ' + 'disabled due to a missing Python module. If you want to use this ' + 'importer, you will need to install the "pyodbc" module.') } } @@ -417,6 +429,7 @@ class SongFormat(object): SongFormat.WorshipAssistant, SongFormat.WorshipCenterPro, SongFormat.ZionWorx, + SongFormat.OPSPro ]) @staticmethod diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py new file mode 100644 index 000000000..e0e727261 --- /dev/null +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`opspro` module provides the functionality for importing +a OPS Pro database into the OpenLP database. +""" +import logging +import re +import pyodbc + +from openlp.core.common import translate +from openlp.plugins.songs.lib.importers.songimport import SongImport + +log = logging.getLogger(__name__) + + +class OpsProImport(SongImport): + """ + The :class:`OpsProImport` class provides the ability to import the + WorshipCenter Pro Access Database + """ + def __init__(self, manager, **kwargs): + """ + Initialise the WorshipCenter Pro importer. + """ + super(OpsProImport, self).__init__(manager, **kwargs) + + def do_import(self): + """ + Receive a single file to import. + """ + try: + conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s' % self.import_source) + except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e: + log.warning('Unable to connect the OPS Pro database %s. %s', self.import_source, str(e)) + # Unfortunately no specific exception type + self.log_error(self.import_source, translate('SongsPlugin.OpsProImport', + 'Unable to connect the OPS Pro database.')) + return + cursor = conn.cursor() + cursor.execute('SELECT Song.ID, Song.SongNumber, Song.SongBookID, Song.Title, Song.CopyrightText, Version, Origin FROM Song ORDER BY Song.Title') + songs = cursor.fetchall() + self.import_wizard.progress_bar.setMaximum(len(songs)) + for song in songs: + if self.stop_import_flag: + break + cursor.execute('SELECT Lyrics FROM Lyrics WHERE SongID = %s ORDER BY Type, Number' + % song.ID) + verses = cursor.fetchall() + cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ON SongCategory.CategoryID = Category.CategoryID ' + 'WHERE SongCategory.SongID = %s' % song.ID) + topics = cursor.fetchall() + + + self.process_song(song, verses, topics) + + def process_song(self, song, verses, verse_order, topics): + """ + Create the song, i.e. title, verse etc. + """ + self.set_defaults() + self.title = song.Title + self.parse_author(song.CopyrightText) + self.add_copyright(song.Origin) + for topic in topics: + self.topics.append(topic.Name) + self.add_verse(verses.Lyrics) + self.finish() From e67ad21740a82be508a6e48680955240a15d3aae Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Mon, 7 Mar 2016 23:27:28 +0100 Subject: [PATCH 02/13] Getting closer... --- openlp/plugins/songs/lib/importer.py | 11 ++ openlp/plugins/songs/lib/importers/opspro.py | 135 ++++++++++++-- .../openlp_plugins/songs/test_opsproimport.py | 166 ++++++++++++++++++ 3 files changed, 296 insertions(+), 16 deletions(-) create mode 100644 tests/functional/openlp_plugins/songs/test_opsproimport.py diff --git a/openlp/plugins/songs/lib/importer.py b/openlp/plugins/songs/lib/importer.py index 409bc897a..d7837ce7d 100644 --- a/openlp/plugins/songs/lib/importer.py +++ b/openlp/plugins/songs/lib/importer.py @@ -48,6 +48,7 @@ from .importers.powerpraise import PowerPraiseImport from .importers.presentationmanager import PresentationManagerImport from .importers.lyrix import LyrixImport from .importers.videopsalm import VideoPsalmImport +from .importers.opspro import OpsProImport log = logging.getLogger(__name__) @@ -78,6 +79,13 @@ if is_win(): HAS_WORSHIPCENTERPRO = True except ImportError: log.exception('Error importing %s', 'WorshipCenterProImport') +HAS_OPSPRO = False +if is_win(): + try: + from .importers.opspro import OpsProImport + HAS_OPSPRO = True + except ImportError: + log.exception('Error importing %s', 'OpsProImport') class SongFormatSelect(object): @@ -478,6 +486,9 @@ if HAS_MEDIASHOUT: SongFormat.set(SongFormat.WorshipCenterPro, 'availability', HAS_WORSHIPCENTERPRO) if HAS_WORSHIPCENTERPRO: SongFormat.set(SongFormat.WorshipCenterPro, 'class', WorshipCenterProImport) +SongFormat.set(SongFormat.OPSPro, 'availability', HAS_OPSPRO) +if HAS_OPSPRO: + SongFormat.set(SongFormat.OPSPro, 'class', OpsProImport) __all__ = ['SongFormat', 'SongFormatSelect'] diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index e0e727261..957f16f81 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -25,7 +25,10 @@ a OPS Pro database into the OpenLP database. """ import logging import re -import pyodbc +import os +if os.name == 'nt': + import pyodbc +import struct from openlp.core.common import translate from openlp.plugins.songs.lib.importers.songimport import SongImport @@ -48,8 +51,9 @@ class OpsProImport(SongImport): """ Receive a single file to import. """ + password = self.extract_mdb_password() try: - conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s' % self.import_source) + conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s;PWD=%s' % (self.import_source, password)) except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e: log.warning('Unable to connect the OPS Pro database %s. %s', self.import_source, str(e)) # Unfortunately no specific exception type @@ -57,31 +61,130 @@ class OpsProImport(SongImport): 'Unable to connect the OPS Pro database.')) return cursor = conn.cursor() - cursor.execute('SELECT Song.ID, Song.SongNumber, Song.SongBookID, Song.Title, Song.CopyrightText, Version, Origin FROM Song ORDER BY Song.Title') + cursor.execute('SELECT Song.ID, SongNumber, SongBookName, Title, CopyrightText, Version, Origin FROM Song ' + 'LEFT JOIN SongBook ON Song.SongBookID = SongBook.ID ORDER BY Title') songs = cursor.fetchall() self.import_wizard.progress_bar.setMaximum(len(songs)) for song in songs: if self.stop_import_flag: break - cursor.execute('SELECT Lyrics FROM Lyrics WHERE SongID = %s ORDER BY Type, Number' - % song.ID) - verses = cursor.fetchall() - cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ON SongCategory.CategoryID = Category.CategoryID ' - 'WHERE SongCategory.SongID = %s' % song.ID) + cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = %d AND Type < 2 ORDER BY Type DESC' % song.ID) + lyrics = cursor.fetchone() + cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ' + 'ON Category.ID = SongCategory.CategoryID WHERE SongCategory.SongID = %d ' + 'ORDER BY CategoryName' % song.ID) topics = cursor.fetchall() + self.process_song(song, lyrics, topics) + break - - self.process_song(song, verses, topics) - - def process_song(self, song, verses, verse_order, topics): + def process_song(self, song, lyrics, topics): """ Create the song, i.e. title, verse etc. """ self.set_defaults() self.title = song.Title - self.parse_author(song.CopyrightText) - self.add_copyright(song.Origin) + if song.CopyrightText: + self.parse_author(song.CopyrightText) + if song.Origin: + self.comments = song.Origin + if song.SongBookName: + self.song_book_name = song.SongBookName + if song.SongNumber: + self.song_number = song.SongNumber for topic in topics: - self.topics.append(topic.Name) - self.add_verse(verses.Lyrics) + self.topics.append(topic.CategoryName) + # Try to split lyrics based on various rules + print(song.ID) + if lyrics: + lyrics_text = lyrics.Lyrics + # Remove whitespaces around the join-tag to keep verses joint + lyrics_text = re.sub('\w*\[join\]\w*', '[join]', lyrics_text, flags=re.IGNORECASE) + lyrics_text = re.sub('\w*\[splits?\]\w*', '[split]', lyrics_text, flags=re.IGNORECASE) + verses = lyrics_text.split('\r\n\r\n') + verse_tag_defs = {} + verse_tag_texts = {} + chorus = '' + for verse_text in verses: + verse_def = 'v' + # Try to detect verse number + verse_number = re.match('^(\d+)\r\n', verse_text) + if verse_number: + verse_text = re.sub('^\d+\r\n', '', verse_text) + verse_def = 'v' + verse_number.group(1) + # Detect verse tags + elif re.match('^.*?:\r\n', verse_text): + tag_match = re.match('^(.*?)(\w.+)?:\r\n(.*)', verse_text) + tag = tag_match.group(1) + verse_text = tag_match.group(3) + if 'refrain' in tag.lower(): + verse_def = 'c' + elif 'bridge' in tag.lower(): + verse_def = 'b' + verse_tag_defs[tag] = verse_def + elif re.match('^\(.*\)$', verse_text): + tag_match = re.match('^\((.*)\)$', verse_text) + tag = tag_match.group(1) + if tag in verse_tag_defs: + verse_text = verse_tag_texts[tag] + verse_def = verse_tag_defs[tag] + # Try to detect end tag + elif re.match('^\[slot\]\r\n', verse_text, re.IGNORECASE): + verse_def = 'e' + verse_text = re.sub('^\[slot\]\r\n', '', verse_text, flags=re.IGNORECASE) + # Handle tags + # Replace the join tag with line breaks + verse_text = re.sub('\[join\]', '\r\n\r\n\r\n', verse_text) + # Replace the split tag with line breaks and an optional split + verse_text = re.sub('\[split\]', '\r\n\r\n[---]\r\n', verse_text) + # Handle translations + #if lyrics.IsDualLanguage: + # ... + + # Remove comments + verse_text = re.sub('\(.*?\)\r\n', '', verse_text, flags=re.IGNORECASE) + self.add_verse(verse_text, verse_def) + print(verse_def) + print(verse_text) self.finish() + + def extract_mdb_password(self): + """ + Extract password from mdb. Based on code from + http://tutorialsto.com/database/access/crack-access-*.-mdb-all-current-versions-of-the-password.html + """ + # The definition of 13 bytes as the source XOR Access2000. Encrypted with the corresponding signs are 0x13 + xor_pattern_2k = (0xa1, 0xec, 0x7a, 0x9c, 0xe1, 0x28, 0x34, 0x8a, 0x73, 0x7b, 0xd2, 0xdf, 0x50) + # Access97 XOR of the source + xor_pattern_97 = (0x86, 0xfb, 0xec, 0x37, 0x5d, 0x44, 0x9c, 0xfa, 0xc6, 0x5e, 0x28, 0xe6, 0x13) + mdb = open(self.import_source, 'rb') + mdb.seek(0x14) + version = struct.unpack('B', mdb.read(1))[0] + # Get encrypted logo + mdb.seek(0x62) + EncrypFlag = struct.unpack('B', mdb.read(1))[0] + # Get encrypted password + mdb.seek(0x42); + encrypted_password = mdb.read(26) + mdb.close() + # "Decrypt" the password based on the version + decrypted_password = '' + if version < 0x01: + # Access 97 + if int (encrypted_password[0] ^ xor_pattern_97[0]) == 0: + # No password + decrypted_password = '' + else: + for j in range(0, 12): + decrypted_password = decrypted_password + chr(encrypted_password[j] ^ xor_pattern_97[j]) + else: + # Access 2000 or 2002 + for j in range(0, 12): + if j% 2 == 0: + # Every byte with a different sign or encrypt. Encryption signs here for the 0x13 + t1 = chr (0x13 ^ EncrypFlag ^ encrypted_password[j * 2] ^ xor_pattern_2k[j]) + else: + t1 = chr(encrypted_password[j * 2] ^ xor_pattern_2k[j]); + decrypted_password = decrypted_password + t1; + if ord(decrypted_password[1]) < 0x20 or ord(decrypted_password[1]) > 0x7e: + decrypted_password = '' + return decrypted_password diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py new file mode 100644 index 000000000..8289ae0dc --- /dev/null +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +This module contains tests for the WorshipCenter Pro song importer. +""" +import os +from unittest import TestCase, SkipTest + +from tests.functional import patch, MagicMock + +from openlp.core.common import Registry +from openlp.plugins.songs.lib.importers.opspro import OpsProImport + + +class TestRecord(object): + """ + Microsoft Access Driver is not available on non Microsoft Systems for this reason the :class:`TestRecord` is used + to simulate a recordset that would be returned by pyobdc. + """ + def __init__(self, id, field, value): + # The case of the following instance variables is important as it needs to be the same as the ones in use in the + # WorshipCenter Pro database. + self.ID = id + self.Field = field + self.Value = value + + +RECORDSET_TEST_DATA = [TestRecord(1, 'TITLE', 'Amazing Grace'), + TestRecord(1, 'AUTHOR', 'John Newton'), + TestRecord(1, 'CCLISONGID', '12345'), + TestRecord(1, 'COMMENTS', 'The original version'), + TestRecord(1, 'COPY', 'Public Domain'), + TestRecord( + 1, 'LYRICS', + 'Amazing grace! How&crlf;sweet the sound&crlf;That saved a wretch like me!&crlf;' + 'I once was lost,&crlf;but now am found;&crlf;Was blind, but now I see.&crlf;&crlf;' + '\'Twas grace that&crlf;taught my heart to fear,&crlf;And grace my fears relieved;&crlf;' + 'How precious did&crlf;that grace appear&crlf;The hour I first believed.&crlf;&crlf;' + 'Through many dangers,&crlf;toils and snares,&crlf;I have already come;&crlf;' + '\'Tis grace hath brought&crlf;me safe thus far,&crlf;' + 'And grace will lead me home.&crlf;&crlf;The Lord has&crlf;promised good to me,&crlf;' + 'His Word my hope secures;&crlf;He will my Shield&crlf;and Portion be,&crlf;' + 'As long as life endures.&crlf;&crlf;Yea, when this flesh&crlf;and heart shall fail,&crlf;' + 'And mortal life shall cease,&crlf;I shall possess,&crlf;within the veil,&crlf;' + 'A life of joy and peace.&crlf;&crlf;The earth shall soon&crlf;dissolve like snow,&crlf;' + 'The sun forbear to shine;&crlf;But God, Who called&crlf;me here below,&crlf;' + 'Shall be forever mine.&crlf;&crlf;When we\'ve been there&crlf;ten thousand years,&crlf;' + 'Bright shining as the sun,&crlf;We\'ve no less days to&crlf;sing God\'s praise&crlf;' + 'Than when we\'d first begun.&crlf;&crlf;'), + TestRecord(2, 'TITLE', 'Beautiful Garden Of Prayer, The'), + TestRecord( + 2, 'LYRICS', + 'There\'s a garden where&crlf;Jesus is waiting,&crlf;' + 'There\'s a place that&crlf;is wondrously fair,&crlf;For it glows with the&crlf;' + 'light of His presence.&crlf;\'Tis the beautiful&crlf;garden of prayer.&crlf;&crlf;' + 'Oh, the beautiful garden,&crlf;the garden of prayer!&crlf;Oh, the beautiful&crlf;' + 'garden of prayer!&crlf;There my Savior awaits,&crlf;and He opens the gates&crlf;' + 'To the beautiful&crlf;garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;' + 'Jesus is waiting,&crlf;And I go with my&crlf;burden and care,&crlf;' + 'Just to learn from His&crlf;lips words of comfort&crlf;In the beautiful&crlf;' + 'garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;Jesus is waiting,&crlf;' + 'And He bids you to come,&crlf;meet Him there;&crlf;Just to bow and&crlf;' + 'receive a new blessing&crlf;In the beautiful&crlf;garden of prayer.&crlf;&crlf;')] +SONG_TEST_DATA = [{'title': 'Amazing Grace', + 'verses': [ + ('Amazing grace! How\nsweet the sound\nThat saved a wretch like me!\nI once was lost,\n' + 'but now am found;\nWas blind, but now I see.'), + ('\'Twas grace that\ntaught my heart to fear,\nAnd grace my fears relieved;\nHow precious did\n' + 'that grace appear\nThe hour I first believed.'), + ('Through many dangers,\ntoils and snares,\nI have already come;\n\'Tis grace hath brought\n' + 'me safe thus far,\nAnd grace will lead me home.'), + ('The Lord has\npromised good to me,\nHis Word my hope secures;\n' + 'He will my Shield\nand Portion be,\nAs long as life endures.'), + ('Yea, when this flesh\nand heart shall fail,\nAnd mortal life shall cease,\nI shall possess,\n' + 'within the veil,\nA life of joy and peace.'), + ('The earth shall soon\ndissolve like snow,\nThe sun forbear to shine;\nBut God, Who called\n' + 'me here below,\nShall be forever mine.'), + ('When we\'ve been there\nten thousand years,\nBright shining as the sun,\n' + 'We\'ve no less days to\nsing God\'s praise\nThan when we\'d first begun.')], + 'author': 'John Newton', + 'comments': 'The original version', + 'copyright': 'Public Domain'}, + {'title': 'Beautiful Garden Of Prayer, The', + 'verses': [ + ('There\'s a garden where\nJesus is waiting,\nThere\'s a place that\nis wondrously fair,\n' + 'For it glows with the\nlight of His presence.\n\'Tis the beautiful\ngarden of prayer.'), + ('Oh, the beautiful garden,\nthe garden of prayer!\nOh, the beautiful\ngarden of prayer!\n' + 'There my Savior awaits,\nand He opens the gates\nTo the beautiful\ngarden of prayer.'), + ('There\'s a garden where\nJesus is waiting,\nAnd I go with my\nburden and care,\n' + 'Just to learn from His\nlips words of comfort\nIn the beautiful\ngarden of prayer.'), + ('There\'s a garden where\nJesus is waiting,\nAnd He bids you to come,\nmeet Him there;\n' + 'Just to bow and\nreceive a new blessing\nIn the beautiful\ngarden of prayer.')]}] + + +class TestOpsProSongImport(TestCase): + """ + Test the functions in the :mod:`opsproimport` module. + """ + def setUp(self): + """ + Create the registry + """ + Registry.create() + + @patch('openlp.plugins.songs.lib.importers.opspro.SongImport') + def create_importer_test(self, mocked_songimport): + """ + Test creating an instance of the OPS Pro file importer + """ + # GIVEN: A mocked out SongImport class, and a mocked out "manager" + mocked_manager = MagicMock() + + # WHEN: An importer object is created + importer = OpsProImport(mocked_manager, filenames=[]) + + # THEN: The importer object should not be None + self.assertIsNotNone(importer, 'Import should not be none') + + @patch('openlp.plugins.songs.lib.importers.opspro.SongImport') + def detect_chorus_test(self, mocked_songimport): + """ + Test importing lyrics with a chorus in OPS Pro + """ + # GIVEN: A mocked out SongImport class, and a mocked out "manager" + mocked_manager = MagicMock() + importer = OpsProImport(mocked_manager, filenames=[]) + song = MagicMock() + song.ID = 100 + song.SongNumber = 123 + song.SongBookName = 'The Song Book' + song.Title = 'Song Title' + song.CopyrightText = 'Music and text by me' + song.Version = '1' + song.Origin = '...' + lyrics = MagicMock() + lyrics.Lyrics = 'sd' + lyrics.Type = 1 + lyrics.IsDualLanguage = True + importer.finish = MagicMock() + + # WHEN: An importer object is created + importer.process_song(song, lyrics, []) + + # THEN: The importer object should not be None + print(importer.verses) + print(importer.verse_order_list) + self.assertIsNone(importer, 'Import should not be none') \ No newline at end of file From 98eb50e9b2dd6bb0a3ff83a7e7e87981bf6b5d91 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 8 Mar 2016 22:43:10 +0100 Subject: [PATCH 03/13] Made the anchor/tag detection more generic. --- openlp/plugins/songs/lib/importers/opspro.py | 31 ++++---- .../openlp_plugins/songs/test_opsproimport.py | 75 ++++++++++--------- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 957f16f81..5f423418b 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -98,13 +98,15 @@ class OpsProImport(SongImport): if lyrics: lyrics_text = lyrics.Lyrics # Remove whitespaces around the join-tag to keep verses joint - lyrics_text = re.sub('\w*\[join\]\w*', '[join]', lyrics_text, flags=re.IGNORECASE) - lyrics_text = re.sub('\w*\[splits?\]\w*', '[split]', lyrics_text, flags=re.IGNORECASE) - verses = lyrics_text.split('\r\n\r\n') + lyrics_text = re.sub('\s*\[join\]\s*', '[join]', lyrics_text, flags=re.IGNORECASE) + lyrics_text = re.sub('\s*\[splits?\]\s*', '[split]', lyrics_text, flags=re.IGNORECASE) + verses = re.split('\r\n\s*?\r\n', lyrics_text) verse_tag_defs = {} verse_tag_texts = {} chorus = '' for verse_text in verses: + if verse_text.strip() == '': + continue verse_def = 'v' # Try to detect verse number verse_number = re.match('^(\d+)\r\n', verse_text) @@ -112,18 +114,21 @@ class OpsProImport(SongImport): verse_text = re.sub('^\d+\r\n', '', verse_text) verse_def = 'v' + verse_number.group(1) # Detect verse tags - elif re.match('^.*?:\r\n', verse_text): - tag_match = re.match('^(.*?)(\w.+)?:\r\n(.*)', verse_text) - tag = tag_match.group(1) - verse_text = tag_match.group(3) - if 'refrain' in tag.lower(): + elif re.match('^.+?\:\r\n', verse_text): + tag_match = re.match('^(.+?)\:\r\n(.*)', verse_text, flags=re.DOTALL) + tag = tag_match.group(1).lower() + tag = tag.split(' ')[0] + verse_text = tag_match.group(2) + if 'refrein' in tag: verse_def = 'c' - elif 'bridge' in tag.lower(): + elif 'bridge' in tag: verse_def = 'b' verse_tag_defs[tag] = verse_def - elif re.match('^\(.*\)$', verse_text): - tag_match = re.match('^\((.*)\)$', verse_text) - tag = tag_match.group(1) + verse_tag_texts[tag] = verse_text + # Detect tag reference + elif re.match('^\(.*?\)$', verse_text): + tag_match = re.match('^\((.*?)\)$', verse_text) + tag = tag_match.group(1).lower() if tag in verse_tag_defs: verse_text = verse_tag_texts[tag] verse_def = verse_tag_defs[tag] @@ -133,7 +138,7 @@ class OpsProImport(SongImport): verse_text = re.sub('^\[slot\]\r\n', '', verse_text, flags=re.IGNORECASE) # Handle tags # Replace the join tag with line breaks - verse_text = re.sub('\[join\]', '\r\n\r\n\r\n', verse_text) + verse_text = re.sub('\[join\]', '\r\n\r\n', verse_text) # Replace the split tag with line breaks and an optional split verse_text = re.sub('\[split\]', '\r\n\r\n[---]\r\n', verse_text) # Handle translations diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index 8289ae0dc..ec6dd14fb 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -43,43 +43,44 @@ class TestRecord(object): self.Field = field self.Value = value +SONG_TEST_DATA1 = ('Refrein 2x:\r\n' +'Kom zing een nieuw lied\r\n' +'want dit is een nieuwe dag.\r\n' +'Zet de poorten open en zing je lied voor Hem.\r\n' +'[splits]\r\n' +'Kom zing een nieuw lied\r\n' +'Hij heeft je roep gehoord.\r\n' +'En de trouw en liefde van God zijn ook voor jou!\r\n' +' \r\n' +'Hij glimlacht en schijnt zijn licht op ons.\r\n' +'Hij redt ons en steunt ons liefdevol.\r\n' +'Mijn Redder, mijn sterkte is de Heer.\r\n' +'Deze dag leef ik voor Hem - en geef Hem eer!\r\n' +' \r\n' +'(refrein)\r\n' +'\r\n' +'Zijn goedheid rust elke dag op ons.\r\n' +'Zijn liefde verdrijft de angst in ons.\r\n' +'Mijn schuilplaats, mijn toevlucht is de Heer.\r\n' +'Deze dag leef ik voor Hem - en geef Hem eer!\r\n' +' \r\n' +'(refrein)\r\n' +'\r\n' +'Bridge 3x:\r\n' +'Breng dank aan de Heer jouw God.\r\n' +'Geef eer met een dankbaar hart,\r\n' +'Hij toont zijn liefde hier vandaag!\r\n' +'[splits]\r\n' +'Breng dank aan de Heer, jouw God.\r\n' +'Geef eer met een dankbaar hart.\r\n' +'Open je hart voor Hem vandaag!\r\n' +'\r\n' +'Ik zing een nieuw lied en breng Hem de hoogste eer\r\n' +'want de nieuwe dag is vol zegen van de Heer!\r\n' +'Ik zing een nieuw lied en breng Hem de hoogste eer.\r\n' +'Zet je hart wijd open en zing je lied voor Hem!\r\n') + -RECORDSET_TEST_DATA = [TestRecord(1, 'TITLE', 'Amazing Grace'), - TestRecord(1, 'AUTHOR', 'John Newton'), - TestRecord(1, 'CCLISONGID', '12345'), - TestRecord(1, 'COMMENTS', 'The original version'), - TestRecord(1, 'COPY', 'Public Domain'), - TestRecord( - 1, 'LYRICS', - 'Amazing grace! How&crlf;sweet the sound&crlf;That saved a wretch like me!&crlf;' - 'I once was lost,&crlf;but now am found;&crlf;Was blind, but now I see.&crlf;&crlf;' - '\'Twas grace that&crlf;taught my heart to fear,&crlf;And grace my fears relieved;&crlf;' - 'How precious did&crlf;that grace appear&crlf;The hour I first believed.&crlf;&crlf;' - 'Through many dangers,&crlf;toils and snares,&crlf;I have already come;&crlf;' - '\'Tis grace hath brought&crlf;me safe thus far,&crlf;' - 'And grace will lead me home.&crlf;&crlf;The Lord has&crlf;promised good to me,&crlf;' - 'His Word my hope secures;&crlf;He will my Shield&crlf;and Portion be,&crlf;' - 'As long as life endures.&crlf;&crlf;Yea, when this flesh&crlf;and heart shall fail,&crlf;' - 'And mortal life shall cease,&crlf;I shall possess,&crlf;within the veil,&crlf;' - 'A life of joy and peace.&crlf;&crlf;The earth shall soon&crlf;dissolve like snow,&crlf;' - 'The sun forbear to shine;&crlf;But God, Who called&crlf;me here below,&crlf;' - 'Shall be forever mine.&crlf;&crlf;When we\'ve been there&crlf;ten thousand years,&crlf;' - 'Bright shining as the sun,&crlf;We\'ve no less days to&crlf;sing God\'s praise&crlf;' - 'Than when we\'d first begun.&crlf;&crlf;'), - TestRecord(2, 'TITLE', 'Beautiful Garden Of Prayer, The'), - TestRecord( - 2, 'LYRICS', - 'There\'s a garden where&crlf;Jesus is waiting,&crlf;' - 'There\'s a place that&crlf;is wondrously fair,&crlf;For it glows with the&crlf;' - 'light of His presence.&crlf;\'Tis the beautiful&crlf;garden of prayer.&crlf;&crlf;' - 'Oh, the beautiful garden,&crlf;the garden of prayer!&crlf;Oh, the beautiful&crlf;' - 'garden of prayer!&crlf;There my Savior awaits,&crlf;and He opens the gates&crlf;' - 'To the beautiful&crlf;garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;' - 'Jesus is waiting,&crlf;And I go with my&crlf;burden and care,&crlf;' - 'Just to learn from His&crlf;lips words of comfort&crlf;In the beautiful&crlf;' - 'garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;Jesus is waiting,&crlf;' - 'And He bids you to come,&crlf;meet Him there;&crlf;Just to bow and&crlf;' - 'receive a new blessing&crlf;In the beautiful&crlf;garden of prayer.&crlf;&crlf;')] SONG_TEST_DATA = [{'title': 'Amazing Grace', 'verses': [ ('Amazing grace! How\nsweet the sound\nThat saved a wretch like me!\nI once was lost,\n' @@ -152,7 +153,7 @@ class TestOpsProSongImport(TestCase): song.Version = '1' song.Origin = '...' lyrics = MagicMock() - lyrics.Lyrics = 'sd' + lyrics.Lyrics = SONG_TEST_DATA1 lyrics.Type = 1 lyrics.IsDualLanguage = True importer.finish = MagicMock() From 51ffb92d40ad29201b935b8bfa41785903fb1f32 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Wed, 9 Mar 2016 22:44:15 +0100 Subject: [PATCH 04/13] Started work on tests --- openlp/plugins/songs/lib/importers/opspro.py | 11 +++ .../openlp_plugins/songs/test_opsproimport.py | 72 +------------------ .../opsprosongs/you are so faithfull.txt | 37 ++++++++++ 3 files changed, 51 insertions(+), 69 deletions(-) create mode 100644 tests/resources/opsprosongs/you are so faithfull.txt diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 5f423418b..9572f8923 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -80,6 +80,17 @@ class OpsProImport(SongImport): def process_song(self, song, lyrics, topics): """ Create the song, i.e. title, verse etc. + + The OPS Pro format is a fairly simple text format using tags and anchors/labels. Linebreaks are \r\n. + Double linebreaks are slide dividers. OPS Pro support dual language using tags. + Tags are in [], see the liste below: + [join] are used to separate verses that should be keept on the same slide. + [split] or [splits] can be used to split a verse over several slides, while still being the same verse + Dual language tags: + [trans off] or [vertaal uit] turns dual language mode off for the following text + [trans on] or [vertaal aan] turns dual language mode on for the following text + [taal a] means the following lines are language a + [taal b] means the following lines are language b """ self.set_defaults() self.title = song.Title diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index ec6dd14fb..0eb03af9f 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -30,6 +30,7 @@ from tests.functional import patch, MagicMock from openlp.core.common import Registry from openlp.plugins.songs.lib.importers.opspro import OpsProImport +TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'opsprosongs')) class TestRecord(object): """ @@ -43,74 +44,6 @@ class TestRecord(object): self.Field = field self.Value = value -SONG_TEST_DATA1 = ('Refrein 2x:\r\n' -'Kom zing een nieuw lied\r\n' -'want dit is een nieuwe dag.\r\n' -'Zet de poorten open en zing je lied voor Hem.\r\n' -'[splits]\r\n' -'Kom zing een nieuw lied\r\n' -'Hij heeft je roep gehoord.\r\n' -'En de trouw en liefde van God zijn ook voor jou!\r\n' -' \r\n' -'Hij glimlacht en schijnt zijn licht op ons.\r\n' -'Hij redt ons en steunt ons liefdevol.\r\n' -'Mijn Redder, mijn sterkte is de Heer.\r\n' -'Deze dag leef ik voor Hem - en geef Hem eer!\r\n' -' \r\n' -'(refrein)\r\n' -'\r\n' -'Zijn goedheid rust elke dag op ons.\r\n' -'Zijn liefde verdrijft de angst in ons.\r\n' -'Mijn schuilplaats, mijn toevlucht is de Heer.\r\n' -'Deze dag leef ik voor Hem - en geef Hem eer!\r\n' -' \r\n' -'(refrein)\r\n' -'\r\n' -'Bridge 3x:\r\n' -'Breng dank aan de Heer jouw God.\r\n' -'Geef eer met een dankbaar hart,\r\n' -'Hij toont zijn liefde hier vandaag!\r\n' -'[splits]\r\n' -'Breng dank aan de Heer, jouw God.\r\n' -'Geef eer met een dankbaar hart.\r\n' -'Open je hart voor Hem vandaag!\r\n' -'\r\n' -'Ik zing een nieuw lied en breng Hem de hoogste eer\r\n' -'want de nieuwe dag is vol zegen van de Heer!\r\n' -'Ik zing een nieuw lied en breng Hem de hoogste eer.\r\n' -'Zet je hart wijd open en zing je lied voor Hem!\r\n') - - -SONG_TEST_DATA = [{'title': 'Amazing Grace', - 'verses': [ - ('Amazing grace! How\nsweet the sound\nThat saved a wretch like me!\nI once was lost,\n' - 'but now am found;\nWas blind, but now I see.'), - ('\'Twas grace that\ntaught my heart to fear,\nAnd grace my fears relieved;\nHow precious did\n' - 'that grace appear\nThe hour I first believed.'), - ('Through many dangers,\ntoils and snares,\nI have already come;\n\'Tis grace hath brought\n' - 'me safe thus far,\nAnd grace will lead me home.'), - ('The Lord has\npromised good to me,\nHis Word my hope secures;\n' - 'He will my Shield\nand Portion be,\nAs long as life endures.'), - ('Yea, when this flesh\nand heart shall fail,\nAnd mortal life shall cease,\nI shall possess,\n' - 'within the veil,\nA life of joy and peace.'), - ('The earth shall soon\ndissolve like snow,\nThe sun forbear to shine;\nBut God, Who called\n' - 'me here below,\nShall be forever mine.'), - ('When we\'ve been there\nten thousand years,\nBright shining as the sun,\n' - 'We\'ve no less days to\nsing God\'s praise\nThan when we\'d first begun.')], - 'author': 'John Newton', - 'comments': 'The original version', - 'copyright': 'Public Domain'}, - {'title': 'Beautiful Garden Of Prayer, The', - 'verses': [ - ('There\'s a garden where\nJesus is waiting,\nThere\'s a place that\nis wondrously fair,\n' - 'For it glows with the\nlight of His presence.\n\'Tis the beautiful\ngarden of prayer.'), - ('Oh, the beautiful garden,\nthe garden of prayer!\nOh, the beautiful\ngarden of prayer!\n' - 'There my Savior awaits,\nand He opens the gates\nTo the beautiful\ngarden of prayer.'), - ('There\'s a garden where\nJesus is waiting,\nAnd I go with my\nburden and care,\n' - 'Just to learn from His\nlips words of comfort\nIn the beautiful\ngarden of prayer.'), - ('There\'s a garden where\nJesus is waiting,\nAnd He bids you to come,\nmeet Him there;\n' - 'Just to bow and\nreceive a new blessing\nIn the beautiful\ngarden of prayer.')]}] - class TestOpsProSongImport(TestCase): """ @@ -153,7 +86,8 @@ class TestOpsProSongImport(TestCase): song.Version = '1' song.Origin = '...' lyrics = MagicMock() - lyrics.Lyrics = SONG_TEST_DATA1 + test_file = open(os.path.join(TEST_PATH, 'you are so faithfull.txt'), 'rb') + lyrics.Lyrics = test_file.read().decode() lyrics.Type = 1 lyrics.IsDualLanguage = True importer.finish = MagicMock() diff --git a/tests/resources/opsprosongs/you are so faithfull.txt b/tests/resources/opsprosongs/you are so faithfull.txt new file mode 100644 index 000000000..ff9ced2c2 --- /dev/null +++ b/tests/resources/opsprosongs/you are so faithfull.txt @@ -0,0 +1,37 @@ +1 +You are so faithful +so faithful, so faithful. +You are so faithful +so faithful, so faithful. + +Refrein: +That's why I praise you +in the morning +That's why I praise you +in the noontime. +That's why I praise you +in the evening +That's why I praise you +all the time. + +2 +You are so loving +so loving, so loving. +You are so loving +so loving, so loving. + +(refrein) + +3 +You are so caring +so caring, so caring. +You are so caring +so caring, so caring. + +(refrein) + +4 +You are so mighty +so mighty, so mighty. +You are so mighty +so mighty, so mighty. From e9e5976d2208939742bb397b9fcf62bd67fa82c1 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Fri, 11 Mar 2016 22:56:07 +0100 Subject: [PATCH 05/13] Finished first test. --- openlp/plugins/songs/lib/importers/opspro.py | 4 +-- .../openlp_plugins/songs/test_opsproimport.py | 18 +++++++---- .../openlp_plugins/songs/test_videopsalm.py | 3 -- .../opsprosongs/You are so faithful.json | 31 +++++++++++++++++++ 4 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 tests/resources/opsprosongs/You are so faithful.json diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 9572f8923..56463d093 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -159,8 +159,8 @@ class OpsProImport(SongImport): # Remove comments verse_text = re.sub('\(.*?\)\r\n', '', verse_text, flags=re.IGNORECASE) self.add_verse(verse_text, verse_def) - print(verse_def) - print(verse_text) + #print(verse_def) + #print(verse_text) self.finish() def extract_mdb_password(self): diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index 0eb03af9f..67b7c5959 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -23,6 +23,7 @@ This module contains tests for the WorshipCenter Pro song importer. """ import os +import json from unittest import TestCase, SkipTest from tests.functional import patch, MagicMock @@ -44,7 +45,6 @@ class TestRecord(object): self.Field = field self.Value = value - class TestOpsProSongImport(TestCase): """ Test the functions in the :mod:`opsproimport` module. @@ -74,7 +74,7 @@ class TestOpsProSongImport(TestCase): """ Test importing lyrics with a chorus in OPS Pro """ - # GIVEN: A mocked out SongImport class, and a mocked out "manager" + # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry mocked_manager = MagicMock() importer = OpsProImport(mocked_manager, filenames=[]) song = MagicMock() @@ -95,7 +95,13 @@ class TestOpsProSongImport(TestCase): # WHEN: An importer object is created importer.process_song(song, lyrics, []) - # THEN: The importer object should not be None - print(importer.verses) - print(importer.verse_order_list) - self.assertIsNone(importer, 'Import should not be none') \ No newline at end of file + # THEN: The imported data should look like expected + result_file = open(os.path.join(TEST_PATH, 'You are so faithful.json'), 'rb') + result_data = json.loads(result_file.read().decode()) + self.assertListEqual(importer.verses, self._get_data(result_data, 'verses')) + self.assertListEqual(importer.verse_order_list_generated, self._get_data(result_data, 'verse_order_list')) + + def _get_data(self, data, key): + if key in data: + return data[key] + return '' diff --git a/tests/functional/openlp_plugins/songs/test_videopsalm.py b/tests/functional/openlp_plugins/songs/test_videopsalm.py index f75a67627..1bf13241d 100644 --- a/tests/functional/openlp_plugins/songs/test_videopsalm.py +++ b/tests/functional/openlp_plugins/songs/test_videopsalm.py @@ -23,11 +23,8 @@ This module contains tests for the VideoPsalm song importer. """ import os -from unittest import TestCase from tests.helpers.songfileimport import SongImportTestHelper -from openlp.core.common import Registry -from tests.functional import patch, MagicMock TEST_PATH = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'videopsalmsongs')) diff --git a/tests/resources/opsprosongs/You are so faithful.json b/tests/resources/opsprosongs/You are so faithful.json new file mode 100644 index 000000000..965d73ab8 --- /dev/null +++ b/tests/resources/opsprosongs/You are so faithful.json @@ -0,0 +1,31 @@ +{ + "title": "You are so faithful", + "verse_order_list": ["v1", "c1", "v2", "c1", "v3", "c1", "v4"], + "verses": [ + [ + "v1", + "You are so faithful\r\nso faithful, so faithful.\r\nYou are so faithful\r\nso faithful, so faithful.", + null + ], + [ + "c1", + "That's why I praise you\r\nin the morning\r\nThat's why I praise you\r\nin the noontime.\r\nThat's why I praise you\r\nin the evening\r\nThat's why I praise you\r\nall the time.", + null + ], + [ + "v2", + "You are so loving\r\nso loving, so loving.\r\nYou are so loving\r\nso loving, so loving.", + null + ], + [ + "v3", + "You are so caring\r\nso caring, so caring.\r\nYou are so caring\r\nso caring, so caring.", + null + ], + [ + "v4", + "You are so mighty\r\nso mighty, so mighty.\r\nYou are so mighty\r\nso mighty, so mighty.", + null + ] + ] +} From 302fcb221b377d77f66cc374b24e79601e82faf4 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Wed, 16 Mar 2016 22:28:29 +0100 Subject: [PATCH 06/13] Added another OPS Pro import test --- openlp/plugins/songs/lib/importers/opspro.py | 2 +- .../openlp_plugins/songs/test_opsproimport.py | 50 +++++++++++++------ .../resources/opsprosongs/Amazing Grace.json | 21 ++++++++ tests/resources/opsprosongs/amazing grace.txt | 24 +++++++++ 4 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 tests/resources/opsprosongs/Amazing Grace.json create mode 100644 tests/resources/opsprosongs/amazing grace.txt diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 56463d093..85d58f7c9 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -130,7 +130,7 @@ class OpsProImport(SongImport): tag = tag_match.group(1).lower() tag = tag.split(' ')[0] verse_text = tag_match.group(2) - if 'refrein' in tag: + if 'refrein' in tag or 'chorus' in tag: verse_def = 'c' elif 'bridge' in tag: verse_def = 'b' diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index 67b7c5959..b3501f2bf 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -77,21 +77,8 @@ class TestOpsProSongImport(TestCase): # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry mocked_manager = MagicMock() importer = OpsProImport(mocked_manager, filenames=[]) - song = MagicMock() - song.ID = 100 - song.SongNumber = 123 - song.SongBookName = 'The Song Book' - song.Title = 'Song Title' - song.CopyrightText = 'Music and text by me' - song.Version = '1' - song.Origin = '...' - lyrics = MagicMock() - test_file = open(os.path.join(TEST_PATH, 'you are so faithfull.txt'), 'rb') - lyrics.Lyrics = test_file.read().decode() - lyrics.Type = 1 - lyrics.IsDualLanguage = True importer.finish = MagicMock() - + song, lyrics = self._build_test_data('you are so faithfull.txt') # WHEN: An importer object is created importer.process_song(song, lyrics, []) @@ -101,7 +88,42 @@ class TestOpsProSongImport(TestCase): self.assertListEqual(importer.verses, self._get_data(result_data, 'verses')) self.assertListEqual(importer.verse_order_list_generated, self._get_data(result_data, 'verse_order_list')) + @patch('openlp.plugins.songs.lib.importers.opspro.SongImport') + def join_and_split_test(self, mocked_songimport): + """ + Test importing lyrics with a split and join tags works in OPS Pro + """ + # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry + mocked_manager = MagicMock() + importer = OpsProImport(mocked_manager, filenames=[]) + importer.finish = MagicMock() + song, lyrics = self._build_test_data('amazing grace.txt') + # WHEN: An importer object is created + importer.process_song(song, lyrics, []) + + # THEN: The imported data should look like expected + result_file = open(os.path.join(TEST_PATH, 'Amazing Grace.json'), 'rb') + result_data = json.loads(result_file.read().decode()) + self.assertListEqual(importer.verses, self._get_data(result_data, 'verses')) + self.assertListEqual(importer.verse_order_list_generated, self._get_data(result_data, 'verse_order_list')) + def _get_data(self, data, key): if key in data: return data[key] return '' + + def _build_test_data(self, test_file): + song = MagicMock() + song.ID = 100 + song.SongNumber = 123 + song.SongBookName = 'The Song Book' + song.Title = 'Song Title' + song.CopyrightText = 'Music and text by me' + song.Version = '1' + song.Origin = '...' + lyrics = MagicMock() + test_file = open(os.path.join(TEST_PATH, test_file), 'rb') + lyrics.Lyrics = test_file.read().decode() + lyrics.Type = 1 + lyrics.IsDualLanguage = True + return song, lyrics diff --git a/tests/resources/opsprosongs/Amazing Grace.json b/tests/resources/opsprosongs/Amazing Grace.json new file mode 100644 index 000000000..9d6df40da --- /dev/null +++ b/tests/resources/opsprosongs/Amazing Grace.json @@ -0,0 +1,21 @@ +{ + "title": "Amazing Grace", + "verse_order_list": ["v1", "v2", "v3"], + "verses": [ + [ + "v1", + "Amazing grace! How sweet the sound!\r\nThat saved a wretch like me!\r\nI once was lost, but now am found;\r\nWas blind, but now I see.\r\n\r\n'Twas grace that taught my heart to fear,\r\nAnd grace my fears relieved.\r\nHow precious did that grace appear,\r\nThe hour I first believed.", + null + ], + [ + "v2", + "The Lord has promised good to me,\r\nHis Word my hope secures.\r\nHe will my shield and portion be\r\nAs long as life endures.", + null + ], + [ + "v3", + "Thro' many dangers, toils and snares\r\nI have already come.\r\n'Tis grace that brought me safe thus far,\r\nAnd grace will lead me home.\r\n\r\n[---]\r\nWhen we've been there ten thousand years,\r\nBright shining as the sun,\r\nWe've no less days to sing God's praise,\r\nThan when we first begun.", + null + ] + ] +} diff --git a/tests/resources/opsprosongs/amazing grace.txt b/tests/resources/opsprosongs/amazing grace.txt new file mode 100644 index 000000000..d12466e85 --- /dev/null +++ b/tests/resources/opsprosongs/amazing grace.txt @@ -0,0 +1,24 @@ +Amazing grace! How sweet the sound! +That saved a wretch like me! +I once was lost, but now am found; +Was blind, but now I see. +[join] +'Twas grace that taught my heart to fear, +And grace my fears relieved. +How precious did that grace appear, +The hour I first believed. + +The Lord has promised good to me, +His Word my hope secures. +He will my shield and portion be +As long as life endures. + +Thro' many dangers, toils and snares +I have already come. +'Tis grace that brought me safe thus far, +And grace will lead me home. +[split] +When we've been there ten thousand years, +Bright shining as the sun, +We've no less days to sing God's praise, +Than when we first begun. From fdc22b4e4cb57e35fefba48f9797951fa8db06cf Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Fri, 18 Mar 2016 23:09:49 +0100 Subject: [PATCH 07/13] Add translations support --- openlp/plugins/songs/lib/importers/opspro.py | 63 ++++++++++++++----- .../openlp_plugins/songs/test_opsproimport.py | 27 ++++++-- .../resources/opsprosongs/amazing grace2.txt | 29 +++++++++ 3 files changed, 101 insertions(+), 18 deletions(-) create mode 100644 tests/resources/opsprosongs/amazing grace2.txt diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 85d58f7c9..69b864116 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -75,7 +75,6 @@ class OpsProImport(SongImport): 'ORDER BY CategoryName' % song.ID) topics = cursor.fetchall() self.process_song(song, lyrics, topics) - break def process_song(self, song, lyrics, topics): """ @@ -105,21 +104,16 @@ class OpsProImport(SongImport): for topic in topics: self.topics.append(topic.CategoryName) # Try to split lyrics based on various rules - print(song.ID) if lyrics: lyrics_text = lyrics.Lyrics - # Remove whitespaces around the join-tag to keep verses joint - lyrics_text = re.sub('\s*\[join\]\s*', '[join]', lyrics_text, flags=re.IGNORECASE) - lyrics_text = re.sub('\s*\[splits?\]\s*', '[split]', lyrics_text, flags=re.IGNORECASE) verses = re.split('\r\n\s*?\r\n', lyrics_text) verse_tag_defs = {} verse_tag_texts = {} - chorus = '' for verse_text in verses: if verse_text.strip() == '': continue verse_def = 'v' - # Try to detect verse number + # Detect verse number verse_number = re.match('^(\d+)\r\n', verse_text) if verse_number: verse_text = re.sub('^\d+\r\n', '', verse_text) @@ -143,19 +137,60 @@ class OpsProImport(SongImport): if tag in verse_tag_defs: verse_text = verse_tag_texts[tag] verse_def = verse_tag_defs[tag] - # Try to detect end tag + # Detect end tag elif re.match('^\[slot\]\r\n', verse_text, re.IGNORECASE): verse_def = 'e' verse_text = re.sub('^\[slot\]\r\n', '', verse_text, flags=re.IGNORECASE) - # Handle tags # Replace the join tag with line breaks - verse_text = re.sub('\[join\]', '\r\n\r\n', verse_text) + verse_text = re.sub('\[join\]', '', verse_text) # Replace the split tag with line breaks and an optional split - verse_text = re.sub('\[split\]', '\r\n\r\n[---]\r\n', verse_text) + verse_text = re.sub('\[split\]', '\r\n[---]', verse_text) # Handle translations - #if lyrics.IsDualLanguage: - # ... - + if lyrics.IsDualLanguage: + language = None + translation = True + translation_verse_text = '' + start_tag = '{translation}' + end_tag = '{/translation}' + verse_text_lines = verse_text.splitlines() + idx = 0 + while idx < len(verse_text_lines): + # Detect if translation is turned on or off + if verse_text_lines[idx] in ['[trans off]', '[vertaal uit]']: + translation = False + idx += 1 + elif verse_text_lines[idx] in ['[trans on]', '[vertaal aan]']: + translation = True + idx += 1 + elif verse_text_lines[idx] == '[taal a]': + language = 'a' + idx += 1 + elif verse_text_lines[idx] == '[taal b]': + language = 'b' + idx += 1 + # Handle the text based on whether translation is off or on + if language: + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + while idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): + if language == 'a': + translation_verse_text += verse_text_lines[idx] + '\r\n' + else: + translation_verse_text += start_tag + verse_text_lines[idx] + end_tag + '\r\n' + idx += 1 + language = None + elif translation: + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + translation_verse_text += start_tag + verse_text_lines[idx] + end_tag + '\r\n' + idx += 1 + else: + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + while idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + verse_text = translation_verse_text # Remove comments verse_text = re.sub('\(.*?\)\r\n', '', verse_text, flags=re.IGNORECASE) self.add_verse(verse_text, verse_def) diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index b3501f2bf..e4ce742b7 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -78,7 +78,7 @@ class TestOpsProSongImport(TestCase): mocked_manager = MagicMock() importer = OpsProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() - song, lyrics = self._build_test_data('you are so faithfull.txt') + song, lyrics = self._build_test_data('you are so faithfull.txt', False) # WHEN: An importer object is created importer.process_song(song, lyrics, []) @@ -97,7 +97,26 @@ class TestOpsProSongImport(TestCase): mocked_manager = MagicMock() importer = OpsProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() - song, lyrics = self._build_test_data('amazing grace.txt') + song, lyrics = self._build_test_data('amazing grace.txt', False) + # WHEN: An importer object is created + importer.process_song(song, lyrics, []) + + # THEN: The imported data should look like expected + result_file = open(os.path.join(TEST_PATH, 'Amazing Grace.json'), 'rb') + result_data = json.loads(result_file.read().decode()) + self.assertListEqual(importer.verses, self._get_data(result_data, 'verses')) + self.assertListEqual(importer.verse_order_list_generated, self._get_data(result_data, 'verse_order_list')) + + @patch('openlp.plugins.songs.lib.importers.opspro.SongImport') + def trans_off_tag_test(self, mocked_songimport): + """ + Test importing lyrics with a split and join and translations tags works in OPS Pro + """ + # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry + mocked_manager = MagicMock() + importer = OpsProImport(mocked_manager, filenames=[]) + importer.finish = MagicMock() + song, lyrics = self._build_test_data('amazing grace2.txt', True) # WHEN: An importer object is created importer.process_song(song, lyrics, []) @@ -112,7 +131,7 @@ class TestOpsProSongImport(TestCase): return data[key] return '' - def _build_test_data(self, test_file): + def _build_test_data(self, test_file, dual_language): song = MagicMock() song.ID = 100 song.SongNumber = 123 @@ -125,5 +144,5 @@ class TestOpsProSongImport(TestCase): test_file = open(os.path.join(TEST_PATH, test_file), 'rb') lyrics.Lyrics = test_file.read().decode() lyrics.Type = 1 - lyrics.IsDualLanguage = True + lyrics.IsDualLanguage = dual_language return song, lyrics diff --git a/tests/resources/opsprosongs/amazing grace2.txt b/tests/resources/opsprosongs/amazing grace2.txt new file mode 100644 index 000000000..1e18a6b62 --- /dev/null +++ b/tests/resources/opsprosongs/amazing grace2.txt @@ -0,0 +1,29 @@ +[trans off] +Amazing grace! How sweet the sound! +That saved a wretch like me! +I once was lost, but now am found; +Was blind, but now I see. +[join] +[trans off] +'Twas grace that taught my heart to fear, +And grace my fears relieved. +How precious did that grace appear, +The hour I first believed. + +[trans off] +The Lord has promised good to me, +His Word my hope secures. +He will my shield and portion be +As long as life endures. + +[trans off] +Thro' many dangers, toils and snares +I have already come. +'Tis grace that brought me safe thus far, +And grace will lead me home. +[trans off] +[split] +When we've been there ten thousand years, +Bright shining as the sun, +We've no less days to sing God's praise, +Than when we first begun. From 7b69634552c89fa8168235df3081235446adfae2 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sat, 19 Mar 2016 07:20:12 +0100 Subject: [PATCH 08/13] Fixes for translation support + test --- openlp/plugins/songs/lib/importers/opspro.py | 18 +++++------ .../openlp_plugins/songs/test_opsproimport.py | 23 ++++++++++++++ .../resources/opsprosongs/Amazing Grace3.json | 31 +++++++++++++++++++ 3 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 tests/resources/opsprosongs/Amazing Grace3.json diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 69b864116..2395fc3f8 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -170,20 +170,20 @@ class OpsProImport(SongImport): idx += 1 # Handle the text based on whether translation is off or on if language: - translation_verse_text += verse_text_lines[idx] + '\r\n' - idx += 1 + if language == 'b': + translation_verse_text += start_tag while idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): - if language == 'a': - translation_verse_text += verse_text_lines[idx] + '\r\n' - else: - translation_verse_text += start_tag + verse_text_lines[idx] + end_tag + '\r\n' + translation_verse_text += verse_text_lines[idx] + '\r\n' idx += 1 + if language == 'b': + translation_verse_text += end_tag language = None elif translation: translation_verse_text += verse_text_lines[idx] + '\r\n' idx += 1 - translation_verse_text += start_tag + verse_text_lines[idx] + end_tag + '\r\n' - idx += 1 + if idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): + translation_verse_text += start_tag + verse_text_lines[idx] + end_tag + '\r\n' + idx += 1 else: translation_verse_text += verse_text_lines[idx] + '\r\n' idx += 1 @@ -194,8 +194,6 @@ class OpsProImport(SongImport): # Remove comments verse_text = re.sub('\(.*?\)\r\n', '', verse_text, flags=re.IGNORECASE) self.add_verse(verse_text, verse_def) - #print(verse_def) - #print(verse_text) self.finish() def extract_mdb_password(self): diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index e4ce742b7..b2f9371dc 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -79,6 +79,7 @@ class TestOpsProSongImport(TestCase): importer = OpsProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() song, lyrics = self._build_test_data('you are so faithfull.txt', False) + # WHEN: An importer object is created importer.process_song(song, lyrics, []) @@ -98,6 +99,7 @@ class TestOpsProSongImport(TestCase): importer = OpsProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() song, lyrics = self._build_test_data('amazing grace.txt', False) + # WHEN: An importer object is created importer.process_song(song, lyrics, []) @@ -117,6 +119,7 @@ class TestOpsProSongImport(TestCase): importer = OpsProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() song, lyrics = self._build_test_data('amazing grace2.txt', True) + # WHEN: An importer object is created importer.process_song(song, lyrics, []) @@ -126,6 +129,26 @@ class TestOpsProSongImport(TestCase): self.assertListEqual(importer.verses, self._get_data(result_data, 'verses')) self.assertListEqual(importer.verse_order_list_generated, self._get_data(result_data, 'verse_order_list')) + @patch('openlp.plugins.songs.lib.importers.opspro.SongImport') + def trans_tag_test(self, mocked_songimport): + """ + Test importing lyrics with various translations tags works in OPS Pro + """ + # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry + mocked_manager = MagicMock() + importer = OpsProImport(mocked_manager, filenames=[]) + importer.finish = MagicMock() + song, lyrics = self._build_test_data('amazing grace3.txt', True) + + # WHEN: An importer object is created + importer.process_song(song, lyrics, []) + + # THEN: The imported data should look like expected + result_file = open(os.path.join(TEST_PATH, 'Amazing Grace3.json'), 'rb') + result_data = json.loads(result_file.read().decode()) + self.assertListEqual(importer.verses, self._get_data(result_data, 'verses')) + self.assertListEqual(importer.verse_order_list_generated, self._get_data(result_data, 'verse_order_list')) + def _get_data(self, data, key): if key in data: return data[key] diff --git a/tests/resources/opsprosongs/Amazing Grace3.json b/tests/resources/opsprosongs/Amazing Grace3.json new file mode 100644 index 000000000..d9ce5cc45 --- /dev/null +++ b/tests/resources/opsprosongs/Amazing Grace3.json @@ -0,0 +1,31 @@ +{ + "title": "Amazing Grace", + "verse_order_list": ["v1", "v2", "v3", "v4", "v5"], + "verses": [ + [ + "v1", + "Amazing grace! How sweet the sound!\r\n{translation}That saved a wretch like me!{/translation}\r\nI once was lost, but now am found;\r\n{translation}Was blind, but now I see.{/translation}", + null + ], + [ + "v2", + "'Twas grace that taught my heart to fear,\r\nAnd grace my fears relieved.\r\n{translation}How precious did that grace appear,\r\nThe hour I first believed.\r\n{/translation}", + null + ], + [ + "v3", + "The Lord has promised good to me,\r\nHis Word my hope secures.\r\nHe will my shield and portion be\r\n{translation}As long as life endures.{/translation}", + null + ], + [ + "v4", + "Thro' many dangers, toils and snares\r\nI have already come.\r\n'Tis grace that brought me safe thus far,\r\n{translation}And grace will lead me home.{/translation}", + null + ], + [ + "v5", + "[end]\r\n{translation}When we've been there ten thousand years,{/translation}\r\nBright shining as the sun,\r\n{translation}We've no less days to sing God's praise,{/translation}\r\nThan when we first begun.", + null + ] + ] +} From 7941d8e2a14af3cfd20fa296ad43f2457348424b Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sat, 19 Mar 2016 21:27:49 +0100 Subject: [PATCH 09/13] improved copyright text handling and restructed a bit. --- openlp/plugins/songs/lib/importers/opspro.py | 103 +++++++++++-------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index 2395fc3f8..d1ffd7aad 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -94,7 +94,11 @@ class OpsProImport(SongImport): self.set_defaults() self.title = song.Title if song.CopyrightText: - self.parse_author(song.CopyrightText) + for line in song.CopyrightText.splitlines(): + if line.startswith('©') or line.lower().startswith('copyright'): + self.add_copyright(line) + else: + self.parse_author(line) if song.Origin: self.comments = song.Origin if song.SongBookName: @@ -147,55 +151,64 @@ class OpsProImport(SongImport): verse_text = re.sub('\[split\]', '\r\n[---]', verse_text) # Handle translations if lyrics.IsDualLanguage: - language = None - translation = True - translation_verse_text = '' - start_tag = '{translation}' - end_tag = '{/translation}' - verse_text_lines = verse_text.splitlines() - idx = 0 - while idx < len(verse_text_lines): - # Detect if translation is turned on or off - if verse_text_lines[idx] in ['[trans off]', '[vertaal uit]']: - translation = False - idx += 1 - elif verse_text_lines[idx] in ['[trans on]', '[vertaal aan]']: - translation = True - idx += 1 - elif verse_text_lines[idx] == '[taal a]': - language = 'a' - idx += 1 - elif verse_text_lines[idx] == '[taal b]': - language = 'b' - idx += 1 - # Handle the text based on whether translation is off or on - if language: - if language == 'b': - translation_verse_text += start_tag - while idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): - translation_verse_text += verse_text_lines[idx] + '\r\n' - idx += 1 - if language == 'b': - translation_verse_text += end_tag - language = None - elif translation: - translation_verse_text += verse_text_lines[idx] + '\r\n' - idx += 1 - if idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): - translation_verse_text += start_tag + verse_text_lines[idx] + end_tag + '\r\n' - idx += 1 - else: - translation_verse_text += verse_text_lines[idx] + '\r\n' - idx += 1 - while idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): - translation_verse_text += verse_text_lines[idx] + '\r\n' - idx += 1 - verse_text = translation_verse_text + verse_text = self.handle_translation(verse_text) # Remove comments verse_text = re.sub('\(.*?\)\r\n', '', verse_text, flags=re.IGNORECASE) self.add_verse(verse_text, verse_def) self.finish() + def handle_translation(self, verse_text): + """ + Replace OPS Pro translation tags with a {translation} tag + + :param verse_text: the verse text + :return: the verse text with replaced tags + """ + language = None + translation = True + translation_verse_text = '' + start_tag = '{translation}' + end_tag = '{/translation}' + verse_text_lines = verse_text.splitlines() + idx = 0 + while idx < len(verse_text_lines): + # Detect if translation is turned on or off + if verse_text_lines[idx] in ['[trans off]', '[vertaal uit]']: + translation = False + idx += 1 + elif verse_text_lines[idx] in ['[trans on]', '[vertaal aan]']: + translation = True + idx += 1 + elif verse_text_lines[idx] == '[taal a]': + language = 'a' + idx += 1 + elif verse_text_lines[idx] == '[taal b]': + language = 'b' + idx += 1 + # Handle the text based on whether translation is off or on + if language: + if language == 'b': + translation_verse_text += start_tag + while idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + if language == 'b': + translation_verse_text += end_tag + language = None + elif translation: + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + if idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): + translation_verse_text += start_tag + verse_text_lines[idx] + end_tag + '\r\n' + idx += 1 + else: + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + while idx < len(verse_text_lines) and not verse_text_lines[idx].startswith('['): + translation_verse_text += verse_text_lines[idx] + '\r\n' + idx += 1 + return translation_verse_text + def extract_mdb_password(self): """ Extract password from mdb. Based on code from From b20eaef0544a2caac58c3240c8a4ed58cfdce5bd Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sat, 19 Mar 2016 21:28:32 +0100 Subject: [PATCH 10/13] fix bug that prevents song book entries to be imported. --- openlp/plugins/songs/lib/importers/songimport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/songs/lib/importers/songimport.py b/openlp/plugins/songs/lib/importers/songimport.py index 1b4550244..54c82da29 100644 --- a/openlp/plugins/songs/lib/importers/songimport.py +++ b/openlp/plugins/songs/lib/importers/songimport.py @@ -371,7 +371,7 @@ class SongImport(QtCore.QObject): song_book = self.manager.get_object_filtered(Book, Book.name == self.song_book_name) if song_book is None: song_book = Book.populate(name=self.song_book_name, publisher=self.song_book_pub) - song.book = song_book + song.add_songbook_entry(song_book, song.song_number) for topic_text in self.topics: if not topic_text: continue From dca65148c48a2a10d976d987058e56b5896088da Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sat, 19 Mar 2016 21:42:59 +0100 Subject: [PATCH 11/13] forgot to add a file and made some pep8 fixes --- openlp/plugins/songs/lib/importers/opspro.py | 20 ++++++------ .../openlp_plugins/songs/test_opsproimport.py | 11 ------- .../resources/opsprosongs/amazing grace3.txt | 31 +++++++++++++++++++ 3 files changed, 42 insertions(+), 20 deletions(-) create mode 100644 tests/resources/opsprosongs/amazing grace3.txt diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index d1ffd7aad..f95611fe5 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -53,7 +53,8 @@ class OpsProImport(SongImport): """ password = self.extract_mdb_password() try: - conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s;PWD=%s' % (self.import_source, password)) + conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s;PWD=%s' % (self.import_source, + password)) except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e: log.warning('Unable to connect the OPS Pro database %s. %s', self.import_source, str(e)) # Unfortunately no specific exception type @@ -68,7 +69,8 @@ class OpsProImport(SongImport): for song in songs: if self.stop_import_flag: break - cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = %d AND Type < 2 ORDER BY Type DESC' % song.ID) + cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = %d AND Type < 2 ' + 'ORDER BY Type DESC' % song.ID) lyrics = cursor.fetchone() cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ' 'ON Category.ID = SongCategory.CategoryID WHERE SongCategory.SongID = %d ' @@ -215,7 +217,7 @@ class OpsProImport(SongImport): http://tutorialsto.com/database/access/crack-access-*.-mdb-all-current-versions-of-the-password.html """ # The definition of 13 bytes as the source XOR Access2000. Encrypted with the corresponding signs are 0x13 - xor_pattern_2k = (0xa1, 0xec, 0x7a, 0x9c, 0xe1, 0x28, 0x34, 0x8a, 0x73, 0x7b, 0xd2, 0xdf, 0x50) + xor_pattern_2k = (0xa1, 0xec, 0x7a, 0x9c, 0xe1, 0x28, 0x34, 0x8a, 0x73, 0x7b, 0xd2, 0xdf, 0x50) # Access97 XOR of the source xor_pattern_97 = (0x86, 0xfb, 0xec, 0x37, 0x5d, 0x44, 0x9c, 0xfa, 0xc6, 0x5e, 0x28, 0xe6, 0x13) mdb = open(self.import_source, 'rb') @@ -225,14 +227,14 @@ class OpsProImport(SongImport): mdb.seek(0x62) EncrypFlag = struct.unpack('B', mdb.read(1))[0] # Get encrypted password - mdb.seek(0x42); + mdb.seek(0x42) encrypted_password = mdb.read(26) mdb.close() # "Decrypt" the password based on the version decrypted_password = '' if version < 0x01: # Access 97 - if int (encrypted_password[0] ^ xor_pattern_97[0]) == 0: + if int(encrypted_password[0] ^ xor_pattern_97[0]) == 0: # No password decrypted_password = '' else: @@ -241,12 +243,12 @@ class OpsProImport(SongImport): else: # Access 2000 or 2002 for j in range(0, 12): - if j% 2 == 0: + if j % 2 == 0: # Every byte with a different sign or encrypt. Encryption signs here for the 0x13 - t1 = chr (0x13 ^ EncrypFlag ^ encrypted_password[j * 2] ^ xor_pattern_2k[j]) + t1 = chr(0x13 ^ EncrypFlag ^ encrypted_password[j * 2] ^ xor_pattern_2k[j]) else: - t1 = chr(encrypted_password[j * 2] ^ xor_pattern_2k[j]); - decrypted_password = decrypted_password + t1; + t1 = chr(encrypted_password[j * 2] ^ xor_pattern_2k[j]) + decrypted_password = decrypted_password + t1 if ord(decrypted_password[1]) < 0x20 or ord(decrypted_password[1]) > 0x7e: decrypted_password = '' return decrypted_password diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index b2f9371dc..294766f49 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -33,17 +33,6 @@ from openlp.plugins.songs.lib.importers.opspro import OpsProImport TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'opsprosongs')) -class TestRecord(object): - """ - Microsoft Access Driver is not available on non Microsoft Systems for this reason the :class:`TestRecord` is used - to simulate a recordset that would be returned by pyobdc. - """ - def __init__(self, id, field, value): - # The case of the following instance variables is important as it needs to be the same as the ones in use in the - # WorshipCenter Pro database. - self.ID = id - self.Field = field - self.Value = value class TestOpsProSongImport(TestCase): """ diff --git a/tests/resources/opsprosongs/amazing grace3.txt b/tests/resources/opsprosongs/amazing grace3.txt new file mode 100644 index 000000000..b42ade8f5 --- /dev/null +++ b/tests/resources/opsprosongs/amazing grace3.txt @@ -0,0 +1,31 @@ +Amazing grace! How sweet the sound! +That saved a wretch like me! +I once was lost, but now am found; +Was blind, but now I see. + +[taal a] +'Twas grace that taught my heart to fear, +And grace my fears relieved. +[taal b] +How precious did that grace appear, +The hour I first believed. + +[trans off] +The Lord has promised good to me, +His Word my hope secures. +[trans on] +He will my shield and portion be +As long as life endures. + +[vertaal uit] +Thro' many dangers, toils and snares +I have already come. +[vertaal aan] +'Tis grace that brought me safe thus far, +And grace will lead me home. + +[end] +When we've been there ten thousand years, +Bright shining as the sun, +We've no less days to sing God's praise, +Than when we first begun. From 45b122f768313b5a121a4284ee87d2a24821a954 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sun, 20 Mar 2016 09:28:41 +0100 Subject: [PATCH 12/13] make stuff prettier... --- openlp/plugins/songs/lib/importer.py | 63 +++++++++---------- openlp/plugins/songs/lib/importers/opspro.py | 12 ++-- .../openlp_plugins/songs/test_opsproimport.py | 15 +++-- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/openlp/plugins/songs/lib/importer.py b/openlp/plugins/songs/lib/importer.py index d7837ce7d..a09bf5ea6 100644 --- a/openlp/plugins/songs/lib/importer.py +++ b/openlp/plugins/songs/lib/importer.py @@ -48,7 +48,6 @@ from .importers.powerpraise import PowerPraiseImport from .importers.presentationmanager import PresentationManagerImport from .importers.lyrix import LyrixImport from .importers.videopsalm import VideoPsalmImport -from .importers.opspro import OpsProImport log = logging.getLogger(__name__) @@ -82,10 +81,10 @@ if is_win(): HAS_OPSPRO = False if is_win(): try: - from .importers.opspro import OpsProImport + from .importers.opspro import OPSProImport HAS_OPSPRO = True except ImportError: - log.exception('Error importing %s', 'OpsProImport') + log.exception('Error importing %s', 'OPSProImport') class SongFormatSelect(object): @@ -164,21 +163,21 @@ class SongFormat(object): Lyrix = 9 MediaShout = 10 OpenSong = 11 - PowerPraise = 12 - PowerSong = 13 - PresentationManager = 14 - ProPresenter = 15 - SongBeamer = 16 - SongPro = 17 - SongShowPlus = 18 - SongsOfFellowship = 19 - SundayPlus = 20 - VideoPsalm = 21 - WordsOfWorship = 22 - WorshipAssistant = 23 - WorshipCenterPro = 24 - ZionWorx = 25 - OPSPro = 26 + OPSPro = 12 + PowerPraise = 13 + PowerSong = 14 + PresentationManager = 15 + ProPresenter = 16 + SongBeamer = 17 + SongPro = 18 + SongShowPlus = 19 + SongsOfFellowship = 20 + SundayPlus = 21 + VideoPsalm = 22 + WordsOfWorship = 23 + WorshipAssistant = 24 + WorshipCenterPro = 25 + ZionWorx = 26 # Set optional attribute defaults __defaults__ = { @@ -281,6 +280,17 @@ class SongFormat(object): 'name': WizardStrings.OS, 'prefix': 'openSong' }, + OPSPro: { + 'name': 'OPS Pro', + 'prefix': 'OPSPro', + 'canDisable': True, + 'selectMode': SongFormatSelect.SingleFile, + 'filter': '%s (*.mdb)' % translate('SongsPlugin.ImportWizardForm', 'OPS Pro database'), + 'disabledLabelText': translate('SongsPlugin.ImportWizardForm', + 'The OPS Pro importer is only supported on Windows. It has been ' + 'disabled due to a missing Python module. If you want to use this ' + 'importer, you will need to install the "pyodbc" module.') + }, PowerPraise: { 'class': PowerPraiseImport, 'name': 'PowerPraise', @@ -391,17 +401,6 @@ class SongFormat(object): 'First convert your ZionWorx database to a CSV text file, as ' 'explained in the User Manual.') - }, - OPSPro: { - 'name': 'OPS Pro', - 'prefix': 'OPSPro', - 'canDisable': True, - 'selectMode': SongFormatSelect.SingleFile, - 'filter': '%s (*.mdb)' % translate('SongsPlugin.ImportWizardForm', 'OPS Pro database'), - 'disabledLabelText': translate('SongsPlugin.ImportWizardForm', - 'The OPS Pro importer is only supported on Windows. It has been ' - 'disabled due to a missing Python module. If you want to use this ' - 'importer, you will need to install the "pyodbc" module.') } } @@ -423,6 +422,7 @@ class SongFormat(object): SongFormat.Lyrix, SongFormat.MediaShout, SongFormat.OpenSong, + SongFormat.OPSPro, SongFormat.PowerPraise, SongFormat.PowerSong, SongFormat.PresentationManager, @@ -436,8 +436,7 @@ class SongFormat(object): SongFormat.WordsOfWorship, SongFormat.WorshipAssistant, SongFormat.WorshipCenterPro, - SongFormat.ZionWorx, - SongFormat.OPSPro + SongFormat.ZionWorx ]) @staticmethod @@ -488,7 +487,7 @@ if HAS_WORSHIPCENTERPRO: SongFormat.set(SongFormat.WorshipCenterPro, 'class', WorshipCenterProImport) SongFormat.set(SongFormat.OPSPro, 'availability', HAS_OPSPRO) if HAS_OPSPRO: - SongFormat.set(SongFormat.OPSPro, 'class', OpsProImport) + SongFormat.set(SongFormat.OPSPro, 'class', OPSProImport) __all__ = ['SongFormat', 'SongFormatSelect'] diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index f95611fe5..df8ff2ab8 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -25,9 +25,7 @@ a OPS Pro database into the OpenLP database. """ import logging import re -import os -if os.name == 'nt': - import pyodbc +import pyodbc import struct from openlp.core.common import translate @@ -36,16 +34,16 @@ from openlp.plugins.songs.lib.importers.songimport import SongImport log = logging.getLogger(__name__) -class OpsProImport(SongImport): +class OPSProImport(SongImport): """ - The :class:`OpsProImport` class provides the ability to import the + The :class:`OPSProImport` class provides the ability to import the WorshipCenter Pro Access Database """ def __init__(self, manager, **kwargs): """ Initialise the WorshipCenter Pro importer. """ - super(OpsProImport, self).__init__(manager, **kwargs) + super(OPSProImport, self).__init__(manager, **kwargs) def do_import(self): """ @@ -58,7 +56,7 @@ class OpsProImport(SongImport): except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e: log.warning('Unable to connect the OPS Pro database %s. %s', self.import_source, str(e)) # Unfortunately no specific exception type - self.log_error(self.import_source, translate('SongsPlugin.OpsProImport', + self.log_error(self.import_source, translate('SongsPlugin.OPSProImport', 'Unable to connect the OPS Pro database.')) return cursor = conn.cursor() diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py index 294766f49..cee21f66e 100644 --- a/tests/functional/openlp_plugins/songs/test_opsproimport.py +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -26,10 +26,13 @@ import os import json from unittest import TestCase, SkipTest +if os.name != 'nt': + raise SkipTest('Not Windows, skipping test') + from tests.functional import patch, MagicMock from openlp.core.common import Registry -from openlp.plugins.songs.lib.importers.opspro import OpsProImport +from openlp.plugins.songs.lib.importers.opspro import OPSProImport TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'opsprosongs')) @@ -53,7 +56,7 @@ class TestOpsProSongImport(TestCase): mocked_manager = MagicMock() # WHEN: An importer object is created - importer = OpsProImport(mocked_manager, filenames=[]) + importer = OPSProImport(mocked_manager, filenames=[]) # THEN: The importer object should not be None self.assertIsNotNone(importer, 'Import should not be none') @@ -65,7 +68,7 @@ class TestOpsProSongImport(TestCase): """ # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry mocked_manager = MagicMock() - importer = OpsProImport(mocked_manager, filenames=[]) + importer = OPSProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() song, lyrics = self._build_test_data('you are so faithfull.txt', False) @@ -85,7 +88,7 @@ class TestOpsProSongImport(TestCase): """ # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry mocked_manager = MagicMock() - importer = OpsProImport(mocked_manager, filenames=[]) + importer = OPSProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() song, lyrics = self._build_test_data('amazing grace.txt', False) @@ -105,7 +108,7 @@ class TestOpsProSongImport(TestCase): """ # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry mocked_manager = MagicMock() - importer = OpsProImport(mocked_manager, filenames=[]) + importer = OPSProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() song, lyrics = self._build_test_data('amazing grace2.txt', True) @@ -125,7 +128,7 @@ class TestOpsProSongImport(TestCase): """ # GIVEN: A mocked out SongImport class, a mocked out "manager" and a mocked song and lyrics entry mocked_manager = MagicMock() - importer = OpsProImport(mocked_manager, filenames=[]) + importer = OPSProImport(mocked_manager, filenames=[]) importer.finish = MagicMock() song, lyrics = self._build_test_data('amazing grace3.txt', True) From 66cea24673f29b777aaa66e0f534ee99ce7f1127 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sun, 20 Mar 2016 21:23:01 +0100 Subject: [PATCH 13/13] some more fixes --- openlp/plugins/songs/lib/importers/opspro.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index df8ff2ab8..08bb1bcc9 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -67,6 +67,7 @@ class OPSProImport(SongImport): for song in songs: if self.stop_import_flag: break + # Type means: 0=Original, 1=Projection, 2=Own cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = %d AND Type < 2 ' 'ORDER BY Type DESC' % song.ID) lyrics = cursor.fetchone() @@ -74,7 +75,12 @@ class OPSProImport(SongImport): 'ON Category.ID = SongCategory.CategoryID WHERE SongCategory.SongID = %d ' 'ORDER BY CategoryName' % song.ID) topics = cursor.fetchall() - self.process_song(song, lyrics, topics) + try: + self.process_song(song, lyrics, topics) + except Exception as e: + self.log_error(self.import_source, + translate('SongsPlugin.OPSProImport', '"%s" could not be imported. %s') + % (song.Title, e)) def process_song(self, song, lyrics, topics): """ @@ -146,9 +152,9 @@ class OPSProImport(SongImport): verse_def = 'e' verse_text = re.sub('^\[slot\]\r\n', '', verse_text, flags=re.IGNORECASE) # Replace the join tag with line breaks - verse_text = re.sub('\[join\]', '', verse_text) + verse_text = verse_text.replace('[join]', '') # Replace the split tag with line breaks and an optional split - verse_text = re.sub('\[split\]', '\r\n[---]', verse_text) + verse_text = re.sub('\[splits?\]', '\r\n[---]', verse_text) # Handle translations if lyrics.IsDualLanguage: verse_text = self.handle_translation(verse_text) @@ -185,6 +191,8 @@ class OPSProImport(SongImport): elif verse_text_lines[idx] == '[taal b]': language = 'b' idx += 1 + if not idx < len(verse_text_lines): + break # Handle the text based on whether translation is off or on if language: if language == 'b':