From e67ad21740a82be508a6e48680955240a15d3aae Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Mon, 7 Mar 2016 23:27:28 +0100 Subject: [PATCH] Getting closer... --- openlp/plugins/songs/lib/importer.py | 11 ++ openlp/plugins/songs/lib/importers/opspro.py | 135 ++++++++++++-- .../openlp_plugins/songs/test_opsproimport.py | 166 ++++++++++++++++++ 3 files changed, 296 insertions(+), 16 deletions(-) create mode 100644 tests/functional/openlp_plugins/songs/test_opsproimport.py diff --git a/openlp/plugins/songs/lib/importer.py b/openlp/plugins/songs/lib/importer.py index 409bc897a..d7837ce7d 100644 --- a/openlp/plugins/songs/lib/importer.py +++ b/openlp/plugins/songs/lib/importer.py @@ -48,6 +48,7 @@ from .importers.powerpraise import PowerPraiseImport from .importers.presentationmanager import PresentationManagerImport from .importers.lyrix import LyrixImport from .importers.videopsalm import VideoPsalmImport +from .importers.opspro import OpsProImport log = logging.getLogger(__name__) @@ -78,6 +79,13 @@ if is_win(): HAS_WORSHIPCENTERPRO = True except ImportError: log.exception('Error importing %s', 'WorshipCenterProImport') +HAS_OPSPRO = False +if is_win(): + try: + from .importers.opspro import OpsProImport + HAS_OPSPRO = True + except ImportError: + log.exception('Error importing %s', 'OpsProImport') class SongFormatSelect(object): @@ -478,6 +486,9 @@ if HAS_MEDIASHOUT: SongFormat.set(SongFormat.WorshipCenterPro, 'availability', HAS_WORSHIPCENTERPRO) if HAS_WORSHIPCENTERPRO: SongFormat.set(SongFormat.WorshipCenterPro, 'class', WorshipCenterProImport) +SongFormat.set(SongFormat.OPSPro, 'availability', HAS_OPSPRO) +if HAS_OPSPRO: + SongFormat.set(SongFormat.OPSPro, 'class', OpsProImport) __all__ = ['SongFormat', 'SongFormatSelect'] diff --git a/openlp/plugins/songs/lib/importers/opspro.py b/openlp/plugins/songs/lib/importers/opspro.py index e0e727261..957f16f81 100644 --- a/openlp/plugins/songs/lib/importers/opspro.py +++ b/openlp/plugins/songs/lib/importers/opspro.py @@ -25,7 +25,10 @@ a OPS Pro database into the OpenLP database. """ import logging import re -import pyodbc +import os +if os.name == 'nt': + import pyodbc +import struct from openlp.core.common import translate from openlp.plugins.songs.lib.importers.songimport import SongImport @@ -48,8 +51,9 @@ class OpsProImport(SongImport): """ Receive a single file to import. """ + password = self.extract_mdb_password() try: - conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s' % self.import_source) + conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s;PWD=%s' % (self.import_source, password)) except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e: log.warning('Unable to connect the OPS Pro database %s. %s', self.import_source, str(e)) # Unfortunately no specific exception type @@ -57,31 +61,130 @@ class OpsProImport(SongImport): 'Unable to connect the OPS Pro database.')) return cursor = conn.cursor() - cursor.execute('SELECT Song.ID, Song.SongNumber, Song.SongBookID, Song.Title, Song.CopyrightText, Version, Origin FROM Song ORDER BY Song.Title') + cursor.execute('SELECT Song.ID, SongNumber, SongBookName, Title, CopyrightText, Version, Origin FROM Song ' + 'LEFT JOIN SongBook ON Song.SongBookID = SongBook.ID ORDER BY Title') songs = cursor.fetchall() self.import_wizard.progress_bar.setMaximum(len(songs)) for song in songs: if self.stop_import_flag: break - cursor.execute('SELECT Lyrics FROM Lyrics WHERE SongID = %s ORDER BY Type, Number' - % song.ID) - verses = cursor.fetchall() - cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ON SongCategory.CategoryID = Category.CategoryID ' - 'WHERE SongCategory.SongID = %s' % song.ID) + cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = %d AND Type < 2 ORDER BY Type DESC' % song.ID) + lyrics = cursor.fetchone() + cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ' + 'ON Category.ID = SongCategory.CategoryID WHERE SongCategory.SongID = %d ' + 'ORDER BY CategoryName' % song.ID) topics = cursor.fetchall() + self.process_song(song, lyrics, topics) + break - - self.process_song(song, verses, topics) - - def process_song(self, song, verses, verse_order, topics): + def process_song(self, song, lyrics, topics): """ Create the song, i.e. title, verse etc. """ self.set_defaults() self.title = song.Title - self.parse_author(song.CopyrightText) - self.add_copyright(song.Origin) + if song.CopyrightText: + self.parse_author(song.CopyrightText) + if song.Origin: + self.comments = song.Origin + if song.SongBookName: + self.song_book_name = song.SongBookName + if song.SongNumber: + self.song_number = song.SongNumber for topic in topics: - self.topics.append(topic.Name) - self.add_verse(verses.Lyrics) + self.topics.append(topic.CategoryName) + # Try to split lyrics based on various rules + print(song.ID) + if lyrics: + lyrics_text = lyrics.Lyrics + # Remove whitespaces around the join-tag to keep verses joint + lyrics_text = re.sub('\w*\[join\]\w*', '[join]', lyrics_text, flags=re.IGNORECASE) + lyrics_text = re.sub('\w*\[splits?\]\w*', '[split]', lyrics_text, flags=re.IGNORECASE) + verses = lyrics_text.split('\r\n\r\n') + verse_tag_defs = {} + verse_tag_texts = {} + chorus = '' + for verse_text in verses: + verse_def = 'v' + # Try to detect verse number + verse_number = re.match('^(\d+)\r\n', verse_text) + if verse_number: + verse_text = re.sub('^\d+\r\n', '', verse_text) + verse_def = 'v' + verse_number.group(1) + # Detect verse tags + elif re.match('^.*?:\r\n', verse_text): + tag_match = re.match('^(.*?)(\w.+)?:\r\n(.*)', verse_text) + tag = tag_match.group(1) + verse_text = tag_match.group(3) + if 'refrain' in tag.lower(): + verse_def = 'c' + elif 'bridge' in tag.lower(): + verse_def = 'b' + verse_tag_defs[tag] = verse_def + elif re.match('^\(.*\)$', verse_text): + tag_match = re.match('^\((.*)\)$', verse_text) + tag = tag_match.group(1) + if tag in verse_tag_defs: + verse_text = verse_tag_texts[tag] + verse_def = verse_tag_defs[tag] + # Try to detect end tag + elif re.match('^\[slot\]\r\n', verse_text, re.IGNORECASE): + verse_def = 'e' + verse_text = re.sub('^\[slot\]\r\n', '', verse_text, flags=re.IGNORECASE) + # Handle tags + # Replace the join tag with line breaks + verse_text = re.sub('\[join\]', '\r\n\r\n\r\n', verse_text) + # Replace the split tag with line breaks and an optional split + verse_text = re.sub('\[split\]', '\r\n\r\n[---]\r\n', verse_text) + # Handle translations + #if lyrics.IsDualLanguage: + # ... + + # Remove comments + verse_text = re.sub('\(.*?\)\r\n', '', verse_text, flags=re.IGNORECASE) + self.add_verse(verse_text, verse_def) + print(verse_def) + print(verse_text) self.finish() + + def extract_mdb_password(self): + """ + Extract password from mdb. Based on code from + http://tutorialsto.com/database/access/crack-access-*.-mdb-all-current-versions-of-the-password.html + """ + # The definition of 13 bytes as the source XOR Access2000. Encrypted with the corresponding signs are 0x13 + xor_pattern_2k = (0xa1, 0xec, 0x7a, 0x9c, 0xe1, 0x28, 0x34, 0x8a, 0x73, 0x7b, 0xd2, 0xdf, 0x50) + # Access97 XOR of the source + xor_pattern_97 = (0x86, 0xfb, 0xec, 0x37, 0x5d, 0x44, 0x9c, 0xfa, 0xc6, 0x5e, 0x28, 0xe6, 0x13) + mdb = open(self.import_source, 'rb') + mdb.seek(0x14) + version = struct.unpack('B', mdb.read(1))[0] + # Get encrypted logo + mdb.seek(0x62) + EncrypFlag = struct.unpack('B', mdb.read(1))[0] + # Get encrypted password + mdb.seek(0x42); + encrypted_password = mdb.read(26) + mdb.close() + # "Decrypt" the password based on the version + decrypted_password = '' + if version < 0x01: + # Access 97 + if int (encrypted_password[0] ^ xor_pattern_97[0]) == 0: + # No password + decrypted_password = '' + else: + for j in range(0, 12): + decrypted_password = decrypted_password + chr(encrypted_password[j] ^ xor_pattern_97[j]) + else: + # Access 2000 or 2002 + for j in range(0, 12): + if j% 2 == 0: + # Every byte with a different sign or encrypt. Encryption signs here for the 0x13 + t1 = chr (0x13 ^ EncrypFlag ^ encrypted_password[j * 2] ^ xor_pattern_2k[j]) + else: + t1 = chr(encrypted_password[j * 2] ^ xor_pattern_2k[j]); + decrypted_password = decrypted_password + t1; + if ord(decrypted_password[1]) < 0x20 or ord(decrypted_password[1]) > 0x7e: + decrypted_password = '' + return decrypted_password diff --git a/tests/functional/openlp_plugins/songs/test_opsproimport.py b/tests/functional/openlp_plugins/songs/test_opsproimport.py new file mode 100644 index 000000000..8289ae0dc --- /dev/null +++ b/tests/functional/openlp_plugins/songs/test_opsproimport.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +This module contains tests for the WorshipCenter Pro song importer. +""" +import os +from unittest import TestCase, SkipTest + +from tests.functional import patch, MagicMock + +from openlp.core.common import Registry +from openlp.plugins.songs.lib.importers.opspro import OpsProImport + + +class TestRecord(object): + """ + Microsoft Access Driver is not available on non Microsoft Systems for this reason the :class:`TestRecord` is used + to simulate a recordset that would be returned by pyobdc. + """ + def __init__(self, id, field, value): + # The case of the following instance variables is important as it needs to be the same as the ones in use in the + # WorshipCenter Pro database. + self.ID = id + self.Field = field + self.Value = value + + +RECORDSET_TEST_DATA = [TestRecord(1, 'TITLE', 'Amazing Grace'), + TestRecord(1, 'AUTHOR', 'John Newton'), + TestRecord(1, 'CCLISONGID', '12345'), + TestRecord(1, 'COMMENTS', 'The original version'), + TestRecord(1, 'COPY', 'Public Domain'), + TestRecord( + 1, 'LYRICS', + 'Amazing grace! How&crlf;sweet the sound&crlf;That saved a wretch like me!&crlf;' + 'I once was lost,&crlf;but now am found;&crlf;Was blind, but now I see.&crlf;&crlf;' + '\'Twas grace that&crlf;taught my heart to fear,&crlf;And grace my fears relieved;&crlf;' + 'How precious did&crlf;that grace appear&crlf;The hour I first believed.&crlf;&crlf;' + 'Through many dangers,&crlf;toils and snares,&crlf;I have already come;&crlf;' + '\'Tis grace hath brought&crlf;me safe thus far,&crlf;' + 'And grace will lead me home.&crlf;&crlf;The Lord has&crlf;promised good to me,&crlf;' + 'His Word my hope secures;&crlf;He will my Shield&crlf;and Portion be,&crlf;' + 'As long as life endures.&crlf;&crlf;Yea, when this flesh&crlf;and heart shall fail,&crlf;' + 'And mortal life shall cease,&crlf;I shall possess,&crlf;within the veil,&crlf;' + 'A life of joy and peace.&crlf;&crlf;The earth shall soon&crlf;dissolve like snow,&crlf;' + 'The sun forbear to shine;&crlf;But God, Who called&crlf;me here below,&crlf;' + 'Shall be forever mine.&crlf;&crlf;When we\'ve been there&crlf;ten thousand years,&crlf;' + 'Bright shining as the sun,&crlf;We\'ve no less days to&crlf;sing God\'s praise&crlf;' + 'Than when we\'d first begun.&crlf;&crlf;'), + TestRecord(2, 'TITLE', 'Beautiful Garden Of Prayer, The'), + TestRecord( + 2, 'LYRICS', + 'There\'s a garden where&crlf;Jesus is waiting,&crlf;' + 'There\'s a place that&crlf;is wondrously fair,&crlf;For it glows with the&crlf;' + 'light of His presence.&crlf;\'Tis the beautiful&crlf;garden of prayer.&crlf;&crlf;' + 'Oh, the beautiful garden,&crlf;the garden of prayer!&crlf;Oh, the beautiful&crlf;' + 'garden of prayer!&crlf;There my Savior awaits,&crlf;and He opens the gates&crlf;' + 'To the beautiful&crlf;garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;' + 'Jesus is waiting,&crlf;And I go with my&crlf;burden and care,&crlf;' + 'Just to learn from His&crlf;lips words of comfort&crlf;In the beautiful&crlf;' + 'garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;Jesus is waiting,&crlf;' + 'And He bids you to come,&crlf;meet Him there;&crlf;Just to bow and&crlf;' + 'receive a new blessing&crlf;In the beautiful&crlf;garden of prayer.&crlf;&crlf;')] +SONG_TEST_DATA = [{'title': 'Amazing Grace', + 'verses': [ + ('Amazing grace! How\nsweet the sound\nThat saved a wretch like me!\nI once was lost,\n' + 'but now am found;\nWas blind, but now I see.'), + ('\'Twas grace that\ntaught my heart to fear,\nAnd grace my fears relieved;\nHow precious did\n' + 'that grace appear\nThe hour I first believed.'), + ('Through many dangers,\ntoils and snares,\nI have already come;\n\'Tis grace hath brought\n' + 'me safe thus far,\nAnd grace will lead me home.'), + ('The Lord has\npromised good to me,\nHis Word my hope secures;\n' + 'He will my Shield\nand Portion be,\nAs long as life endures.'), + ('Yea, when this flesh\nand heart shall fail,\nAnd mortal life shall cease,\nI shall possess,\n' + 'within the veil,\nA life of joy and peace.'), + ('The earth shall soon\ndissolve like snow,\nThe sun forbear to shine;\nBut God, Who called\n' + 'me here below,\nShall be forever mine.'), + ('When we\'ve been there\nten thousand years,\nBright shining as the sun,\n' + 'We\'ve no less days to\nsing God\'s praise\nThan when we\'d first begun.')], + 'author': 'John Newton', + 'comments': 'The original version', + 'copyright': 'Public Domain'}, + {'title': 'Beautiful Garden Of Prayer, The', + 'verses': [ + ('There\'s a garden where\nJesus is waiting,\nThere\'s a place that\nis wondrously fair,\n' + 'For it glows with the\nlight of His presence.\n\'Tis the beautiful\ngarden of prayer.'), + ('Oh, the beautiful garden,\nthe garden of prayer!\nOh, the beautiful\ngarden of prayer!\n' + 'There my Savior awaits,\nand He opens the gates\nTo the beautiful\ngarden of prayer.'), + ('There\'s a garden where\nJesus is waiting,\nAnd I go with my\nburden and care,\n' + 'Just to learn from His\nlips words of comfort\nIn the beautiful\ngarden of prayer.'), + ('There\'s a garden where\nJesus is waiting,\nAnd He bids you to come,\nmeet Him there;\n' + 'Just to bow and\nreceive a new blessing\nIn the beautiful\ngarden of prayer.')]}] + + +class TestOpsProSongImport(TestCase): + """ + Test the functions in the :mod:`opsproimport` module. + """ + def setUp(self): + """ + Create the registry + """ + Registry.create() + + @patch('openlp.plugins.songs.lib.importers.opspro.SongImport') + def create_importer_test(self, mocked_songimport): + """ + Test creating an instance of the OPS Pro file importer + """ + # GIVEN: A mocked out SongImport class, and a mocked out "manager" + mocked_manager = MagicMock() + + # WHEN: An importer object is created + importer = OpsProImport(mocked_manager, filenames=[]) + + # THEN: The importer object should not be None + self.assertIsNotNone(importer, 'Import should not be none') + + @patch('openlp.plugins.songs.lib.importers.opspro.SongImport') + def detect_chorus_test(self, mocked_songimport): + """ + Test importing lyrics with a chorus in OPS Pro + """ + # GIVEN: A mocked out SongImport class, and a mocked out "manager" + mocked_manager = MagicMock() + importer = OpsProImport(mocked_manager, filenames=[]) + song = MagicMock() + song.ID = 100 + song.SongNumber = 123 + song.SongBookName = 'The Song Book' + song.Title = 'Song Title' + song.CopyrightText = 'Music and text by me' + song.Version = '1' + song.Origin = '...' + lyrics = MagicMock() + lyrics.Lyrics = 'sd' + lyrics.Type = 1 + lyrics.IsDualLanguage = True + importer.finish = MagicMock() + + # WHEN: An importer object is created + importer.process_song(song, lyrics, []) + + # THEN: The importer object should not be None + print(importer.verses) + print(importer.verse_order_list) + self.assertIsNone(importer, 'Import should not be none') \ No newline at end of file