Getting closer...

This commit is contained in:
Tomas Groth 2016-03-07 23:27:28 +01:00
parent a66fcff441
commit e67ad21740
3 changed files with 296 additions and 16 deletions

View File

@ -48,6 +48,7 @@ from .importers.powerpraise import PowerPraiseImport
from .importers.presentationmanager import PresentationManagerImport
from .importers.lyrix import LyrixImport
from .importers.videopsalm import VideoPsalmImport
from .importers.opspro import OpsProImport
log = logging.getLogger(__name__)
@ -78,6 +79,13 @@ if is_win():
HAS_WORSHIPCENTERPRO = True
except ImportError:
log.exception('Error importing %s', 'WorshipCenterProImport')
HAS_OPSPRO = False
if is_win():
try:
from .importers.opspro import OpsProImport
HAS_OPSPRO = True
except ImportError:
log.exception('Error importing %s', 'OpsProImport')
class SongFormatSelect(object):
@ -478,6 +486,9 @@ if HAS_MEDIASHOUT:
SongFormat.set(SongFormat.WorshipCenterPro, 'availability', HAS_WORSHIPCENTERPRO)
if HAS_WORSHIPCENTERPRO:
SongFormat.set(SongFormat.WorshipCenterPro, 'class', WorshipCenterProImport)
SongFormat.set(SongFormat.OPSPro, 'availability', HAS_OPSPRO)
if HAS_OPSPRO:
SongFormat.set(SongFormat.OPSPro, 'class', OpsProImport)
__all__ = ['SongFormat', 'SongFormatSelect']

View File

@ -25,7 +25,10 @@ a OPS Pro database into the OpenLP database.
"""
import logging
import re
import pyodbc
import os
if os.name == 'nt':
import pyodbc
import struct
from openlp.core.common import translate
from openlp.plugins.songs.lib.importers.songimport import SongImport
@ -48,8 +51,9 @@ class OpsProImport(SongImport):
"""
Receive a single file to import.
"""
password = self.extract_mdb_password()
try:
conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s' % self.import_source)
conn = pyodbc.connect('DRIVER={Microsoft Access Driver (*.mdb)};DBQ=%s;PWD=%s' % (self.import_source, password))
except (pyodbc.DatabaseError, pyodbc.IntegrityError, pyodbc.InternalError, pyodbc.OperationalError) as e:
log.warning('Unable to connect the OPS Pro database %s. %s', self.import_source, str(e))
# Unfortunately no specific exception type
@ -57,31 +61,130 @@ class OpsProImport(SongImport):
'Unable to connect the OPS Pro database.'))
return
cursor = conn.cursor()
cursor.execute('SELECT Song.ID, Song.SongNumber, Song.SongBookID, Song.Title, Song.CopyrightText, Version, Origin FROM Song ORDER BY Song.Title')
cursor.execute('SELECT Song.ID, SongNumber, SongBookName, Title, CopyrightText, Version, Origin FROM Song '
'LEFT JOIN SongBook ON Song.SongBookID = SongBook.ID ORDER BY Title')
songs = cursor.fetchall()
self.import_wizard.progress_bar.setMaximum(len(songs))
for song in songs:
if self.stop_import_flag:
break
cursor.execute('SELECT Lyrics FROM Lyrics WHERE SongID = %s ORDER BY Type, Number'
% song.ID)
verses = cursor.fetchall()
cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory ON SongCategory.CategoryID = Category.CategoryID '
'WHERE SongCategory.SongID = %s' % song.ID)
cursor.execute('SELECT Lyrics, Type, IsDualLanguage FROM Lyrics WHERE SongID = %d AND Type < 2 ORDER BY Type DESC' % song.ID)
lyrics = cursor.fetchone()
cursor.execute('SELECT CategoryName FROM Category INNER JOIN SongCategory '
'ON Category.ID = SongCategory.CategoryID WHERE SongCategory.SongID = %d '
'ORDER BY CategoryName' % song.ID)
topics = cursor.fetchall()
self.process_song(song, lyrics, topics)
break
self.process_song(song, verses, topics)
def process_song(self, song, verses, verse_order, topics):
def process_song(self, song, lyrics, topics):
"""
Create the song, i.e. title, verse etc.
"""
self.set_defaults()
self.title = song.Title
if song.CopyrightText:
self.parse_author(song.CopyrightText)
self.add_copyright(song.Origin)
if song.Origin:
self.comments = song.Origin
if song.SongBookName:
self.song_book_name = song.SongBookName
if song.SongNumber:
self.song_number = song.SongNumber
for topic in topics:
self.topics.append(topic.Name)
self.add_verse(verses.Lyrics)
self.topics.append(topic.CategoryName)
# Try to split lyrics based on various rules
print(song.ID)
if lyrics:
lyrics_text = lyrics.Lyrics
# Remove whitespaces around the join-tag to keep verses joint
lyrics_text = re.sub('\w*\[join\]\w*', '[join]', lyrics_text, flags=re.IGNORECASE)
lyrics_text = re.sub('\w*\[splits?\]\w*', '[split]', lyrics_text, flags=re.IGNORECASE)
verses = lyrics_text.split('\r\n\r\n')
verse_tag_defs = {}
verse_tag_texts = {}
chorus = ''
for verse_text in verses:
verse_def = 'v'
# Try to detect verse number
verse_number = re.match('^(\d+)\r\n', verse_text)
if verse_number:
verse_text = re.sub('^\d+\r\n', '', verse_text)
verse_def = 'v' + verse_number.group(1)
# Detect verse tags
elif re.match('^.*?:\r\n', verse_text):
tag_match = re.match('^(.*?)(\w.+)?:\r\n(.*)', verse_text)
tag = tag_match.group(1)
verse_text = tag_match.group(3)
if 'refrain' in tag.lower():
verse_def = 'c'
elif 'bridge' in tag.lower():
verse_def = 'b'
verse_tag_defs[tag] = verse_def
elif re.match('^\(.*\)$', verse_text):
tag_match = re.match('^\((.*)\)$', verse_text)
tag = tag_match.group(1)
if tag in verse_tag_defs:
verse_text = verse_tag_texts[tag]
verse_def = verse_tag_defs[tag]
# Try to detect end tag
elif re.match('^\[slot\]\r\n', verse_text, re.IGNORECASE):
verse_def = 'e'
verse_text = re.sub('^\[slot\]\r\n', '', verse_text, flags=re.IGNORECASE)
# Handle tags
# Replace the join tag with line breaks
verse_text = re.sub('\[join\]', '\r\n\r\n\r\n', verse_text)
# Replace the split tag with line breaks and an optional split
verse_text = re.sub('\[split\]', '\r\n\r\n[---]\r\n', verse_text)
# Handle translations
#if lyrics.IsDualLanguage:
# ...
# Remove comments
verse_text = re.sub('\(.*?\)\r\n', '', verse_text, flags=re.IGNORECASE)
self.add_verse(verse_text, verse_def)
print(verse_def)
print(verse_text)
self.finish()
def extract_mdb_password(self):
"""
Extract password from mdb. Based on code from
http://tutorialsto.com/database/access/crack-access-*.-mdb-all-current-versions-of-the-password.html
"""
# The definition of 13 bytes as the source XOR Access2000. Encrypted with the corresponding signs are 0x13
xor_pattern_2k = (0xa1, 0xec, 0x7a, 0x9c, 0xe1, 0x28, 0x34, 0x8a, 0x73, 0x7b, 0xd2, 0xdf, 0x50)
# Access97 XOR of the source
xor_pattern_97 = (0x86, 0xfb, 0xec, 0x37, 0x5d, 0x44, 0x9c, 0xfa, 0xc6, 0x5e, 0x28, 0xe6, 0x13)
mdb = open(self.import_source, 'rb')
mdb.seek(0x14)
version = struct.unpack('B', mdb.read(1))[0]
# Get encrypted logo
mdb.seek(0x62)
EncrypFlag = struct.unpack('B', mdb.read(1))[0]
# Get encrypted password
mdb.seek(0x42);
encrypted_password = mdb.read(26)
mdb.close()
# "Decrypt" the password based on the version
decrypted_password = ''
if version < 0x01:
# Access 97
if int (encrypted_password[0] ^ xor_pattern_97[0]) == 0:
# No password
decrypted_password = ''
else:
for j in range(0, 12):
decrypted_password = decrypted_password + chr(encrypted_password[j] ^ xor_pattern_97[j])
else:
# Access 2000 or 2002
for j in range(0, 12):
if j% 2 == 0:
# Every byte with a different sign or encrypt. Encryption signs here for the 0x13
t1 = chr (0x13 ^ EncrypFlag ^ encrypted_password[j * 2] ^ xor_pattern_2k[j])
else:
t1 = chr(encrypted_password[j * 2] ^ xor_pattern_2k[j]);
decrypted_password = decrypted_password + t1;
if ord(decrypted_password[1]) < 0x20 or ord(decrypted_password[1]) > 0x7e:
decrypted_password = ''
return decrypted_password

View File

@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
###############################################################################
# OpenLP - Open Source Lyrics Projection #
# --------------------------------------------------------------------------- #
# Copyright (c) 2008-2016 OpenLP Developers #
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it #
# under the terms of the GNU General Public License as published by the Free #
# Software Foundation; version 2 of the License. #
# #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
# more details. #
# #
# You should have received a copy of the GNU General Public License along #
# with this program; if not, write to the Free Software Foundation, Inc., 59 #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
###############################################################################
"""
This module contains tests for the WorshipCenter Pro song importer.
"""
import os
from unittest import TestCase, SkipTest
from tests.functional import patch, MagicMock
from openlp.core.common import Registry
from openlp.plugins.songs.lib.importers.opspro import OpsProImport
class TestRecord(object):
"""
Microsoft Access Driver is not available on non Microsoft Systems for this reason the :class:`TestRecord` is used
to simulate a recordset that would be returned by pyobdc.
"""
def __init__(self, id, field, value):
# The case of the following instance variables is important as it needs to be the same as the ones in use in the
# WorshipCenter Pro database.
self.ID = id
self.Field = field
self.Value = value
RECORDSET_TEST_DATA = [TestRecord(1, 'TITLE', 'Amazing Grace'),
TestRecord(1, 'AUTHOR', 'John Newton'),
TestRecord(1, 'CCLISONGID', '12345'),
TestRecord(1, 'COMMENTS', 'The original version'),
TestRecord(1, 'COPY', 'Public Domain'),
TestRecord(
1, 'LYRICS',
'Amazing grace! How&crlf;sweet the sound&crlf;That saved a wretch like me!&crlf;'
'I once was lost,&crlf;but now am found;&crlf;Was blind, but now I see.&crlf;&crlf;'
'\'Twas grace that&crlf;taught my heart to fear,&crlf;And grace my fears relieved;&crlf;'
'How precious did&crlf;that grace appear&crlf;The hour I first believed.&crlf;&crlf;'
'Through many dangers,&crlf;toils and snares,&crlf;I have already come;&crlf;'
'\'Tis grace hath brought&crlf;me safe thus far,&crlf;'
'And grace will lead me home.&crlf;&crlf;The Lord has&crlf;promised good to me,&crlf;'
'His Word my hope secures;&crlf;He will my Shield&crlf;and Portion be,&crlf;'
'As long as life endures.&crlf;&crlf;Yea, when this flesh&crlf;and heart shall fail,&crlf;'
'And mortal life shall cease,&crlf;I shall possess,&crlf;within the veil,&crlf;'
'A life of joy and peace.&crlf;&crlf;The earth shall soon&crlf;dissolve like snow,&crlf;'
'The sun forbear to shine;&crlf;But God, Who called&crlf;me here below,&crlf;'
'Shall be forever mine.&crlf;&crlf;When we\'ve been there&crlf;ten thousand years,&crlf;'
'Bright shining as the sun,&crlf;We\'ve no less days to&crlf;sing God\'s praise&crlf;'
'Than when we\'d first begun.&crlf;&crlf;'),
TestRecord(2, 'TITLE', 'Beautiful Garden Of Prayer, The'),
TestRecord(
2, 'LYRICS',
'There\'s a garden where&crlf;Jesus is waiting,&crlf;'
'There\'s a place that&crlf;is wondrously fair,&crlf;For it glows with the&crlf;'
'light of His presence.&crlf;\'Tis the beautiful&crlf;garden of prayer.&crlf;&crlf;'
'Oh, the beautiful garden,&crlf;the garden of prayer!&crlf;Oh, the beautiful&crlf;'
'garden of prayer!&crlf;There my Savior awaits,&crlf;and He opens the gates&crlf;'
'To the beautiful&crlf;garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;'
'Jesus is waiting,&crlf;And I go with my&crlf;burden and care,&crlf;'
'Just to learn from His&crlf;lips words of comfort&crlf;In the beautiful&crlf;'
'garden of prayer.&crlf;&crlf;There\'s a garden where&crlf;Jesus is waiting,&crlf;'
'And He bids you to come,&crlf;meet Him there;&crlf;Just to bow and&crlf;'
'receive a new blessing&crlf;In the beautiful&crlf;garden of prayer.&crlf;&crlf;')]
SONG_TEST_DATA = [{'title': 'Amazing Grace',
'verses': [
('Amazing grace! How\nsweet the sound\nThat saved a wretch like me!\nI once was lost,\n'
'but now am found;\nWas blind, but now I see.'),
('\'Twas grace that\ntaught my heart to fear,\nAnd grace my fears relieved;\nHow precious did\n'
'that grace appear\nThe hour I first believed.'),
('Through many dangers,\ntoils and snares,\nI have already come;\n\'Tis grace hath brought\n'
'me safe thus far,\nAnd grace will lead me home.'),
('The Lord has\npromised good to me,\nHis Word my hope secures;\n'
'He will my Shield\nand Portion be,\nAs long as life endures.'),
('Yea, when this flesh\nand heart shall fail,\nAnd mortal life shall cease,\nI shall possess,\n'
'within the veil,\nA life of joy and peace.'),
('The earth shall soon\ndissolve like snow,\nThe sun forbear to shine;\nBut God, Who called\n'
'me here below,\nShall be forever mine.'),
('When we\'ve been there\nten thousand years,\nBright shining as the sun,\n'
'We\'ve no less days to\nsing God\'s praise\nThan when we\'d first begun.')],
'author': 'John Newton',
'comments': 'The original version',
'copyright': 'Public Domain'},
{'title': 'Beautiful Garden Of Prayer, The',
'verses': [
('There\'s a garden where\nJesus is waiting,\nThere\'s a place that\nis wondrously fair,\n'
'For it glows with the\nlight of His presence.\n\'Tis the beautiful\ngarden of prayer.'),
('Oh, the beautiful garden,\nthe garden of prayer!\nOh, the beautiful\ngarden of prayer!\n'
'There my Savior awaits,\nand He opens the gates\nTo the beautiful\ngarden of prayer.'),
('There\'s a garden where\nJesus is waiting,\nAnd I go with my\nburden and care,\n'
'Just to learn from His\nlips words of comfort\nIn the beautiful\ngarden of prayer.'),
('There\'s a garden where\nJesus is waiting,\nAnd He bids you to come,\nmeet Him there;\n'
'Just to bow and\nreceive a new blessing\nIn the beautiful\ngarden of prayer.')]}]
class TestOpsProSongImport(TestCase):
"""
Test the functions in the :mod:`opsproimport` module.
"""
def setUp(self):
"""
Create the registry
"""
Registry.create()
@patch('openlp.plugins.songs.lib.importers.opspro.SongImport')
def create_importer_test(self, mocked_songimport):
"""
Test creating an instance of the OPS Pro file importer
"""
# GIVEN: A mocked out SongImport class, and a mocked out "manager"
mocked_manager = MagicMock()
# WHEN: An importer object is created
importer = OpsProImport(mocked_manager, filenames=[])
# THEN: The importer object should not be None
self.assertIsNotNone(importer, 'Import should not be none')
@patch('openlp.plugins.songs.lib.importers.opspro.SongImport')
def detect_chorus_test(self, mocked_songimport):
"""
Test importing lyrics with a chorus in OPS Pro
"""
# GIVEN: A mocked out SongImport class, and a mocked out "manager"
mocked_manager = MagicMock()
importer = OpsProImport(mocked_manager, filenames=[])
song = MagicMock()
song.ID = 100
song.SongNumber = 123
song.SongBookName = 'The Song Book'
song.Title = 'Song Title'
song.CopyrightText = 'Music and text by me'
song.Version = '1'
song.Origin = '...'
lyrics = MagicMock()
lyrics.Lyrics = 'sd'
lyrics.Type = 1
lyrics.IsDualLanguage = True
importer.finish = MagicMock()
# WHEN: An importer object is created
importer.process_song(song, lyrics, [])
# THEN: The importer object should not be None
print(importer.verses)
print(importer.verse_order_list)
self.assertIsNone(importer, 'Import should not be none')