From f709c90f0ad3413f89563ed0c3b1cf2c345fcfd5 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 21:03:47 +0100 Subject: [PATCH 01/42] Start opensong importer --- openlp/plugins/songs/lib/opensongimport.py | 116 ++++++++++++++++++ openlp/plugins/songs/lib/test.opensong | 45 +++++++ .../plugins/songs/lib/test_opensongimport.py | 9 ++ 3 files changed, 170 insertions(+) create mode 100644 openlp/plugins/songs/lib/opensongimport.py create mode 100644 openlp/plugins/songs/lib/test.opensong create mode 100644 openlp/plugins/songs/lib/test_opensongimport.py diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py new file mode 100644 index 000000000..4dcc7f20d --- /dev/null +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin # +# Thompson, Jon Tibble, Carsten Tinggaard # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### + +import os +import re + +# from songimport import SongImport + +class opensongimport: + """ + Import songs exported from OpenSong - the format is described loosly here: + http://www.opensong.org/d/manual/song_file_format_specification + + However, it doesn't describe the section, so here's an attempt: + + Verses can be expressed in one of 2 ways: + + [v1]List of words + Another Line + + [v2]Some words for the 2nd verse + etc... + + + or: + + 1List of words + 2Some words for the 2nd Verse + + 1Another Line + 2etc... + + + Either or both forms can be used in one song. + + The [v1] labels can have either upper or loewr case Vs + Other labels can be used also: + C - Chorus + B - Bridge + + Guitar chords can be provided 'above' the lyrics (the line is preceeded by a'.') and _s can be used to signify long-drawn-out words: + + . A7 Bm + 1 Some____ Words + + Chords and _s are removed by this importer. + + The verses etc. are imported and tagged appropriately. + + The tag is used to populate the OpenLP verse + display order field. The Author and Copyright tags are also + imported to the appropriate places. + + """ + def __init__(self, songmanager): + """ + Initialise the class. Requires a songmanager class which is passed + to SongImport for writing song to disk + """ + self.songmanager=songmanager + self.song = None + + def osimport(self, filename): + """ + Process the OpenSong file + """ + self.new_song() + f=open(filename) + tree=objectify.parse(f) + root=tree.getroot() + print "Title", zroot.title + # data storage while importing + self.verses=[] + + + # xxx this is common with SOF + def new_song(self): + """ + A change of song. Store the old, create a new + ... but only if the last song was complete. If not, stick with it + """ + if self.song: + self.finish_verse() + if not self.song.check_complete(): + return + self.song.finish() + + self.song = SongImport(self.manager) + self.skip_to_close_bracket = False + self.is_chorus = False + self.italics = False + self.currentverse = u'' + + diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong new file mode 100644 index 000000000..0fde072ca --- /dev/null +++ b/openlp/plugins/songs/lib/test.opensong @@ -0,0 +1,45 @@ + + + Martins Test + Martin Thompson + 2010 Martin Thompson + 1 + V1 C1 V2 C2 B1 V1 + Blah + + + + + + + + + + + ;Comment +[V] +. A B C +1 v1 Line 1___ +2 v2 Line 1___ +. A B C7 +1 V1 Line 2 +2 V1 Line 2 + +[b1] + Bridge 1 + +[C1] + Chorus 1 + +[C2] + Chorus 2 + + \ No newline at end of file diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py new file mode 100644 index 000000000..79b8e9a69 --- /dev/null +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -0,0 +1,9 @@ +import openlp.plugins.songs.lib.opensongimport + +def test(): + o=opensongimport.opensongimport(0)# xxx needs a song manager here + o.osimport(u'test.opensong') + pass + +if __name__=="__main__": + test() From 2f077c398208d8b79b2bfc02ef1e7c1f2f45f802 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 22:24:49 +0100 Subject: [PATCH 02/42] Renamed so as not to conflict with the Pthon XML library which is still used by the Song XML class --- openlp/plugins/songs/lib/{xml.py => lyrics_xml.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openlp/plugins/songs/lib/{xml.py => lyrics_xml.py} (100%) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/lyrics_xml.py similarity index 100% rename from openlp/plugins/songs/lib/xml.py rename to openlp/plugins/songs/lib/lyrics_xml.py From fc27f8107e7b042043fc3145526aaae888b28188 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 22:25:50 +0100 Subject: [PATCH 03/42] First test looks promsing --- openlp/plugins/songs/lib/__init__.py | 1 + openlp/plugins/songs/lib/opensongimport.py | 111 ++++++++++++++---- openlp/plugins/songs/lib/test.opensong | 6 +- .../plugins/songs/lib/test_opensongimport.py | 12 +- 4 files changed, 99 insertions(+), 31 deletions(-) diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index 6c637ea9e..37e738664 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -97,3 +97,4 @@ from mediaitem import SongMediaItem from sofimport import SofImport from oooimport import OooImport from songimport import SongImport +from opensongimport import OpenSongImport diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 4dcc7f20d..7b066c858 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -26,9 +26,11 @@ import os import re -# from songimport import SongImport +from songimport import SongImport +from lxml.etree import Element +from lxml import objectify -class opensongimport: +class OpenSongImport: """ Import songs exported from OpenSong - the format is described loosly here: http://www.opensong.org/d/manual/song_file_format_specification @@ -82,35 +84,94 @@ class opensongimport: self.songmanager=songmanager self.song = None - def osimport(self, filename): + def do_import(self, filename): """ Process the OpenSong file """ - self.new_song() + self.song = SongImport(self.songmanager) f=open(filename) tree=objectify.parse(f) root=tree.getroot() - print "Title", zroot.title + # xxx this bit ought to be more "iterable"... esp. if song had attributes not getters and setters... + if root.copyright: + self.song.add_copyright(unicode(root.copyright)) + if root.author: + self.song.parse_author(unicode(root.author)) + if root.title: + self.song.set_title(unicode(root.title)) + if root.aka: + self.song.set_alternate_title(unicode(root.aka)) + if root.hymn_number: + self.song.set_song_number(unicode(root.hymn_number)) + # data storage while importing - self.verses=[] - - - # xxx this is common with SOF - def new_song(self): - """ - A change of song. Store the old, create a new - ... but only if the last song was complete. If not, stick with it - """ - if self.song: - self.finish_verse() - if not self.song.check_complete(): - return - self.song.finish() - - self.song = SongImport(self.manager) - self.skip_to_close_bracket = False - self.is_chorus = False - self.italics = False - self.currentverse = u'' + verses={} + lyrics=str(root.lyrics) + # xxx what to do if no presentation order - need to figure it out on the fly + for l in lyrics.split('\n'): + # remove comments + semicolon = l.find(';') + if semicolon >= 0: + l=l[:semicolon] + l=l.strip() + if l=='': + continue + # skip inline guitar chords + if l[0] == u'.': + continue + + # verse/chorus/etc. marker + if l[0] == u'[': + versetype=l[1].upper() + if not verses.has_key(versetype): + verses[versetype]={} + if l[2] != u']': + # there's a number to go with it - extract that as well + right_bracket=l.find(u']') + versenum=int(l[2:right_bracket]) + else: + versenum = None # allow error trap + continue + words=None + + # number at start of line => verse number + if l[0] >= u'0' and l[0] <= u'9': + versenum=int(l[0]) + words=l[1:].strip() + + if words is None and \ + versenum is not None and \ + versetype is not None: + words=l + if versenum is not None and \ + not verses[versetype].has_key(versenum): + verses[versetype][versenum]=[] # storage for lines in this verse + if words: + # remove the ____s from extended words + words=words.replace(u'_', u'') + verses[versetype][versenum].append(words) + # done parsing + print u'Title:', root.title + versetypes=verses.keys() + versetypes.sort() + versetags={} + for v in versetypes: + versenums=verses[v].keys() + versenums.sort() + for n in versenums: + versetag= u'%s%s' %(v,n) + lines=u'\n'.join(verses[v][n]) + self.song.verses.append([versetag, lines]) + versetags[versetag]=1 # keep track of what we have for error checking later + # now figure out the presentation order + if root.presentation: + order=unicode(root.presentation).split(u' ') + for tag in order: + if not versetags.has_key(tag): + raise OpenSongImportError + else: + self.song.verse_order_list.append(tag) + + self.song.print_song() diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index 0fde072ca..20206cecb 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -23,11 +23,11 @@ 2 v2 Line 1___ . A B C7 1 V1 Line 2 -2 V1 Line 2 +2 V2 Line 2 [b1] Bridge 1 - + Bridge 1 line 2 [C1] Chorus 1 @@ -42,4 +42,4 @@ - \ No newline at end of file + diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 79b8e9a69..655fc83cc 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -1,8 +1,14 @@ -import openlp.plugins.songs.lib.opensongimport +from openlp.plugins.songs.lib.opensongimport import OpenSongImport +from openlp.plugins.songs.lib.manager import SongManager def test(): - o=opensongimport.opensongimport(0)# xxx needs a song manager here - o.osimport(u'test.opensong') + manager=SongManager() + o=OpenSongImport(manager) + o.do_import(u'test.opensong') + # xxx need some more asserts in here to test it... + assert (1) + # now to XML + # asserts pass if __name__=="__main__": From 3417d85a58b27df804f06e235c688359007e276e Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 22:26:35 +0100 Subject: [PATCH 04/42] Elementtree now from lxml --- openlp/core/lib/songxmlhandler.py | 4 ++-- openlp/core/lib/xmlrootclass.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/core/lib/songxmlhandler.py b/openlp/core/lib/songxmlhandler.py index 76b01e376..5e5abc76b 100644 --- a/openlp/core/lib/songxmlhandler.py +++ b/openlp/core/lib/songxmlhandler.py @@ -39,9 +39,9 @@ The basic XML is of the format:: import logging -from xml.dom.minidom import Document -from xml.etree.ElementTree import ElementTree, XML, dump +from lxml.etree import ElementTree, XML, dump from xml.parsers.expat import ExpatError +from xml.dom.minidom import Document log = logging.getLogger(__name__) diff --git a/openlp/core/lib/xmlrootclass.py b/openlp/core/lib/xmlrootclass.py index 1ea1d41a2..de1339839 100644 --- a/openlp/core/lib/xmlrootclass.py +++ b/openlp/core/lib/xmlrootclass.py @@ -26,7 +26,7 @@ import os import sys -from xml.etree.ElementTree import ElementTree, XML +from lxml.etree import ElementTree, XML sys.path.append(os.path.abspath(os.path.join(u'.', u'..', u'..'))) From e8a397d3ecf246b519a2e29cd852b9a149d60d08 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 22 Jun 2010 21:41:31 +0100 Subject: [PATCH 05/42] Handles songs without a preliminary V tag --- openlp/plugins/songs/lib/opensongimport.py | 86 +++++++++++-------- openlp/plugins/songs/lib/test.opensong | 1 - .../plugins/songs/lib/test_opensongimport.py | 39 ++++++++- 3 files changed, 84 insertions(+), 42 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 7b066c858..dd3bc3df0 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -83,31 +83,36 @@ class OpenSongImport: """ self.songmanager=songmanager self.song = None - + def do_import(self, filename): + file=open(filename) + self.do_import_file(file) + + def do_import_file(self, file): """ Process the OpenSong file """ self.song = SongImport(self.songmanager) - f=open(filename) - tree=objectify.parse(f) + tree=objectify.parse(file) root=tree.getroot() - # xxx this bit ought to be more "iterable"... esp. if song had attributes not getters and setters... - if root.copyright: - self.song.add_copyright(unicode(root.copyright)) - if root.author: - self.song.parse_author(unicode(root.author)) - if root.title: - self.song.set_title(unicode(root.title)) - if root.aka: - self.song.set_alternate_title(unicode(root.aka)) - if root.hymn_number: - self.song.set_song_number(unicode(root.hymn_number)) + fields=dir(root) + decode={u'copyright':self.song.add_copyright, + u'author':self.song.parse_author, + u'title':self.song.set_title, + u'aka':self.song.set_alternate_title, + u'hymn_number':self.song.set_song_number} + for (attr, fn) in decode.items(): + if attr in fields: + fn(unicode(root.__getattr__(attr))) # data storage while importing verses={} - lyrics=str(root.lyrics) - # xxx what to do if no presentation order - need to figure it out on the fly + lyrics=unicode(root.lyrics) + # keep track of a "default" verse order, in case none is specified + our_verse_order=[] + verses_seen={} + # in the absence of any other indication, verses are the default, erm, versetype! + versetype=u'V' for l in lyrics.split('\n'): # remove comments semicolon = l.find(';') @@ -123,35 +128,39 @@ class OpenSongImport: # verse/chorus/etc. marker if l[0] == u'[': versetype=l[1].upper() - if not verses.has_key(versetype): - verses[versetype]={} if l[2] != u']': # there's a number to go with it - extract that as well right_bracket=l.find(u']') versenum=int(l[2:right_bracket]) + versetag=u'%s%d'%(versetype,versenum) else: versenum = None # allow error trap continue words=None - # number at start of line => verse number + # number at start of line.. it's verse number if l[0] >= u'0' and l[0] <= u'9': versenum=int(l[0]) words=l[1:].strip() - + versetag=u'%s%d'%(versetype,versenum) if words is None and \ versenum is not None and \ versetype is not None: words=l - if versenum is not None and \ - not verses[versetype].has_key(versenum): - verses[versetype][versenum]=[] # storage for lines in this verse + if versenum is not None: + if not verses.has_key(versetype): + verses[versetype]={} + if not verses[versetype].has_key(versenum): + verses[versetype][versenum]=[] # storage for lines in this verse + if not verses_seen.has_key(versetag): + verses_seen[versetag] = 1 + our_verse_order.append(versetag) if words: - # remove the ____s from extended words - words=words.replace(u'_', u'') + # Tidy text and remove the ____s from extended words + # words=self.song.tidy_text(words) + words=words.replace('_', '') verses[versetype][versenum].append(words) # done parsing - print u'Title:', root.title versetypes=verses.keys() versetypes.sort() versetags={} @@ -164,14 +173,17 @@ class OpenSongImport: self.song.verses.append([versetag, lines]) versetags[versetag]=1 # keep track of what we have for error checking later # now figure out the presentation order - if root.presentation: - order=unicode(root.presentation).split(u' ') - for tag in order: - if not versetags.has_key(tag): - raise OpenSongImportError - else: - self.song.verse_order_list.append(tag) - - - self.song.print_song() - + if 'presentation' in fields and root.presentation != u'': + order=unicode(root.presentation) + order=order.split() + else: + assert len(our_verse_order)>0 + order=our_verse_order + for tag in order: + if not versetags.has_key(tag): + raise OpenSongImportError + else: + self.song.verse_order_list.append(tag) + def finish(self): + """ Separate function, allows test suite to not pollute database""" + self.song.finish() diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index 20206cecb..ea6303c89 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -17,7 +17,6 @@ ;Comment -[V] . A B C 1 v1 Line 1___ 2 v2 Line 1___ diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 655fc83cc..e6d5eec2d 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -5,10 +5,41 @@ def test(): manager=SongManager() o=OpenSongImport(manager) o.do_import(u'test.opensong') - # xxx need some more asserts in here to test it... - assert (1) - # now to XML - # asserts + # o.finish() + o.song.print_song() + assert o.song.copyright == u'2010 Martin Thompson' + assert o.song.authors == [u'Martin Thompson'] + assert o.song.title == u'Martins Test' + assert o.song.alternate_title == u'' + assert o.song.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses + assert [u'C1', u'Chorus 1'] in o.song.verses + assert [u'C2', u'Chorus 2'] in o.song.verses + assert not [u'C3', u'Chorus 3'] in o.song.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses + assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'B1', u'V1'] + + o=OpenSongImport(manager) + o.do_import(u'test2.opensong') + # o.finish() + o.song.print_song() + assert o.song.copyright == u'2010 Martin Thompson' + assert o.song.authors == [u'Martin Thompson'] + assert o.song.title == u'Martins 2nd Test' + assert o.song.alternate_title == u'' + assert o.song.song_number == u'2' + print o.song.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses + assert [u'C1', u'Chorus 1'] in o.song.verses + assert [u'C2', u'Chorus 2'] in o.song.verses + assert not [u'C3', u'Chorus 3'] in o.song.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses + print o.song.verse_order_list + assert o.song.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] + + print "Tests passed" pass if __name__=="__main__": From 9c573d7bef6af30c32d3aeba5c8eb8dfac18d826 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 22 Jun 2010 22:24:24 +0100 Subject: [PATCH 06/42] Handles [C] followed by non-numbered lines, by assigned them all to [C1] --- openlp/plugins/songs/lib/opensongimport.py | 8 +++++--- openlp/plugins/songs/lib/test.opensong | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index dd3bc3df0..753aacfc7 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -48,6 +48,7 @@ class OpenSongImport: or: + [V] 1List of words 2Some words for the 2nd Verse @@ -55,9 +56,9 @@ class OpenSongImport: 2etc... - Either or both forms can be used in one song. + Either or both forms can be used in one song. The Number does not necessarily appear at the start of the line - The [v1] labels can have either upper or loewr case Vs + The [v1] labels can have either upper or lower case Vs Other labels can be used also: C - Chorus B - Bridge @@ -134,7 +135,7 @@ class OpenSongImport: versenum=int(l[2:right_bracket]) versetag=u'%s%d'%(versetype,versenum) else: - versenum = None # allow error trap + versenum = 1 continue words=None @@ -180,6 +181,7 @@ class OpenSongImport: assert len(our_verse_order)>0 order=our_verse_order for tag in order: + print tag if not versetags.has_key(tag): raise OpenSongImportError else: diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index ea6303c89..8ef2a1103 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -27,10 +27,12 @@ [b1] Bridge 1 Bridge 1 line 2 -[C1] +[C] +. A B Chorus 1 [C2] +. A B Chorus 2 diff --git a/openlp/plugins/songs/lib/test_importing_lots.py b/openlp/plugins/songs/lib/test_importing_lots.py new file mode 100644 index 000000000..7543de60e --- /dev/null +++ b/openlp/plugins/songs/lib/test_importing_lots.py @@ -0,0 +1,49 @@ +from openlp.plugins.songs.lib.opensongimport import OpenSongImport +from openlp.plugins.songs.lib.manager import SongManager +from glob import glob +from zipfile import ZipFile +import os +from traceback import print_exc +import sys +import codecs +def opensong_import_lots(): + ziploc=u'/home/mjt/openlp/OpenSong_Data/' + files=[] + files.extend(glob(ziploc+u'Songs.zip')) + files.extend(glob(ziploc+u'SOF.zip')) +# files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) +# files.extend(glob(ziploc+u'opensong_*.zip')) + errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') + manager=SongManager() + for file in files: + print u'Importing', file + z=ZipFile(file, u'r') + for song in z.infolist(): + filename=song.filename.decode('cp852') + parts=os.path.split(filename) + if parts[-1] == u'': + #No final part => directory + continue + # xxx need to handle unicode filenames (CP437?? Winzip does this) + print " ", file, ":",filename, + songfile=z.open(song) + + o=OpenSongImport(manager) + try: + o.do_import_file(songfile) + except: + print "Failure", + + errfile.write(u'Failure: %s:%s\n' %(file, filename)) + songfile=z.open(song) + for l in songfile.readlines(): + l=l.decode('utf8') + print(u' |%s\n'%l.strip()) + errfile.write(u' |%s\n'%l.strip()) + print_exc(3, file=errfile) + continue + # o.finish() + print "OK" + # o.song.print_song() +if __name__=="__main__": + opensong_import_lots() From baec9d5c2f2591ff835ca8768663dc02478ac204 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Fri, 25 Jun 2010 19:20:45 +0100 Subject: [PATCH 09/42] Unicode chars in my testcase work OK --- openlp/plugins/songs/lib/test.opensong | 6 ++--- .../plugins/songs/lib/test_importing_lots.py | 22 +++++++++++-------- .../plugins/songs/lib/test_opensongimport.py | 3 ++- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index 8533e8943..af0f039ed 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -1,7 +1,7 @@ Martins Test - Martin Thompson + MartiÑ Thómpson 2010 Martin Thompson 1 V1 C V2 C2 V3 B1 V1 @@ -25,8 +25,8 @@ 2 V2 Line 2 [3] -V3 Line 1 -V3 Line 2 + V3 Line 1 + V3 Line 2 [b1] Bridge 1 diff --git a/openlp/plugins/songs/lib/test_importing_lots.py b/openlp/plugins/songs/lib/test_importing_lots.py index 7543de60e..5161f4a30 100644 --- a/openlp/plugins/songs/lib/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test_importing_lots.py @@ -9,9 +9,9 @@ import codecs def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] - files.extend(glob(ziploc+u'Songs.zip')) - files.extend(glob(ziploc+u'SOF.zip')) -# files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) +# files.extend(glob(ziploc+u'Songs.zip')) +# files.extend(glob(ziploc+u'SOF.zip')) + files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager=SongManager() @@ -19,31 +19,35 @@ def opensong_import_lots(): print u'Importing', file z=ZipFile(file, u'r') for song in z.infolist(): - filename=song.filename.decode('cp852') + # need to handle unicode filenames (CP437 - Winzip does this) + filename=song.filename#.decode('cp852') parts=os.path.split(filename) if parts[-1] == u'': #No final part => directory continue - # xxx need to handle unicode filenames (CP437?? Winzip does this) print " ", file, ":",filename, - songfile=z.open(song) - + # songfile=z.open(song) + z.extract(song) + songfile=open(filename, u'r') o=OpenSongImport(manager) try: o.do_import_file(songfile) except: print "Failure", - errfile.write(u'Failure: %s:%s\n' %(file, filename)) + errfile.write(u'Failure: %s:%s\n' %(file, filename.decode('cp437'))) songfile=z.open(song) for l in songfile.readlines(): l=l.decode('utf8') print(u' |%s\n'%l.strip()) errfile.write(u' |%s\n'%l.strip()) print_exc(3, file=errfile) - continue + print_exc(3) + sys.exit(1) + # continue # o.finish() print "OK" + os.unlink(filename) # o.song.print_song() if __name__=="__main__": opensong_import_lots() diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 40b09a23b..6291f7106 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.plugins.songs.lib.manager import SongManager @@ -8,7 +9,7 @@ def test(): # o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'Martin Thompson'] + assert o.song.authors == [u'MartiÑ Thómpson'] assert o.song.title == u'Martins Test' assert o.song.alternate_title == u'' assert o.song.song_number == u'1' From 75c6ba22842c81d4bf03050e1d69ab3feb5f5ef2 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Mon, 28 Jun 2010 20:55:04 +0100 Subject: [PATCH 10/42] Improved =s --- openlp/plugins/songs/lib/opensongimport.py | 68 +++++++++---------- .../plugins/songs/lib/test_opensongimport.py | 29 +++++++- 2 files changed, 60 insertions(+), 37 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 170a650bd..f1a83540e 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -86,11 +86,11 @@ class OpenSongImport: Initialise the class. Requires a songmanager class which is passed to SongImport for writing song to disk """ - self.songmanager=songmanager + self.songmanager = songmanager self.song = None def do_import(self, filename): - file=open(filename) + file = open(filename) self.do_import_file(file) def do_import_file(self, file): @@ -98,10 +98,10 @@ class OpenSongImport: Process the OpenSong file """ self.song = SongImport(self.songmanager) - tree=objectify.parse(file) - root=tree.getroot() - fields=dir(root) - decode={u'copyright':self.song.add_copyright, + tree = objectify.parse(file) + root = tree.getroot() + fields = dir(root) + decode = {u'copyright':self.song.add_copyright, u'author':self.song.parse_author, u'title':self.song.set_title, u'aka':self.song.set_alternate_title, @@ -111,20 +111,20 @@ class OpenSongImport: fn(unicode(root.__getattr__(attr))) # data storage while importing - verses={} - lyrics=unicode(root.lyrics) + verses = {} + lyrics = unicode(root.lyrics) # keep track of a "default" verse order, in case none is specified - our_verse_order=[] - verses_seen={} + our_verse_order = [] + verses_seen = {} # in the absence of any other indication, verses are the default, erm, versetype! - versetype=u'V' + versetype = u'V' for l in lyrics.split(u'\n'): # remove comments semicolon = l.find(u';') if semicolon >= 0: - l=l[:semicolon] - l=l.strip() - if l==u'': + l = l[:semicolon] + l = l.strip() + if len(l) == 0: continue # skip inline guitar chords if l[0] == u'.': @@ -132,14 +132,14 @@ class OpenSongImport: # verse/chorus/etc. marker if l[0] == u'[': - versetype=l[1].upper() + versetype = l[1].upper() if versetype.isdigit(): - versenum=versetype - versetype=u'V' + versenum = versetype + versetype = u'V' elif l[2] != u']': # there's a number to go with it - extract that as well - right_bracket=l.find(u']') - versenum=l[2:right_bracket] + right_bracket = l.find(u']') + versenum = l[2:right_bracket] else: versenum = u'' continue @@ -147,18 +147,18 @@ class OpenSongImport: # number at start of line.. it's verse number if l[0].isdigit(): - versenum=l[0] - words=l[1:].strip() + versenum = l[0] + words = l[1:].strip() if words is None and \ versenum is not None and \ versetype is not None: words=l if versenum is not None: - versetag=u'%s%s'%(versetype,versenum) + versetag = u'%s%s'%(versetype,versenum) if not verses.has_key(versetype): - verses[versetype]={} + verses[versetype] = {} if not verses[versetype].has_key(versenum): - verses[versetype][versenum]=[] # storage for lines in this verse + verses[versetype][versenum] = [] # storage for lines in this verse if not verses_seen.has_key(versetag): verses_seen[versetag] = 1 our_verse_order.append(versetag) @@ -168,28 +168,28 @@ class OpenSongImport: words=words.replace('_', '') verses[versetype][versenum].append(words) # done parsing - versetypes=verses.keys() + versetypes = verses.keys() versetypes.sort() - versetags={} + versetags = {} for v in versetypes: - versenums=verses[v].keys() + versenums = verses[v].keys() versenums.sort() for n in versenums: - versetag= u'%s%s' %(v,n) - lines=u'\n'.join(verses[v][n]) + versetag = u'%s%s' %(v,n) + lines = u'\n'.join(verses[v][n]) self.song.verses.append([versetag, lines]) - versetags[versetag]=1 # keep track of what we have for error checking later + versetags[versetag] = 1 # keep track of what we have for error checking later # now figure out the presentation order if u'presentation' in fields and root.presentation != u'': - order=unicode(root.presentation) - order=order.split() + order = unicode(root.presentation) + order = order.split() else: assert len(our_verse_order)>0 - order=our_verse_order + order = our_verse_order for tag in order: if not versetags.has_key(tag): print u'Got order', tag, u'but not in versetags, skipping' - raise OpenSongImportError + raise OpenSongImportError # xxx keep error, or just warn? else: self.song.verse_order_list.append(tag) def finish(self): diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 6291f7106..95fc93cdd 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -1,10 +1,33 @@ # -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin # +# Thompson, Jon Tibble, Carsten Tinggaard # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.plugins.songs.lib.manager import SongManager def test(): - manager=SongManager() - o=OpenSongImport(manager) + manager = SongManager() + o = OpenSongImport(manager) o.do_import(u'test.opensong') # o.finish() o.song.print_song() @@ -21,7 +44,7 @@ def test(): assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] - o=OpenSongImport(manager) + o = OpenSongImport(manager) o.do_import(u'test2.opensong') # o.finish() o.song.print_song() From 7ce01d0e65547fbe862e0022fd63b0b99d2006b2 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 30 Jun 2010 20:53:08 +0100 Subject: [PATCH 11/42] Moved test scripts and data to a test folder --- openlp/plugins/songs/lib/test.opensong | 50 ------------------- .../songs/lib/{ => test}/test2.opensong | 0 .../lib/{ => test}/test_importing_lots.py | 4 +- .../lib/{ => test}/test_opensongimport.py | 0 4 files changed, 3 insertions(+), 51 deletions(-) delete mode 100644 openlp/plugins/songs/lib/test.opensong rename openlp/plugins/songs/lib/{ => test}/test2.opensong (100%) rename openlp/plugins/songs/lib/{ => test}/test_importing_lots.py (93%) rename openlp/plugins/songs/lib/{ => test}/test_opensongimport.py (100%) diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong deleted file mode 100644 index af0f039ed..000000000 --- a/openlp/plugins/songs/lib/test.opensong +++ /dev/null @@ -1,50 +0,0 @@ - - - Martins Test - MartiÑ Thómpson - 2010 Martin Thompson - 1 - V1 C V2 C2 V3 B1 V1 - Blah - - - - - - - - - - - ;Comment -. A B C -1 v1 Line 1___ -2 v2 Line 1___ -. A B C7 -1 V1 Line 2 -2 V2 Line 2 - -[3] - V3 Line 1 - V3 Line 2 - -[b1] - Bridge 1 - Bridge 1 line 2 -[C] -. A B - Chorus 1 - -[C2] -. A B - Chorus 2 - - diff --git a/openlp/plugins/songs/lib/test2.opensong b/openlp/plugins/songs/lib/test/test2.opensong similarity index 100% rename from openlp/plugins/songs/lib/test2.opensong rename to openlp/plugins/songs/lib/test/test2.opensong diff --git a/openlp/plugins/songs/lib/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py similarity index 93% rename from openlp/plugins/songs/lib/test_importing_lots.py rename to openlp/plugins/songs/lib/test/test_importing_lots.py index 5161f4a30..59d72b34b 100644 --- a/openlp/plugins/songs/lib/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -9,9 +9,10 @@ import codecs def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] + files=['test.opensong.zip'] # files.extend(glob(ziploc+u'Songs.zip')) # files.extend(glob(ziploc+u'SOF.zip')) - files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) + # files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager=SongManager() @@ -32,6 +33,7 @@ def opensong_import_lots(): o=OpenSongImport(manager) try: o.do_import_file(songfile) + o.song.print_song() except: print "Failure", diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py similarity index 100% rename from openlp/plugins/songs/lib/test_opensongimport.py rename to openlp/plugins/songs/lib/test/test_opensongimport.py From 02fd6fe4da0d18b9a455ed70ada1a47140a3e1be Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 30 Jun 2010 21:05:43 +0100 Subject: [PATCH 12/42] Main class handles zipfiles --- openlp/plugins/songs/lib/opensongimport.py | 29 ++++++++++++++++--- .../songs/lib/test/test_importing_lots.py | 8 ++--- .../songs/lib/test/test_opensongimport.py | 20 +++++++++++-- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index f1a83540e..e7878c08a 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -30,6 +30,11 @@ from songimport import SongImport from lxml.etree import Element from lxml import objectify +from zipfile import ZipFile + +import logging +log = logging.getLogger(__name__) + class OpenSongImportError(Exception): pass @@ -89,10 +94,26 @@ class OpenSongImport: self.songmanager = songmanager self.song = None - def do_import(self, filename): - file = open(filename) - self.do_import_file(file) - + def do_import(self, filename, commit=True): + ext=os.path.splitext(filename)[1] + if ext.lower() == ".zip": + log.info('Zipfile found %s', filename) + z=ZipFile(filename, u'r') + for song in z.infolist(): + parts=os.path.split(song.filename) + if parts[-1] == u'': + #No final part => directory + continue + songfile=z.open(song) + self.do_import_file(songfile) + if commit: + self.finish() + else: + log.info('Direct import %s', filename) + file = open(filename) + self.do_import_file(file) + if commit: + self.finish() def do_import_file(self, file): """ Process the OpenSong file diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index 59d72b34b..9c90ad50c 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -27,9 +27,9 @@ def opensong_import_lots(): #No final part => directory continue print " ", file, ":",filename, - # songfile=z.open(song) - z.extract(song) - songfile=open(filename, u'r') + songfile=z.open(song) + #z.extract(song) + #songfile=open(filename, u'r') o=OpenSongImport(manager) try: o.do_import_file(songfile) @@ -49,7 +49,7 @@ def opensong_import_lots(): # continue # o.finish() print "OK" - os.unlink(filename) + #os.unlink(filename) # o.song.print_song() if __name__=="__main__": opensong_import_lots() diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 95fc93cdd..f847b273b 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -28,7 +28,23 @@ from openlp.plugins.songs.lib.manager import SongManager def test(): manager = SongManager() o = OpenSongImport(manager) - o.do_import(u'test.opensong') + o.do_import(u'test.opensong', commit=False) + # o.finish() + o.song.print_song() + assert o.song.copyright == u'2010 Martin Thompson' + assert o.song.authors == [u'MartiÑ Thómpson'] + assert o.song.title == u'Martins Test' + assert o.song.alternate_title == u'' + assert o.song.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses + assert [u'C', u'Chorus 1'] in o.song.verses + assert [u'C2', u'Chorus 2'] in o.song.verses + assert not [u'C3', u'Chorus 3'] in o.song.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses + assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] + + o.do_import(u'test.opensong.zip', commit=False) # o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' @@ -45,7 +61,7 @@ def test(): assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] o = OpenSongImport(manager) - o.do_import(u'test2.opensong') + o.do_import(u'test2.opensong', commit=False) # o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' From 38ccc21f89a4390970a38014bf39648e20b6352e Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 30 Jun 2010 22:19:18 +0100 Subject: [PATCH 13/42] Added GUI for importing OpenSong --- openlp/plugins/songs/lib/opensongimport.py | 9 ++-- openlp/plugins/songs/lib/songimport.py | 9 ++-- .../songs/lib/test/test_importing_lots.py | 6 +-- openlp/plugins/songs/songsplugin.py | 41 +++++++++++++++++++ 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index e7878c08a..b65eb74b3 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -95,6 +95,10 @@ class OpenSongImport: self.song = None def do_import(self, filename, commit=True): + """ + Import either a single opensong file, or a zipfile containing multiple opensong files + If the commit parameter is set False, the import will not be committed to the database (useful for test scripts) + """ ext=os.path.splitext(filename)[1] if ext.lower() == ".zip": log.info('Zipfile found %s', filename) @@ -116,7 +120,7 @@ class OpenSongImport: self.finish() def do_import_file(self, file): """ - Process the OpenSong file + Process the OpenSong file - pass in a file-like object, not a filename """ self.song = SongImport(self.songmanager) tree = objectify.parse(file) @@ -209,8 +213,7 @@ class OpenSongImport: order = our_verse_order for tag in order: if not versetags.has_key(tag): - print u'Got order', tag, u'but not in versetags, skipping' - raise OpenSongImportError # xxx keep error, or just warn? + log.warn(u'Got order %s but not in versetags, skipping', tag) else: self.song.verse_order_list.append(tag) def finish(self): diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 08855f01a..a87023527 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -302,8 +302,7 @@ class SongImport(object): song.theme_name = self.theme_name song.ccli_number = self.ccli_number for authortext in self.authors: - filter_string = u'display_name=%s' % authortext - author = self.manager.get_object_filtered(Author, filter_string) + author = self.manager.get_object_filtered(Author, Author.display_name == authortext) if author is None: author = Author() author.display_name = authortext @@ -312,8 +311,7 @@ class SongImport(object): self.manager.save_object(author) song.authors.append(author) if self.song_book_name: - filter_string = u'name=%s' % self.song_book_name - song_book = self.manager.get_object_filtered(Book, filter_string) + song_book = self.manager.get_object_filtered(Book, Book.name == self.song_book_name) if song_book is None: song_book = Book() song_book.name = self.song_book_name @@ -321,8 +319,7 @@ class SongImport(object): self.manager.save_object(song_book) song.song_book_id = song_book.id for topictext in self.topics: - filter_string = u'name=%s' % topictext - topic = self.manager.get_object_filtered(Topic, filter_string) + topic = self.manager.get_object_filtered(Topic.name == topictext) if topic is None: topic = Topic() topic.name = topictext diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index 9c90ad50c..0abc6d3e1 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -10,8 +10,8 @@ def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] files=['test.opensong.zip'] -# files.extend(glob(ziploc+u'Songs.zip')) -# files.extend(glob(ziploc+u'SOF.zip')) + files.extend(glob(ziploc+u'Songs.zip')) + files.extend(glob(ziploc+u'SOF.zip')) # files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') @@ -47,7 +47,7 @@ def opensong_import_lots(): print_exc(3) sys.exit(1) # continue - # o.finish() + o.finish() print "OK" #os.unlink(filename) # o.song.print_song() diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 82a780fa9..08a136e42 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -38,6 +38,8 @@ try: except ImportError: OOo_available = False +from openlp.plugins.songs.lib import OpenSongImport + log = logging.getLogger(__name__) class SongsPlugin(Plugin): @@ -143,6 +145,25 @@ class SongsPlugin(Plugin): QtCore.SIGNAL(u'triggered()'), self.onImportSofItemClick) QtCore.QObject.connect(self.ImportOooItem, QtCore.SIGNAL(u'triggered()'), self.onImportOooItemClick) + # OpenSong import menu item - will be removed and the + # functionality will be contained within the import wizard + self.ImportOpenSongItem = QtGui.QAction(import_menu) + self.ImportOpenSongItem.setObjectName(u'ImportOpenSongItem') + self.ImportOpenSongItem.setText( + translate('SongsPlugin', + 'OpenSong (temp menu item)')) + self.ImportOpenSongItem.setToolTip( + translate('SongsPlugin', + 'Import songs from OpenSong files' + + '(either raw text or ZIPfiles)')) + self.ImportOpenSongItem.setStatusTip( + translate('SongsPlugin', + 'Import songs from OpenSong files' + + '(either raw text or ZIPfiles)')) + import_menu.addAction(self.ImportOpenSongItem) + QtCore.QObject.connect(self.ImportOpenSongItem, + QtCore.SIGNAL(u'triggered()'), self.onImportOpenSongItemClick) + def add_export_menu_item(self, export_menu): """ @@ -183,6 +204,26 @@ class SongsPlugin(Plugin): QtGui.QMessageBox.Ok) Receiver.send_message(u'songs_load_list') + def onImportOpenSongItemClick(self): + filenames = QtGui.QFileDialog.getOpenFileNames( + None, translate('SongsPlugin', + 'Open OpenSong file'), + u'', u'OpenSong file (*. *.zip *.ZIP)') + try: + for filename in filenames: + importer = OpenSongImport(self.manager) + importer.do_import(unicode(filename)) + except: + log.exception('Could not import OpenSong file') + QtGui.QMessageBox.critical(None, + translate('SongsPlugin', + 'Import Error'), + translate('SongsPlugin', + 'Error importing OpenSong file'), + QtGui.QMessageBox.StandardButtons(QtGui.QMessageBox.Ok), + QtGui.QMessageBox.Ok) + Receiver.send_message(u'songs_load_list') + def onImportOooItemClick(self): filenames = QtGui.QFileDialog.getOpenFileNames( None, translate('SongsPlugin', From ab1f040aaaa520e29fa813fefd9747ca5fca03c3 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Mon, 5 Jul 2010 21:00:05 +0100 Subject: [PATCH 14/42] Working on non-ascii characters in zipfiles --- openlp/plugins/songs/lib/test/test_importing_lots.py | 10 +++++----- openlp/plugins/songs/lib/test/test_opensongimport.py | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index 0abc6d3e1..4b97bf42b 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -9,10 +9,10 @@ import codecs def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] - files=['test.opensong.zip'] - files.extend(glob(ziploc+u'Songs.zip')) - files.extend(glob(ziploc+u'SOF.zip')) - # files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) + files=[u'test.opensong.zip', ziploc+u'ADond.zip'] + #files.extend(glob(ziploc+u'Songs.zip')) + #files.extend(glob(ziploc+u'SOF.zip')) + files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager=SongManager() @@ -47,7 +47,7 @@ def opensong_import_lots(): print_exc(3) sys.exit(1) # continue - o.finish() + #o.finish() print "OK" #os.unlink(filename) # o.song.print_song() diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index f847b273b..d3d8d356a 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -28,6 +28,9 @@ from openlp.plugins.songs.lib.manager import SongManager def test(): manager = SongManager() o = OpenSongImport(manager) + o.do_import(u'/home/mjt/openlp/OpenSong_Data/ADond', commit=False) + o.song.print_song() + sys.exit(1) o.do_import(u'test.opensong', commit=False) # o.finish() o.song.print_song() From b3559a993d2f8116392d23442cbf1028c09d4db7 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:16:14 +0100 Subject: [PATCH 15/42] test theme, alttheme -> our theme --- openlp/plugins/songs/lib/opensongimport.py | 9 +++++++++ openlp/plugins/songs/lib/test/test_opensongimport.py | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 1a5c94a39..df8492976 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -118,6 +118,8 @@ class OpenSongImport: self.do_import_file(file) if commit: self.finish() + + def do_import_file(self, file): """ Process the OpenSong file - pass in a file-like object, not a filename @@ -135,6 +137,13 @@ class OpenSongImport: for (attr, fn) in decode.items(): if attr in fields: fn(unicode(root.__getattr__(attr))) + + res = [] + if u'theme' in fields: + res.append(unicode(root.theme)) + if u'alttheme' in fields: + res.append(unicode(root.alttheme)) + self.song.theme=u', '.join(res) # data storage while importing verses = {} diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index a027042a7..c1025d4fe 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -30,7 +30,7 @@ def test(): manager = SongManager() o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) - # o.finish() + o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' assert o.song.authors == [u'MartiÑ Thómpson'] @@ -45,7 +45,8 @@ def test(): assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] assert o.song.ccli_number == u'Blah' - + print u':%s:'%o.song.theme + assert o.song.theme == u'TestTheme, TestAltTheme' o.do_import(u'test.opensong.zip', commit=False) # o.finish() o.song.print_song() From 860d3e2923eca9aa0233588238a189a2845e042a Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:17:16 +0100 Subject: [PATCH 16/42] Removed opensong code from songxml --- openlp/plugins/songs/lib/songxml.py | 203 ---------------------------- 1 file changed, 203 deletions(-) diff --git a/openlp/plugins/songs/lib/songxml.py b/openlp/plugins/songs/lib/songxml.py index 2965c579b..6e99933ec 100644 --- a/openlp/plugins/songs/lib/songxml.py +++ b/openlp/plugins/songs/lib/songxml.py @@ -60,175 +60,6 @@ class SongFeatureError(SongException): # TODO: Song: Import ChangingSong # TODO: Song: Export ChangingSong -_BLANK_OPENSONG_XML = \ -''' - - - - - - - - - - -''' - -class _OpenSong(object): - """ - Class for import of OpenSong - """ - def __init__(self, xmlContent = None): - """ - Initialize from given xml content - """ - self._set_from_xml(_BLANK_OPENSONG_XML, 'song') - if xmlContent: - self._set_from_xml(xmlContent, 'song') - - def _set_from_xml(self, xml, root_tag): - """ - Set song properties from given xml content. - - ``xml`` - Formatted xml tags and values. - ``root_tag`` - The root tag of the xml. - """ - root = ElementTree(element=XML(xml)) - xml_iter = root.getiterator() - for element in xml_iter: - if element.tag != root_tag: - text = element.text - if text is None: - val = text - elif isinstance(text, basestring): - # Strings need special handling to sort the colours out - if text[0] == u'$': - # This might be a hex number, let's try to convert it. - try: - val = int(text[1:], 16) - except ValueError: - pass - else: - # Let's just see if it's a integer. - try: - val = int(text) - except ValueError: - # Ok, it seems to be a string. - val = text - if hasattr(self, u'post_tag_hook'): - (element.tag, val) = \ - self.post_tag_hook(element.tag, val) - setattr(self, element.tag, val) - - def __str__(self): - """ - Return string with all public attributes - - The string is formatted with one attribute per line - If the string is split on newline then the length of the - list is equal to the number of attributes - """ - attributes = [] - for attrib in dir(self): - if not attrib.startswith(u'_'): - attributes.append( - u'%30s : %s' % (attrib, getattr(self, attrib))) - return u'\n'.join(attributes) - - def _get_as_string(self): - """ - Return one string with all public attributes - """ - result = u'' - for attrib in dir(self): - if not attrib.startswith(u'_'): - result += u'_%s_' % getattr(self, attrib) - return result - - def get_author_list(self): - """Convert author field to an authorlist - - in OpenSong an author list may be separated by '/' - return as a string - """ - if self.author: - list = self.author.split(u' and ') - res = [item.strip() for item in list] - return u', '.join(res) - - def get_category_array(self): - """Convert theme and alttheme into category_array - - return as a string - """ - res = [] - if self.theme: - res.append(self.theme) - if self.alttheme: - res.append(self.alttheme) - return u', u'.join(res) - - def _reorder_verse(self, tag, tmpVerse): - """ - Reorder the verse in case of first char is a number - tag -- the tag of this verse / verse group - tmpVerse -- list of strings - """ - res = [] - for digit in '1234567890 ': - tagPending = True - for line in tmpVerse: - if line.startswith(digit): - if tagPending: - tagPending = False - tagChar = tag.strip(u'[]').lower() - if 'v' == tagChar: - newtag = "Verse" - elif 'c' == tagChar: - newtag = "Chorus" - elif 'b' == tagChar: - newtag = "Bridge" - elif 'p' == tagChar: - newtag = "Pre-chorus" - else: - newtag = tagChar - tagString = (u'# %s %s' % (newtag, digit)).rstrip() - res.append(tagString) - res.append(line[1:]) - if (len(line) == 0) and (not tagPending): - res.append(line) - return res - - def get_lyrics(self): - """ - Convert the lyrics to openlp lyrics format - return as list of strings - """ - lyrics = self.lyrics.split(u'\n') - tmpVerse = [] - finalLyrics = [] - tag = "" - for lyric in lyrics: - line = lyric.rstrip() - if not line.startswith(u'.'): - # drop all chords - tmpVerse.append(line) - if line: - if line.startswith(u'['): - tag = line - else: - reorderedVerse = self._reorder_verse(tag, tmpVerse) - finalLyrics.extend(reorderedVerse) - tag = "" - tmpVerse = [] - # catch up final verse - reorderedVerse = self._reorder_verse(tag, tmpVerse) - finalLyrics.extend(reorderedVerse) - return finalLyrics - - class Song(object): """Handling song properties and methods @@ -307,40 +138,6 @@ class Song(object): self.set_lyrics(u'') return - def from_opensong_buffer(self, xmlcontent): - """Initialize from buffer(string) of xml lines in opensong format""" - self._reset() - opensong = _OpenSong(xmlcontent) - if opensong.title: - self.set_title(opensong.title) - if opensong.copyright: - self.set_copyright(opensong.copyright) - if opensong.presentation: - self.set_verse_order(opensong.presentation) - if opensong.ccli: - self.set_song_cclino(opensong.ccli) - self.set_author_list(opensong.get_author_list()) - self.set_category_array(opensong.get_category_array()) - self.set_lyrics(opensong.get_lyrics()) - - def from_opensong_file(self, xmlfilename): - """ - Initialize from file containing xml - xmlfilename -- path to xml file - """ - osfile = None - try: - osfile = open(xmlfilename, 'r') - list = [line for line in osfile] - osfile.close() - xml = "".join(list) - self.from_opensong_buffer(xml) - except IOError: - log.exception(u'Failed to load opensong xml file') - finally: - if osfile: - osfile.close() - def _remove_punctuation(self, title): """Remove the puntuation chars from title From b04c93e9a7bf304dddd9e49a2d60546d001730d5 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:17:58 +0100 Subject: [PATCH 17/42] test theme, alttheme -> our theme --- openlp/plugins/songs/lib/lyrics_xml.py | 4 ++-- openlp/plugins/songs/lib/songimport.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/openlp/plugins/songs/lib/lyrics_xml.py b/openlp/plugins/songs/lib/lyrics_xml.py index 359c28eda..38985ca1b 100644 --- a/openlp/plugins/songs/lib/lyrics_xml.py +++ b/openlp/plugins/songs/lib/lyrics_xml.py @@ -76,8 +76,8 @@ class SongXMLBuilder(object): ``content`` The actual text of the verse to be stored. """ - #log.debug(u'add_verse_to_lyrics %s, %s\n%s' % (type, number, content)) - verse = etree.Element(u'verse', type=type, label=number) + # log.debug(u'add_verse_to_lyrics %s, %s\n%s' % (type, number, content)) + verse = etree.Element(u'verse', type=unicode(type), label=unicode(number)) verse.text = etree.CDATA(content) self.lyrics.append(verse) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 5fbcce540..3b507140b 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -51,8 +51,8 @@ class SongImport(object): self.alternate_title = u'' self.copyright = u'' self.comment = u'' - self.theme_name = u'' - self.ccli_number = u'' + self.theme = u'' + self.song_cclino = u'' self.authors = [] self.topics = [] self.song_book_name = u'' @@ -303,8 +303,8 @@ class SongImport(object): song.verse_order = u' '.join(self.verse_order_list) song.copyright = self.copyright song.comment = self.comment - song.theme_name = self.theme_name - song.ccli_number = self.ccli_number + song.theme = self.theme + song.song_cclino = self.song_cclino for authortext in self.authors: author = self.manager.get_object_filtered(Author, Author.display_name == authortext) @@ -358,7 +358,7 @@ class SongImport(object): print u'TOPIC: ' + topictext if self.comment: print u'COMMENT: ' + self.comment - if self.theme_name: - print u'THEME: ' + self.theme_name - if self.ccli_number: - print u'CCLI: ' + self.ccli_number + if self.theme: + print u'THEME: ' + self.theme + if self.song_cclino: + print u'CCLI: ' + self.song_cclino From e4b7e23c42bf04997b8aa9764b32559eba2f3ee1 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:18:38 +0100 Subject: [PATCH 18/42] Added test data --- openlp/plugins/songs/lib/test/test.opensong | 51 ++++++++++++++++++ .../plugins/songs/lib/test/test.opensong.zip | Bin 0 -> 848 bytes 2 files changed, 51 insertions(+) create mode 100644 openlp/plugins/songs/lib/test/test.opensong create mode 100644 openlp/plugins/songs/lib/test/test.opensong.zip diff --git a/openlp/plugins/songs/lib/test/test.opensong b/openlp/plugins/songs/lib/test/test.opensong new file mode 100644 index 000000000..469cd7722 --- /dev/null +++ b/openlp/plugins/songs/lib/test/test.opensong @@ -0,0 +1,51 @@ + + + Martins Test + MartiÑ Thómpson + 2010 Martin Thompson + 1 + V1 C V2 C2 V3 B1 V1 + Blah + + + + + + + + TestTheme + TestAltTheme + + + ;Comment +. A B C +1 v1 Line 1___ +2 v2 Line 1___ +. A B C7 +1 V1 Line 2 +2 V2 Line 2 + +[3] + V3 Line 1 + V3 Line 2 + +[b1] + Bridge 1 + Bridge 1 line 2 +[C] +. A B + Chorus 1 + +[C2] +. A B + Chorus 2 + + diff --git a/openlp/plugins/songs/lib/test/test.opensong.zip b/openlp/plugins/songs/lib/test/test.opensong.zip new file mode 100644 index 0000000000000000000000000000000000000000..1072367a844712f5b433e9c80a621a00e8245f20 GIT binary patch literal 848 zcmWIWW@Zs#U}E542$^-$#ykH@-vTBEhDde>23`gkhLY6c621I_)V$*Sy!6lzP6lR% z&niCCWEFi%E4UdLSza(RFo21xq27774FvAK7GW-%`{MBe-hfEQ+ygHb-CcT*h0)P- zi%W|T>!hN`_ZFqU6fD}v=XtVh(mJj!8L0;z?fLG(cfaT5pJy8o7bgbzAC?qLt*r_tRy~@#%eKV!?eTR_c6ogMVN>#D!On+!Q^h{( zZ_`|pIQyVg^IN7c&78tx#;y1FPg)e=dF@eH-U~xB4T&u;-cEWm!(@)g38fb|+3OBm zsQvA1XV7!!`;kD+Dmj+K7auMb%vF-OXFSQ*qs&E2coWCBZF`EnTerJcv_~pDy1$S` zcDb7g-<)>7=`W&EyS`^%EvG;L8K4#La_ct_jzbaUks+j3&tFE7Z>9gp8LMT|7vnwl9oF;c+xY%y@qtwjZFs6HQkKUs*FGsazjlQL zqw>Cozivh|EuD1yP0P`mRhCj8Gj8oH^;r@W^is8Q^T+<#s{@T*JG-O@K9aw)Ds}5G zt8S++UR#8NV}e5cvGV@>R*w zRs8zB?pqfvOZrqYZ>QMY71<}_uP$|eR*`Y*>{ZQSvz)OK0P|K6q?vp7LL zzA&jZK62KFn?J9q%Gh!!Rblg{{~@t&?48G?UV x0=yZS Date: Fri, 9 Jul 2010 19:17:48 +0100 Subject: [PATCH 19/42] Changed opensong import file extension to All files *.* --- openlp/plugins/songs/songsplugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 6552f4c41..99e3fca74 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -201,7 +201,7 @@ class SongsPlugin(Plugin): filenames = QtGui.QFileDialog.getOpenFileNames( None, translate('SongsPlugin', 'Open OpenSong file'), - u'', u'OpenSong file (*. *.zip *.ZIP)') + u'', u'All files (*.*)') try: for filename in filenames: importer = OpenSongImport(self.manager) @@ -236,4 +236,4 @@ class SongsPlugin(Plugin): if not self.manager.get_all_objects_filtered(Song, Song.theme_name == theme): return True - return False \ No newline at end of file + return False From 0f4c7276b3160a49390173f4e7ba1951e29b2f94 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:10:27 +0100 Subject: [PATCH 20/42] Using new manager --- openlp/plugins/songs/lib/test/test_opensongimport.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 7ae5c3412..96b5018a3 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -23,11 +23,13 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### from openlp.plugins.songs.lib.opensongimport import OpenSongImport -from openlp.plugins.songs.lib.manager import SongManager +from openlp.core.lib.db import Manager +from openlp.plugins.songs.lib.db import init_schema#, Song +from openlp.plugins.songs.songsplugin import SongsPlugin import sys def test(): - manager = SongManager() + manager = Manager(u'songs', init_schema) o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) o.finish() From d54770646dab95891957b2f215bcb55d6daed6fd Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:38:46 +0100 Subject: [PATCH 21/42] Rename cclino to ccli_number and theme to theme_name in all song import related files --- openlp/plugins/songs/lib/opensongimport.py | 4 +- openlp/plugins/songs/lib/songimport.py | 22 ++++---- openlp/plugins/songs/lib/songxml.py | 52 +++++++++---------- .../songs/lib/test/test_importing_lots.py | 11 ++-- .../songs/lib/test/test_opensongimport.py | 7 ++- openlp/plugins/songs/songsplugin.py | 4 +- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index a5efb2808..93c682d90 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -135,7 +135,7 @@ class OpenSongImport: root = tree.getroot() fields = dir(root) decode = {u'copyright':self.song.add_copyright, - u'ccli':self.song.set_song_cclino, + u'ccli':self.song.set_ccli_number, u'author':self.song.parse_author, u'title':self.song.set_title, u'aka':self.song.set_alternate_title, @@ -149,7 +149,7 @@ class OpenSongImport: res.append(unicode(root.theme)) if u'alttheme' in fields: res.append(unicode(root.alttheme)) - self.song.theme = u', '.join(res) + self.song.theme_name = u', '.join(res) # data storage while importing verses = {} diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 1253c8536..893def7f0 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -51,8 +51,8 @@ class SongImport(object): self.alternate_title = u'' self.copyright = u'' self.comment = u'' - self.theme = u'' - self.song_cclino = u'' + self.theme_name = u'' + self.ccli_number = u'' self.authors = [] self.topics = [] self.song_book_name = u'' @@ -163,11 +163,11 @@ class SongImport(object): """ self.song_number = song_number - def set_song_cclino(self, cclino): + def set_ccli_number(self, cclino): """ Set the ccli number """ - self.song_cclino = cclino + self.ccli_number = cclino def set_song_book(self, song_book, publisher): """ @@ -273,7 +273,7 @@ class SongImport(object): def commit_song(self): """ - Write the song and it's fields to disk + Write the song and its fields to disk """ song = Song() song.title = self.title @@ -303,8 +303,8 @@ class SongImport(object): song.verse_order = u' '.join(self.verse_order_list) song.copyright = self.copyright song.comment = self.comment - song.theme = self.theme - song.song_cclino = self.song_cclino + song.theme_name = self.theme_name + song.ccli_number = self.ccli_number for authortext in self.authors: author = self.manager.get_object_filtered(Author, Author.display_name == authortext) @@ -358,7 +358,7 @@ class SongImport(object): print u'TOPIC: ' + topictext if self.comment: print u'COMMENT: ' + self.comment - if self.theme: - print u'THEME: ' + self.theme - if self.song_cclino: - print u'CCLI: ' + self.song_cclino + if self.theme_name: + print u'THEME: ' + self.theme_name + if self.ccli_number: + print u'CCLI: ' + self.ccli_number diff --git a/openlp/plugins/songs/lib/songxml.py b/openlp/plugins/songs/lib/songxml.py index 6e99933ec..c5b3df49a 100644 --- a/openlp/plugins/songs/lib/songxml.py +++ b/openlp/plugins/songs/lib/songxml.py @@ -106,7 +106,7 @@ class Song(object): show_author_list -- 0: no show, 1: show show_copyright -- 0: no show, 1: show show_song_cclino -- 0: no show, 1: show - theme -- name of theme or blank + theme_name -- name of theme or blank category_array -- list of user defined properties (hymn, gospel) song_book -- name of originating book song_number -- number of the song, related to a songbook @@ -129,7 +129,7 @@ class Song(object): self.show_copyright = 1 self.show_song_cclino = 1 self.show_title = 1 - self.theme = "" + self.theme_name = "" self.category_array = None self.song_book = "" self.song_number = "" @@ -221,7 +221,7 @@ class Song(object): self.set_title(sName) self.set_author_list(author_list) self.set_copyright(sCopyright) - self.set_song_cclino(sCcli) + self.set_ccli_number(sCcli) self.set_lyrics(lyrics) def from_ccli_text_file(self, textFileName): @@ -276,21 +276,21 @@ class Song(object): """Set the copyright string""" self.copyright = copyright - def get_song_cclino(self): + def get_ccli_number(self): """Return the songCclino""" - return self._assure_string(self.song_cclino) + return self._assure_string(self.ccli_number) - def set_song_cclino(self, song_cclino): - """Set the song_cclino""" - self.song_cclino = song_cclino + def set_ccli_number(self, ccli_number): + """Set the ccli_number""" + self.ccli_number = ccli_number - def get_theme(self): + def get_theme_name(self): """Return the theme name for the song""" - return self._assure_string(self.theme) + return self._assure_string(self.theme_name) - def set_theme(self, theme): + def set_theme_name(self, theme_name): """Set the theme name (string)""" - self.theme = theme + self.theme_name = theme_name def get_song_book(self): """Return the song_book (string)""" @@ -329,9 +329,9 @@ class Song(object): asOneString True -- string: - "John Newton, A Parker" + 'John Newton, A Parker' False -- list of strings - ["John Newton", u'A Parker"] + ['John Newton', u'A Parker'] """ if asOneString: res = self._assure_string(self.author_list) @@ -354,9 +354,9 @@ class Song(object): asOneString True -- string: - "Hymn, Gospel" + 'Hymn, Gospel' False -- list of strings - ["Hymn", u'Gospel"] + ['Hymn', u'Gospel'] """ if asOneString: res = self._assure_string(self.category_array) @@ -398,13 +398,13 @@ class Song(object): """Set the show_copyright flag (bool)""" self.show_copyright = show_copyright - def get_show_song_cclino(self): + def get_show_ccli_number(self): """Return the showSongCclino (string)""" - return self.show_song_cclino + return self.show_ccli_number - def set_show_song_cclino(self, show_song_cclino): - """Set the show_song_cclino flag (bool)""" - self.show_song_cclino = show_song_cclino + def set_show_ccli_number(self, show_ccli_number): + """Set the show_ccli_number flag (bool)""" + self.show_ccli_number = show_ccli_number def get_lyrics(self): """Return the lyrics as a list of strings @@ -471,7 +471,7 @@ class Song(object): slideNumber -- 1 .. numberOfSlides Returns a list as: - [theme (string), + [theme_name (string), title (string), authorlist (string), copyright (string), @@ -496,13 +496,13 @@ class Song(object): cpright = self.get_copyright() else: cpright = "" - if self.show_song_cclino: - ccli = self.get_song_cclino() + if self.show_ccli_number: + ccli = self.get_ccli_number() else: ccli = "" - theme = self.get_theme() + theme_name = self.get_theme_name() # examine the slide for a theme - res.append(theme) + res.append(theme_name) res.append(title) res.append(author) res.append(cpright) diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index d896cf5ba..e7ed339b0 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -1,5 +1,6 @@ from openlp.plugins.songs.lib.opensongimport import OpenSongImport -from openlp.plugins.songs.lib.manager import SongManager +from openlp.plugins.songs.lib.db import init_schema +from openlp.core.lib.db import Manager from glob import glob from zipfile import ZipFile import os @@ -9,13 +10,13 @@ import codecs def opensong_import_lots(): ziploc = u'/home/mjt/openlp/OpenSong_Data/' files = [] - files = [u'test.opensong.zip', ziploc+u'ADond.zip'] - #files.extend(glob(ziploc+u'Songs.zip')) + #files = [u'test.opensong.zip', ziploc+u'ADond.zip'] + files.extend(glob(ziploc+u'Songs.zip')) #files.extend(glob(ziploc+u'SOF.zip')) - files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) + #files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') - manager=SongManager() + manager = Manager(u'songs', init_schema) for file in files: print u'Importing', file z = ZipFile(file, u'r') diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 96b5018a3..aede61416 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -24,7 +24,7 @@ ############################################################################### from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.core.lib.db import Manager -from openlp.plugins.songs.lib.db import init_schema#, Song +from openlp.plugins.songs.lib.db import init_schema from openlp.plugins.songs.songsplugin import SongsPlugin import sys @@ -46,9 +46,8 @@ def test(): assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] - assert o.song.song_cclino == u'Blah' - print u':%s:'%o.song.theme - assert o.song.theme == u'TestTheme, TestAltTheme' + assert o.song.ccli_number == u'Blah' + assert o.song.theme_name == u'TestTheme, TestAltTheme' o.do_import(u'test.opensong.zip', commit=False) o.finish() o.song.print_song() diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 99e3fca74..b6795cd32 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -232,8 +232,8 @@ class SongsPlugin(Plugin): 'This plugin allows songs to be managed and displayed.') return about_text - def canDeleteTheme(self, theme): + def canDeleteTheme(self, theme_name): if not self.manager.get_all_objects_filtered(Song, - Song.theme_name == theme): + Song.theme_name == theme_name): return True return False From 490e2451e00dc6343639d224a4837632cb5b9252 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:46:23 +0100 Subject: [PATCH 22/42] spaces --- openlp/plugins/songs/lib/test/test_importing_lots.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index e7ed339b0..dc3eeeb2a 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -15,7 +15,7 @@ def opensong_import_lots(): #files.extend(glob(ziploc+u'SOF.zip')) #files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) - errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') + errfile = codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager = Manager(u'songs', init_schema) for file in files: print u'Importing', file @@ -41,8 +41,8 @@ def opensong_import_lots(): errfile.write(u'Failure: %s:%s\n' %(file, filename.decode('cp437'))) songfile = z.open(song) for l in songfile.readlines(): - l=l.decode('utf8') - print(u' |%s\n'%l.strip()) + l = l.decode('utf8') + print(u' |%s\n' % l.strip()) errfile.write(u' |%s\n'%l.strip()) print_exc(3, file = errfile) print_exc(3) From 222f10ba4574db947adbd812e9b8d969b1738e8e Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:52:45 +0100 Subject: [PATCH 23/42] Now fixed the trunk merge --- openlp/plugins/songs/songsplugin.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index b9129be7a..ab36d13d0 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -233,10 +233,6 @@ class SongsPlugin(Plugin): 'This plugin allows songs to be managed and displayed.') return about_text - def canDeleteTheme(self, theme_name): - if not self.manager.get_all_objects_filtered(Song, - Song.theme_name == theme_name): - def usesTheme(self, theme): """ Called to find out if the song plugin is currently using a theme. From 294a02af458e0b031d9e2438034a313d51a7e0bc Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Thu, 15 Jul 2010 21:26:57 +0100 Subject: [PATCH 24/42] Renamed back and fixed a couple of = signs --- openlp/plugins/songs/lib/{lyrics_xml.py => xml.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename openlp/plugins/songs/lib/{lyrics_xml.py => xml.py} (97%) diff --git a/openlp/plugins/songs/lib/lyrics_xml.py b/openlp/plugins/songs/lib/xml.py similarity index 97% rename from openlp/plugins/songs/lib/lyrics_xml.py rename to openlp/plugins/songs/lib/xml.py index d3c595b15..1e9678c55 100644 --- a/openlp/plugins/songs/lib/lyrics_xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -49,7 +49,7 @@ class SongXMLBuilder(object): """ log.info(u'SongXMLBuilder Loaded') - def __init__(self, song_language = None): + def __init__(self, song_language=None): """ Set up the song builder. @@ -85,14 +85,14 @@ class SongXMLBuilder(object): """ Debugging aid to dump XML so that we can see what we have. """ - return etree.tostring(self.song_xml, encoding = u'UTF-8', + return etree.tostring(self.song_xml, encoding=u'UTF-8', xml_declaration=True, pretty_print=True) def extract_xml(self): """ Extract our newly created XML song. """ - return etree.tostring(self.song_xml, encoding = u'UTF-8', + return etree.tostring(self.song_xml, encoding=u'UTF-8', xml_declaration=True) From bb34a304a9be75c43e02f3cc3768259e536894ca Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Thu, 15 Jul 2010 21:27:44 +0100 Subject: [PATCH 25/42] Tweaked Topics handler --- openlp/plugins/songs/lib/songimport.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 893def7f0..e5045a5f5 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -190,8 +190,8 @@ class SongImport(object): """ Add the author. OpenLP stores them individually so split by 'and', '&' and comma. - However need to check for "Mr and Mrs Smith" and turn it to - "Mr Smith" and "Mrs Smith". + However need to check for 'Mr and Mrs Smith' and turn it to + 'Mr Smith' and 'Mrs Smith'. """ for author in text.split(u','): authors = author.split(u'&') @@ -325,12 +325,14 @@ class SongImport(object): self.manager.save_object(song_book) song.song_book_id = song_book.id for topictext in self.topics: - topic = self.manager.get_object_filtered(Topic.name == topictext) + if len(topictext) == 0: + continue + topic = self.manager.get_object_filtered(Topic, Topic.name == topictext) if topic is None: topic = Topic() topic.name = topictext self.manager.save_object(topic) - song.topics.append(topictext) + song.topics.append(topic) self.manager.save_object(song) def print_song(self): From 059032f23736a18990cee8a122c37180c69ec925 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Thu, 15 Jul 2010 21:29:24 +0100 Subject: [PATCH 26/42] Changed internal 'song' attribute to clearer 'song_import'. OpenSong themes now map to Topics. Text cleaning called --- openlp/plugins/songs/lib/opensongimport.py | 30 +++---- .../songs/lib/test/test_importing_lots.py | 5 +- .../songs/lib/test/test_opensongimport.py | 86 +++++++++---------- 3 files changed, 60 insertions(+), 61 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 93c682d90..b8d30c535 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -38,7 +38,7 @@ log = logging.getLogger(__name__) class OpenSongImportError(Exception): pass -class OpenSongImport: +class OpenSongImport(object): """ Import songs exported from OpenSong - the format is described loosly here: http://www.opensong.org/d/manual/song_file_format_specification @@ -130,27 +130,25 @@ class OpenSongImport: Process the OpenSong file - pass in a file-like object, not a filename """ - self.song = SongImport(self.songmanager) + self.song_import = SongImport(self.songmanager) tree = objectify.parse(file) root = tree.getroot() fields = dir(root) - decode = {u'copyright':self.song.add_copyright, - u'ccli':self.song.set_ccli_number, - u'author':self.song.parse_author, - u'title':self.song.set_title, - u'aka':self.song.set_alternate_title, - u'hymn_number':self.song.set_song_number} + decode = {u'copyright':self.song_import.add_copyright, + u'ccli':self.song_import.set_ccli_number, + u'author':self.song_import.parse_author, + u'title':self.song_import.set_title, + u'aka':self.song_import.set_alternate_title, + u'hymn_number':self.song_import.set_song_number} for (attr, fn) in decode.items(): if attr in fields: fn(unicode(root.__getattr__(attr))) res = [] if u'theme' in fields: - res.append(unicode(root.theme)) + self.song_import.topics.append(unicode(root.theme)) if u'alttheme' in fields: - res.append(unicode(root.alttheme)) - self.song.theme_name = u', '.join(res) - + self.song_import.topics.append(unicode(root.alttheme)) # data storage while importing verses = {} lyrics = unicode(root.lyrics) @@ -207,7 +205,7 @@ class OpenSongImport: our_verse_order.append(versetag) if words: # Tidy text and remove the ____s from extended words - # words=self.song.tidy_text(words) + words=self.song_import.tidy_text(words) words=words.replace('_', '') verses[versetype][versenum].append(words) # done parsing @@ -221,7 +219,7 @@ class OpenSongImport: for n in versenums: versetag = u'%s%s' %(v,n) lines = u'\n'.join(verses[v][n]) - self.song.verses.append([versetag, lines]) + self.song_import.verses.append([versetag, lines]) versetags[versetag] = 1 # keep track of what we have for error checking later # now figure out the presentation order if u'presentation' in fields and root.presentation != u'': @@ -236,7 +234,7 @@ class OpenSongImport: if not versetags.has_key(tag): log.warn(u'Got order %s but not in versetags, skipping', tag) else: - self.song.verse_order_list.append(tag) + self.song_import.verse_order_list.append(tag) def finish(self): """ Separate function, allows test suite to not pollute database""" - self.song.finish() + self.song_import.finish() diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index dc3eeeb2a..9f1908bca 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -7,6 +7,7 @@ import os from traceback import print_exc import sys import codecs + def opensong_import_lots(): ziploc = u'/home/mjt/openlp/OpenSong_Data/' files = [] @@ -34,7 +35,7 @@ def opensong_import_lots(): o = OpenSongImport(manager) try: o.do_import_file(songfile) - o.song.print_song() + # o.song_import.print_song() except: print "Failure", @@ -51,6 +52,6 @@ def opensong_import_lots(): #o.finish() print "OK" #os.unlink(filename) - # o.song.print_song() + # o.song_import.print_song() if __name__ == "__main__": opensong_import_lots() diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index aede61416..7f6c0f45f 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -33,55 +33,55 @@ def test(): o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) o.finish() - o.song.print_song() - assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'MartiÑ Thómpson'] - assert o.song.title == u'Martins Test' - assert o.song.alternate_title == u'' - assert o.song.song_number == u'1' - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses - assert [u'C1', u'Chorus 1'] in o.song.verses - assert [u'C2', u'Chorus 2'] in o.song.verses - assert not [u'C3', u'Chorus 3'] in o.song.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses - assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] - assert o.song.ccli_number == u'Blah' - assert o.song.theme_name == u'TestTheme, TestAltTheme' + o.song_import.print_song() + assert o.song_import.copyright == u'2010 Martin Thompson' + assert o.song_import.authors == [u'MartiÑ Thómpson'] + assert o.song_import.title == u'Martins Test' + assert o.song_import.alternate_title == u'' + assert o.song_import.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses + assert [u'C1', u'Chorus 1'] in o.song_import.verses + assert [u'C2', u'Chorus 2'] in o.song_import.verses + assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses + assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] + assert o.song_import.ccli_number == u'Blah' + assert o.song_import.topics == [u'TestTheme', u'TestAltTheme'] o.do_import(u'test.opensong.zip', commit=False) + o.song_import.print_song() o.finish() - o.song.print_song() - assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'MartiÑ Thómpson'] - assert o.song.title == u'Martins Test' - assert o.song.alternate_title == u'' - assert o.song.song_number == u'1' - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses - assert [u'C1', u'Chorus 1'] in o.song.verses - assert [u'C2', u'Chorus 2'] in o.song.verses - assert not [u'C3', u'Chorus 3'] in o.song.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses - assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] + assert o.song_import.copyright == u'2010 Martin Thompson' + assert o.song_import.authors == [u'MartiÑ Thómpson'] + assert o.song_import.title == u'Martins Test' + assert o.song_import.alternate_title == u'' + assert o.song_import.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses + assert [u'C1', u'Chorus 1'] in o.song_import.verses + assert [u'C2', u'Chorus 2'] in o.song_import.verses + assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses + assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] o = OpenSongImport(manager) o.do_import(u'test2.opensong', commit=False) # o.finish() - o.song.print_song() - assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'Martin Thompson'] - assert o.song.title == u'Martins 2nd Test' - assert o.song.alternate_title == u'' - assert o.song.song_number == u'2' - print o.song.verses - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses - assert [u'C1', u'Chorus 1'] in o.song.verses - assert [u'C2', u'Chorus 2'] in o.song.verses - assert not [u'C3', u'Chorus 3'] in o.song.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses - print o.song.verse_order_list - assert o.song.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] + o.song_import.print_song() + assert o.song_import.copyright == u'2010 Martin Thompson' + assert o.song_import.authors == [u'Martin Thompson'] + assert o.song_import.title == u'Martins 2nd Test' + assert o.song_import.alternate_title == u'' + assert o.song_import.song_number == u'2' + print o.song_import.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses + assert [u'C1', u'Chorus 1'] in o.song_import.verses + assert [u'C2', u'Chorus 2'] in o.song_import.verses + assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses + print o.song_import.verse_order_list + assert o.song_import.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] print "Tests passed" pass From f964c5d1ce57e0d66f04a2309bde5dcbf1238d31 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Sat, 17 Jul 2010 18:36:03 +0100 Subject: [PATCH 27/42] Add alternate_title field --- openlp/plugins/songs/forms/editsongform.py | 5 ++--- openlp/plugins/songs/lib/db.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index c0db7b741..e2646b07b 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -180,7 +180,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): self.loadBooks() self.song = self.songmanager.get_object(Song, id) self.TitleEditItem.setText(self.song.title) - title = self.song.search_title.split(u'@') + self.AlternativeEdit.setText(self.song.alternate_title) if self.song.song_book_id != 0: book_name = self.songmanager.get_object(Book, self.song.song_book_id) @@ -198,8 +198,6 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): id = 0 self.song.theme_name = None self.ThemeSelectionComboItem.setCurrentIndex(id) - if len(title) > 1: - self.AlternativeEdit.setText(title[1]) if self.song.copyright: self.CopyrightEditItem.setText(self.song.copyright) else: @@ -622,6 +620,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): def saveSong(self): self.song.title = unicode(self.TitleEditItem.text()) + self.song.alternate_title = unicode(self.AlternativeEdit.text()) self.song.copyright = unicode(self.CopyrightEditItem.text()) self.song.search_title = self.song.title + u'@' + \ unicode(self.AlternativeEdit.text()) diff --git a/openlp/plugins/songs/lib/db.py b/openlp/plugins/songs/lib/db.py index 655043144..794501cc7 100644 --- a/openlp/plugins/songs/lib/db.py +++ b/openlp/plugins/songs/lib/db.py @@ -88,6 +88,7 @@ def init_schema(url): Column(u'song_book_id', types.Integer, ForeignKey(u'song_books.id'), default=0), Column(u'title', types.Unicode(255), nullable=False), + Column(u'alternate_title', types.Unicode(255)), Column(u'lyrics', types.UnicodeText, nullable=False), Column(u'verse_order', types.Unicode(128)), Column(u'copyright', types.Unicode(255)), From 168aeea3e1de937d71efc8f50520c2c8dd6b78cf Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Sun, 18 Jul 2010 20:27:27 +0100 Subject: [PATCH 28/42] Tweaks from review --- openlp/plugins/songs/lib/opensongimport.py | 58 ++++++++++--------- .../songs/lib/test/test_opensongimport.py | 3 +- openlp/plugins/songs/lib/xml.py | 3 +- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index b8d30c535..4974f79f7 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -26,11 +26,12 @@ import os import re -from songimport import SongImport +from zipfile import ZipFile + from lxml.etree import Element from lxml import objectify -from zipfile import ZipFile +from openlp.plugins.songs.lib.songimport import SongImport import logging log = logging.getLogger(__name__) @@ -104,12 +105,12 @@ class OpenSongImport(object): set False, the import will not be committed to the database (useful for test scripts) """ - ext=os.path.splitext(filename)[1] + ext = os.path.splitext(filename)[1] if ext.lower() == ".zip": log.info('Zipfile found %s', filename) - z=ZipFile(filename, u'r') + z = ZipFile(filename, u'r') for song in z.infolist(): - parts=os.path.split(song.filename) + parts = os.path.split(song.filename) if parts[-1] == u'': #No final part => directory continue @@ -158,28 +159,29 @@ class OpenSongImport(object): # in the absence of any other indication, verses are the default, # erm, versetype! versetype = u'V' - for l in lyrics.split(u'\n'): + for thisline in lyrics.split(u'\n'): # remove comments - semicolon = l.find(u';') + semicolon = thisline.find(u';') if semicolon >= 0: - l = l[:semicolon] - l = l.strip() - if len(l) == 0: + thisline = thisline[:semicolon] + thisline = thisline.strip() + if len(thisline) == 0: continue - # skip inline guitar chords and page and column breaks - if l[0] == u'.' or l.startswith(u'---') or l.startswith(u'-!!'): + # skip inthisline guitar chords and page and column breaks + if thisline[0] == u'.' or thisline.startswith(u'---') \ + or thisline.startswith(u'-!!'): continue # verse/chorus/etc. marker - if l[0] == u'[': - versetype = l[1].upper() + if thisline[0] == u'[': + versetype = thisline[1].upper() if versetype.isdigit(): versenum = versetype versetype = u'V' - elif l[2] != u']': + elif thisline[2] != u']': # there's a number to go with it - extract that as well - right_bracket = l.find(u']') - versenum = l[2:right_bracket] + right_bracket = thisline.find(u']') + versenum = thisline[2:right_bracket] else: # if there's no number, assume it's no.1 versenum = u'1' @@ -187,13 +189,13 @@ class OpenSongImport(object): words = None # number at start of line.. it's verse number - if l[0].isdigit(): - versenum = l[0] - words = l[1:].strip() + if thisline[0].isdigit(): + versenum = thisline[0] + words = thisline[1:].strip() if words is None and \ versenum is not None and \ versetype is not None: - words = l + words = thisline if versenum is not None: versetag = u'%s%s'%(versetype,versenum) if not verses.has_key(versetype): @@ -205,20 +207,20 @@ class OpenSongImport(object): our_verse_order.append(versetag) if words: # Tidy text and remove the ____s from extended words - words=self.song_import.tidy_text(words) - words=words.replace('_', '') + words = self.song_import.tidy_text(words) + words = words.replace('_', '') verses[versetype][versenum].append(words) # done parsing versetypes = verses.keys() versetypes.sort() versetags = {} verse_renames = {} - for v in versetypes: - versenums = verses[v].keys() + for versetype in versetypes: + versenums = verses[versetype].keys() versenums.sort() - for n in versenums: - versetag = u'%s%s' %(v,n) - lines = u'\n'.join(verses[v][n]) + for num in versenums: + versetag = u'%s%s' %(versetype,num) + lines = u'\n'.join(verses[versetype][num]) self.song_import.verses.append([versetag, lines]) versetags[versetag] = 1 # keep track of what we have for error checking later # now figure out the presentation order diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 7f6c0f45f..8c974adbc 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -32,17 +32,16 @@ def test(): manager = Manager(u'songs', init_schema) o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) - o.finish() o.song_import.print_song() assert o.song_import.copyright == u'2010 Martin Thompson' assert o.song_import.authors == [u'MartiÑ Thómpson'] assert o.song_import.title == u'Martins Test' assert o.song_import.alternate_title == u'' assert o.song_import.song_number == u'1' - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses assert [u'C1', u'Chorus 1'] in o.song_import.verses assert [u'C2', u'Chorus 2'] in o.song_import.verses assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 1e9678c55..336c1ebf1 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -77,7 +77,8 @@ class SongXMLBuilder(object): The actual text of the verse to be stored. """ # log.debug(u'add_verse_to_lyrics %s, %s\n%s' % (type, number, content)) - verse = etree.Element(u'verse', type = unicode(type), label = unicode(number)) + verse = etree.Element(u'verse', type = unicode(type), + label = unicode(number)) verse.text = etree.CDATA(content) self.lyrics.append(verse) From f71f27c890886ea9328a5edc8891f4956f90b259 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Mon, 19 Jul 2010 00:37:24 +0100 Subject: [PATCH 29/42] Database refactor --- openlp/core/lib/db.py | 38 ++++----- openlp/plugins/alerts/forms/alertform.py | 3 +- openlp/plugins/bibles/lib/manager.py | 3 +- openlp/plugins/custom/customplugin.py | 6 +- openlp/plugins/custom/lib/mediaitem.py | 2 +- openlp/plugins/songs/forms/editsongform.py | 8 +- .../songs/forms/songmaintenanceform.py | 14 ++-- openlp/plugins/songs/lib/mediaitem.py | 8 +- openlp/plugins/songs/songsplugin.py | 7 +- .../songusage/forms/songusagedeleteform.py | 7 +- .../songusage/forms/songusagedetailform.py | 10 ++- openlp/plugins/songusage/lib/__init__.py | 1 - openlp/plugins/songusage/lib/manager.py | 81 ------------------- openlp/plugins/songusage/songusageplugin.py | 8 +- 14 files changed, 57 insertions(+), 139 deletions(-) delete mode 100644 openlp/plugins/songusage/lib/manager.py diff --git a/openlp/core/lib/db.py b/openlp/core/lib/db.py index 8acc79541..735df4efb 100644 --- a/openlp/core/lib/db.py +++ b/openlp/core/lib/db.py @@ -178,36 +178,24 @@ class Manager(object): """ return self.session.query(object_class).filter(filter_clause).first() - def get_all_objects(self, object_class, order_by_ref=None): + def get_all_objects(self, object_class, filter_clause=None, + order_by_ref=None): """ Returns all the objects from the database ``object_class`` The type of objects to return + ``filter_clause`` + The filter governing selection of objects to return. Defaults to + None. + ``order_by_ref`` Any parameters to order the returned objects by. Defaults to None. """ query = self.session.query(object_class) - if order_by_ref is not None: - return query.order_by(order_by_ref).all() - return query.all() - - def get_all_objects_filtered(self, object_class, filter_clause, - order_by_ref=None): - """ - Returns a selection of objects from the database - - ``object_class`` - The type of objects to return - - ``filter_clause`` - The filter governing selection of objects to return - - ``order_by_ref`` - Any parameters to order the returned objects by. Defaults to None. - """ - query = self.session.query(object_class).filter(filter_clause) + if filter_clause: + query = query.filter(filter_clause) if order_by_ref is not None: return query.order_by(order_by_ref).all() return query.all() @@ -235,7 +223,7 @@ class Manager(object): else: return True - def delete_all_objects(self, object_class): + def delete_all_objects(self, object_class, filter_clause=None): """ Delete all object records @@ -243,11 +231,13 @@ class Manager(object): The type of object to delete """ try: - self.session.query(object_class).delete(synchronize_session=False) + query = self.session.query(object_class) + if filter_clause: + query = query.filter(filter_clause) + query.delete(synchronize_session=False) self.session.commit() return True except InvalidRequestError: self.session.rollback() - log.exception(u'Failed to delete all %s records', - object_class.__name__) + log.exception(u'Failed to delete %s records', object_class.__name__) return False diff --git a/openlp/plugins/alerts/forms/alertform.py b/openlp/plugins/alerts/forms/alertform.py index 388d35751..71edd8e9f 100644 --- a/openlp/plugins/alerts/forms/alertform.py +++ b/openlp/plugins/alerts/forms/alertform.py @@ -62,7 +62,8 @@ class AlertForm(QtGui.QDialog, Ui_AlertDialog): def loadList(self): self.AlertListWidget.clear() - alerts = self.manager.get_all_objects(AlertItem, AlertItem.text) + alerts = self.manager.get_all_objects(AlertItem, + order_by_ref=AlertItem.text) for alert in alerts: item_name = QtGui.QListWidgetItem(alert.text) item_name.setData(QtCore.Qt.UserRole, QtCore.QVariant(alert.id)) diff --git a/openlp/plugins/bibles/lib/manager.py b/openlp/plugins/bibles/lib/manager.py index 39f3c255b..f862e1d1c 100644 --- a/openlp/plugins/bibles/lib/manager.py +++ b/openlp/plugins/bibles/lib/manager.py @@ -198,7 +198,8 @@ class BibleManager(object): u'name': book.name, u'chapters': self.db_cache[bible].get_chapter_count(book.name) } - for book in self.db_cache[bible].get_all_objects(Book, Book.id) + for book in self.db_cache[bible].get_all_objects(Book, + order_by_ref=Book.id) ] def get_chapter_count(self, bible, book): diff --git a/openlp/plugins/custom/customplugin.py b/openlp/plugins/custom/customplugin.py index 2cdbac362..79ccb1f9b 100644 --- a/openlp/plugins/custom/customplugin.py +++ b/openlp/plugins/custom/customplugin.py @@ -75,7 +75,7 @@ class CustomPlugin(Plugin): Returns True if the theme is being used, otherwise returns False. """ - if self.custommanager.get_all_objects_filtered(CustomSlide, + if self.custommanager.get_all_objects(CustomSlide, CustomSlide.theme_name == theme): return True return False @@ -91,8 +91,8 @@ class CustomPlugin(Plugin): ``newTheme`` The new name the plugin should now use. """ - customsUsingTheme = self.custommanager.get_all_objects_filtered( - CustomSlide, CustomSlide.theme_name == oldTheme) + customsUsingTheme = self.custommanager.get_all_objects(CustomSlide, + CustomSlide.theme_name == oldTheme) for custom in customsUsingTheme: custom.theme_name = newTheme self.custommanager.save_object(custom) diff --git a/openlp/plugins/custom/lib/mediaitem.py b/openlp/plugins/custom/lib/mediaitem.py index 34ffeeac4..cdea8cb48 100644 --- a/openlp/plugins/custom/lib/mediaitem.py +++ b/openlp/plugins/custom/lib/mediaitem.py @@ -73,7 +73,7 @@ class CustomMediaItem(MediaManagerItem): def initialise(self): self.loadCustomListView(self.parent.custommanager.get_all_objects( - CustomSlide, CustomSlide.title)) + CustomSlide, order_by_ref=CustomSlide.title)) #Called to redisplay the song list screen edith from a search #or from the exit of the Song edit dialog. If remote editing is active #Trigger it and clean up so it will not update again. diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index e2646b07b..3fa801595 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -118,7 +118,8 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): self.TopicRemoveButton.setEnabled(False) def loadAuthors(self): - authors = self.songmanager.get_all_objects(Author, Author.display_name) + authors = self.songmanager.get_all_objects(Author, + order_by_ref=Author.display_name) self.AuthorsSelectionComboItem.clear() self.AuthorsSelectionComboItem.addItem(u'') for author in authors: @@ -128,7 +129,8 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): row, QtCore.QVariant(author.id)) def loadTopics(self): - topics = self.songmanager.get_all_objects(Topic, Topic.name) + topics = self.songmanager.get_all_objects(Topic, + order_by_ref=Topic.name) self.SongTopicCombo.clear() self.SongTopicCombo.addItem(u'') for topic in topics: @@ -137,7 +139,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): self.SongTopicCombo.setItemData(row, QtCore.QVariant(topic.id)) def loadBooks(self): - books = self.songmanager.get_all_objects(Book, Book.name) + books = self.songmanager.get_all_objects(Book, order_by_ref=Book.name) self.SongbookCombo.clear() self.SongbookCombo.addItem(u'') for book in books: diff --git a/openlp/plugins/songs/forms/songmaintenanceform.py b/openlp/plugins/songs/forms/songmaintenanceform.py index 2a848ad98..259dbcaf7 100644 --- a/openlp/plugins/songs/forms/songmaintenanceform.py +++ b/openlp/plugins/songs/forms/songmaintenanceform.py @@ -102,7 +102,8 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): Reloads the Authors list. """ self.AuthorsListWidget.clear() - authors = self.songmanager.get_all_objects(Author, Author.display_name) + authors = self.songmanager.get_all_objects(Author, + order_by_ref=Author.display_name) for author in authors: if author.display_name: author_name = QtGui.QListWidgetItem(author.display_name) @@ -117,7 +118,8 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): Reloads the Topics list. """ self.TopicsListWidget.clear() - topics = self.songmanager.get_all_objects(Topic, Topic.name) + topics = self.songmanager.get_all_objects(Topic, + order_by_ref=Topic.name) for topic in topics: topic_name = QtGui.QListWidgetItem(topic.name) topic_name.setData(QtCore.Qt.UserRole, QtCore.QVariant(topic.id)) @@ -128,7 +130,7 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): Reloads the Books list. """ self.BooksListWidget.clear() - books = self.songmanager.get_all_objects(Book, Book.name) + books = self.songmanager.get_all_objects(Book, order_by_ref=Book.name) for book in books: book_name = QtGui.QListWidgetItem(u'%s (%s)' % (book.name, book.publisher)) @@ -140,7 +142,7 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): Returns False if the given Author is already in the list otherwise True. """ - authors = self.songmanager.get_all_objects_filtered(Author, + authors = self.songmanager.get_all_objects(Author, and_( Author.first_name == new_author.first_name, Author.last_name == new_author.last_name, @@ -165,7 +167,7 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): """ Returns False if the given Topic is already in the list otherwise True. """ - topics = self.songmanager.get_all_objects_filtered(Topic, + topics = self.songmanager.get_all_objects(Topic, Topic.name == new_topic.name) if len(topics) > 0: # If we edit an existing Topic, we need to make sure that we do @@ -185,7 +187,7 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): """ Returns False if the given Book is already in the list otherwise True. """ - books = self.songmanager.get_all_objects_filtered(Book, + books = self.songmanager.get_all_objects(Book, and_(Book.name == new_book.name, Book.publisher == new_book.publisher)) if len(books) > 0: diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index 93f376d1f..0b9ef789d 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -164,20 +164,20 @@ class SongMediaItem(MediaManagerItem): search_type = self.SearchTypeComboBox.currentIndex() if search_type == 0: log.debug(u'Titles Search') - search_results = self.parent.manager.get_all_objects_filtered(Song, + search_results = self.parent.manager.get_all_objects(Song, Song.search_title.like(u'%' + search_keywords + u'%'), Song.search_title.asc()) self.displayResultsSong(search_results) elif search_type == 1: log.debug(u'Lyrics Search') - search_results = self.parent.manager.get_all_objects_filtered(Song, + search_results = self.parent.manager.get_all_objects(Song, Song.search_lyrics.like(u'%' + search_keywords + u'%'), Song.search_lyrics.asc()) self.displayResultsSong(search_results) elif search_type == 2: log.debug(u'Authors Search') - search_results = self.parent.manager.get_all_objects_filtered( - Author, Author.display_name.like(u'%' + search_keywords + u'%'), + search_results = self.parent.manager.get_all_objects(Author, + Author.display_name.like(u'%' + search_keywords + u'%'), Author.display_name.asc()) self.displayResultsAuthor(search_results) #Called to redisplay the song list screen edith from a search diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index e7be7cae3..11a057f92 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -69,7 +69,7 @@ class SongsPlugin(Plugin): log.info(u'Songs Initialising') Plugin.initialise(self) self.mediaItem.displayResultsSong( - self.manager.get_all_objects(Song, Song.title)) + self.manager.get_all_objects(Song, order_by_ref=Song.title)) def getMediaManagerItem(self): """ @@ -198,8 +198,7 @@ class SongsPlugin(Plugin): Returns True if the theme is being used, otherwise returns False. """ - if self.manager.get_all_objects_filtered(Song, - Song.theme_name == theme): + if self.manager.get_all_objects(Song, Song.theme_name == theme): return True return False @@ -214,7 +213,7 @@ class SongsPlugin(Plugin): ``newTheme`` The new name the plugin should now use. """ - songsUsingTheme = self.manager.get_all_objects_filtered(Song, + songsUsingTheme = self.manager.get_all_objects(Song, Song.theme_name == oldTheme) for song in songsUsingTheme: song.theme_name = newTheme diff --git a/openlp/plugins/songusage/forms/songusagedeleteform.py b/openlp/plugins/songusage/forms/songusagedeleteform.py index 97e032413..4ded872f0 100644 --- a/openlp/plugins/songusage/forms/songusagedeleteform.py +++ b/openlp/plugins/songusage/forms/songusagedeleteform.py @@ -25,8 +25,9 @@ from PyQt4 import QtGui -from songusagedeletedialog import Ui_SongUsageDeleteDialog from openlp.core.lib import translate +from openlp.plugins.songusage.lib.db import SongUsageItem +from songusagedeletedialog import Ui_SongUsageDeleteDialog class SongUsageDeleteForm(QtGui.QDialog, Ui_SongUsageDeleteDialog): """ @@ -52,6 +53,6 @@ class SongUsageDeleteForm(QtGui.QDialog, Ui_SongUsageDeleteDialog): QtGui.QMessageBox.Cancel) if ret == QtGui.QMessageBox.Ok: deleteDate = self.DeleteCalendar.selectedDate().toPyDate() - self.songusagemanager.delete_to_date(deleteDate) + self.songusagemanager.delete_all_objects(SongUsageItem, + SongUsageItem.usagedate <= deleteDate) self.close() - diff --git a/openlp/plugins/songusage/forms/songusagedetailform.py b/openlp/plugins/songusage/forms/songusagedetailform.py index be1b8221c..ac65ce857 100644 --- a/openlp/plugins/songusage/forms/songusagedetailform.py +++ b/openlp/plugins/songusage/forms/songusagedetailform.py @@ -27,9 +27,10 @@ import logging import os from PyQt4 import QtCore, QtGui +from sqlalchemy.sql import and_ from openlp.core.lib import SettingsManager, translate - +from openlp.plugins.songusage.lib.db import SongUsageItem from songusagedetaildialog import Ui_SongUsageDetailDialog log = logging.getLogger(__name__) @@ -74,8 +75,11 @@ class SongUsageDetailForm(QtGui.QDialog, Ui_SongUsageDetailDialog): filename = u'usage_detail_%s_%s.txt' % ( self.FromDate.selectedDate().toString(u'ddMMyyyy'), self.ToDate.selectedDate().toString(u'ddMMyyyy')) - usage = self.parent.songusagemanager.get_songusage_for_period( - self.FromDate.selectedDate(), self.ToDate.selectedDate()) + usage = self.parent.songusagemanager.get_all_objects( + SongUsageItem, and_( + SongUsageItem.usagedate >= self.FromDate.selectedDate().toPyDate(), + SongUsageItem.usagedate < self.ToDate.selectedDate().toPyDate()), + [SongUsageItem.usagedate, SongUsageItem.usagetime]) outname = os.path.join(unicode(self.FileLineEdit.text()), filename) file = None try: diff --git a/openlp/plugins/songusage/lib/__init__.py b/openlp/plugins/songusage/lib/__init__.py index ae8425317..bd83e0532 100644 --- a/openlp/plugins/songusage/lib/__init__.py +++ b/openlp/plugins/songusage/lib/__init__.py @@ -25,4 +25,3 @@ """ The :mod:`lib` module contains the library functions for the songusage plugin. """ -from openlp.plugins.songusage.lib.manager import SongUsageManager diff --git a/openlp/plugins/songusage/lib/manager.py b/openlp/plugins/songusage/lib/manager.py deleted file mode 100644 index 2c34f3a54..000000000 --- a/openlp/plugins/songusage/lib/manager.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2010 Raoul Snyman # -# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # -# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin # -# Thompson, Jon Tibble, Carsten Tinggaard # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### -""" -The :mod:`manager` module provides song usage specific database query code -""" -import logging - -from sqlalchemy.exceptions import InvalidRequestError - -from openlp.core.lib.db import Manager -from openlp.plugins.songusage.lib.db import init_schema, SongUsageItem - -log = logging.getLogger(__name__) - -class SongUsageManager(Manager): - """ - The Song Manager provides a central location for all database code. This - class takes care of connecting to the database and running all the queries. - """ - log.info(u'SongUsage manager loaded') - - def __init__(self): - """ - Creates the connection to the database, and creates the tables if they - don't exist. - """ - log.debug(u'SongUsage Initialising') - Manager.__init__(self, u'songusage', init_schema) - log.debug(u'SongUsage Initialised') - - def get_songusage_for_period(self, start_date, end_date): - """ - Returns the details of SongUsage for a designated time period - - ``start_date`` - The start of the period to return - - ``end_date`` - The end of the period to return - """ - return self.session.query(SongUsageItem) \ - .filter(SongUsageItem.usagedate >= start_date.toPyDate()) \ - .filter(SongUsageItem.usagedate < end_date.toPyDate()) \ - .order_by(SongUsageItem.usagedate, SongUsageItem.usagetime).all() - - def delete_to_date(self, date): - """ - Delete SongUsage records before given date - """ - try: - self.session.query(SongUsageItem) \ - .filter(SongUsageItem.usagedate <= date) \ - .delete(synchronize_session=False) - self.session.commit() - return True - except InvalidRequestError: - self.session.rollback() - log.exception(u'Failed to delete all Song Usage items to %s' % date) - return False diff --git a/openlp/plugins/songusage/songusageplugin.py b/openlp/plugins/songusage/songusageplugin.py index 07e7271a1..c7a8a30aa 100644 --- a/openlp/plugins/songusage/songusageplugin.py +++ b/openlp/plugins/songusage/songusageplugin.py @@ -24,15 +24,15 @@ ############################################################################### import logging - from datetime import datetime + from PyQt4 import QtCore, QtGui from openlp.core.lib import Plugin, Receiver, build_icon, translate -from openlp.plugins.songusage.lib import SongUsageManager +from openlp.core.lib.db import Manager from openlp.plugins.songusage.forms import SongUsageDetailForm, \ SongUsageDeleteForm -from openlp.plugins.songusage.lib.db import SongUsageItem +from openlp.plugins.songusage.lib.db import init_schema, SongUsageItem log = logging.getLogger(__name__) @@ -117,7 +117,7 @@ class SongUsagePlugin(Plugin): QtCore.QVariant(False)).toBool() self.SongUsageStatus.setChecked(self.SongUsageActive) if self.songusagemanager is None: - self.songusagemanager = SongUsageManager() + self.songusagemanager = Manager(u'songusage', init_schema) self.SongUsagedeleteform = SongUsageDeleteForm(self.songusagemanager) self.SongUsagedetailform = SongUsageDetailForm(self) self.SongUsageMenu.menuAction().setVisible(True) From dd8c6f378b6e03b49938af032c95eaeb590224e8 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Mon, 19 Jul 2010 09:50:16 +0100 Subject: [PATCH 30/42] Fix alt title field --- openlp/plugins/songs/forms/editsongform.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index 3fa801595..6a626e65a 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -182,7 +182,10 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): self.loadBooks() self.song = self.songmanager.get_object(Song, id) self.TitleEditItem.setText(self.song.title) - self.AlternativeEdit.setText(self.song.alternate_title) + if self.song.alternate_title: + self.AlternativeEdit.setText(self.song.alternate_title) + else: + self.AlternativeEdit.setText(u'') if self.song.song_book_id != 0: book_name = self.songmanager.get_object(Book, self.song.song_book_id) From 9e2b876be350d9d3460a475c73341a6dcdcdcc8c Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Mon, 19 Jul 2010 13:18:56 +0100 Subject: [PATCH 31/42] Add audio files to DB --- openlp/plugins/songs/lib/db.py | 36 +++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/lib/db.py b/openlp/plugins/songs/lib/db.py index 794501cc7..e090d7442 100644 --- a/openlp/plugins/songs/lib/db.py +++ b/openlp/plugins/songs/lib/db.py @@ -32,6 +32,12 @@ from sqlalchemy.orm import mapper, relation from openlp.core.lib.db import BaseModel, init_db +class AudioFile(BaseModel): + """ + AudioFile model + """ + pass + class Author(BaseModel): """ Author model @@ -75,6 +81,12 @@ def init_schema(url): Column(u'display_name', types.Unicode(255), nullable=False) ) + # Definition of the "audio_files" table + audio_files_table = Table(u'audio_files', metadata, + Column(u'id', types.Integer, primary_key=True), + Column(u'file_name', types.Unicode(255), nullable=False) + ) + # Definition of the "song_books" table song_books_table = Table(u'song_books', metadata, Column(u'id', types.Integer, primary_key=True), @@ -114,6 +126,14 @@ def init_schema(url): ForeignKey(u'songs.id'), primary_key=True) ) + # Definition of the "songs_audio_files" table + songs_audio_files_table = Table(u'songs_audio_files', metadata, + Column(u'song_id', types.Integer, + ForeignKey(u'songs.id'), primary_key=True), + Column(u'audio_file_id', types.Integer, + ForeignKey(u'audio_files.id'), primary_key=True) + ) + # Definition of the "songs_topics" table songs_topics_table = Table(u'songs_topics', metadata, Column(u'song_id', types.Integer, @@ -123,6 +143,7 @@ def init_schema(url): ) # Define table indexes + Index(u'audio_files_id', audio_files_table.c.id) Index(u'authors_id', authors_table.c.id) Index(u'authors_display_name_id', authors_table.c.display_name, authors_table.c.id) @@ -133,19 +154,28 @@ def init_schema(url): authors_songs_table.c.song_id) Index(u'authors_songs_song', authors_songs_table.c.song_id, authors_songs_table.c.author_id) + Index(u'songs_audio_files_file', songs_audio_files_table.c.audio_file_id, + songs_audio_files_table.c.song_id) + Index(u'songs_audio_files_song', songs_audio_files_table.c.song_id, + songs_audio_files_table.c.audio_file_id) Index(u'topics_song_topic', songs_topics_table.c.topic_id, songs_topics_table.c.song_id) Index(u'topics_song_song', songs_topics_table.c.song_id, songs_topics_table.c.topic_id) + mapper(AudioFile, audio_files_table) mapper(Author, authors_table) mapper(Book, song_books_table) mapper(Song, songs_table, - properties={'authors': relation(Author, backref='songs', - secondary=authors_songs_table), + properties={ + 'audio_files': relation(AudioFile, backref='songs', + secondary=songs_audio_files_table), + 'authors': relation(Author, backref='songs', + secondary=authors_songs_table), 'book': relation(Book, backref='songs'), 'topics': relation(Topic, backref='songs', - secondary=songs_topics_table)}) + secondary=songs_topics_table) + }) mapper(Topic, topics_table) metadata.create_all(checkfirst=True) From 86ad4b1071b8cfe72883ab72b8d6e30d0c886395 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Mon, 19 Jul 2010 16:39:42 +0100 Subject: [PATCH 32/42] Fix DB definition ordering --- openlp/plugins/songs/lib/db.py | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/openlp/plugins/songs/lib/db.py b/openlp/plugins/songs/lib/db.py index e090d7442..acf388035 100644 --- a/openlp/plugins/songs/lib/db.py +++ b/openlp/plugins/songs/lib/db.py @@ -73,6 +73,12 @@ def init_schema(url): """ session, metadata = init_db(url, auto_flush=False) + # Definition of the "audio_files" table + audio_files_table = Table(u'audio_files', metadata, + Column(u'id', types.Integer, primary_key=True), + Column(u'file_name', types.Unicode(255), nullable=False) + ) + # Definition of the "authors" table authors_table = Table(u'authors', metadata, Column(u'id', types.Integer, primary_key=True), @@ -81,12 +87,6 @@ def init_schema(url): Column(u'display_name', types.Unicode(255), nullable=False) ) - # Definition of the "audio_files" table - audio_files_table = Table(u'audio_files', metadata, - Column(u'id', types.Integer, primary_key=True), - Column(u'file_name', types.Unicode(255), nullable=False) - ) - # Definition of the "song_books" table song_books_table = Table(u'song_books', metadata, Column(u'id', types.Integer, primary_key=True), @@ -118,6 +118,14 @@ def init_schema(url): Column(u'name', types.Unicode(128), nullable=False) ) + # Definition of the "audio_files_songs" table + audio_files_songs_table = Table(u'audio_files_songs', metadata, + Column(u'audio_file_id', types.Integer, + ForeignKey(u'audio_files.id'), primary_key=True), + Column(u'song_id', types.Integer, + ForeignKey(u'songs.id'), primary_key=True) + ) + # Definition of the "authors_songs" table authors_songs_table = Table(u'authors_songs', metadata, Column(u'author_id', types.Integer, @@ -126,14 +134,6 @@ def init_schema(url): ForeignKey(u'songs.id'), primary_key=True) ) - # Definition of the "songs_audio_files" table - songs_audio_files_table = Table(u'songs_audio_files', metadata, - Column(u'song_id', types.Integer, - ForeignKey(u'songs.id'), primary_key=True), - Column(u'audio_file_id', types.Integer, - ForeignKey(u'audio_files.id'), primary_key=True) - ) - # Definition of the "songs_topics" table songs_topics_table = Table(u'songs_topics', metadata, Column(u'song_id', types.Integer, @@ -150,14 +150,14 @@ def init_schema(url): Index(u'song_books_id', song_books_table.c.id) Index(u'songs_id', songs_table.c.id) Index(u'topics_id', topics_table.c.id) + Index(u'audio_files_songs_file', audio_files_songs_table.c.audio_file_id, + audio_files_songs_table.c.song_id) + Index(u'audio_files_songs_song', audio_files_songs_table.c.song_id, + audio_files_songs_table.c.audio_file_id) Index(u'authors_songs_author', authors_songs_table.c.author_id, authors_songs_table.c.song_id) Index(u'authors_songs_song', authors_songs_table.c.song_id, authors_songs_table.c.author_id) - Index(u'songs_audio_files_file', songs_audio_files_table.c.audio_file_id, - songs_audio_files_table.c.song_id) - Index(u'songs_audio_files_song', songs_audio_files_table.c.song_id, - songs_audio_files_table.c.audio_file_id) Index(u'topics_song_topic', songs_topics_table.c.topic_id, songs_topics_table.c.song_id) Index(u'topics_song_song', songs_topics_table.c.song_id, @@ -169,7 +169,7 @@ def init_schema(url): mapper(Song, songs_table, properties={ 'audio_files': relation(AudioFile, backref='songs', - secondary=songs_audio_files_table), + secondary=audio_files_songs_table), 'authors': relation(Author, backref='songs', secondary=authors_songs_table), 'book': relation(Book, backref='songs'), From 978372ca9f82227e18bf83ce8f31f7bd57080d74 Mon Sep 17 00:00:00 2001 From: andreas Date: Mon, 19 Jul 2010 20:34:03 +0200 Subject: [PATCH 33/42] added check preventing adding a topic/author twice to the song --- openlp/plugins/songs/forms/editsongform.py | 31 ++++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index c0db7b741..0a3ed0a3d 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -304,10 +304,18 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): elif item > 0: item_id = (self.AuthorsSelectionComboItem.itemData(item)).toInt()[0] author = self.songmanager.get_object(Author, item_id) - self.song.authors.append(author) - author_item = QtGui.QListWidgetItem(unicode(author.display_name)) - author_item.setData(QtCore.Qt.UserRole, QtCore.QVariant(author.id)) - self.AuthorsListView.addItem(author_item) + if author in self.song.authors: + QtGui.QMessageBox.warning(self, + translate('SongsPlugin.EditSongForm', 'Error'), + translate('SongsPlugin.EditSongForm', 'This author is ' + 'already in the list.')) + else: + self.song.authors.append(author) + author_item = QtGui.QListWidgetItem(unicode( + author.display_name)) + author_item.setData(QtCore.Qt.UserRole, + QtCore.QVariant(author.id)) + self.AuthorsListView.addItem(author_item) self.AuthorsSelectionComboItem.setCurrentIndex(0) else: QtGui.QMessageBox.warning(self, @@ -355,10 +363,17 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): elif item > 0: item_id = (self.SongTopicCombo.itemData(item)).toInt()[0] topic = self.songmanager.get_object(Topic, item_id) - self.song.topics.append(topic) - topic_item = QtGui.QListWidgetItem(unicode(topic.name)) - topic_item.setData(QtCore.Qt.UserRole, QtCore.QVariant(topic.id)) - self.TopicsListView.addItem(topic_item) + if topic in self.song.topics: + QtGui.QMessageBox.warning(self, + translate('SongsPlugin.EditSongForm', 'Error'), + translate('SongsPlugin.EditSongForm', 'This topic is ' + 'already in the list.')) + else: + self.song.topics.append(topic) + topic_item = QtGui.QListWidgetItem(unicode(topic.name)) + topic_item.setData(QtCore.Qt.UserRole, + QtCore.QVariant(topic.id)) + self.TopicsListView.addItem(topic_item) self.SongTopicCombo.setCurrentIndex(0) else: QtGui.QMessageBox.warning(self, From acb9cca5d32373ea48278f6194ddddfd61417ea2 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Mon, 19 Jul 2010 20:35:33 +0100 Subject: [PATCH 34/42] Missed a rename --- openlp/plugins/songs/lib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index e6d56de20..8be820d82 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -137,7 +137,7 @@ class VerseType(object): unicode(VerseType.to_string(VerseType.Other)).lower(): return VerseType.Other -from lyrics_xml import LyricsXML, SongXMLBuilder, SongXMLParser +from xml import LyricsXML, SongXMLBuilder, SongXMLParser from songstab import SongsTab from mediaitem import SongMediaItem from songimport import SongImport From 42cff457623dc970241e624171732c3f8782f7f5 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Mon, 19 Jul 2010 21:45:53 +0100 Subject: [PATCH 35/42] Complete refactor for new code --- openlp/plugins/songs/forms/songmaintenanceform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/forms/songmaintenanceform.py b/openlp/plugins/songs/forms/songmaintenanceform.py index c9d3a7978..67f60bfee 100644 --- a/openlp/plugins/songs/forms/songmaintenanceform.py +++ b/openlp/plugins/songs/forms/songmaintenanceform.py @@ -394,7 +394,7 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): and_(Author.first_name == old_author.first_name, Author.last_name == old_author.last_name, Author.display_name == old_author.display_name)) - songs = self.songmanager.get_all_objects_filtered(Song, + songs = self.songmanager.get_all_objects(Song, Song.authors.contains(old_author)) for song in songs: # We check if the song has already existing_author as author. If @@ -414,7 +414,7 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): ''' existing_topic = self.songmanager.get_object_filtered(Topic, Topic.name == old_topic.name) - songs = self.songmanager.get_all_objects_filtered(Song, + songs = self.songmanager.get_all_objects(Song, Song.topics.contains(old_topic)) for song in songs: # We check if the song has already existing_topic as topic. If that @@ -435,7 +435,7 @@ class SongMaintenanceForm(QtGui.QDialog, Ui_SongMaintenanceDialog): existing_book = self.songmanager.get_object_filtered(Book, and_(Book.name == old_book.name, Book.publisher == old_book.publisher)) - songs = self.songmanager.get_all_objects_filtered(Song, + songs = self.songmanager.get_all_objects(Song, Song.song_book_id == old_book.id) for song in songs: song.song_book_id = existing_book.id From 80eb8e36af87086a5b08312ad08023487ed3abaf Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Mon, 19 Jul 2010 21:48:10 +0100 Subject: [PATCH 36/42] Import fix --- openlp/plugins/songs/lib/test/test_opensongimport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 8c974adbc..bcd74ba17 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -22,11 +22,10 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### + from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.core.lib.db import Manager from openlp.plugins.songs.lib.db import init_schema -from openlp.plugins.songs.songsplugin import SongsPlugin -import sys def test(): manager = Manager(u'songs', init_schema) From 7e623ea2268d3c4b09b0441b21749968b9dad7c5 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Tue, 20 Jul 2010 09:33:22 +0100 Subject: [PATCH 37/42] OpenLP v2 song DB importer --- openlp/plugins/songs/lib/__init__.py | 1 + openlp/plugins/songs/lib/olpimport.py | 198 +++++++++++++++++++++++++ openlp/plugins/songs/lib/songimport.py | 11 +- openlp/plugins/songs/songsplugin.py | 35 ++++- 4 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 openlp/plugins/songs/lib/olpimport.py diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index 8be820d82..0e2b93bd6 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -142,6 +142,7 @@ from songstab import SongsTab from mediaitem import SongMediaItem from songimport import SongImport from opensongimport import OpenSongImport +from olpimport import OpenLPSongImport try: from sofimport import SofImport from oooimport import OooImport diff --git a/openlp/plugins/songs/lib/olpimport.py b/openlp/plugins/songs/lib/olpimport.py new file mode 100644 index 000000000..433e59b28 --- /dev/null +++ b/openlp/plugins/songs/lib/olpimport.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin # +# Thompson, Jon Tibble, Carsten Tinggaard # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`olpimport` module provides the functionality for importing OpenLP +song databases into the current installation database. +""" +import logging + +from sqlalchemy import create_engine, MetaData +from sqlalchemy.orm import class_mapper, mapper, relation, scoped_session, \ + sessionmaker +from sqlalchemy.orm.exc import UnmappedClassError + +from openlp.core.lib.db import BaseModel +from openlp.plugins.songs.lib.db import Author, Book, Song, Topic #, AudioFile + +log = logging.getLogger(__name__) + +class OldAudioFile(BaseModel): + """ + AudioFile model + """ + pass + +class OldAuthor(BaseModel): + """ + Author model + """ + pass + +class OldBook(BaseModel): + """ + Book model + """ + pass + +class OldSong(BaseModel): + """ + Song model + """ + pass + +class OldTopic(BaseModel): + """ + Topic model + """ + pass + +class OpenLPSongImport(object): + """ + + """ + def __init__(self, master_manager, source_db): + """ + + """ + self.master_manager = master_manager + self.import_source = source_db + self.source_session = None + + def import_source_v2_db(self): + """ + + """ + engine = create_engine(self.import_source) + source_meta = MetaData() + source_meta.reflect(engine) + self.source_session = scoped_session(sessionmaker(bind=engine)) + if u'audio_files' in source_meta.tables.keys(): + has_audio_files = True + else: + has_audio_files = False + source_authors_table = source_meta.tables[u'authors'] + source_song_books_table = source_meta.tables[u'song_books'] + source_songs_table = source_meta.tables[u'songs'] + source_topics_table = source_meta.tables[u'topics'] + source_authors_songs_table = source_meta.tables[u'authors_songs'] + source_songs_topics_table = source_meta.tables[u'songs_topics'] + if has_audio_files: + source_audio_files_table = source_meta.tables[u'audio_files'] + source_audio_files_songs_table = \ + source_meta.tables[u'audio_files_songs'] + try: + class_mapper(OldAudioFile) + except UnmappedClassError: + mapper(OldAudioFile, source_audio_files_table) + song_props = { + 'authors': relation(OldAuthor, backref='songs', + secondary=source_authors_songs_table), + 'book': relation(OldBook, backref='songs'), + 'topics': relation(OldTopic, backref='songs', + secondary=source_songs_topics_table) + } + if has_audio_files: + song_props['audio_files'] = relation(OldAudioFile, backref='songs', + secondary=source_audio_files_songs_table) + try: + class_mapper(OldAuthor) + except UnmappedClassError: + mapper(OldAuthor, source_authors_table) + try: + class_mapper(OldBook) + except UnmappedClassError: + mapper(OldBook, source_song_books_table) + try: + class_mapper(OldSong) + except UnmappedClassError: + mapper(OldSong, source_songs_table, properties=song_props) + try: + class_mapper(OldTopic) + except UnmappedClassError: + mapper(OldTopic, source_topics_table) + + source_songs = self.source_session.query(OldSong).all() + for song in source_songs: + new_song = Song() + new_song.title = song.title + if has_audio_files: + new_song.alternate_title = song.alternate_title + else: + new_song.alternate_title = u'' + new_song.search_title = song.search_title + new_song.song_number = song.song_number + new_song.lyrics = song.lyrics + new_song.search_lyrics = song.search_lyrics + new_song.verse_order = song.verse_order + new_song.copyright = song.copyright + new_song.comments = song.comments + new_song.theme_name = song.theme_name + new_song.ccli_number = song.ccli_number + if song.authors: + for author in song.authors: + existing_author = self.master_manager.get_object_filtered( + Author, Author.display_name == author.display_name) + if existing_author: + new_song.authors.append(existing_author) + else: + new_song.authors.append(Author.populate( + first_name=author.first_name, + last_name=author.last_name, + display_name=author.display_name)) + else: + au = self.master_manager.get_object_filtered(Author, + Author.display_name == u'Author Unknown') + if au: + new_song.authors.append(au) + else: + new_song.authors.append(Author.populate( + display_name=u'Author Unknown')) + if song.song_book_id != 0: + existing_song_book = self.master_manager.get_object_filtered( + Book, Book.name == song.book.name) + if existing_song_book: + new_song.book = existing_song_book + else: + new_song.book = Book.populate(name=song.book.name, + publisher=song.book.publisher) + if song.topics: + for topic in song.topics: + existing_topic = self.master_manager.get_object_filtered( + Topic, Topic.name == topic.name) + if existing_topic: + new_song.topics.append(existing_topic) + else: + new_song.topics.append(Topic.populate(name=topic.name)) +# if has_audio_files: +# if song.audio_files: +# for audio_file in song.audio_files: +# existing_audio_file = \ +# self.master_manager.get_object_filtered(AudioFile, +# AudioFile.file_name == audio_file.file_name) +# if existing_audio_file: +# new_song.audio_files.remove(audio_file) +# new_song.audio_files.append(existing_audio_file) + self.master_manager.save_object(new_song) + engine.dispose() diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 9787d9e20..222bbeedd 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -50,7 +50,7 @@ class SongImport(object): self.song_number = u'' self.alternate_title = u'' self.copyright = u'' - self.comment = u'' + self.comments = u'' self.theme_name = u'' self.ccli_number = u'' self.authors = [] @@ -253,7 +253,7 @@ class SongImport(object): song.lyrics = unicode(sxml.extract_xml(), u'utf-8') song.verse_order = u' '.join(self.verse_order_list) song.copyright = self.copyright - song.comment = self.comment + song.comments = self.comments song.theme_name = self.theme_name song.ccli_number = self.ccli_number for authortext in self.authors: @@ -274,7 +274,8 @@ class SongImport(object): for topictext in self.topics: if len(topictext) == 0: continue - topic = self.manager.get_object_filtered(Topic, Topic.name == topictext) + topic = self.manager.get_object_filtered(Topic, + Topic.name == topictext) if topic is None: topic = Topic.populate(name=topictext) song.topics.append(topic) @@ -303,8 +304,8 @@ class SongImport(object): print u'NUMBER: ' + self.song_number for topictext in self.topics: print u'TOPIC: ' + topictext - if self.comment: - print u'COMMENT: ' + self.comment + if self.comments: + print u'COMMENTS: ' + self.comments if self.theme_name: print u'THEME: ' + self.theme_name if self.ccli_number: diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 69063ec19..5e1c94e57 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -30,7 +30,7 @@ from PyQt4 import QtCore, QtGui from openlp.core.lib import Plugin, build_icon, PluginStatus, Receiver, \ translate from openlp.core.lib.db import Manager -from openlp.plugins.songs.lib import SongMediaItem, SongsTab +from openlp.plugins.songs.lib import OpenLPSongImport, SongMediaItem, SongsTab from openlp.plugins.songs.lib.db import init_schema, Song try: @@ -157,7 +157,19 @@ class SongsPlugin(Plugin): import_menu.addAction(self.ImportOpenSongItem) QtCore.QObject.connect(self.ImportOpenSongItem, QtCore.SIGNAL(u'triggered()'), self.onImportOpenSongItemClick) - + # OpenLP v2 import menu item - ditto above regarding refactoring into + # an import wizard + self.ImportOpenLPSongItem = QtGui.QAction(import_menu) + self.ImportOpenLPSongItem.setObjectName(u'ImportOpenLPSongItem') + self.ImportOpenLPSongItem.setText(translate('SongsPlugin', + 'OpenLP v2 (temporary)')) + self.ImportOpenLPSongItem.setToolTip(translate('SongsPlugin', + 'Import an OpenLP v2 song database')) + self.ImportOpenLPSongItem.setStatusTip(translate('SongsPlugin', + 'Import an OpenLP v2 song database')) + import_menu.addAction(self.ImportOpenLPSongItem) + QtCore.QObject.connect(self.ImportOpenLPSongItem, + QtCore.SIGNAL(u'triggered()'), self.onImportOpenLPSongItemClick) def addExportMenuItem(self, export_menu): """ @@ -218,6 +230,25 @@ class SongsPlugin(Plugin): QtGui.QMessageBox.Ok) Receiver.send_message(u'songs_load_list') + def onImportOpenLPSongItemClick(self): + filenames = QtGui.QFileDialog.getOpenFileNames(None, + translate('SongsPlugin', 'Select OpenLP database(s) to import...'), + u'', u'OpenLP databases (*.sqlite);;All Files (*)') + try: + for filename in filenames: + db_url = u'sqlite:///%s' % filename + importer = OpenLPSongImport(self.manager, db_url) + importer.import_source_v2_db() + QtGui.QMessageBox.information(None, translate('SongsPlugin', + 'Database(s) imported'), translate('SongsPlugin', 'Your ' + 'OpenLP v2 song databases have been successfully imported')) + except: + log.exception(u'Failed to import OpenLP v2 database(s)') + QtGui.QMessageBox.critical(None, translate('SongsPlugin', + 'Import Error'), translate('SongsPlugin', + 'Error importing OpenLP v2 database(s)')) + Receiver.send_message(u'songs_load_list') + def onImportOooItemClick(self): filenames = QtGui.QFileDialog.getOpenFileNames( None, translate('SongsPlugin', From 11c90c3eb79da633e655c7f696e07abd4d2bbab0 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Tue, 20 Jul 2010 09:37:15 +0100 Subject: [PATCH 38/42] Fix song book form ampersands --- openlp/plugins/songs/forms/songbookdialog.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openlp/plugins/songs/forms/songbookdialog.py b/openlp/plugins/songs/forms/songbookdialog.py index 0401ff75b..a2e5658cd 100644 --- a/openlp/plugins/songs/forms/songbookdialog.py +++ b/openlp/plugins/songs/forms/songbookdialog.py @@ -41,6 +41,7 @@ class Ui_SongBookDialog(object): QtGui.QFormLayout.LabelRole, self.NameLabel) self.NameEdit = QtGui.QLineEdit(SongBookDialog) self.NameEdit.setObjectName(u'NameEdit') + self.NameLabel.setBuddy(self.NameEdit) self.SongBookLayout.setWidget(0, QtGui.QFormLayout.FieldRole, self.NameEdit) self.PublisherLabel = QtGui.QLabel(SongBookDialog) @@ -49,6 +50,7 @@ class Ui_SongBookDialog(object): QtGui.QFormLayout.LabelRole, self.PublisherLabel) self.PublisherEdit = QtGui.QLineEdit(SongBookDialog) self.PublisherEdit.setObjectName(u'PublisherEdit') + self.PublisherLabel.setBuddy(self.PublisherEdit) self.SongBookLayout.setWidget(1, QtGui.QFormLayout.FieldRole, self.PublisherEdit) self.ButtonBox = QtGui.QDialogButtonBox(SongBookDialog) From a4bbb0ef282947945b49da0835e7fe9a88c95c97 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Tue, 20 Jul 2010 10:21:15 +0100 Subject: [PATCH 39/42] Clarify import type --- openlp/plugins/songs/songsplugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 5e1c94e57..bf1c0cb23 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -162,7 +162,7 @@ class SongsPlugin(Plugin): self.ImportOpenLPSongItem = QtGui.QAction(import_menu) self.ImportOpenLPSongItem.setObjectName(u'ImportOpenLPSongItem') self.ImportOpenLPSongItem.setText(translate('SongsPlugin', - 'OpenLP v2 (temporary)')) + 'OpenLP v2 Songs (temporary)')) self.ImportOpenLPSongItem.setToolTip(translate('SongsPlugin', 'Import an OpenLP v2 song database')) self.ImportOpenLPSongItem.setStatusTip(translate('SongsPlugin', From 1738040b8713bf8c8326caa885cb4ccdca35b55b Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Tue, 20 Jul 2010 12:02:03 +0100 Subject: [PATCH 40/42] Save Song Book (Bug #607030) --- openlp/plugins/songs/forms/editsongform.py | 33 +++++++++++----------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index c0db7b741..f2dc826e9 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -382,23 +382,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): self.TopicsListView.takeItem(row) def onSongBookComboChanged(self, item): - item = int(self.SongbookCombo.currentIndex()) - text = unicode(self.SongbookCombo.currentText()) - if item == 0 and text: - if QtGui.QMessageBox.question(self, - translate('SongsPlugin.EditSongForm', 'Add Book'), - translate('SongsPlugin.EditSongForm', 'This song book does ' - 'not exist, do you want to add it?'), - QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, - QtGui.QMessageBox.Yes) == QtGui.QMessageBox.Yes: - book = Book.populate(name=text) - self.songmanager.save_object(book) - self.song.book = book - self.loadBooks() - else: - return - elif item >= 1: - item = int(self.SongbookCombo.currentIndex()) + if item >= 1: self.song.song_book_id = \ (self.SongbookCombo.itemData(item)).toInt()[0] else: @@ -616,6 +600,21 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): def accept(self): log.debug(u'accept') + item = int(self.SongbookCombo.currentIndex()) + text = unicode(self.SongbookCombo.currentText()) + if item == 0 and text: + if QtGui.QMessageBox.question(self, + translate('SongsPlugin.EditSongForm', 'Add Book'), + translate('SongsPlugin.EditSongForm', 'This song book does ' + 'not exist, do you want to add it?'), + QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, + QtGui.QMessageBox.Yes) == QtGui.QMessageBox.Yes: + book = Book.populate(name=text) + self.songmanager.save_object(book) + self.song.book = book + self.loadBooks() + else: + return if self.saveSong(): Receiver.send_message(u'songs_load_list') self.close() From cb17897d4726e66163a9fa59f3d16eb1021e3f32 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Tue, 20 Jul 2010 13:43:21 +0100 Subject: [PATCH 41/42] Media rather than Audio --- openlp/plugins/songs/lib/db.py | 57 ++++++++++++------------- openlp/plugins/songs/lib/olpimport.py | 60 ++++++++++++++------------- 2 files changed, 60 insertions(+), 57 deletions(-) diff --git a/openlp/plugins/songs/lib/db.py b/openlp/plugins/songs/lib/db.py index acf388035..156a5e383 100644 --- a/openlp/plugins/songs/lib/db.py +++ b/openlp/plugins/songs/lib/db.py @@ -32,12 +32,6 @@ from sqlalchemy.orm import mapper, relation from openlp.core.lib.db import BaseModel, init_db -class AudioFile(BaseModel): - """ - AudioFile model - """ - pass - class Author(BaseModel): """ Author model @@ -52,6 +46,12 @@ class Book(BaseModel): return u'' % ( str(self.id), self.name, self.publisher) +class MediaFile(BaseModel): + """ + MediaFile model + """ + pass + class Song(BaseModel): """ Song model @@ -73,12 +73,6 @@ def init_schema(url): """ session, metadata = init_db(url, auto_flush=False) - # Definition of the "audio_files" table - audio_files_table = Table(u'audio_files', metadata, - Column(u'id', types.Integer, primary_key=True), - Column(u'file_name', types.Unicode(255), nullable=False) - ) - # Definition of the "authors" table authors_table = Table(u'authors', metadata, Column(u'id', types.Integer, primary_key=True), @@ -87,6 +81,13 @@ def init_schema(url): Column(u'display_name', types.Unicode(255), nullable=False) ) + # Definition of the "media_files" table + media_files_table = Table(u'media_files', metadata, + Column(u'id', types.Integer, primary_key=True), + Column(u'file_name', types.Unicode(255), nullable=False), + Column(u'type', types.Unicode(64), nullable=False, default=u'audio') + ) + # Definition of the "song_books" table song_books_table = Table(u'song_books', metadata, Column(u'id', types.Integer, primary_key=True), @@ -118,14 +119,6 @@ def init_schema(url): Column(u'name', types.Unicode(128), nullable=False) ) - # Definition of the "audio_files_songs" table - audio_files_songs_table = Table(u'audio_files_songs', metadata, - Column(u'audio_file_id', types.Integer, - ForeignKey(u'audio_files.id'), primary_key=True), - Column(u'song_id', types.Integer, - ForeignKey(u'songs.id'), primary_key=True) - ) - # Definition of the "authors_songs" table authors_songs_table = Table(u'authors_songs', metadata, Column(u'author_id', types.Integer, @@ -134,6 +127,14 @@ def init_schema(url): ForeignKey(u'songs.id'), primary_key=True) ) + # Definition of the "media_files_songs" table + media_files_songs_table = Table(u'media_files_songs', metadata, + Column(u'media_file_id', types.Integer, + ForeignKey(u'media_files.id'), primary_key=True), + Column(u'song_id', types.Integer, + ForeignKey(u'songs.id'), primary_key=True) + ) + # Definition of the "songs_topics" table songs_topics_table = Table(u'songs_topics', metadata, Column(u'song_id', types.Integer, @@ -143,36 +144,36 @@ def init_schema(url): ) # Define table indexes - Index(u'audio_files_id', audio_files_table.c.id) Index(u'authors_id', authors_table.c.id) Index(u'authors_display_name_id', authors_table.c.display_name, authors_table.c.id) + Index(u'media_files_id', media_files_table.c.id) Index(u'song_books_id', song_books_table.c.id) Index(u'songs_id', songs_table.c.id) Index(u'topics_id', topics_table.c.id) - Index(u'audio_files_songs_file', audio_files_songs_table.c.audio_file_id, - audio_files_songs_table.c.song_id) - Index(u'audio_files_songs_song', audio_files_songs_table.c.song_id, - audio_files_songs_table.c.audio_file_id) Index(u'authors_songs_author', authors_songs_table.c.author_id, authors_songs_table.c.song_id) Index(u'authors_songs_song', authors_songs_table.c.song_id, authors_songs_table.c.author_id) + Index(u'media_files_songs_file', media_files_songs_table.c.media_file_id, + media_files_songs_table.c.song_id) + Index(u'media_files_songs_song', media_files_songs_table.c.song_id, + media_files_songs_table.c.media_file_id) Index(u'topics_song_topic', songs_topics_table.c.topic_id, songs_topics_table.c.song_id) Index(u'topics_song_song', songs_topics_table.c.song_id, songs_topics_table.c.topic_id) - mapper(AudioFile, audio_files_table) mapper(Author, authors_table) mapper(Book, song_books_table) + mapper(MediaFile, media_files_table) mapper(Song, songs_table, properties={ - 'audio_files': relation(AudioFile, backref='songs', - secondary=audio_files_songs_table), 'authors': relation(Author, backref='songs', secondary=authors_songs_table), 'book': relation(Book, backref='songs'), + 'media_files': relation(MediaFile, backref='songs', + secondary=media_files_songs_table), 'topics': relation(Topic, backref='songs', secondary=songs_topics_table) }) diff --git a/openlp/plugins/songs/lib/olpimport.py b/openlp/plugins/songs/lib/olpimport.py index 433e59b28..a73684e00 100644 --- a/openlp/plugins/songs/lib/olpimport.py +++ b/openlp/plugins/songs/lib/olpimport.py @@ -34,16 +34,10 @@ from sqlalchemy.orm import class_mapper, mapper, relation, scoped_session, \ from sqlalchemy.orm.exc import UnmappedClassError from openlp.core.lib.db import BaseModel -from openlp.plugins.songs.lib.db import Author, Book, Song, Topic #, AudioFile +from openlp.plugins.songs.lib.db import Author, Book, Song, Topic #, MediaFile log = logging.getLogger(__name__) -class OldAudioFile(BaseModel): - """ - AudioFile model - """ - pass - class OldAuthor(BaseModel): """ Author model @@ -56,6 +50,12 @@ class OldBook(BaseModel): """ pass +class OldMediaFile(BaseModel): + """ + MediaFile model + """ + pass + class OldSong(BaseModel): """ Song model @@ -88,24 +88,24 @@ class OpenLPSongImport(object): source_meta = MetaData() source_meta.reflect(engine) self.source_session = scoped_session(sessionmaker(bind=engine)) - if u'audio_files' in source_meta.tables.keys(): - has_audio_files = True + if u'media_files' in source_meta.tables.keys(): + has_media_files = True else: - has_audio_files = False + has_media_files = False source_authors_table = source_meta.tables[u'authors'] source_song_books_table = source_meta.tables[u'song_books'] source_songs_table = source_meta.tables[u'songs'] source_topics_table = source_meta.tables[u'topics'] source_authors_songs_table = source_meta.tables[u'authors_songs'] source_songs_topics_table = source_meta.tables[u'songs_topics'] - if has_audio_files: - source_audio_files_table = source_meta.tables[u'audio_files'] - source_audio_files_songs_table = \ - source_meta.tables[u'audio_files_songs'] + if has_media_files: + source_media_files_table = source_meta.tables[u'media_files'] + source_media_files_songs_table = \ + source_meta.tables[u'media_files_songs'] try: - class_mapper(OldAudioFile) + class_mapper(OldMediaFile) except UnmappedClassError: - mapper(OldAudioFile, source_audio_files_table) + mapper(OldMediaFile, source_media_files_table) song_props = { 'authors': relation(OldAuthor, backref='songs', secondary=source_authors_songs_table), @@ -113,9 +113,9 @@ class OpenLPSongImport(object): 'topics': relation(OldTopic, backref='songs', secondary=source_songs_topics_table) } - if has_audio_files: - song_props['audio_files'] = relation(OldAudioFile, backref='songs', - secondary=source_audio_files_songs_table) + if has_media_files: + song_props['media_files'] = relation(OldMediaFile, backref='songs', + secondary=source_media_files_songs_table) try: class_mapper(OldAuthor) except UnmappedClassError: @@ -137,7 +137,7 @@ class OpenLPSongImport(object): for song in source_songs: new_song = Song() new_song.title = song.title - if has_audio_files: + if has_media_files: new_song.alternate_title = song.alternate_title else: new_song.alternate_title = u'' @@ -185,14 +185,16 @@ class OpenLPSongImport(object): new_song.topics.append(existing_topic) else: new_song.topics.append(Topic.populate(name=topic.name)) -# if has_audio_files: -# if song.audio_files: -# for audio_file in song.audio_files: -# existing_audio_file = \ -# self.master_manager.get_object_filtered(AudioFile, -# AudioFile.file_name == audio_file.file_name) -# if existing_audio_file: -# new_song.audio_files.remove(audio_file) -# new_song.audio_files.append(existing_audio_file) +# if has_media_files: +# if song.media_files: +# for media_file in song.media_files: +# existing_media_file = \ +# self.master_manager.get_object_filtered(MediaFile, +# MediaFile.file_name == media_file.file_name) +# if existing_media_file: +# new_song.media_files.append(existing_media_file) +# else: +# new_song.media_files.append(MediaFile.populate( +# file_name=media_file.file_name)) self.master_manager.save_object(new_song) engine.dispose() From cb431fea2af21a1f0cb18684765f49de79206563 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Tue, 20 Jul 2010 16:22:05 +0100 Subject: [PATCH 42/42] Fix song object creation (Bug #607034) --- openlp/core/lib/db.py | 8 ++++++-- openlp/plugins/songs/forms/editsongform.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/openlp/core/lib/db.py b/openlp/core/lib/db.py index 8acc79541..70def9490 100644 --- a/openlp/core/lib/db.py +++ b/openlp/core/lib/db.py @@ -135,16 +135,20 @@ class Manager(object): settings.endGroup() self.session = init_schema(self.db_url) - def save_object(self, object_instance): + def save_object(self, object_instance, commit=True): """ Save an object to the database ``object_instance`` The object to save + + ``commit`` + Commit the session with this object """ try: self.session.add(object_instance) - self.session.commit() + if commit: + self.session.commit() return True except InvalidRequestError: self.session.rollback() diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index f2dc826e9..691918782 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -290,7 +290,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): QtGui.QMessageBox.Yes) == QtGui.QMessageBox.Yes: author = Author.populate(first_name=text.rsplit(u' ', 1)[0], last_name=text.rsplit(u' ', 1)[1], display_name=text) - self.songmanager.save_object(author) + self.songmanager.save_object(author, False) self.song.authors.append(author) author_item = QtGui.QListWidgetItem( unicode(author.display_name)) @@ -342,7 +342,7 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, QtGui.QMessageBox.Yes) == QtGui.QMessageBox.Yes: topic = Topic.populate(name=text) - self.songmanager.save_object(topic) + self.songmanager.save_object(topic, False) self.song.topics.append(topic) topic_item = QtGui.QListWidgetItem(unicode(topic.name)) topic_item.setData(QtCore.Qt.UserRole,