From f709c90f0ad3413f89563ed0c3b1cf2c345fcfd5 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 21:03:47 +0100 Subject: [PATCH 01/28] Start opensong importer --- openlp/plugins/songs/lib/opensongimport.py | 116 ++++++++++++++++++ openlp/plugins/songs/lib/test.opensong | 45 +++++++ .../plugins/songs/lib/test_opensongimport.py | 9 ++ 3 files changed, 170 insertions(+) create mode 100644 openlp/plugins/songs/lib/opensongimport.py create mode 100644 openlp/plugins/songs/lib/test.opensong create mode 100644 openlp/plugins/songs/lib/test_opensongimport.py diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py new file mode 100644 index 000000000..4dcc7f20d --- /dev/null +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin # +# Thompson, Jon Tibble, Carsten Tinggaard # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### + +import os +import re + +# from songimport import SongImport + +class opensongimport: + """ + Import songs exported from OpenSong - the format is described loosly here: + http://www.opensong.org/d/manual/song_file_format_specification + + However, it doesn't describe the section, so here's an attempt: + + Verses can be expressed in one of 2 ways: + + [v1]List of words + Another Line + + [v2]Some words for the 2nd verse + etc... + + + or: + + 1List of words + 2Some words for the 2nd Verse + + 1Another Line + 2etc... + + + Either or both forms can be used in one song. + + The [v1] labels can have either upper or loewr case Vs + Other labels can be used also: + C - Chorus + B - Bridge + + Guitar chords can be provided 'above' the lyrics (the line is preceeded by a'.') and _s can be used to signify long-drawn-out words: + + . A7 Bm + 1 Some____ Words + + Chords and _s are removed by this importer. + + The verses etc. are imported and tagged appropriately. + + The tag is used to populate the OpenLP verse + display order field. The Author and Copyright tags are also + imported to the appropriate places. + + """ + def __init__(self, songmanager): + """ + Initialise the class. Requires a songmanager class which is passed + to SongImport for writing song to disk + """ + self.songmanager=songmanager + self.song = None + + def osimport(self, filename): + """ + Process the OpenSong file + """ + self.new_song() + f=open(filename) + tree=objectify.parse(f) + root=tree.getroot() + print "Title", zroot.title + # data storage while importing + self.verses=[] + + + # xxx this is common with SOF + def new_song(self): + """ + A change of song. Store the old, create a new + ... but only if the last song was complete. If not, stick with it + """ + if self.song: + self.finish_verse() + if not self.song.check_complete(): + return + self.song.finish() + + self.song = SongImport(self.manager) + self.skip_to_close_bracket = False + self.is_chorus = False + self.italics = False + self.currentverse = u'' + + diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong new file mode 100644 index 000000000..0fde072ca --- /dev/null +++ b/openlp/plugins/songs/lib/test.opensong @@ -0,0 +1,45 @@ + + + Martins Test + Martin Thompson + 2010 Martin Thompson + 1 + V1 C1 V2 C2 B1 V1 + Blah + + + + + + + + + + + ;Comment +[V] +. A B C +1 v1 Line 1___ +2 v2 Line 1___ +. A B C7 +1 V1 Line 2 +2 V1 Line 2 + +[b1] + Bridge 1 + +[C1] + Chorus 1 + +[C2] + Chorus 2 + + \ No newline at end of file diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py new file mode 100644 index 000000000..79b8e9a69 --- /dev/null +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -0,0 +1,9 @@ +import openlp.plugins.songs.lib.opensongimport + +def test(): + o=opensongimport.opensongimport(0)# xxx needs a song manager here + o.osimport(u'test.opensong') + pass + +if __name__=="__main__": + test() From 2f077c398208d8b79b2bfc02ef1e7c1f2f45f802 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 22:24:49 +0100 Subject: [PATCH 02/28] Renamed so as not to conflict with the Pthon XML library which is still used by the Song XML class --- openlp/plugins/songs/lib/{xml.py => lyrics_xml.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openlp/plugins/songs/lib/{xml.py => lyrics_xml.py} (100%) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/lyrics_xml.py similarity index 100% rename from openlp/plugins/songs/lib/xml.py rename to openlp/plugins/songs/lib/lyrics_xml.py From fc27f8107e7b042043fc3145526aaae888b28188 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 22:25:50 +0100 Subject: [PATCH 03/28] First test looks promsing --- openlp/plugins/songs/lib/__init__.py | 1 + openlp/plugins/songs/lib/opensongimport.py | 111 ++++++++++++++---- openlp/plugins/songs/lib/test.opensong | 6 +- .../plugins/songs/lib/test_opensongimport.py | 12 +- 4 files changed, 99 insertions(+), 31 deletions(-) diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index 6c637ea9e..37e738664 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -97,3 +97,4 @@ from mediaitem import SongMediaItem from sofimport import SofImport from oooimport import OooImport from songimport import SongImport +from opensongimport import OpenSongImport diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 4dcc7f20d..7b066c858 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -26,9 +26,11 @@ import os import re -# from songimport import SongImport +from songimport import SongImport +from lxml.etree import Element +from lxml import objectify -class opensongimport: +class OpenSongImport: """ Import songs exported from OpenSong - the format is described loosly here: http://www.opensong.org/d/manual/song_file_format_specification @@ -82,35 +84,94 @@ class opensongimport: self.songmanager=songmanager self.song = None - def osimport(self, filename): + def do_import(self, filename): """ Process the OpenSong file """ - self.new_song() + self.song = SongImport(self.songmanager) f=open(filename) tree=objectify.parse(f) root=tree.getroot() - print "Title", zroot.title + # xxx this bit ought to be more "iterable"... esp. if song had attributes not getters and setters... + if root.copyright: + self.song.add_copyright(unicode(root.copyright)) + if root.author: + self.song.parse_author(unicode(root.author)) + if root.title: + self.song.set_title(unicode(root.title)) + if root.aka: + self.song.set_alternate_title(unicode(root.aka)) + if root.hymn_number: + self.song.set_song_number(unicode(root.hymn_number)) + # data storage while importing - self.verses=[] - - - # xxx this is common with SOF - def new_song(self): - """ - A change of song. Store the old, create a new - ... but only if the last song was complete. If not, stick with it - """ - if self.song: - self.finish_verse() - if not self.song.check_complete(): - return - self.song.finish() - - self.song = SongImport(self.manager) - self.skip_to_close_bracket = False - self.is_chorus = False - self.italics = False - self.currentverse = u'' + verses={} + lyrics=str(root.lyrics) + # xxx what to do if no presentation order - need to figure it out on the fly + for l in lyrics.split('\n'): + # remove comments + semicolon = l.find(';') + if semicolon >= 0: + l=l[:semicolon] + l=l.strip() + if l=='': + continue + # skip inline guitar chords + if l[0] == u'.': + continue + + # verse/chorus/etc. marker + if l[0] == u'[': + versetype=l[1].upper() + if not verses.has_key(versetype): + verses[versetype]={} + if l[2] != u']': + # there's a number to go with it - extract that as well + right_bracket=l.find(u']') + versenum=int(l[2:right_bracket]) + else: + versenum = None # allow error trap + continue + words=None + + # number at start of line => verse number + if l[0] >= u'0' and l[0] <= u'9': + versenum=int(l[0]) + words=l[1:].strip() + + if words is None and \ + versenum is not None and \ + versetype is not None: + words=l + if versenum is not None and \ + not verses[versetype].has_key(versenum): + verses[versetype][versenum]=[] # storage for lines in this verse + if words: + # remove the ____s from extended words + words=words.replace(u'_', u'') + verses[versetype][versenum].append(words) + # done parsing + print u'Title:', root.title + versetypes=verses.keys() + versetypes.sort() + versetags={} + for v in versetypes: + versenums=verses[v].keys() + versenums.sort() + for n in versenums: + versetag= u'%s%s' %(v,n) + lines=u'\n'.join(verses[v][n]) + self.song.verses.append([versetag, lines]) + versetags[versetag]=1 # keep track of what we have for error checking later + # now figure out the presentation order + if root.presentation: + order=unicode(root.presentation).split(u' ') + for tag in order: + if not versetags.has_key(tag): + raise OpenSongImportError + else: + self.song.verse_order_list.append(tag) + + self.song.print_song() diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index 0fde072ca..20206cecb 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -23,11 +23,11 @@ 2 v2 Line 1___ . A B C7 1 V1 Line 2 -2 V1 Line 2 +2 V2 Line 2 [b1] Bridge 1 - + Bridge 1 line 2 [C1] Chorus 1 @@ -42,4 +42,4 @@ - \ No newline at end of file + diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 79b8e9a69..655fc83cc 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -1,8 +1,14 @@ -import openlp.plugins.songs.lib.opensongimport +from openlp.plugins.songs.lib.opensongimport import OpenSongImport +from openlp.plugins.songs.lib.manager import SongManager def test(): - o=opensongimport.opensongimport(0)# xxx needs a song manager here - o.osimport(u'test.opensong') + manager=SongManager() + o=OpenSongImport(manager) + o.do_import(u'test.opensong') + # xxx need some more asserts in here to test it... + assert (1) + # now to XML + # asserts pass if __name__=="__main__": From 3417d85a58b27df804f06e235c688359007e276e Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 15 Jun 2010 22:26:35 +0100 Subject: [PATCH 04/28] Elementtree now from lxml --- openlp/core/lib/songxmlhandler.py | 4 ++-- openlp/core/lib/xmlrootclass.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/core/lib/songxmlhandler.py b/openlp/core/lib/songxmlhandler.py index 76b01e376..5e5abc76b 100644 --- a/openlp/core/lib/songxmlhandler.py +++ b/openlp/core/lib/songxmlhandler.py @@ -39,9 +39,9 @@ The basic XML is of the format:: import logging -from xml.dom.minidom import Document -from xml.etree.ElementTree import ElementTree, XML, dump +from lxml.etree import ElementTree, XML, dump from xml.parsers.expat import ExpatError +from xml.dom.minidom import Document log = logging.getLogger(__name__) diff --git a/openlp/core/lib/xmlrootclass.py b/openlp/core/lib/xmlrootclass.py index 1ea1d41a2..de1339839 100644 --- a/openlp/core/lib/xmlrootclass.py +++ b/openlp/core/lib/xmlrootclass.py @@ -26,7 +26,7 @@ import os import sys -from xml.etree.ElementTree import ElementTree, XML +from lxml.etree import ElementTree, XML sys.path.append(os.path.abspath(os.path.join(u'.', u'..', u'..'))) From e8a397d3ecf246b519a2e29cd852b9a149d60d08 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 22 Jun 2010 21:41:31 +0100 Subject: [PATCH 05/28] Handles songs without a preliminary V tag --- openlp/plugins/songs/lib/opensongimport.py | 86 +++++++++++-------- openlp/plugins/songs/lib/test.opensong | 1 - .../plugins/songs/lib/test_opensongimport.py | 39 ++++++++- 3 files changed, 84 insertions(+), 42 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 7b066c858..dd3bc3df0 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -83,31 +83,36 @@ class OpenSongImport: """ self.songmanager=songmanager self.song = None - + def do_import(self, filename): + file=open(filename) + self.do_import_file(file) + + def do_import_file(self, file): """ Process the OpenSong file """ self.song = SongImport(self.songmanager) - f=open(filename) - tree=objectify.parse(f) + tree=objectify.parse(file) root=tree.getroot() - # xxx this bit ought to be more "iterable"... esp. if song had attributes not getters and setters... - if root.copyright: - self.song.add_copyright(unicode(root.copyright)) - if root.author: - self.song.parse_author(unicode(root.author)) - if root.title: - self.song.set_title(unicode(root.title)) - if root.aka: - self.song.set_alternate_title(unicode(root.aka)) - if root.hymn_number: - self.song.set_song_number(unicode(root.hymn_number)) + fields=dir(root) + decode={u'copyright':self.song.add_copyright, + u'author':self.song.parse_author, + u'title':self.song.set_title, + u'aka':self.song.set_alternate_title, + u'hymn_number':self.song.set_song_number} + for (attr, fn) in decode.items(): + if attr in fields: + fn(unicode(root.__getattr__(attr))) # data storage while importing verses={} - lyrics=str(root.lyrics) - # xxx what to do if no presentation order - need to figure it out on the fly + lyrics=unicode(root.lyrics) + # keep track of a "default" verse order, in case none is specified + our_verse_order=[] + verses_seen={} + # in the absence of any other indication, verses are the default, erm, versetype! + versetype=u'V' for l in lyrics.split('\n'): # remove comments semicolon = l.find(';') @@ -123,35 +128,39 @@ class OpenSongImport: # verse/chorus/etc. marker if l[0] == u'[': versetype=l[1].upper() - if not verses.has_key(versetype): - verses[versetype]={} if l[2] != u']': # there's a number to go with it - extract that as well right_bracket=l.find(u']') versenum=int(l[2:right_bracket]) + versetag=u'%s%d'%(versetype,versenum) else: versenum = None # allow error trap continue words=None - # number at start of line => verse number + # number at start of line.. it's verse number if l[0] >= u'0' and l[0] <= u'9': versenum=int(l[0]) words=l[1:].strip() - + versetag=u'%s%d'%(versetype,versenum) if words is None and \ versenum is not None and \ versetype is not None: words=l - if versenum is not None and \ - not verses[versetype].has_key(versenum): - verses[versetype][versenum]=[] # storage for lines in this verse + if versenum is not None: + if not verses.has_key(versetype): + verses[versetype]={} + if not verses[versetype].has_key(versenum): + verses[versetype][versenum]=[] # storage for lines in this verse + if not verses_seen.has_key(versetag): + verses_seen[versetag] = 1 + our_verse_order.append(versetag) if words: - # remove the ____s from extended words - words=words.replace(u'_', u'') + # Tidy text and remove the ____s from extended words + # words=self.song.tidy_text(words) + words=words.replace('_', '') verses[versetype][versenum].append(words) # done parsing - print u'Title:', root.title versetypes=verses.keys() versetypes.sort() versetags={} @@ -164,14 +173,17 @@ class OpenSongImport: self.song.verses.append([versetag, lines]) versetags[versetag]=1 # keep track of what we have for error checking later # now figure out the presentation order - if root.presentation: - order=unicode(root.presentation).split(u' ') - for tag in order: - if not versetags.has_key(tag): - raise OpenSongImportError - else: - self.song.verse_order_list.append(tag) - - - self.song.print_song() - + if 'presentation' in fields and root.presentation != u'': + order=unicode(root.presentation) + order=order.split() + else: + assert len(our_verse_order)>0 + order=our_verse_order + for tag in order: + if not versetags.has_key(tag): + raise OpenSongImportError + else: + self.song.verse_order_list.append(tag) + def finish(self): + """ Separate function, allows test suite to not pollute database""" + self.song.finish() diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index 20206cecb..ea6303c89 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -17,7 +17,6 @@ ;Comment -[V] . A B C 1 v1 Line 1___ 2 v2 Line 1___ diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 655fc83cc..e6d5eec2d 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -5,10 +5,41 @@ def test(): manager=SongManager() o=OpenSongImport(manager) o.do_import(u'test.opensong') - # xxx need some more asserts in here to test it... - assert (1) - # now to XML - # asserts + # o.finish() + o.song.print_song() + assert o.song.copyright == u'2010 Martin Thompson' + assert o.song.authors == [u'Martin Thompson'] + assert o.song.title == u'Martins Test' + assert o.song.alternate_title == u'' + assert o.song.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses + assert [u'C1', u'Chorus 1'] in o.song.verses + assert [u'C2', u'Chorus 2'] in o.song.verses + assert not [u'C3', u'Chorus 3'] in o.song.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses + assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'B1', u'V1'] + + o=OpenSongImport(manager) + o.do_import(u'test2.opensong') + # o.finish() + o.song.print_song() + assert o.song.copyright == u'2010 Martin Thompson' + assert o.song.authors == [u'Martin Thompson'] + assert o.song.title == u'Martins 2nd Test' + assert o.song.alternate_title == u'' + assert o.song.song_number == u'2' + print o.song.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses + assert [u'C1', u'Chorus 1'] in o.song.verses + assert [u'C2', u'Chorus 2'] in o.song.verses + assert not [u'C3', u'Chorus 3'] in o.song.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses + print o.song.verse_order_list + assert o.song.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] + + print "Tests passed" pass if __name__=="__main__": From 9c573d7bef6af30c32d3aeba5c8eb8dfac18d826 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Tue, 22 Jun 2010 22:24:24 +0100 Subject: [PATCH 06/28] Handles [C] followed by non-numbered lines, by assigned them all to [C1] --- openlp/plugins/songs/lib/opensongimport.py | 8 +++++--- openlp/plugins/songs/lib/test.opensong | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index dd3bc3df0..753aacfc7 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -48,6 +48,7 @@ class OpenSongImport: or: + [V] 1List of words 2Some words for the 2nd Verse @@ -55,9 +56,9 @@ class OpenSongImport: 2etc... - Either or both forms can be used in one song. + Either or both forms can be used in one song. The Number does not necessarily appear at the start of the line - The [v1] labels can have either upper or loewr case Vs + The [v1] labels can have either upper or lower case Vs Other labels can be used also: C - Chorus B - Bridge @@ -134,7 +135,7 @@ class OpenSongImport: versenum=int(l[2:right_bracket]) versetag=u'%s%d'%(versetype,versenum) else: - versenum = None # allow error trap + versenum = 1 continue words=None @@ -180,6 +181,7 @@ class OpenSongImport: assert len(our_verse_order)>0 order=our_verse_order for tag in order: + print tag if not versetags.has_key(tag): raise OpenSongImportError else: diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index ea6303c89..8ef2a1103 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -27,10 +27,12 @@ [b1] Bridge 1 Bridge 1 line 2 -[C1] +[C] +. A B Chorus 1 [C2] +. A B Chorus 2 diff --git a/openlp/plugins/songs/lib/test_importing_lots.py b/openlp/plugins/songs/lib/test_importing_lots.py new file mode 100644 index 000000000..7543de60e --- /dev/null +++ b/openlp/plugins/songs/lib/test_importing_lots.py @@ -0,0 +1,49 @@ +from openlp.plugins.songs.lib.opensongimport import OpenSongImport +from openlp.plugins.songs.lib.manager import SongManager +from glob import glob +from zipfile import ZipFile +import os +from traceback import print_exc +import sys +import codecs +def opensong_import_lots(): + ziploc=u'/home/mjt/openlp/OpenSong_Data/' + files=[] + files.extend(glob(ziploc+u'Songs.zip')) + files.extend(glob(ziploc+u'SOF.zip')) +# files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) +# files.extend(glob(ziploc+u'opensong_*.zip')) + errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') + manager=SongManager() + for file in files: + print u'Importing', file + z=ZipFile(file, u'r') + for song in z.infolist(): + filename=song.filename.decode('cp852') + parts=os.path.split(filename) + if parts[-1] == u'': + #No final part => directory + continue + # xxx need to handle unicode filenames (CP437?? Winzip does this) + print " ", file, ":",filename, + songfile=z.open(song) + + o=OpenSongImport(manager) + try: + o.do_import_file(songfile) + except: + print "Failure", + + errfile.write(u'Failure: %s:%s\n' %(file, filename)) + songfile=z.open(song) + for l in songfile.readlines(): + l=l.decode('utf8') + print(u' |%s\n'%l.strip()) + errfile.write(u' |%s\n'%l.strip()) + print_exc(3, file=errfile) + continue + # o.finish() + print "OK" + # o.song.print_song() +if __name__=="__main__": + opensong_import_lots() From baec9d5c2f2591ff835ca8768663dc02478ac204 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Fri, 25 Jun 2010 19:20:45 +0100 Subject: [PATCH 09/28] Unicode chars in my testcase work OK --- openlp/plugins/songs/lib/test.opensong | 6 ++--- .../plugins/songs/lib/test_importing_lots.py | 22 +++++++++++-------- .../plugins/songs/lib/test_opensongimport.py | 3 ++- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong index 8533e8943..af0f039ed 100644 --- a/openlp/plugins/songs/lib/test.opensong +++ b/openlp/plugins/songs/lib/test.opensong @@ -1,7 +1,7 @@ Martins Test - Martin Thompson + MartiÑ Thómpson 2010 Martin Thompson 1 V1 C V2 C2 V3 B1 V1 @@ -25,8 +25,8 @@ 2 V2 Line 2 [3] -V3 Line 1 -V3 Line 2 + V3 Line 1 + V3 Line 2 [b1] Bridge 1 diff --git a/openlp/plugins/songs/lib/test_importing_lots.py b/openlp/plugins/songs/lib/test_importing_lots.py index 7543de60e..5161f4a30 100644 --- a/openlp/plugins/songs/lib/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test_importing_lots.py @@ -9,9 +9,9 @@ import codecs def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] - files.extend(glob(ziploc+u'Songs.zip')) - files.extend(glob(ziploc+u'SOF.zip')) -# files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) +# files.extend(glob(ziploc+u'Songs.zip')) +# files.extend(glob(ziploc+u'SOF.zip')) + files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager=SongManager() @@ -19,31 +19,35 @@ def opensong_import_lots(): print u'Importing', file z=ZipFile(file, u'r') for song in z.infolist(): - filename=song.filename.decode('cp852') + # need to handle unicode filenames (CP437 - Winzip does this) + filename=song.filename#.decode('cp852') parts=os.path.split(filename) if parts[-1] == u'': #No final part => directory continue - # xxx need to handle unicode filenames (CP437?? Winzip does this) print " ", file, ":",filename, - songfile=z.open(song) - + # songfile=z.open(song) + z.extract(song) + songfile=open(filename, u'r') o=OpenSongImport(manager) try: o.do_import_file(songfile) except: print "Failure", - errfile.write(u'Failure: %s:%s\n' %(file, filename)) + errfile.write(u'Failure: %s:%s\n' %(file, filename.decode('cp437'))) songfile=z.open(song) for l in songfile.readlines(): l=l.decode('utf8') print(u' |%s\n'%l.strip()) errfile.write(u' |%s\n'%l.strip()) print_exc(3, file=errfile) - continue + print_exc(3) + sys.exit(1) + # continue # o.finish() print "OK" + os.unlink(filename) # o.song.print_song() if __name__=="__main__": opensong_import_lots() diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 40b09a23b..6291f7106 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.plugins.songs.lib.manager import SongManager @@ -8,7 +9,7 @@ def test(): # o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'Martin Thompson'] + assert o.song.authors == [u'MartiÑ Thómpson'] assert o.song.title == u'Martins Test' assert o.song.alternate_title == u'' assert o.song.song_number == u'1' From 75c6ba22842c81d4bf03050e1d69ab3feb5f5ef2 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Mon, 28 Jun 2010 20:55:04 +0100 Subject: [PATCH 10/28] Improved =s --- openlp/plugins/songs/lib/opensongimport.py | 68 +++++++++---------- .../plugins/songs/lib/test_opensongimport.py | 29 +++++++- 2 files changed, 60 insertions(+), 37 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 170a650bd..f1a83540e 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -86,11 +86,11 @@ class OpenSongImport: Initialise the class. Requires a songmanager class which is passed to SongImport for writing song to disk """ - self.songmanager=songmanager + self.songmanager = songmanager self.song = None def do_import(self, filename): - file=open(filename) + file = open(filename) self.do_import_file(file) def do_import_file(self, file): @@ -98,10 +98,10 @@ class OpenSongImport: Process the OpenSong file """ self.song = SongImport(self.songmanager) - tree=objectify.parse(file) - root=tree.getroot() - fields=dir(root) - decode={u'copyright':self.song.add_copyright, + tree = objectify.parse(file) + root = tree.getroot() + fields = dir(root) + decode = {u'copyright':self.song.add_copyright, u'author':self.song.parse_author, u'title':self.song.set_title, u'aka':self.song.set_alternate_title, @@ -111,20 +111,20 @@ class OpenSongImport: fn(unicode(root.__getattr__(attr))) # data storage while importing - verses={} - lyrics=unicode(root.lyrics) + verses = {} + lyrics = unicode(root.lyrics) # keep track of a "default" verse order, in case none is specified - our_verse_order=[] - verses_seen={} + our_verse_order = [] + verses_seen = {} # in the absence of any other indication, verses are the default, erm, versetype! - versetype=u'V' + versetype = u'V' for l in lyrics.split(u'\n'): # remove comments semicolon = l.find(u';') if semicolon >= 0: - l=l[:semicolon] - l=l.strip() - if l==u'': + l = l[:semicolon] + l = l.strip() + if len(l) == 0: continue # skip inline guitar chords if l[0] == u'.': @@ -132,14 +132,14 @@ class OpenSongImport: # verse/chorus/etc. marker if l[0] == u'[': - versetype=l[1].upper() + versetype = l[1].upper() if versetype.isdigit(): - versenum=versetype - versetype=u'V' + versenum = versetype + versetype = u'V' elif l[2] != u']': # there's a number to go with it - extract that as well - right_bracket=l.find(u']') - versenum=l[2:right_bracket] + right_bracket = l.find(u']') + versenum = l[2:right_bracket] else: versenum = u'' continue @@ -147,18 +147,18 @@ class OpenSongImport: # number at start of line.. it's verse number if l[0].isdigit(): - versenum=l[0] - words=l[1:].strip() + versenum = l[0] + words = l[1:].strip() if words is None and \ versenum is not None and \ versetype is not None: words=l if versenum is not None: - versetag=u'%s%s'%(versetype,versenum) + versetag = u'%s%s'%(versetype,versenum) if not verses.has_key(versetype): - verses[versetype]={} + verses[versetype] = {} if not verses[versetype].has_key(versenum): - verses[versetype][versenum]=[] # storage for lines in this verse + verses[versetype][versenum] = [] # storage for lines in this verse if not verses_seen.has_key(versetag): verses_seen[versetag] = 1 our_verse_order.append(versetag) @@ -168,28 +168,28 @@ class OpenSongImport: words=words.replace('_', '') verses[versetype][versenum].append(words) # done parsing - versetypes=verses.keys() + versetypes = verses.keys() versetypes.sort() - versetags={} + versetags = {} for v in versetypes: - versenums=verses[v].keys() + versenums = verses[v].keys() versenums.sort() for n in versenums: - versetag= u'%s%s' %(v,n) - lines=u'\n'.join(verses[v][n]) + versetag = u'%s%s' %(v,n) + lines = u'\n'.join(verses[v][n]) self.song.verses.append([versetag, lines]) - versetags[versetag]=1 # keep track of what we have for error checking later + versetags[versetag] = 1 # keep track of what we have for error checking later # now figure out the presentation order if u'presentation' in fields and root.presentation != u'': - order=unicode(root.presentation) - order=order.split() + order = unicode(root.presentation) + order = order.split() else: assert len(our_verse_order)>0 - order=our_verse_order + order = our_verse_order for tag in order: if not versetags.has_key(tag): print u'Got order', tag, u'but not in versetags, skipping' - raise OpenSongImportError + raise OpenSongImportError # xxx keep error, or just warn? else: self.song.verse_order_list.append(tag) def finish(self): diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test_opensongimport.py index 6291f7106..95fc93cdd 100644 --- a/openlp/plugins/songs/lib/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test_opensongimport.py @@ -1,10 +1,33 @@ # -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Christian Richter, Maikel Stuivenberg, Martin # +# Thompson, Jon Tibble, Carsten Tinggaard # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.plugins.songs.lib.manager import SongManager def test(): - manager=SongManager() - o=OpenSongImport(manager) + manager = SongManager() + o = OpenSongImport(manager) o.do_import(u'test.opensong') # o.finish() o.song.print_song() @@ -21,7 +44,7 @@ def test(): assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] - o=OpenSongImport(manager) + o = OpenSongImport(manager) o.do_import(u'test2.opensong') # o.finish() o.song.print_song() From 7ce01d0e65547fbe862e0022fd63b0b99d2006b2 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 30 Jun 2010 20:53:08 +0100 Subject: [PATCH 11/28] Moved test scripts and data to a test folder --- openlp/plugins/songs/lib/test.opensong | 50 ------------------- .../songs/lib/{ => test}/test2.opensong | 0 .../lib/{ => test}/test_importing_lots.py | 4 +- .../lib/{ => test}/test_opensongimport.py | 0 4 files changed, 3 insertions(+), 51 deletions(-) delete mode 100644 openlp/plugins/songs/lib/test.opensong rename openlp/plugins/songs/lib/{ => test}/test2.opensong (100%) rename openlp/plugins/songs/lib/{ => test}/test_importing_lots.py (93%) rename openlp/plugins/songs/lib/{ => test}/test_opensongimport.py (100%) diff --git a/openlp/plugins/songs/lib/test.opensong b/openlp/plugins/songs/lib/test.opensong deleted file mode 100644 index af0f039ed..000000000 --- a/openlp/plugins/songs/lib/test.opensong +++ /dev/null @@ -1,50 +0,0 @@ - - - Martins Test - MartiÑ Thómpson - 2010 Martin Thompson - 1 - V1 C V2 C2 V3 B1 V1 - Blah - - - - - - - - - - - ;Comment -. A B C -1 v1 Line 1___ -2 v2 Line 1___ -. A B C7 -1 V1 Line 2 -2 V2 Line 2 - -[3] - V3 Line 1 - V3 Line 2 - -[b1] - Bridge 1 - Bridge 1 line 2 -[C] -. A B - Chorus 1 - -[C2] -. A B - Chorus 2 - - diff --git a/openlp/plugins/songs/lib/test2.opensong b/openlp/plugins/songs/lib/test/test2.opensong similarity index 100% rename from openlp/plugins/songs/lib/test2.opensong rename to openlp/plugins/songs/lib/test/test2.opensong diff --git a/openlp/plugins/songs/lib/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py similarity index 93% rename from openlp/plugins/songs/lib/test_importing_lots.py rename to openlp/plugins/songs/lib/test/test_importing_lots.py index 5161f4a30..59d72b34b 100644 --- a/openlp/plugins/songs/lib/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -9,9 +9,10 @@ import codecs def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] + files=['test.opensong.zip'] # files.extend(glob(ziploc+u'Songs.zip')) # files.extend(glob(ziploc+u'SOF.zip')) - files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) + # files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager=SongManager() @@ -32,6 +33,7 @@ def opensong_import_lots(): o=OpenSongImport(manager) try: o.do_import_file(songfile) + o.song.print_song() except: print "Failure", diff --git a/openlp/plugins/songs/lib/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py similarity index 100% rename from openlp/plugins/songs/lib/test_opensongimport.py rename to openlp/plugins/songs/lib/test/test_opensongimport.py From 02fd6fe4da0d18b9a455ed70ada1a47140a3e1be Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 30 Jun 2010 21:05:43 +0100 Subject: [PATCH 12/28] Main class handles zipfiles --- openlp/plugins/songs/lib/opensongimport.py | 29 ++++++++++++++++--- .../songs/lib/test/test_importing_lots.py | 8 ++--- .../songs/lib/test/test_opensongimport.py | 20 +++++++++++-- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index f1a83540e..e7878c08a 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -30,6 +30,11 @@ from songimport import SongImport from lxml.etree import Element from lxml import objectify +from zipfile import ZipFile + +import logging +log = logging.getLogger(__name__) + class OpenSongImportError(Exception): pass @@ -89,10 +94,26 @@ class OpenSongImport: self.songmanager = songmanager self.song = None - def do_import(self, filename): - file = open(filename) - self.do_import_file(file) - + def do_import(self, filename, commit=True): + ext=os.path.splitext(filename)[1] + if ext.lower() == ".zip": + log.info('Zipfile found %s', filename) + z=ZipFile(filename, u'r') + for song in z.infolist(): + parts=os.path.split(song.filename) + if parts[-1] == u'': + #No final part => directory + continue + songfile=z.open(song) + self.do_import_file(songfile) + if commit: + self.finish() + else: + log.info('Direct import %s', filename) + file = open(filename) + self.do_import_file(file) + if commit: + self.finish() def do_import_file(self, file): """ Process the OpenSong file diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index 59d72b34b..9c90ad50c 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -27,9 +27,9 @@ def opensong_import_lots(): #No final part => directory continue print " ", file, ":",filename, - # songfile=z.open(song) - z.extract(song) - songfile=open(filename, u'r') + songfile=z.open(song) + #z.extract(song) + #songfile=open(filename, u'r') o=OpenSongImport(manager) try: o.do_import_file(songfile) @@ -49,7 +49,7 @@ def opensong_import_lots(): # continue # o.finish() print "OK" - os.unlink(filename) + #os.unlink(filename) # o.song.print_song() if __name__=="__main__": opensong_import_lots() diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 95fc93cdd..f847b273b 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -28,7 +28,23 @@ from openlp.plugins.songs.lib.manager import SongManager def test(): manager = SongManager() o = OpenSongImport(manager) - o.do_import(u'test.opensong') + o.do_import(u'test.opensong', commit=False) + # o.finish() + o.song.print_song() + assert o.song.copyright == u'2010 Martin Thompson' + assert o.song.authors == [u'MartiÑ Thómpson'] + assert o.song.title == u'Martins Test' + assert o.song.alternate_title == u'' + assert o.song.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses + assert [u'C', u'Chorus 1'] in o.song.verses + assert [u'C2', u'Chorus 2'] in o.song.verses + assert not [u'C3', u'Chorus 3'] in o.song.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses + assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] + + o.do_import(u'test.opensong.zip', commit=False) # o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' @@ -45,7 +61,7 @@ def test(): assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] o = OpenSongImport(manager) - o.do_import(u'test2.opensong') + o.do_import(u'test2.opensong', commit=False) # o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' From 38ccc21f89a4390970a38014bf39648e20b6352e Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 30 Jun 2010 22:19:18 +0100 Subject: [PATCH 13/28] Added GUI for importing OpenSong --- openlp/plugins/songs/lib/opensongimport.py | 9 ++-- openlp/plugins/songs/lib/songimport.py | 9 ++-- .../songs/lib/test/test_importing_lots.py | 6 +-- openlp/plugins/songs/songsplugin.py | 41 +++++++++++++++++++ 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index e7878c08a..b65eb74b3 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -95,6 +95,10 @@ class OpenSongImport: self.song = None def do_import(self, filename, commit=True): + """ + Import either a single opensong file, or a zipfile containing multiple opensong files + If the commit parameter is set False, the import will not be committed to the database (useful for test scripts) + """ ext=os.path.splitext(filename)[1] if ext.lower() == ".zip": log.info('Zipfile found %s', filename) @@ -116,7 +120,7 @@ class OpenSongImport: self.finish() def do_import_file(self, file): """ - Process the OpenSong file + Process the OpenSong file - pass in a file-like object, not a filename """ self.song = SongImport(self.songmanager) tree = objectify.parse(file) @@ -209,8 +213,7 @@ class OpenSongImport: order = our_verse_order for tag in order: if not versetags.has_key(tag): - print u'Got order', tag, u'but not in versetags, skipping' - raise OpenSongImportError # xxx keep error, or just warn? + log.warn(u'Got order %s but not in versetags, skipping', tag) else: self.song.verse_order_list.append(tag) def finish(self): diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 08855f01a..a87023527 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -302,8 +302,7 @@ class SongImport(object): song.theme_name = self.theme_name song.ccli_number = self.ccli_number for authortext in self.authors: - filter_string = u'display_name=%s' % authortext - author = self.manager.get_object_filtered(Author, filter_string) + author = self.manager.get_object_filtered(Author, Author.display_name == authortext) if author is None: author = Author() author.display_name = authortext @@ -312,8 +311,7 @@ class SongImport(object): self.manager.save_object(author) song.authors.append(author) if self.song_book_name: - filter_string = u'name=%s' % self.song_book_name - song_book = self.manager.get_object_filtered(Book, filter_string) + song_book = self.manager.get_object_filtered(Book, Book.name == self.song_book_name) if song_book is None: song_book = Book() song_book.name = self.song_book_name @@ -321,8 +319,7 @@ class SongImport(object): self.manager.save_object(song_book) song.song_book_id = song_book.id for topictext in self.topics: - filter_string = u'name=%s' % topictext - topic = self.manager.get_object_filtered(Topic, filter_string) + topic = self.manager.get_object_filtered(Topic.name == topictext) if topic is None: topic = Topic() topic.name = topictext diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index 9c90ad50c..0abc6d3e1 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -10,8 +10,8 @@ def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] files=['test.opensong.zip'] -# files.extend(glob(ziploc+u'Songs.zip')) -# files.extend(glob(ziploc+u'SOF.zip')) + files.extend(glob(ziploc+u'Songs.zip')) + files.extend(glob(ziploc+u'SOF.zip')) # files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') @@ -47,7 +47,7 @@ def opensong_import_lots(): print_exc(3) sys.exit(1) # continue - # o.finish() + o.finish() print "OK" #os.unlink(filename) # o.song.print_song() diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 82a780fa9..08a136e42 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -38,6 +38,8 @@ try: except ImportError: OOo_available = False +from openlp.plugins.songs.lib import OpenSongImport + log = logging.getLogger(__name__) class SongsPlugin(Plugin): @@ -143,6 +145,25 @@ class SongsPlugin(Plugin): QtCore.SIGNAL(u'triggered()'), self.onImportSofItemClick) QtCore.QObject.connect(self.ImportOooItem, QtCore.SIGNAL(u'triggered()'), self.onImportOooItemClick) + # OpenSong import menu item - will be removed and the + # functionality will be contained within the import wizard + self.ImportOpenSongItem = QtGui.QAction(import_menu) + self.ImportOpenSongItem.setObjectName(u'ImportOpenSongItem') + self.ImportOpenSongItem.setText( + translate('SongsPlugin', + 'OpenSong (temp menu item)')) + self.ImportOpenSongItem.setToolTip( + translate('SongsPlugin', + 'Import songs from OpenSong files' + + '(either raw text or ZIPfiles)')) + self.ImportOpenSongItem.setStatusTip( + translate('SongsPlugin', + 'Import songs from OpenSong files' + + '(either raw text or ZIPfiles)')) + import_menu.addAction(self.ImportOpenSongItem) + QtCore.QObject.connect(self.ImportOpenSongItem, + QtCore.SIGNAL(u'triggered()'), self.onImportOpenSongItemClick) + def add_export_menu_item(self, export_menu): """ @@ -183,6 +204,26 @@ class SongsPlugin(Plugin): QtGui.QMessageBox.Ok) Receiver.send_message(u'songs_load_list') + def onImportOpenSongItemClick(self): + filenames = QtGui.QFileDialog.getOpenFileNames( + None, translate('SongsPlugin', + 'Open OpenSong file'), + u'', u'OpenSong file (*. *.zip *.ZIP)') + try: + for filename in filenames: + importer = OpenSongImport(self.manager) + importer.do_import(unicode(filename)) + except: + log.exception('Could not import OpenSong file') + QtGui.QMessageBox.critical(None, + translate('SongsPlugin', + 'Import Error'), + translate('SongsPlugin', + 'Error importing OpenSong file'), + QtGui.QMessageBox.StandardButtons(QtGui.QMessageBox.Ok), + QtGui.QMessageBox.Ok) + Receiver.send_message(u'songs_load_list') + def onImportOooItemClick(self): filenames = QtGui.QFileDialog.getOpenFileNames( None, translate('SongsPlugin', From ab1f040aaaa520e29fa813fefd9747ca5fca03c3 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Mon, 5 Jul 2010 21:00:05 +0100 Subject: [PATCH 14/28] Working on non-ascii characters in zipfiles --- openlp/plugins/songs/lib/test/test_importing_lots.py | 10 +++++----- openlp/plugins/songs/lib/test/test_opensongimport.py | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index 0abc6d3e1..4b97bf42b 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -9,10 +9,10 @@ import codecs def opensong_import_lots(): ziploc=u'/home/mjt/openlp/OpenSong_Data/' files=[] - files=['test.opensong.zip'] - files.extend(glob(ziploc+u'Songs.zip')) - files.extend(glob(ziploc+u'SOF.zip')) - # files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) + files=[u'test.opensong.zip', ziploc+u'ADond.zip'] + #files.extend(glob(ziploc+u'Songs.zip')) + #files.extend(glob(ziploc+u'SOF.zip')) + files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager=SongManager() @@ -47,7 +47,7 @@ def opensong_import_lots(): print_exc(3) sys.exit(1) # continue - o.finish() + #o.finish() print "OK" #os.unlink(filename) # o.song.print_song() diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index f847b273b..d3d8d356a 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -28,6 +28,9 @@ from openlp.plugins.songs.lib.manager import SongManager def test(): manager = SongManager() o = OpenSongImport(manager) + o.do_import(u'/home/mjt/openlp/OpenSong_Data/ADond', commit=False) + o.song.print_song() + sys.exit(1) o.do_import(u'test.opensong', commit=False) # o.finish() o.song.print_song() From b3559a993d2f8116392d23442cbf1028c09d4db7 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:16:14 +0100 Subject: [PATCH 15/28] test theme, alttheme -> our theme --- openlp/plugins/songs/lib/opensongimport.py | 9 +++++++++ openlp/plugins/songs/lib/test/test_opensongimport.py | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 1a5c94a39..df8492976 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -118,6 +118,8 @@ class OpenSongImport: self.do_import_file(file) if commit: self.finish() + + def do_import_file(self, file): """ Process the OpenSong file - pass in a file-like object, not a filename @@ -135,6 +137,13 @@ class OpenSongImport: for (attr, fn) in decode.items(): if attr in fields: fn(unicode(root.__getattr__(attr))) + + res = [] + if u'theme' in fields: + res.append(unicode(root.theme)) + if u'alttheme' in fields: + res.append(unicode(root.alttheme)) + self.song.theme=u', '.join(res) # data storage while importing verses = {} diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index a027042a7..c1025d4fe 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -30,7 +30,7 @@ def test(): manager = SongManager() o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) - # o.finish() + o.finish() o.song.print_song() assert o.song.copyright == u'2010 Martin Thompson' assert o.song.authors == [u'MartiÑ Thómpson'] @@ -45,7 +45,8 @@ def test(): assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses assert o.song.verse_order_list == [u'V1', u'C', u'V2', u'C2', u'V3', u'B1', u'V1'] assert o.song.ccli_number == u'Blah' - + print u':%s:'%o.song.theme + assert o.song.theme == u'TestTheme, TestAltTheme' o.do_import(u'test.opensong.zip', commit=False) # o.finish() o.song.print_song() From 860d3e2923eca9aa0233588238a189a2845e042a Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:17:16 +0100 Subject: [PATCH 16/28] Removed opensong code from songxml --- openlp/plugins/songs/lib/songxml.py | 203 ---------------------------- 1 file changed, 203 deletions(-) diff --git a/openlp/plugins/songs/lib/songxml.py b/openlp/plugins/songs/lib/songxml.py index 2965c579b..6e99933ec 100644 --- a/openlp/plugins/songs/lib/songxml.py +++ b/openlp/plugins/songs/lib/songxml.py @@ -60,175 +60,6 @@ class SongFeatureError(SongException): # TODO: Song: Import ChangingSong # TODO: Song: Export ChangingSong -_BLANK_OPENSONG_XML = \ -''' - - - - - - - - - - -''' - -class _OpenSong(object): - """ - Class for import of OpenSong - """ - def __init__(self, xmlContent = None): - """ - Initialize from given xml content - """ - self._set_from_xml(_BLANK_OPENSONG_XML, 'song') - if xmlContent: - self._set_from_xml(xmlContent, 'song') - - def _set_from_xml(self, xml, root_tag): - """ - Set song properties from given xml content. - - ``xml`` - Formatted xml tags and values. - ``root_tag`` - The root tag of the xml. - """ - root = ElementTree(element=XML(xml)) - xml_iter = root.getiterator() - for element in xml_iter: - if element.tag != root_tag: - text = element.text - if text is None: - val = text - elif isinstance(text, basestring): - # Strings need special handling to sort the colours out - if text[0] == u'$': - # This might be a hex number, let's try to convert it. - try: - val = int(text[1:], 16) - except ValueError: - pass - else: - # Let's just see if it's a integer. - try: - val = int(text) - except ValueError: - # Ok, it seems to be a string. - val = text - if hasattr(self, u'post_tag_hook'): - (element.tag, val) = \ - self.post_tag_hook(element.tag, val) - setattr(self, element.tag, val) - - def __str__(self): - """ - Return string with all public attributes - - The string is formatted with one attribute per line - If the string is split on newline then the length of the - list is equal to the number of attributes - """ - attributes = [] - for attrib in dir(self): - if not attrib.startswith(u'_'): - attributes.append( - u'%30s : %s' % (attrib, getattr(self, attrib))) - return u'\n'.join(attributes) - - def _get_as_string(self): - """ - Return one string with all public attributes - """ - result = u'' - for attrib in dir(self): - if not attrib.startswith(u'_'): - result += u'_%s_' % getattr(self, attrib) - return result - - def get_author_list(self): - """Convert author field to an authorlist - - in OpenSong an author list may be separated by '/' - return as a string - """ - if self.author: - list = self.author.split(u' and ') - res = [item.strip() for item in list] - return u', '.join(res) - - def get_category_array(self): - """Convert theme and alttheme into category_array - - return as a string - """ - res = [] - if self.theme: - res.append(self.theme) - if self.alttheme: - res.append(self.alttheme) - return u', u'.join(res) - - def _reorder_verse(self, tag, tmpVerse): - """ - Reorder the verse in case of first char is a number - tag -- the tag of this verse / verse group - tmpVerse -- list of strings - """ - res = [] - for digit in '1234567890 ': - tagPending = True - for line in tmpVerse: - if line.startswith(digit): - if tagPending: - tagPending = False - tagChar = tag.strip(u'[]').lower() - if 'v' == tagChar: - newtag = "Verse" - elif 'c' == tagChar: - newtag = "Chorus" - elif 'b' == tagChar: - newtag = "Bridge" - elif 'p' == tagChar: - newtag = "Pre-chorus" - else: - newtag = tagChar - tagString = (u'# %s %s' % (newtag, digit)).rstrip() - res.append(tagString) - res.append(line[1:]) - if (len(line) == 0) and (not tagPending): - res.append(line) - return res - - def get_lyrics(self): - """ - Convert the lyrics to openlp lyrics format - return as list of strings - """ - lyrics = self.lyrics.split(u'\n') - tmpVerse = [] - finalLyrics = [] - tag = "" - for lyric in lyrics: - line = lyric.rstrip() - if not line.startswith(u'.'): - # drop all chords - tmpVerse.append(line) - if line: - if line.startswith(u'['): - tag = line - else: - reorderedVerse = self._reorder_verse(tag, tmpVerse) - finalLyrics.extend(reorderedVerse) - tag = "" - tmpVerse = [] - # catch up final verse - reorderedVerse = self._reorder_verse(tag, tmpVerse) - finalLyrics.extend(reorderedVerse) - return finalLyrics - - class Song(object): """Handling song properties and methods @@ -307,40 +138,6 @@ class Song(object): self.set_lyrics(u'') return - def from_opensong_buffer(self, xmlcontent): - """Initialize from buffer(string) of xml lines in opensong format""" - self._reset() - opensong = _OpenSong(xmlcontent) - if opensong.title: - self.set_title(opensong.title) - if opensong.copyright: - self.set_copyright(opensong.copyright) - if opensong.presentation: - self.set_verse_order(opensong.presentation) - if opensong.ccli: - self.set_song_cclino(opensong.ccli) - self.set_author_list(opensong.get_author_list()) - self.set_category_array(opensong.get_category_array()) - self.set_lyrics(opensong.get_lyrics()) - - def from_opensong_file(self, xmlfilename): - """ - Initialize from file containing xml - xmlfilename -- path to xml file - """ - osfile = None - try: - osfile = open(xmlfilename, 'r') - list = [line for line in osfile] - osfile.close() - xml = "".join(list) - self.from_opensong_buffer(xml) - except IOError: - log.exception(u'Failed to load opensong xml file') - finally: - if osfile: - osfile.close() - def _remove_punctuation(self, title): """Remove the puntuation chars from title From b04c93e9a7bf304dddd9e49a2d60546d001730d5 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:17:58 +0100 Subject: [PATCH 17/28] test theme, alttheme -> our theme --- openlp/plugins/songs/lib/lyrics_xml.py | 4 ++-- openlp/plugins/songs/lib/songimport.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/openlp/plugins/songs/lib/lyrics_xml.py b/openlp/plugins/songs/lib/lyrics_xml.py index 359c28eda..38985ca1b 100644 --- a/openlp/plugins/songs/lib/lyrics_xml.py +++ b/openlp/plugins/songs/lib/lyrics_xml.py @@ -76,8 +76,8 @@ class SongXMLBuilder(object): ``content`` The actual text of the verse to be stored. """ - #log.debug(u'add_verse_to_lyrics %s, %s\n%s' % (type, number, content)) - verse = etree.Element(u'verse', type=type, label=number) + # log.debug(u'add_verse_to_lyrics %s, %s\n%s' % (type, number, content)) + verse = etree.Element(u'verse', type=unicode(type), label=unicode(number)) verse.text = etree.CDATA(content) self.lyrics.append(verse) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 5fbcce540..3b507140b 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -51,8 +51,8 @@ class SongImport(object): self.alternate_title = u'' self.copyright = u'' self.comment = u'' - self.theme_name = u'' - self.ccli_number = u'' + self.theme = u'' + self.song_cclino = u'' self.authors = [] self.topics = [] self.song_book_name = u'' @@ -303,8 +303,8 @@ class SongImport(object): song.verse_order = u' '.join(self.verse_order_list) song.copyright = self.copyright song.comment = self.comment - song.theme_name = self.theme_name - song.ccli_number = self.ccli_number + song.theme = self.theme + song.song_cclino = self.song_cclino for authortext in self.authors: author = self.manager.get_object_filtered(Author, Author.display_name == authortext) @@ -358,7 +358,7 @@ class SongImport(object): print u'TOPIC: ' + topictext if self.comment: print u'COMMENT: ' + self.comment - if self.theme_name: - print u'THEME: ' + self.theme_name - if self.ccli_number: - print u'CCLI: ' + self.ccli_number + if self.theme: + print u'THEME: ' + self.theme + if self.song_cclino: + print u'CCLI: ' + self.song_cclino From e4b7e23c42bf04997b8aa9764b32559eba2f3ee1 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 7 Jul 2010 21:18:38 +0100 Subject: [PATCH 18/28] Added test data --- openlp/plugins/songs/lib/test/test.opensong | 51 ++++++++++++++++++ .../plugins/songs/lib/test/test.opensong.zip | Bin 0 -> 848 bytes 2 files changed, 51 insertions(+) create mode 100644 openlp/plugins/songs/lib/test/test.opensong create mode 100644 openlp/plugins/songs/lib/test/test.opensong.zip diff --git a/openlp/plugins/songs/lib/test/test.opensong b/openlp/plugins/songs/lib/test/test.opensong new file mode 100644 index 000000000..469cd7722 --- /dev/null +++ b/openlp/plugins/songs/lib/test/test.opensong @@ -0,0 +1,51 @@ + + + Martins Test + MartiÑ Thómpson + 2010 Martin Thompson + 1 + V1 C V2 C2 V3 B1 V1 + Blah + + + + + + + + TestTheme + TestAltTheme + + + ;Comment +. A B C +1 v1 Line 1___ +2 v2 Line 1___ +. A B C7 +1 V1 Line 2 +2 V2 Line 2 + +[3] + V3 Line 1 + V3 Line 2 + +[b1] + Bridge 1 + Bridge 1 line 2 +[C] +. A B + Chorus 1 + +[C2] +. A B + Chorus 2 + + diff --git a/openlp/plugins/songs/lib/test/test.opensong.zip b/openlp/plugins/songs/lib/test/test.opensong.zip new file mode 100644 index 0000000000000000000000000000000000000000..1072367a844712f5b433e9c80a621a00e8245f20 GIT binary patch literal 848 zcmWIWW@Zs#U}E542$^-$#ykH@-vTBEhDde>23`gkhLY6c621I_)V$*Sy!6lzP6lR% z&niCCWEFi%E4UdLSza(RFo21xq27774FvAK7GW-%`{MBe-hfEQ+ygHb-CcT*h0)P- zi%W|T>!hN`_ZFqU6fD}v=XtVh(mJj!8L0;z?fLG(cfaT5pJy8o7bgbzAC?qLt*r_tRy~@#%eKV!?eTR_c6ogMVN>#D!On+!Q^h{( zZ_`|pIQyVg^IN7c&78tx#;y1FPg)e=dF@eH-U~xB4T&u;-cEWm!(@)g38fb|+3OBm zsQvA1XV7!!`;kD+Dmj+K7auMb%vF-OXFSQ*qs&E2coWCBZF`EnTerJcv_~pDy1$S` zcDb7g-<)>7=`W&EyS`^%EvG;L8K4#La_ct_jzbaUks+j3&tFE7Z>9gp8LMT|7vnwl9oF;c+xY%y@qtwjZFs6HQkKUs*FGsazjlQL zqw>Cozivh|EuD1yP0P`mRhCj8Gj8oH^;r@W^is8Q^T+<#s{@T*JG-O@K9aw)Ds}5G zt8S++UR#8NV}e5cvGV@>R*w zRs8zB?pqfvOZrqYZ>QMY71<}_uP$|eR*`Y*>{ZQSvz)OK0P|K6q?vp7LL zzA&jZK62KFn?J9q%Gh!!Rblg{{~@t&?48G?UV x0=yZS Date: Fri, 9 Jul 2010 19:17:48 +0100 Subject: [PATCH 19/28] Changed opensong import file extension to All files *.* --- openlp/plugins/songs/songsplugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 6552f4c41..99e3fca74 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -201,7 +201,7 @@ class SongsPlugin(Plugin): filenames = QtGui.QFileDialog.getOpenFileNames( None, translate('SongsPlugin', 'Open OpenSong file'), - u'', u'OpenSong file (*. *.zip *.ZIP)') + u'', u'All files (*.*)') try: for filename in filenames: importer = OpenSongImport(self.manager) @@ -236,4 +236,4 @@ class SongsPlugin(Plugin): if not self.manager.get_all_objects_filtered(Song, Song.theme_name == theme): return True - return False \ No newline at end of file + return False From 0f4c7276b3160a49390173f4e7ba1951e29b2f94 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:10:27 +0100 Subject: [PATCH 20/28] Using new manager --- openlp/plugins/songs/lib/test/test_opensongimport.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 7ae5c3412..96b5018a3 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -23,11 +23,13 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### from openlp.plugins.songs.lib.opensongimport import OpenSongImport -from openlp.plugins.songs.lib.manager import SongManager +from openlp.core.lib.db import Manager +from openlp.plugins.songs.lib.db import init_schema#, Song +from openlp.plugins.songs.songsplugin import SongsPlugin import sys def test(): - manager = SongManager() + manager = Manager(u'songs', init_schema) o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) o.finish() From d54770646dab95891957b2f215bcb55d6daed6fd Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:38:46 +0100 Subject: [PATCH 21/28] Rename cclino to ccli_number and theme to theme_name in all song import related files --- openlp/plugins/songs/lib/opensongimport.py | 4 +- openlp/plugins/songs/lib/songimport.py | 22 ++++---- openlp/plugins/songs/lib/songxml.py | 52 +++++++++---------- .../songs/lib/test/test_importing_lots.py | 11 ++-- .../songs/lib/test/test_opensongimport.py | 7 ++- openlp/plugins/songs/songsplugin.py | 4 +- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index a5efb2808..93c682d90 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -135,7 +135,7 @@ class OpenSongImport: root = tree.getroot() fields = dir(root) decode = {u'copyright':self.song.add_copyright, - u'ccli':self.song.set_song_cclino, + u'ccli':self.song.set_ccli_number, u'author':self.song.parse_author, u'title':self.song.set_title, u'aka':self.song.set_alternate_title, @@ -149,7 +149,7 @@ class OpenSongImport: res.append(unicode(root.theme)) if u'alttheme' in fields: res.append(unicode(root.alttheme)) - self.song.theme = u', '.join(res) + self.song.theme_name = u', '.join(res) # data storage while importing verses = {} diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 1253c8536..893def7f0 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -51,8 +51,8 @@ class SongImport(object): self.alternate_title = u'' self.copyright = u'' self.comment = u'' - self.theme = u'' - self.song_cclino = u'' + self.theme_name = u'' + self.ccli_number = u'' self.authors = [] self.topics = [] self.song_book_name = u'' @@ -163,11 +163,11 @@ class SongImport(object): """ self.song_number = song_number - def set_song_cclino(self, cclino): + def set_ccli_number(self, cclino): """ Set the ccli number """ - self.song_cclino = cclino + self.ccli_number = cclino def set_song_book(self, song_book, publisher): """ @@ -273,7 +273,7 @@ class SongImport(object): def commit_song(self): """ - Write the song and it's fields to disk + Write the song and its fields to disk """ song = Song() song.title = self.title @@ -303,8 +303,8 @@ class SongImport(object): song.verse_order = u' '.join(self.verse_order_list) song.copyright = self.copyright song.comment = self.comment - song.theme = self.theme - song.song_cclino = self.song_cclino + song.theme_name = self.theme_name + song.ccli_number = self.ccli_number for authortext in self.authors: author = self.manager.get_object_filtered(Author, Author.display_name == authortext) @@ -358,7 +358,7 @@ class SongImport(object): print u'TOPIC: ' + topictext if self.comment: print u'COMMENT: ' + self.comment - if self.theme: - print u'THEME: ' + self.theme - if self.song_cclino: - print u'CCLI: ' + self.song_cclino + if self.theme_name: + print u'THEME: ' + self.theme_name + if self.ccli_number: + print u'CCLI: ' + self.ccli_number diff --git a/openlp/plugins/songs/lib/songxml.py b/openlp/plugins/songs/lib/songxml.py index 6e99933ec..c5b3df49a 100644 --- a/openlp/plugins/songs/lib/songxml.py +++ b/openlp/plugins/songs/lib/songxml.py @@ -106,7 +106,7 @@ class Song(object): show_author_list -- 0: no show, 1: show show_copyright -- 0: no show, 1: show show_song_cclino -- 0: no show, 1: show - theme -- name of theme or blank + theme_name -- name of theme or blank category_array -- list of user defined properties (hymn, gospel) song_book -- name of originating book song_number -- number of the song, related to a songbook @@ -129,7 +129,7 @@ class Song(object): self.show_copyright = 1 self.show_song_cclino = 1 self.show_title = 1 - self.theme = "" + self.theme_name = "" self.category_array = None self.song_book = "" self.song_number = "" @@ -221,7 +221,7 @@ class Song(object): self.set_title(sName) self.set_author_list(author_list) self.set_copyright(sCopyright) - self.set_song_cclino(sCcli) + self.set_ccli_number(sCcli) self.set_lyrics(lyrics) def from_ccli_text_file(self, textFileName): @@ -276,21 +276,21 @@ class Song(object): """Set the copyright string""" self.copyright = copyright - def get_song_cclino(self): + def get_ccli_number(self): """Return the songCclino""" - return self._assure_string(self.song_cclino) + return self._assure_string(self.ccli_number) - def set_song_cclino(self, song_cclino): - """Set the song_cclino""" - self.song_cclino = song_cclino + def set_ccli_number(self, ccli_number): + """Set the ccli_number""" + self.ccli_number = ccli_number - def get_theme(self): + def get_theme_name(self): """Return the theme name for the song""" - return self._assure_string(self.theme) + return self._assure_string(self.theme_name) - def set_theme(self, theme): + def set_theme_name(self, theme_name): """Set the theme name (string)""" - self.theme = theme + self.theme_name = theme_name def get_song_book(self): """Return the song_book (string)""" @@ -329,9 +329,9 @@ class Song(object): asOneString True -- string: - "John Newton, A Parker" + 'John Newton, A Parker' False -- list of strings - ["John Newton", u'A Parker"] + ['John Newton', u'A Parker'] """ if asOneString: res = self._assure_string(self.author_list) @@ -354,9 +354,9 @@ class Song(object): asOneString True -- string: - "Hymn, Gospel" + 'Hymn, Gospel' False -- list of strings - ["Hymn", u'Gospel"] + ['Hymn', u'Gospel'] """ if asOneString: res = self._assure_string(self.category_array) @@ -398,13 +398,13 @@ class Song(object): """Set the show_copyright flag (bool)""" self.show_copyright = show_copyright - def get_show_song_cclino(self): + def get_show_ccli_number(self): """Return the showSongCclino (string)""" - return self.show_song_cclino + return self.show_ccli_number - def set_show_song_cclino(self, show_song_cclino): - """Set the show_song_cclino flag (bool)""" - self.show_song_cclino = show_song_cclino + def set_show_ccli_number(self, show_ccli_number): + """Set the show_ccli_number flag (bool)""" + self.show_ccli_number = show_ccli_number def get_lyrics(self): """Return the lyrics as a list of strings @@ -471,7 +471,7 @@ class Song(object): slideNumber -- 1 .. numberOfSlides Returns a list as: - [theme (string), + [theme_name (string), title (string), authorlist (string), copyright (string), @@ -496,13 +496,13 @@ class Song(object): cpright = self.get_copyright() else: cpright = "" - if self.show_song_cclino: - ccli = self.get_song_cclino() + if self.show_ccli_number: + ccli = self.get_ccli_number() else: ccli = "" - theme = self.get_theme() + theme_name = self.get_theme_name() # examine the slide for a theme - res.append(theme) + res.append(theme_name) res.append(title) res.append(author) res.append(cpright) diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index d896cf5ba..e7ed339b0 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -1,5 +1,6 @@ from openlp.plugins.songs.lib.opensongimport import OpenSongImport -from openlp.plugins.songs.lib.manager import SongManager +from openlp.plugins.songs.lib.db import init_schema +from openlp.core.lib.db import Manager from glob import glob from zipfile import ZipFile import os @@ -9,13 +10,13 @@ import codecs def opensong_import_lots(): ziploc = u'/home/mjt/openlp/OpenSong_Data/' files = [] - files = [u'test.opensong.zip', ziploc+u'ADond.zip'] - #files.extend(glob(ziploc+u'Songs.zip')) + #files = [u'test.opensong.zip', ziploc+u'ADond.zip'] + files.extend(glob(ziploc+u'Songs.zip')) #files.extend(glob(ziploc+u'SOF.zip')) - files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) + #files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') - manager=SongManager() + manager = Manager(u'songs', init_schema) for file in files: print u'Importing', file z = ZipFile(file, u'r') diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 96b5018a3..aede61416 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -24,7 +24,7 @@ ############################################################################### from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.core.lib.db import Manager -from openlp.plugins.songs.lib.db import init_schema#, Song +from openlp.plugins.songs.lib.db import init_schema from openlp.plugins.songs.songsplugin import SongsPlugin import sys @@ -46,9 +46,8 @@ def test(): assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] - assert o.song.song_cclino == u'Blah' - print u':%s:'%o.song.theme - assert o.song.theme == u'TestTheme, TestAltTheme' + assert o.song.ccli_number == u'Blah' + assert o.song.theme_name == u'TestTheme, TestAltTheme' o.do_import(u'test.opensong.zip', commit=False) o.finish() o.song.print_song() diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 99e3fca74..b6795cd32 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -232,8 +232,8 @@ class SongsPlugin(Plugin): 'This plugin allows songs to be managed and displayed.') return about_text - def canDeleteTheme(self, theme): + def canDeleteTheme(self, theme_name): if not self.manager.get_all_objects_filtered(Song, - Song.theme_name == theme): + Song.theme_name == theme_name): return True return False From 490e2451e00dc6343639d224a4837632cb5b9252 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:46:23 +0100 Subject: [PATCH 22/28] spaces --- openlp/plugins/songs/lib/test/test_importing_lots.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index e7ed339b0..dc3eeeb2a 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -15,7 +15,7 @@ def opensong_import_lots(): #files.extend(glob(ziploc+u'SOF.zip')) #files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) - errfile=codecs.open(u'import_lots_errors.txt', u'w', u'utf8') + errfile = codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager = Manager(u'songs', init_schema) for file in files: print u'Importing', file @@ -41,8 +41,8 @@ def opensong_import_lots(): errfile.write(u'Failure: %s:%s\n' %(file, filename.decode('cp437'))) songfile = z.open(song) for l in songfile.readlines(): - l=l.decode('utf8') - print(u' |%s\n'%l.strip()) + l = l.decode('utf8') + print(u' |%s\n' % l.strip()) errfile.write(u' |%s\n'%l.strip()) print_exc(3, file = errfile) print_exc(3) From 222f10ba4574db947adbd812e9b8d969b1738e8e Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Wed, 14 Jul 2010 21:52:45 +0100 Subject: [PATCH 23/28] Now fixed the trunk merge --- openlp/plugins/songs/songsplugin.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index b9129be7a..ab36d13d0 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -233,10 +233,6 @@ class SongsPlugin(Plugin): 'This plugin allows songs to be managed and displayed.') return about_text - def canDeleteTheme(self, theme_name): - if not self.manager.get_all_objects_filtered(Song, - Song.theme_name == theme_name): - def usesTheme(self, theme): """ Called to find out if the song plugin is currently using a theme. From 294a02af458e0b031d9e2438034a313d51a7e0bc Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Thu, 15 Jul 2010 21:26:57 +0100 Subject: [PATCH 24/28] Renamed back and fixed a couple of = signs --- openlp/plugins/songs/lib/{lyrics_xml.py => xml.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename openlp/plugins/songs/lib/{lyrics_xml.py => xml.py} (97%) diff --git a/openlp/plugins/songs/lib/lyrics_xml.py b/openlp/plugins/songs/lib/xml.py similarity index 97% rename from openlp/plugins/songs/lib/lyrics_xml.py rename to openlp/plugins/songs/lib/xml.py index d3c595b15..1e9678c55 100644 --- a/openlp/plugins/songs/lib/lyrics_xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -49,7 +49,7 @@ class SongXMLBuilder(object): """ log.info(u'SongXMLBuilder Loaded') - def __init__(self, song_language = None): + def __init__(self, song_language=None): """ Set up the song builder. @@ -85,14 +85,14 @@ class SongXMLBuilder(object): """ Debugging aid to dump XML so that we can see what we have. """ - return etree.tostring(self.song_xml, encoding = u'UTF-8', + return etree.tostring(self.song_xml, encoding=u'UTF-8', xml_declaration=True, pretty_print=True) def extract_xml(self): """ Extract our newly created XML song. """ - return etree.tostring(self.song_xml, encoding = u'UTF-8', + return etree.tostring(self.song_xml, encoding=u'UTF-8', xml_declaration=True) From bb34a304a9be75c43e02f3cc3768259e536894ca Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Thu, 15 Jul 2010 21:27:44 +0100 Subject: [PATCH 25/28] Tweaked Topics handler --- openlp/plugins/songs/lib/songimport.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 893def7f0..e5045a5f5 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -190,8 +190,8 @@ class SongImport(object): """ Add the author. OpenLP stores them individually so split by 'and', '&' and comma. - However need to check for "Mr and Mrs Smith" and turn it to - "Mr Smith" and "Mrs Smith". + However need to check for 'Mr and Mrs Smith' and turn it to + 'Mr Smith' and 'Mrs Smith'. """ for author in text.split(u','): authors = author.split(u'&') @@ -325,12 +325,14 @@ class SongImport(object): self.manager.save_object(song_book) song.song_book_id = song_book.id for topictext in self.topics: - topic = self.manager.get_object_filtered(Topic.name == topictext) + if len(topictext) == 0: + continue + topic = self.manager.get_object_filtered(Topic, Topic.name == topictext) if topic is None: topic = Topic() topic.name = topictext self.manager.save_object(topic) - song.topics.append(topictext) + song.topics.append(topic) self.manager.save_object(song) def print_song(self): From 059032f23736a18990cee8a122c37180c69ec925 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Thu, 15 Jul 2010 21:29:24 +0100 Subject: [PATCH 26/28] Changed internal 'song' attribute to clearer 'song_import'. OpenSong themes now map to Topics. Text cleaning called --- openlp/plugins/songs/lib/opensongimport.py | 30 +++---- .../songs/lib/test/test_importing_lots.py | 5 +- .../songs/lib/test/test_opensongimport.py | 86 +++++++++---------- 3 files changed, 60 insertions(+), 61 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index 93c682d90..b8d30c535 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -38,7 +38,7 @@ log = logging.getLogger(__name__) class OpenSongImportError(Exception): pass -class OpenSongImport: +class OpenSongImport(object): """ Import songs exported from OpenSong - the format is described loosly here: http://www.opensong.org/d/manual/song_file_format_specification @@ -130,27 +130,25 @@ class OpenSongImport: Process the OpenSong file - pass in a file-like object, not a filename """ - self.song = SongImport(self.songmanager) + self.song_import = SongImport(self.songmanager) tree = objectify.parse(file) root = tree.getroot() fields = dir(root) - decode = {u'copyright':self.song.add_copyright, - u'ccli':self.song.set_ccli_number, - u'author':self.song.parse_author, - u'title':self.song.set_title, - u'aka':self.song.set_alternate_title, - u'hymn_number':self.song.set_song_number} + decode = {u'copyright':self.song_import.add_copyright, + u'ccli':self.song_import.set_ccli_number, + u'author':self.song_import.parse_author, + u'title':self.song_import.set_title, + u'aka':self.song_import.set_alternate_title, + u'hymn_number':self.song_import.set_song_number} for (attr, fn) in decode.items(): if attr in fields: fn(unicode(root.__getattr__(attr))) res = [] if u'theme' in fields: - res.append(unicode(root.theme)) + self.song_import.topics.append(unicode(root.theme)) if u'alttheme' in fields: - res.append(unicode(root.alttheme)) - self.song.theme_name = u', '.join(res) - + self.song_import.topics.append(unicode(root.alttheme)) # data storage while importing verses = {} lyrics = unicode(root.lyrics) @@ -207,7 +205,7 @@ class OpenSongImport: our_verse_order.append(versetag) if words: # Tidy text and remove the ____s from extended words - # words=self.song.tidy_text(words) + words=self.song_import.tidy_text(words) words=words.replace('_', '') verses[versetype][versenum].append(words) # done parsing @@ -221,7 +219,7 @@ class OpenSongImport: for n in versenums: versetag = u'%s%s' %(v,n) lines = u'\n'.join(verses[v][n]) - self.song.verses.append([versetag, lines]) + self.song_import.verses.append([versetag, lines]) versetags[versetag] = 1 # keep track of what we have for error checking later # now figure out the presentation order if u'presentation' in fields and root.presentation != u'': @@ -236,7 +234,7 @@ class OpenSongImport: if not versetags.has_key(tag): log.warn(u'Got order %s but not in versetags, skipping', tag) else: - self.song.verse_order_list.append(tag) + self.song_import.verse_order_list.append(tag) def finish(self): """ Separate function, allows test suite to not pollute database""" - self.song.finish() + self.song_import.finish() diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index dc3eeeb2a..9f1908bca 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -7,6 +7,7 @@ import os from traceback import print_exc import sys import codecs + def opensong_import_lots(): ziploc = u'/home/mjt/openlp/OpenSong_Data/' files = [] @@ -34,7 +35,7 @@ def opensong_import_lots(): o = OpenSongImport(manager) try: o.do_import_file(songfile) - o.song.print_song() + # o.song_import.print_song() except: print "Failure", @@ -51,6 +52,6 @@ def opensong_import_lots(): #o.finish() print "OK" #os.unlink(filename) - # o.song.print_song() + # o.song_import.print_song() if __name__ == "__main__": opensong_import_lots() diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index aede61416..7f6c0f45f 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -33,55 +33,55 @@ def test(): o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) o.finish() - o.song.print_song() - assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'MartiÑ Thómpson'] - assert o.song.title == u'Martins Test' - assert o.song.alternate_title == u'' - assert o.song.song_number == u'1' - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses - assert [u'C1', u'Chorus 1'] in o.song.verses - assert [u'C2', u'Chorus 2'] in o.song.verses - assert not [u'C3', u'Chorus 3'] in o.song.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses - assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] - assert o.song.ccli_number == u'Blah' - assert o.song.theme_name == u'TestTheme, TestAltTheme' + o.song_import.print_song() + assert o.song_import.copyright == u'2010 Martin Thompson' + assert o.song_import.authors == [u'MartiÑ Thómpson'] + assert o.song_import.title == u'Martins Test' + assert o.song_import.alternate_title == u'' + assert o.song_import.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses + assert [u'C1', u'Chorus 1'] in o.song_import.verses + assert [u'C2', u'Chorus 2'] in o.song_import.verses + assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses + assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] + assert o.song_import.ccli_number == u'Blah' + assert o.song_import.topics == [u'TestTheme', u'TestAltTheme'] o.do_import(u'test.opensong.zip', commit=False) + o.song_import.print_song() o.finish() - o.song.print_song() - assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'MartiÑ Thómpson'] - assert o.song.title == u'Martins Test' - assert o.song.alternate_title == u'' - assert o.song.song_number == u'1' - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses - assert [u'C1', u'Chorus 1'] in o.song.verses - assert [u'C2', u'Chorus 2'] in o.song.verses - assert not [u'C3', u'Chorus 3'] in o.song.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses - assert o.song.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] + assert o.song_import.copyright == u'2010 Martin Thompson' + assert o.song_import.authors == [u'MartiÑ Thómpson'] + assert o.song_import.title == u'Martins Test' + assert o.song_import.alternate_title == u'' + assert o.song_import.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses + assert [u'C1', u'Chorus 1'] in o.song_import.verses + assert [u'C2', u'Chorus 2'] in o.song_import.verses + assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses + assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] o = OpenSongImport(manager) o.do_import(u'test2.opensong', commit=False) # o.finish() - o.song.print_song() - assert o.song.copyright == u'2010 Martin Thompson' - assert o.song.authors == [u'Martin Thompson'] - assert o.song.title == u'Martins 2nd Test' - assert o.song.alternate_title == u'' - assert o.song.song_number == u'2' - print o.song.verses - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song.verses - assert [u'C1', u'Chorus 1'] in o.song.verses - assert [u'C2', u'Chorus 2'] in o.song.verses - assert not [u'C3', u'Chorus 3'] in o.song.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song.verses - print o.song.verse_order_list - assert o.song.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] + o.song_import.print_song() + assert o.song_import.copyright == u'2010 Martin Thompson' + assert o.song_import.authors == [u'Martin Thompson'] + assert o.song_import.title == u'Martins 2nd Test' + assert o.song_import.alternate_title == u'' + assert o.song_import.song_number == u'2' + print o.song_import.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses + assert [u'C1', u'Chorus 1'] in o.song_import.verses + assert [u'C2', u'Chorus 2'] in o.song_import.verses + assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses + print o.song_import.verse_order_list + assert o.song_import.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] print "Tests passed" pass From 168aeea3e1de937d71efc8f50520c2c8dd6b78cf Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Sun, 18 Jul 2010 20:27:27 +0100 Subject: [PATCH 27/28] Tweaks from review --- openlp/plugins/songs/lib/opensongimport.py | 58 ++++++++++--------- .../songs/lib/test/test_opensongimport.py | 3 +- openlp/plugins/songs/lib/xml.py | 3 +- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index b8d30c535..4974f79f7 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -26,11 +26,12 @@ import os import re -from songimport import SongImport +from zipfile import ZipFile + from lxml.etree import Element from lxml import objectify -from zipfile import ZipFile +from openlp.plugins.songs.lib.songimport import SongImport import logging log = logging.getLogger(__name__) @@ -104,12 +105,12 @@ class OpenSongImport(object): set False, the import will not be committed to the database (useful for test scripts) """ - ext=os.path.splitext(filename)[1] + ext = os.path.splitext(filename)[1] if ext.lower() == ".zip": log.info('Zipfile found %s', filename) - z=ZipFile(filename, u'r') + z = ZipFile(filename, u'r') for song in z.infolist(): - parts=os.path.split(song.filename) + parts = os.path.split(song.filename) if parts[-1] == u'': #No final part => directory continue @@ -158,28 +159,29 @@ class OpenSongImport(object): # in the absence of any other indication, verses are the default, # erm, versetype! versetype = u'V' - for l in lyrics.split(u'\n'): + for thisline in lyrics.split(u'\n'): # remove comments - semicolon = l.find(u';') + semicolon = thisline.find(u';') if semicolon >= 0: - l = l[:semicolon] - l = l.strip() - if len(l) == 0: + thisline = thisline[:semicolon] + thisline = thisline.strip() + if len(thisline) == 0: continue - # skip inline guitar chords and page and column breaks - if l[0] == u'.' or l.startswith(u'---') or l.startswith(u'-!!'): + # skip inthisline guitar chords and page and column breaks + if thisline[0] == u'.' or thisline.startswith(u'---') \ + or thisline.startswith(u'-!!'): continue # verse/chorus/etc. marker - if l[0] == u'[': - versetype = l[1].upper() + if thisline[0] == u'[': + versetype = thisline[1].upper() if versetype.isdigit(): versenum = versetype versetype = u'V' - elif l[2] != u']': + elif thisline[2] != u']': # there's a number to go with it - extract that as well - right_bracket = l.find(u']') - versenum = l[2:right_bracket] + right_bracket = thisline.find(u']') + versenum = thisline[2:right_bracket] else: # if there's no number, assume it's no.1 versenum = u'1' @@ -187,13 +189,13 @@ class OpenSongImport(object): words = None # number at start of line.. it's verse number - if l[0].isdigit(): - versenum = l[0] - words = l[1:].strip() + if thisline[0].isdigit(): + versenum = thisline[0] + words = thisline[1:].strip() if words is None and \ versenum is not None and \ versetype is not None: - words = l + words = thisline if versenum is not None: versetag = u'%s%s'%(versetype,versenum) if not verses.has_key(versetype): @@ -205,20 +207,20 @@ class OpenSongImport(object): our_verse_order.append(versetag) if words: # Tidy text and remove the ____s from extended words - words=self.song_import.tidy_text(words) - words=words.replace('_', '') + words = self.song_import.tidy_text(words) + words = words.replace('_', '') verses[versetype][versenum].append(words) # done parsing versetypes = verses.keys() versetypes.sort() versetags = {} verse_renames = {} - for v in versetypes: - versenums = verses[v].keys() + for versetype in versetypes: + versenums = verses[versetype].keys() versenums.sort() - for n in versenums: - versetag = u'%s%s' %(v,n) - lines = u'\n'.join(verses[v][n]) + for num in versenums: + versetag = u'%s%s' %(versetype,num) + lines = u'\n'.join(verses[versetype][num]) self.song_import.verses.append([versetag, lines]) versetags[versetag] = 1 # keep track of what we have for error checking later # now figure out the presentation order diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index 7f6c0f45f..8c974adbc 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -32,17 +32,16 @@ def test(): manager = Manager(u'songs', init_schema) o = OpenSongImport(manager) o.do_import(u'test.opensong', commit=False) - o.finish() o.song_import.print_song() assert o.song_import.copyright == u'2010 Martin Thompson' assert o.song_import.authors == [u'MartiÑ Thómpson'] assert o.song_import.title == u'Martins Test' assert o.song_import.alternate_title == u'' assert o.song_import.song_number == u'1' - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses assert [u'C1', u'Chorus 1'] in o.song_import.verses assert [u'C2', u'Chorus 2'] in o.song_import.verses assert not [u'C3', u'Chorus 3'] in o.song_import.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 1e9678c55..336c1ebf1 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -77,7 +77,8 @@ class SongXMLBuilder(object): The actual text of the verse to be stored. """ # log.debug(u'add_verse_to_lyrics %s, %s\n%s' % (type, number, content)) - verse = etree.Element(u'verse', type = unicode(type), label = unicode(number)) + verse = etree.Element(u'verse', type = unicode(type), + label = unicode(number)) verse.text = etree.CDATA(content) self.lyrics.append(verse) From acb9cca5d32373ea48278f6194ddddfd61417ea2 Mon Sep 17 00:00:00 2001 From: Martin Thompson Date: Mon, 19 Jul 2010 20:35:33 +0100 Subject: [PATCH 28/28] Missed a rename --- openlp/plugins/songs/lib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index e6d56de20..8be820d82 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -137,7 +137,7 @@ class VerseType(object): unicode(VerseType.to_string(VerseType.Other)).lower(): return VerseType.Other -from lyrics_xml import LyricsXML, SongXMLBuilder, SongXMLParser +from xml import LyricsXML, SongXMLBuilder, SongXMLParser from songstab import SongsTab from mediaitem import SongMediaItem from songimport import SongImport