From cfb896e3b102a350fa5e4a4dd4ca23b098907e69 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Sun, 2 Jan 2011 17:42:09 +0100 Subject: [PATCH 01/17] started work on OpenLyrics importer --- openlp/plugins/songs/forms/songimportform.py | 49 +++++++++-------- .../plugins/songs/forms/songimportwizard.py | 11 ++-- openlp/plugins/songs/lib/importer.py | 6 ++- openlp/plugins/songs/lib/xml.py | 54 +++++++++++-------- 4 files changed, 65 insertions(+), 55 deletions(-) diff --git a/openlp/plugins/songs/forms/songimportform.py b/openlp/plugins/songs/forms/songimportform.py index 5ec7f45e1..5776dd21a 100644 --- a/openlp/plugins/songs/forms/songimportform.py +++ b/openlp/plugins/songs/forms/songimportform.py @@ -73,12 +73,12 @@ class SongImportForm(QtGui.QWizard, Ui_SongImportWizard): QtCore.QObject.connect(self.openLP1BrowseButton, QtCore.SIGNAL(u'clicked()'), self.onOpenLP1BrowseButtonClicked) - #QtCore.QObject.connect(self.openLyricsAddButton, - # QtCore.SIGNAL(u'clicked()'), - # self.onOpenLyricsAddButtonClicked) - #QtCore.QObject.connect(self.openLyricsRemoveButton, - # QtCore.SIGNAL(u'clicked()'), - # self.onOpenLyricsRemoveButtonClicked) + QtCore.QObject.connect(self.openLyricsAddButton, + QtCore.SIGNAL(u'clicked()'), + self.onOpenLyricsAddButtonClicked) + QtCore.QObject.connect(self.openLyricsRemoveButton, + QtCore.SIGNAL(u'clicked()'), + self.onOpenLyricsRemoveButtonClicked) QtCore.QObject.connect(self.openSongAddButton, QtCore.SIGNAL(u'clicked()'), self.onOpenSongAddButtonClicked) @@ -167,16 +167,15 @@ class SongImportForm(QtGui.QWizard, Ui_SongImportWizard): self.openLP1BrowseButton.setFocus() return False elif source_format == SongFormat.OpenLyrics: -# if self.openLyricsFileListWidget.count() == 0: -# QtGui.QMessageBox.critical(self, -# translate('SongsPlugin.ImportWizardForm', -# 'No OpenLyrics Files Selected'), -# translate('SongsPlugin.ImportWizardForm', -# 'You need to add at least one OpenLyrics ' -# 'song file to import from.')) -# self.openLyricsAddButton.setFocus() -# return False - return False + if self.openLyricsFileListWidget.count() == 0: + QtGui.QMessageBox.critical(self, + translate('SongsPlugin.ImportWizardForm', + 'No OpenLyrics Files Selected'), + translate('SongsPlugin.ImportWizardForm', + 'You need to add at least one OpenLyrics ' + 'song file to import from.')) + self.openLyricsAddButton.setFocus() + return False elif source_format == SongFormat.OpenSong: if self.openSongFileListWidget.count() == 0: QtGui.QMessageBox.critical(self, @@ -337,15 +336,15 @@ class SongImportForm(QtGui.QWizard, Ui_SongImportWizard): 'openlp.org v1.x Databases') ) - #def onOpenLyricsAddButtonClicked(self): - # self.getFiles( - # translate('SongsPlugin.ImportWizardForm', - # 'Select OpenLyrics Files'), - # self.openLyricsFileListWidget - # ) + def onOpenLyricsAddButtonClicked(self): + self.getFiles( + translate('SongsPlugin.ImportWizardForm', + 'Select OpenLyrics Files'), + self.openLyricsFileListWidget + ) - #def onOpenLyricsRemoveButtonClicked(self): - # self.removeSelectedItems(self.openLyricsFileListWidget) + def onOpenLyricsRemoveButtonClicked(self): + self.removeSelectedItems(self.openLyricsFileListWidget) def onOpenSongAddButtonClicked(self): self.getFiles( @@ -435,7 +434,7 @@ class SongImportForm(QtGui.QWizard, Ui_SongImportWizard): self.formatComboBox.setCurrentIndex(0) self.openLP2FilenameEdit.setText(u'') self.openLP1FilenameEdit.setText(u'') - #self.openLyricsFileListWidget.clear() + self.openLyricsFileListWidget.clear() self.openSongFileListWidget.clear() self.wordsOfWorshipFileListWidget.clear() self.ccliFileListWidget.clear() diff --git a/openlp/plugins/songs/forms/songimportwizard.py b/openlp/plugins/songs/forms/songimportwizard.py index 85fbb07fe..6eccff9b4 100644 --- a/openlp/plugins/songs/forms/songimportwizard.py +++ b/openlp/plugins/songs/forms/songimportwizard.py @@ -81,9 +81,6 @@ class Ui_SongImportWizard(object): self.addSingleFileSelectItem(u'openLP1', None, True) # OpenLyrics self.addMultiFileSelectItem(u'openLyrics', u'OpenLyrics', True) - # set OpenLyrics to disabled by default - self.openLyricsDisabledWidget.setVisible(True) - self.openLyricsImportWidget.setVisible(False) # Open Song self.addMultiFileSelectItem(u'openSong', u'OpenSong') # Words of Worship @@ -177,10 +174,10 @@ class Ui_SongImportWizard(object): 'importer has been disabled due to a missing Python module. If ' 'you want to use this importer, you will need to install the ' '"python-sqlite" module.')) - #self.openLyricsAddButton.setText( - # translate('SongsPlugin.ImportWizardForm', 'Add Files...')) - #self.openLyricsRemoveButton.setText( - # translate('SongsPlugin.ImportWizardForm', 'Remove File(s)')) + self.openLyricsAddButton.setText( + translate('SongsPlugin.ImportWizardForm', 'Add Files...')) + self.openLyricsRemoveButton.setText( + translate('SongsPlugin.ImportWizardForm', 'Remove File(s)')) self.openLyricsDisabledLabel.setText( translate('SongsPlugin.ImportWizardForm', 'The OpenLyrics ' 'importer has not yet been developed, but as you can see, we are ' diff --git a/openlp/plugins/songs/lib/importer.py b/openlp/plugins/songs/lib/importer.py index b82e14c12..128d80138 100644 --- a/openlp/plugins/songs/lib/importer.py +++ b/openlp/plugins/songs/lib/importer.py @@ -26,6 +26,7 @@ from opensongimport import OpenSongImport from olpimport import OpenLPSongImport +from openlyricsimport import OpenLyricsImport from wowimport import WowImport from cclifileimport import CCLIFileImport from ewimport import EasyWorshipSongImport @@ -77,8 +78,10 @@ class SongFormat(object): """ if format == SongFormat.OpenLP2: return OpenLPSongImport - if format == SongFormat.OpenLP1: + elif format == SongFormat.OpenLP1: return OpenLP1SongImport + elif format == SongFormat.OpenLyrics: + return OpenLyricsImport elif format == SongFormat.OpenSong: return OpenSongImport elif format == SongFormat.SongsOfFellowship: @@ -93,7 +96,6 @@ class SongFormat(object): return EasyWorshipSongImport elif format == SongFormat.SongBeamer: return SongBeamerImport -# else: return None @staticmethod diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index f00711bb6..a6558d669 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -80,8 +80,8 @@ class SongXMLBuilder(object): ``content`` The actual text of the verse to be stored. """ - verse = etree.Element(u'verse', type = unicode(type), - label = unicode(number)) + verse = etree.Element(u'verse', type=unicode(type), + label=unicode(number)) verse.text = etree.CDATA(content) self.lyrics.append(verse) @@ -194,9 +194,7 @@ class LyricsXML(object): text = text.replace('\r\n', '\n') verses = text.split('\n\n') self.languages = [{u'language': u'en', u'verses': []}] - counter = 0 - for verse in verses: - counter = counter + 1 + for counter, verse in enumerate(verses): self.languages[0][u'verses'].append({ u'type': u'verse', u'label': unicode(counter), @@ -245,14 +243,16 @@ class LyricsXML(object): class OpenLyricsParser(object): """ - This class represents the converter for Song to/from OpenLyrics XML. + This class represents the converter for Song to/from + `OpenLyrics `_ XML. """ + # TODO: complete OpenLyrics standard implementation! def __init__(self, manager): self.manager = manager def song_to_xml(self, song): """ - Convert the song to OpenLyrics Format + Convert the song to OpenLyrics Format. """ song_xml_parser = SongXMLParser(song.lyrics) verse_list = song_xml_parser.get_verses() @@ -286,7 +286,7 @@ class OpenLyricsParser(object): def xml_to_song(self, xml): """ - Create a Song from OpenLyrics format xml + Create and save a Song from OpenLyrics format xml. """ # No xml get out of here if not xml: @@ -299,23 +299,15 @@ class OpenLyricsParser(object): song.copyright = unicode(properties.copyright.text) if song.copyright == u'None': song.copyright = u'' - song.verse_order = unicode(properties.verseOrder.text) - if song.verse_order == u'None': - song.verse_order = u'' song.topics = [] song.book = None - theme_name = None try: song.ccli_number = unicode(properties.ccliNo.text) - except: + except AttributeError: song.ccli_number = u'' try: - theme_name = unicode(properties.themes.theme) - except: - pass - if theme_name: - song.theme_name = theme_name - else: + song.theme_name = unicode(properties.themes.theme) + except AttributeError: song.theme_name = u'' # Process Titles for title in properties.titles.title: @@ -331,6 +323,7 @@ class OpenLyricsParser(object): # Process Lyrics sxml = SongXMLBuilder() search_text = u'' + song.verse_order = u'' for lyrics in song_xml.lyrics: for verse in song_xml.lyrics.verse: text = u'' @@ -341,17 +334,36 @@ class OpenLyricsParser(object): else: text += u'\n' + line type = VerseType.expand_string(verse.attrib[u'name'][0]) + # Here we need to create the verse order for the case that the + # song does not have a verseOrder property. sxml.add_verse_to_lyrics(type, verse.attrib[u'name'][1], text) search_text = search_text + text song.search_lyrics = search_text.lower() song.lyrics = unicode(sxml.extract_xml(), u'utf-8') + try: + song.verse_order = unicode(properties.verseOrder.text) + except AttributeError: + # TODO: Do not allow empty verse order. + # Do not worry! + pass + if song.verse_order == u'None': + song.verse_order = u'' + # Process Comments song.comments = u'' + try: + for comment in properties.comments.comment: + if not song.comments: + song.comments = comment + else: + song.comments += u'\n' + comment + except AttributeError: + pass song.song_number = u'' # Process Authors try: for author in properties.authors.author: self._process_author(author.text, song) - except: + except AttributeError: # No Author in XML so ignore pass self.manager.save_object(song) @@ -396,4 +408,4 @@ class OpenLyricsParser(object): new_author = Author.populate(first_name=name.rsplit(u' ', 1)[0], last_name=name.rsplit(u' ', 1)[1], display_name=name) self.manager.save_object(new_author) - song.authors.append(new_author) \ No newline at end of file + song.authors.append(new_author) From 909fa25e7b08a6d9fc0de438c6c430b3929816c0 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Sun, 2 Jan 2011 18:24:47 +0100 Subject: [PATCH 02/17] clean ups, added missing file --- openlp/plugins/songs/lib/mediaitem.py | 13 ++-- openlp/plugins/songs/lib/openlyricsimport.py | 77 ++++++++++++++++++++ openlp/plugins/songs/lib/xml.py | 1 + 3 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 openlp/plugins/songs/lib/openlyricsimport.py diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index cd305877c..011373d68 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -314,15 +314,14 @@ class SongMediaItem(MediaManagerItem): translate('SongsPlugin.MediaItem', 'You must select an item to delete.')): items = self.listView.selectedIndexes() - ans = QtGui.QMessageBox.question(self, + if QtGui.QMessageBox.question(self, translate('SongsPlugin.MediaItem', 'Delete Song(s)?'), translate('SongsPlugin.MediaItem', 'Are you sure you want to delete the %n selected song(s)?', '', QtCore.QCoreApplication.CodecForTr, len(items)), - QtGui.QMessageBox.StandardButtons(QtGui.QMessageBox.Ok| - QtGui.QMessageBox.Cancel), - QtGui.QMessageBox.Ok) - if ans == QtGui.QMessageBox.Cancel: + QtGui.QMessageBox.StandardButtons(QtGui.QMessageBox.Ok | + QtGui.QMessageBox.Cancel), + QtGui.QMessageBox.Ok) == QtGui.QMessageBox.Cancel: return for item in items: item_id = (item.data(QtCore.Qt.UserRole)).toInt()[0] @@ -396,8 +395,8 @@ class SongMediaItem(MediaManagerItem): service_item.audit = [ song.title, author_audit, song.copyright, unicode(song.ccli_number) ] - service_item.data_string = {u'title':song.search_title, - u'authors':author_list} + service_item.data_string = {u'title': song.search_title, + u'authors': author_list} service_item.xml_version = self.openLyrics.song_to_xml(song) return True diff --git a/openlp/plugins/songs/lib/openlyricsimport.py b/openlp/plugins/songs/lib/openlyricsimport.py new file mode 100644 index 000000000..6deb9cc09 --- /dev/null +++ b/openlp/plugins/songs/lib/openlyricsimport.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2011 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian # +# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, # +# Carsten Tinggaard, Frode Woldsund # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`openlyricsimport` module provides the functionality for importing +songs which are saved as OpenLyrics files. +""" + +#import logging +import os + +from openlp.core.lib import translate +from openlp.plugins.songs.lib.songimport import SongImport +from openlp.plugins.songs.lib import OpenLyricsParser + +#log = logging.getLogger(__name__) + +class OpenLyricsImport(SongImport): + """ + This provides the Openlyrics import. + """ + def __init__(self, master_manager, **kwargs): + """ + Initialise the import. + """ + SongImport.__init__(self, master_manager) + self.master_manager = master_manager + self.openLyricsParser = OpenLyricsParser(master_manager) + if kwargs.has_key(u'filename'): + self.import_source = kwargs[u'filename'] + if kwargs.has_key(u'filenames'): + self.import_source = kwargs[u'filenames'] + + def do_import(self): + """ + Imports the songs. + """ + success = True + self.import_wizard.importProgressBar.setMaximum(len(self.import_source)) + for file_path in self.import_source: + if self.stop_import_flag: + success = False + break + file = open(file_path) + xml = file.read() + file.close() + self.import_wizard.incrementProgressBar(unicode(translate( + 'SongsPlugin.OpenLyricsImport', 'Importing %s...')) % + os.path.basename(file_path)) + if self.openLyricsParser.xml_to_song(xml) == 0: + success = false + break + if success: + return True + return False diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index a6558d669..65be3f5e4 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -367,6 +367,7 @@ class OpenLyricsParser(object): # No Author in XML so ignore pass self.manager.save_object(song) + # TODO: better return song itself, instead of song.id return song.id def _add_text_to_element(self, tag, parent, text=None, label=None): From 9fc113abceae1a21a3ce1747ba58b86af79edfd0 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Mon, 3 Jan 2011 08:16:21 +0100 Subject: [PATCH 03/17] fix wrong 'themes' usage (themes -> topics), improved methods, make sure a missing property does not crash openlp --- openlp/plugins/songs/lib/xml.py | 112 ++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 36 deletions(-) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 65be3f5e4..290d83d38 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -42,8 +42,10 @@ import logging import re from lxml import etree, objectify + +from openlp.core.lib import translate from openlp.plugins.songs.lib import VerseType -from openlp.plugins.songs.lib.db import Author, Song +from openlp.plugins.songs.lib.db import Author, Song, Topic log = logging.getLogger(__name__) @@ -246,7 +248,7 @@ class OpenLyricsParser(object): This class represents the converter for Song to/from `OpenLyrics `_ XML. """ - # TODO: complete OpenLyrics standard implementation! + # TODO: complete OpenLyrics standard implementation as fare as possible! def __init__(self, manager): self.manager = manager @@ -288,7 +290,7 @@ class OpenLyricsParser(object): """ Create and save a Song from OpenLyrics format xml. """ - # No xml get out of here + # No xml get out of here. if not xml: return 0 song = Song() @@ -296,19 +298,18 @@ class OpenLyricsParser(object): xml = xml[38:] song_xml = objectify.fromstring(xml) properties = song_xml.properties - song.copyright = unicode(properties.copyright.text) - if song.copyright == u'None': + # Process Copyright + try: + song.copyright = unicode(properties.copyright.text) + if song.copyright == u'None': + song.copyright = u'' + except AttributeError: song.copyright = u'' - song.topics = [] - song.book = None + # Process CCLI number try: song.ccli_number = unicode(properties.ccliNo.text) except AttributeError: song.ccli_number = u'' - try: - song.theme_name = unicode(properties.themes.theme) - except AttributeError: - song.theme_name = u'' # Process Titles for title in properties.titles.title: if not song.title: @@ -325,26 +326,29 @@ class OpenLyricsParser(object): search_text = u'' song.verse_order = u'' for lyrics in song_xml.lyrics: - for verse in song_xml.lyrics.verse: + for verse in lyrics.verse: text = u'' - for line in verse.lines.line: - line = unicode(line) - if not text: - text = line - else: - text += u'\n' + line - type = VerseType.expand_string(verse.attrib[u'name'][0]) - # Here we need to create the verse order for the case that the - # song does not have a verseOrder property. - sxml.add_verse_to_lyrics(type, verse.attrib[u'name'][1], text) + # Note that the element will not be in OpenLyrics 0.8: + # http://code.google.com/p/openlyrics/issues/detail?id=8 + for line in verse.lines: + for line in line.line: + line = unicode(line) + if not text: + text = line + else: + text += u'\n' + line + type_ = VerseType.expand_string(verse.attrib[u'name'][0]) + # TODO: Here we need to create the verse order for the case that + # the song does not have a verseOrder property. + sxml.add_verse_to_lyrics(type_, verse.attrib[u'name'][1], text) search_text = search_text + text song.search_lyrics = search_text.lower() song.lyrics = unicode(sxml.extract_xml(), u'utf-8') + # Process verse order try: song.verse_order = unicode(properties.verseOrder.text) except AttributeError: # TODO: Do not allow empty verse order. - # Do not worry! pass if song.verse_order == u'None': song.verse_order = u'' @@ -358,16 +362,26 @@ class OpenLyricsParser(object): song.comments += u'\n' + comment except AttributeError: pass - song.song_number = u'' # Process Authors try: for author in properties.authors.author: self._process_author(author.text, song) except AttributeError: - # No Author in XML so ignore pass + if not song.authors: + # Add "Author unknown" (can be translated) + self._process_author(translate('SongsPlugin.XML', + 'Author unknown'), song) + # Process Topcis + try: + for topic in properties.themes.theme: + self._process_topic(topic.text, song) + except AttributeError: + pass + # Properties not yet supported. + song.book = None + song.song_number = u'' self.manager.save_object(song) - # TODO: better return song itself, instead of song.id return song.id def _add_text_to_element(self, tag, parent, text=None, label=None): @@ -396,17 +410,43 @@ class OpenLyricsParser(object): def _process_author(self, name, song): """ - Find or create an Author from display_name. + Finds an existing Author or creates a new Author and adds it to the song + object. + + ``name`` + The display_name of the song (string). + + ``song`` + The song the Author will be added to. """ + if not name: + return name = unicode(name) author = self.manager.get_object_filtered(Author, Author.display_name == name) - if author: - # should only be one! so take the first - song.authors.append(author) - else: - # Need a new author - new_author = Author.populate(first_name=name.rsplit(u' ', 1)[0], - last_name=name.rsplit(u' ', 1)[1], display_name=name) - self.manager.save_object(new_author) - song.authors.append(new_author) + if author is None: + # We need to create a new author, as the author does not exist. + author = Author.populate(first_name=name.rsplit(u' ', 1)[0], + last_name=name.rsplit(u' ', 1)[1], display_name=name) + self.manager.save_object(author) + song.authors.append(author) + + def _process_topic(self, topictext, song): + """ + Finds an existing Topic or creates a new Topic and adds it to the song + object. + + ``topictext`` + The topictext we add to the song (string). + + ``song`` + The song the Topic will be added to. + """ + topictext = unicode(topictext) + # Check if topic already exists in the database. + topic = self.manager.get_object_filtered(Topic, Topic.name == topictext) + if topic is None: + # We need to create a new topic, as the topic does not exist. + topic = Topic.populate(name=topictext) + self.manager.save_object(topic) + song.topics.append(topic) From a127019f28c3d3810cb1392f7949e0cbdb23b9ed Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Mon, 3 Jan 2011 14:43:09 +0100 Subject: [PATCH 04/17] fixed wrong use of 'theme', implemented songbooks and songnumber, continued with openlyrics implementation, clean ups --- openlp/plugins/songs/lib/mediaitem.py | 7 +- openlp/plugins/songs/lib/openlyricsimport.py | 11 +-- openlp/plugins/songs/lib/xml.py | 91 ++++++++++++++++---- 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index 011373d68..d2565ad8f 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -404,11 +404,13 @@ class SongMediaItem(MediaManagerItem): """ Triggered by a song being loaded by the service item """ + # TODO: fix db flooding log.debug(u'serviceLoad') if item.data_string: + print item.data_string search_results = self.parent.manager.get_all_objects(Song, Song.search_title == - item.data_string[u'title'].split(u'@')[0].lower() , + item.data_string[u'title'].split(u'@')[0].lower(), Song.search_title.asc()) author_list = item.data_string[u'authors'].split(u', ') # The service item always has an author (at least it has u'' as @@ -417,7 +419,6 @@ class SongMediaItem(MediaManagerItem): if u'' in author_list: author_list.remove(u'') editId = 0 - uuid = item._uuid add_song = True if search_results: for song in search_results: @@ -444,7 +445,7 @@ class SongMediaItem(MediaManagerItem): # Update service with correct song id. if editId != 0: Receiver.send_message(u'service_item_update', - u'%s:%s' %(editId, uuid)) + u'%s:%s' % (editId, item._uuid)) def collateSongTitles(self, song_1, song_2): """ diff --git a/openlp/plugins/songs/lib/openlyricsimport.py b/openlp/plugins/songs/lib/openlyricsimport.py index 6deb9cc09..f4f49e1b5 100644 --- a/openlp/plugins/songs/lib/openlyricsimport.py +++ b/openlp/plugins/songs/lib/openlyricsimport.py @@ -61,8 +61,7 @@ class OpenLyricsImport(SongImport): self.import_wizard.importProgressBar.setMaximum(len(self.import_source)) for file_path in self.import_source: if self.stop_import_flag: - success = False - break + return False file = open(file_path) xml = file.read() file.close() @@ -70,8 +69,6 @@ class OpenLyricsImport(SongImport): 'SongsPlugin.OpenLyricsImport', 'Importing %s...')) % os.path.basename(file_path)) if self.openLyricsParser.xml_to_song(xml) == 0: - success = false - break - if success: - return True - return False + # Importing this song failed! For now we stop import. + return False + return True diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 290d83d38..2a3ec4944 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -45,7 +45,7 @@ from lxml import etree, objectify from openlp.core.lib import translate from openlp.plugins.songs.lib import VerseType -from openlp.plugins.songs.lib.db import Author, Song, Topic +from openlp.plugins.songs.lib.db import Author, Book, Song, Topic log = logging.getLogger(__name__) @@ -265,22 +265,41 @@ class OpenLyricsParser(object): self._add_text_to_element(u'title', titles, song.title) if song.alternate_title: self._add_text_to_element(u'title', titles, song.alternate_title) - if song.theme_name: - themes = etree.SubElement(properties, u'themes') - self._add_text_to_element(u'theme', themes, song.theme_name) - self._add_text_to_element(u'copyright', properties, song.copyright) - self._add_text_to_element(u'verseOrder', properties, song.verse_order) + if song.comments: + comments = etree.SubElement(properties, u'comments') + self._add_text_to_element(u'comment', comments, song.comments) + if song.copyright: + self._add_text_to_element(u'copyright', properties, song.copyright) + if song.verse_order: + self._add_text_to_element( + u'verseOrder', properties, song.verse_order) if song.ccli_number: self._add_text_to_element(u'ccliNo', properties, song.ccli_number) - authors = etree.SubElement(properties, u'authors') - for author in song.authors: - self._add_text_to_element(u'author', authors, author.display_name) + if song.authors: + authors = etree.SubElement(properties, u'authors') + for author in song.authors: + self._add_text_to_element( + u'author', authors, author.display_name) + book = self.manager.get_object_filtered( + Book, Book.id == song.song_book_id) + if book is not None: + book = book.name + songbooks = etree.SubElement(properties, u'songbooks') + element = self._add_text_to_element( + u'songbook', songbooks, None, book) + element.set(u'entry', song.song_number) + if song.topics: + themes = etree.SubElement(properties, u'themes') + for topic in song.topics: + self._add_text_to_element(u'theme', themes, topic.name) lyrics = etree.SubElement(song_xml, u'lyrics') for verse in verse_list: verse_tag = u'%s%s' % ( verse[0][u'type'][0].lower(), verse[0][u'label']) element = \ self._add_text_to_element(u'verse', lyrics, None, verse_tag) + # Note that the element will not be in OpenLyrics 0.8: + # http://code.google.com/p/openlyrics/issues/detail?id=8 element = self._add_text_to_element(u'lines', element) for line in unicode(verse[1]).split(u'\n'): self._add_text_to_element(u'line', element, line) @@ -357,9 +376,9 @@ class OpenLyricsParser(object): try: for comment in properties.comments.comment: if not song.comments: - song.comments = comment + song.comments = unicode(comment.text) else: - song.comments += u'\n' + comment + song.comments += u'\n' + unicode(comment.text) except AttributeError: pass # Process Authors @@ -372,6 +391,18 @@ class OpenLyricsParser(object): # Add "Author unknown" (can be translated) self._process_author(translate('SongsPlugin.XML', 'Author unknown'), song) + # Process Song Book and Song Number + song.song_book_id = 0 + song.song_number = u'' + try: + for songbook in properties.songbooks.songbook: + self._process_songbook(songbook.get(u'name'), song) + if songbook.get(u'entry'): + song.song_number = unicode(songbook.get(u'entry')) + # OpenLp does only support one song book, so take the first one. + break + except AttributeError: + pass # Process Topcis try: for topic in properties.themes.theme: @@ -379,8 +410,7 @@ class OpenLyricsParser(object): except AttributeError: pass # Properties not yet supported. - song.book = None - song.song_number = u'' + song.theme_name = u'' self.manager.save_object(song) return song.id @@ -417,9 +447,10 @@ class OpenLyricsParser(object): The display_name of the song (string). ``song`` - The song the Author will be added to. + The song the object. """ if not name: + # Wrong use of XML here, as no text has been supplied. return name = unicode(name) author = self.manager.get_object_filtered(Author, @@ -433,20 +464,44 @@ class OpenLyricsParser(object): def _process_topic(self, topictext, song): """ - Finds an existing Topic or creates a new Topic and adds it to the song + Finds an existing topic or creates a new topic and adds it to the song object. ``topictext`` - The topictext we add to the song (string). + The topictext of the topic (string). ``song`` - The song the Topic will be added to. + The song object. """ + if not topictext: + # Wrong use of XML here, as no text has been supplied. + return topictext = unicode(topictext) - # Check if topic already exists in the database. topic = self.manager.get_object_filtered(Topic, Topic.name == topictext) if topic is None: # We need to create a new topic, as the topic does not exist. topic = Topic.populate(name=topictext) self.manager.save_object(topic) song.topics.append(topic) + + def _process_songbook(self, bookname, song): + """ + Finds an existing book or creates a new book and adds it to the song + object. + + ``bookname`` + The name of the book (string). + + ``song`` + The song object. + """ + if not bookname: + # Wrong use of XML here, as no text has been supplied. + return + bookname = unicode(bookname) + book = self.manager.get_object_filtered(Book, Book.name == bookname) + if book is None: + # We need to create a new book, as the book does not exist. + book = Book.populate(name=bookname, publisher=u'') + self.manager.save_object(book) + song.song_book_id = book.id From a5f73713d4192ef0ac9b961255c1f1eebbac8fda Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Mon, 3 Jan 2011 22:47:49 +0100 Subject: [PATCH 05/17] add standard verseOrder, if xml does not have any verseOrder property; added '_get' and '_text' methods which do the same as 'get' and 'text', but make sure that they never return 'None' (but u'' instead) --- openlp/plugins/songs/lib/mediaitem.py | 2 - openlp/plugins/songs/lib/openlyricsimport.py | 8 +- openlp/plugins/songs/lib/xml.py | 86 +++++++++++++------- 3 files changed, 58 insertions(+), 38 deletions(-) diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index d2565ad8f..93d648146 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -404,10 +404,8 @@ class SongMediaItem(MediaManagerItem): """ Triggered by a song being loaded by the service item """ - # TODO: fix db flooding log.debug(u'serviceLoad') if item.data_string: - print item.data_string search_results = self.parent.manager.get_all_objects(Song, Song.search_title == item.data_string[u'title'].split(u'@')[0].lower(), diff --git a/openlp/plugins/songs/lib/openlyricsimport.py b/openlp/plugins/songs/lib/openlyricsimport.py index f4f49e1b5..5346b5803 100644 --- a/openlp/plugins/songs/lib/openlyricsimport.py +++ b/openlp/plugins/songs/lib/openlyricsimport.py @@ -28,15 +28,12 @@ The :mod:`openlyricsimport` module provides the functionality for importing songs which are saved as OpenLyrics files. """ -#import logging import os from openlp.core.lib import translate from openlp.plugins.songs.lib.songimport import SongImport from openlp.plugins.songs.lib import OpenLyricsParser -#log = logging.getLogger(__name__) - class OpenLyricsImport(SongImport): """ This provides the Openlyrics import. @@ -57,14 +54,15 @@ class OpenLyricsImport(SongImport): """ Imports the songs. """ - success = True self.import_wizard.importProgressBar.setMaximum(len(self.import_source)) for file_path in self.import_source: if self.stop_import_flag: return False file = open(file_path) - xml = file.read() + lines = file.readlines() file.close() + lines = [line.strip() for line in lines] + xml = u''.join(lines) self.import_wizard.incrementProgressBar(unicode(translate( 'SongsPlugin.OpenLyricsImport', 'Importing %s...')) % os.path.basename(file_path)) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 2a3ec4944..326f97537 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -298,8 +298,6 @@ class OpenLyricsParser(object): verse[0][u'type'][0].lower(), verse[0][u'label']) element = \ self._add_text_to_element(u'verse', lyrics, None, verse_tag) - # Note that the element will not be in OpenLyrics 0.8: - # http://code.google.com/p/openlyrics/issues/detail?id=8 element = self._add_text_to_element(u'lines', element) for line in unicode(verse[1]).split(u'\n'): self._add_text_to_element(u'line', element, line) @@ -307,7 +305,10 @@ class OpenLyricsParser(object): def xml_to_song(self, xml): """ - Create and save a Song from OpenLyrics format xml. + Create and save a song from OpenLyrics format xml to the database. Since + we also export XML from external sources (e. g. OpenLyrics import), we + cannot ensure, that it completely conforms to the OpenLyrics standard. + That means, that we for example have to remove chords. """ # No xml get out of here. if not xml: @@ -315,28 +316,28 @@ class OpenLyricsParser(object): song = Song() if xml[:5] == u'').sub(u'', xml) song_xml = objectify.fromstring(xml) properties = song_xml.properties # Process Copyright try: - song.copyright = unicode(properties.copyright.text) - if song.copyright == u'None': - song.copyright = u'' + song.copyright = self._text(properties.copyright) except AttributeError: song.copyright = u'' # Process CCLI number try: - song.ccli_number = unicode(properties.ccliNo.text) + song.ccli_number = self._text(properties.ccliNo) except AttributeError: song.ccli_number = u'' # Process Titles for title in properties.titles.title: if not song.title: - song.title = unicode(title.text) + song.title = self._text(title) song.search_title = unicode(song.title) song.alternate_title = u'' else: - song.alternate_title = unicode(title.text) + song.alternate_title = self._text(title) song.search_title += u'@' + song.alternate_title song.search_title = re.sub(r'[\'"`,;:(){}?]+', u'', unicode(song.search_title)).lower() @@ -347,8 +348,6 @@ class OpenLyricsParser(object): for lyrics in song_xml.lyrics: for verse in lyrics.verse: text = u'' - # Note that the element will not be in OpenLyrics 0.8: - # http://code.google.com/p/openlyrics/issues/detail?id=8 for line in verse.lines: for line in line.line: line = unicode(line) @@ -356,18 +355,21 @@ class OpenLyricsParser(object): text = line else: text += u'\n' + line - type_ = VerseType.expand_string(verse.attrib[u'name'][0]) - # TODO: Here we need to create the verse order for the case that - # the song does not have a verseOrder property. - sxml.add_verse_to_lyrics(type_, verse.attrib[u'name'][1], text) + type = VerseType.expand_string(verse.attrib[u'name'][0]) + sxml.add_verse_to_lyrics(type, verse.attrib[u'name'][1], text) + # TODO: test this verse_order thing! + song.verse_order += u'%s%s ' % (type[0], + verse.attrib[u'name'][1]) search_text = search_text + text song.search_lyrics = search_text.lower() song.lyrics = unicode(sxml.extract_xml(), u'utf-8') + song.verse_order = song.verse_order.strip() # Process verse order try: - song.verse_order = unicode(properties.verseOrder.text) + song.verse_order = self._text(properties.verseOrder) except AttributeError: - # TODO: Do not allow empty verse order. + # Do not worry, as the verse order has cautionary already been + # saved while creating the verses. pass if song.verse_order == u'None': song.verse_order = u'' @@ -376,29 +378,29 @@ class OpenLyricsParser(object): try: for comment in properties.comments.comment: if not song.comments: - song.comments = unicode(comment.text) + song.comments = self._text(comment) else: - song.comments += u'\n' + unicode(comment.text) + song.comments += u'\n' + self._text(comment) except AttributeError: pass # Process Authors try: for author in properties.authors.author: - self._process_author(author.text, song) + self._process_author(self._text(author), song) except AttributeError: pass if not song.authors: # Add "Author unknown" (can be translated) - self._process_author(translate('SongsPlugin.XML', - 'Author unknown'), song) + self._process_author(unicode(translate('SongsPlugin.XML', + 'Author unknown')), song) # Process Song Book and Song Number song.song_book_id = 0 song.song_number = u'' try: for songbook in properties.songbooks.songbook: - self._process_songbook(songbook.get(u'name'), song) + self._process_songbook(self._get(songbook, u'name'), song) if songbook.get(u'entry'): - song.song_number = unicode(songbook.get(u'entry')) + song.song_number = self._get(songbook, u'entry') # OpenLp does only support one song book, so take the first one. break except AttributeError: @@ -406,7 +408,7 @@ class OpenLyricsParser(object): # Process Topcis try: for topic in properties.themes.theme: - self._process_topic(topic.text, song) + self._process_topic(self._text(topic), song) except AttributeError: pass # Properties not yet supported. @@ -414,6 +416,31 @@ class OpenLyricsParser(object): self.manager.save_object(song) return song.id + def _get(self, element, attribute): + """ + This takes care of empty attributes. It returns the element's attribute. + + ``element`` + The element. + + ``attribute`` + The element's attribute (unicode). + """ + if element.get(attribute) is not None: + return element.get(attribute) + return u'' + + def _text(self, element): + """ + This takes care of empty texts. It returns the element's text. + + ``element`` + The element. + """ + if element.text is not None: + return unicode(element.text) + return u'' + def _add_text_to_element(self, tag, parent, text=None, label=None): if label: element = etree.Element(tag, name=unicode(label)) @@ -444,7 +471,7 @@ class OpenLyricsParser(object): object. ``name`` - The display_name of the song (string). + The display_name of the song (unicode). ``song`` The song the object. @@ -452,7 +479,6 @@ class OpenLyricsParser(object): if not name: # Wrong use of XML here, as no text has been supplied. return - name = unicode(name) author = self.manager.get_object_filtered(Author, Author.display_name == name) if author is None: @@ -468,7 +494,7 @@ class OpenLyricsParser(object): object. ``topictext`` - The topictext of the topic (string). + The topictext of the topic (unicode). ``song`` The song object. @@ -476,7 +502,6 @@ class OpenLyricsParser(object): if not topictext: # Wrong use of XML here, as no text has been supplied. return - topictext = unicode(topictext) topic = self.manager.get_object_filtered(Topic, Topic.name == topictext) if topic is None: # We need to create a new topic, as the topic does not exist. @@ -490,7 +515,7 @@ class OpenLyricsParser(object): object. ``bookname`` - The name of the book (string). + The name of the book (unicode). ``song`` The song object. @@ -498,7 +523,6 @@ class OpenLyricsParser(object): if not bookname: # Wrong use of XML here, as no text has been supplied. return - bookname = unicode(bookname) book = self.manager.get_object_filtered(Book, Book.name == bookname) if book is None: # We need to create a new book, as the book does not exist. From 449610d50e2de222b3669dcb243b465fb1ff26a9 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Tue, 4 Jan 2011 23:06:43 +0100 Subject: [PATCH 07/17] - fixed author wihtou first_name e. g. "Luther" - open xml file properly - tweakes --- openlp/plugins/songs/lib/openlyricsimport.py | 15 ++++++--- openlp/plugins/songs/lib/xml.py | 33 ++++++++++---------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/openlp/plugins/songs/lib/openlyricsimport.py b/openlp/plugins/songs/lib/openlyricsimport.py index 5346b5803..27c2308f4 100644 --- a/openlp/plugins/songs/lib/openlyricsimport.py +++ b/openlp/plugins/songs/lib/openlyricsimport.py @@ -28,12 +28,17 @@ The :mod:`openlyricsimport` module provides the functionality for importing songs which are saved as OpenLyrics files. """ +import logging import os +from lxml import etree + from openlp.core.lib import translate from openlp.plugins.songs.lib.songimport import SongImport from openlp.plugins.songs.lib import OpenLyricsParser +log = logging.getLogger(__name__) + class OpenLyricsImport(SongImport): """ This provides the Openlyrics import. @@ -42,6 +47,7 @@ class OpenLyricsImport(SongImport): """ Initialise the import. """ + log.debug(u'initialise OpenLyricsImport') SongImport.__init__(self, master_manager) self.master_manager = master_manager self.openLyricsParser = OpenLyricsParser(master_manager) @@ -58,15 +64,14 @@ class OpenLyricsImport(SongImport): for file_path in self.import_source: if self.stop_import_flag: return False - file = open(file_path) - lines = file.readlines() - file.close() - lines = [line.strip() for line in lines] - xml = u''.join(lines) self.import_wizard.incrementProgressBar(unicode(translate( 'SongsPlugin.OpenLyricsImport', 'Importing %s...')) % os.path.basename(file_path)) + parser = etree.XMLParser(remove_blank_text=True) + file = etree.parse(file_path, parser) + xml = etree.tostring(file) if self.openLyricsParser.xml_to_song(xml) == 0: + log.debug(u'File could not be imported: %s' % file_path) # Importing this song failed! For now we stop import. return False return True diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 326f97537..4045a412a 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -248,7 +248,8 @@ class OpenLyricsParser(object): This class represents the converter for Song to/from `OpenLyrics `_ XML. """ - # TODO: complete OpenLyrics standard implementation as fare as possible! + # TODO: Complete OpenLyrics standard implementation and document what is + # supported and what not! def __init__(self, manager): self.manager = manager @@ -316,8 +317,6 @@ class OpenLyricsParser(object): song = Song() if xml[:5] == u'').sub(u'', xml) song_xml = objectify.fromstring(xml) properties = song_xml.properties # Process Copyright @@ -349,17 +348,17 @@ class OpenLyricsParser(object): for verse in lyrics.verse: text = u'' for line in verse.lines: - for line in line.line: - line = unicode(line) - if not text: - text = line - else: - text += u'\n' + line - type = VerseType.expand_string(verse.attrib[u'name'][0]) - sxml.add_verse_to_lyrics(type, verse.attrib[u'name'][1], text) - # TODO: test this verse_order thing! + text = u'\n'.join([unicode(line) for line in line.line]) + # Remove chords + text = re.compile(u'').sub(u'', text) + # OpenLyrics allows e. g. "c", but we need "c1". + if self._get(verse, u'name').isalpha(): + verse.set(u'name', self._get(verse, u'name') + u'1') + type = VerseType.expand_string(self._get(verse, u'name')[0]) + sxml.add_verse_to_lyrics( + type, self._get(verse, u'name')[1], text) song.verse_order += u'%s%s ' % (type[0], - verse.attrib[u'name'][1]) + self._get(verse, u'name')[1]) search_text = search_text + text song.search_lyrics = search_text.lower() song.lyrics = unicode(sxml.extract_xml(), u'utf-8') @@ -399,7 +398,7 @@ class OpenLyricsParser(object): try: for songbook in properties.songbooks.songbook: self._process_songbook(self._get(songbook, u'name'), song) - if songbook.get(u'entry'): + if self._get(songbook, u'entry'): song.song_number = self._get(songbook, u'entry') # OpenLp does only support one song book, so take the first one. break @@ -427,7 +426,7 @@ class OpenLyricsParser(object): The element's attribute (unicode). """ if element.get(attribute) is not None: - return element.get(attribute) + return unicode(element.get(attribute)) return u'' def _text(self, element): @@ -483,8 +482,8 @@ class OpenLyricsParser(object): Author.display_name == name) if author is None: # We need to create a new author, as the author does not exist. - author = Author.populate(first_name=name.rsplit(u' ', 1)[0], - last_name=name.rsplit(u' ', 1)[1], display_name=name) + author = Author.populate(last_name=name.split(u' ')[-1], + first_name=u' '.join(name.split(u' ')[:-1]), display_name=name) self.manager.save_object(author) song.authors.append(author) From 4246c3b76ee8c3619bd201849a596a331a12550c Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Wed, 5 Jan 2011 17:31:04 +0100 Subject: [PATCH 08/17] - added documentation - commented not needed class out - split class 'OpenLyricsParser' - relocated and imporved some code in methods --- openlp/plugins/songs/lib/__init__.py | 3 +- openlp/plugins/songs/lib/mediaitem.py | 10 +- openlp/plugins/songs/lib/xml.py | 587 +++++++++++++++----------- 3 files changed, 341 insertions(+), 259 deletions(-) diff --git a/openlp/plugins/songs/lib/__init__.py b/openlp/plugins/songs/lib/__init__.py index 5093d6180..491c7e6c5 100644 --- a/openlp/plugins/songs/lib/__init__.py +++ b/openlp/plugins/songs/lib/__init__.py @@ -123,6 +123,7 @@ class VerseType(object): unicode(VerseType.to_string(VerseType.Other)).lower(): return VerseType.Other -from xml import LyricsXML, SongXMLBuilder, SongXMLParser, OpenLyricsParser +from xml import OpenLyricsBuilder, OpenLyricsParser, SongXMLBuilder, \ + SongXMLParser from songstab import SongsTab from mediaitem import SongMediaItem \ No newline at end of file diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index 93d648146..d2776e1a4 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -35,7 +35,8 @@ from openlp.core.lib import MediaManagerItem, BaseListWithDnD, Receiver, \ ItemCapabilities, translate, check_item_selected from openlp.plugins.songs.forms import EditSongForm, SongMaintenanceForm, \ SongImportForm -from openlp.plugins.songs.lib import SongXMLParser, OpenLyricsParser +from openlp.plugins.songs.lib import OpenLyricsBuilder, OpenLyricsParser, \ + SongXMLParser from openlp.plugins.songs.lib.db import Author, Song from openlp.core.lib.searchedit import SearchEdit @@ -58,7 +59,8 @@ class SongMediaItem(MediaManagerItem): self.ListViewWithDnD_class = SongListView MediaManagerItem.__init__(self, parent, self, icon) self.edit_song_form = EditSongForm(self, self.parent.manager) - self.openLyrics = OpenLyricsParser(self.parent.manager) + self.openLyricsParser = OpenLyricsParser(self.parent.manager) + self.openLyricsBuilder = OpenLyricsBuilder(self.parent.manager) self.singleServiceItem = False self.song_maintenance_form = SongMaintenanceForm( self.parent.manager, self) @@ -397,7 +399,7 @@ class SongMediaItem(MediaManagerItem): ] service_item.data_string = {u'title': song.search_title, u'authors': author_list} - service_item.xml_version = self.openLyrics.song_to_xml(song) + service_item.xml_version = self.openLyricsBuilder.song_to_xml(song) return True def serviceLoad(self, item): @@ -439,7 +441,7 @@ class SongMediaItem(MediaManagerItem): break if add_song: if self.addSongFromService: - editId = self.openLyrics.xml_to_song(item.xml_version) + editId = self.openLyricsParser.xml_to_song(item.xml_version) # Update service with correct song id. if editId != 0: Receiver.send_message(u'service_item_update', diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 4045a412a..4d23ebd78 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -24,9 +24,9 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ -The :mod:`xml` module provides the XML functionality for songs +The :mod:`xml` module provides the XML functionality. -The basic XML is of the format:: +The basic XML for storing the lyrics in the song database is of the format:: @@ -36,6 +36,28 @@ The basic XML is of the format:: + + +The XML of `OpenLyrics `_ songs is of the format:: + + + + + Amazing Grace + + + + + + Amazing grace how sweet the sound + + + + """ import logging @@ -144,112 +166,109 @@ class SongXMLParser(object): return etree.dump(self.song_xml) -class LyricsXML(object): +#class LyricsXML(object): +# """ +# This class represents the XML in the ``lyrics`` field of a song. +# """ +# def __init__(self, song=None): +# if song: +# if song.lyrics.startswith(u'' % \ +# (verse[u'type'], verse[u'label'], verse[u'text']) +# lyrics_output = lyrics_output + \ +# u'%s' % \ +# (language[u'language'], verse_output) +# song_output = u'' + \ +# u'%s' % lyrics_output +# return song_output + + +class OpenLyricsBuilder(object): """ - This class represents the XML in the ``lyrics`` field of a song. + This class represents the converter for song to OpenLyrics XML. """ - def __init__(self, song=None): - if song: - if song.lyrics.startswith(u'' % \ - (verse[u'type'], verse[u'label'], verse[u'text']) - lyrics_output = lyrics_output + \ - u'%s' % \ - (language[u'language'], verse_output) - song_output = u'' + \ - u'%s' % lyrics_output - return song_output - - -class OpenLyricsParser(object): - """ - This class represents the converter for Song to/from - `OpenLyrics `_ XML. - """ - # TODO: Complete OpenLyrics standard implementation and document what is - # supported and what not! def __init__(self, manager): self.manager = manager @@ -304,6 +323,98 @@ class OpenLyricsParser(object): self._add_text_to_element(u'line', element, line) return self._extract_xml(song_xml) + def _add_text_to_element(self, tag, parent, text=None, label=None): + if label: + element = etree.Element(tag, name=unicode(label)) + else: + element = etree.Element(tag) + if text: + element.text = unicode(text) + parent.append(element) + return element + + def _extract_xml(self, xml): + """ + Extract our newly created XML song. + """ + return etree.tostring(xml, encoding=u'UTF-8', + xml_declaration=True) + + def _dump_xml(self, xml): + """ + Debugging aid to dump XML so that we can see what we have. + """ + return etree.tostring(xml, encoding=u'UTF-8', + xml_declaration=True, pretty_print=True) + + +class OpenLyricsParser(object): + """ + This class represents the converter for OpenLyrics XML to a song. + + As OpenLyrics has a rich set of different features, we cannot support them + all. The following features are supported by the :class:`OpenLyricsParser`:: + + ```` + OpenLP does not support the author ``type`` and consequently not + ``lang`` for the author of the type ``translation``. + + ```` + This property is not supported. + + ```` + The ```` property is fully supported. But comments in lyrics + are not supported. + + ```` + This property is fully supported. + + ```` + This property is not supported. + + ```` + This property is not supported. + + ```` + This property is not supported. + + ```` + The attribute ``part`` is not supported. + + ```` + This property is not supported. + + ```` + As OpenLP does only support one songbook, we cannot consider more than + one songbook. + + ```` + This property is not supported. + + ```` + Topics, as they are called in OpenLP, are fully supported, whereby only + the topic text (e. g. Grace) is considered, but neither the ``id`` nor + ``lang``. + + ```` + This property is not supported. + + ```` + This property is not supported. + + ```` + The attribute ``translit`` and ``lang`` are not supported. + This class support verse names of the format ```` and + ````. Whereas this class does not support verse names of + the format ```` as OpenLP does not support splitting + verses into different parts. + + ```` + OpenLP supports this property. + """ + def __init__(self, manager): + self.manager = manager + def xml_to_song(self, xml): """ Create and save a song from OpenLyrics format xml to the database. Since @@ -317,6 +428,8 @@ class OpenLyricsParser(object): song = Song() if xml[:5] == u'').sub(u'', xml) song_xml = objectify.fromstring(xml) properties = song_xml.properties # Process Copyright @@ -329,41 +442,11 @@ class OpenLyricsParser(object): song.ccli_number = self._text(properties.ccliNo) except AttributeError: song.ccli_number = u'' - # Process Titles - for title in properties.titles.title: - if not song.title: - song.title = self._text(title) - song.search_title = unicode(song.title) - song.alternate_title = u'' - else: - song.alternate_title = self._text(title) - song.search_title += u'@' + song.alternate_title - song.search_title = re.sub(r'[\'"`,;:(){}?]+', u'', - unicode(song.search_title)).lower() - # Process Lyrics - sxml = SongXMLBuilder() - search_text = u'' + self._process_titles(properties, song) song.verse_order = u'' - for lyrics in song_xml.lyrics: - for verse in lyrics.verse: - text = u'' - for line in verse.lines: - text = u'\n'.join([unicode(line) for line in line.line]) - # Remove chords - text = re.compile(u'').sub(u'', text) - # OpenLyrics allows e. g. "c", but we need "c1". - if self._get(verse, u'name').isalpha(): - verse.set(u'name', self._get(verse, u'name') + u'1') - type = VerseType.expand_string(self._get(verse, u'name')[0]) - sxml.add_verse_to_lyrics( - type, self._get(verse, u'name')[1], text) - song.verse_order += u'%s%s ' % (type[0], - self._get(verse, u'name')[1]) - search_text = search_text + text - song.search_lyrics = search_text.lower() - song.lyrics = unicode(sxml.extract_xml(), u'utf-8') - song.verse_order = song.verse_order.strip() + self._process_lyrics(song_xml, song) # Process verse order + song.verse_order = song.verse_order.strip() try: song.verse_order = self._text(properties.verseOrder) except AttributeError: @@ -372,46 +455,10 @@ class OpenLyricsParser(object): pass if song.verse_order == u'None': song.verse_order = u'' - # Process Comments - song.comments = u'' - try: - for comment in properties.comments.comment: - if not song.comments: - song.comments = self._text(comment) - else: - song.comments += u'\n' + self._text(comment) - except AttributeError: - pass - # Process Authors - try: - for author in properties.authors.author: - self._process_author(self._text(author), song) - except AttributeError: - pass - if not song.authors: - # Add "Author unknown" (can be translated) - self._process_author(unicode(translate('SongsPlugin.XML', - 'Author unknown')), song) - # Process Song Book and Song Number - song.song_book_id = 0 - song.song_number = u'' - try: - for songbook in properties.songbooks.songbook: - self._process_songbook(self._get(songbook, u'name'), song) - if self._get(songbook, u'entry'): - song.song_number = self._get(songbook, u'entry') - # OpenLp does only support one song book, so take the first one. - break - except AttributeError: - pass - # Process Topcis - try: - for topic in properties.themes.theme: - self._process_topic(self._text(topic), song) - except AttributeError: - pass - # Properties not yet supported. - song.theme_name = u'' + self._process_comments(properties, song) + self._process_authors(properties, song) + self._process_songbooks(properties, song) + self._process_topics(properties, song) self.manager.save_object(song) return song.id @@ -440,91 +487,123 @@ class OpenLyricsParser(object): return unicode(element.text) return u'' - def _add_text_to_element(self, tag, parent, text=None, label=None): - if label: - element = etree.Element(tag, name=unicode(label)) - else: - element = etree.Element(tag) - if text: - element.text = unicode(text) - parent.append(element) - return element - - def _dump_xml(self, xml): - """ - Debugging aid to dump XML so that we can see what we have. - """ - return etree.tostring(xml, encoding=u'UTF-8', - xml_declaration=True, pretty_print=True) - - def _extract_xml(self, xml): - """ - Extract our newly created XML song. - """ - return etree.tostring(xml, encoding=u'UTF-8', - xml_declaration=True) - - def _process_author(self, name, song): + def _process_authors(self, properties, song): """ Finds an existing Author or creates a new Author and adds it to the song object. - - ``name`` - The display_name of the song (unicode). - - ``song`` - The song the object. """ - if not name: - # Wrong use of XML here, as no text has been supplied. - return - author = self.manager.get_object_filtered(Author, - Author.display_name == name) - if author is None: - # We need to create a new author, as the author does not exist. - author = Author.populate(last_name=name.split(u' ')[-1], - first_name=u' '.join(name.split(u' ')[:-1]), display_name=name) + authors = [] + try: + for author in properties.authors.author: + display_name = self._text(author) + if display_name: + authors.append(display_name) + except AttributeError: + pass + if not authors: + # Add "Author unknown" (can be translated). + authors.append((unicode(translate('SongsPlugin.XML', + 'Author unknown')))) + for display_name in authors: + author = self.manager.get_object_filtered(Author, + Author.display_name == display_name) + if author is None: + # We need to create a new author, as the author does not exist. + author = Author.populate(display_name=display_name, + last_name=display_name.split(u' ')[-1], + first_name=u' '.join(display_name.split(u' ')[:-1])) self.manager.save_object(author) - song.authors.append(author) + song.authors.append(author) - def _process_topic(self, topictext, song): + def _process_comments(self, properties, song): """ - Finds an existing topic or creates a new topic and adds it to the song - object. - - ``topictext`` - The topictext of the topic (unicode). - - ``song`` - The song object. """ - if not topictext: - # Wrong use of XML here, as no text has been supplied. - return - topic = self.manager.get_object_filtered(Topic, Topic.name == topictext) - if topic is None: - # We need to create a new topic, as the topic does not exist. - topic = Topic.populate(name=topictext) - self.manager.save_object(topic) - song.topics.append(topic) + try: + song.comments = u'\n'.join( + [self._text(comment) for comment in properties.comments.comment] + ) + except AttributeError: + song.comments = u'' - def _process_songbook(self, bookname, song): + def _process_lyrics(self, song_xml, song): + """ + """ + sxml = SongXMLBuilder() + search_text = u'' + for verse in song_xml.lyrics.verse: + text = u'' + for lines in verse.lines: + if text: + text += u'\n' + text += u'\n'.join([unicode(line) for line in lines.line]) + # OpenLyrics allows e. g. "c", but we need "c1". + if self._get(verse, u'name').isalpha(): + verse.set(u'name', self._get(verse, u'name') + u'1') + type = VerseType.expand_string(self._get(verse, u'name')[0]) + sxml.add_verse_to_lyrics(type, self._get(verse, u'name')[1], text) + song.verse_order += u'%s%s ' % (type[0], + self._get(verse, u'name')[1]) + search_text = search_text + text + song.search_lyrics = search_text.lower() + song.lyrics = unicode(sxml.extract_xml(), u'utf-8') + #TODO: make sure "c" becomes "c1" + + def _process_songbooks(self, properties, song): """ Finds an existing book or creates a new book and adds it to the song object. - - ``bookname`` - The name of the book (unicode). - - ``song`` - The song object. """ - if not bookname: - # Wrong use of XML here, as no text has been supplied. - return - book = self.manager.get_object_filtered(Book, Book.name == bookname) - if book is None: - # We need to create a new book, as the book does not exist. - book = Book.populate(name=bookname, publisher=u'') - self.manager.save_object(book) - song.song_book_id = book.id + song.song_book_id = 0 + song.song_number = u'' + try: + for songbook in properties.songbooks.songbook: + bookname = self._get(songbook, u'name') + if bookname: + book = self.manager.get_object_filtered(Book, + Book.name == bookname) + if book is None: + # We need to create a book, because it does not exist. + book = Book.populate(name=bookname, publisher=u'') + self.manager.save_object(book) + song.song_book_id = book.id + if self._get(songbook, u'entry'): + song.song_number = self._get(songbook, u'entry') + # We does only support one song book, so take the first one. + break + except AttributeError: + pass + + def _process_titles(self, properties, song): + """ + """ + for title in properties.titles.title: + if not song.title: + song.title = self._text(title) + song.search_title = unicode(song.title) + song.alternate_title = u'' + else: + song.alternate_title = self._text(title) + song.search_title += u'@' + song.alternate_title + song.search_title = re.sub(r'[\'"`,;:(){}?]+', u'', + unicode(song.search_title)).lower() + + def _process_topics(self, properties, song): + """ + Finds an existing topic or creates a new topic and adds it to the song + object. + """ + try: + for topictext in properties.themes.theme: + topictext = self._text(topictext) + if not topictext: + # Wrong use of XML here, as no text has been supplied. + return + topic = self.manager.get_object_filtered(Topic, + Topic.name == topictext) + if topic is None: + # We need to create a new topic, as the topic does not exist. + topic = Topic.populate(name=topictext) + self.manager.save_object(topic) + song.topics.append(topic) + except AttributeError: + pass From 09e2fb7691c79c48ef431882f06f448b34283985 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Wed, 5 Jan 2011 17:45:29 +0100 Subject: [PATCH 09/17] clean up --- openlp/plugins/songs/lib/xml.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 4d23ebd78..957c377ec 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -420,7 +420,6 @@ class OpenLyricsParser(object): Create and save a song from OpenLyrics format xml to the database. Since we also export XML from external sources (e. g. OpenLyrics import), we cannot ensure, that it completely conforms to the OpenLyrics standard. - That means, that we for example have to remove chords. """ # No xml get out of here. if not xml: @@ -453,8 +452,6 @@ class OpenLyricsParser(object): # Do not worry, as the verse order has cautionary already been # saved while creating the verses. pass - if song.verse_order == u'None': - song.verse_order = u'' self._process_comments(properties, song) self._process_authors(properties, song) self._process_songbooks(properties, song) From b4dfeca4d7be4984bc128a14541355e0302dd3d0 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Wed, 5 Jan 2011 20:34:56 +0100 Subject: [PATCH 10/17] --- openlp/plugins/songs/lib/xml.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 957c377ec..8a9cc6930 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -545,6 +545,11 @@ class OpenLyricsParser(object): song.lyrics = unicode(sxml.extract_xml(), u'utf-8') #TODO: make sure "c" becomes "c1" +# name = self._get(verse, u'name') +# type = name[0] +# number = re.compile(u'[a-zA-Z]*').sub(u'', name) +# part = re.compile(u'[0-9]*').sub(u'', name[1:]) + def _process_songbooks(self, properties, song): """ Finds an existing book or creates a new book and adds it to the song From fc2142849932f289c0e4429254144188af8b8644 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Thu, 6 Jan 2011 10:34:26 +0100 Subject: [PATCH 11/17] fixed docs --- openlp/plugins/songs/lib/xml.py | 45 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 14d2f426f..4a676b1ad 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -355,57 +355,56 @@ class OpenLyricsParser(object): As OpenLyrics has a rich set of different features, we cannot support them all. The following features are supported by the :class:`OpenLyricsParser`:: - ```` - OpenLP does not support the author ``type`` and consequently not - ``lang`` for the author of the type ``translation``. + ** + OpenLP does not support the attribute *type* and *lang*. - ```` + ** This property is not supported. - ```` - The ```` property is fully supported. But comments in lyrics + ** + The ** property is fully supported. But comments in lyrics are not supported. - ```` + ** This property is fully supported. - ```` + ** This property is not supported. - ```` + ** This property is not supported. - ```` + ** This property is not supported. - ```` - The attribute ``part`` is not supported. + ** + The attribute *part* is not supported. - ```` + ** This property is not supported. - ```` + ** As OpenLP does only support one songbook, we cannot consider more than one songbook. - ```` + ** This property is not supported. - ```` + ** Topics, as they are called in OpenLP, are fully supported, whereby only - the topic text (e. g. Grace) is considered, but neither the ``id`` nor - ``lang``. + the topic text (e. g. Grace) is considered, but neither the *id* nor + *lang*. - ```` + ** This property is not supported. - ```` + ** This property is not supported. - ```` - The attribute ``translit`` and ``lang`` are not supported. + ** + The attribute *translit* and *lang* are not supported. - ```` + ** OpenLP supports this property. """ def __init__(self, manager): From 68c550cd711dab7160702645999fdfd9853bcbcd Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Thu, 6 Jan 2011 14:01:30 +0100 Subject: [PATCH 12/17] -basic work for OpenLyrics exporter (songexport wizard is missing) --- openlp/plugins/songs/lib/openlyricsexport.py | 72 ++++++++++++++++++++ openlp/plugins/songs/lib/xml.py | 8 +-- 2 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 openlp/plugins/songs/lib/openlyricsexport.py diff --git a/openlp/plugins/songs/lib/openlyricsexport.py b/openlp/plugins/songs/lib/openlyricsexport.py new file mode 100644 index 000000000..95c96517d --- /dev/null +++ b/openlp/plugins/songs/lib/openlyricsexport.py @@ -0,0 +1,72 @@ +## -*- coding: utf-8 -*- +## vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 +# +################################################################################ +## OpenLP - Open Source Lyrics Projection # +## --------------------------------------------------------------------------- # +## Copyright (c) 2008-2011 Raoul Snyman # +## Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +## Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian # +## Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, # +## Carsten Tinggaard, Frode Woldsund # +## --------------------------------------------------------------------------- # +## This program is free software; you can redistribute it and/or modify it # +## under the terms of the GNU General Public License as published by the Free # +## Software Foundation; version 2 of the License. # +## # +## This program is distributed in the hope that it will be useful, but WITHOUT # +## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +## FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +## more details. # +## # +## You should have received a copy of the GNU General Public License along # +## with this program; if not, write to the Free Software Foundation, Inc., 59 # +## Temple Place, Suite 330, Boston, MA 02111-1307 USA # +################################################################################ +#""" +#The :mod:`openlyricsexport` module provides the functionality for exporting +#songs from the database. +#""" +# +#import logging +#import os +# +#from lxml import etree +# +#from openlp.core.lib import translate +#from openlp.plugins.songs.lib import OpenLyricsBuilder +#from openlp.plugins.songs.lib.songimport import SongImport +# +#log = logging.getLogger(__name__) +# +#class OpenLyricsExport(SongExport): +# """ +# This provides the Openlyrics export. +# """ +# def __init__(self, master_manager, songs=None, save_path=u''): +# """ +# Initialise the export. +# """ +# log.debug(u'initialise OpenLyricsExport') +# SongExport.__init__(self, master_manager) +# self.master_manager = master_manager +# self.songs = songs +# self.save_path = save_path +# +# def do_export(self): +# """ +# Exports the songs. +# """ +# openLyricsBuilder = OpenLyricsBuilder(self.master_manager) +# self.export_wizard.exportProgressBar.setMaximum(len(songs)) +# for song in self.songs: +# if self.stop_export_flag: +# return False +# self.import_wizard.incrementProgressBar(unicode(translate( +# 'SongsPlugin.OpenLyricsExport', 'Importing %s...')) % +# song.title) +# xml = openLyricsBuilder.song_to_xml(song, True) +# tree = etree.ElementTree(etree.fromstring(xml)) +# path = os.path.join(self.save_path, song.title) +# tree.write(path) +# return True diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 4a676b1ad..a5c8ecd06 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -272,7 +272,7 @@ class OpenLyricsBuilder(object): def __init__(self, manager): self.manager = manager - def song_to_xml(self, song): + def song_to_xml(self, song, pretty_print=False): """ Convert the song to OpenLyrics Format. """ @@ -321,7 +321,7 @@ class OpenLyricsBuilder(object): element = self._add_text_to_element(u'lines', element) for line in unicode(verse[1]).split(u'\n'): self._add_text_to_element(u'line', element, line) - return self._extract_xml(song_xml) + return self._extract_xml(song_xml, pretty_print) def _add_text_to_element(self, tag, parent, text=None, label=None): if label: @@ -333,12 +333,12 @@ class OpenLyricsBuilder(object): parent.append(element) return element - def _extract_xml(self, xml): + def _extract_xml(self, xml, pretty_print): """ Extract our newly created XML song. """ return etree.tostring(xml, encoding=u'UTF-8', - xml_declaration=True) + xml_declaration=True, pretty_print=pretty_print) def _dump_xml(self, xml): """ From a51ad14322efafc6966eb7a9ff680685a11178d8 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Sat, 8 Jan 2011 21:35:31 +0100 Subject: [PATCH 13/17] removed OpenLyrics export for now --- openlp/plugins/songs/lib/openlyricsexport.py | 72 -------------------- 1 file changed, 72 deletions(-) delete mode 100644 openlp/plugins/songs/lib/openlyricsexport.py diff --git a/openlp/plugins/songs/lib/openlyricsexport.py b/openlp/plugins/songs/lib/openlyricsexport.py deleted file mode 100644 index 95c96517d..000000000 --- a/openlp/plugins/songs/lib/openlyricsexport.py +++ /dev/null @@ -1,72 +0,0 @@ -## -*- coding: utf-8 -*- -## vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 -# -################################################################################ -## OpenLP - Open Source Lyrics Projection # -## --------------------------------------------------------------------------- # -## Copyright (c) 2008-2011 Raoul Snyman # -## Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # -## Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian # -## Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, # -## Carsten Tinggaard, Frode Woldsund # -## --------------------------------------------------------------------------- # -## This program is free software; you can redistribute it and/or modify it # -## under the terms of the GNU General Public License as published by the Free # -## Software Foundation; version 2 of the License. # -## # -## This program is distributed in the hope that it will be useful, but WITHOUT # -## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -## FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -## more details. # -## # -## You should have received a copy of the GNU General Public License along # -## with this program; if not, write to the Free Software Foundation, Inc., 59 # -## Temple Place, Suite 330, Boston, MA 02111-1307 USA # -################################################################################ -#""" -#The :mod:`openlyricsexport` module provides the functionality for exporting -#songs from the database. -#""" -# -#import logging -#import os -# -#from lxml import etree -# -#from openlp.core.lib import translate -#from openlp.plugins.songs.lib import OpenLyricsBuilder -#from openlp.plugins.songs.lib.songimport import SongImport -# -#log = logging.getLogger(__name__) -# -#class OpenLyricsExport(SongExport): -# """ -# This provides the Openlyrics export. -# """ -# def __init__(self, master_manager, songs=None, save_path=u''): -# """ -# Initialise the export. -# """ -# log.debug(u'initialise OpenLyricsExport') -# SongExport.__init__(self, master_manager) -# self.master_manager = master_manager -# self.songs = songs -# self.save_path = save_path -# -# def do_export(self): -# """ -# Exports the songs. -# """ -# openLyricsBuilder = OpenLyricsBuilder(self.master_manager) -# self.export_wizard.exportProgressBar.setMaximum(len(songs)) -# for song in self.songs: -# if self.stop_export_flag: -# return False -# self.import_wizard.incrementProgressBar(unicode(translate( -# 'SongsPlugin.OpenLyricsExport', 'Importing %s...')) % -# song.title) -# xml = openLyricsBuilder.song_to_xml(song, True) -# tree = etree.ElementTree(etree.fromstring(xml)) -# path = os.path.join(self.save_path, song.title) -# tree.write(path) -# return True From f2784fc4dd6874cba00112669018a4d3781c53fb Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Sat, 8 Jan 2011 21:59:46 +0100 Subject: [PATCH 14/17] removed whitespaces --- openlp/core/ui/__init__.py | 4 ++-- openlp/plugins/bibles/forms/bibleimportwizard.py | 2 +- openlp/plugins/songs/forms/authorsdialog.py | 2 +- openlp/plugins/songs/forms/songimportwizard.py | 2 +- openlp/plugins/songs/lib/xml.py | 16 ++++++++-------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/openlp/core/ui/__init__.py b/openlp/core/ui/__init__.py index 26c59ed0f..ca2123c87 100644 --- a/openlp/core/ui/__init__.py +++ b/openlp/core/ui/__init__.py @@ -35,11 +35,11 @@ class HideMode(object): ``Blank`` This mode is used to hide all output, specifically by covering the display with a black screen. - + ``Theme`` This mode is used to hide all output, but covers the display with the current theme background, as opposed to black. - + ``Desktop`` This mode hides all output by minimising the display, leaving the user's desktop showing. diff --git a/openlp/plugins/bibles/forms/bibleimportwizard.py b/openlp/plugins/bibles/forms/bibleimportwizard.py index a85e430a1..3200bf974 100644 --- a/openlp/plugins/bibles/forms/bibleimportwizard.py +++ b/openlp/plugins/bibles/forms/bibleimportwizard.py @@ -234,7 +234,7 @@ class Ui_BibleImportWizard(object): QtGui.QSizePolicy.Minimum) self.openlp1Layout.setItem(1, QtGui.QFormLayout.LabelRole, self.openlp1Spacer) - self.selectStack.addWidget(self.openlp1Widget) + self.selectStack.addWidget(self.openlp1Widget) self.selectPageLayout.addLayout(self.selectStack) bibleImportWizard.addPage(self.selectPage) # License Page diff --git a/openlp/plugins/songs/forms/authorsdialog.py b/openlp/plugins/songs/forms/authorsdialog.py index 28083ae05..6f1c7f2a4 100644 --- a/openlp/plugins/songs/forms/authorsdialog.py +++ b/openlp/plugins/songs/forms/authorsdialog.py @@ -54,7 +54,7 @@ class Ui_AuthorsDialog(object): self.displayEdit.setObjectName(u'displayEdit') self.displayLabel.setBuddy(self.displayEdit) self.authorLayout.addRow(self.displayLabel, self.displayEdit) - self.dialogLayout.addLayout(self.authorLayout) + self.dialogLayout.addLayout(self.authorLayout) self.buttonBox = QtGui.QDialogButtonBox(authorsDialog) self.buttonBox.setStandardButtons( QtGui.QDialogButtonBox.Save | QtGui.QDialogButtonBox.Cancel) diff --git a/openlp/plugins/songs/forms/songimportwizard.py b/openlp/plugins/songs/forms/songimportwizard.py index 6eccff9b4..903300d0c 100644 --- a/openlp/plugins/songs/forms/songimportwizard.py +++ b/openlp/plugins/songs/forms/songimportwizard.py @@ -39,7 +39,7 @@ class Ui_SongImportWizard(object): QtGui.QWizard.IndependentPages | QtGui.QWizard.NoBackButtonOnStartPage | QtGui.QWizard.NoBackButtonOnLastPage) - # Welcome Page + # Welcome Page self.welcomePage = QtGui.QWizardPage() self.welcomePage.setPixmap(QtGui.QWizard.WatermarkPixmap, QtGui.QPixmap(u':/wizards/wizard_importsong.bmp')) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index a5c8ecd06..10d93b100 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -359,10 +359,10 @@ class OpenLyricsParser(object): OpenLP does not support the attribute *type* and *lang*. ** - This property is not supported. + This property is not supported. ** - The ** property is fully supported. But comments in lyrics + The ** property is fully supported. But comments in lyrics are not supported. ** @@ -372,23 +372,23 @@ class OpenLyricsParser(object): This property is not supported. ** - This property is not supported. + This property is not supported. ** - This property is not supported. + This property is not supported. ** The attribute *part* is not supported. ** - This property is not supported. + This property is not supported. ** As OpenLP does only support one songbook, we cannot consider more than one songbook. ** - This property is not supported. + This property is not supported. ** Topics, as they are called in OpenLP, are fully supported, whereby only @@ -399,9 +399,9 @@ class OpenLyricsParser(object): This property is not supported. ** - This property is not supported. + This property is not supported. - ** + ** The attribute *translit* and *lang* are not supported. ** From ce1241aa0f9528cd06d0e1045e1f31f3e5b4b560 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Sun, 9 Jan 2011 17:52:31 +0100 Subject: [PATCH 15/17] merged classes together --- openlp/plugins/songs/forms/editsongform.py | 8 +- openlp/plugins/songs/lib/__init__.py | 3 +- openlp/plugins/songs/lib/mediaitem.py | 14 +- openlp/plugins/songs/lib/openlyricsimport.py | 6 +- openlp/plugins/songs/lib/songimport.py | 4 +- openlp/plugins/songs/lib/xml.py | 314 ++++++------------- openlp/plugins/songs/songsplugin.py | 4 +- 7 files changed, 109 insertions(+), 244 deletions(-) diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index 202cc43fe..86249f024 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -31,7 +31,7 @@ from PyQt4 import QtCore, QtGui from openlp.core.lib import Receiver, translate from openlp.plugins.songs.forms import EditVerseForm -from openlp.plugins.songs.lib import SongXMLBuilder, SongXMLParser, VerseType +from openlp.plugins.songs.lib import SongXML, VerseType from openlp.plugins.songs.lib.db import Book, Song, Author, Topic from editsongdialog import Ui_EditSongDialog @@ -263,8 +263,8 @@ class EditSongForm(QtGui.QDialog, Ui_EditSongDialog): if isinstance(self.song.lyrics, buffer): self.song.lyrics = unicode(self.song.lyrics) if self.song.lyrics.startswith(u' - + @@ -71,32 +71,26 @@ from openlp.plugins.songs.lib.db import Author, Book, Song, Topic log = logging.getLogger(__name__) -class SongXMLBuilder(object): +class SongXML(object): """ - This class builds the XML used to describe songs. + This class builds and parses the XML used to describe songs. """ - log.info(u'SongXMLBuilder Loaded') + log.info(u'SongXML Loaded') - def __init__(self, song_language=None): + def __init__(self): """ - Set up the song builder. - - ``song_language`` - The language used in this song + Set up the default variables. """ - lang = u'en' - if song_language: - lang = song_language self.song_xml = objectify.fromstring(u'') - self.lyrics = etree.SubElement(self.song_xml, u'lyrics', language=lang) + self.lyrics = etree.SubElement(self.song_xml, u'lyrics') def add_verse_to_lyrics(self, type, number, content): """ - Add a verse to the ```` tag. + Add a verse to the ** tag. ``type`` - A string denoting the type of verse. Possible values are "Chorus", - "Verse", "Bridge", and "Custom". + A string denoting the type of verse. Possible values are "V", + "C", "B", "P", "I", "E" and "O". ``number`` An integer denoting the number of the item, for example: verse 1. @@ -109,13 +103,6 @@ class SongXMLBuilder(object): verse.text = etree.CDATA(content) self.lyrics.append(verse) - def dump_xml(self): - """ - Debugging aid to dump XML so that we can see what we have. - """ - return etree.tostring(self.song_xml, encoding=u'UTF-8', - xml_declaration=True, pretty_print=True) - def extract_xml(self): """ Extract our newly created XML song. @@ -123,16 +110,10 @@ class SongXMLBuilder(object): return etree.tostring(self.song_xml, encoding=u'UTF-8', xml_declaration=True) - -class SongXMLParser(object): - """ - A class to read in and parse a song's XML. - """ - log.info(u'SongXMLParser Loaded') - - def __init__(self, xml): + def get_verses(self, xml): """ - Set up our song XML parser. + Iterates through the verses in the XML and returns a list of verses + and their attributes. ``xml`` The XML of the song to be parsed. @@ -144,12 +125,6 @@ class SongXMLParser(object): self.song_xml = objectify.fromstring(xml) except etree.XMLSyntaxError: log.exception(u'Invalid xml %s', xml) - - def get_verses(self): - """ - Iterates through the verses in the XML and returns a list of verses - and their attributes. - """ xml_iter = self.song_xml.getiterator() verse_list = [] for element in xml_iter: @@ -166,191 +141,9 @@ class SongXMLParser(object): return etree.dump(self.song_xml) -#class LyricsXML(object): -# """ -# This class represents the XML in the ``lyrics`` field of a song. -# """ -# def __init__(self, song=None): -# if song: -# if song.lyrics.startswith(u'' % \ -# (verse[u'type'], verse[u'label'], verse[u'text']) -# lyrics_output = lyrics_output + \ -# u'%s' % \ -# (language[u'language'], verse_output) -# song_output = u'' + \ -# u'%s' % lyrics_output -# return song_output - - -class OpenLyricsBuilder(object): +class OpenLyrics(object): """ - This class represents the converter for song to OpenLyrics XML. - """ - def __init__(self, manager): - self.manager = manager - - def song_to_xml(self, song, pretty_print=False): - """ - Convert the song to OpenLyrics Format. - """ - song_xml_parser = SongXMLParser(song.lyrics) - verse_list = song_xml_parser.get_verses() - song_xml = objectify.fromstring( - u'') - properties = etree.SubElement(song_xml, u'properties') - titles = etree.SubElement(properties, u'titles') - self._add_text_to_element(u'title', titles, song.title) - if song.alternate_title: - self._add_text_to_element(u'title', titles, song.alternate_title) - if song.comments: - comments = etree.SubElement(properties, u'comments') - self._add_text_to_element(u'comment', comments, song.comments) - if song.copyright: - self._add_text_to_element(u'copyright', properties, song.copyright) - if song.verse_order: - self._add_text_to_element( - u'verseOrder', properties, song.verse_order) - if song.ccli_number: - self._add_text_to_element(u'ccliNo', properties, song.ccli_number) - if song.authors: - authors = etree.SubElement(properties, u'authors') - for author in song.authors: - self._add_text_to_element( - u'author', authors, author.display_name) - book = self.manager.get_object_filtered( - Book, Book.id == song.song_book_id) - if book is not None: - book = book.name - songbooks = etree.SubElement(properties, u'songbooks') - element = self._add_text_to_element( - u'songbook', songbooks, None, book) - element.set(u'entry', song.song_number) - if song.topics: - themes = etree.SubElement(properties, u'themes') - for topic in song.topics: - self._add_text_to_element(u'theme', themes, topic.name) - lyrics = etree.SubElement(song_xml, u'lyrics') - for verse in verse_list: - verse_tag = u'%s%s' % ( - verse[0][u'type'][0].lower(), verse[0][u'label']) - element = \ - self._add_text_to_element(u'verse', lyrics, None, verse_tag) - element = self._add_text_to_element(u'lines', element) - for line in unicode(verse[1]).split(u'\n'): - self._add_text_to_element(u'line', element, line) - return self._extract_xml(song_xml, pretty_print) - - def _add_text_to_element(self, tag, parent, text=None, label=None): - if label: - element = etree.Element(tag, name=unicode(label)) - else: - element = etree.Element(tag) - if text: - element.text = unicode(text) - parent.append(element) - return element - - def _extract_xml(self, xml, pretty_print): - """ - Extract our newly created XML song. - """ - return etree.tostring(xml, encoding=u'UTF-8', - xml_declaration=True, pretty_print=pretty_print) - - def _dump_xml(self, xml): - """ - Debugging aid to dump XML so that we can see what we have. - """ - return etree.tostring(xml, encoding=u'UTF-8', - xml_declaration=True, pretty_print=True) - - -class OpenLyricsParser(object): - """ - This class represents the converter for OpenLyrics XML to a song. + This class represents the converter for OpenLyrics XML to/from a song. As OpenLyrics has a rich set of different features, we cannot support them all. The following features are supported by the :class:`OpenLyricsParser`:: @@ -410,6 +203,57 @@ class OpenLyricsParser(object): def __init__(self, manager): self.manager = manager + def song_to_xml(self, song, pretty_print=False): + """ + Convert the song to OpenLyrics Format. + """ + sxml = SongXML() + verse_list = sxml.get_verses(song.lyrics) + song_xml = objectify.fromstring( + u'') + properties = etree.SubElement(song_xml, u'properties') + titles = etree.SubElement(properties, u'titles') + self._add_text_to_element(u'title', titles, song.title) + if song.alternate_title: + self._add_text_to_element(u'title', titles, song.alternate_title) + if song.comments: + comments = etree.SubElement(properties, u'comments') + self._add_text_to_element(u'comment', comments, song.comments) + if song.copyright: + self._add_text_to_element(u'copyright', properties, song.copyright) + if song.verse_order: + self._add_text_to_element( + u'verseOrder', properties, song.verse_order) + if song.ccli_number: + self._add_text_to_element(u'ccliNo', properties, song.ccli_number) + if song.authors: + authors = etree.SubElement(properties, u'authors') + for author in song.authors: + self._add_text_to_element( + u'author', authors, author.display_name) + book = self.manager.get_object_filtered( + Book, Book.id == song.song_book_id) + if book is not None: + book = book.name + songbooks = etree.SubElement(properties, u'songbooks') + element = self._add_text_to_element( + u'songbook', songbooks, None, book) + element.set(u'entry', song.song_number) + if song.topics: + themes = etree.SubElement(properties, u'themes') + for topic in song.topics: + self._add_text_to_element(u'theme', themes, topic.name) + lyrics = etree.SubElement(song_xml, u'lyrics') + for verse in verse_list: + verse_tag = u'%s%s' % ( + verse[0][u'type'][0].lower(), verse[0][u'label']) + element = \ + self._add_text_to_element(u'verse', lyrics, None, verse_tag) + element = self._add_text_to_element(u'lines', element) + for line in unicode(verse[1]).split(u'\n'): + self._add_text_to_element(u'line', element, line) + return self._extract_xml(song_xml, pretty_print) + def xml_to_song(self, xml): """ Create and save a song from OpenLyrics format xml to the database. Since @@ -441,6 +285,23 @@ class OpenLyricsParser(object): self.manager.save_object(song) return song.id + def _add_text_to_element(self, tag, parent, text=None, label=None): + if label: + element = etree.Element(tag, name=unicode(label)) + else: + element = etree.Element(tag) + if text: + element.text = unicode(text) + parent.append(element) + return element + + def _extract_xml(self, xml, pretty_print): + """ + Extract our newly created XML song. + """ + return etree.tostring(xml, encoding=u'UTF-8', + xml_declaration=True, pretty_print=pretty_print) + def _get(self, element, attribute): """ This returns the element's attribute as unicode string. @@ -562,7 +423,7 @@ class OpenLyricsParser(object): ``song`` The song object. """ - sxml = SongXMLBuilder() + sxml = SongXML() search_text = u'' temp_verse_order = [] for verse in lyrics.verse: @@ -682,3 +543,10 @@ class OpenLyricsParser(object): song.topics.append(topic) except AttributeError: pass + + def _dump_xml(self, xml): + """ + Debugging aid to dump XML so that we can see what we have. + """ + return etree.tostring(xml, encoding=u'UTF-8', + xml_declaration=True, pretty_print=True) diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 545497acb..17e609fd4 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -31,7 +31,7 @@ from PyQt4 import QtCore, QtGui from openlp.core.lib import Plugin, StringContent, build_icon, translate from openlp.core.lib.db import Manager -from openlp.plugins.songs.lib import SongMediaItem, SongsTab, SongXMLParser +from openlp.plugins.songs.lib import SongMediaItem, SongsTab, SongXML from openlp.plugins.songs.lib.db import init_schema, Song from openlp.plugins.songs.lib.importer import SongFormat @@ -153,7 +153,7 @@ class SongsPlugin(Plugin): song.search_title = self.whitespace.sub(u' ', song.title.lower() + \ u' ' + song.alternate_title.lower()) lyrics = u'' - verses = SongXMLParser(song.lyrics).get_verses() + verses = SongXML().get_verses(song.lyrics) for verse in verses: lyrics = lyrics + self.whitespace.sub(u' ', verse[1]) + u' ' song.search_lyrics = lyrics.lower() From 62794614f91682fd87b2db82963433ce2f09d425 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Sun, 9 Jan 2011 18:07:17 +0100 Subject: [PATCH 16/17] clean ups --- openlp/core/ui/__init__.py | 4 ++-- openlp/plugins/songs/lib/xml.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/core/ui/__init__.py b/openlp/core/ui/__init__.py index ca2123c87..c509fe023 100644 --- a/openlp/core/ui/__init__.py +++ b/openlp/core/ui/__init__.py @@ -35,11 +35,11 @@ class HideMode(object): ``Blank`` This mode is used to hide all output, specifically by covering the display with a black screen. - + ``Theme`` This mode is used to hide all output, but covers the display with the current theme background, as opposed to black. - + ``Desktop`` This mode hides all output by minimising the display, leaving the user's desktop showing. diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 463e30d87..753383af3 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -269,7 +269,7 @@ class OpenLyrics(object): song = Song() if xml[:5] == u'').sub(u'', xml) song_xml = objectify.fromstring(xml) properties = song_xml.properties From b52d0aafd5da669a9ee5436663e18e0558a0bbd5 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Sun, 9 Jan 2011 20:18:08 +0100 Subject: [PATCH 17/17] re.compile code from reindex tool --- openlp/plugins/songs/lib/mediaitem.py | 7 +++---- openlp/plugins/songs/lib/xml.py | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openlp/plugins/songs/lib/mediaitem.py b/openlp/plugins/songs/lib/mediaitem.py index 3f3755d14..73c8eaa10 100644 --- a/openlp/plugins/songs/lib/mediaitem.py +++ b/openlp/plugins/songs/lib/mediaitem.py @@ -351,8 +351,7 @@ class SongMediaItem(MediaManagerItem): service_item.theme = song.theme_name service_item.edit_id = item_id if song.lyrics.startswith(u'') properties = etree.SubElement(song_xml, u'properties') titles = etree.SubElement(properties, u'titles') - self._add_text_to_element(u'title', titles, song.title) + self._add_text_to_element(u'title', titles, song.title.strip()) if song.alternate_title: - self._add_text_to_element(u'title', titles, song.alternate_title) + self._add_text_to_element( + u'title', titles, song.alternate_title.strip()) if song.comments: comments = etree.SubElement(properties, u'comments') self._add_text_to_element(u'comment', comments, song.comments)