From cbeb289875643be3d130013f0bd2a8519897aa1b Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Wed, 19 Jan 2011 20:22:43 +0100 Subject: [PATCH 1/5] add verse language to db (like OpenLyrics) --- openlp/plugins/songs/lib/openlyricsimport.py | 7 +++++-- openlp/plugins/songs/lib/xml.py | 22 ++++++++++++++++---- openlp/plugins/songs/songsplugin.py | 12 +++++++---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/openlp/plugins/songs/lib/openlyricsimport.py b/openlp/plugins/songs/lib/openlyricsimport.py index 220160b1d..60d032b7e 100644 --- a/openlp/plugins/songs/lib/openlyricsimport.py +++ b/openlp/plugins/songs/lib/openlyricsimport.py @@ -61,14 +61,17 @@ class OpenLyricsImport(SongImport): Imports the songs. """ self.import_wizard.progressBar.setMaximum(len(self.import_source)) + parser = etree.XMLParser(remove_blank_text=True) for file_path in self.import_source: if self.stop_import_flag: return False self.import_wizard.incrementProgressBar(unicode(translate( 'SongsPlugin.OpenLyricsImport', 'Importing %s...')) % os.path.basename(file_path)) - parser = etree.XMLParser(remove_blank_text=True) - parsed_file = etree.parse(file_path, parser) + try: + parsed_file = etree.parse(file_path, parser) + except etree.XMLSyntaxError: + return False xml = unicode(etree.tostring(parsed_file)) if self.openLyrics.xml_to_song(xml) is None: log.debug(u'File could not be imported: %s' % file_path) diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 88459b797..2d8babf65 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -31,7 +31,7 @@ The basic XML for storing the lyrics in the song database is of the format:: - + @@ -84,7 +84,7 @@ class SongXML(object): self.song_xml = objectify.fromstring(u'') self.lyrics = etree.SubElement(self.song_xml, u'lyrics') - def add_verse_to_lyrics(self, type, number, content): + def add_verse_to_lyrics(self, type, number, content, lang=None): """ Add a verse to the ** tag. @@ -97,9 +97,15 @@ class SongXML(object): ``content`` The actual text of the verse to be stored. + + ``lang`` + The verse's language code. This is not required, but should be added + if available. """ verse = etree.Element(u'verse', type=unicode(type), label=unicode(number)) + if lang: + verse.set(u'lang', lang) verse.text = etree.CDATA(content) self.lyrics.append(verse) @@ -117,6 +123,11 @@ class SongXML(object): ``xml`` The XML of the song to be parsed. + + The returned list has the following format:: + + [[{'lang': 'en', 'type': 'V', 'label': '1'}, u"The English verse."], + [{'lang': 'en', 'type': 'C', 'label': '1'}, u"The English chorus."]] """ self.song_xml = None if xml[:5] == u'* - The attribute *translit* and *lang* are not supported. + The attribute *translit* is not supported. ** OpenLP supports this property. @@ -442,7 +453,10 @@ class OpenLyrics(object): if not verse_number: verse_number = u'1' temp_verse_order.append((verse_type, verse_number, verse_part)) - sxml.add_verse_to_lyrics(verse_type, verse_number, text) + lang = None + if self._get(verse, u'lang'): + lang = self._get(verse, u'lang') + sxml.add_verse_to_lyrics(verse_type, verse_number, text, lang) search_text = search_text + text song.search_lyrics = search_text.lower() song.lyrics = unicode(sxml.extract_xml(), u'utf-8') diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 17e609fd4..66f457d24 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -134,7 +134,7 @@ class SongsPlugin(Plugin): def onToolsReindexItemTriggered(self): """ - Rebuild the search title of each song. + Rebuild each song. """ maxSongs = self.manager.get_object_count(Song) progressDialog = QtGui.QProgressDialog( @@ -150,8 +150,13 @@ class SongsPlugin(Plugin): song.title = u'' if song.alternate_title is None: song.alternate_title = u'' - song.search_title = self.whitespace.sub(u' ', song.title.lower() + \ + song.search_title = self.whitespace.sub(u' ', song.title.lower() + u' ' + song.alternate_title.lower()) + # Remove the "language" attribute from lyrics tag. This is not very + # important, but this keeps the database clean. This can be removed + # when everybody has run the reindex tool once. + song.lyrics = song.lyrics.replace( + u'', u'') lyrics = u'' verses = SongXML().get_verses(song.lyrics) for verse in verses: @@ -159,8 +164,7 @@ class SongsPlugin(Plugin): song.search_lyrics = lyrics.lower() progressDialog.setValue(counter) self.manager.save_objects(songs) - counter += 1 - progressDialog.setValue(counter) + progressDialog.setValue(counter + 1) self.mediaItem.displayResultsSong( self.manager.get_all_objects(Song, order_by_ref=Song.search_title)) From da1070468b7638472b140c84d86b3b5f2820ec52 Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Wed, 19 Jan 2011 20:37:44 +0100 Subject: [PATCH 2/5] removed white spaces --- openlp/plugins/songs/forms/songimportform.py | 2 +- openlp/plugins/songs/lib/easislidesimport.py | 2 +- openlp/plugins/songs/songsplugin.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/forms/songimportform.py b/openlp/plugins/songs/forms/songimportform.py index d14ab9c36..e3d7b635e 100644 --- a/openlp/plugins/songs/forms/songimportform.py +++ b/openlp/plugins/songs/forms/songimportform.py @@ -654,7 +654,7 @@ class SongImportForm(OpenLPWizard): 'Select EasiSlides songfile'), self.easiSlidesFilenameEdit ) - + def onEWBrowseButtonClicked(self): """ Get EasyWorship song database files diff --git a/openlp/plugins/songs/lib/easislidesimport.py b/openlp/plugins/songs/lib/easislidesimport.py index 8685b6934..85384efd7 100644 --- a/openlp/plugins/songs/lib/easislidesimport.py +++ b/openlp/plugins/songs/lib/easislidesimport.py @@ -313,7 +313,7 @@ class EasiSlidesImport(SongImport): tag = SeqTypes[tag.lower()] else: continue - + if tag in versetags: self.verse_order_list.append(tag) else: diff --git a/openlp/plugins/songs/songsplugin.py b/openlp/plugins/songs/songsplugin.py index 66f457d24..7efe73db2 100644 --- a/openlp/plugins/songs/songsplugin.py +++ b/openlp/plugins/songs/songsplugin.py @@ -153,7 +153,7 @@ class SongsPlugin(Plugin): song.search_title = self.whitespace.sub(u' ', song.title.lower() + u' ' + song.alternate_title.lower()) # Remove the "language" attribute from lyrics tag. This is not very - # important, but this keeps the database clean. This can be removed + # important, but this keeps the database clean. This can be removed # when everybody has run the reindex tool once. song.lyrics = song.lyrics.replace( u'', u'') From 6f344bd85aefbb5255f8460daafe73b5271bedec Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Wed, 19 Jan 2011 20:57:08 +0100 Subject: [PATCH 3/5] added possibility to add language to songimporter --- openlp/plugins/songs/lib/songimport.py | 32 ++++++++++++++++---------- openlp/plugins/songs/lib/xml.py | 4 ++-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 3ce271466..67aa8094f 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -197,16 +197,24 @@ class SongImport(QtCore.QObject): return self.media_files.append(filename) - def add_verse(self, verse, versetag=u'V'): + def add_verse(self, versetext, versetag=u'V', lang=None): """ - Add a verse. This is the whole verse, lines split by \n - Verse tag can be V1/C1/B etc, or 'V' and 'C' (will count the verses/ - choruses itself) or None, where it will assume verse - It will also attempt to detect duplicates. In this case it will just - add to the verse order + Add a verse. This is the whole verse, lines split by \n. It will also + attempt to detect duplicates. In this case it will just add to the verse + order. + + ``versetext`` + The text of the verse. + + ``versetag`` + The verse tag can be V1/C1/B etc, or 'V' and 'C' (will count the verses/ + choruses itself) or None, where it will assume verse. + + ``lang`` + The language code (ISO-639) of the verse, for example *en* or *de*. """ - for (oldversetag, oldverse) in self.verses: - if oldverse.strip() == verse.strip(): + for (oldversetag, oldverse, oldlang) in self.verses: + if oldverse.strip() == versetext.strip(): self.verse_order_list.append(oldversetag) return if versetag[0] in self.versecounts: @@ -217,7 +225,7 @@ class SongImport(QtCore.QObject): versetag += unicode(self.versecounts[versetag[0]]) elif int(versetag[1:]) > self.versecounts[versetag[0]]: self.versecounts[versetag[0]] = int(versetag[1:]) - self.verses.append([versetag, verse.rstrip()]) + self.verses.append([versetag, versetext.rstrip(), lang]) self.verse_order_list.append(versetag) if versetag.startswith(u'V') and self.contains_verse(u'C1'): self.verse_order_list.append(u'C1') @@ -266,7 +274,7 @@ class SongImport(QtCore.QObject): verses_changed_to_other = {} sxml = SongXML() other_count = 1 - for (versetag, versetext) in self.verses: + for (versetag, versetext, lang) in self.verses: if versetag[0] == u'C': versetype = VerseType.to_string(VerseType.Chorus) elif versetag[0] == u'V': @@ -286,7 +294,7 @@ class SongImport(QtCore.QObject): versetype = VerseType.to_string(VerseType.Other) log.info(u'Versetype %s changing to %s' , versetag, newversetag) versetag = newversetag - sxml.add_verse_to_lyrics(versetype, versetag[1:], versetext) + sxml.add_verse_to_lyrics(versetype, versetag[1:], versetext, lang) song.search_lyrics += u' ' + self.remove_punctuation(versetext) song.search_lyrics = song.search_lyrics.lower() song.lyrics = unicode(sxml.extract_xml(), u'utf-8') @@ -338,7 +346,7 @@ class SongImport(QtCore.QObject): + u'========================================' print u'TITLE: ' + self.title print u'ALT TITLE: ' + self.alternate_title - for (versetag, versetext) in self.verses: + for (versetag, versetext, lang) in self.verses: print u'VERSE ' + versetag + u': ' + versetext print u'ORDER: ' + u' '.join(self.verse_order_list) for author in self.authors: diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 2d8babf65..7fc102e81 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -99,8 +99,8 @@ class SongXML(object): The actual text of the verse to be stored. ``lang`` - The verse's language code. This is not required, but should be added - if available. + The verse's language code (ISO-639). This is not required, but + should be added if available. """ verse = etree.Element(u'verse', type=unicode(type), label=unicode(number)) From 57abf340576595d55a4124bf2ea5d1c28dbd629c Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Wed, 19 Jan 2011 21:08:07 +0100 Subject: [PATCH 4/5] fixed long line --- openlp/plugins/songs/lib/songimport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 67aa8094f..cab0aacf6 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -207,8 +207,8 @@ class SongImport(QtCore.QObject): The text of the verse. ``versetag`` - The verse tag can be V1/C1/B etc, or 'V' and 'C' (will count the verses/ - choruses itself) or None, where it will assume verse. + The verse tag can be V1/C1/B etc, or 'V' and 'C' (will count the + verses/choruses itself) or None, where it will assume verse. ``lang`` The language code (ISO-639) of the verse, for example *en* or *de*. From 17d3ea7573be7287589f78a8b66e1d624d510fdf Mon Sep 17 00:00:00 2001 From: Andreas Preikschat Date: Thu, 20 Jan 2011 06:42:27 +0100 Subject: [PATCH 5/5] fixes for merge, catch another exception --- openlp/plugins/songs/lib/easislidesimport.py | 1 - openlp/plugins/songs/lib/openlyricsimport.py | 10 ++++------ openlp/plugins/songs/lib/xml.py | 7 +++++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/openlp/plugins/songs/lib/easislidesimport.py b/openlp/plugins/songs/lib/easislidesimport.py index 85384efd7..316b78d8c 100644 --- a/openlp/plugins/songs/lib/easislidesimport.py +++ b/openlp/plugins/songs/lib/easislidesimport.py @@ -313,7 +313,6 @@ class EasiSlidesImport(SongImport): tag = SeqTypes[tag.lower()] else: continue - if tag in versetags: self.verse_order_list.append(tag) else: diff --git a/openlp/plugins/songs/lib/openlyricsimport.py b/openlp/plugins/songs/lib/openlyricsimport.py index 60d032b7e..37407c75c 100644 --- a/openlp/plugins/songs/lib/openlyricsimport.py +++ b/openlp/plugins/songs/lib/openlyricsimport.py @@ -70,11 +70,9 @@ class OpenLyricsImport(SongImport): os.path.basename(file_path)) try: parsed_file = etree.parse(file_path, parser) + xml = unicode(etree.tostring(parsed_file)) + if self.openLyrics.xml_to_song(xml) is None: + log.debug(u'File could not be imported: %s' % file_path) except etree.XMLSyntaxError: - return False - xml = unicode(etree.tostring(parsed_file)) - if self.openLyrics.xml_to_song(xml) is None: - log.debug(u'File could not be imported: %s' % file_path) - # Importing this song failed! For now we stop import. - return False + log.exception(u'XML syntax error in file %s' % file_path) return True diff --git a/openlp/plugins/songs/lib/xml.py b/openlp/plugins/songs/lib/xml.py index 7fc102e81..91bfdb025 100644 --- a/openlp/plugins/songs/lib/xml.py +++ b/openlp/plugins/songs/lib/xml.py @@ -279,13 +279,16 @@ class OpenLyrics(object): # No xml get out of here. if not xml: return None - song = Song() if xml[:5] == u'').sub(u'', xml) song_xml = objectify.fromstring(xml) - properties = song_xml.properties + try: + properties = song_xml.properties + except AttributeError: + return None + song = Song() self._process_copyright(properties, song) self._process_cclinumber(properties, song) self._process_titles(properties, song)