From dfe8bbae9c9b018c30479659c2f294fe9a4c627f Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Thu, 26 Aug 2010 22:52:54 +0200 Subject: [PATCH 1/4] Some minor syntactic sugar. --- openlp/plugins/songs/lib/olpimport.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openlp/plugins/songs/lib/olpimport.py b/openlp/plugins/songs/lib/olpimport.py index a4c15718e..63f99dc7b 100644 --- a/openlp/plugins/songs/lib/olpimport.py +++ b/openlp/plugins/songs/lib/olpimport.py @@ -75,18 +75,18 @@ class OpenLPSongImport(SongImport): The :class:`OpenLPSongImport` class provides OpenLP with the ability to import song databases from other installations of OpenLP. """ - def __init__(self, master_manager, **kwargs): + def __init__(self, manager, **kwargs): """ Initialise the import. - ``master_manager`` + ``manager`` The song manager for the running OpenLP installation. ``source_db`` The database providing the data to import. """ - SongImport.__init__(self, master_manager) - self.master_manager = master_manager + SongImport.__init__(self, manager) + #self.master_manager = master_manager self.import_source = u'sqlite:///%s' % kwargs[u'filename'] log.debug(self.import_source) self.source_session = None @@ -167,7 +167,7 @@ class OpenLPSongImport(SongImport): new_song.ccli_number = song.ccli_number if song.authors: for author in song.authors: - existing_author = self.master_manager.get_object_filtered( + existing_author = self.manager.get_object_filtered( Author, Author.display_name == author.display_name) if existing_author: new_song.authors.append(existing_author) @@ -177,7 +177,7 @@ class OpenLPSongImport(SongImport): last_name=author.last_name, display_name=author.display_name)) else: - au = self.master_manager.get_object_filtered(Author, + au = self.manager.get_object_filtered(Author, Author.display_name == u'Author Unknown') if au: new_song.authors.append(au) @@ -185,7 +185,7 @@ class OpenLPSongImport(SongImport): new_song.authors.append(Author.populate( display_name=u'Author Unknown')) if song.book: - existing_song_book = self.master_manager.get_object_filtered( + existing_song_book = self.manager.get_object_filtered( Book, Book.name == song.book.name) if existing_song_book: new_song.book = existing_song_book @@ -194,7 +194,7 @@ class OpenLPSongImport(SongImport): publisher=song.book.publisher) if song.topics: for topic in song.topics: - existing_topic = self.master_manager.get_object_filtered( + existing_topic = self.manager.get_object_filtered( Topic, Topic.name == topic.name) if existing_topic: new_song.topics.append(existing_topic) @@ -204,12 +204,12 @@ class OpenLPSongImport(SongImport): # if song.media_files: # for media_file in song.media_files: # existing_media_file = \ -# self.master_manager.get_object_filtered(MediaFile, +# self.manager.get_object_filtered(MediaFile, # MediaFile.file_name == media_file.file_name) # if existing_media_file: # new_song.media_files.append(existing_media_file) # else: # new_song.media_files.append(MediaFile.populate( # file_name=media_file.file_name)) - self.master_manager.save_object(new_song) + self.manager.save_object(new_song) engine.dispose() From 3d60cc894de45caa7c037f4bfc0fc19cd2fea0be Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Thu, 26 Aug 2010 22:53:24 +0200 Subject: [PATCH 2/4] Added the initial openlp.org 1.x importer. --- openlp/plugins/songs/lib/olp1import.py | 131 +++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 openlp/plugins/songs/lib/olp1import.py diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py new file mode 100644 index 000000000..1625bfff2 --- /dev/null +++ b/openlp/plugins/songs/lib/olp1import.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian # +# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, # +# Carsten Tinggaard, Frode Woldsund # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`olp1import` module provides the functionality for importing +openlp.org 1.x song databases into the current installation database. +""" +import logging +import sqlite + +#from openlp.core.lib.db import BaseModel +from openlp.plugins.songs.lib.db import Author, Book, Song, Topic #, MediaFile +from songimport import SongImport + +log = logging.getLogger(__name__) + +class OpenLP1SongImport(SongImport): + """ + The :class:`OpenLP1SongImport` class provides OpenLP with the ability to + import song databases from installations of openlp.org 1.x. + """ + def __init__(self, manager, **kwargs): + """ + Initialise the import. + + ``manager`` + The song manager for the running OpenLP installation. + + ``filename`` + The database providing the data to import. + """ + SongImport.__init__(self, manager) + self.manager = manager + self.import_source = kwargs[u'filename'] + + def do_import(self): + """ + Run the import for an openlp.org 1.x song database. + """ + connection = sqlite.connect(self.import_source) + cursor = connection.cursor() + +# for song in source_songs: +# new_song = Song() +# new_song.title = song.title +# if has_media_files: +# new_song.alternate_title = song.alternate_title +# else: +# old_titles = song.search_title.split(u'@') +# if len(old_titles) > 1: +# new_song.alternate_title = old_titles[1] +# else: +# new_song.alternate_title = u'' +# new_song.search_title = song.search_title +# new_song.song_number = song.song_number +# new_song.lyrics = song.lyrics +# new_song.search_lyrics = song.search_lyrics +# new_song.verse_order = song.verse_order +# new_song.copyright = song.copyright +# new_song.comments = song.comments +# new_song.theme_name = song.theme_name +# new_song.ccli_number = song.ccli_number +# if song.authors: +# for author in song.authors: +# existing_author = self.master_manager.get_object_filtered( +# Author, Author.display_name == author.display_name) +# if existing_author: +# new_song.authors.append(existing_author) +# else: +# new_song.authors.append(Author.populate( +# first_name=author.first_name, +# last_name=author.last_name, +# display_name=author.display_name)) +# else: +# au = self.master_manager.get_object_filtered(Author, +# Author.display_name == u'Author Unknown') +# if au: +# new_song.authors.append(au) +# else: +# new_song.authors.append(Author.populate( +# display_name=u'Author Unknown')) +# if song.book: +# existing_song_book = self.master_manager.get_object_filtered( +# Book, Book.name == song.book.name) +# if existing_song_book: +# new_song.book = existing_song_book +# else: +# new_song.book = Book.populate(name=song.book.name, +# publisher=song.book.publisher) +# if song.topics: +# for topic in song.topics: +# existing_topic = self.master_manager.get_object_filtered( +# Topic, Topic.name == topic.name) +# if existing_topic: +# new_song.topics.append(existing_topic) +# else: +# new_song.topics.append(Topic.populate(name=topic.name)) +## if has_media_files: +## if song.media_files: +## for media_file in song.media_files: +## existing_media_file = \ +## self.master_manager.get_object_filtered(MediaFile, +## MediaFile.file_name == media_file.file_name) +## if existing_media_file: +## new_song.media_files.append(existing_media_file) +## else: +## new_song.media_files.append(MediaFile.populate( +## file_name=media_file.file_name)) +# self.master_manager.save_object(new_song) From 59d2e2dab644d33eba211f441c7da94d03979836 Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Sun, 29 Aug 2010 01:09:05 +0200 Subject: [PATCH 3/4] Fixed up the OpenSong importer. --- openlp/plugins/songs/lib/opensongimport.py | 194 ++++++++++++--------- openlp/plugins/songs/lib/songimport.py | 2 +- 2 files changed, 110 insertions(+), 86 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index e1d683000..ccf8479bf 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -36,119 +36,148 @@ log = logging.getLogger(__name__) class OpenSongImportError(Exception): pass -class OpenSongImport(object): +class OpenSongImport(SongImport): """ - Import songs exported from OpenSong - the format is described loosly here: - http://www.opensong.org/d/manual/song_file_format_specification + Import songs exported from OpenSong - However, it doesn't describe the section, so here's an attempt: + The format is described loosly on the `OpenSong File Format Specification + `_ page on + the OpenSong web site. However, it doesn't describe the section, + so here's an attempt: - Verses can be expressed in one of 2 ways: - - [v1]List of words - Another Line + Verses can be expressed in one of 2 ways, either in complete verses, or by + line grouping, i.e. grouping all line 1's of a verse together, all line 2's + of a verse together, and so on. - [v2]Some words for the 2nd verse - etc... - + An example of complete verses:: - The 'v' can be left out - it is implied - or: - - [V] - 1List of words - 2Some words for the 2nd Verse + + [v1] + List of words + Another Line - 1Another Line - 2etc... - + [v2] + Some words for the 2nd verse + etc... + - Either or both forms can be used in one song. The Number does not - necessarily appear at the start of the line + The 'v' in the verse specifiers above can be left out, it is implied. + + An example of line grouping:: + + + [V] + 1List of words + 2Some words for the 2nd Verse + + 1Another Line + 2etc... + + + Either or both forms can be used in one song. The number does not + necessarily appear at the start of the line. Additionally, the [v1] labels + can have either upper or lower case Vs. - The [v1] labels can have either upper or lower case Vs Other labels can be used also: - C - Chorus - B - Bridge - Guitar chords can be provided 'above' the lyrics (the line is - preceeded by a'.') and _s can be used to signify long-drawn-out - words: + C + Chorus - . A7 Bm - 1 Some____ Words + B + Bridge - Chords and _s are removed by this importer. + All verses are imported and tagged appropriately. - The verses etc. are imported and tagged appropriately. + Guitar chords can be provided "above" the lyrics (the line is preceeded by + a period "."), and one or more "_" can be used to signify long-drawn-out + words. Chords and "_" are removed by this importer. For example:: - The tag is used to populate the OpenLP verse - display order field. The Author and Copyright tags are also - imported to the appropriate places. + . A7 Bm + 1 Some____ Words + + The tag is used to populate the OpenLP verse display order + field. The Author and Copyright tags are also imported to the appropriate + places. """ - def __init__(self, songmanager): + def __init__(self, manager, **kwargs): """ - Initialise the class. Requires a songmanager class which - is passed to SongImport for writing song to disk + Initialise the class. """ - self.songmanager = songmanager + SongImport.__init__(self, manager) + self.filenames = kwargs[u'filenames'] self.song = None + self.commit = True - def do_import(self, filename, commit=True): + def do_import(self): """ - Import either a single opensong file, or a zipfile - containing multiple opensong files If the commit parameter is - set False, the import will not be committed to the database - (useful for test scripts) + Import either a single opensong file, or a zipfile containing multiple + opensong files. If `self.commit` is set False, the import will not be + committed to the database (useful for test scripts). """ - ext = os.path.splitext(filename)[1] - if ext.lower() == ".zip": - log.info('Zipfile found %s', filename) - z = ZipFile(filename, u'r') - for song in z.infolist(): - parts = os.path.split(song.filename) - if parts[-1] == u'': - #No final part => directory - continue - songfile = z.open(song) - self.do_import_file(songfile) - if commit: + success = False + self.import_wizard.importProgressBar.setMaximum(len(self.filenames)) + for filename in self.filenames: + if self.stop_import_flag: + break + ext = os.path.splitext(filename)[1] + if ext.lower() == u'.zip': + log.debug(u'Zipfile found %s', filename) + z = ZipFile(filename, u'r') + for song in z.infolist(): + if self.stop_import_flag: + break + parts = os.path.split(song.filename) + if parts[-1] == u'': + #No final part => directory + continue + self.import_wizard.incrementProgressBar(u'Importing %s...' \ + % parts[-1]) + songfile = z.open(song) + self.do_import_file(songfile) + if self.commit: + self.finish() + if self.stop_import_flag: + break + else: + log.info('Direct import %s', filename) + self.import_wizard.incrementProgressBar(u'Importing %s...' \ + % os.path.split(filename)[-1]) + file = open(filename) + self.do_import_file(file) + if self.commit: self.finish() - else: - log.info('Direct import %s', filename) - file = open(filename) - self.do_import_file(file) - if commit: - self.finish() + if not self.commit: + self.finish() + - def do_import_file(self, file): """ Process the OpenSong file - pass in a file-like object, not a filename - """ - self.song_import = SongImport(self.songmanager) + """ tree = objectify.parse(file) root = tree.getroot() fields = dir(root) - decode = {u'copyright':self.song_import.add_copyright, - u'ccli':u'ccli_number', - u'author':self.song_import.parse_author, - u'title':u'title', - u'aka':u'alternate_title', - u'hymn_number':u'song_number'} + decode = { + u'copyright': self.add_copyright, + u'ccli': u'ccli_number', + u'author': self.parse_author, + u'title': u'title', + u'aka': u'alternate_title', + u'hymn_number': u'song_number' + } for (attr, fn_or_string) in decode.items(): if attr in fields: ustring = unicode(root.__getattr__(attr)) if type(fn_or_string) == type(u''): - self.song_import.__setattr__(fn_or_string, ustring) + self.__setattr__(fn_or_string, ustring) else: fn_or_string(ustring) - if u'theme' in fields: - self.song_import.topics.append(unicode(root.theme)) - if u'alttheme' in fields: - self.song_import.topics.append(unicode(root.alttheme)) + if u'theme' in fields and unicode(root.theme) not in self.topics: + self.topics.append(unicode(root.theme)) + if u'alttheme' in fields and unicode(root.alttheme) not in self.topics: + self.topics.append(unicode(root.alttheme)) # data storage while importing verses = {} lyrics = unicode(root.lyrics) @@ -158,6 +187,7 @@ class OpenSongImport(object): # in the absence of any other indication, verses are the default, # erm, versetype! versetype = u'V' + versenum = None for thisline in lyrics.split(u'\n'): # remove comments semicolon = thisline.find(u';') @@ -170,7 +200,6 @@ class OpenSongImport(object): if thisline[0] == u'.' or thisline.startswith(u'---') \ or thisline.startswith(u'-!!'): continue - # verse/chorus/etc. marker if thisline[0] == u'[': versetype = thisline[1].upper() @@ -186,7 +215,6 @@ class OpenSongImport(object): versenum = u'1' continue words = None - # number at start of line.. it's verse number if thisline[0].isdigit(): versenum = thisline[0] @@ -207,7 +235,7 @@ class OpenSongImport(object): our_verse_order.append(versetag) if words: # Tidy text and remove the ____s from extended words - words = self.song_import.tidy_text(words) + words = self.tidy_text(words) words = words.replace('_', '') verses[versetype][versenum].append(words) # done parsing @@ -220,7 +248,7 @@ class OpenSongImport(object): for num in versenums: versetag = u'%s%s' % (versetype, num) lines = u'\n'.join(verses[versetype][num]) - self.song_import.verses.append([versetag, lines]) + self.verses.append([versetag, lines]) # Keep track of what we have for error checking later versetags[versetag] = 1 # now figure out the presentation order @@ -236,8 +264,4 @@ class OpenSongImport(object): if not versetags.has_key(tag): log.warn(u'Got order %s but not in versetags, skipping', tag) else: - self.song_import.verse_order_list.append(tag) - - def finish(self): - """ Separate function, allows test suite to not pollute database""" - self.song_import.finish() + self.verse_order_list.append(tag) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 2ffb0beda..0bfebba47 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -236,7 +236,7 @@ class SongImport(QtCore.QObject): """ All fields have been set to this song. Write it away """ - if len(self.authors) == 0: + if not self.authors: self.authors.append(u'Author unknown') self.commit_song() From 2d9b39986a513e651e144ca50b726f26bf9dd0bc Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Sun, 29 Aug 2010 22:05:36 +0200 Subject: [PATCH 4/4] Some more fixes for the OpenSong import. --- openlp/plugins/songs/lib/opensongimport.py | 21 +++++++++++++++------ openlp/plugins/songs/lib/songimport.py | 3 +-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index ccf8479bf..d51ff5b49 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -28,6 +28,7 @@ import logging import os from zipfile import ZipFile from lxml import objectify +from lxml.etree import Error, LxmlError from openlp.plugins.songs.lib.songimport import SongImport @@ -156,7 +157,12 @@ class OpenSongImport(SongImport): Process the OpenSong file - pass in a file-like object, not a filename """ - tree = objectify.parse(file) + self.authors = [] + try: + tree = objectify.parse(file) + except Error, LxmlError: + log.exception(u'Error parsing XML') + return root = tree.getroot() fields = dir(root) decode = { @@ -167,11 +173,11 @@ class OpenSongImport(SongImport): u'aka': u'alternate_title', u'hymn_number': u'song_number' } - for (attr, fn_or_string) in decode.items(): + for attr, fn_or_string in decode.items(): if attr in fields: ustring = unicode(root.__getattr__(attr)) - if type(fn_or_string) == type(u''): - self.__setattr__(fn_or_string, ustring) + if isinstance(fn_or_string, basestring): + setattr(self, fn_or_string, ustring) else: fn_or_string(ustring) if u'theme' in fields and unicode(root.theme) not in self.topics: @@ -252,12 +258,15 @@ class OpenSongImport(SongImport): # Keep track of what we have for error checking later versetags[versetag] = 1 # now figure out the presentation order + order = [] if u'presentation' in fields and root.presentation != u'': order = unicode(root.presentation) order = order.split() else: - assert len(our_verse_order)>0 - order = our_verse_order + if len(our_verse_order) > 0: + order = our_verse_order + else: + log.warn(u'No verse order available for %s, skipping.', self.title) for tag in order: if len(tag) == 1: tag = tag + u'1' # Assume it's no.1 if it's not there diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 0bfebba47..5889a3774 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -158,8 +158,7 @@ class SongImport(QtCore.QObject): def parse_author(self, text): """ Add the author. OpenLP stores them individually so split by 'and', '&' - and comma. - However need to check for 'Mr and Mrs Smith' and turn it to + and comma. However need to check for 'Mr and Mrs Smith' and turn it to 'Mr Smith' and 'Mrs Smith'. """ for author in text.split(u','):