diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py new file mode 100644 index 000000000..1625bfff2 --- /dev/null +++ b/openlp/plugins/songs/lib/olp1import.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian # +# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, # +# Carsten Tinggaard, Frode Woldsund # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`olp1import` module provides the functionality for importing +openlp.org 1.x song databases into the current installation database. +""" +import logging +import sqlite + +#from openlp.core.lib.db import BaseModel +from openlp.plugins.songs.lib.db import Author, Book, Song, Topic #, MediaFile +from songimport import SongImport + +log = logging.getLogger(__name__) + +class OpenLP1SongImport(SongImport): + """ + The :class:`OpenLP1SongImport` class provides OpenLP with the ability to + import song databases from installations of openlp.org 1.x. + """ + def __init__(self, manager, **kwargs): + """ + Initialise the import. + + ``manager`` + The song manager for the running OpenLP installation. + + ``filename`` + The database providing the data to import. + """ + SongImport.__init__(self, manager) + self.manager = manager + self.import_source = kwargs[u'filename'] + + def do_import(self): + """ + Run the import for an openlp.org 1.x song database. + """ + connection = sqlite.connect(self.import_source) + cursor = connection.cursor() + +# for song in source_songs: +# new_song = Song() +# new_song.title = song.title +# if has_media_files: +# new_song.alternate_title = song.alternate_title +# else: +# old_titles = song.search_title.split(u'@') +# if len(old_titles) > 1: +# new_song.alternate_title = old_titles[1] +# else: +# new_song.alternate_title = u'' +# new_song.search_title = song.search_title +# new_song.song_number = song.song_number +# new_song.lyrics = song.lyrics +# new_song.search_lyrics = song.search_lyrics +# new_song.verse_order = song.verse_order +# new_song.copyright = song.copyright +# new_song.comments = song.comments +# new_song.theme_name = song.theme_name +# new_song.ccli_number = song.ccli_number +# if song.authors: +# for author in song.authors: +# existing_author = self.master_manager.get_object_filtered( +# Author, Author.display_name == author.display_name) +# if existing_author: +# new_song.authors.append(existing_author) +# else: +# new_song.authors.append(Author.populate( +# first_name=author.first_name, +# last_name=author.last_name, +# display_name=author.display_name)) +# else: +# au = self.master_manager.get_object_filtered(Author, +# Author.display_name == u'Author Unknown') +# if au: +# new_song.authors.append(au) +# else: +# new_song.authors.append(Author.populate( +# display_name=u'Author Unknown')) +# if song.book: +# existing_song_book = self.master_manager.get_object_filtered( +# Book, Book.name == song.book.name) +# if existing_song_book: +# new_song.book = existing_song_book +# else: +# new_song.book = Book.populate(name=song.book.name, +# publisher=song.book.publisher) +# if song.topics: +# for topic in song.topics: +# existing_topic = self.master_manager.get_object_filtered( +# Topic, Topic.name == topic.name) +# if existing_topic: +# new_song.topics.append(existing_topic) +# else: +# new_song.topics.append(Topic.populate(name=topic.name)) +## if has_media_files: +## if song.media_files: +## for media_file in song.media_files: +## existing_media_file = \ +## self.master_manager.get_object_filtered(MediaFile, +## MediaFile.file_name == media_file.file_name) +## if existing_media_file: +## new_song.media_files.append(existing_media_file) +## else: +## new_song.media_files.append(MediaFile.populate( +## file_name=media_file.file_name)) +# self.master_manager.save_object(new_song) diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index e1d683000..8c675b9ab 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -28,6 +28,7 @@ import logging import os from zipfile import ZipFile from lxml import objectify +from lxml.etree import Error, LxmlError from openlp.plugins.songs.lib.songimport import SongImport @@ -36,119 +37,156 @@ log = logging.getLogger(__name__) class OpenSongImportError(Exception): pass -class OpenSongImport(object): +class OpenSongImport(SongImport): """ - Import songs exported from OpenSong - the format is described loosly here: - http://www.opensong.org/d/manual/song_file_format_specification + Import songs exported from OpenSong - However, it doesn't describe the section, so here's an attempt: + The format is described loosly on the `OpenSong File Format Specification + `_ page on + the OpenSong web site. However, it doesn't describe the section, + so here's an attempt: - Verses can be expressed in one of 2 ways: - - [v1]List of words - Another Line + Verses can be expressed in one of 2 ways, either in complete verses, or by + line grouping, i.e. grouping all line 1's of a verse together, all line 2's + of a verse together, and so on. - [v2]Some words for the 2nd verse - etc... - + An example of complete verses:: - The 'v' can be left out - it is implied - or: - - [V] - 1List of words - 2Some words for the 2nd Verse + + [v1] + List of words + Another Line - 1Another Line - 2etc... - + [v2] + Some words for the 2nd verse + etc... + - Either or both forms can be used in one song. The Number does not - necessarily appear at the start of the line + The 'v' in the verse specifiers above can be left out, it is implied. + + An example of line grouping:: + + + [V] + 1List of words + 2Some words for the 2nd Verse + + 1Another Line + 2etc... + + + Either or both forms can be used in one song. The number does not + necessarily appear at the start of the line. Additionally, the [v1] labels + can have either upper or lower case Vs. - The [v1] labels can have either upper or lower case Vs Other labels can be used also: - C - Chorus - B - Bridge - Guitar chords can be provided 'above' the lyrics (the line is - preceeded by a'.') and _s can be used to signify long-drawn-out - words: + C + Chorus - . A7 Bm - 1 Some____ Words + B + Bridge - Chords and _s are removed by this importer. + All verses are imported and tagged appropriately. - The verses etc. are imported and tagged appropriately. + Guitar chords can be provided "above" the lyrics (the line is preceeded by + a period "."), and one or more "_" can be used to signify long-drawn-out + words. Chords and "_" are removed by this importer. For example:: - The tag is used to populate the OpenLP verse - display order field. The Author and Copyright tags are also - imported to the appropriate places. + . A7 Bm + 1 Some____ Words + + The tag is used to populate the OpenLP verse display order + field. The Author and Copyright tags are also imported to the appropriate + places. """ - def __init__(self, songmanager): + def __init__(self, manager, **kwargs): """ - Initialise the class. Requires a songmanager class which - is passed to SongImport for writing song to disk + Initialise the class. """ - self.songmanager = songmanager + SongImport.__init__(self, manager) + self.filenames = kwargs[u'filenames'] self.song = None + self.commit = True - def do_import(self, filename, commit=True): + def do_import(self): """ - Import either a single opensong file, or a zipfile - containing multiple opensong files If the commit parameter is - set False, the import will not be committed to the database - (useful for test scripts) + Import either each of the files in self.filenames - each element of + which can be either a single opensong file, or a zipfile containing + multiple opensong files. If `self.commit` is set False, the + import will not be committed to the database (useful for test scripts). """ - ext = os.path.splitext(filename)[1] - if ext.lower() == ".zip": - log.info('Zipfile found %s', filename) - z = ZipFile(filename, u'r') - for song in z.infolist(): - parts = os.path.split(song.filename) - if parts[-1] == u'': - #No final part => directory - continue - songfile = z.open(song) - self.do_import_file(songfile) - if commit: + success = False + self.import_wizard.importProgressBar.setMaximum(len(self.filenames)) + for filename in self.filenames: + if self.stop_import_flag: + break + ext = os.path.splitext(filename)[1] + if ext.lower() == u'.zip': + log.info(u'Zipfile found %s', filename) + z = ZipFile(filename, u'r') + for song in z.infolist(): + if self.stop_import_flag: + break + parts = os.path.split(song.filename) + if parts[-1] == u'': + #No final part => directory + continue + log.info(u'Zip importing %s', parts[-1]) + self.import_wizard.incrementProgressBar(u'Importing %s...' \ + % parts[-1]) + songfile = z.open(song) + self.do_import_file(songfile) + if self.commit: + self.finish() + if self.stop_import_flag: + break + else: + log.info('Direct import %s', filename) + self.import_wizard.incrementProgressBar(u'Importing %s...' \ + % os.path.split(filename)[-1]) + file = open(filename) + self.do_import_file(file) + if self.commit: self.finish() - else: - log.info('Direct import %s', filename) - file = open(filename) - self.do_import_file(file) - if commit: - self.finish() + if not self.commit: + self.finish() + - def do_import_file(self, file): """ Process the OpenSong file - pass in a file-like object, not a filename - """ - self.song_import = SongImport(self.songmanager) - tree = objectify.parse(file) + """ + self.authors = [] + self.verse_order_list = [] + try: + tree = objectify.parse(file) + except Error, LxmlError: + log.exception(u'Error parsing XML') + return root = tree.getroot() fields = dir(root) - decode = {u'copyright':self.song_import.add_copyright, - u'ccli':u'ccli_number', - u'author':self.song_import.parse_author, - u'title':u'title', - u'aka':u'alternate_title', - u'hymn_number':u'song_number'} - for (attr, fn_or_string) in decode.items(): + decode = { + u'copyright': self.add_copyright, + u'ccli': u'ccli_number', + u'author': self.parse_author, + u'title': u'title', + u'aka': u'alternate_title', + u'hymn_number': u'song_number' + } + for attr, fn_or_string in decode.items(): if attr in fields: ustring = unicode(root.__getattr__(attr)) - if type(fn_or_string) == type(u''): - self.song_import.__setattr__(fn_or_string, ustring) + if isinstance(fn_or_string, basestring): + setattr(self, fn_or_string, ustring) else: fn_or_string(ustring) - if u'theme' in fields: - self.song_import.topics.append(unicode(root.theme)) - if u'alttheme' in fields: - self.song_import.topics.append(unicode(root.alttheme)) + if u'theme' in fields and unicode(root.theme) not in self.topics: + self.topics.append(unicode(root.theme)) + if u'alttheme' in fields and unicode(root.alttheme) not in self.topics: + self.topics.append(unicode(root.alttheme)) # data storage while importing verses = {} lyrics = unicode(root.lyrics) @@ -158,6 +196,7 @@ class OpenSongImport(object): # in the absence of any other indication, verses are the default, # erm, versetype! versetype = u'V' + versenum = None for thisline in lyrics.split(u'\n'): # remove comments semicolon = thisline.find(u';') @@ -170,7 +209,6 @@ class OpenSongImport(object): if thisline[0] == u'.' or thisline.startswith(u'---') \ or thisline.startswith(u'-!!'): continue - # verse/chorus/etc. marker if thisline[0] == u'[': versetype = thisline[1].upper() @@ -186,7 +224,6 @@ class OpenSongImport(object): versenum = u'1' continue words = None - # number at start of line.. it's verse number if thisline[0].isdigit(): versenum = thisline[0] @@ -207,7 +244,7 @@ class OpenSongImport(object): our_verse_order.append(versetag) if words: # Tidy text and remove the ____s from extended words - words = self.song_import.tidy_text(words) + words = self.tidy_text(words) words = words.replace('_', '') verses[versetype][versenum].append(words) # done parsing @@ -220,24 +257,23 @@ class OpenSongImport(object): for num in versenums: versetag = u'%s%s' % (versetype, num) lines = u'\n'.join(verses[versetype][num]) - self.song_import.verses.append([versetag, lines]) + self.verses.append([versetag, lines]) # Keep track of what we have for error checking later versetags[versetag] = 1 # now figure out the presentation order + order = [] if u'presentation' in fields and root.presentation != u'': order = unicode(root.presentation) order = order.split() else: - assert len(our_verse_order)>0 - order = our_verse_order + if len(our_verse_order) > 0: + order = our_verse_order + else: + log.warn(u'No verse order available for %s, skipping.', self.title) for tag in order: if len(tag) == 1: tag = tag + u'1' # Assume it's no.1 if it's not there if not versetags.has_key(tag): log.warn(u'Got order %s but not in versetags, skipping', tag) else: - self.song_import.verse_order_list.append(tag) - - def finish(self): - """ Separate function, allows test suite to not pollute database""" - self.song_import.finish() + self.verse_order_list.append(tag) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 2ffb0beda..5889a3774 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -158,8 +158,7 @@ class SongImport(QtCore.QObject): def parse_author(self, text): """ Add the author. OpenLP stores them individually so split by 'and', '&' - and comma. - However need to check for 'Mr and Mrs Smith' and turn it to + and comma. However need to check for 'Mr and Mrs Smith' and turn it to 'Mr Smith' and 'Mrs Smith'. """ for author in text.split(u','): @@ -236,7 +235,7 @@ class SongImport(QtCore.QObject): """ All fields have been set to this song. Write it away """ - if len(self.authors) == 0: + if not self.authors: self.authors.append(u'Author unknown') self.commit_song() diff --git a/openlp/plugins/songs/lib/test/test_importing_lots.py b/openlp/plugins/songs/lib/test/test_importing_lots.py index 9f1908bca..43ceeab04 100644 --- a/openlp/plugins/songs/lib/test/test_importing_lots.py +++ b/openlp/plugins/songs/lib/test/test_importing_lots.py @@ -8,50 +8,25 @@ from traceback import print_exc import sys import codecs +import logging +LOG_FILENAME = 'import.log' +logging.basicConfig(filename=LOG_FILENAME,level=logging.INFO) + +from test_opensongimport import wizard_stub, progbar_stub def opensong_import_lots(): ziploc = u'/home/mjt/openlp/OpenSong_Data/' files = [] #files = [u'test.opensong.zip', ziploc+u'ADond.zip'] - files.extend(glob(ziploc+u'Songs.zip')) + # files.extend(glob(ziploc+u'Songs.zip')) + files.extend(glob(ziploc+u'RaoulSongs.zip')) #files.extend(glob(ziploc+u'SOF.zip')) #files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip')) # files.extend(glob(ziploc+u'opensong_*.zip')) errfile = codecs.open(u'import_lots_errors.txt', u'w', u'utf8') manager = Manager(u'songs', init_schema) - for file in files: - print u'Importing', file - z = ZipFile(file, u'r') - for song in z.infolist(): - # need to handle unicode filenames (CP437 - Winzip does this) - filename = song.filename#.decode('cp852') - parts = os.path.split(filename) - if parts[-1] == u'': - #No final part => directory - continue - print " ", file, ":",filename, - songfile = z.open(song) - #z.extract(song) - #songfile=open(filename, u'r') - o = OpenSongImport(manager) - try: - o.do_import_file(songfile) - # o.song_import.print_song() - except: - print "Failure", - - errfile.write(u'Failure: %s:%s\n' %(file, filename.decode('cp437'))) - songfile = z.open(song) - for l in songfile.readlines(): - l = l.decode('utf8') - print(u' |%s\n' % l.strip()) - errfile.write(u' |%s\n'%l.strip()) - print_exc(3, file = errfile) - print_exc(3) - sys.exit(1) - # continue - #o.finish() - print "OK" - #os.unlink(filename) - # o.song_import.print_song() + o = OpenSongImport(manager, filenames=files) + o.import_wizard=wizard_stub() + o.do_import() + if __name__ == "__main__": opensong_import_lots() diff --git a/openlp/plugins/songs/lib/test/test_opensongimport.py b/openlp/plugins/songs/lib/test/test_opensongimport.py index d72114ce0..faad1b0f2 100644 --- a/openlp/plugins/songs/lib/test/test_opensongimport.py +++ b/openlp/plugins/songs/lib/test/test_opensongimport.py @@ -28,59 +28,81 @@ from openlp.plugins.songs.lib.opensongimport import OpenSongImport from openlp.core.lib.db import Manager from openlp.plugins.songs.lib.db import init_schema +import logging +LOG_FILENAME = 'test.log' +logging.basicConfig(filename=LOG_FILENAME,level=logging.INFO) + +# Stubs to replace the UI functions for raw testing +class wizard_stub: + def __init__(self): + self.importProgressBar=progbar_stub() + def incrementProgressBar(self, str): + pass +class progbar_stub: + def __init__(self): + pass + def setMaximum(self, arg): + pass + + def test(): manager = Manager(u'songs', init_schema) - o = OpenSongImport(manager) - o.do_import(u'test.opensong', commit=False) - o.song_import.print_song() - assert o.song_import.copyright == u'2010 Martin Thompson' - assert o.song_import.authors == [u'MartiÑ Thómpson'] - assert o.song_import.title == u'Martins Test' - assert o.song_import.alternate_title == u'' - assert o.song_import.song_number == u'1' - assert [u'C1', u'Chorus 1'] in o.song_import.verses - assert [u'C2', u'Chorus 2'] in o.song_import.verses - assert not [u'C3', u'Chorus 3'] in o.song_import.verses - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses - assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] - assert o.song_import.ccli_number == u'Blah' - assert o.song_import.topics == [u'TestTheme', u'TestAltTheme'] - o.do_import(u'test.opensong.zip', commit=False) - o.song_import.print_song() - o.finish() - assert o.song_import.copyright == u'2010 Martin Thompson' - assert o.song_import.authors == [u'MartiÑ Thómpson'] - assert o.song_import.title == u'Martins Test' - assert o.song_import.alternate_title == u'' - assert o.song_import.song_number == u'1' - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses - assert [u'C1', u'Chorus 1'] in o.song_import.verses - assert [u'C2', u'Chorus 2'] in o.song_import.verses - assert not [u'C3', u'Chorus 3'] in o.song_import.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses - assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] + o = OpenSongImport(manager, filenames=[u'test.opensong']) + o.import_wizard = wizard_stub() + o.commit = False + o.do_import() + o.print_song() + assert o.copyright == u'2010 Martin Thompson' + assert o.authors == [u'MartiÑ Thómpson'] + assert o.title == u'Martins Test' + assert o.alternate_title == u'' + assert o.song_number == u'1' + assert [u'C1', u'Chorus 1'] in o.verses + assert [u'C2', u'Chorus 2'] in o.verses + assert not [u'C3', u'Chorus 3'] in o.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.verses + assert o.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] + assert o.ccli_number == u'Blah' + assert o.topics == [u'TestTheme', u'TestAltTheme'] - o = OpenSongImport(manager) - o.do_import(u'test2.opensong', commit=False) + o.filenames = [u'test.opensong.zip'] + o.do_import() + o.print_song() + o.finish() + assert o.copyright == u'2010 Martin Thompson' + assert o.authors == [u'MartiÑ Thómpson'] + assert o.title == u'Martins Test' + assert o.alternate_title == u'' + assert o.song_number == u'1' + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.verses + assert [u'C1', u'Chorus 1'] in o.verses + assert [u'C2', u'Chorus 2'] in o.verses + assert not [u'C3', u'Chorus 3'] in o.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.verses + print o.verse_order_list + assert o.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1'] + + o.filenames = [u'test2.opensong'] + o.do_import() # o.finish() - o.song_import.print_song() - assert o.song_import.copyright == u'2010 Martin Thompson' - assert o.song_import.authors == [u'Martin Thompson'] - assert o.song_import.title == u'Martins 2nd Test' - assert o.song_import.alternate_title == u'' - assert o.song_import.song_number == u'2' - print o.song_import.verses - assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses - assert [u'C1', u'Chorus 1'] in o.song_import.verses - assert [u'C2', u'Chorus 2'] in o.song_import.verses - assert not [u'C3', u'Chorus 3'] in o.song_import.verses - assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses - assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses - print o.song_import.verse_order_list - assert o.song_import.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] + o.print_song() + assert o.copyright == u'2010 Martin Thompson' + assert o.authors == [u'Martin Thompson'] + assert o.title == u'Martins 2nd Test' + assert o.alternate_title == u'' + assert o.song_number == u'2' + print o.verses + assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.verses + assert [u'C1', u'Chorus 1'] in o.verses + assert [u'C2', u'Chorus 2'] in o.verses + assert not [u'C3', u'Chorus 3'] in o.verses + assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.verses + assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.verses + print o.verse_order_list + assert o.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2'] print "Tests passed" pass