diff --git a/openlp/plugins/songs/forms/songimportform.py b/openlp/plugins/songs/forms/songimportform.py index f2a59ba81..0c58b8650 100644 --- a/openlp/plugins/songs/forms/songimportform.py +++ b/openlp/plugins/songs/forms/songimportform.py @@ -31,7 +31,6 @@ from PyQt4 import QtCore, QtGui from songimportwizard import Ui_SongImportWizard from openlp.core.lib import Receiver, SettingsManager, translate -#from openlp.core.utils import AppLocation from openlp.plugins.songs.lib.importer import SongFormat log = logging.getLogger(__name__) @@ -136,7 +135,7 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard): self.openLP2BrowseButton.setFocus() return False elif source_format == SongFormat.OpenLP1: - if self.openSongFilenameEdit.text().isEmpty(): + if self.openLP1FilenameEdit.text().isEmpty(): QtGui.QMessageBox.critical(self, translate('SongsPlugin.ImportWizardForm', 'No openlp.org 1.x Song Database Selected'), @@ -292,7 +291,7 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard): def onCCLIRemoveButtonClicked(self): self.removeSelectedItems(self.ccliFileListWidget) - + def onSongsOfFellowshipAddButtonClicked(self): self.getFiles( translate('SongsPlugin.ImportWizardForm', @@ -374,11 +373,11 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard): importer = self.plugin.importSongs(SongFormat.OpenLP2, filename=unicode(self.openLP2FilenameEdit.text()) ) - #elif source_format == SongFormat.OpenLP1: - # # Import an openlp.org database - # importer = self.plugin.importSongs(SongFormat.OpenLP1, - # filename=unicode(self.field(u'openlp1_filename').toString()) - # ) + elif source_format == SongFormat.OpenLP1: + # Import an openlp.org database + importer = self.plugin.importSongs(SongFormat.OpenLP1, + filename=unicode(self.openLP1FilenameEdit.text()) + ) elif source_format == SongFormat.OpenLyrics: # Import OpenLyrics songs importer = self.plugin.importSongs(SongFormat.OpenLyrics, diff --git a/openlp/plugins/songs/lib/importer.py b/openlp/plugins/songs/lib/importer.py index 5801ea44a..da9618ef2 100644 --- a/openlp/plugins/songs/lib/importer.py +++ b/openlp/plugins/songs/lib/importer.py @@ -26,6 +26,7 @@ from opensongimport import OpenSongImport from olpimport import OpenLPSongImport +from olp1import import OpenLP1SongImport try: from sofimport import SofImport from oooimport import OooImport @@ -61,6 +62,8 @@ class SongFormat(object): """ if format == SongFormat.OpenLP2: return OpenLPSongImport + if format == SongFormat.OpenLP1: + return OpenLP1SongImport elif format == SongFormat.OpenSong: return OpenSongImport elif format == SongFormat.SongsOfFellowship: @@ -70,7 +73,7 @@ class SongFormat(object): elif format == SongFormat.Generic: return OooImport elif format == SongFormat.CCLI: - return CCLIFileImport + return CCLIFileImport # else: return None diff --git a/openlp/plugins/songs/lib/olp1import.py b/openlp/plugins/songs/lib/olp1import.py new file mode 100644 index 000000000..953b509b4 --- /dev/null +++ b/openlp/plugins/songs/lib/olp1import.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2010 Raoul Snyman # +# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael # +# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian # +# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, # +# Carsten Tinggaard, Frode Woldsund # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`olp1import` module provides the functionality for importing +openlp.org 1.x song databases into the current installation database. +""" +import logging +import sqlite + +from openlp.core.lib import translate +from songimport import SongImport + +log = logging.getLogger(__name__) + +class OpenLP1SongImport(SongImport): + """ + The :class:`OpenLP1SongImport` class provides OpenLP with the ability to + import song databases from installations of openlp.org 1.x. + """ + def __init__(self, manager, **kwargs): + """ + Initialise the import. + + ``manager`` + The song manager for the running OpenLP installation. + + ``filename`` + The database providing the data to import. + """ + SongImport.__init__(self, manager) + self.import_source = kwargs[u'filename'] + + def do_import(self): + """ + Run the import for an openlp.org 1.x song database. + """ + # Connect to the database + connection = sqlite.connect(self.import_source) + cursor = connection.cursor() + # Count the number of records we need to import, for the progress bar + cursor.execute(u'SELECT COUNT(songid) FROM songs') + count = int(cursor.fetchone()[0]) + success = True + self.import_wizard.importProgressBar.setMaximum(count) + # "cache" our list of authors + cursor.execute(u'SELECT authorid, authorname FROM authors') + authors = cursor.fetchall() + # "cache" our list of tracks + cursor.execute(u'SELECT trackid, fulltrackname FROM tracks') + tracks = cursor.fetchall() + # Import the songs + cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, ' + u'copyrightinfo FROM songs') + songs = cursor.fetchall() + for song in songs: + self.set_defaults() + if self.stop_import_flag: + success = False + break + song_id = song[0] + title = unicode(song[1], u'cp1252') + lyrics = unicode(song[2], u'cp1252').replace(u'\r', u'') + copyright = unicode(song[3], u'cp1252') + self.import_wizard.incrementProgressBar( + unicode(translate('SongsPlugin.ImportWizardForm', + 'Importing "%s"...')) % title) + self.title = title + self.process_song_text(lyrics) + self.add_copyright(copyright) + cursor.execute(u'SELECT authorid FROM songauthors ' + u'WHERE songid = %s' % song_id) + author_ids = cursor.fetchall() + for author_id in author_ids: + if self.stop_import_flag: + success = False + break + for author in authors: + if author[0] == author_id[0]: + self.parse_author(unicode(author[1], u'cp1252')) + break + if self.stop_import_flag: + success = False + break + cursor.execute(u'SELECT name FROM sqlite_master ' + u'WHERE type = \'table\' AND name = \'tracks\'') + table_list = cursor.fetchall() + if len(table_list) > 0: + cursor.execute(u'SELECT trackid FROM songtracks ' + u'WHERE songid = %s ORDER BY listindex' % song_id) + track_ids = cursor.fetchall() + for track_id in track_ids: + if self.stop_import_flag: + success = False + break + for track in tracks: + if track[0] == track_id[0]: + self.add_media_file(unicode(track[1], u'cp1252')) + break + if self.stop_import_flag: + success = False + break + self.finish() + return success diff --git a/openlp/plugins/songs/lib/opensongimport.py b/openlp/plugins/songs/lib/opensongimport.py index e1d683000..f6048d566 100644 --- a/openlp/plugins/songs/lib/opensongimport.py +++ b/openlp/plugins/songs/lib/opensongimport.py @@ -28,6 +28,7 @@ import logging import os from zipfile import ZipFile from lxml import objectify +from lxml.etree import Error, LxmlError from openlp.plugins.songs.lib.songimport import SongImport @@ -36,119 +37,163 @@ log = logging.getLogger(__name__) class OpenSongImportError(Exception): pass -class OpenSongImport(object): +class OpenSongImport(SongImport): """ - Import songs exported from OpenSong - the format is described loosly here: - http://www.opensong.org/d/manual/song_file_format_specification + Import songs exported from OpenSong - However, it doesn't describe the section, so here's an attempt: + The format is described loosly on the `OpenSong File Format Specification + `_ page on + the OpenSong web site. However, it doesn't describe the section, + so here's an attempt: - Verses can be expressed in one of 2 ways: - - [v1]List of words - Another Line + Verses can be expressed in one of 2 ways, either in complete verses, or by + line grouping, i.e. grouping all line 1's of a verse together, all line 2's + of a verse together, and so on. - [v2]Some words for the 2nd verse - etc... - + An example of complete verses:: - The 'v' can be left out - it is implied - or: - - [V] - 1List of words - 2Some words for the 2nd Verse + + [v1] + List of words + Another Line - 1Another Line - 2etc... - + [v2] + Some words for the 2nd verse + etc... + - Either or both forms can be used in one song. The Number does not - necessarily appear at the start of the line + The 'v' in the verse specifiers above can be left out, it is implied. + + An example of line grouping:: + + + [V] + 1List of words + 2Some words for the 2nd Verse + + 1Another Line + 2etc... + + + Either or both forms can be used in one song. The number does not + necessarily appear at the start of the line. Additionally, the [v1] labels + can have either upper or lower case Vs. - The [v1] labels can have either upper or lower case Vs Other labels can be used also: - C - Chorus - B - Bridge - Guitar chords can be provided 'above' the lyrics (the line is - preceeded by a'.') and _s can be used to signify long-drawn-out - words: + C + Chorus - . A7 Bm - 1 Some____ Words + B + Bridge - Chords and _s are removed by this importer. + All verses are imported and tagged appropriately. - The verses etc. are imported and tagged appropriately. + Guitar chords can be provided "above" the lyrics (the line is preceeded by + a period "."), and one or more "_" can be used to signify long-drawn-out + words. Chords and "_" are removed by this importer. For example:: - The tag is used to populate the OpenLP verse - display order field. The Author and Copyright tags are also - imported to the appropriate places. + . A7 Bm + 1 Some____ Words + + The tag is used to populate the OpenLP verse display order + field. The Author and Copyright tags are also imported to the appropriate + places. """ - def __init__(self, songmanager): + def __init__(self, manager, **kwargs): """ - Initialise the class. Requires a songmanager class which - is passed to SongImport for writing song to disk + Initialise the class. """ - self.songmanager = songmanager + SongImport.__init__(self, manager) + self.filenames = kwargs[u'filenames'] self.song = None + self.commit = True - def do_import(self, filename, commit=True): + def do_import(self): """ - Import either a single opensong file, or a zipfile - containing multiple opensong files If the commit parameter is - set False, the import will not be committed to the database - (useful for test scripts) + Import either a single opensong file, or a zipfile containing multiple + opensong files. If `self.commit` is set False, the import will not be + committed to the database (useful for test scripts). """ - ext = os.path.splitext(filename)[1] - if ext.lower() == ".zip": - log.info('Zipfile found %s', filename) - z = ZipFile(filename, u'r') - for song in z.infolist(): - parts = os.path.split(song.filename) - if parts[-1] == u'': - #No final part => directory - continue - songfile = z.open(song) - self.do_import_file(songfile) - if commit: + success = True + self.import_wizard.importProgressBar.setMaximum(len(self.filenames)) + for filename in self.filenames: + if self.stop_import_flag: + success = False + break + ext = os.path.splitext(filename)[1] + if ext.lower() == u'.zip': + log.debug(u'Zipfile found %s', filename) + z = ZipFile(filename, u'r') + self.import_wizard.importProgressBar.setMaximum( + self.import_wizard.importProgressBar.maximum() + + len(z.infolist())) + for song in z.infolist(): + if self.stop_import_flag: + success = False + break + parts = os.path.split(song.filename) + if parts[-1] == u'': + #No final part => directory + continue + self.import_wizard.incrementProgressBar( + unicode(translate('SongsPlugin.ImportWizardForm', + 'Importing %s...')) % parts[-1]) + songfile = z.open(song) + self.do_import_file(songfile) + if self.commit: + self.finish() + self.set_defaults() + if self.stop_import_flag: + success = False + break + else: + log.info('Direct import %s', filename) + self.import_wizard.incrementProgressBar( + unicode(translate('SongsPlugin.ImportWizardForm', + 'Importing %s...')) % os.path.split(filename)[-1]) + file = open(filename) + self.do_import_file(file) + if self.commit: self.finish() - else: - log.info('Direct import %s', filename) - file = open(filename) - self.do_import_file(file) - if commit: - self.finish() + self.set_defaults() + if not self.commit: + self.finish() + return success - def do_import_file(self, file): """ Process the OpenSong file - pass in a file-like object, not a filename - """ - self.song_import = SongImport(self.songmanager) - tree = objectify.parse(file) + """ + self.authors = [] + try: + tree = objectify.parse(file) + except Error, LxmlError: + log.exception(u'Error parsing XML') + return root = tree.getroot() fields = dir(root) - decode = {u'copyright':self.song_import.add_copyright, - u'ccli':u'ccli_number', - u'author':self.song_import.parse_author, - u'title':u'title', - u'aka':u'alternate_title', - u'hymn_number':u'song_number'} - for (attr, fn_or_string) in decode.items(): + decode = { + u'copyright': self.add_copyright, + u'ccli': u'ccli_number', + u'author': self.parse_author, + u'title': u'title', + u'aka': u'alternate_title', + u'hymn_number': u'song_number' + } + for attr, fn_or_string in decode.items(): if attr in fields: ustring = unicode(root.__getattr__(attr)) - if type(fn_or_string) == type(u''): - self.song_import.__setattr__(fn_or_string, ustring) + if isinstance(fn_or_string, basestring): + setattr(self, fn_or_string, ustring) else: fn_or_string(ustring) - if u'theme' in fields: - self.song_import.topics.append(unicode(root.theme)) - if u'alttheme' in fields: - self.song_import.topics.append(unicode(root.alttheme)) + if u'theme' in fields and unicode(root.theme) not in self.topics: + self.topics.append(unicode(root.theme)) + if u'alttheme' in fields and unicode(root.alttheme) not in self.topics: + self.topics.append(unicode(root.alttheme)) # data storage while importing verses = {} lyrics = unicode(root.lyrics) @@ -158,6 +203,7 @@ class OpenSongImport(object): # in the absence of any other indication, verses are the default, # erm, versetype! versetype = u'V' + versenum = None for thisline in lyrics.split(u'\n'): # remove comments semicolon = thisline.find(u';') @@ -170,7 +216,6 @@ class OpenSongImport(object): if thisline[0] == u'.' or thisline.startswith(u'---') \ or thisline.startswith(u'-!!'): continue - # verse/chorus/etc. marker if thisline[0] == u'[': versetype = thisline[1].upper() @@ -186,7 +231,6 @@ class OpenSongImport(object): versenum = u'1' continue words = None - # number at start of line.. it's verse number if thisline[0].isdigit(): versenum = thisline[0] @@ -207,7 +251,7 @@ class OpenSongImport(object): our_verse_order.append(versetag) if words: # Tidy text and remove the ____s from extended words - words = self.song_import.tidy_text(words) + words = self.tidy_text(words) words = words.replace('_', '') verses[versetype][versenum].append(words) # done parsing @@ -220,24 +264,23 @@ class OpenSongImport(object): for num in versenums: versetag = u'%s%s' % (versetype, num) lines = u'\n'.join(verses[versetype][num]) - self.song_import.verses.append([versetag, lines]) + self.verses.append([versetag, lines]) # Keep track of what we have for error checking later versetags[versetag] = 1 # now figure out the presentation order + order = [] if u'presentation' in fields and root.presentation != u'': order = unicode(root.presentation) order = order.split() else: - assert len(our_verse_order)>0 - order = our_verse_order + if len(our_verse_order) > 0: + order = our_verse_order + else: + log.warn(u'No verse order available for %s, skipping.', self.title) for tag in order: if len(tag) == 1: tag = tag + u'1' # Assume it's no.1 if it's not there if not versetags.has_key(tag): log.warn(u'Got order %s but not in versetags, skipping', tag) else: - self.song_import.verse_order_list.append(tag) - - def finish(self): - """ Separate function, allows test suite to not pollute database""" - self.song_import.finish() + self.verse_order_list.append(tag) diff --git a/openlp/plugins/songs/lib/songimport.py b/openlp/plugins/songs/lib/songimport.py index 17000a2ba..ea763c1ee 100644 --- a/openlp/plugins/songs/lib/songimport.py +++ b/openlp/plugins/songs/lib/songimport.py @@ -30,7 +30,7 @@ from PyQt4 import QtCore from openlp.core.lib import Receiver, translate from openlp.plugins.songs.lib import VerseType -from openlp.plugins.songs.lib.db import Song, Author, Topic, Book +from openlp.plugins.songs.lib.db import Song, Author, Topic, Book, MediaFile from openlp.plugins.songs.lib.xml import SongXMLBuilder log = logging.getLogger(__name__) @@ -66,6 +66,7 @@ class SongImport(QtCore.QObject): self.ccli_number = u'' self.authors = [] self.topics = [] + self.media_files = [] self.song_book_name = u'' self.song_book_pub = u'' self.verse_order_list = [] @@ -76,7 +77,7 @@ class SongImport(QtCore.QObject): 'SongsPlugin.SongImport', 'copyright')) self.copyright_symbol = unicode(translate( 'SongsPlugin.SongImport', '\xa9')) - + def stop_import(self): """ Sets the flag for importers to stop their import @@ -184,6 +185,14 @@ class SongImport(QtCore.QObject): return self.authors.append(author) + def add_media_file(self, filename): + """ + Add a media file to the list + """ + if filename in self.media_files: + return + self.media_files.append(filename) + def add_verse(self, verse, versetag=None): """ Add a verse. This is the whole verse, lines split by \n @@ -279,11 +288,16 @@ class SongImport(QtCore.QObject): for authortext in self.authors: author = self.manager.get_object_filtered(Author, Author.display_name == authortext) - if author is None: + if not author: author = Author.populate(display_name = authortext, last_name=authortext.split(u' ')[-1], first_name=u' '.join(authortext.split(u' ')[:-1])) song.authors.append(author) + for filename in self.media_files: + media_file = self.manager.get_object_filtered(MediaFile, + MediaFile.file_name == filename) + if not media_file: + song.media_files.append(MediaFile.populate(file_name=filename)) if self.song_book_name: song_book = self.manager.get_object_filtered(Book, Book.name == self.song_book_name)