Merge in the openlp.org 1.x importer.

bzr-revno: 1017

openlp/plugins/songs/forms/songimportform.py

@@ -31,7 +31,6 @@ from PyQt4 import QtCore, QtGui
 
 from songimportwizard import Ui_SongImportWizard
 from openlp.core.lib import Receiver, SettingsManager, translate
-#from openlp.core.utils import AppLocation
 from openlp.plugins.songs.lib.importer import SongFormat
 
 log = logging.getLogger(__name__)
@@ -136,7 +135,7 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard):
                     self.openLP2BrowseButton.setFocus()
                     return False
             elif source_format == SongFormat.OpenLP1:
-                if self.openSongFilenameEdit.text().isEmpty():
+                if self.openLP1FilenameEdit.text().isEmpty():
                     QtGui.QMessageBox.critical(self,
                         translate('SongsPlugin.ImportWizardForm',
                             'No openlp.org 1.x Song Database Selected'),
@@ -374,11 +373,11 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard):
             importer = self.plugin.importSongs(SongFormat.OpenLP2,
                 filename=unicode(self.openLP2FilenameEdit.text())
             )
-        #elif source_format == SongFormat.OpenLP1:
+        elif source_format == SongFormat.OpenLP1:
-        #    # Import an openlp.org database
+            # Import an openlp.org database
-        #    importer = self.plugin.importSongs(SongFormat.OpenLP1,
+            importer = self.plugin.importSongs(SongFormat.OpenLP1,
-        #        filename=unicode(self.field(u'openlp1_filename').toString())
+                filename=unicode(self.openLP1FilenameEdit.text())
-        #    )
+            )
         elif source_format == SongFormat.OpenLyrics:
             # Import OpenLyrics songs
             importer = self.plugin.importSongs(SongFormat.OpenLyrics,

openlp/plugins/songs/lib/importer.py

@@ -26,6 +26,7 @@
 
 from opensongimport import OpenSongImport
 from olpimport import OpenLPSongImport
+from olp1import import OpenLP1SongImport
 try:
     from sofimport import SofImport
     from oooimport import OooImport
@@ -61,6 +62,8 @@ class SongFormat(object):
         """
         if format == SongFormat.OpenLP2:
             return OpenLPSongImport
+        if format == SongFormat.OpenLP1:
+            return OpenLP1SongImport
         elif format == SongFormat.OpenSong:
             return OpenSongImport
         elif format == SongFormat.SongsOfFellowship:

openlp/plugins/songs/lib/olp1import.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
+
+###############################################################################
+# OpenLP - Open Source Lyrics Projection                                      #
+# --------------------------------------------------------------------------- #
+# Copyright (c) 2008-2010 Raoul Snyman                                        #
+# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael      #
+# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian      #
+# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble,    #
+# Carsten Tinggaard, Frode Woldsund                                           #
+# --------------------------------------------------------------------------- #
+# This program is free software; you can redistribute it and/or modify it     #
+# under the terms of the GNU General Public License as published by the Free  #
+# Software Foundation; version 2 of the License.                              #
+#                                                                             #
+# This program is distributed in the hope that it will be useful, but WITHOUT #
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with this program; if not, write to the Free Software Foundation, Inc., 59  #
+# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
+###############################################################################
+"""
+The :mod:`olp1import` module provides the functionality for importing
+openlp.org 1.x song databases into the current installation database.
+"""
+import logging
+import sqlite
+
+from openlp.core.lib import translate
+from songimport import SongImport
+
+log = logging.getLogger(__name__)
+
+class OpenLP1SongImport(SongImport):
+    """
+    The :class:`OpenLP1SongImport` class provides OpenLP with the ability to
+    import song databases from installations of openlp.org 1.x.
+    """
+    def __init__(self, manager, **kwargs):
+        """
+        Initialise the import.
+
+        ``manager``
+            The song manager for the running OpenLP installation.
+
+        ``filename``
+            The database providing the data to import.
+        """
+        SongImport.__init__(self, manager)
+        self.import_source = kwargs[u'filename']
+
+    def do_import(self):
+        """
+        Run the import for an openlp.org 1.x song database.
+        """
+        # Connect to the database
+        connection = sqlite.connect(self.import_source)
+        cursor = connection.cursor()
+        # Count the number of records we need to import, for the progress bar
+        cursor.execute(u'SELECT COUNT(songid) FROM songs')
+        count = int(cursor.fetchone()[0])
+        success = True
+        self.import_wizard.importProgressBar.setMaximum(count)
+        # "cache" our list of authors
+        cursor.execute(u'SELECT authorid, authorname FROM authors')
+        authors = cursor.fetchall()
+        # "cache" our list of tracks
+        cursor.execute(u'SELECT trackid, fulltrackname FROM tracks')
+        tracks = cursor.fetchall()
+        # Import the songs
+        cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, '
+            u'copyrightinfo FROM songs')
+        songs = cursor.fetchall()
+        for song in songs:
+            self.set_defaults()
+            if self.stop_import_flag:
+                success = False
+                break
+            song_id = song[0]
+            title = unicode(song[1], u'cp1252')
+            lyrics = unicode(song[2], u'cp1252').replace(u'\r', u'')
+            copyright = unicode(song[3], u'cp1252')
+            self.import_wizard.incrementProgressBar(
+                unicode(translate('SongsPlugin.ImportWizardForm',
+                    'Importing "%s"...')) % title)
+            self.title = title
+            self.process_song_text(lyrics)
+            self.add_copyright(copyright)
+            cursor.execute(u'SELECT authorid FROM songauthors '
+                u'WHERE songid = %s' % song_id)
+            author_ids = cursor.fetchall()
+            for author_id in author_ids:
+                if self.stop_import_flag:
+                    success = False
+                    break
+                for author in authors:
+                    if author[0] == author_id[0]:
+                        self.parse_author(unicode(author[1], u'cp1252'))
+                        break
+            if self.stop_import_flag:
+                success = False
+                break
+            cursor.execute(u'SELECT name FROM sqlite_master '
+                u'WHERE type = \'table\' AND name = \'tracks\'')
+            table_list = cursor.fetchall()
+            if len(table_list) > 0:
+                cursor.execute(u'SELECT trackid FROM songtracks '
+                    u'WHERE songid = %s ORDER BY listindex' % song_id)
+                track_ids = cursor.fetchall()
+                for track_id in track_ids:
+                    if self.stop_import_flag:
+                        success = False
+                        break
+                    for track in tracks:
+                        if track[0] == track_id[0]:
+                            self.add_media_file(unicode(track[1], u'cp1252'))
+                            break
+            if self.stop_import_flag:
+                success = False
+                break
+            self.finish()
+        return success

openlp/plugins/songs/lib/opensongimport.py

@@ -28,6 +28,7 @@ import logging
 import os
 from zipfile import ZipFile
 from lxml import objectify
+from lxml.etree import Error, LxmlError
 
 from openlp.plugins.songs.lib.songimport import SongImport
 
@@ -36,119 +37,163 @@ log = logging.getLogger(__name__)
 class OpenSongImportError(Exception):
     pass
 
-class OpenSongImport(object):
+class OpenSongImport(SongImport):
     """
-    Import songs exported from OpenSong - the format is described loosly here:
+    Import songs exported from OpenSong
-    http://www.opensong.org/d/manual/song_file_format_specification
 
-    However, it doesn't describe the <lyrics> section, so here's an attempt:
+    The format is described loosly on the `OpenSong File Format Specification
+    <http://www.opensong.org/d/manual/song_file_format_specification>`_ page on
+    the OpenSong web site. However, it doesn't describe the <lyrics> section,
+    so here's an attempt:
 
-    Verses can be expressed in one of 2 ways:
+    Verses can be expressed in one of 2 ways, either in complete verses, or by
-    <lyrics>
+    line grouping, i.e. grouping all line 1's of a verse together, all line 2's
-    [v1]List of words
+    of a verse together, and so on.
-    Another Line
 
-    [v2]Some words for the 2nd verse
+    An example of complete verses::
-    etc...
-    </lyrics>
 
-    The 'v' can be left out - it is implied
+        <lyrics>
-    or:
+        [v1]
-    <lyrics>
+         List of words
-    [V]
+         Another Line
-    1List of words
-    2Some words for the 2nd Verse
 
-    1Another Line
+        [v2]
-    2etc...
+         Some words for the 2nd verse
-    </lyrics>
+         etc...
+        </lyrics>
 
-    Either or both forms can be used in one song.  The Number does not
+    The 'v' in the verse specifiers above can be left out, it is implied.
-    necessarily appear at the start of the line
+
+    An example of line grouping::
+
+        <lyrics>
+        [V]
+        1List of words
+        2Some words for the 2nd Verse
+
+        1Another Line
+        2etc...
+        </lyrics>
+
+    Either or both forms can be used in one song. The number does not
+    necessarily appear at the start of the line. Additionally, the [v1] labels
+    can have either upper or lower case Vs.
 
-    The [v1] labels can have either upper or lower case Vs
     Other labels can be used also:
-      C - Chorus
-      B - Bridge
 
-    Guitar chords can be provided 'above' the lyrics (the line is
+    C
-    preceeded by a'.') and _s can be used to signify long-drawn-out
+        Chorus
-    words:
 
-    . A7        Bm
+    B
-    1 Some____ Words
+        Bridge
 
-    Chords and _s are removed by this importer.
+    All verses are imported and tagged appropriately.
 
-    The verses etc. are imported and tagged appropriately.
+    Guitar chords can be provided "above" the lyrics (the line is preceeded by
+    a period "."), and one or more "_" can be used to signify long-drawn-out
+    words. Chords and "_" are removed by this importer. For example::
 
-    The <presentation> tag is used to populate the OpenLP verse
+        . A7        Bm
-    display order field.  The Author and Copyright tags are also
+        1 Some____ Words
-    imported to the appropriate places.
+
+    The <presentation> tag is used to populate the OpenLP verse display order
+    field. The Author and Copyright tags are also imported to the appropriate
+    places.
 
     """
-    def __init__(self, songmanager):
+    def __init__(self, manager, **kwargs):
         """
-        Initialise the class. Requires a songmanager class which 
+        Initialise the class.
-        is passed to SongImport for writing song to disk
         """
-        self.songmanager = songmanager
+        SongImport.__init__(self, manager)
+        self.filenames = kwargs[u'filenames']
         self.song = None
+        self.commit = True
 
-    def do_import(self, filename, commit=True):
+    def do_import(self):
         """
-        Import either a single opensong file, or a zipfile
+        Import either a single opensong file, or a zipfile containing multiple
-        containing multiple opensong files If the commit parameter is
+        opensong files. If `self.commit` is set False, the import will not be
-        set False, the import will not be committed to the database
+        committed to the database (useful for test scripts).
-        (useful for test scripts)
         """
-        ext = os.path.splitext(filename)[1]
+        success = True
-        if ext.lower() == ".zip":
+        self.import_wizard.importProgressBar.setMaximum(len(self.filenames))
-            log.info('Zipfile found %s', filename)
+        for filename in self.filenames:
-            z = ZipFile(filename, u'r')
+            if self.stop_import_flag:
-            for song in z.infolist():
+                success = False
-                parts = os.path.split(song.filename)
+                break
-                if parts[-1] == u'':
+            ext = os.path.splitext(filename)[1]
-                    #No final part => directory
+            if ext.lower() == u'.zip':
-                    continue
+                log.debug(u'Zipfile found %s', filename)
-                songfile = z.open(song)
+                z = ZipFile(filename, u'r')
-                self.do_import_file(songfile)
+                self.import_wizard.importProgressBar.setMaximum(
-                if commit:
+                    self.import_wizard.importProgressBar.maximum() +
+                    len(z.infolist()))
+                for song in z.infolist():
+                    if self.stop_import_flag:
+                        success = False
+                        break
+                    parts = os.path.split(song.filename)
+                    if parts[-1] == u'':
+                        #No final part => directory
+                        continue
+                    self.import_wizard.incrementProgressBar(
+                        unicode(translate('SongsPlugin.ImportWizardForm',
+                            'Importing %s...')) % parts[-1])
+                    songfile = z.open(song)
+                    self.do_import_file(songfile)
+                    if self.commit:
+                        self.finish()
+                    self.set_defaults()
+                if self.stop_import_flag:
+                    success = False
+                    break
+            else:
+                log.info('Direct import %s', filename)
+                self.import_wizard.incrementProgressBar(
+                    unicode(translate('SongsPlugin.ImportWizardForm',
+                        'Importing %s...')) % os.path.split(filename)[-1])
+                file = open(filename)
+                self.do_import_file(file)
+                if self.commit:
                     self.finish()
-        else:
+                self.set_defaults()
-            log.info('Direct import %s', filename)
+        if not self.commit:
-            file = open(filename)
+            self.finish()
-            self.do_import_file(file)
+        return success
-            if commit:
-                self.finish()
-
 
     def do_import_file(self, file):
         """
         Process the OpenSong file - pass in a file-like object,
         not a filename
         """
-        self.song_import = SongImport(self.songmanager)
+        self.authors = []
-        tree = objectify.parse(file)
+        try:
+            tree = objectify.parse(file)
+        except Error, LxmlError:
+            log.exception(u'Error parsing XML')
+            return
         root = tree.getroot()
         fields = dir(root)
-        decode = {u'copyright':self.song_import.add_copyright,
+        decode = {
-                u'ccli':u'ccli_number',
+            u'copyright': self.add_copyright,
-                u'author':self.song_import.parse_author,
+            u'ccli': u'ccli_number',
-                u'title':u'title',
+            u'author': self.parse_author,
-                u'aka':u'alternate_title',
+            u'title': u'title',
-                u'hymn_number':u'song_number'}
+            u'aka': u'alternate_title',
-        for (attr, fn_or_string) in decode.items():
+            u'hymn_number': u'song_number'
+        }
+        for attr, fn_or_string in decode.items():
             if attr in fields:
                 ustring = unicode(root.__getattr__(attr))
-                if type(fn_or_string) == type(u''):
+                if isinstance(fn_or_string, basestring):
-                    self.song_import.__setattr__(fn_or_string, ustring)
+                    setattr(self, fn_or_string, ustring)
                 else:
                     fn_or_string(ustring)
-        if u'theme' in fields:
+        if u'theme' in fields and unicode(root.theme) not in self.topics:
-            self.song_import.topics.append(unicode(root.theme))
+            self.topics.append(unicode(root.theme))
-        if u'alttheme' in fields:
+        if u'alttheme' in fields and unicode(root.alttheme) not in self.topics:
-            self.song_import.topics.append(unicode(root.alttheme))
+            self.topics.append(unicode(root.alttheme))
         # data storage while importing
         verses = {}
         lyrics = unicode(root.lyrics)
@@ -158,6 +203,7 @@ class OpenSongImport(object):
         # in the absence of any other indication, verses are the default,
         # erm, versetype!
         versetype = u'V'
+        versenum = None
         for thisline in lyrics.split(u'\n'):
             # remove comments
             semicolon = thisline.find(u';')
@@ -170,7 +216,6 @@ class OpenSongImport(object):
             if thisline[0] == u'.' or thisline.startswith(u'---') \
                 or thisline.startswith(u'-!!'):
                 continue
-            
             # verse/chorus/etc. marker
             if thisline[0] == u'[':
                 versetype = thisline[1].upper()
@@ -186,7 +231,6 @@ class OpenSongImport(object):
                     versenum = u'1'
                 continue
             words = None
-
             # number at start of line.. it's verse number
             if thisline[0].isdigit():
                 versenum = thisline[0]
@@ -207,7 +251,7 @@ class OpenSongImport(object):
                     our_verse_order.append(versetag)
             if words:
                 # Tidy text and remove the ____s from extended words
-                words = self.song_import.tidy_text(words)
+                words = self.tidy_text(words)
                 words = words.replace('_', '')
                 verses[versetype][versenum].append(words)
         # done parsing
@@ -220,24 +264,23 @@ class OpenSongImport(object):
             for num in versenums:
                 versetag = u'%s%s' % (versetype, num)
                 lines = u'\n'.join(verses[versetype][num])
-                self.song_import.verses.append([versetag, lines])
+                self.verses.append([versetag, lines])
                 # Keep track of what we have for error checking later
                 versetags[versetag] = 1
         # now figure out the presentation order
+        order = []
         if u'presentation' in fields and root.presentation != u'':
             order = unicode(root.presentation)
             order = order.split()
         else:
-            assert len(our_verse_order)>0
+            if len(our_verse_order) > 0:
-            order = our_verse_order
+                order = our_verse_order
+            else:
+                log.warn(u'No verse order available for %s, skipping.', self.title)
         for tag in order:
             if len(tag) == 1:
                 tag = tag + u'1' # Assume it's no.1 if it's not there
             if not versetags.has_key(tag):
                 log.warn(u'Got order %s but not in versetags, skipping', tag)
             else:
-                self.song_import.verse_order_list.append(tag)
+                self.verse_order_list.append(tag)
-
-    def finish(self):
-        """ Separate function, allows test suite to not pollute database"""
-        self.song_import.finish()

openlp/plugins/songs/lib/songimport.py

@@ -30,7 +30,7 @@ from PyQt4 import QtCore
 
 from openlp.core.lib import Receiver, translate
 from openlp.plugins.songs.lib import VerseType
-from openlp.plugins.songs.lib.db import Song, Author, Topic, Book
+from openlp.plugins.songs.lib.db import Song, Author, Topic, Book, MediaFile
 from openlp.plugins.songs.lib.xml import SongXMLBuilder
 
 log = logging.getLogger(__name__)
@@ -66,6 +66,7 @@ class SongImport(QtCore.QObject):
         self.ccli_number = u''
         self.authors = []
         self.topics = []
+        self.media_files = []
         self.song_book_name = u''
         self.song_book_pub = u''
         self.verse_order_list = []
@@ -184,6 +185,14 @@ class SongImport(QtCore.QObject):
             return
         self.authors.append(author)
 
+    def add_media_file(self, filename):
+        """
+        Add a media file to the list
+        """
+        if filename in self.media_files:
+            return
+        self.media_files.append(filename)
+
     def add_verse(self, verse, versetag=None):
         """
         Add a verse. This is the whole verse, lines split by \n
@@ -279,11 +288,16 @@ class SongImport(QtCore.QObject):
         for authortext in self.authors:
             author = self.manager.get_object_filtered(Author,
                 Author.display_name == authortext)
-            if author is None:
+            if not author:
                 author = Author.populate(display_name = authortext,
                     last_name=authortext.split(u' ')[-1],
                     first_name=u' '.join(authortext.split(u' ')[:-1]))
             song.authors.append(author)
+        for filename in self.media_files:
+            media_file = self.manager.get_object_filtered(MediaFile,
+                MediaFile.file_name == filename)
+            if not media_file:
+                song.media_files.append(MediaFile.populate(file_name=filename))
         if self.song_book_name:
             song_book = self.manager.get_object_filtered(Book,
                 Book.name == self.song_book_name)