raoul
/
openlp
forked from openlp/openlp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge in the openlp.org 1.x importer. 

bzr-revno: 1017
This commit is contained in:
Raoul Snyman 2010-09-08 06:57:13 +02:00
parent 1dd5ee185c f30aecfd46
commit b8893f4ab2
5 changed files with 287 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
openlp/plugins/songs/forms/songimportform.py
View File

@ -31,7 +31,6 @@ from PyQt4 import QtCore, QtGui
from songimportwizard import Ui_SongImportWizard
from openlp.core.lib import Receiver, SettingsManager, translate
#from openlp.core.utils import AppLocation
from openlp.plugins.songs.lib.importer import SongFormat
log = logging.getLogger(__name__)
@ -136,7 +135,7 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard):
self.openLP2BrowseButton.setFocus()
return False
elif source_format == SongFormat.OpenLP1:
if self.openSongFilenameEdit.text().isEmpty():
if self.openLP1FilenameEdit.text().isEmpty():
QtGui.QMessageBox.critical(self,
translate('SongsPlugin.ImportWizardForm',
'No openlp.org 1.x Song Database Selected'),
@ -292,7 +291,7 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard):
def onCCLIRemoveButtonClicked(self):
self.removeSelectedItems(self.ccliFileListWidget)
def onSongsOfFellowshipAddButtonClicked(self):
self.getFiles(
translate('SongsPlugin.ImportWizardForm',
@ -374,11 +373,11 @@ class ImportWizardForm(QtGui.QWizard, Ui_SongImportWizard):
importer = self.plugin.importSongs(SongFormat.OpenLP2,
filename=unicode(self.openLP2FilenameEdit.text())
)
#elif source_format == SongFormat.OpenLP1:
# # Import an openlp.org database
# importer = self.plugin.importSongs(SongFormat.OpenLP1,
# filename=unicode(self.field(u'openlp1_filename').toString())
# )
elif source_format == SongFormat.OpenLP1:
# Import an openlp.org database
importer = self.plugin.importSongs(SongFormat.OpenLP1,
filename=unicode(self.openLP1FilenameEdit.text())
)
elif source_format == SongFormat.OpenLyrics:
# Import OpenLyrics songs
importer = self.plugin.importSongs(SongFormat.OpenLyrics,

5
openlp/plugins/songs/lib/importer.py
View File

@ -26,6 +26,7 @@
from opensongimport import OpenSongImport
from olpimport import OpenLPSongImport
from olp1import import OpenLP1SongImport
try:
from sofimport import SofImport
from oooimport import OooImport
@ -61,6 +62,8 @@ class SongFormat(object):
"""
if format == SongFormat.OpenLP2:
return OpenLPSongImport
if format == SongFormat.OpenLP1:
return OpenLP1SongImport
elif format == SongFormat.OpenSong:
return OpenSongImport
elif format == SongFormat.SongsOfFellowship:
@ -70,7 +73,7 @@ class SongFormat(object):
elif format == SongFormat.Generic:
return OooImport
elif format == SongFormat.CCLI:
return CCLIFileImport
return CCLIFileImport
# else:
return None

126
openlp/plugins/songs/lib/olp1import.py Normal file
View File

@ -0,0 +1,126 @@
# -*- coding: utf-8 -*-
# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
###############################################################################
# OpenLP - Open Source Lyrics Projection #
# --------------------------------------------------------------------------- #
# Copyright (c) 2008-2010 Raoul Snyman #
# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael #
# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian #
# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, #
# Carsten Tinggaard, Frode Woldsund #
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it #
# under the terms of the GNU General Public License as published by the Free #
# Software Foundation; version 2 of the License. #
# #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
# more details. #
# #
# You should have received a copy of the GNU General Public License along #
# with this program; if not, write to the Free Software Foundation, Inc., 59 #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
###############################################################################
"""
The :mod:`olp1import` module provides the functionality for importing
openlp.org 1.x song databases into the current installation database.
"""
import logging
import sqlite
from openlp.core.lib import translate
from songimport import SongImport
log = logging.getLogger(__name__)
class OpenLP1SongImport(SongImport):
"""
The :class:`OpenLP1SongImport` class provides OpenLP with the ability to
import song databases from installations of openlp.org 1.x.
"""
def __init__(self, manager, **kwargs):
"""
Initialise the import.
``manager``
The song manager for the running OpenLP installation.
``filename``
The database providing the data to import.
"""
SongImport.__init__(self, manager)
self.import_source = kwargs[u'filename']
def do_import(self):
"""
Run the import for an openlp.org 1.x song database.
"""
# Connect to the database
connection = sqlite.connect(self.import_source)
cursor = connection.cursor()
# Count the number of records we need to import, for the progress bar
cursor.execute(u'SELECT COUNT(songid) FROM songs')
count = int(cursor.fetchone()[0])
success = True
self.import_wizard.importProgressBar.setMaximum(count)
# "cache" our list of authors
cursor.execute(u'SELECT authorid, authorname FROM authors')
authors = cursor.fetchall()
# "cache" our list of tracks
cursor.execute(u'SELECT trackid, fulltrackname FROM tracks')
tracks = cursor.fetchall()
# Import the songs
cursor.execute(u'SELECT songid, songtitle, lyrics || \'\' AS lyrics, '
u'copyrightinfo FROM songs')
songs = cursor.fetchall()
for song in songs:
self.set_defaults()
if self.stop_import_flag:
success = False
break
song_id = song[0]
title = unicode(song[1], u'cp1252')
lyrics = unicode(song[2], u'cp1252').replace(u'\r', u'')
copyright = unicode(song[3], u'cp1252')
self.import_wizard.incrementProgressBar(
unicode(translate('SongsPlugin.ImportWizardForm',
'Importing "%s"...')) % title)
self.title = title
self.process_song_text(lyrics)
self.add_copyright(copyright)
cursor.execute(u'SELECT authorid FROM songauthors '
u'WHERE songid = %s' % song_id)
author_ids = cursor.fetchall()
for author_id in author_ids:
if self.stop_import_flag:
success = False
break
for author in authors:
if author[0] == author_id[0]:
self.parse_author(unicode(author[1], u'cp1252'))
break
if self.stop_import_flag:
success = False
break
cursor.execute(u'SELECT name FROM sqlite_master '
u'WHERE type = \'table\' AND name = \'tracks\'')
table_list = cursor.fetchall()
if len(table_list) > 0:
cursor.execute(u'SELECT trackid FROM songtracks '
u'WHERE songid = %s ORDER BY listindex' % song_id)
track_ids = cursor.fetchall()
for track_id in track_ids:
if self.stop_import_flag:
success = False
break
for track in tracks:
if track[0] == track_id[0]:
self.add_media_file(unicode(track[1], u'cp1252'))
break
if self.stop_import_flag:
success = False
break
self.finish()
return success

223
openlp/plugins/songs/lib/opensongimport.py
View File

@ -28,6 +28,7 @@ import logging
import os
from zipfile import ZipFile
from lxml import objectify
from lxml.etree import Error, LxmlError
from openlp.plugins.songs.lib.songimport import SongImport
@ -36,119 +37,163 @@ log = logging.getLogger(__name__)
class OpenSongImportError(Exception):
pass
class OpenSongImport(object):
class OpenSongImport(SongImport):
"""
Import songs exported from OpenSong - the format is described loosly here:
http://www.opensong.org/d/manual/song_file_format_specification
Import songs exported from OpenSong
However, it doesn't describe the <lyrics> section, so here's an attempt:
The format is described loosly on the `OpenSong File Format Specification
<http://www.opensong.org/d/manual/song_file_format_specification>`_ page on
the OpenSong web site. However, it doesn't describe the <lyrics> section,
so here's an attempt:
Verses can be expressed in one of 2 ways:
<lyrics>
[v1]List of words
Another Line
Verses can be expressed in one of 2 ways, either in complete verses, or by
line grouping, i.e. grouping all line 1's of a verse together, all line 2's
of a verse together, and so on.
[v2]Some words for the 2nd verse
etc...
</lyrics>
An example of complete verses::
The 'v' can be left out - it is implied
or:
<lyrics>
[V]
1List of words
2Some words for the 2nd Verse
<lyrics>
[v1]
List of words
Another Line
1Another Line
2etc...
</lyrics>
[v2]
Some words for the 2nd verse
etc...
</lyrics>
Either or both forms can be used in one song. The Number does not
necessarily appear at the start of the line
The 'v' in the verse specifiers above can be left out, it is implied.
An example of line grouping::
<lyrics>
[V]
1List of words
2Some words for the 2nd Verse
1Another Line
2etc...
</lyrics>
Either or both forms can be used in one song. The number does not
necessarily appear at the start of the line. Additionally, the [v1] labels
can have either upper or lower case Vs.
The [v1] labels can have either upper or lower case Vs
Other labels can be used also:
C - Chorus
B - Bridge
Guitar chords can be provided 'above' the lyrics (the line is
preceeded by a'.') and _s can be used to signify long-drawn-out
words:
C
Chorus
. A7 Bm
1 Some____ Words
B
Bridge
Chords and _s are removed by this importer.
All verses are imported and tagged appropriately.
The verses etc. are imported and tagged appropriately.
Guitar chords can be provided "above" the lyrics (the line is preceeded by
a period "."), and one or more "_" can be used to signify long-drawn-out
words. Chords and "_" are removed by this importer. For example::
The <presentation> tag is used to populate the OpenLP verse
display order field. The Author and Copyright tags are also
imported to the appropriate places.
. A7 Bm
1 Some____ Words
The <presentation> tag is used to populate the OpenLP verse display order
field. The Author and Copyright tags are also imported to the appropriate
places.
"""
def __init__(self, songmanager):
def __init__(self, manager, **kwargs):
"""
Initialise the class. Requires a songmanager class which
is passed to SongImport for writing song to disk
Initialise the class.
"""
self.songmanager = songmanager
SongImport.__init__(self, manager)
self.filenames = kwargs[u'filenames']
self.song = None
self.commit = True
def do_import(self, filename, commit=True):
def do_import(self):
"""
Import either a single opensong file, or a zipfile
containing multiple opensong files If the commit parameter is
set False, the import will not be committed to the database
(useful for test scripts)
Import either a single opensong file, or a zipfile containing multiple
opensong files. If `self.commit` is set False, the import will not be
committed to the database (useful for test scripts).
"""
ext = os.path.splitext(filename)[1]
if ext.lower() == ".zip":
log.info('Zipfile found %s', filename)
z = ZipFile(filename, u'r')
for song in z.infolist():
parts = os.path.split(song.filename)
if parts[-1] == u'':
#No final part => directory
continue
songfile = z.open(song)
self.do_import_file(songfile)
if commit:
success = True
self.import_wizard.importProgressBar.setMaximum(len(self.filenames))
for filename in self.filenames:
if self.stop_import_flag:
success = False
break
ext = os.path.splitext(filename)[1]
if ext.lower() == u'.zip':
log.debug(u'Zipfile found %s', filename)
z = ZipFile(filename, u'r')
self.import_wizard.importProgressBar.setMaximum(
self.import_wizard.importProgressBar.maximum() +
len(z.infolist()))
for song in z.infolist():
if self.stop_import_flag:
success = False
break
parts = os.path.split(song.filename)
if parts[-1] == u'':
#No final part => directory
continue
self.import_wizard.incrementProgressBar(
unicode(translate('SongsPlugin.ImportWizardForm',
'Importing %s...')) % parts[-1])
songfile = z.open(song)
self.do_import_file(songfile)
if self.commit:
self.finish()
self.set_defaults()
if self.stop_import_flag:
success = False
break
else:
log.info('Direct import %s', filename)
self.import_wizard.incrementProgressBar(
unicode(translate('SongsPlugin.ImportWizardForm',
'Importing %s...')) % os.path.split(filename)[-1])
file = open(filename)
self.do_import_file(file)
if self.commit:
self.finish()
else:
log.info('Direct import %s', filename)
file = open(filename)
self.do_import_file(file)
if commit:
self.finish()
self.set_defaults()
if not self.commit:
self.finish()
return success
def do_import_file(self, file):
"""
Process the OpenSong file - pass in a file-like object,
not a filename
"""
self.song_import = SongImport(self.songmanager)
tree = objectify.parse(file)
"""
self.authors = []
try:
tree = objectify.parse(file)
except Error, LxmlError:
log.exception(u'Error parsing XML')
return
root = tree.getroot()
fields = dir(root)
decode = {u'copyright':self.song_import.add_copyright,
u'ccli':u'ccli_number',
u'author':self.song_import.parse_author,
u'title':u'title',
u'aka':u'alternate_title',
u'hymn_number':u'song_number'}
for (attr, fn_or_string) in decode.items():
decode = {
u'copyright': self.add_copyright,
u'ccli': u'ccli_number',
u'author': self.parse_author,
u'title': u'title',
u'aka': u'alternate_title',
u'hymn_number': u'song_number'
}
for attr, fn_or_string in decode.items():
if attr in fields:
ustring = unicode(root.__getattr__(attr))
if type(fn_or_string) == type(u''):
self.song_import.__setattr__(fn_or_string, ustring)
if isinstance(fn_or_string, basestring):
setattr(self, fn_or_string, ustring)
else:
fn_or_string(ustring)
if u'theme' in fields:
self.song_import.topics.append(unicode(root.theme))
if u'alttheme' in fields:
self.song_import.topics.append(unicode(root.alttheme))
if u'theme' in fields and unicode(root.theme) not in self.topics:
self.topics.append(unicode(root.theme))
if u'alttheme' in fields and unicode(root.alttheme) not in self.topics:
self.topics.append(unicode(root.alttheme))
# data storage while importing
verses = {}
lyrics = unicode(root.lyrics)
@ -158,6 +203,7 @@ class OpenSongImport(object):
# in the absence of any other indication, verses are the default,
# erm, versetype!
versetype = u'V'
versenum = None
for thisline in lyrics.split(u'\n'):
# remove comments
semicolon = thisline.find(u';')
@ -170,7 +216,6 @@ class OpenSongImport(object):
if thisline[0] == u'.' or thisline.startswith(u'---') \
or thisline.startswith(u'-!!'):
continue
# verse/chorus/etc. marker
if thisline[0] == u'[':
versetype = thisline[1].upper()
@ -186,7 +231,6 @@ class OpenSongImport(object):
versenum = u'1'
continue
words = None
# number at start of line.. it's verse number
if thisline[0].isdigit():
versenum = thisline[0]
@ -207,7 +251,7 @@ class OpenSongImport(object):
our_verse_order.append(versetag)
if words:
# Tidy text and remove the ____s from extended words
words = self.song_import.tidy_text(words)
words = self.tidy_text(words)
words = words.replace('_', '')
verses[versetype][versenum].append(words)
# done parsing
@ -220,24 +264,23 @@ class OpenSongImport(object):
for num in versenums:
versetag = u'%s%s' % (versetype, num)
lines = u'\n'.join(verses[versetype][num])
self.song_import.verses.append([versetag, lines])
self.verses.append([versetag, lines])
# Keep track of what we have for error checking later
versetags[versetag] = 1
# now figure out the presentation order
order = []
if u'presentation' in fields and root.presentation != u'':
order = unicode(root.presentation)
order = order.split()
else:
assert len(our_verse_order)>0
order = our_verse_order
if len(our_verse_order) > 0:
order = our_verse_order
else:
log.warn(u'No verse order available for %s, skipping.', self.title)
for tag in order:
if len(tag) == 1:
tag = tag + u'1' # Assume it's no.1 if it's not there
if not versetags.has_key(tag):
log.warn(u'Got order %s but not in versetags, skipping', tag)
else:
self.song_import.verse_order_list.append(tag)
def finish(self):
""" Separate function, allows test suite to not pollute database"""
self.song_import.finish()
self.verse_order_list.append(tag)

20
openlp/plugins/songs/lib/songimport.py
View File

@ -30,7 +30,7 @@ from PyQt4 import QtCore
from openlp.core.lib import Receiver, translate
from openlp.plugins.songs.lib import VerseType
from openlp.plugins.songs.lib.db import Song, Author, Topic, Book
from openlp.plugins.songs.lib.db import Song, Author, Topic, Book, MediaFile
from openlp.plugins.songs.lib.xml import SongXMLBuilder
log = logging.getLogger(__name__)
@ -66,6 +66,7 @@ class SongImport(QtCore.QObject):
self.ccli_number = u''
self.authors = []
self.topics = []
self.media_files = []
self.song_book_name = u''
self.song_book_pub = u''
self.verse_order_list = []
@ -76,7 +77,7 @@ class SongImport(QtCore.QObject):
'SongsPlugin.SongImport', 'copyright'))
self.copyright_symbol = unicode(translate(
'SongsPlugin.SongImport', '\xa9'))
def stop_import(self):
"""
Sets the flag for importers to stop their import
@ -184,6 +185,14 @@ class SongImport(QtCore.QObject):
return
self.authors.append(author)
def add_media_file(self, filename):
"""
Add a media file to the list
"""
if filename in self.media_files:
return
self.media_files.append(filename)
def add_verse(self, verse, versetag=None):
"""
Add a verse. This is the whole verse, lines split by \n
@ -279,11 +288,16 @@ class SongImport(QtCore.QObject):
for authortext in self.authors:
author = self.manager.get_object_filtered(Author,
Author.display_name == authortext)
if author is None:
if not author:
author = Author.populate(display_name = authortext,
last_name=authortext.split(u' ')[-1],
first_name=u' '.join(authortext.split(u' ')[:-1]))
song.authors.append(author)
for filename in self.media_files:
media_file = self.manager.get_object_filtered(MediaFile,
MediaFile.file_name == filename)
if not media_file:
song.media_files.append(MediaFile.populate(file_name=filename))
if self.song_book_name:
song_book = self.manager.get_object_filtered(Book,
Book.name == self.song_book_name)