Passes first tests again

This commit is contained in:
Martin Thompson 2010-08-30 21:19:16 +01:00
commit 733bedebe8
5 changed files with 341 additions and 178 deletions

View File

@ -0,0 +1,131 @@
# -*- coding: utf-8 -*-
# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
###############################################################################
# OpenLP - Open Source Lyrics Projection #
# --------------------------------------------------------------------------- #
# Copyright (c) 2008-2010 Raoul Snyman #
# Portions copyright (c) 2008-2010 Tim Bentley, Jonathan Corwin, Michael #
# Gorven, Scott Guerrieri, Meinert Jordan, Andreas Preikschat, Christian #
# Richter, Philip Ridout, Maikel Stuivenberg, Martin Thompson, Jon Tibble, #
# Carsten Tinggaard, Frode Woldsund #
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it #
# under the terms of the GNU General Public License as published by the Free #
# Software Foundation; version 2 of the License. #
# #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
# more details. #
# #
# You should have received a copy of the GNU General Public License along #
# with this program; if not, write to the Free Software Foundation, Inc., 59 #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
###############################################################################
"""
The :mod:`olp1import` module provides the functionality for importing
openlp.org 1.x song databases into the current installation database.
"""
import logging
import sqlite
#from openlp.core.lib.db import BaseModel
from openlp.plugins.songs.lib.db import Author, Book, Song, Topic #, MediaFile
from songimport import SongImport
log = logging.getLogger(__name__)
class OpenLP1SongImport(SongImport):
"""
The :class:`OpenLP1SongImport` class provides OpenLP with the ability to
import song databases from installations of openlp.org 1.x.
"""
def __init__(self, manager, **kwargs):
"""
Initialise the import.
``manager``
The song manager for the running OpenLP installation.
``filename``
The database providing the data to import.
"""
SongImport.__init__(self, manager)
self.manager = manager
self.import_source = kwargs[u'filename']
def do_import(self):
"""
Run the import for an openlp.org 1.x song database.
"""
connection = sqlite.connect(self.import_source)
cursor = connection.cursor()
# for song in source_songs:
# new_song = Song()
# new_song.title = song.title
# if has_media_files:
# new_song.alternate_title = song.alternate_title
# else:
# old_titles = song.search_title.split(u'@')
# if len(old_titles) > 1:
# new_song.alternate_title = old_titles[1]
# else:
# new_song.alternate_title = u''
# new_song.search_title = song.search_title
# new_song.song_number = song.song_number
# new_song.lyrics = song.lyrics
# new_song.search_lyrics = song.search_lyrics
# new_song.verse_order = song.verse_order
# new_song.copyright = song.copyright
# new_song.comments = song.comments
# new_song.theme_name = song.theme_name
# new_song.ccli_number = song.ccli_number
# if song.authors:
# for author in song.authors:
# existing_author = self.master_manager.get_object_filtered(
# Author, Author.display_name == author.display_name)
# if existing_author:
# new_song.authors.append(existing_author)
# else:
# new_song.authors.append(Author.populate(
# first_name=author.first_name,
# last_name=author.last_name,
# display_name=author.display_name))
# else:
# au = self.master_manager.get_object_filtered(Author,
# Author.display_name == u'Author Unknown')
# if au:
# new_song.authors.append(au)
# else:
# new_song.authors.append(Author.populate(
# display_name=u'Author Unknown'))
# if song.book:
# existing_song_book = self.master_manager.get_object_filtered(
# Book, Book.name == song.book.name)
# if existing_song_book:
# new_song.book = existing_song_book
# else:
# new_song.book = Book.populate(name=song.book.name,
# publisher=song.book.publisher)
# if song.topics:
# for topic in song.topics:
# existing_topic = self.master_manager.get_object_filtered(
# Topic, Topic.name == topic.name)
# if existing_topic:
# new_song.topics.append(existing_topic)
# else:
# new_song.topics.append(Topic.populate(name=topic.name))
## if has_media_files:
## if song.media_files:
## for media_file in song.media_files:
## existing_media_file = \
## self.master_manager.get_object_filtered(MediaFile,
## MediaFile.file_name == media_file.file_name)
## if existing_media_file:
## new_song.media_files.append(existing_media_file)
## else:
## new_song.media_files.append(MediaFile.populate(
## file_name=media_file.file_name))
# self.master_manager.save_object(new_song)

View File

@ -28,6 +28,7 @@ import logging
import os
from zipfile import ZipFile
from lxml import objectify
from lxml.etree import Error, LxmlError
from openlp.plugins.songs.lib.songimport import SongImport
@ -36,119 +37,156 @@ log = logging.getLogger(__name__)
class OpenSongImportError(Exception):
pass
class OpenSongImport(object):
class OpenSongImport(SongImport):
"""
Import songs exported from OpenSong - the format is described loosly here:
http://www.opensong.org/d/manual/song_file_format_specification
Import songs exported from OpenSong
However, it doesn't describe the <lyrics> section, so here's an attempt:
The format is described loosly on the `OpenSong File Format Specification
<http://www.opensong.org/d/manual/song_file_format_specification>`_ page on
the OpenSong web site. However, it doesn't describe the <lyrics> section,
so here's an attempt:
Verses can be expressed in one of 2 ways:
<lyrics>
[v1]List of words
Another Line
Verses can be expressed in one of 2 ways, either in complete verses, or by
line grouping, i.e. grouping all line 1's of a verse together, all line 2's
of a verse together, and so on.
[v2]Some words for the 2nd verse
etc...
</lyrics>
An example of complete verses::
The 'v' can be left out - it is implied
or:
<lyrics>
[V]
1List of words
2Some words for the 2nd Verse
<lyrics>
[v1]
List of words
Another Line
1Another Line
2etc...
</lyrics>
[v2]
Some words for the 2nd verse
etc...
</lyrics>
Either or both forms can be used in one song. The Number does not
necessarily appear at the start of the line
The 'v' in the verse specifiers above can be left out, it is implied.
An example of line grouping::
<lyrics>
[V]
1List of words
2Some words for the 2nd Verse
1Another Line
2etc...
</lyrics>
Either or both forms can be used in one song. The number does not
necessarily appear at the start of the line. Additionally, the [v1] labels
can have either upper or lower case Vs.
The [v1] labels can have either upper or lower case Vs
Other labels can be used also:
C - Chorus
B - Bridge
Guitar chords can be provided 'above' the lyrics (the line is
preceeded by a'.') and _s can be used to signify long-drawn-out
words:
C
Chorus
. A7 Bm
1 Some____ Words
B
Bridge
Chords and _s are removed by this importer.
All verses are imported and tagged appropriately.
The verses etc. are imported and tagged appropriately.
Guitar chords can be provided "above" the lyrics (the line is preceeded by
a period "."), and one or more "_" can be used to signify long-drawn-out
words. Chords and "_" are removed by this importer. For example::
The <presentation> tag is used to populate the OpenLP verse
display order field. The Author and Copyright tags are also
imported to the appropriate places.
. A7 Bm
1 Some____ Words
The <presentation> tag is used to populate the OpenLP verse display order
field. The Author and Copyright tags are also imported to the appropriate
places.
"""
def __init__(self, songmanager):
def __init__(self, manager, **kwargs):
"""
Initialise the class. Requires a songmanager class which
is passed to SongImport for writing song to disk
Initialise the class.
"""
self.songmanager = songmanager
SongImport.__init__(self, manager)
self.filenames = kwargs[u'filenames']
self.song = None
self.commit = True
def do_import(self, filename, commit=True):
def do_import(self):
"""
Import either a single opensong file, or a zipfile
containing multiple opensong files If the commit parameter is
set False, the import will not be committed to the database
(useful for test scripts)
Import either each of the files in self.filenames - each element of
which can be either a single opensong file, or a zipfile containing
multiple opensong files. If `self.commit` is set False, the
import will not be committed to the database (useful for test scripts).
"""
ext = os.path.splitext(filename)[1]
if ext.lower() == ".zip":
log.info('Zipfile found %s', filename)
z = ZipFile(filename, u'r')
for song in z.infolist():
parts = os.path.split(song.filename)
if parts[-1] == u'':
#No final part => directory
continue
songfile = z.open(song)
self.do_import_file(songfile)
if commit:
success = False
self.import_wizard.importProgressBar.setMaximum(len(self.filenames))
for filename in self.filenames:
if self.stop_import_flag:
break
ext = os.path.splitext(filename)[1]
if ext.lower() == u'.zip':
log.info(u'Zipfile found %s', filename)
z = ZipFile(filename, u'r')
for song in z.infolist():
if self.stop_import_flag:
break
parts = os.path.split(song.filename)
if parts[-1] == u'':
#No final part => directory
continue
log.info(u'Zip importing %s', parts[-1])
self.import_wizard.incrementProgressBar(u'Importing %s...' \
% parts[-1])
songfile = z.open(song)
self.do_import_file(songfile)
if self.commit:
self.finish()
if self.stop_import_flag:
break
else:
log.info('Direct import %s', filename)
self.import_wizard.incrementProgressBar(u'Importing %s...' \
% os.path.split(filename)[-1])
file = open(filename)
self.do_import_file(file)
if self.commit:
self.finish()
else:
log.info('Direct import %s', filename)
file = open(filename)
self.do_import_file(file)
if commit:
self.finish()
if not self.commit:
self.finish()
def do_import_file(self, file):
"""
Process the OpenSong file - pass in a file-like object,
not a filename
"""
self.song_import = SongImport(self.songmanager)
tree = objectify.parse(file)
"""
self.authors = []
self.verse_order_list = []
try:
tree = objectify.parse(file)
except Error, LxmlError:
log.exception(u'Error parsing XML')
return
root = tree.getroot()
fields = dir(root)
decode = {u'copyright':self.song_import.add_copyright,
u'ccli':u'ccli_number',
u'author':self.song_import.parse_author,
u'title':u'title',
u'aka':u'alternate_title',
u'hymn_number':u'song_number'}
for (attr, fn_or_string) in decode.items():
decode = {
u'copyright': self.add_copyright,
u'ccli': u'ccli_number',
u'author': self.parse_author,
u'title': u'title',
u'aka': u'alternate_title',
u'hymn_number': u'song_number'
}
for attr, fn_or_string in decode.items():
if attr in fields:
ustring = unicode(root.__getattr__(attr))
if type(fn_or_string) == type(u''):
self.song_import.__setattr__(fn_or_string, ustring)
if isinstance(fn_or_string, basestring):
setattr(self, fn_or_string, ustring)
else:
fn_or_string(ustring)
if u'theme' in fields:
self.song_import.topics.append(unicode(root.theme))
if u'alttheme' in fields:
self.song_import.topics.append(unicode(root.alttheme))
if u'theme' in fields and unicode(root.theme) not in self.topics:
self.topics.append(unicode(root.theme))
if u'alttheme' in fields and unicode(root.alttheme) not in self.topics:
self.topics.append(unicode(root.alttheme))
# data storage while importing
verses = {}
lyrics = unicode(root.lyrics)
@ -158,6 +196,7 @@ class OpenSongImport(object):
# in the absence of any other indication, verses are the default,
# erm, versetype!
versetype = u'V'
versenum = None
for thisline in lyrics.split(u'\n'):
# remove comments
semicolon = thisline.find(u';')
@ -170,7 +209,6 @@ class OpenSongImport(object):
if thisline[0] == u'.' or thisline.startswith(u'---') \
or thisline.startswith(u'-!!'):
continue
# verse/chorus/etc. marker
if thisline[0] == u'[':
versetype = thisline[1].upper()
@ -186,7 +224,6 @@ class OpenSongImport(object):
versenum = u'1'
continue
words = None
# number at start of line.. it's verse number
if thisline[0].isdigit():
versenum = thisline[0]
@ -207,7 +244,7 @@ class OpenSongImport(object):
our_verse_order.append(versetag)
if words:
# Tidy text and remove the ____s from extended words
words = self.song_import.tidy_text(words)
words = self.tidy_text(words)
words = words.replace('_', '')
verses[versetype][versenum].append(words)
# done parsing
@ -220,24 +257,23 @@ class OpenSongImport(object):
for num in versenums:
versetag = u'%s%s' % (versetype, num)
lines = u'\n'.join(verses[versetype][num])
self.song_import.verses.append([versetag, lines])
self.verses.append([versetag, lines])
# Keep track of what we have for error checking later
versetags[versetag] = 1
# now figure out the presentation order
order = []
if u'presentation' in fields and root.presentation != u'':
order = unicode(root.presentation)
order = order.split()
else:
assert len(our_verse_order)>0
order = our_verse_order
if len(our_verse_order) > 0:
order = our_verse_order
else:
log.warn(u'No verse order available for %s, skipping.', self.title)
for tag in order:
if len(tag) == 1:
tag = tag + u'1' # Assume it's no.1 if it's not there
if not versetags.has_key(tag):
log.warn(u'Got order %s but not in versetags, skipping', tag)
else:
self.song_import.verse_order_list.append(tag)
def finish(self):
""" Separate function, allows test suite to not pollute database"""
self.song_import.finish()
self.verse_order_list.append(tag)

View File

@ -158,8 +158,7 @@ class SongImport(QtCore.QObject):
def parse_author(self, text):
"""
Add the author. OpenLP stores them individually so split by 'and', '&'
and comma.
However need to check for 'Mr and Mrs Smith' and turn it to
and comma. However need to check for 'Mr and Mrs Smith' and turn it to
'Mr Smith' and 'Mrs Smith'.
"""
for author in text.split(u','):
@ -236,7 +235,7 @@ class SongImport(QtCore.QObject):
"""
All fields have been set to this song. Write it away
"""
if len(self.authors) == 0:
if not self.authors:
self.authors.append(u'Author unknown')
self.commit_song()

View File

@ -8,50 +8,25 @@ from traceback import print_exc
import sys
import codecs
import logging
LOG_FILENAME = 'import.log'
logging.basicConfig(filename=LOG_FILENAME,level=logging.INFO)
from test_opensongimport import wizard_stub, progbar_stub
def opensong_import_lots():
ziploc = u'/home/mjt/openlp/OpenSong_Data/'
files = []
#files = [u'test.opensong.zip', ziploc+u'ADond.zip']
files.extend(glob(ziploc+u'Songs.zip'))
# files.extend(glob(ziploc+u'Songs.zip'))
files.extend(glob(ziploc+u'RaoulSongs.zip'))
#files.extend(glob(ziploc+u'SOF.zip'))
#files.extend(glob(ziploc+u'spanish_songs_for_opensong.zip'))
# files.extend(glob(ziploc+u'opensong_*.zip'))
errfile = codecs.open(u'import_lots_errors.txt', u'w', u'utf8')
manager = Manager(u'songs', init_schema)
for file in files:
print u'Importing', file
z = ZipFile(file, u'r')
for song in z.infolist():
# need to handle unicode filenames (CP437 - Winzip does this)
filename = song.filename#.decode('cp852')
parts = os.path.split(filename)
if parts[-1] == u'':
#No final part => directory
continue
print " ", file, ":",filename,
songfile = z.open(song)
#z.extract(song)
#songfile=open(filename, u'r')
o = OpenSongImport(manager)
try:
o.do_import_file(songfile)
# o.song_import.print_song()
except:
print "Failure",
errfile.write(u'Failure: %s:%s\n' %(file, filename.decode('cp437')))
songfile = z.open(song)
for l in songfile.readlines():
l = l.decode('utf8')
print(u' |%s\n' % l.strip())
errfile.write(u' |%s\n'%l.strip())
print_exc(3, file = errfile)
print_exc(3)
sys.exit(1)
# continue
#o.finish()
print "OK"
#os.unlink(filename)
# o.song_import.print_song()
o = OpenSongImport(manager, filenames=files)
o.import_wizard=wizard_stub()
o.do_import()
if __name__ == "__main__":
opensong_import_lots()

View File

@ -28,59 +28,81 @@ from openlp.plugins.songs.lib.opensongimport import OpenSongImport
from openlp.core.lib.db import Manager
from openlp.plugins.songs.lib.db import init_schema
import logging
LOG_FILENAME = 'test.log'
logging.basicConfig(filename=LOG_FILENAME,level=logging.INFO)
# Stubs to replace the UI functions for raw testing
class wizard_stub:
def __init__(self):
self.importProgressBar=progbar_stub()
def incrementProgressBar(self, str):
pass
class progbar_stub:
def __init__(self):
pass
def setMaximum(self, arg):
pass
def test():
manager = Manager(u'songs', init_schema)
o = OpenSongImport(manager)
o.do_import(u'test.opensong', commit=False)
o.song_import.print_song()
assert o.song_import.copyright == u'2010 Martin Thompson'
assert o.song_import.authors == [u'MartiÑ Thómpson']
assert o.song_import.title == u'Martins Test'
assert o.song_import.alternate_title == u''
assert o.song_import.song_number == u'1'
assert [u'C1', u'Chorus 1'] in o.song_import.verses
assert [u'C2', u'Chorus 2'] in o.song_import.verses
assert not [u'C3', u'Chorus 3'] in o.song_import.verses
assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses
assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses
assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses
assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1']
assert o.song_import.ccli_number == u'Blah'
assert o.song_import.topics == [u'TestTheme', u'TestAltTheme']
o.do_import(u'test.opensong.zip', commit=False)
o.song_import.print_song()
o.finish()
assert o.song_import.copyright == u'2010 Martin Thompson'
assert o.song_import.authors == [u'MartiÑ Thómpson']
assert o.song_import.title == u'Martins Test'
assert o.song_import.alternate_title == u''
assert o.song_import.song_number == u'1'
assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses
assert [u'C1', u'Chorus 1'] in o.song_import.verses
assert [u'C2', u'Chorus 2'] in o.song_import.verses
assert not [u'C3', u'Chorus 3'] in o.song_import.verses
assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses
assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses
assert o.song_import.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1']
o = OpenSongImport(manager, filenames=[u'test.opensong'])
o.import_wizard = wizard_stub()
o.commit = False
o.do_import()
o.print_song()
assert o.copyright == u'2010 Martin Thompson'
assert o.authors == [u'MartiÑ Thómpson']
assert o.title == u'Martins Test'
assert o.alternate_title == u''
assert o.song_number == u'1'
assert [u'C1', u'Chorus 1'] in o.verses
assert [u'C2', u'Chorus 2'] in o.verses
assert not [u'C3', u'Chorus 3'] in o.verses
assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.verses
assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.verses
assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.verses
assert o.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1']
assert o.ccli_number == u'Blah'
assert o.topics == [u'TestTheme', u'TestAltTheme']
o = OpenSongImport(manager)
o.do_import(u'test2.opensong', commit=False)
o.filenames = [u'test.opensong.zip']
o.do_import()
o.print_song()
o.finish()
assert o.copyright == u'2010 Martin Thompson'
assert o.authors == [u'MartiÑ Thómpson']
assert o.title == u'Martins Test'
assert o.alternate_title == u''
assert o.song_number == u'1'
assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.verses
assert [u'C1', u'Chorus 1'] in o.verses
assert [u'C2', u'Chorus 2'] in o.verses
assert not [u'C3', u'Chorus 3'] in o.verses
assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.verses
assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.verses
print o.verse_order_list
assert o.verse_order_list == [u'V1', u'C1', u'V2', u'C2', u'V3', u'B1', u'V1']
o.filenames = [u'test2.opensong']
o.do_import()
# o.finish()
o.song_import.print_song()
assert o.song_import.copyright == u'2010 Martin Thompson'
assert o.song_import.authors == [u'Martin Thompson']
assert o.song_import.title == u'Martins 2nd Test'
assert o.song_import.alternate_title == u''
assert o.song_import.song_number == u'2'
print o.song_import.verses
assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.song_import.verses
assert [u'C1', u'Chorus 1'] in o.song_import.verses
assert [u'C2', u'Chorus 2'] in o.song_import.verses
assert not [u'C3', u'Chorus 3'] in o.song_import.verses
assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.song_import.verses
assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.song_import.verses
print o.song_import.verse_order_list
assert o.song_import.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2']
o.print_song()
assert o.copyright == u'2010 Martin Thompson'
assert o.authors == [u'Martin Thompson']
assert o.title == u'Martins 2nd Test'
assert o.alternate_title == u''
assert o.song_number == u'2'
print o.verses
assert [u'B1', u'Bridge 1\nBridge 1 line 2'] in o.verses
assert [u'C1', u'Chorus 1'] in o.verses
assert [u'C2', u'Chorus 2'] in o.verses
assert not [u'C3', u'Chorus 3'] in o.verses
assert [u'V1', u'v1 Line 1\nV1 Line 2'] in o.verses
assert [u'V2', u'v2 Line 1\nV2 Line 2'] in o.verses
print o.verse_order_list
assert o.verse_order_list == [u'V1', u'V2', u'B1', u'C1', u'C2']
print "Tests passed"
pass