raoul/openlp
1
0
Fork 0
forked from openlp/openlp
Code Issues Pull Requests Projects Releases Wiki Activity

Fix CSV bible importing (bug #687309)

Browse Source 
bzr-revno: 1250
This commit is contained in:
Jon Tibble 2011-01-28 13:39:08 +00:00
commit 972442a14c
8 changed files with 179 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
openlp/plugins/bibles/forms/bibleimportform.py
View File

@ -124,9 +124,12 @@ class BibleImportForm(OpenLPWizard):
QtCore.QObject.connect(self.osisBrowseButton,
QtCore.SIGNAL(u'clicked()'),
self.onOsisBrowseButtonClicked)
QtCore.QObject.connect(self.csvTestamentsButton,
QtCore.SIGNAL(u'clicked()'),
self.onCsvTestamentsBrowseButtonClicked)
QtCore.QObject.connect(self.csvBooksButton,
QtCore.SIGNAL(u'clicked()'),
self.onBooksBrowseButtonClicked)
self.onCsvBooksBrowseButtonClicked)
QtCore.QObject.connect(self.csvVersesButton,
QtCore.SIGNAL(u'clicked()'),
self.onCsvVersesBrowseButtonClicked)
@ -187,6 +190,18 @@ class BibleImportForm(OpenLPWizard):
self.csvLayout = QtGui.QFormLayout(self.csvWidget)
self.csvLayout.setMargin(0)
self.csvLayout.setObjectName(u'CsvLayout')
self.csvTestamentsLabel = QtGui.QLabel(self.csvWidget)
self.csvTestamentsLabel.setObjectName(u'CsvTestamentsLabel')
self.csvTestamentsLayout = QtGui.QHBoxLayout()
self.csvTestamentsLayout.setObjectName(u'CsvTestamentsLayout')
self.csvTestamentsEdit = QtGui.QLineEdit(self.csvWidget)
self.csvTestamentsEdit.setObjectName(u'CsvTestamentsEdit')
self.csvTestamentsLayout.addWidget(self.csvTestamentsEdit)
self.csvTestamentsButton = QtGui.QToolButton(self.csvWidget)
self.csvTestamentsButton.setIcon(self.openIcon)
self.csvTestamentsButton.setObjectName(u'CsvTestamentsButton')
self.csvTestamentsLayout.addWidget(self.csvTestamentsButton)
self.csvLayout.addRow(self.csvTestamentsLabel, self.csvTestamentsLayout)
self.csvBooksLabel = QtGui.QLabel(self.csvWidget)
self.csvBooksLabel.setObjectName(u'CsvBooksLabel')
self.csvBooksLayout = QtGui.QHBoxLayout()
@ -213,7 +228,7 @@ class BibleImportForm(OpenLPWizard):
self.csvLayout.addRow(self.csvVersesLabel, self.csvVersesLayout)
self.csvSpacer = QtGui.QSpacerItem(10, 0, QtGui.QSizePolicy.Fixed,
QtGui.QSizePolicy.Minimum)
self.csvLayout.setItem(2, QtGui.QFormLayout.LabelRole, self.csvSpacer)
self.csvLayout.setItem(3, QtGui.QFormLayout.LabelRole, self.csvSpacer)
self.selectStack.addWidget(self.csvWidget)
self.openSongWidget = QtGui.QWidget(self.selectPage)
self.openSongWidget.setObjectName(u'OpenSongWidget')
@ -389,6 +404,8 @@ class BibleImportForm(OpenLPWizard):
translate('BiblesPlugin.ImportWizardForm', 'File location:'))
self.osisFileLabel.setText(
translate('BiblesPlugin.ImportWizardForm', 'File location:'))
self.csvTestamentsLabel.setText(
translate('BiblesPlugin.ImportWizardForm', 'Testaments location:'))
self.csvBooksLabel.setText(
translate('BiblesPlugin.ImportWizardForm', 'Books location:'))
self.csvVersesLabel.setText(
@ -478,7 +495,16 @@ class BibleImportForm(OpenLPWizard):
self.osisFileEdit.setFocus()
return False
elif self.field(u'source_format').toInt()[0] == BibleFormat.CSV:
if not self.field(u'csv_booksfile').toString():
if not self.field(u'csv_testamentsfile').toString():
answer = criticalErrorMessageBox(translate(
'BiblesPlugin.ImportWizardForm', 'No Testaments File'),
translate('BiblesPlugin.ImportWizardForm',
'You have not specified a testaments file. Do you '
'want to proceed with the import?'), question=True)
if answer == QtGui.QMessageBox.No:
self.csvTestamentsEdit.setFocus()
return False
elif not self.field(u'csv_booksfile').toString():
criticalErrorMessageBox(
translate('BiblesPlugin.ImportWizardForm',
'Invalid Books File'),
@ -572,7 +598,15 @@ class BibleImportForm(OpenLPWizard):
translate('BiblesPlugin.ImportWizardForm', 'Open OSIS File'),
self.osisFileEdit)
def onBooksBrowseButtonClicked(self):
def onCsvTestamentsBrowseButtonClicked(self):
"""
Show the file open dialog for the testaments CSV file.
"""
self.getFileName(translate('BiblesPlugin.ImportWizardForm',
'Open Testaments CSV File'), self.csvTestamentsEdit, u'%s (*.csv)'
% translate('BiblesPlugin.ImportWizardForm', 'CSV File'))
def onCsvBooksBrowseButtonClicked(self):
"""
Show the file open dialog for the books CSV file.
"""
@ -613,12 +647,14 @@ class BibleImportForm(OpenLPWizard):
"""
self.selectPage.registerField(u'source_format', self.formatComboBox)
self.selectPage.registerField(u'osis_location', self.osisFileEdit)
self.selectPage.registerField(
u'csv_testamentsfile', self.csvTestamentsEdit)
self.selectPage.registerField(u'csv_booksfile', self.csvBooksEdit)
self.selectPage.registerField(u'csv_versefile', self.csvVersesEdit)
self.selectPage.registerField(u'opensong_file', self.openSongFileEdit)
self.selectPage.registerField(u'web_location', self.webSourceComboBox)
self.selectPage.registerField(u'web_biblename',
self.webTranslationComboBox)
self.selectPage.registerField(
u'web_biblename', self.webTranslationComboBox)
self.selectPage.registerField(u'proxy_server', self.webServerEdit)
self.selectPage.registerField(u'proxy_username', self.webUserEdit)
self.selectPage.registerField(u'proxy_password', self.webPasswordEdit)
@ -641,6 +677,7 @@ class BibleImportForm(OpenLPWizard):
self.cancelButton.setVisible(True)
self.setField(u'source_format', QtCore.QVariant(0))
self.setField(u'osis_location', QtCore.QVariant(''))
self.setField(u'csv_testamentsfile', QtCore.QVariant(''))
self.setField(u'csv_booksfile', QtCore.QVariant(''))
self.setField(u'csv_versefile', QtCore.QVariant(''))
self.setField(u'opensong_file', QtCore.QVariant(''))
@ -770,7 +807,8 @@ class BibleImportForm(OpenLPWizard):
elif bible_type == BibleFormat.CSV:
# Import a CSV bible.
importer = self.manager.import_bible(BibleFormat.CSV,
name=license_version,
name=license_version, testamentsfile=unicode(
self.field(u'csv_testamentsfile').toString()),
booksfile=unicode(self.field(u'csv_booksfile').toString()),
versefile=unicode(self.field(u'csv_versefile').toString())
)
@ -795,8 +833,7 @@ class BibleImportForm(OpenLPWizard):
bible = \
self.web_bible_list[WebDownload.Bibleserver][bible_version]
importer = self.manager.import_bible(
BibleFormat.WebDownload,
name=license_version,
BibleFormat.WebDownload, name=license_version,
download_source=WebDownload.get_name(download_location),
download_name=bible,
proxy_server=unicode(self.field(u'proxy_server').toString()),

150
openlp/plugins/bibles/lib/csvbible.py
View File

@ -23,7 +23,45 @@
# with this program; if not, write to the Free Software Foundation, Inc., 59 #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
###############################################################################
"""
The :mod:`cvsbible` modules provides a facility to import bibles from a set of
CSV files.
The module expects two mandatory files containing the books and the verses and
will accept an optional third file containing the testaments.
The format of the testament file is:
<testament_id>,<testament_name>
For example:
1,Old Testament
2,New Testament
The format of the books file is:
<book_id>,<testament_id>,<book_name>,<book_abbreviation>
For example
1,1,Genesis,Gen
2,1,Exodus,Exod
...
40,2,Matthew,Matt
The format of the verses file is:
<book_id>,<chapter_number>,<verse_number>,<verse_text>
For example:
1,1,1,"In the beginning God created the heaven and the earth."
1,1,2,"And the earth was without form, and void; and darkness...."
All CSV files are expected to use a comma (',') as the delimeter and double
quotes ('"') as the quote symbol.
"""
import logging
import chardet
import csv
@ -31,7 +69,7 @@ import csv
from PyQt4 import QtCore
from openlp.core.lib import Receiver, translate
from openlp.plugins.bibles.lib.db import BibleDB
from openlp.plugins.bibles.lib.db import BibleDB, Testament
log = logging.getLogger(__name__)
@ -39,68 +77,113 @@ class CSVBible(BibleDB):
"""
This class provides a specialisation for importing of CSV Bibles.
"""
def __init__(self, parent, **kwargs):
"""
Loads a Bible from a pair of CVS files passed in
Loads a Bible from a set of CVS files.
This class assumes the files contain all the information and
a clean bible is being loaded.
"""
log.info(self.__class__.__name__)
BibleDB.__init__(self, parent, **kwargs)
try:
self.testamentsfile = kwargs[u'testamentsfile']
except KeyError:
self.testamentsfile = None
self.booksfile = kwargs[u'booksfile']
self.versesfile = kwargs[u'versefile']
QtCore.QObject.connect(Receiver.get_receiver(),
QtCore.SIGNAL(u'openlp_stop_wizard'), self.stop_import)
def setup_testaments(self):
"""
Overrides parent method so we can handle importing a testament file.
"""
if self.testamentsfile:
self.wizard.progressBar.setMinimum(0)
self.wizard.progressBar.setMaximum(2)
self.wizard.progressBar.setValue(0)
testaments_file = None
try:
details = get_file_encoding(self.testamentsfile)
testaments_file = open(self.testamentsfile, 'rb')
testaments_reader = csv.reader(testaments_file, delimiter=',',
quotechar='"')
for line in testaments_reader:
if self.stop_import_flag:
break
self.wizard.incrementProgressBar(unicode(
translate('BibleDB.Wizard',
'Importing testaments... %s')) %
unicode(line[1], details['encoding']), 0)
self.save_object(Testament.populate(
name=unicode(line[1], details['encoding'])))
Receiver.send_message(u'openlp_process_events')
except (IOError, IndexError):
log.exception(u'Loading testaments from file failed')
finally:
if testaments_file:
testaments_file.close()
self.wizard.incrementProgressBar(unicode(translate(
'BibleDB.Wizard', 'Importing testaments... done.')), 2)
else:
BibleDB.setup_testaments(self)
def do_import(self):
"""
Import the bible books and verses.
"""
self.wizard.progressBar.setValue(0)
self.wizard.progressBar.setMinimum(0)
self.wizard.progressBar.setMaximum(66)
success = True
books_file = None
book_ptr = None
verse_file = None
book_list = {}
# Populate the Tables
try:
details = get_file_encoding(self.booksfile)
books_file = open(self.booksfile, 'r')
dialect = csv.Sniffer().sniff(books_file.read(1024))
books_file.seek(0)
books_reader = csv.reader(books_file, dialect)
books_reader = csv.reader(books_file, delimiter=',', quotechar='"')
for line in books_reader:
# cancel pressed
if self.stop_import_flag:
break
details = chardet.detect(line[1])
self.create_book(unicode(line[1], details['encoding']),
line[2], int(line[0]))
Receiver.send_message(u'openlp_process_events')
self.wizard.incrementProgressBar(unicode(
translate('BibleDB.Wizard', 'Importing books... %s')) %
unicode(line[2], details['encoding']))
self.create_book(unicode(line[2], details['encoding']),
unicode(line[3], details['encoding']), int(line[1]))
book_list[int(line[0])] = unicode(line[2], details['encoding'])
Receiver.send_message(u'openlp_process_events')
except (IOError, IndexError):
log.exception(u'Loading books from file failed')
success = False
finally:
if books_file:
books_file.close()
if not success:
if self.stop_import_flag or not success:
return False
self.wizard.progressBar.setValue(0)
self.wizard.progressBar.setMaximum(67)
verse_file = None
try:
verse_file = open(self.versesfile, 'r')
dialect = csv.Sniffer().sniff(verse_file.read(1024))
verse_file.seek(0)
verse_reader = csv.reader(verse_file, dialect)
book_ptr = None
details = get_file_encoding(self.versesfile)
verse_file = open(self.versesfile, 'rb')
verse_reader = csv.reader(verse_file, delimiter=',', quotechar='"')
for line in verse_reader:
if self.stop_import_flag:
# cancel pressed
break
details = chardet.detect(line[3])
if book_ptr != line[0]:
book = self.get_book(line[0])
if book_ptr != book_list[int(line[0])]:
book = self.get_book(book_list[int(line[0])])
book_ptr = book.name
self.wizard.incrementProgressBar(unicode(translate(
'BiblesPlugin.CSVImport', 'Importing %s %s...',
'Importing <book name> <chapter>...')) %
(book.name, int(line[1])))
'BibleDB.Wizard', 'Importing verses from %s...',
'Importing verses from <book name>...')) % book.name)
self.session.commit()
self.create_verse(book.id, line[1], line[2],
unicode(line[3], details['encoding']))
Receiver.send_message(u'openlp_process_events')
self.wizard.incrementProgressBar(translate('BibleDB.Wizard',
'Importing verses... done.'))
Receiver.send_message(u'openlp_process_events')
self.session.commit()
except IOError:
log.exception(u'Loading verses from file failed')
@ -112,3 +195,18 @@ class CSVBible(BibleDB):
return False
else:
return success
def get_file_encoding(filename):
"""
Utility function to get the file encoding.
"""
detect_file = None
try:
detect_file = open(filename, 'r')
details = chardet.detect(detect_file.read(1024))
except IOError:
log.exception(u'Error detecting file encoding')
finally:
if detect_file:
detect_file.close()
return details

8
openlp/plugins/bibles/lib/db.py
View File

@ -206,10 +206,16 @@ class BibleDB(QtCore.QObject, Manager):
"""
self.wizard = wizard
self.create_meta(u'dbversion', u'2')
self.setup_testaments()
return self.name
def setup_testaments(self):
"""
Initialise the testaments section of a bible with suitable defaults.
"""
self.save_object(Testament.populate(name=u'Old Testament'))
self.save_object(Testament.populate(name=u'New Testament'))
self.save_object(Testament.populate(name=u'Apocrypha'))
return self.name
def create_book(self, name, abbrev, testament=1):
"""

3
openlp/plugins/bibles/lib/mediaitem.py
View File

@ -254,9 +254,6 @@ class BibleMediaItem(MediaManagerItem):
QtCore.QObject.connect(self.quickSearchEdit,
QtCore.SIGNAL(u'returnPressed()'), self.onQuickSearchButton)
def addListViewToToolBar(self):
MediaManagerItem.addListViewToToolBar(self)
def configUpdated(self):
log.debug(u'configUpdated')
if QtCore.QSettings().value(self.settingsSection + u'/second bibles',

1
openlp/plugins/images/lib/mediaitem.py
View File

@ -96,7 +96,6 @@ class ImageMediaItem(MediaManagerItem):
def addListViewToToolBar(self):
MediaManagerItem.addListViewToToolBar(self)
self.listView.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu)
self.listView.addAction(self.replaceAction)
def addEndHeaderBar(self):

1
openlp/plugins/media/lib/mediaitem.py
View File

@ -84,7 +84,6 @@ class MediaMediaItem(MediaManagerItem):
def addListViewToToolBar(self):
MediaManagerItem.addListViewToToolBar(self)
self.listView.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu)
self.listView.addAction(self.replaceAction)
def addEndHeaderBar(self):

5
openlp/plugins/songs/lib/sofimport.py
View File

@ -39,9 +39,7 @@ from oooimport import OooImport
if os.name == u'nt':
BOLD = 150.0
ITALIC = 2
PAGE_BEFORE = 4
PAGE_AFTER = 5
PAGE_BOTH = 6
from oooimport import PAGE_BEFORE, PAGE_AFTER, PAGE_BOTH
else:
try:
from com.sun.star.awt.FontWeight import BOLD
@ -306,7 +304,6 @@ class SofImport(OooImport):
self.currentverse = u''
self.is_chorus = False
def uncap_text(self, text):
"""
Words in the title are in all capitals, so we lowercase them.

5
openlp/plugins/songs/lib/songimport.py
View File

@ -227,7 +227,7 @@ class SongImport(QtCore.QObject):
self.versecounts[versetag[0]] = int(versetag[1:])
self.verses.append([versetag, versetext.rstrip(), lang])
self.verse_order_list.append(versetag)
if versetag.startswith(u'V') and self.contains_verse(u'C1'):
if versetag.startswith(u'V') and u'C1' in self.verse_order_list:
self.verse_order_list.append(u'C1')
def repeat_verse(self):
@ -236,9 +236,6 @@ class SongImport(QtCore.QObject):
"""
self.verse_order_list.append(self.verse_order_list[-1])
def contains_verse(self, versetag):
return versetag in self.verse_order_list
def check_complete(self):
"""
Check the mandatory fields are entered (i.e. title and a verse)