Fix CSV bible imports part 1

This commit is contained in:
Jon Tibble 2011-01-27 03:13:43 +00:00
parent 6e435e47e6
commit 9211c49bb2
2 changed files with 127 additions and 26 deletions

View File

@ -23,7 +23,45 @@
# with this program; if not, write to the Free Software Foundation, Inc., 59 # # with this program; if not, write to the Free Software Foundation, Inc., 59 #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA # # Temple Place, Suite 330, Boston, MA 02111-1307 USA #
############################################################################### ###############################################################################
"""
The :mod:`cvsbible` modules provides a facility to import bibles from a set of
CSV files.
The module expects two mandatory files containing the books and the verses and
will accept an optional third file containing the testaments.
The format of the testament file is:
<testament_id>,<testament_name>
For example:
1,Old Testament
2,New Testament
The format of the books file is:
<book_id>,<testament_id>,<book_name>,<book_abbreviation>
For example
1,1,Genesis,Gen
2,1,Exodus,Exod
...
40,2,Matthew,Matt
The format of the verses file is:
<book_id>,<chapter_number>,<verse_number>,<verse_text>
For example:
1,1,1,"In the beginning God created the heaven and the earth."
1,1,2,"And the earth was without form, and void; and darkness...."
All CSV files are expected to use a comma (',') as the delimeter and double
quotes ('"') as the quote symbol.
"""
import logging import logging
import chardet import chardet
import csv import csv
@ -39,38 +77,81 @@ class CSVBible(BibleDB):
""" """
This class provides a specialisation for importing of CSV Bibles. This class provides a specialisation for importing of CSV Bibles.
""" """
def __init__(self, parent, **kwargs): def __init__(self, parent, **kwargs):
""" """
Loads a Bible from a pair of CVS files passed in Loads a Bible from a set of CVS files.
This class assumes the files contain all the information and This class assumes the files contain all the information and
a clean bible is being loaded. a clean bible is being loaded.
""" """
log.info(self.__class__.__name__) log.info(self.__class__.__name__)
BibleDB.__init__(self, parent, **kwargs) BibleDB.__init__(self, parent, **kwargs)
try:
self.testamentsfile = kwargs[u'testamentsfile']
except KeyError:
self.testamentsfile = None
self.booksfile = kwargs[u'booksfile'] self.booksfile = kwargs[u'booksfile']
self.versesfile = kwargs[u'versefile'] self.versesfile = kwargs[u'versefile']
QtCore.QObject.connect(Receiver.get_receiver(), QtCore.QObject.connect(Receiver.get_receiver(),
QtCore.SIGNAL(u'openlp_stop_wizard'), self.stop_import) QtCore.SIGNAL(u'openlp_stop_wizard'), self.stop_import)
def do_import(self): def setup_testaments(self):
success = True """
books_file = None Overrides parent method so we can handle importing a testament file.
book_ptr = None """
verse_file = None if self.testamentsfile:
# Populate the Tables self.wizard.progressBar.setMinimum(0)
self.wizard.progressBar.setMaximum(2)
self.wizard.progressBar.setValue(0)
testaments_file = None
try: try:
books_file = open(self.booksfile, 'r') details = get_file_encoding(self.testamentsfile)
dialect = csv.Sniffer().sniff(books_file.read(1024)) testaments_file = open(self.testamentsfile, 'rb')
books_file.seek(0) testaments_reader = csv.reader(testaments_file, delimiter=',',
books_reader = csv.reader(books_file, dialect) quotechar='"')
for line in books_reader: for line in testaments_reader:
# cancel pressed
if self.stop_import_flag: if self.stop_import_flag:
break break
details = chardet.detect(line[1]) self.wizard.incrementProgressBar(unicode(
self.create_book(unicode(line[1], details['encoding']), translate('BibleDB.Wizard',
line[2], int(line[0])) 'Importing testaments... %s')) %
unicode(line[1], details['encoding']), 0)
self.save_object(Testament.populate(
name=unicode(line[1], details['encoding'])))
Receiver.send_message(u'openlp_process_events')
except (IOError, IndexError):
log.exception(u'Loading testaments from file failed')
finally:
if testaments_file:
testaments_file.close()
self.wizard.incrementProgressBar(unicode(translate(
'BibleDB.Wizard', 'Importing testaments... done.')), 2)
else:
BibleDB.setup_testaments(self)
def do_import(self):
"""
Import the bible books and verses.
"""
self.wizard.progressBar.setValue(0)
self.wizard.progressBar.setMinimum(0)
self.wizard.progressBar.setMaximum(65)
success = True
books_file = None
book_list = {}
# Populate the Tables
try:
details = get_file_encoding(self.booksfile)
books_file = open(self.booksfile, 'r')
books_reader = csv.reader(books_file, delimiter=',', quotechar='"')
for line in books_reader:
if self.stop_import_flag:
break
self.wizard.incrementProgressBar(unicode(
translate('BibleDB.Wizard', 'Importing books... %s')) %
unicode(line[2], details['encoding']))
self.create_book(unicode(line[2], details['encoding']),
unicode(line[3], details['encoding']), int(line[1]))
book_list[int(line[0])] = unicode(line[2], details['encoding'])
Receiver.send_message(u'openlp_process_events') Receiver.send_message(u'openlp_process_events')
except (IOError, IndexError): except (IOError, IndexError):
log.exception(u'Loading books from file failed') log.exception(u'Loading books from file failed')
@ -78,25 +159,24 @@ class CSVBible(BibleDB):
finally: finally:
if books_file: if books_file:
books_file.close() books_file.close()
if not success: if self.stop_import_flag or not success:
return False return False
self.wizard.progressBar.setValue(0)
verse_file = None
try: try:
verse_file = open(self.versesfile, 'r') book_ptr = None
dialect = csv.Sniffer().sniff(verse_file.read(1024)) details = get_file_encoding(self.versesfile)
verse_file.seek(0) verse_file = open(self.versesfile, 'rb')
verse_reader = csv.reader(verse_file, dialect) verse_reader = csv.reader(verse_file, delimiter=',', quotechar='"')
for line in verse_reader: for line in verse_reader:
if self.stop_import_flag: if self.stop_import_flag:
# cancel pressed
break break
details = chardet.detect(line[3]) if book_ptr != book_list[int(line[0])]:
if book_ptr != line[0]: book = self.get_book(book_list[int(line[0])])
book = self.get_book(line[0])
book_ptr = book.name book_ptr = book.name
self.wizard.incrementProgressBar(unicode(translate( self.wizard.incrementProgressBar(unicode(translate(
'BiblesPlugin.CSVImport', 'Importing %s %s...', 'BibleDB.Wizard', 'Importing verses from %s...',
'Importing <book name> <chapter>...')) % 'Importing verses from <book name>...')) % book.name)
(book.name, int(line[1])))
self.session.commit() self.session.commit()
self.create_verse(book.id, line[1], line[2], self.create_verse(book.id, line[1], line[2],
unicode(line[3], details['encoding'])) unicode(line[3], details['encoding']))
@ -112,3 +192,18 @@ class CSVBible(BibleDB):
return False return False
else: else:
return success return success
def get_file_encoding(filename):
"""
Utility function to get the file encoding.
"""
detect_file = None
try:
detect_file = open(filename, 'r')
details = chardet.detect(detect_file.read(1024))
except IOError:
log.exception(u'Error detecting file encoding')
finally:
if detect_file:
detect_file.close()
return details

View File

@ -206,10 +206,16 @@ class BibleDB(QtCore.QObject, Manager):
""" """
self.wizard = wizard self.wizard = wizard
self.create_meta(u'dbversion', u'2') self.create_meta(u'dbversion', u'2')
self.setup_testaments()
return self.name
def setup_testaments(self):
"""
Initialise the testaments section of a bible with suitable defaults.
"""
self.save_object(Testament.populate(name=u'Old Testament')) self.save_object(Testament.populate(name=u'Old Testament'))
self.save_object(Testament.populate(name=u'New Testament')) self.save_object(Testament.populate(name=u'New Testament'))
self.save_object(Testament.populate(name=u'Apocrypha')) self.save_object(Testament.populate(name=u'Apocrypha'))
return self.name
def create_book(self, name, abbrev, testament=1): def create_book(self, name, abbrev, testament=1):
""" """