From 9211c49bb2aaa79afe037eb772d6183846918e60 Mon Sep 17 00:00:00 2001 From: Jon Tibble Date: Thu, 27 Jan 2011 03:13:43 +0000 Subject: [PATCH] Fix CSV bible imports part 1 --- openlp/plugins/bibles/lib/csvbible.py | 145 +++++++++++++++++++++----- openlp/plugins/bibles/lib/db.py | 8 +- 2 files changed, 127 insertions(+), 26 deletions(-) diff --git a/openlp/plugins/bibles/lib/csvbible.py b/openlp/plugins/bibles/lib/csvbible.py index 8959167a6..87b08d8b8 100644 --- a/openlp/plugins/bibles/lib/csvbible.py +++ b/openlp/plugins/bibles/lib/csvbible.py @@ -23,7 +23,45 @@ # with this program; if not, write to the Free Software Foundation, Inc., 59 # # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### +""" +The :mod:`cvsbible` modules provides a facility to import bibles from a set of +CSV files. +The module expects two mandatory files containing the books and the verses and +will accept an optional third file containing the testaments. + +The format of the testament file is: + + , + + For example: + + 1,Old Testament + 2,New Testament + +The format of the books file is: + + ,,, + + For example + + 1,1,Genesis,Gen + 2,1,Exodus,Exod + ... + 40,2,Matthew,Matt + +The format of the verses file is: + + ,,, + + For example: + + 1,1,1,"In the beginning God created the heaven and the earth." + 1,1,2,"And the earth was without form, and void; and darkness...." + +All CSV files are expected to use a comma (',') as the delimeter and double +quotes ('"') as the quote symbol. +""" import logging import chardet import csv @@ -39,68 +77,110 @@ class CSVBible(BibleDB): """ This class provides a specialisation for importing of CSV Bibles. """ - def __init__(self, parent, **kwargs): """ - Loads a Bible from a pair of CVS files passed in + Loads a Bible from a set of CVS files. This class assumes the files contain all the information and a clean bible is being loaded. """ log.info(self.__class__.__name__) BibleDB.__init__(self, parent, **kwargs) + try: + self.testamentsfile = kwargs[u'testamentsfile'] + except KeyError: + self.testamentsfile = None self.booksfile = kwargs[u'booksfile'] self.versesfile = kwargs[u'versefile'] QtCore.QObject.connect(Receiver.get_receiver(), QtCore.SIGNAL(u'openlp_stop_wizard'), self.stop_import) + def setup_testaments(self): + """ + Overrides parent method so we can handle importing a testament file. + """ + if self.testamentsfile: + self.wizard.progressBar.setMinimum(0) + self.wizard.progressBar.setMaximum(2) + self.wizard.progressBar.setValue(0) + testaments_file = None + try: + details = get_file_encoding(self.testamentsfile) + testaments_file = open(self.testamentsfile, 'rb') + testaments_reader = csv.reader(testaments_file, delimiter=',', + quotechar='"') + for line in testaments_reader: + if self.stop_import_flag: + break + self.wizard.incrementProgressBar(unicode( + translate('BibleDB.Wizard', + 'Importing testaments... %s')) % + unicode(line[1], details['encoding']), 0) + self.save_object(Testament.populate( + name=unicode(line[1], details['encoding']))) + Receiver.send_message(u'openlp_process_events') + except (IOError, IndexError): + log.exception(u'Loading testaments from file failed') + finally: + if testaments_file: + testaments_file.close() + self.wizard.incrementProgressBar(unicode(translate( + 'BibleDB.Wizard', 'Importing testaments... done.')), 2) + else: + BibleDB.setup_testaments(self) + def do_import(self): + """ + Import the bible books and verses. + """ + self.wizard.progressBar.setValue(0) + self.wizard.progressBar.setMinimum(0) + self.wizard.progressBar.setMaximum(65) success = True books_file = None - book_ptr = None - verse_file = None + book_list = {} # Populate the Tables try: + details = get_file_encoding(self.booksfile) books_file = open(self.booksfile, 'r') - dialect = csv.Sniffer().sniff(books_file.read(1024)) - books_file.seek(0) - books_reader = csv.reader(books_file, dialect) + books_reader = csv.reader(books_file, delimiter=',', quotechar='"') for line in books_reader: - # cancel pressed if self.stop_import_flag: break - details = chardet.detect(line[1]) - self.create_book(unicode(line[1], details['encoding']), - line[2], int(line[0])) - Receiver.send_message(u'openlp_process_events') + self.wizard.incrementProgressBar(unicode( + translate('BibleDB.Wizard', 'Importing books... %s')) % + unicode(line[2], details['encoding'])) + self.create_book(unicode(line[2], details['encoding']), + unicode(line[3], details['encoding']), int(line[1])) + book_list[int(line[0])] = unicode(line[2], details['encoding']) + Receiver.send_message(u'openlp_process_events') except (IOError, IndexError): log.exception(u'Loading books from file failed') success = False finally: if books_file: books_file.close() - if not success: + if self.stop_import_flag or not success: return False + self.wizard.progressBar.setValue(0) + verse_file = None try: - verse_file = open(self.versesfile, 'r') - dialect = csv.Sniffer().sniff(verse_file.read(1024)) - verse_file.seek(0) - verse_reader = csv.reader(verse_file, dialect) + book_ptr = None + details = get_file_encoding(self.versesfile) + verse_file = open(self.versesfile, 'rb') + verse_reader = csv.reader(verse_file, delimiter=',', quotechar='"') for line in verse_reader: if self.stop_import_flag: - # cancel pressed break - details = chardet.detect(line[3]) - if book_ptr != line[0]: - book = self.get_book(line[0]) + if book_ptr != book_list[int(line[0])]: + book = self.get_book(book_list[int(line[0])]) book_ptr = book.name self.wizard.incrementProgressBar(unicode(translate( - 'BiblesPlugin.CSVImport', 'Importing %s %s...', - 'Importing ...')) % - (book.name, int(line[1]))) + 'BibleDB.Wizard', 'Importing verses from %s...', + 'Importing verses from ...')) % book.name) self.session.commit() self.create_verse(book.id, line[1], line[2], unicode(line[3], details['encoding'])) - Receiver.send_message(u'openlp_process_events') + Receiver.send_message(u'openlp_process_events') self.session.commit() except IOError: log.exception(u'Loading verses from file failed') @@ -112,3 +192,18 @@ class CSVBible(BibleDB): return False else: return success + +def get_file_encoding(filename): + """ + Utility function to get the file encoding. + """ + detect_file = None + try: + detect_file = open(filename, 'r') + details = chardet.detect(detect_file.read(1024)) + except IOError: + log.exception(u'Error detecting file encoding') + finally: + if detect_file: + detect_file.close() + return details diff --git a/openlp/plugins/bibles/lib/db.py b/openlp/plugins/bibles/lib/db.py index f442a9cd6..a9444d88b 100644 --- a/openlp/plugins/bibles/lib/db.py +++ b/openlp/plugins/bibles/lib/db.py @@ -206,10 +206,16 @@ class BibleDB(QtCore.QObject, Manager): """ self.wizard = wizard self.create_meta(u'dbversion', u'2') + self.setup_testaments() + return self.name + + def setup_testaments(self): + """ + Initialise the testaments section of a bible with suitable defaults. + """ self.save_object(Testament.populate(name=u'Old Testament')) self.save_object(Testament.populate(name=u'New Testament')) self.save_object(Testament.populate(name=u'Apocrypha')) - return self.name def create_book(self, name, abbrev, testament=1): """