Fix CSV bible imports part 1

openlp/plugins/bibles/lib/csvbible.py

@@ -23,7 +23,45 @@
 # with this program; if not, write to the Free Software Foundation, Inc., 59  #
 # Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
 ###############################################################################
+"""
+The :mod:`cvsbible` modules provides a facility to import bibles from a set of
+CSV files.
 
+The module expects two mandatory files containing the books and the verses and
+will accept an optional third file containing the testaments.
+
+The format of the testament file is:
+
+    <testament_id>,<testament_name>
+
+    For example:
+
+        1,Old Testament
+        2,New Testament
+
+The format of the books file is:
+
+    <book_id>,<testament_id>,<book_name>,<book_abbreviation>
+
+    For example
+
+        1,1,Genesis,Gen
+        2,1,Exodus,Exod
+        ...
+        40,2,Matthew,Matt
+
+The format of the verses file is:
+
+    <book_id>,<chapter_number>,<verse_number>,<verse_text>
+
+    For example:
+
+        1,1,1,"In the beginning God created the heaven and the earth."
+        1,1,2,"And the earth was without form, and void; and darkness...."
+
+All CSV files are expected to use a comma (',') as the delimeter and double
+quotes ('"') as the quote symbol.
+"""
 import logging
 import chardet
 import csv
@@ -39,68 +77,110 @@ class CSVBible(BibleDB):
     """
     This class provides a specialisation for importing of CSV Bibles.
     """
-
     def __init__(self, parent, **kwargs):
         """
-        Loads a Bible from a pair of CVS files passed in
+        Loads a Bible from a set of CVS files.
         This class assumes the files contain all the information and
         a clean bible is being loaded.
         """
         log.info(self.__class__.__name__)
         BibleDB.__init__(self, parent, **kwargs)
+        try:
+            self.testamentsfile = kwargs[u'testamentsfile']
+        except KeyError:
+            self.testamentsfile = None
         self.booksfile = kwargs[u'booksfile']
         self.versesfile = kwargs[u'versefile']
         QtCore.QObject.connect(Receiver.get_receiver(),
             QtCore.SIGNAL(u'openlp_stop_wizard'), self.stop_import)
 
+    def setup_testaments(self):
+        """
+        Overrides parent method so we can handle importing a testament file.
+        """
+        if self.testamentsfile:
+            self.wizard.progressBar.setMinimum(0)
+            self.wizard.progressBar.setMaximum(2)
+            self.wizard.progressBar.setValue(0)
+            testaments_file = None
+            try:
+                details = get_file_encoding(self.testamentsfile)
+                testaments_file = open(self.testamentsfile, 'rb')
+                testaments_reader = csv.reader(testaments_file, delimiter=',',
+                    quotechar='"')
+                for line in testaments_reader:
+                    if self.stop_import_flag:
+                        break
+                    self.wizard.incrementProgressBar(unicode(
+                        translate('BibleDB.Wizard',
+                        'Importing testaments... %s')) %
+                        unicode(line[1], details['encoding']), 0)
+                    self.save_object(Testament.populate(
+                        name=unicode(line[1], details['encoding'])))
+                Receiver.send_message(u'openlp_process_events')
+            except (IOError, IndexError):
+                log.exception(u'Loading testaments from file failed')
+            finally:
+                if testaments_file:
+                    testaments_file.close()
+            self.wizard.incrementProgressBar(unicode(translate(
+                'BibleDB.Wizard', 'Importing testaments... done.')), 2)
+        else:
+            BibleDB.setup_testaments(self)
+
     def do_import(self):
+        """
+        Import the bible books and verses.
+        """
+        self.wizard.progressBar.setValue(0)
+        self.wizard.progressBar.setMinimum(0)
+        self.wizard.progressBar.setMaximum(65)
         success = True
         books_file = None
-        book_ptr = None
+        book_list = {}
-        verse_file = None
         # Populate the Tables
         try:
+            details = get_file_encoding(self.booksfile)
             books_file = open(self.booksfile, 'r')
-            dialect = csv.Sniffer().sniff(books_file.read(1024))
+            books_reader = csv.reader(books_file, delimiter=',', quotechar='"')
-            books_file.seek(0)
-            books_reader = csv.reader(books_file, dialect)
             for line in books_reader:
-                # cancel pressed
                 if self.stop_import_flag:
                     break
-                details = chardet.detect(line[1])
+                self.wizard.incrementProgressBar(unicode(
-                self.create_book(unicode(line[1], details['encoding']),
+                    translate('BibleDB.Wizard', 'Importing books... %s')) %
-                    line[2], int(line[0]))
+                    unicode(line[2], details['encoding']))
-                Receiver.send_message(u'openlp_process_events')
+                self.create_book(unicode(line[2], details['encoding']),
+                    unicode(line[3], details['encoding']), int(line[1]))
+                book_list[int(line[0])] = unicode(line[2], details['encoding'])
+            Receiver.send_message(u'openlp_process_events')
         except (IOError, IndexError):
             log.exception(u'Loading books from file failed')
             success = False
         finally:
             if books_file:
                 books_file.close()
-        if not success:
+        if self.stop_import_flag or not success:
             return False
+        self.wizard.progressBar.setValue(0)
+        verse_file = None
         try:
-            verse_file = open(self.versesfile, 'r')
+            book_ptr = None
-            dialect = csv.Sniffer().sniff(verse_file.read(1024))
+            details = get_file_encoding(self.versesfile)
-            verse_file.seek(0)
+            verse_file = open(self.versesfile, 'rb')
-            verse_reader = csv.reader(verse_file, dialect)
+            verse_reader = csv.reader(verse_file, delimiter=',', quotechar='"')
             for line in verse_reader:
                 if self.stop_import_flag:
-                    # cancel pressed
                     break
-                details = chardet.detect(line[3])
+                if book_ptr != book_list[int(line[0])]:
-                if book_ptr != line[0]:
+                    book = self.get_book(book_list[int(line[0])])
-                    book = self.get_book(line[0])
                     book_ptr = book.name
                     self.wizard.incrementProgressBar(unicode(translate(
-                        'BiblesPlugin.CSVImport', 'Importing %s %s...',
+                        'BibleDB.Wizard', 'Importing verses from %s...',
-                        'Importing <book name> <chapter>...')) %
+                        'Importing verses from <book name>...')) % book.name)
-                        (book.name, int(line[1])))
                     self.session.commit()
                 self.create_verse(book.id, line[1], line[2],
                     unicode(line[3], details['encoding']))
-                Receiver.send_message(u'openlp_process_events')
+            Receiver.send_message(u'openlp_process_events')
             self.session.commit()
         except IOError:
             log.exception(u'Loading verses from file failed')
@@ -112,3 +192,18 @@ class CSVBible(BibleDB):
             return False
         else:
             return success
+
+def get_file_encoding(filename):
+    """
+    Utility function to get the file encoding.
+    """
+    detect_file = None
+    try:
+        detect_file = open(filename, 'r')
+        details = chardet.detect(detect_file.read(1024))
+    except IOError:
+        log.exception(u'Error detecting file encoding')
+    finally:
+        if detect_file:
+            detect_file.close()
+    return details

openlp/plugins/bibles/lib/db.py

@@ -206,10 +206,16 @@ class BibleDB(QtCore.QObject, Manager):
         """
         self.wizard = wizard
         self.create_meta(u'dbversion', u'2')
+        self.setup_testaments()
+        return self.name
+
+    def setup_testaments(self):
+        """
+        Initialise the testaments section of a bible with suitable defaults.
+        """
         self.save_object(Testament.populate(name=u'Old Testament'))
         self.save_object(Testament.populate(name=u'New Testament'))
         self.save_object(Testament.populate(name=u'Apocrypha'))
-        return self.name
 
     def create_book(self, name, abbrev, testament=1):
         """