Fix CSV bible importing (bug #687309)

bzr-revno: 1250

openlp/plugins/bibles/forms/bibleimportform.py

@@ -124,9 +124,12 @@ class BibleImportForm(OpenLPWizard):
         QtCore.QObject.connect(self.osisBrowseButton,
             QtCore.SIGNAL(u'clicked()'),
             self.onOsisBrowseButtonClicked)
+        QtCore.QObject.connect(self.csvTestamentsButton,
+            QtCore.SIGNAL(u'clicked()'),
+            self.onCsvTestamentsBrowseButtonClicked)
         QtCore.QObject.connect(self.csvBooksButton,
             QtCore.SIGNAL(u'clicked()'),
-            self.onBooksBrowseButtonClicked)
+            self.onCsvBooksBrowseButtonClicked)
         QtCore.QObject.connect(self.csvVersesButton,
             QtCore.SIGNAL(u'clicked()'),
             self.onCsvVersesBrowseButtonClicked)
@@ -187,6 +190,18 @@ class BibleImportForm(OpenLPWizard):
         self.csvLayout = QtGui.QFormLayout(self.csvWidget)
         self.csvLayout.setMargin(0)
         self.csvLayout.setObjectName(u'CsvLayout')
+        self.csvTestamentsLabel = QtGui.QLabel(self.csvWidget)
+        self.csvTestamentsLabel.setObjectName(u'CsvTestamentsLabel')
+        self.csvTestamentsLayout = QtGui.QHBoxLayout()
+        self.csvTestamentsLayout.setObjectName(u'CsvTestamentsLayout')
+        self.csvTestamentsEdit = QtGui.QLineEdit(self.csvWidget)
+        self.csvTestamentsEdit.setObjectName(u'CsvTestamentsEdit')
+        self.csvTestamentsLayout.addWidget(self.csvTestamentsEdit)
+        self.csvTestamentsButton = QtGui.QToolButton(self.csvWidget)
+        self.csvTestamentsButton.setIcon(self.openIcon)
+        self.csvTestamentsButton.setObjectName(u'CsvTestamentsButton')
+        self.csvTestamentsLayout.addWidget(self.csvTestamentsButton)
+        self.csvLayout.addRow(self.csvTestamentsLabel, self.csvTestamentsLayout)
         self.csvBooksLabel = QtGui.QLabel(self.csvWidget)
         self.csvBooksLabel.setObjectName(u'CsvBooksLabel')
         self.csvBooksLayout = QtGui.QHBoxLayout()
@@ -213,7 +228,7 @@ class BibleImportForm(OpenLPWizard):
         self.csvLayout.addRow(self.csvVersesLabel, self.csvVersesLayout)
         self.csvSpacer = QtGui.QSpacerItem(10, 0, QtGui.QSizePolicy.Fixed,
             QtGui.QSizePolicy.Minimum)
-        self.csvLayout.setItem(2, QtGui.QFormLayout.LabelRole, self.csvSpacer)
+        self.csvLayout.setItem(3, QtGui.QFormLayout.LabelRole, self.csvSpacer)
         self.selectStack.addWidget(self.csvWidget)
         self.openSongWidget = QtGui.QWidget(self.selectPage)
         self.openSongWidget.setObjectName(u'OpenSongWidget')
@@ -389,6 +404,8 @@ class BibleImportForm(OpenLPWizard):
             translate('BiblesPlugin.ImportWizardForm', 'File location:'))
         self.osisFileLabel.setText(
             translate('BiblesPlugin.ImportWizardForm', 'File location:'))
+        self.csvTestamentsLabel.setText(
+            translate('BiblesPlugin.ImportWizardForm', 'Testaments location:'))
         self.csvBooksLabel.setText(
             translate('BiblesPlugin.ImportWizardForm', 'Books location:'))
         self.csvVersesLabel.setText(
@@ -478,7 +495,16 @@ class BibleImportForm(OpenLPWizard):
                     self.osisFileEdit.setFocus()
                     return False
             elif self.field(u'source_format').toInt()[0] == BibleFormat.CSV:
-                if not self.field(u'csv_booksfile').toString():
+                if not self.field(u'csv_testamentsfile').toString():
+                    answer = criticalErrorMessageBox(translate(
+                        'BiblesPlugin.ImportWizardForm', 'No Testaments File'),
+                        translate('BiblesPlugin.ImportWizardForm',
+                        'You have not specified a testaments file. Do you '
+                        'want to proceed with the import?'), question=True)
+                    if answer == QtGui.QMessageBox.No:
+                        self.csvTestamentsEdit.setFocus()
+                        return False
+                elif not self.field(u'csv_booksfile').toString():
                     criticalErrorMessageBox(
                         translate('BiblesPlugin.ImportWizardForm',
                         'Invalid Books File'),
@@ -572,7 +598,15 @@ class BibleImportForm(OpenLPWizard):
             translate('BiblesPlugin.ImportWizardForm', 'Open OSIS File'),
             self.osisFileEdit)
 
-    def onBooksBrowseButtonClicked(self):
+    def onCsvTestamentsBrowseButtonClicked(self):
+        """
+        Show the file open dialog for the testaments CSV file.
+        """
+        self.getFileName(translate('BiblesPlugin.ImportWizardForm',
+            'Open Testaments CSV File'), self.csvTestamentsEdit, u'%s (*.csv)'
+            % translate('BiblesPlugin.ImportWizardForm', 'CSV File'))
+
+    def onCsvBooksBrowseButtonClicked(self):
         """
         Show the file open dialog for the books CSV file.
         """
@@ -613,12 +647,14 @@ class BibleImportForm(OpenLPWizard):
         """
         self.selectPage.registerField(u'source_format', self.formatComboBox)
         self.selectPage.registerField(u'osis_location', self.osisFileEdit)
+        self.selectPage.registerField(
+            u'csv_testamentsfile', self.csvTestamentsEdit)
         self.selectPage.registerField(u'csv_booksfile', self.csvBooksEdit)
         self.selectPage.registerField(u'csv_versefile', self.csvVersesEdit)
         self.selectPage.registerField(u'opensong_file', self.openSongFileEdit)
         self.selectPage.registerField(u'web_location', self.webSourceComboBox)
-        self.selectPage.registerField(u'web_biblename',
+        self.selectPage.registerField(
-            self.webTranslationComboBox)
+            u'web_biblename', self.webTranslationComboBox)
         self.selectPage.registerField(u'proxy_server', self.webServerEdit)
         self.selectPage.registerField(u'proxy_username', self.webUserEdit)
         self.selectPage.registerField(u'proxy_password', self.webPasswordEdit)
@@ -641,6 +677,7 @@ class BibleImportForm(OpenLPWizard):
         self.cancelButton.setVisible(True)
         self.setField(u'source_format', QtCore.QVariant(0))
         self.setField(u'osis_location', QtCore.QVariant(''))
+        self.setField(u'csv_testamentsfile', QtCore.QVariant(''))
         self.setField(u'csv_booksfile', QtCore.QVariant(''))
         self.setField(u'csv_versefile', QtCore.QVariant(''))
         self.setField(u'opensong_file', QtCore.QVariant(''))
@@ -770,7 +807,8 @@ class BibleImportForm(OpenLPWizard):
         elif bible_type == BibleFormat.CSV:
             # Import a CSV bible.
             importer = self.manager.import_bible(BibleFormat.CSV,
-                name=license_version,
+                name=license_version, testamentsfile=unicode(
+                self.field(u'csv_testamentsfile').toString()),
                 booksfile=unicode(self.field(u'csv_booksfile').toString()),
                 versefile=unicode(self.field(u'csv_versefile').toString())
             )
@@ -795,8 +833,7 @@ class BibleImportForm(OpenLPWizard):
                 bible = \
                     self.web_bible_list[WebDownload.Bibleserver][bible_version]
             importer = self.manager.import_bible(
-                BibleFormat.WebDownload,
+                BibleFormat.WebDownload, name=license_version,
-                name=license_version,
                 download_source=WebDownload.get_name(download_location),
                 download_name=bible,
                 proxy_server=unicode(self.field(u'proxy_server').toString()),

openlp/plugins/bibles/lib/csvbible.py

@@ -23,7 +23,45 @@
 # with this program; if not, write to the Free Software Foundation, Inc., 59  #
 # Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
 ###############################################################################
+"""
+The :mod:`cvsbible` modules provides a facility to import bibles from a set of
+CSV files.
 
+The module expects two mandatory files containing the books and the verses and
+will accept an optional third file containing the testaments.
+
+The format of the testament file is:
+
+    <testament_id>,<testament_name>
+
+    For example:
+
+        1,Old Testament
+        2,New Testament
+
+The format of the books file is:
+
+    <book_id>,<testament_id>,<book_name>,<book_abbreviation>
+
+    For example
+
+        1,1,Genesis,Gen
+        2,1,Exodus,Exod
+        ...
+        40,2,Matthew,Matt
+
+The format of the verses file is:
+
+    <book_id>,<chapter_number>,<verse_number>,<verse_text>
+
+    For example:
+
+        1,1,1,"In the beginning God created the heaven and the earth."
+        1,1,2,"And the earth was without form, and void; and darkness...."
+
+All CSV files are expected to use a comma (',') as the delimeter and double
+quotes ('"') as the quote symbol.
+"""
 import logging
 import chardet
 import csv
@@ -31,7 +69,7 @@ import csv
 from PyQt4 import QtCore
 
 from openlp.core.lib import Receiver, translate
-from openlp.plugins.bibles.lib.db import BibleDB
+from openlp.plugins.bibles.lib.db import BibleDB, Testament
 
 log = logging.getLogger(__name__)
 
@@ -39,68 +77,113 @@ class CSVBible(BibleDB):
     """
     This class provides a specialisation for importing of CSV Bibles.
     """
-
     def __init__(self, parent, **kwargs):
         """
-        Loads a Bible from a pair of CVS files passed in
+        Loads a Bible from a set of CVS files.
         This class assumes the files contain all the information and
         a clean bible is being loaded.
         """
         log.info(self.__class__.__name__)
         BibleDB.__init__(self, parent, **kwargs)
+        try:
+            self.testamentsfile = kwargs[u'testamentsfile']
+        except KeyError:
+            self.testamentsfile = None
         self.booksfile = kwargs[u'booksfile']
         self.versesfile = kwargs[u'versefile']
         QtCore.QObject.connect(Receiver.get_receiver(),
             QtCore.SIGNAL(u'openlp_stop_wizard'), self.stop_import)
 
+    def setup_testaments(self):
+        """
+        Overrides parent method so we can handle importing a testament file.
+        """
+        if self.testamentsfile:
+            self.wizard.progressBar.setMinimum(0)
+            self.wizard.progressBar.setMaximum(2)
+            self.wizard.progressBar.setValue(0)
+            testaments_file = None
+            try:
+                details = get_file_encoding(self.testamentsfile)
+                testaments_file = open(self.testamentsfile, 'rb')
+                testaments_reader = csv.reader(testaments_file, delimiter=',',
+                    quotechar='"')
+                for line in testaments_reader:
+                    if self.stop_import_flag:
+                        break
+                    self.wizard.incrementProgressBar(unicode(
+                        translate('BibleDB.Wizard',
+                        'Importing testaments... %s')) %
+                        unicode(line[1], details['encoding']), 0)
+                    self.save_object(Testament.populate(
+                        name=unicode(line[1], details['encoding'])))
+                Receiver.send_message(u'openlp_process_events')
+            except (IOError, IndexError):
+                log.exception(u'Loading testaments from file failed')
+            finally:
+                if testaments_file:
+                    testaments_file.close()
+            self.wizard.incrementProgressBar(unicode(translate(
+                'BibleDB.Wizard', 'Importing testaments... done.')), 2)
+        else:
+            BibleDB.setup_testaments(self)
+
     def do_import(self):
+        """
+        Import the bible books and verses.
+        """
+        self.wizard.progressBar.setValue(0)
+        self.wizard.progressBar.setMinimum(0)
+        self.wizard.progressBar.setMaximum(66)
         success = True
         books_file = None
-        book_ptr = None
+        book_list = {}
-        verse_file = None
         # Populate the Tables
         try:
+            details = get_file_encoding(self.booksfile)
             books_file = open(self.booksfile, 'r')
-            dialect = csv.Sniffer().sniff(books_file.read(1024))
+            books_reader = csv.reader(books_file, delimiter=',', quotechar='"')
-            books_file.seek(0)
-            books_reader = csv.reader(books_file, dialect)
             for line in books_reader:
-                # cancel pressed
                 if self.stop_import_flag:
                     break
-                details = chardet.detect(line[1])
+                self.wizard.incrementProgressBar(unicode(
-                self.create_book(unicode(line[1], details['encoding']),
+                    translate('BibleDB.Wizard', 'Importing books... %s')) %
-                    line[2], int(line[0]))
+                    unicode(line[2], details['encoding']))
-                Receiver.send_message(u'openlp_process_events')
+                self.create_book(unicode(line[2], details['encoding']),
+                    unicode(line[3], details['encoding']), int(line[1]))
+                book_list[int(line[0])] = unicode(line[2], details['encoding'])
+            Receiver.send_message(u'openlp_process_events')
         except (IOError, IndexError):
             log.exception(u'Loading books from file failed')
             success = False
         finally:
             if books_file:
                 books_file.close()
-        if not success:
+        if self.stop_import_flag or not success:
             return False
+        self.wizard.progressBar.setValue(0)
+        self.wizard.progressBar.setMaximum(67)
+        verse_file = None
         try:
-            verse_file = open(self.versesfile, 'r')
+            book_ptr = None
-            dialect = csv.Sniffer().sniff(verse_file.read(1024))
+            details = get_file_encoding(self.versesfile)
-            verse_file.seek(0)
+            verse_file = open(self.versesfile, 'rb')
-            verse_reader = csv.reader(verse_file, dialect)
+            verse_reader = csv.reader(verse_file, delimiter=',', quotechar='"')
             for line in verse_reader:
                 if self.stop_import_flag:
-                    # cancel pressed
                     break
-                details = chardet.detect(line[3])
+                if book_ptr != book_list[int(line[0])]:
-                if book_ptr != line[0]:
+                    book = self.get_book(book_list[int(line[0])])
-                    book = self.get_book(line[0])
                     book_ptr = book.name
                     self.wizard.incrementProgressBar(unicode(translate(
-                        'BiblesPlugin.CSVImport', 'Importing %s %s...',
+                        'BibleDB.Wizard', 'Importing verses from %s...',
-                        'Importing <book name> <chapter>...')) %
+                        'Importing verses from <book name>...')) % book.name)
-                        (book.name, int(line[1])))
                     self.session.commit()
                 self.create_verse(book.id, line[1], line[2],
                     unicode(line[3], details['encoding']))
-                Receiver.send_message(u'openlp_process_events')
+            self.wizard.incrementProgressBar(translate('BibleDB.Wizard',
+                'Importing verses... done.'))
+            Receiver.send_message(u'openlp_process_events')
             self.session.commit()
         except IOError:
             log.exception(u'Loading verses from file failed')
@@ -112,3 +195,18 @@ class CSVBible(BibleDB):
             return False
         else:
             return success
+
+def get_file_encoding(filename):
+    """
+    Utility function to get the file encoding.
+    """
+    detect_file = None
+    try:
+        detect_file = open(filename, 'r')
+        details = chardet.detect(detect_file.read(1024))
+    except IOError:
+        log.exception(u'Error detecting file encoding')
+    finally:
+        if detect_file:
+            detect_file.close()
+    return details

openlp/plugins/bibles/lib/db.py

@@ -206,10 +206,16 @@ class BibleDB(QtCore.QObject, Manager):
         """
         self.wizard = wizard
         self.create_meta(u'dbversion', u'2')
+        self.setup_testaments()
+        return self.name
+
+    def setup_testaments(self):
+        """
+        Initialise the testaments section of a bible with suitable defaults.
+        """
         self.save_object(Testament.populate(name=u'Old Testament'))
         self.save_object(Testament.populate(name=u'New Testament'))
         self.save_object(Testament.populate(name=u'Apocrypha'))
-        return self.name
 
     def create_book(self, name, abbrev, testament=1):
         """

openlp/plugins/bibles/lib/mediaitem.py

@@ -254,9 +254,6 @@ class BibleMediaItem(MediaManagerItem):
         QtCore.QObject.connect(self.quickSearchEdit,
             QtCore.SIGNAL(u'returnPressed()'), self.onQuickSearchButton)
 
-    def addListViewToToolBar(self):
-        MediaManagerItem.addListViewToToolBar(self)
-
     def configUpdated(self):
         log.debug(u'configUpdated')
         if QtCore.QSettings().value(self.settingsSection + u'/second bibles',

openlp/plugins/images/lib/mediaitem.py

@@ -96,7 +96,6 @@ class ImageMediaItem(MediaManagerItem):
 
     def addListViewToToolBar(self):
         MediaManagerItem.addListViewToToolBar(self)
-        self.listView.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu)
         self.listView.addAction(self.replaceAction)
 
     def addEndHeaderBar(self):

openlp/plugins/media/lib/mediaitem.py

@@ -84,7 +84,6 @@ class MediaMediaItem(MediaManagerItem):
 
     def addListViewToToolBar(self):
         MediaManagerItem.addListViewToToolBar(self)
-        self.listView.setContextMenuPolicy(QtCore.Qt.ActionsContextMenu)
         self.listView.addAction(self.replaceAction)
 
     def addEndHeaderBar(self):

openlp/plugins/songs/lib/sofimport.py

@@ -39,9 +39,7 @@ from oooimport import OooImport
 if os.name == u'nt':
     BOLD = 150.0
     ITALIC = 2
-    PAGE_BEFORE = 4
+    from oooimport import PAGE_BEFORE, PAGE_AFTER, PAGE_BOTH
-    PAGE_AFTER = 5
-    PAGE_BOTH = 6
 else:
     try:
         from com.sun.star.awt.FontWeight import BOLD
@@ -306,7 +304,6 @@ class SofImport(OooImport):
         self.currentverse = u''
         self.is_chorus = False
 
-
     def uncap_text(self, text):
         """
         Words in the title are in all capitals, so we lowercase them.

openlp/plugins/songs/lib/songimport.py

@@ -227,7 +227,7 @@ class SongImport(QtCore.QObject):
             self.versecounts[versetag[0]] = int(versetag[1:])
         self.verses.append([versetag, versetext.rstrip(), lang])
         self.verse_order_list.append(versetag)
-        if versetag.startswith(u'V') and self.contains_verse(u'C1'):
+        if versetag.startswith(u'V') and u'C1' in self.verse_order_list:
             self.verse_order_list.append(u'C1')
 
     def repeat_verse(self):
@@ -236,9 +236,6 @@ class SongImport(QtCore.QObject):
         """
         self.verse_order_list.append(self.verse_order_list[-1])
 
-    def contains_verse(self, versetag):
-        return versetag in self.verse_order_list
-
     def check_complete(self):
         """
         Check the mandatory fields are entered (i.e. title and a verse)