2009-09-08 19:58:05 +00:00
|
|
|
# -*- coding: utf-8 -*-
|
2013-01-01 16:33:41 +00:00
|
|
|
# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
|
2008-11-17 21:22:14 +00:00
|
|
|
|
2009-09-08 19:58:05 +00:00
|
|
|
###############################################################################
|
|
|
|
# OpenLP - Open Source Lyrics Projection #
|
|
|
|
# --------------------------------------------------------------------------- #
|
2012-12-29 20:56:56 +00:00
|
|
|
# Copyright (c) 2008-2013 Raoul Snyman #
|
|
|
|
# Portions copyright (c) 2008-2013 Tim Bentley, Gerald Britton, Jonathan #
|
2012-06-22 14:14:53 +00:00
|
|
|
# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub, #
|
2012-11-11 21:16:14 +00:00
|
|
|
# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer. #
|
2012-10-21 13:16:22 +00:00
|
|
|
# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru, #
|
|
|
|
# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith, #
|
|
|
|
# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock, #
|
2012-12-01 07:57:54 +00:00
|
|
|
# Frode Woldsund, Martin Zibricky, Patrick Zimmermann #
|
2009-09-08 19:58:05 +00:00
|
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
# This program is free software; you can redistribute it and/or modify it #
|
|
|
|
# under the terms of the GNU General Public License as published by the Free #
|
|
|
|
# Software Foundation; version 2 of the License. #
|
|
|
|
# #
|
|
|
|
# This program is distributed in the hope that it will be useful, but WITHOUT #
|
|
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
|
|
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
|
|
|
|
# more details. #
|
|
|
|
# #
|
|
|
|
# You should have received a copy of the GNU General Public License along #
|
|
|
|
# with this program; if not, write to the Free Software Foundation, Inc., 59 #
|
|
|
|
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
|
|
|
|
###############################################################################
|
2011-01-27 03:13:43 +00:00
|
|
|
"""
|
2013-04-18 17:45:14 +00:00
|
|
|
The :mod:`cvsbible` modules provides a facility to import bibles from a set of CSV files.
|
2008-11-17 21:22:14 +00:00
|
|
|
|
2011-12-10 18:21:18 +00:00
|
|
|
The module expects two mandatory files containing the books and the verses.
|
2011-01-27 03:13:43 +00:00
|
|
|
|
|
|
|
The format of the books file is:
|
|
|
|
|
|
|
|
<book_id>,<testament_id>,<book_name>,<book_abbreviation>
|
|
|
|
|
|
|
|
For example
|
|
|
|
|
|
|
|
1,1,Genesis,Gen
|
|
|
|
2,1,Exodus,Exod
|
|
|
|
...
|
|
|
|
40,2,Matthew,Matt
|
|
|
|
|
2011-01-30 09:43:07 +00:00
|
|
|
There are two acceptable formats of the verses file. They are:
|
2011-01-27 03:13:43 +00:00
|
|
|
|
|
|
|
<book_id>,<chapter_number>,<verse_number>,<verse_text>
|
2011-01-30 09:43:07 +00:00
|
|
|
or
|
|
|
|
<book_name>,<chapter_number>,<verse_number>,<verse_text>
|
2011-01-27 03:13:43 +00:00
|
|
|
|
|
|
|
For example:
|
|
|
|
|
|
|
|
1,1,1,"In the beginning God created the heaven and the earth."
|
2011-01-30 09:43:07 +00:00
|
|
|
or
|
|
|
|
"Genesis",1,2,"And the earth was without form, and void; and...."
|
2011-01-27 03:13:43 +00:00
|
|
|
|
2013-04-18 17:45:14 +00:00
|
|
|
All CSV files are expected to use a comma (',') as the delimiter and double quotes ('"') as the quote symbol.
|
2011-01-27 03:13:43 +00:00
|
|
|
"""
|
2009-02-02 19:54:38 +00:00
|
|
|
import logging
|
2009-07-09 16:51:25 +00:00
|
|
|
import chardet
|
2010-01-24 07:08:14 +00:00
|
|
|
import csv
|
2009-01-20 19:50:37 +00:00
|
|
|
|
2013-02-05 06:47:03 +00:00
|
|
|
from openlp.core.lib import translate
|
2011-05-03 17:09:30 +00:00
|
|
|
from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
|
2008-11-17 21:22:14 +00:00
|
|
|
|
2013-04-18 17:45:14 +00:00
|
|
|
|
2009-12-27 23:54:00 +00:00
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
2013-04-18 17:45:14 +00:00
|
|
|
|
2009-12-27 23:54:00 +00:00
|
|
|
class CSVBible(BibleDB):
|
|
|
|
"""
|
|
|
|
This class provides a specialisation for importing of CSV Bibles.
|
|
|
|
"""
|
2011-04-03 18:44:08 +00:00
|
|
|
log.info(u'CSVBible loaded')
|
|
|
|
|
2010-01-31 19:49:01 +00:00
|
|
|
def __init__(self, parent, **kwargs):
|
2008-11-17 21:22:14 +00:00
|
|
|
"""
|
2013-04-18 17:45:14 +00:00
|
|
|
Loads a Bible from a set of CSV files. This class assumes the files contain all the information and a clean
|
|
|
|
bible is being loaded.
|
2009-07-09 16:51:25 +00:00
|
|
|
"""
|
2010-01-24 07:08:14 +00:00
|
|
|
log.info(self.__class__.__name__)
|
2010-12-06 19:30:04 +00:00
|
|
|
BibleDB.__init__(self, parent, **kwargs)
|
2009-12-30 17:29:08 +00:00
|
|
|
self.booksfile = kwargs[u'booksfile']
|
2010-09-03 05:33:03 +00:00
|
|
|
self.versesfile = kwargs[u'versefile']
|
2011-04-14 20:33:02 +00:00
|
|
|
|
2011-05-26 19:13:11 +00:00
|
|
|
def do_import(self, bible_name=None):
|
2011-01-27 03:13:43 +00:00
|
|
|
"""
|
|
|
|
Import the bible books and verses.
|
|
|
|
"""
|
2013-03-07 08:05:43 +00:00
|
|
|
self.wizard.progress_bar.setValue(0)
|
|
|
|
self.wizard.progress_bar.setMinimum(0)
|
|
|
|
self.wizard.progress_bar.setMaximum(66)
|
2009-12-27 22:19:39 +00:00
|
|
|
success = True
|
2011-05-26 19:13:11 +00:00
|
|
|
language_id = self.get_language(bible_name)
|
2011-04-02 20:22:35 +00:00
|
|
|
if not language_id:
|
2011-05-26 19:13:11 +00:00
|
|
|
log.exception(u'Importing books from "%s" failed' % self.filename)
|
2011-03-18 09:52:54 +00:00
|
|
|
return False
|
2010-01-24 07:08:14 +00:00
|
|
|
books_file = None
|
2011-01-27 03:13:43 +00:00
|
|
|
book_list = {}
|
2010-12-06 19:30:04 +00:00
|
|
|
# Populate the Tables
|
2009-11-07 00:00:36 +00:00
|
|
|
try:
|
2011-01-27 03:13:43 +00:00
|
|
|
details = get_file_encoding(self.booksfile)
|
2010-01-24 07:08:14 +00:00
|
|
|
books_file = open(self.booksfile, 'r')
|
2011-12-10 18:21:18 +00:00
|
|
|
if not books_file.read(3) == '\xEF\xBB\xBF':
|
|
|
|
# no BOM was found
|
|
|
|
books_file.seek(0)
|
2011-01-27 03:13:43 +00:00
|
|
|
books_reader = csv.reader(books_file, delimiter=',', quotechar='"')
|
2010-01-24 07:08:14 +00:00
|
|
|
for line in books_reader:
|
2010-02-06 15:33:23 +00:00
|
|
|
if self.stop_import_flag:
|
2009-11-07 00:00:36 +00:00
|
|
|
break
|
2013-03-07 08:05:43 +00:00
|
|
|
self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible', 'Importing books... %s') %
|
2011-01-27 03:13:43 +00:00
|
|
|
unicode(line[2], details['encoding']))
|
2013-01-01 16:33:41 +00:00
|
|
|
book_ref_id = self.get_book_ref_id_by_name(unicode(line[2], details['encoding']), 67, language_id)
|
2011-03-18 09:52:54 +00:00
|
|
|
if not book_ref_id:
|
2013-01-01 16:33:41 +00:00
|
|
|
log.exception(u'Importing books from "%s" failed' % self.booksfile)
|
2011-03-18 09:52:54 +00:00
|
|
|
return False
|
2011-03-17 19:40:01 +00:00
|
|
|
book_details = BiblesResourcesDB.get_book_by_id(book_ref_id)
|
2013-01-01 16:33:41 +00:00
|
|
|
self.create_book(unicode(line[2], details['encoding']), book_ref_id, book_details[u'testament_id'])
|
2011-01-27 03:13:43 +00:00
|
|
|
book_list[int(line[0])] = unicode(line[2], details['encoding'])
|
2013-02-03 19:23:12 +00:00
|
|
|
self.application.process_events()
|
2011-01-13 02:28:03 +00:00
|
|
|
except (IOError, IndexError):
|
2009-11-07 00:00:36 +00:00
|
|
|
log.exception(u'Loading books from file failed')
|
2009-12-27 22:19:39 +00:00
|
|
|
success = False
|
2009-11-07 00:00:36 +00:00
|
|
|
finally:
|
2010-01-24 07:08:14 +00:00
|
|
|
if books_file:
|
|
|
|
books_file.close()
|
2011-01-27 03:13:43 +00:00
|
|
|
if self.stop_import_flag or not success:
|
2009-12-27 22:19:39 +00:00
|
|
|
return False
|
2013-03-07 08:05:43 +00:00
|
|
|
self.wizard.progress_bar.setValue(0)
|
|
|
|
self.wizard.progress_bar.setMaximum(67)
|
2011-01-27 03:13:43 +00:00
|
|
|
verse_file = None
|
2009-11-07 00:00:36 +00:00
|
|
|
try:
|
2011-01-27 03:13:43 +00:00
|
|
|
book_ptr = None
|
|
|
|
details = get_file_encoding(self.versesfile)
|
|
|
|
verse_file = open(self.versesfile, 'rb')
|
2011-12-10 18:21:18 +00:00
|
|
|
if not verse_file.read(3) == '\xEF\xBB\xBF':
|
|
|
|
# no BOM was found
|
|
|
|
verse_file.seek(0)
|
2011-01-27 03:13:43 +00:00
|
|
|
verse_reader = csv.reader(verse_file, delimiter=',', quotechar='"')
|
2010-01-24 07:08:14 +00:00
|
|
|
for line in verse_reader:
|
2010-12-27 22:57:35 +00:00
|
|
|
if self.stop_import_flag:
|
2009-11-07 00:00:36 +00:00
|
|
|
break
|
2011-01-30 09:32:09 +00:00
|
|
|
try:
|
|
|
|
line_book = book_list[int(line[0])]
|
|
|
|
except ValueError:
|
|
|
|
line_book = unicode(line[0], details['encoding'])
|
|
|
|
if book_ptr != line_book:
|
|
|
|
book = self.get_book(line_book)
|
2009-11-07 00:00:36 +00:00
|
|
|
book_ptr = book.name
|
2013-03-07 08:05:43 +00:00
|
|
|
self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible',
|
2013-01-01 16:33:41 +00:00
|
|
|
'Importing verses from %s... Importing verses from <book name>...') % book.name)
|
2010-06-15 00:44:06 +00:00
|
|
|
self.session.commit()
|
2011-01-31 02:43:37 +00:00
|
|
|
try:
|
|
|
|
verse_text = unicode(line[3], details['encoding'])
|
|
|
|
except UnicodeError:
|
|
|
|
verse_text = unicode(line[3], u'cp1252')
|
|
|
|
self.create_verse(book.id, line[1], line[2], verse_text)
|
2013-03-07 08:05:43 +00:00
|
|
|
self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible', 'Importing verses... done.'))
|
2013-02-03 19:23:12 +00:00
|
|
|
self.application.process_events()
|
2010-06-15 00:44:06 +00:00
|
|
|
self.session.commit()
|
2010-05-29 19:50:50 +00:00
|
|
|
except IOError:
|
2009-11-07 00:00:36 +00:00
|
|
|
log.exception(u'Loading verses from file failed')
|
2009-12-27 22:19:39 +00:00
|
|
|
success = False
|
2009-11-07 00:00:36 +00:00
|
|
|
finally:
|
2010-01-24 07:08:14 +00:00
|
|
|
if verse_file:
|
|
|
|
verse_file.close()
|
2010-02-06 15:33:23 +00:00
|
|
|
if self.stop_import_flag:
|
2009-12-27 22:19:39 +00:00
|
|
|
return False
|
|
|
|
else:
|
2010-12-27 22:57:35 +00:00
|
|
|
return success
|
2011-01-27 03:13:43 +00:00
|
|
|
|
|
|
|
def get_file_encoding(filename):
|
|
|
|
"""
|
|
|
|
Utility function to get the file encoding.
|
|
|
|
"""
|
|
|
|
detect_file = None
|
|
|
|
try:
|
|
|
|
detect_file = open(filename, 'r')
|
|
|
|
details = chardet.detect(detect_file.read(1024))
|
|
|
|
except IOError:
|
|
|
|
log.exception(u'Error detecting file encoding')
|
|
|
|
finally:
|
|
|
|
if detect_file:
|
|
|
|
detect_file.close()
|
|
|
|
return details
|