raoul/openlp
1
0
Fork 0
forked from openlp/openlp
Code Issues Pull Requests Projects Releases Wiki Activity

attempt to clean up bible importers

Browse Source
This commit is contained in:
Andreas Preikschat 2010-12-06 20:30:04 +01:00
parent 9db57568b0
commit d5e0643b45
7 changed files with 56 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
openlp/plugins/bibles/forms/bibleimportform.py
View File

@ -405,6 +405,9 @@ class BibleImportForm(QtGui.QWizard, Ui_BibleImportWizard):
Receiver.send_message(u'openlp_process_events') Receiver.send_message(u'openlp_process_events')
def preImport(self): def preImport(self):
"""
Prepare the UI for the import.
"""
bible_type = self.field(u'source_format').toInt()[0] bible_type = self.field(u'source_format').toInt()[0]
self.finishButton.setVisible(False) self.finishButton.setVisible(False)
self.ImportProgressBar.setMinimum(0) self.ImportProgressBar.setMinimum(0)
@ -420,6 +423,9 @@ class BibleImportForm(QtGui.QWizard, Ui_BibleImportWizard):
Receiver.send_message(u'openlp_process_events') Receiver.send_message(u'openlp_process_events')
def performImport(self): def performImport(self):
"""
Perform the actual import.
"""
bible_type = self.field(u'source_format').toInt()[0] bible_type = self.field(u'source_format').toInt()[0]
license_version = unicode(self.field(u'license_version').toString()) license_version = unicode(self.field(u'license_version').toString())
license_copyright = unicode(self.field(u'license_copyright').toString()) license_copyright = unicode(self.field(u'license_copyright').toString())
@ -469,7 +475,7 @@ class BibleImportForm(QtGui.QWizard, Ui_BibleImportWizard):
elif bible_type == BibleFormat.OLP1: elif bible_type == BibleFormat.OLP1:
# Import an openlp.org 1.x bible. # Import an openlp.org 1.x bible.
importer = self.manager.import_bible(BibleFormat.OLP1, importer = self.manager.import_bible(BibleFormat.OLP1,
name=license_version, name=license_version,
filename=unicode(self.field(u'OLP1_location').toString()) filename=unicode(self.field(u'OLP1_location').toString())
) )
if importer.do_import(): if importer.do_import():
@ -485,9 +491,8 @@ class BibleImportForm(QtGui.QWizard, Ui_BibleImportWizard):
self.ImportProgressLabel.setText(translate( self.ImportProgressLabel.setText(translate(
'BiblesPlugin.ImportWizardForm', 'Finished import.')) 'BiblesPlugin.ImportWizardForm', 'Finished import.'))
else: else:
self.ImportProgressLabel.setText( self.ImportProgressLabel.setText(translate(
translate('BiblesPlugin.ImportWizardForm', 'BiblesPlugin.ImportWizardForm', 'Your Bible import failed.'))
'Your Bible import failed.'))
delete_database(self.bibleplugin.settingsSection, importer.file) delete_database(self.bibleplugin.settingsSection, importer.file)
def postImport(self): def postImport(self):

25
openlp/plugins/bibles/lib/csvbible.py
View File

@ -30,7 +30,7 @@ import csv
from PyQt4 import QtCore from PyQt4 import QtCore
from openlp.core.lib import Receiver from openlp.core.lib import Receiver, translate
from db import BibleDB from db import BibleDB
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -46,28 +46,25 @@ class CSVBible(BibleDB):
This class assumes the files contain all the information and This class assumes the files contain all the information and
a clean bible is being loaded. a clean bible is being loaded.
""" """
BibleDB.__init__(self, parent, **kwargs)
log.info(self.__class__.__name__) log.info(self.__class__.__name__)
if u'booksfile' not in kwargs: BibleDB.__init__(self, parent, **kwargs)
raise KeyError(u'You have to supply a file to import books from.')
self.booksfile = kwargs[u'booksfile'] self.booksfile = kwargs[u'booksfile']
if u'versefile' not in kwargs:
raise KeyError(u'You have to supply a file to import verses from.')
self.versesfile = kwargs[u'versefile'] self.versesfile = kwargs[u'versefile']
QtCore.QObject.connect(Receiver.get_receiver(), QtCore.QObject.connect(Receiver.get_receiver(),
QtCore.SIGNAL(u'bibles_stop_import'), self.stop_import) QtCore.SIGNAL(u'bibles_stop_import'), self.stop_import)
def do_import(self): def do_import(self):
#Populate the Tables
success = True success = True
books_file = None books_file = None
book_ptr = None
verse_file = None
# Populate the Tables
try: try:
books_file = open(self.booksfile, 'r') books_file = open(self.booksfile, 'r')
dialect = csv.Sniffer().sniff(books_file.read(1024)) dialect = csv.Sniffer().sniff(books_file.read(1024))
books_file.seek(0) books_file.seek(0)
books_reader = csv.reader(books_file, dialect) books_reader = csv.reader(books_file, dialect)
for line in books_reader: for line in books_reader:
# cancel pressed
if self.stop_import_flag: if self.stop_import_flag:
break break
details = chardet.detect(line[1]) details = chardet.detect(line[1])
@ -82,25 +79,24 @@ class CSVBible(BibleDB):
books_file.close() books_file.close()
if not success: if not success:
return False return False
verse_file = None
try: try:
book_ptr = None
verse_file = open(self.versesfile, 'r') verse_file = open(self.versesfile, 'r')
dialect = csv.Sniffer().sniff(verse_file.read(1024)) dialect = csv.Sniffer().sniff(verse_file.read(1024))
verse_file.seek(0) verse_file.seek(0)
verse_reader = csv.reader(verse_file, dialect) verse_reader = csv.reader(verse_file, dialect)
for line in verse_reader: for line in verse_reader:
if self.stop_import_flag: # cancel pressed if self.stop_import_flag:
break break
details = chardet.detect(line[3]) details = chardet.detect(line[3])
if book_ptr != line[0]: if book_ptr != line[0]:
book = self.get_book(line[0]) book = self.get_book(line[0])
book_ptr = book.name book_ptr = book.name
self.wizard.incrementProgressBar( self.wizard.incrementProgressBar(u'%s %s %s...' % (
u'Importing %s %s' % (book.name, line[1])) translate('BiblesPlugin.CSVImport', 'Importing'),
book.name, line[1]))
self.session.commit() self.session.commit()
self.create_verse(book.id, line[1], line[2], self.create_verse(book.id, line[1], line[2],
unicode(line[3], details['encoding'])) unicode(line[3], details['encoding']))
Receiver.send_message(u'openlp_process_events') Receiver.send_message(u'openlp_process_events')
self.session.commit() self.session.commit()
except IOError: except IOError:
@ -110,7 +106,6 @@ class CSVBible(BibleDB):
if verse_file: if verse_file:
verse_file.close() verse_file.close()
if self.stop_import_flag: if self.stop_import_flag:
self.wizard.incrementProgressBar(u'Import canceled!')
return False return False
else: else:
return success return success

13
openlp/plugins/bibles/lib/http.py
View File

@ -333,24 +333,17 @@ class HTTPBible(BibleDB):
Init confirms the bible exists and stores the database path. Init confirms the bible exists and stores the database path.
""" """
BibleDB.__init__(self, parent, **kwargs) BibleDB.__init__(self, parent, **kwargs)
if u'download_source' not in kwargs:
raise KeyError(u'Missing keyword argument "download_source"')
if u'download_name' not in kwargs:
raise KeyError(u'Missing keyword argument "download_name"')
self.download_source = kwargs[u'download_source'] self.download_source = kwargs[u'download_source']
self.download_name = kwargs[u'download_name'] self.download_name = kwargs[u'download_name']
self.proxy_server = None
self.proxy_username = None
self.proxy_password = None
if u'proxy_server' in kwargs: if u'proxy_server' in kwargs:
self.proxy_server = kwargs[u'proxy_server'] self.proxy_server = kwargs[u'proxy_server']
else:
self.proxy_server = None
if u'proxy_username' in kwargs: if u'proxy_username' in kwargs:
self.proxy_username = kwargs[u'proxy_username'] self.proxy_username = kwargs[u'proxy_username']
else:
self.proxy_username = None
if u'proxy_password' in kwargs: if u'proxy_password' in kwargs:
self.proxy_password = kwargs[u'proxy_password'] self.proxy_password = kwargs[u'proxy_password']
else:
self.proxy_password = None
def do_import(self): def do_import(self):
""" """

2
openlp/plugins/bibles/lib/manager.py
View File

@ -34,10 +34,10 @@ from openlp.plugins.bibles.lib import parse_reference
from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta
from csvbible import CSVBible from csvbible import CSVBible
from http import HTTPBible
from olp1 import OpenLP1Bible from olp1 import OpenLP1Bible
from opensong import OpenSongBible from opensong import OpenSongBible
from osis import OSISBible from osis import OSISBible
from http import HTTPBible
log = logging.getLogger(__name__) log = logging.getLogger(__name__)

27
openlp/plugins/bibles/lib/olp1.py
View File

@ -27,8 +27,9 @@
import logging import logging
import sqlite import sqlite
#from openlp.core.lib import Receiver, translate from PyQt4 import QtCore
from openlp.core.lib import translate
from openlp.core.lib import Receiver, translate
from db import BibleDB from db import BibleDB
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -41,22 +42,19 @@ class OpenLP1Bible(BibleDB):
""" """
Constructor. Constructor.
""" """
log.debug(__name__) log.debug(self.__class__.__name__)
BibleDB.__init__(self, parent, **kwargs) BibleDB.__init__(self, parent, **kwargs)
if 'filename' not in kwargs: self.filename = kwargs[u'filename']
raise KeyError(u'You have to supply a file name to import from.') self.name = kwargs[u'name']
self.filename = kwargs['filename'] QtCore.QObject.connect(Receiver.get_receiver(),
# QtCore.QObject.connect(Receiver.get_receiver(), QtCore.SIGNAL(u'bibles_stop_import'), self.stop_import)
# QtCore.SIGNAL(u'bibles_stop_import'), self.stop_import)
def do_import(self): def do_import(self):
""" """
Imports an openlp.org v1 bible. Imports an openlp.org v1 bible.
""" """
# TODO: stop_import_flag
connection = None connection = None
cursor = None cursor = None
self.wizard.incrementProgressBar(u'Preparing for import...')
try: try:
connection = sqlite.connect(self.filename) connection = sqlite.connect(self.filename)
cursor = connection.cursor() cursor = connection.cursor()
@ -66,18 +64,23 @@ class OpenLP1Bible(BibleDB):
cursor.execute(u'SELECT id, testament_id, name, abbreviation FROM book') cursor.execute(u'SELECT id, testament_id, name, abbreviation FROM book')
books = cursor.fetchall() books = cursor.fetchall()
for book in books: for book in books:
if self.stop_import_flag:
return False
book_id = int(book[0]) book_id = int(book[0])
testament_id = int(book[1]) testament_id = int(book[1])
name = unicode(book[2], u'cp1252') name = unicode(book[2], u'cp1252')
abbreviation = unicode(book[3], u'cp1252') abbreviation = unicode(book[3], u'cp1252')
self.wizard.incrementProgressBar(unicode('%s %s' % (translate(
'BiblesPlugin.olp1', 'Importing'), name)))
self.create_book(name, abbreviation, testament_id) self.create_book(name, abbreviation, testament_id)
# Update the progess bar.
self.wizard.incrementProgressBar(u'%s %s...' % (translate(
'BiblesPlugin.OpenLP1Import', 'Importing'), name))
# Import the verses for this book. # Import the verses for this book.
cursor.execute(u'SELECT chapter, verse, text || \'\' AS text FROM ' cursor.execute(u'SELECT chapter, verse, text || \'\' AS text FROM '
'verse WHERE book_id=%s' % book_id) 'verse WHERE book_id=%s' % book_id)
verses = cursor.fetchall() verses = cursor.fetchall()
for verse in verses: for verse in verses:
if self.stop_import_flag:
return False
chapter = int(verse[0]) chapter = int(verse[0])
verse_number = int(verse[1]) verse_number = int(verse[1])
text = unicode(verse[2], u'cp1252') text = unicode(verse[2], u'cp1252')

11
openlp/plugins/bibles/lib/opensong.py
View File

@ -44,10 +44,8 @@ class OpenSongBible(BibleDB):
Constructor to create and set up an instance of the OpenSongBible Constructor to create and set up an instance of the OpenSongBible
class. This class is used to import Bibles from OpenSong's XML format. class. This class is used to import Bibles from OpenSong's XML format.
""" """
log.debug(__name__) log.debug(self.__class__.__name__)
BibleDB.__init__(self, parent, **kwargs) BibleDB.__init__(self, parent, **kwargs)
if 'filename' not in kwargs:
raise KeyError(u'You have to supply a file name to import from.')
self.filename = kwargs['filename'] self.filename = kwargs['filename']
QtCore.QObject.connect(Receiver.get_receiver(), QtCore.QObject.connect(Receiver.get_receiver(),
QtCore.SIGNAL(u'bibles_stop_import'), self.stop_import) QtCore.SIGNAL(u'bibles_stop_import'), self.stop_import)
@ -59,7 +57,6 @@ class OpenSongBible(BibleDB):
log.debug(u'Starting OpenSong import from "%s"' % self.filename) log.debug(u'Starting OpenSong import from "%s"' % self.filename)
if not isinstance(self.filename, unicode): if not isinstance(self.filename, unicode):
self.filename = unicode(self.filename, u'utf8') self.filename = unicode(self.filename, u'utf8')
self.wizard.incrementProgressBar(u'Preparing for import...')
file = None file = None
success = True success = True
try: try:
@ -87,10 +84,9 @@ class OpenSongBible(BibleDB):
unicode(verse.text) unicode(verse.text)
) )
Receiver.send_message(u'openlp_process_events') Receiver.send_message(u'openlp_process_events')
self.wizard.incrementProgressBar( self.wizard.incrementProgressBar(u'%s %s %s...' % (
QtCore.QString('%s %s %s' % (
translate('BiblesPlugin.Opensong', 'Importing'), translate('BiblesPlugin.Opensong', 'Importing'),
db_book.name, chapter.attrib[u'n']))) db_book.name, chapter.attrib[u'n']))
self.session.commit() self.session.commit()
except IOError: except IOError:
log.exception(u'Loading bible from OpenSong file failed') log.exception(u'Loading bible from OpenSong file failed')
@ -99,7 +95,6 @@ class OpenSongBible(BibleDB):
if file: if file:
file.close() file.close()
if self.stop_import_flag: if self.stop_import_flag:
self.wizard.incrementProgressBar(u'Import canceled!')
return False return False
else: else:
return success return success

34
openlp/plugins/bibles/lib/osis.py
View File

@ -33,7 +33,7 @@ import re
from PyQt4 import QtCore from PyQt4 import QtCore
from openlp.core.lib import Receiver from openlp.core.lib import Receiver, translate
from openlp.core.utils import AppLocation from openlp.core.utils import AppLocation
from db import BibleDB from db import BibleDB
@ -50,11 +50,11 @@ class OSISBible(BibleDB):
Constructor to create and set up an instance of the OpenSongBible Constructor to create and set up an instance of the OpenSongBible
class. This class is used to import Bibles from OpenSong's XML format. class. This class is used to import Bibles from OpenSong's XML format.
""" """
log.debug(__name__) log.debug(self.__class__.__name__)
BibleDB.__init__(self, parent, **kwargs) BibleDB.__init__(self, parent, **kwargs)
if u'filename' not in kwargs:
raise KeyError(u'You have to supply a file name to import from.')
self.filename = kwargs[u'filename'] self.filename = kwargs[u'filename']
fbibles = None
self.books = {}
self.verse_regex = re.compile( self.verse_regex = re.compile(
r'<verse osisID="([a-zA-Z0-9 ]*).([0-9]*).([0-9]*)">(.*?)</verse>') r'<verse osisID="([a-zA-Z0-9 ]*).([0-9]*).([0-9]*)">(.*?)</verse>')
self.note_regex = re.compile(r'<note(.*?)>(.*?)</note>') self.note_regex = re.compile(r'<note(.*?)>(.*?)</note>')
@ -72,11 +72,9 @@ class OSISBible(BibleDB):
self.divineName_regex = re.compile( self.divineName_regex = re.compile(
r'<divineName(.*?)>(.*?)</divineName>') r'<divineName(.*?)>(.*?)</divineName>')
self.spaces_regex = re.compile(r'([ ]{2,})') self.spaces_regex = re.compile(r'([ ]{2,})')
self.books = {}
filepath = os.path.join( filepath = os.path.join(
AppLocation.get_directory(AppLocation.PluginsDir), u'bibles', AppLocation.get_directory(AppLocation.PluginsDir), u'bibles',
u'resources', u'osisbooks.csv') u'resources', u'osisbooks.csv')
fbibles = None
try: try:
fbibles = open(filepath, u'r') fbibles = open(filepath, u'r')
for line in fbibles: for line in fbibles:
@ -96,9 +94,15 @@ class OSISBible(BibleDB):
Loads a Bible from file. Loads a Bible from file.
""" """
log.debug(u'Starting OSIS import from "%s"' % self.filename) log.debug(u'Starting OSIS import from "%s"' % self.filename)
self.wizard.incrementProgressBar(
u'Detecting encoding (this may take a few minutes)...')
detect_file = None detect_file = None
db_book = None
osis = None
success = True
last_chapter = 0
testament = 1
match_count = 0
self.wizard.incrementProgressBar(translate('BiblesPlugin.OsisImport',
'Detecting encoding (this may take a few minutes)...'))
try: try:
detect_file = open(self.filename, u'r') detect_file = open(self.filename, u'r')
details = chardet.detect(detect_file.read(1048576)) details = chardet.detect(detect_file.read(1048576))
@ -108,14 +112,8 @@ class OSISBible(BibleDB):
finally: finally:
if detect_file: if detect_file:
detect_file.close() detect_file.close()
osis = None
success = True
try: try:
osis = codecs.open(self.filename, u'r', details['encoding']) osis = codecs.open(self.filename, u'r', details['encoding'])
last_chapter = 0
testament = 1
match_count = 0
db_book = None
for file_record in osis: for file_record in osis:
if self.stop_import_flag: if self.stop_import_flag:
break break
@ -142,9 +140,9 @@ class OSISBible(BibleDB):
if last_chapter != chapter: if last_chapter != chapter:
if last_chapter != 0: if last_chapter != 0:
self.session.commit() self.session.commit()
self.wizard.incrementProgressBar( self.wizard.incrementProgressBar(u'%s %s %s...' % (
u'Importing %s %s...' % \ translate('BiblesPlugin.OsisImport', 'Importing'),
(self.books[match.group(1)][0], chapter)) self.books[match.group(1)][0], chapter))
last_chapter = chapter last_chapter = chapter
# All of this rigmarol below is because the mod2osis # All of this rigmarol below is because the mod2osis
# tool from the Sword library embeds XML in the OSIS # tool from the Sword library embeds XML in the OSIS
@ -171,7 +169,6 @@ class OSISBible(BibleDB):
self.create_verse(db_book.id, chapter, verse, verse_text) self.create_verse(db_book.id, chapter, verse, verse_text)
Receiver.send_message(u'openlp_process_events') Receiver.send_message(u'openlp_process_events')
self.session.commit() self.session.commit()
self.wizard.incrementProgressBar(u'Finishing import...')
if match_count == 0: if match_count == 0:
success = False success = False
except (ValueError, IOError): except (ValueError, IOError):
@ -181,7 +178,6 @@ class OSISBible(BibleDB):
if osis: if osis:
osis.close() osis.close()
if self.stop_import_flag: if self.stop_import_flag:
self.wizard.incrementProgressBar(u'Import canceled!')
return False return False
else: else:
return success return success