raoul/openlp
1
0
Fork 0
forked from openlp/openlp
Code Issues Pull Requests Projects Releases Wiki Activity

Rewrote OSIS import, added tests.

Browse Source
This commit is contained in:
Tomas Groth 2014-08-24 15:40:45 +01:00
parent 43d3823bfb
commit 3101d72b48
9 changed files with 522 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
openlp/plugins/bibles/lib/opensong.py
View File

@ -30,7 +30,7 @@
import logging import logging
from lxml import etree, objectify from lxml import etree, objectify
from openlp.core.common import translate from openlp.core.common import translate, trace_error_handler
from openlp.core.lib.ui import critical_error_message_box from openlp.core.lib.ui import critical_error_message_box
from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
@ -134,6 +134,7 @@ class OpenSongBible(BibleDB):
self.session.commit() self.session.commit()
self.application.process_events() self.application.process_events()
except etree.XMLSyntaxError as inst: except etree.XMLSyntaxError as inst:
trace_error_handler(log)
critical_error_message_box( critical_error_message_box(
message=translate('BiblesPlugin.OpenSongImport', message=translate('BiblesPlugin.OpenSongImport',
'Incorrect Bible file type supplied. OpenSong Bibles may be ' 'Incorrect Bible file type supplied. OpenSong Bibles may be '

255
openlp/plugins/bibles/lib/osis.py
View File

@ -27,14 +27,12 @@
# Temple Place, Suite 330, Boston, MA 02111-1307 USA # # Temple Place, Suite 330, Boston, MA 02111-1307 USA #
############################################################################### ###############################################################################
import os
import logging import logging
import chardet from lxml import etree
import codecs
import re
from openlp.core.common import AppLocation, translate from openlp.core.common import translate, trace_error_handler
from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB
from openlp.core.lib.ui import critical_error_message_box
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -53,143 +51,146 @@ class OSISBible(BibleDB):
log.debug(self.__class__.__name__) log.debug(self.__class__.__name__)
BibleDB.__init__(self, parent, **kwargs) BibleDB.__init__(self, parent, **kwargs)
self.filename = kwargs['filename'] self.filename = kwargs['filename']
self.language_regex = re.compile(r'<language.*>(.*?)</language>')
self.verse_regex = re.compile(r'<verse osisID="([a-zA-Z0-9 ]*).([0-9]*).([0-9]*)">(.*?)</verse>')
self.note_regex = re.compile(r'<note(.*?)>(.*?)</note>')
self.title_regex = re.compile(r'<title(.*?)>(.*?)</title>')
self.milestone_regex = re.compile(r'<milestone(.*?)/>')
self.fi_regex = re.compile(r'<FI>(.*?)<Fi>')
self.rf_regex = re.compile(r'<RF>(.*?)<Rf>')
self.lb_regex = re.compile(r'<lb(.*?)>')
self.lg_regex = re.compile(r'<lg(.*?)>')
self.l_regex = re.compile(r'<l (.*?)>')
self.w_regex = re.compile(r'<w (.*?)>')
self.q_regex = re.compile(r'<q(.*?)>')
self.q1_regex = re.compile(r'<q(.*?)level="1"(.*?)>')
self.q2_regex = re.compile(r'<q(.*?)level="2"(.*?)>')
self.trans_regex = re.compile(r'<transChange(.*?)>(.*?)</transChange>')
self.divine_name_regex = re.compile(r'<divineName(.*?)>(.*?)</divineName>')
self.spaces_regex = re.compile(r'([ ]{2,})')
filepath = os.path.join(
AppLocation.get_directory(AppLocation.PluginsDir), 'bibles', 'resources', 'osisbooks.csv')
def do_import(self, bible_name=None): def do_import(self, bible_name=None):
""" """
Loads a Bible from file. Loads a Bible from file.
""" """
log.debug('Starting OSIS import from "%s"' % self.filename) log.debug('Starting OSIS import from "%s"' % self.filename)
detect_file = None if not isinstance(self.filename, str):
db_book = None self.filename = str(self.filename, 'utf8')
osis = None import_file = None
success = True success = True
last_chapter = 0
match_count = 0
self.wizard.increment_progress_bar(
translate('BiblesPlugin.OsisImport', 'Detecting encoding (this may take a few minutes)...'))
try: try:
detect_file = open(self.filename, 'r') # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding
details = chardet.detect(detect_file.read(1048576)) # detection, and the two mechanisms together interfere with each other.
detect_file.seek(0) import_file = open(self.filename, 'rb')
lines_in_file = int(len(detect_file.readlines())) language_id = self.get_language(bible_name)
except IOError: if not language_id:
log.exception('Failed to detect OSIS file encoding') log.error('Importing books from "%s" failed' % self.filename)
return return False
finally: osis_bible_tree = etree.parse(import_file)
if detect_file: namespace = {'ns': 'http://www.bibletechnologies.net/2003/OSIS/namespace'}
detect_file.close() num_books = int(osis_bible_tree.xpath("count(//ns:div[@type='book'])", namespaces=namespace))
try: log.debug('number of books: %d' % num_books)
osis = codecs.open(self.filename, 'r', details['encoding']) self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport',
repl = replacement 'Removing unused tags (this may take a few minutes)...'))
language_id = False # We strip unused tags from the XML, this should leave us with only chapter, verse and div tags.
# Decide if the bible probably contains only NT or AT and NT or # Strip tags we don't use - keep content
# AT, NT and Apocrypha etree.strip_tags(osis_bible_tree, ('{http://www.bibletechnologies.net/2003/OSIS/namespace}p',
if lines_in_file < 11500: '{http://www.bibletechnologies.net/2003/OSIS/namespace}l',
book_count = 27 '{http://www.bibletechnologies.net/2003/OSIS/namespace}lg',
chapter_count = 260 '{http://www.bibletechnologies.net/2003/OSIS/namespace}q',
elif lines_in_file < 34200: '{http://www.bibletechnologies.net/2003/OSIS/namespace}a',
book_count = 66 '{http://www.bibletechnologies.net/2003/OSIS/namespace}w',
chapter_count = 1188 '{http://www.bibletechnologies.net/2003/OSIS/namespace}divineName',
else: '{http://www.bibletechnologies.net/2003/OSIS/namespace}foreign',
book_count = 67 '{http://www.bibletechnologies.net/2003/OSIS/namespace}hi',
chapter_count = 1336 '{http://www.bibletechnologies.net/2003/OSIS/namespace}inscription',
for file_record in osis: '{http://www.bibletechnologies.net/2003/OSIS/namespace}mentioned',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}name',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}reference',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}seg',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}transChange',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}salute',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}signed',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}closer',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}speech',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}speaker',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}list',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}item',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}table',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}head',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}row',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}cell',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}caption'))
# Strip tags we don't use - remove content
etree.strip_elements(osis_bible_tree, ('{http://www.bibletechnologies.net/2003/OSIS/namespace}note',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}milestone',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}title',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}abbr',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}catchWord',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}index',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}rdg',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}rdgGroup',
'{http://www.bibletechnologies.net/2003/OSIS/namespace}figure'),
with_tail=False)
# Precompile a few xpath-querys
verse_in_chapter = etree.XPath('count(//ns:chapter[1]/ns:verse)', namespaces=namespace)
text_in_verse = etree.XPath('count(//ns:verse[1]/text())', namespaces=namespace)
# Find books in the bible
bible_books = osis_bible_tree.xpath("//ns:div[@type='book']", namespaces=namespace)
for book in bible_books:
if self.stop_import_flag: if self.stop_import_flag:
break break
# Try to find the bible language # Remove div-tags in the book
if not language_id: etree.strip_tags(book, ('{http://www.bibletechnologies.net/2003/OSIS/namespace}div'))
language_match = self.language_regex.search(file_record) book_ref_id = self.get_book_ref_id_by_name(book.get('osisID'), num_books)
if language_match: if not book_ref_id:
language = BiblesResourcesDB.get_language( book_ref_id = self.get_book_ref_id_by_localised_name(book.get('osisID'))
language_match.group(1)) if not book_ref_id:
if language: log.error('Importing books from "%s" failed' % self.filename)
language_id = language['id'] return False
self.save_meta('language_id', language_id) book_details = BiblesResourcesDB.get_book_by_id(book_ref_id)
continue db_book = self.create_book(book_details['name'], book_ref_id, book_details['testament_id'])
match = self.verse_regex.search(file_record) # Find out if chapter-tags contains the verses, or if it is used as milestone/anchor
if match: if int(verse_in_chapter(book)) > 0:
# Set meta language_id if not detected till now # The chapter tags contains the verses
if not language_id: for chapter in book:
language_id = self.get_language(bible_name) chapter_number = chapter.get("osisID").split('.')[1]
if not language_id: # Find out if verse-tags contains the text, or if it is used as milestone/anchor
log.error('Importing books from "%s" failed' % self.filename) if int(text_in_verse(chapter)) == 0:
return False # verse-tags are used as milestone
match_count += 1 for verse in chapter:
book = str(match.group(1)) # If this tag marks the start of a verse, the verse text is between this tag and
chapter = int(match.group(2)) # the next tag, which the "tail" attribute gives us.
verse = int(match.group(3)) if verse.get('sID'):
verse_text = match.group(4) verse_number = verse.get("osisID").split('.')[2]
book_ref_id = self.get_book_ref_id_by_name(book, book_count, language_id) verse_text = verse.tail
if not book_ref_id: if verse_text:
log.error('Importing books from "%s" failed' % self.filename) self.create_verse(db_book.id, chapter_number, verse_number, verse_text.strip())
return False else:
book_details = BiblesResourcesDB.get_book_by_id(book_ref_id) # Verse-tags contains the text
if not db_book or db_book.name != book_details['name']: for verse in chapter:
log.debug('New book: "%s"' % book_details['name']) verse_number = verse.get("osisID").split('.')[2]
db_book = self.create_book( self.create_verse(db_book.id, chapter_number, verse_number, verse.text.strip())
book_details['name'],
book_ref_id,
book_details['testament_id'])
if last_chapter == 0:
self.wizard.progress_bar.setMaximum(chapter_count)
if last_chapter != chapter:
if last_chapter != 0:
self.session.commit()
self.wizard.increment_progress_bar( self.wizard.increment_progress_bar(
translate('BiblesPlugin.OsisImport', 'Importing %s %s...', translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...' %
'Importing <book name> <chapter>...') % (book_details['name'], chapter)) {'bookname': db_book.name, 'chapter': chapter_number}))
last_chapter = chapter else:
# All of this rigmarole below is because the mod2osis tool from the Sword library embeds XML in the log.debug('chapters are milestones')
# OSIS but neglects to enclose the verse text (with XML) in <[CDATA[ ]]> tags. # The chapter tags is used as milestones. For now we assume verses is also milestones
verse_text = self.note_regex.sub('', verse_text) chapter_number = 0
verse_text = self.title_regex.sub('', verse_text) for element in book:
verse_text = self.milestone_regex.sub('', verse_text) if element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' \
verse_text = self.fi_regex.sub('', verse_text) and element.get('sID'):
verse_text = self.rf_regex.sub('', verse_text) chapter_number = element.get("osisID").split('.')[1]
verse_text = self.lb_regex.sub(' ', verse_text) self.wizard.increment_progress_bar(
verse_text = self.lg_regex.sub('', verse_text) translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...' %
verse_text = self.l_regex.sub(' ', verse_text) {'bookname': db_book.name, 'chapter': chapter_number}))
verse_text = self.w_regex.sub('', verse_text) elif element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}verse' \
verse_text = self.q1_regex.sub('"', verse_text) and element.get('sID'):
verse_text = self.q2_regex.sub('\'', verse_text) # If this tag marks the start of a verse, the verse text is between this tag and
verse_text = self.q_regex.sub('', verse_text) # the next tag, which the "tail" attribute gives us.
verse_text = self.divine_name_regex.sub(repl, verse_text) verse_number = element.get("osisID").split('.')[2]
verse_text = self.trans_regex.sub('', verse_text) verse_text = element.tail
verse_text = verse_text.replace('</lb>', '') \ if verse_text:
.replace('</l>', '').replace('<lg>', '') \ self.create_verse(db_book.id, chapter_number, verse_number, verse_text.strip())
.replace('</lg>', '').replace('</q>', '') \ self.session.commit()
.replace('</div>', '').replace('</w>', '') self.application.process_events()
verse_text = self.spaces_regex.sub(' ', verse_text)
self.create_verse(db_book.id, chapter, verse, verse_text)
self.application.process_events()
self.session.commit()
if match_count == 0:
success = False
except (ValueError, IOError): except (ValueError, IOError):
log.exception('Loading bible from OSIS file failed') log.exception('Loading bible from OSIS file failed')
trace_error_handler(log)
success = False success = False
except etree.XMLSyntaxError as e:
log.exception('Loading bible from OSIS file failed')
trace_error_handler(log)
success = False
critical_error_message_box(message=translate('BiblesPlugin.OsisImport',
'The file is not a valid OSIS-XML file: \n%s' % e.msg))
finally: finally:
if osis: if import_file:
osis.close() import_file.close()
if self.stop_import_flag: if self.stop_import_flag:
return False return False
else: else:

162
tests/functional/openlp_plugins/bibles/test_osisimport.py Normal file
View File

@ -0,0 +1,162 @@
# -*- coding: utf-8 -*-
# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
###############################################################################
# OpenLP - Open Source Lyrics Projection #
# --------------------------------------------------------------------------- #
# Copyright (c) 2008-2014 Raoul Snyman #
# Portions copyright (c) 2008-2014 Tim Bentley, Gerald Britton, Jonathan #
# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub, #
# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer. #
# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru, #
# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith, #
# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock, #
# Frode Woldsund, Martin Zibricky, Patrick Zimmermann #
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it #
# under the terms of the GNU General Public License as published by the Free #
# Software Foundation; version 2 of the License. #
# #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for #
# more details. #
# #
# You should have received a copy of the GNU General Public License along #
# with this program; if not, write to the Free Software Foundation, Inc., 59 #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
###############################################################################
"""
This module contains tests for the OSIS Bible importer.
"""
import os
import json
from unittest import TestCase
from tests.functional import MagicMock, patch
from openlp.plugins.bibles.lib.osis import OSISBible
from openlp.plugins.bibles.lib.db import BibleDB
TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..', '..', 'resources', 'bibles'))
class TestOsisImport(TestCase):
"""
Test the functions in the :mod:`osisimport` module.
"""
def setUp(self):
self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry')
self.registry_patcher.start()
self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager')
self.manager_patcher.start()
def tearDown(self):
self.registry_patcher.stop()
self.manager_patcher.stop()
def create_importer_test(self):
"""
Test creating an instance of the OSIS file importer
"""
# GIVEN: A mocked out "manager"
mocked_manager = MagicMock()
# WHEN: An importer object is created
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
# THEN: The importer should be an instance of BibleDB
self.assertIsInstance(importer, BibleDB)
def file_import_nested_tags_test(self):
"""
Test the actual import of OSIS Bible file, with nested chapter and verse tags
"""
# GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions
# get_book_ref_id_by_name, create_verse, create_book, session and get_language.
result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'osis-dk1933.xml'
with patch('openlp.plugins.bibles.lib.osis.OSISBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
importer.wizard = mocked_import_wizard
importer.get_book_ref_id_by_name = MagicMock()
importer.create_verse = MagicMock()
importer.create_book = MagicMock()
importer.session = MagicMock()
importer.get_language = MagicMock()
importer.get_language.return_value = 'Danish'
# WHEN: Importing bible file
importer.filename = os.path.join(TEST_PATH, bible_file)
importer.do_import()
# THEN: The create_verse() method should have been called with each verse in the file.
self.assertTrue(importer.create_verse.called)
for verse_tag, verse_text in test_data['verses']:
importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text)
def file_import_mixed_tags_test(self):
"""
Test the actual import of OSIS Bible file, with nested chapter and milestone verse tags.
"""
# GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions
# get_book_ref_id_by_name, create_verse, create_book, session and get_language.
result_file = open(os.path.join(TEST_PATH, 'kjv.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'osis-kjv.xml'
with patch('openlp.plugins.bibles.lib.osis.OSISBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
importer.wizard = mocked_import_wizard
importer.get_book_ref_id_by_name = MagicMock()
importer.create_verse = MagicMock()
importer.create_book = MagicMock()
importer.session = MagicMock()
importer.get_language = MagicMock()
importer.get_language.return_value = 'English'
# WHEN: Importing bible file
importer.filename = os.path.join(TEST_PATH, bible_file)
importer.do_import()
# THEN: The create_verse() method should have been called with each verse in the file.
self.assertTrue(importer.create_verse.called)
for verse_tag, verse_text in test_data['verses']:
importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text)
def file_import_milestone_tags_test(self):
"""
Test the actual import of OSIS Bible file, with milestone chapter and verse tags.
"""
# GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions
# get_book_ref_id_by_name, create_verse, create_book, session and get_language.
result_file = open(os.path.join(TEST_PATH, 'web.json'), 'rb')
test_data = json.loads(result_file.read().decode())
bible_file = 'osis-web.xml'
with patch('openlp.plugins.bibles.lib.osis.OSISBible.application'):
mocked_manager = MagicMock()
mocked_import_wizard = MagicMock()
importer = OSISBible(mocked_manager, path='.', name='.', filename='')
importer.wizard = mocked_import_wizard
importer.get_book_ref_id_by_name = MagicMock()
importer.create_verse = MagicMock()
importer.create_book = MagicMock()
importer.session = MagicMock()
importer.get_language = MagicMock()
importer.get_language.return_value = 'English'
# WHEN: Importing bible file
importer.filename = os.path.join(TEST_PATH, bible_file)
importer.do_import()
# THEN: The create_verse() method should have been called with each verse in the file.
self.assertTrue(importer.create_verse.called)
print(importer.create_verse.call_list())
for verse_tag, verse_text in test_data['verses']:
importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text)

16
tests/resources/bibles/dk1933.json Normal file
View File

@ -0,0 +1,16 @@
{
"book": "Genesis",
"chapter": 1,
"verses": [
[ "1", "I Begyndelsen skabte Gud Himmelen og Jorden."],
[ "2", "Og Jorden var øde og tom, og der var Mørke over Verdensdybet. Men Guds Ånd svævede over Vandene." ],
[ "3", "Og Gud sagde: \"Der blive Lys!\" Og der blev Lys." ],
[ "4", "Og Gud så, at Lyset var godt, og Gud satte Skel mellem Lyset og Mørket," ],
[ "5", "og Gud kaldte Lyset Dag, og Mørket kaldte han Nat. Og det blev Aften, og det blev Morgen, første Dag." ],
[ "6", "Derpå sagde Gud: \"Der blive en Hvælving midt i Vandene til at skille Vandene ad!\"" ],
[ "7", "Og således skete det: Gud gjorde Hvælvingen og skilte Vandet under Hvælvingen fra Vandet over Hvælvingen;" ],
[ "8", "og Gud kaldte Hvælvingen Himmel. Og det blev Aften, og det blev Morgen, anden Dag." ],
[ "9", "Derpå sagde Gud: \"Vandet under Himmelen samle sig på eet Sted, så det faste Land kommer til Syne!\" Og således skete det;" ],
[ "10", "og Gud kaldte det faste Land Jord, og Stedet, hvor Vandet samlede sig, kaldte han Hav. Og Gud så, at det var godt." ]
]
}

16
tests/resources/bibles/kjv.json Normal file
View File

@ -0,0 +1,16 @@
{
"book": "Genesis",
"chapter": 1,
"verses": [
[ "1", "In the beginning God created the heaven and the earth."],
[ "2", "And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters." ],
[ "3", "And God said, Let there be light: and there was light." ],
[ "4", "And God saw the light, that it was good: and God divided the light from the darkness." ],
[ "5", "And God called the light Day, and the darkness he called Night. And the evening and the morning were the first day." ],
[ "6", "And God said, Let there be a firmament in the midst of the waters, and let it divide the waters from the waters." ],
[ "7", "And God made the firmament, and divided the waters which were under the firmament from the waters which were above the firmament: and it was so." ],
[ "8", "And God called the firmament Heaven. And the evening and the morning were the second day." ],
[ "9", "And God said, Let the waters under the heaven be gathered together unto one place, and let the dry land appear: and it was so." ],
[ "10", "And God called the dry land Earth; and the gathering together of the waters called he Seas: and God saw that it was good." ]
]
}

32
tests/resources/bibles/osis-dk1933.xml Normal file
View File

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8" ?>
<osis xmlns="http://www.bibletechnologies.net/2003/OSIS/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.bibletechnologies.net/2003/OSIS/namespace http://www.bibletechnologies.net/osisCore.2.1.1.xsd">
<osisText osisIDWork="DanDetteBiblen" osisRefWork="defaultReferenceScheme" xml:lang="da">
<header>
<work osisWork="DanDetteBiblen">
<title>Dette er Biblen</title>
<identifier type="OSIS">Bible.DanDetteBiblen</identifier>
<refSystem>Bible.KJV</refSystem>
</work>
<work osisWork="defaultReferenceScheme">
<refSystem>Bible.KJV</refSystem>
</work>
</header>
<div type="x-testament">
<div type="book" osisID="Gen">
<chapter osisID="Gen.1">
<verse osisID="Gen.1.1">I Begyndelsen skabte Gud Himmelen og Jorden.</verse>
<verse osisID="Gen.1.2">Og Jorden var øde og tom, og der var Mørke over Verdensdybet. Men Guds Ånd svævede over Vandene.<p /></verse>
<verse osisID="Gen.1.3">Og Gud sagde: "Der blive Lys!" Og der blev Lys.</verse>
<verse osisID="Gen.1.4">Og Gud så, at Lyset var godt, og Gud satte Skel mellem Lyset og Mørket,</verse>
<verse osisID="Gen.1.5">og Gud kaldte Lyset Dag, og Mørket kaldte han Nat. Og det blev Aften, og det blev Morgen, første Dag.<p /></verse>
<verse osisID="Gen.1.6">Derpå sagde Gud: "Der blive en Hvælving midt i Vandene til at skille Vandene ad!"</verse>
<verse osisID="Gen.1.7">Og således skete det: Gud gjorde Hvælvingen og skilte Vandet under Hvælvingen fra Vandet over Hvælvingen;</verse>
<verse osisID="Gen.1.8">og Gud kaldte Hvælvingen Himmel. Og det blev Aften, og det blev Morgen, anden Dag.<p /></verse>
<verse osisID="Gen.1.9">Derpå sagde Gud: "Vandet under Himmelen samle sig på eet Sted, så det faste Land kommer til Syne!" Og således skete det;</verse>
<verse osisID="Gen.1.10">og Gud kaldte det faste Land Jord, og Stedet, hvor Vandet samlede sig, kaldte han Hav. Og Gud så, at det var godt.</verse>
</chapter>
</div>
</div>
</osisText>
</osis>

41
tests/resources/bibles/osis-kjv.xml Normal file
View File

@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<osis xmlns="http://www.bibletechnologies.net/2003/OSIS/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.bibletechnologies.net/2003/OSIS/namespace http://www.bibletechnologies.net/osisCore.2.1.1.xsd">
<osisText osisIDWork="KJV" osisRefWork="defaultReferenceScheme" xml:lang="en">
<header>
<work osisWork="KJV">
<title>King James Version (1769) with Strongs Numbers and Morphology</title>
<identifier type="OSIS">Bible.KJV</identifier>
<scope>Gen-Rev</scope>
<refSystem>Bible.KJV</refSystem>
</work>
<work osisWork="defaultReferenceScheme">
<refSystem>Bible.KJV</refSystem>
</work>
<work osisWork="strong">
<refSystem>Dict.Strongs</refSystem>
</work>
<work osisWork="robinson">
<refSystem>Dict.Robinsons</refSystem>
</work>
<work osisWork="strongMorph">
<refSystem>Dict.strongMorph</refSystem>
</work>
</header>
<div type="book" osisID="Gen" canonical="true">
<title type="main">THE FIRST BOOK OF MOSES CALLED GENESIS</title>
<chapter osisID="Gen.1" chapterTitle="CHAPTER 1.">
<title type="chapter">CHAPTER 1.</title>
<verse osisID="Gen.1.1" sID="Gen.1.1"/><w lemma="strong:H07225">In the beginning</w> <w lemma="strong:H0430">God</w> <w morph="strongMorph:TH8804" lemma="strong:H0853 strong:H01254">created</w> <w lemma="strong:H08064">the heaven</w> <w lemma="strong:H0853">and</w> <w lemma="strong:H0776">the earth</w>.<verse eID="Gen.1.1"/>
<verse osisID="Gen.1.2" sID="Gen.1.2"/><w lemma="strong:H0776">And the earth</w> <w morph="strongMorph:TH8804" lemma="strong:H01961">was</w> <w lemma="strong:H08414">without form</w>, <w lemma="strong:H0922">and void</w>; <w lemma="strong:H02822">and darkness</w> <transChange type="added">was</transChange> <w lemma="strong:H06440">upon the face</w> <w lemma="strong:H08415">of the deep</w>. <w lemma="strong:H07307">And the Spirit</w> <w lemma="strong:H0430">of God</w> <w morph="strongMorph:TH8764" lemma="strong:H07363">moved</w> <w lemma="strong:H05921">upon</w> <w lemma="strong:H06440">the face</w> <w lemma="strong:H04325">of the waters</w>.<verse eID="Gen.1.2"/>
<verse osisID="Gen.1.3" sID="Gen.1.3"/><milestone type="x-extra-p"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H0559">said</w>, <w morph="strongMorph:TH8799" lemma="strong:H01961">Let there be</w> <w lemma="strong:H0216">light</w>: <w lemma="strong:H0216">and there was light</w>.<verse eID="Gen.1.3"/>
<verse osisID="Gen.1.4" sID="Gen.1.4"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H0853 strong:H07200">saw</w> <w lemma="strong:H0216">the light</w>, <w lemma="strong:H03588">that</w> <transChange type="added">it was</transChange> <w lemma="strong:H02896">good</w>: <w lemma="strong:H0430">and God</w> <w morph="strongMorph:TH8686" lemma="strong:H0996 strong:H0914">divided</w> <w lemma="strong:H0216">the light</w> <w lemma="strong:H0996">from</w> <w lemma="strong:H02822">the darkness</w>.<note type="study">the light from…: Heb. between the light and between the darkness</note><verse eID="Gen.1.4"/>
<verse osisID="Gen.1.5" sID="Gen.1.5"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H07121">called</w> <w lemma="strong:H0216">the light</w> <w lemma="strong:H03117">Day</w>, <w lemma="strong:H02822">and the darkness</w> <w morph="strongMorph:TH8804" lemma="strong:H07121">he called</w> <w lemma="strong:H03915">Night</w>. <w lemma="strong:H06153">And the evening</w> <w lemma="strong:H01242">and the morning</w> <w lemma="strong:H0259">were the first</w> <w lemma="strong:H03117">day</w>.<note type="study">And the evening…: Heb. And the evening was, and the morning was etc.</note><verse eID="Gen.1.5"/>
<verse osisID="Gen.1.6" sID="Gen.1.6"/><milestone type="x-p" marker="¶"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H0559">said</w>, <w lemma="strong:H07549">Let there be a firmament</w> <w lemma="strong:H08432">in the midst</w> <w lemma="strong:H04325">of the waters</w>, <w morph="strongMorph:TH8688" lemma="strong:H0914">and let it divide</w> <w lemma="strong:H04325">the waters</w> <w lemma="strong:H04325">from the waters</w>.<note type="study">firmament: Heb. expansion</note><verse eID="Gen.1.6"/>
<verse osisID="Gen.1.7" sID="Gen.1.7"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H06213">made</w> <w lemma="strong:H07549">the firmament</w>, <w morph="strongMorph:TH8686" lemma="strong:H0914">and divided</w> <w lemma="strong:H04325">the waters</w> <w lemma="strong:H0834">which</w> <transChange type="added">were</transChange> <w lemma="strong:H08478">under</w> <w lemma="strong:H07549">the firmament</w> <w lemma="strong:H04325">from the waters</w> <w lemma="strong:H0834">which</w> <transChange type="added">were</transChange> <w lemma="strong:H05921">above</w> <w lemma="strong:H07549">the firmament</w>: <w lemma="strong:H03651">and it was so</w>.<verse eID="Gen.1.7"/>
<verse osisID="Gen.1.8" sID="Gen.1.8"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H07121">called</w> <w lemma="strong:H07549">the firmament</w> <w lemma="strong:H08064">Heaven</w>. <w lemma="strong:H06153">And the evening</w> <w lemma="strong:H01242">and the morning</w> <w lemma="strong:H08145">were the second</w> <w lemma="strong:H03117">day</w>.<note type="study">And the evening…: Heb. And the evening was, and the morning was etc.</note><verse eID="Gen.1.8"/>
<verse osisID="Gen.1.9" sID="Gen.1.9"/><milestone type="x-p" marker="¶"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H0559">said</w>, <w lemma="strong:H04325">Let the waters</w> <w lemma="strong:H08064">under the heaven</w> <w morph="strongMorph:TH8735" lemma="strong:H06960">be gathered together</w> <w lemma="strong:H0413">unto</w> <w lemma="strong:H0259">one</w> <w lemma="strong:H04725">place</w>, <w lemma="strong:H03004">and let the dry</w> <transChange type="added">land</transChange> <w morph="strongMorph:TH8735" lemma="strong:H07200">appear</w>: and it was so.<verse eID="Gen.1.9"/>
<verse osisID="Gen.1.10" sID="Gen.1.10"/><w lemma="strong:H0430">And God</w> <w morph="strongMorph:TH8799" lemma="strong:H07121">called</w> <w lemma="strong:H03004">the dry</w> <transChange type="added">land</transChange> <w lemma="strong:H0776">Earth</w>; <w lemma="strong:H04723">and the gathering together</w> <w lemma="strong:H04325">of the waters</w> <w morph="strongMorph:TH8804" lemma="strong:H07121">called</w> <w lemma="strong:H03220">he Seas</w>: <w lemma="strong:H0430">and God</w> <w morph="strongMorph:TH8799" lemma="strong:H07200">saw</w> that <transChange type="added">it was</transChange> <w lemma="strong:H02896">good</w>.<verse eID="Gen.1.10"/>
</chapter>
</div>
</osisText>
</osis>

109
tests/resources/bibles/osis-web.xml Normal file
View File

@ -0,0 +1,109 @@
<?xml version="1.0" encoding="utf-8"?>
<osis xmlns="http://www.bibletechnologies.net/2003/OSIS/namespace" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.bibletechnologies.net/2003/OSIS/namespace osisCore.2.0.xsd">
<osisText osisIDWork="WEB" osisRefWork="Bible" xml:lang="en">
<header>
<revisionDesc resp="Rainbow Missions, Inc.">
<date>2007-08-26T08.23.41</date>
<p> This draft version of the World English Bible is
substantially complete in the New Testament, Genesis, Exodus, Job, Psalms, Proverbs, Ecclesiastes, Song of Solomon, and the “minor” prophets. Editing continues on the other books of the Old Testament. All WEB companion Apocrypha books are still in
rough draft form. </p>
<p>Converted web.gbf in GBF to web.osis.xml in
an XML format that is mostly compliant with OSIS 2.0 using gbf2osis.exe.
(Please see http://ebt.cx/translation/ for links to this software.)</p>
<p>GBF and OSIS metadata fields do not exactly correspond to each other, so
the conversion is not perfect in the metadata. However, the Scripture portion
should be correct.</p>
<p>No attempt was to convert quotation marks to structural markers using q or
speech elements, because this would require language and style-dependent
processing, and because the current OSIS specification is deficient in that
quotation mark processing is not guaranteed to produce the correct results
for all languages and translations. In English texts, the hard part of the
conversion to markup is figuring out what ’ means.
The other difficulty is that OSIS in no way guarantees that these punctuation
marks would be reconstituted properly by software that reads OSIS files
for anything other than modern English, and even then, it does not
accommodate all styles of punctuation and all cases.
We strongly recommend that anyone using OSIS NOT replace quotation mark
punctuation in any existing text with q or speech elements. It is better
for multiple language processing capabilities to leave the quotation
punctuation as part of the text. If you need the q or speech markup, then you
may supplement those punctuation marks with those markup elements, but specify
the n='' parameter in those elements to indicate that no generation of any
punctuation from those markup elements is required or desired. That way you
can have BOTH correct punctuation already in the text AND markup so that you
can automatically determine when you are in a quotation or not, independent
of language. This may be useful for a search by speaker, for example.</p>
<p>The output of gbf2osis marks Jesus' words in a non-standard way using the q
element AND quotation marks if they were marked with FR/Fr markers in the GBF
file. The OSIS 2.0 specification requires that quotation marks be stripped out,
and reinserted by software that reads the OSIS files when q elements are used.
This is not acceptable for the reasons given above, and we choose not to do
that, but we used the q element with who='Jesus' to indicate Jesus' words.
Do not generate any additional punctuation due to these markers. The correct
punctuation is already in the text.</p>
<p>OSIS does not currently support footnote start anchors. Therefore, these
start anchors have been represented with milestone elements, in case someone
might like to use them, for example, to start an href element in a conversion
to HTML. (OSIS sort of supports the same idea by allowing a catchword to be
defined within a footnote, but I did not implement the processing to convert
to this different way of doing things, and it isn't exactly the same, anyway.)</p>
<p>Traditional psalm book titles are rendered as text rather than titles, because
the title element does not support containing transChange elements, as would be
required to encode the KJV text using OSIS title elements. This may actually be
a superior solution, anyway, in that the Masoretic text makes no such distinction
(even though many modern typeset Bibles do make a typographic distinction in this
case).</p>
<p>The schema location headers were modified to use local copies rather than the
standard locations so that these files could be validated and used without an
Internet connection active at all times (very important for the developer's
remote island location), but you may wish to change them back.</p>
</revisionDesc>
<work osisWork="WEB">
<title>World English Bible</title>
<creator>WEB committee</creator>
<date event="eversion" type="Gregorian">2007-08-26</date>
<publisher>Rainbow Missions, Inc.</publisher>
<type type="OSIS">Bible</type>
<identifier type="OSIS">Bible.en.WEB.draft.2007-08-26</identifier>
<source>http://eBible.org/web/</source>
<language type="SIL">ENG</language>
<coverage>Wherever English is spoken in the world.</coverage>
<rights>The World English Bible is dedicated to the Public Domain by the translators and editors. It is not copyrighted. “World English Bible” and the World English Bible logo are a trademarks of Rainbow
Missions, Inc. They may only be used to identify this translation of the Holy Bible as published by Rainbow Missions, Inc., and faithful copies and quotations. “Faithful copies” include copies converted to other formats (i. e. HTML, PDF, etc.) or
typeset differently, without altering the text of the Scriptures, except that changing the spellings between preferred American and British usage is allowed. Use of the markings of direct quotes of Jesus Christ for different rendition (i. e. red text)
is optional. Comments and typo reports are welcome at http://eBible.org/cgi-bin/comment.cgi. Please see http://eBible.org/web/ for updates, revision status, free downloads, and printed edition purchase information.</rights>
<scope>Gen-Mal</scope>
<scope>Tob-AddEsth</scope>
<scope>Bar-EpJer</scope>
<scope>AddDan</scope>
<scope>Matt-Rev</scope>
<refSystem>Bible.WEB</refSystem>
</work>
</header>
<div type="bookGroup" canonical="true">
<div type="book" osisID="Gen" scope="Gen">
<title type="main" short="Genesis">Genesis </title>
<chapter sID="Gen.1" osisID="Gen.1" />
<p>
<verse sID="Gen.1.1" osisID="Gen.1.1" />In the beginning <milestone type="x-noteStartAnchor" />God<note type="translation">The Hebrew word rendered “God” is “Elohim.” After “God,” the Hebrew has the two letters “Aleph Tav” (the first and last letters of the Hebrew alphabet) as a grammatical marker.</note> created the heavens and the earth.
<verse eID="Gen.1.1" /><verse sID="Gen.1.2" osisID="Gen.1.2" />Now the earth was formless and empty. Darkness was on the surface of the deep. God’s Spirit was hovering over the surface of the waters.</p>
<p>
<verse eID="Gen.1.2" />
<verse sID="Gen.1.3" osisID="Gen.1.3" />God said, “Let there be light,” and there was light.
<verse eID="Gen.1.3" /><verse sID="Gen.1.4" osisID="Gen.1.4" />God saw the light, and saw that it was good. God divided the light from the darkness.
<verse eID="Gen.1.4" /><verse sID="Gen.1.5" osisID="Gen.1.5" />God called the light “day,” and the darkness he called “night.” There was evening and there was morning, one day.</p>
<p>
<verse eID="Gen.1.5" />
<verse sID="Gen.1.6" osisID="Gen.1.6" />God said, “Let there be an expanse in the middle of the waters, and let it divide the waters from the waters.”
<verse eID="Gen.1.6" /><verse sID="Gen.1.7" osisID="Gen.1.7" />God made the expanse, and divided the waters which were under the expanse from the waters which were above the expanse; and it was so.
<verse eID="Gen.1.7" /><verse sID="Gen.1.8" osisID="Gen.1.8" />God called the expanse “sky.” There was evening and there was morning, a second day.</p>
<p>
<verse eID="Gen.1.8" />
<verse sID="Gen.1.9" osisID="Gen.1.9" />God said, “Let the waters under the sky be gathered together to one place, and let the dry land appear;” and it was so.
<verse eID="Gen.1.9" /><verse sID="Gen.1.10" osisID="Gen.1.10" />God called the dry land “earth,” and the gathering together of the waters he called “seas.” God saw that it was good.
<verse eID="Gen.1.10" /></p>
</div>
</div>
</osisText>
</osis>

16
tests/resources/bibles/web.json Normal file
View File

@ -0,0 +1,16 @@
{
"book": "Genesis",
"chapter": "1",
"verses": [
[ "1", "In the beginning God created the heavens and the earth."],
[ "2", "Now the earth was formless and empty. Darkness was on the surface of the deep. God’s Spirit was hovering over the surface of the waters." ],
[ "3", "God said, “Let there be light,” and there was light." ],
[ "4", "God saw the light, and saw that it was good. God divided the light from the darkness." ],
[ "5", "God called the light “day,” and the darkness he called “night.” There was evening and there was morning, one day." ],
[ "6", "God said, “Let there be an expanse in the middle of the waters, and let it divide the waters from the waters.”" ],
[ "7", "God made the expanse, and divided the waters which were under the expanse from the waters which were above the expanse; and it was so." ],
[ "8", "God called the expanse “sky.” There was evening and there was morning, a second day." ],
[ "9", "God said, “Let the waters under the sky be gathered together to one place, and let the dry land appear;” and it was so." ],
[ "10", "God called the dry land “earth,” and the gathering together of the waters he called “seas.” God saw that it was good." ]
]
}