From eed14070577a2ed222abbdacd11df01eb5f71917 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Thu, 11 Aug 2016 00:22:07 +0300 Subject: [PATCH 01/65] - Fixed the bug where Bible "Advanced" clear button gives focus to "Quick" search field, thus making typing in it possible. --- openlp/plugins/bibles/lib/mediaitem.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openlp/plugins/bibles/lib/mediaitem.py b/openlp/plugins/bibles/lib/mediaitem.py index b8ff3a752..bdcda9942 100644 --- a/openlp/plugins/bibles/lib/mediaitem.py +++ b/openlp/plugins/bibles/lib/mediaitem.py @@ -254,7 +254,7 @@ class BibleMediaItem(MediaManagerItem): self.quickStyleComboBox.activated.connect(self.on_quick_style_combo_box_changed) self.advancedStyleComboBox.activated.connect(self.on_advanced_style_combo_box_changed) # Buttons - self.advancedClearButton.clicked.connect(self.on_clear_button) + self.advancedClearButton.clicked.connect(self.on_clear_button_advanced) self.quickClearButton.clicked.connect(self.on_clear_button) self.advancedSearchButton.clicked.connect(self.on_advanced_search_button) self.quickSearchButton.clicked.connect(self.on_quick_search_button) @@ -555,6 +555,11 @@ class BibleMediaItem(MediaManagerItem): self.quick_search_edit.clear() self.quick_search_edit.setFocus() + def on_clear_button_advanced(self): + # The same as the on_clear_button, but does not give focus to Quick search field. + self.list_view.clear() + self.check_search_result() + def on_lock_button_toggled(self, checked): self.quick_search_edit.setFocus() if checked: From 4cb3debb655c5ccf01b743d691d45fb86e1af936 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Thu, 11 Aug 2016 14:32:44 +0300 Subject: [PATCH 02/65] - Fixed bug: https://bugs.launchpad.net/openlp/+bug/1612187 - Changed the default hotkey for "Blank to Desktop" to "Esc", added "D" as an alternative shortcut. - Removed the default shortcut for "Escape item". --- openlp/core/common/settings.py | 5 +++-- openlp/core/ui/shortcutlistform.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/openlp/core/common/settings.py b/openlp/core/common/settings.py index 2caf04dab..a53c55244 100644 --- a/openlp/core/common/settings.py +++ b/openlp/core/common/settings.py @@ -260,11 +260,12 @@ class Settings(QtCore.QSettings): 'shortcuts/displayTagItem': [], 'shortcuts/blankScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Period)], 'shortcuts/collapse': [QtGui.QKeySequence(QtCore.Qt.Key_Minus)], - 'shortcuts/desktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_D)], + 'shortcuts/desktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Escape), + QtGui.QKeySequence(QtCore.Qt.Key_D)], 'shortcuts/delete': [QtGui.QKeySequence(QtGui.QKeySequence.Delete)], 'shortcuts/down': [QtGui.QKeySequence(QtCore.Qt.Key_Down)], 'shortcuts/editSong': [], - 'shortcuts/escapeItem': [QtGui.QKeySequence(QtCore.Qt.Key_Escape)], + 'shortcuts/escapeItem': [], 'shortcuts/expand': [QtGui.QKeySequence(QtCore.Qt.Key_Plus)], 'shortcuts/exportThemeItem': [], 'shortcuts/fileNewItem': [QtGui.QKeySequence(QtGui.QKeySequence.New)], diff --git a/openlp/core/ui/shortcutlistform.py b/openlp/core/ui/shortcutlistform.py index 1266d1cc4..e23fa0a27 100644 --- a/openlp/core/ui/shortcutlistform.py +++ b/openlp/core/ui/shortcutlistform.py @@ -430,7 +430,8 @@ class ShortcutListForm(QtWidgets.QDialog, Ui_ShortcutListDialog, RegistryPropert ' use a different shortcut.' ).format(key=self.get_shortcut_string(key_sequence)) self.main_window.warning_message(translate('OpenLP.ShortcutListDialog', 'Duplicate Shortcut'), - text, for_display=True) + text) + for_display = True self.dialog_was_shown = True return is_valid From 9794d26e9550031056433f0619a9a63509daf606 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Thu, 11 Aug 2016 16:28:30 +0300 Subject: [PATCH 03/65] - Changed the order of blank to modes in Shortcuts list from: Black, theme, desktop to desktop, theme, black. - Pep8 fix (Ident) --- openlp/core/common/settings.py | 2 +- openlp/core/ui/slidecontroller.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/openlp/core/common/settings.py b/openlp/core/common/settings.py index a53c55244..0be8d819a 100644 --- a/openlp/core/common/settings.py +++ b/openlp/core/common/settings.py @@ -261,7 +261,7 @@ class Settings(QtCore.QSettings): 'shortcuts/blankScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Period)], 'shortcuts/collapse': [QtGui.QKeySequence(QtCore.Qt.Key_Minus)], 'shortcuts/desktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Escape), - QtGui.QKeySequence(QtCore.Qt.Key_D)], + QtGui.QKeySequence(QtCore.Qt.Key_D)], 'shortcuts/delete': [QtGui.QKeySequence(QtGui.QKeySequence.Delete)], 'shortcuts/down': [QtGui.QKeySequence(QtCore.Qt.Key_Down)], 'shortcuts/editSong': [], diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index 7121e5227..43348a980 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -234,21 +234,21 @@ class SlideController(DisplayController, RegistryProperties): self.hide_menu.setPopupMode(QtWidgets.QToolButton.MenuButtonPopup) self.hide_menu.setMenu(QtWidgets.QMenu(translate('OpenLP.SlideController', 'Hide'), self.toolbar)) self.toolbar.add_toolbar_widget(self.hide_menu) - self.blank_screen = create_action(self, 'blankScreen', - text=translate('OpenLP.SlideController', 'Blank Screen'), - icon=':/slides/slide_blank.png', - checked=False, can_shortcuts=True, category=self.category, - triggers=self.on_blank_display) - self.theme_screen = create_action(self, 'themeScreen', - text=translate('OpenLP.SlideController', 'Blank to Theme'), - icon=':/slides/slide_theme.png', - checked=False, can_shortcuts=True, category=self.category, - triggers=self.on_theme_display) self.desktop_screen = create_action(self, 'desktopScreen', text=translate('OpenLP.SlideController', 'Show Desktop'), icon=':/slides/slide_desktop.png', checked=False, can_shortcuts=True, category=self.category, triggers=self.on_hide_display) + self.theme_screen = create_action(self, 'themeScreen', + text=translate('OpenLP.SlideController', 'Blank to Theme'), + icon=':/slides/slide_theme.png', + checked=False, can_shortcuts=True, category=self.category, + triggers=self.on_theme_display) + self.blank_screen = create_action(self, 'blankScreen', + text=translate('OpenLP.SlideController', 'Blank Screen'), + icon=':/slides/slide_blank.png', + checked=False, can_shortcuts=True, category=self.category, + triggers=self.on_blank_display) self.hide_menu.setDefaultAction(self.blank_screen) self.hide_menu.menu().addAction(self.blank_screen) self.hide_menu.menu().addAction(self.theme_screen) From 5ac5c6cd68f17cca78b3358c8f18b7a67b391659 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Thu, 11 Aug 2016 20:02:29 +0100 Subject: [PATCH 04/65] split the web bible importers out io their own files --- .../plugins/bibles/forms/bibleimportform.py | 4 +- openlp/plugins/bibles/lib/importers/http.py | 535 ------------------ openlp/plugins/bibles/lib/importers/osis.py | 2 +- .../openlp_plugins/bibles/test_lib_http.py | 4 +- 4 files changed, 7 insertions(+), 538 deletions(-) diff --git a/openlp/plugins/bibles/forms/bibleimportform.py b/openlp/plugins/bibles/forms/bibleimportform.py index 3d02228ca..e9eee88d5 100644 --- a/openlp/plugins/bibles/forms/bibleimportform.py +++ b/openlp/plugins/bibles/forms/bibleimportform.py @@ -40,7 +40,9 @@ from openlp.core.ui.lib.wizard import OpenLPWizard, WizardStrings from openlp.core.common.languagemanager import get_locale_key from openlp.plugins.bibles.lib.manager import BibleFormat from openlp.plugins.bibles.lib.db import clean_filename -from openlp.plugins.bibles.lib.importers.http import CWExtract, BGExtract, BSExtract +from openlp.plugins.bibles.lib.importers.biblegateway import BGExtract +from openlp.plugins.bibles.lib.importers.bibleserver import BSExtract +from openlp.plugins.bibles.lib.importers.crosswalk import CWExtract log = logging.getLogger(__name__) diff --git a/openlp/plugins/bibles/lib/importers/http.py b/openlp/plugins/bibles/lib/importers/http.py index 6921c9005..5afd107f6 100644 --- a/openlp/plugins/bibles/lib/importers/http.py +++ b/openlp/plugins/bibles/lib/importers/http.py @@ -38,545 +38,10 @@ from openlp.plugins.bibles.lib.bibleimport import BibleImport from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB, Book CLEANER_REGEX = re.compile(r' |
|\'\+\'') -FIX_PUNKCTUATION_REGEX = re.compile(r'[ ]+([.,;])') -REDUCE_SPACES_REGEX = re.compile(r'[ ]{2,}') -UGLY_CHARS = { - '\u2014': ' - ', - '\u2018': '\'', - '\u2019': '\'', - '\u201c': '"', - '\u201d': '"', - ' ': ' ' -} -VERSE_NUMBER_REGEX = re.compile(r'v(\d{1,2})(\d{3})(\d{3}) verse.*') - -BIBLESERVER_LANGUAGE_CODE = { - 'fl_1': 'de', - 'fl_2': 'en', - 'fl_3': 'fr', - 'fl_4': 'it', - 'fl_5': 'es', - 'fl_6': 'pt', - 'fl_7': 'ru', - 'fl_8': 'sv', - 'fl_9': 'no', - 'fl_10': 'nl', - 'fl_11': 'cs', - 'fl_12': 'sk', - 'fl_13': 'ro', - 'fl_14': 'hr', - 'fl_15': 'hu', - 'fl_16': 'bg', - 'fl_17': 'ar', - 'fl_18': 'tr', - 'fl_19': 'pl', - 'fl_20': 'da', - 'fl_21': 'zh' -} - -CROSSWALK_LANGUAGES = { - 'Portuguese': 'pt', - 'German': 'de', - 'Italian': 'it', - 'Español': 'es', - 'French': 'fr', - 'Dutch': 'nl' -} log = logging.getLogger(__name__) -class BGExtract(RegistryProperties): - """ - Extract verses from BibleGateway - """ - def __init__(self, proxy_url=None): - log.debug('BGExtract.init("{url}")'.format(url=proxy_url)) - self.proxy_url = proxy_url - socket.setdefaulttimeout(30) - - def _remove_elements(self, parent, tag, class_=None): - """ - Remove a particular element from the BeautifulSoup tree. - - :param parent: The element from which items need to be removed. - :param tag: A string of the tab type, e.g. "div" - :param class_: An HTML class attribute for further qualification. - """ - if class_: - all_tags = parent.find_all(tag, class_) - else: - all_tags = parent.find_all(tag) - for element in all_tags: - element.extract() - - def _extract_verse(self, tag): - """ - Extract a verse (or part of a verse) from a tag. - - :param tag: The BeautifulSoup Tag element with the stuff we want. - """ - if isinstance(tag, NavigableString): - return None, str(tag) - elif tag.get('class') and (tag.get('class')[0] == 'versenum' or tag.get('class')[0] == 'versenum mid-line'): - verse = str(tag.string).replace('[', '').replace(']', '').strip() - return verse, None - elif tag.get('class') and tag.get('class')[0] == 'chapternum': - verse = '1' - return verse, None - else: - verse = None - text = '' - for child in tag.contents: - c_verse, c_text = self._extract_verse(child) - if c_verse: - verse = c_verse - if text and c_text: - text += c_text - elif c_text is not None: - text = c_text - return verse, text - - def _clean_soup(self, tag): - """ - Remove all the rubbish from the HTML page. - - :param tag: The base tag within which we want to remove stuff. - """ - self._remove_elements(tag, 'sup', 'crossreference') - self._remove_elements(tag, 'sup', 'footnote') - self._remove_elements(tag, 'div', 'footnotes') - self._remove_elements(tag, 'div', 'crossrefs') - self._remove_elements(tag, 'h3') - self._remove_elements(tag, 'h4') - self._remove_elements(tag, 'h5') - - def _extract_verses(self, tags): - """ - Extract all the verses from a pre-prepared list of HTML tags. - - :param tags: A list of BeautifulSoup Tag elements. - """ - verses = [] - tags = tags[::-1] - current_text = '' - for tag in tags: - verse = None - text = '' - for child in tag.contents: - c_verse, c_text = self._extract_verse(child) - if c_verse: - verse = c_verse - if text and c_text: - text += c_text - elif c_text is not None: - text = c_text - if not verse: - current_text = text + ' ' + current_text - else: - text += ' ' + current_text - current_text = '' - if text: - for old, new in UGLY_CHARS.items(): - text = text.replace(old, new) - text = ' '.join(text.split()) - if verse and text: - verse = verse.strip() - try: - verse = int(verse) - except ValueError: - verse_parts = verse.split('-') - if len(verse_parts) > 1: - verse = int(verse_parts[0]) - except TypeError: - log.warning('Illegal verse number: {verse:d}'.format(verse=verse)) - verses.append((verse, text)) - verse_list = {} - for verse, text in verses[::-1]: - verse_list[verse] = text - return verse_list - - def _extract_verses_old(self, div): - """ - Use the old style of parsing for those Bibles on BG who mysteriously have not been migrated to the new (still - broken) HTML. - - :param div: The parent div. - """ - verse_list = {} - # Cater for inconsistent mark up in the first verse of a chapter. - first_verse = div.find('versenum') - if first_verse and first_verse.contents: - verse_list[1] = str(first_verse.contents[0]) - for verse in div('sup', 'versenum'): - raw_verse_num = verse.next_element - clean_verse_num = 0 - # Not all verses exist in all translations and may or may not be represented by a verse number. If they are - # not fine, if they are it will probably be in a format that breaks int(). We will then have no idea what - # garbage may be sucked in to the verse text so if we do not get a clean int() then ignore the verse - # completely. - try: - clean_verse_num = int(str(raw_verse_num)) - except ValueError: - verse_parts = str(raw_verse_num).split('-') - if len(verse_parts) > 1: - clean_verse_num = int(verse_parts[0]) - except TypeError: - log.warning('Illegal verse number: {verse:d}'.format(verse=raw_verse_num)) - if clean_verse_num: - verse_text = raw_verse_num.next_element - part = raw_verse_num.next_element.next_element - while not (isinstance(part, Tag) and part.get('class')[0] == 'versenum'): - # While we are still in the same verse grab all the text. - if isinstance(part, NavigableString): - verse_text += part - if isinstance(part.next_element, Tag) and part.next_element.name == 'div': - # Run out of verses so stop. - break - part = part.next_element - verse_list[clean_verse_num] = str(verse_text) - return verse_list - - def get_bible_chapter(self, version, book_name, chapter): - """ - Access and decode Bibles via the BibleGateway website. - - :param version: The version of the Bible like 31 for New International version. - :param book_name: Name of the Book. - :param chapter: Chapter number. - """ - log.debug('BGExtract.get_bible_chapter("{version}", "{name}", "{chapter}")'.format(version=version, - name=book_name, - chapter=chapter)) - url_book_name = urllib.parse.quote(book_name.encode("utf-8")) - url_params = 'search={name}+{chapter}&version={version}'.format(name=url_book_name, - chapter=chapter, - version=version) - soup = get_soup_for_bible_ref( - 'http://biblegateway.com/passage/?{url}'.format(url=url_params), - pre_parse_regex=r'', pre_parse_substitute='') - if not soup: - return None - div = soup.find('div', 'result-text-style-normal') - if not div: - return None - self._clean_soup(div) - span_list = div.find_all('span', 'text') - log.debug('Span list: {span}'.format(span=span_list)) - if not span_list: - # If we don't get any spans then we must have the old HTML format - verse_list = self._extract_verses_old(div) - else: - verse_list = self._extract_verses(span_list) - if not verse_list: - log.debug('No content found in the BibleGateway response.') - send_error_message('parse') - return None - return SearchResults(book_name, chapter, verse_list) - - def get_books_from_http(self, version): - """ - Load a list of all books a Bible contains from BibleGateway website. - - :param version: The version of the Bible like NIV for New International Version - """ - log.debug('BGExtract.get_books_from_http("{version}")'.format(version=version)) - url_params = urllib.parse.urlencode({'action': 'getVersionInfo', 'vid': '{version}'.format(version=version)}) - reference_url = 'http://biblegateway.com/versions/?{url}#books'.format(url=url_params) - page = get_web_page(reference_url) - if not page: - send_error_message('download') - return None - page_source = page.read() - try: - page_source = str(page_source, 'utf8') - except UnicodeDecodeError: - page_source = str(page_source, 'cp1251') - try: - soup = BeautifulSoup(page_source, 'lxml') - except Exception: - log.error('BeautifulSoup could not parse the Bible page.') - send_error_message('parse') - return None - if not soup: - send_error_message('parse') - return None - self.application.process_events() - content = soup.find('table', 'infotable') - if content: - content = content.find_all('tr') - if not content: - log.error('No books found in the Biblegateway response.') - send_error_message('parse') - return None - books = [] - for book in content: - book = book.find('td') - if book: - books.append(book.contents[1]) - return books - - def get_bibles_from_http(self): - """ - Load a list of bibles from BibleGateway website. - - returns a list in the form [(biblename, biblekey, language_code)] - """ - log.debug('BGExtract.get_bibles_from_http') - bible_url = 'https://biblegateway.com/versions/' - soup = get_soup_for_bible_ref(bible_url) - if not soup: - return None - bible_select = soup.find('select', {'class': 'search-translation-select'}) - if not bible_select: - log.debug('No select tags found - did site change?') - return None - option_tags = bible_select.find_all('option') - if not option_tags: - log.debug('No option tags found - did site change?') - return None - current_lang = '' - bibles = [] - for ot in option_tags: - tag_class = '' - try: - tag_class = ot['class'][0] - except KeyError: - tag_class = '' - tag_text = ot.get_text() - if tag_class == 'lang': - current_lang = tag_text[tag_text.find('(') + 1:tag_text.find(')')].lower() - elif tag_class == 'spacer': - continue - else: - bibles.append((tag_text, ot['value'], current_lang)) - return bibles - - -class BSExtract(RegistryProperties): - """ - Extract verses from Bibleserver.com - """ - def __init__(self, proxy_url=None): - log.debug('BSExtract.init("{url}")'.format(url=proxy_url)) - self.proxy_url = proxy_url - socket.setdefaulttimeout(30) - - def get_bible_chapter(self, version, book_name, chapter): - """ - Access and decode bibles via Bibleserver mobile website - - :param version: The version of the bible like NIV for New International Version - :param book_name: Text name of bible book e.g. Genesis, 1. John, 1John or Offenbarung - :param chapter: Chapter number - """ - log.debug('BSExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, - book=book_name, - chapter=chapter)) - url_version = urllib.parse.quote(version.encode("utf-8")) - url_book_name = urllib.parse.quote(book_name.encode("utf-8")) - chapter_url = 'http://m.bibleserver.com/text/{version}/{name}{chapter:d}'.format(version=url_version, - name=url_book_name, - chapter=chapter) - header = ('Accept-Language', 'en') - soup = get_soup_for_bible_ref(chapter_url, header) - if not soup: - return None - self.application.process_events() - content = soup.find('div', 'content') - if not content: - log.error('No verses found in the Bibleserver response.') - send_error_message('parse') - return None - content = content.find('div').find_all('div') - verses = {} - for verse in content: - self.application.process_events() - versenumber = int(VERSE_NUMBER_REGEX.sub(r'\3', ' '.join(verse['class']))) - verses[versenumber] = verse.contents[1].rstrip('\n') - return SearchResults(book_name, chapter, verses) - - def get_books_from_http(self, version): - """ - Load a list of all books a Bible contains from Bibleserver mobile website. - - :param version: The version of the Bible like NIV for New International Version - """ - log.debug('BSExtract.get_books_from_http("{version}")'.format(version=version)) - url_version = urllib.parse.quote(version.encode("utf-8")) - chapter_url = 'http://m.bibleserver.com/overlay/selectBook?translation={version}'.format(version=url_version) - soup = get_soup_for_bible_ref(chapter_url) - if not soup: - return None - content = soup.find('ul') - if not content: - log.error('No books found in the Bibleserver response.') - send_error_message('parse') - return None - content = content.find_all('li') - return [book.contents[0].contents[0] for book in content if len(book.contents[0].contents)] - - def get_bibles_from_http(self): - """ - Load a list of bibles from Bibleserver website. - - returns a list in the form [(biblename, biblekey, language_code)] - """ - log.debug('BSExtract.get_bibles_from_http') - bible_url = 'http://www.bibleserver.com/index.php?language=2' - soup = get_soup_for_bible_ref(bible_url) - if not soup: - return None - bible_links = soup.find_all('a', {'class': 'trlCell'}) - if not bible_links: - log.debug('No a tags found - did site change?') - return None - bibles = [] - for link in bible_links: - bible_name = link.get_text() - # Skip any audio - if 'audio' in bible_name.lower(): - continue - try: - bible_link = link['href'] - bible_key = bible_link[bible_link.rfind('/') + 1:] - css_classes = link['class'] - except KeyError: - log.debug('No href/class attribute found - did site change?') - language_code = '' - for css_class in css_classes: - if css_class.startswith('fl_'): - try: - language_code = BIBLESERVER_LANGUAGE_CODE[css_class] - except KeyError: - language_code = '' - bibles.append((bible_name, bible_key, language_code)) - return bibles - - -class CWExtract(RegistryProperties): - """ - Extract verses from CrossWalk/BibleStudyTools - """ - def __init__(self, proxy_url=None): - log.debug('CWExtract.init("{url}")'.format(url=proxy_url)) - self.proxy_url = proxy_url - socket.setdefaulttimeout(30) - - def get_bible_chapter(self, version, book_name, chapter): - """ - Access and decode bibles via the Crosswalk website - - :param version: The version of the Bible like niv for New International Version - :param book_name: Text name of in english e.g. 'gen' for Genesis - :param chapter: Chapter number - """ - log.debug('CWExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, - book=book_name, - chapter=chapter)) - url_book_name = book_name.replace(' ', '-') - url_book_name = url_book_name.lower() - url_book_name = urllib.parse.quote(url_book_name.encode("utf-8")) - chapter_url = 'http://www.biblestudytools.com/{version}/{book}/{chapter}.html'.format(version=version, - book=url_book_name, - chapter=chapter) - soup = get_soup_for_bible_ref(chapter_url) - if not soup: - return None - self.application.process_events() - verses_div = soup.find_all('div', 'verse') - if not verses_div: - log.error('No verses found in the CrossWalk response.') - send_error_message('parse') - return None - verses = {} - for verse in verses_div: - self.application.process_events() - verse_number = int(verse.find('strong').contents[0]) - verse_span = verse.find('span') - tags_to_remove = verse_span.find_all(['a', 'sup']) - for tag in tags_to_remove: - tag.decompose() - verse_text = verse_span.get_text() - self.application.process_events() - # Fix up leading and trailing spaces, multiple spaces, and spaces between text and , and . - verse_text = verse_text.strip('\n\r\t ') - verse_text = REDUCE_SPACES_REGEX.sub(' ', verse_text) - verse_text = FIX_PUNKCTUATION_REGEX.sub(r'\1', verse_text) - verses[verse_number] = verse_text - return SearchResults(book_name, chapter, verses) - - def get_books_from_http(self, version): - """ - Load a list of all books a Bible contain from the Crosswalk website. - - :param version: The version of the bible like NIV for New International Version - """ - log.debug('CWExtract.get_books_from_http("{version}")'.format(version=version)) - chapter_url = 'http://www.biblestudytools.com/{version}/'.format(version=version) - soup = get_soup_for_bible_ref(chapter_url) - if not soup: - return None - content = soup.find_all('h4', {'class': 'small-header'}) - if not content: - log.error('No books found in the Crosswalk response.') - send_error_message('parse') - return None - books = [] - for book in content: - books.append(book.contents[0]) - return books - - def get_bibles_from_http(self): - """ - Load a list of bibles from Crosswalk website. - returns a list in the form [(biblename, biblekey, language_code)] - """ - log.debug('CWExtract.get_bibles_from_http') - bible_url = 'http://www.biblestudytools.com/bible-versions/' - soup = get_soup_for_bible_ref(bible_url) - if not soup: - return None - h4_tags = soup.find_all('h4', {'class': 'small-header'}) - if not h4_tags: - log.debug('No h4 tags found - did site change?') - return None - bibles = [] - for h4t in h4_tags: - short_name = None - if h4t.span: - short_name = h4t.span.get_text().strip().lower() - else: - log.error('No span tag found - did site change?') - return None - if not short_name: - continue - h4t.span.extract() - tag_text = h4t.get_text().strip() - # The names of non-english bibles has their language in parentheses at the end - if tag_text.endswith(')'): - language = tag_text[tag_text.rfind('(') + 1:-1] - if language in CROSSWALK_LANGUAGES: - language_code = CROSSWALK_LANGUAGES[language] - else: - language_code = '' - # ... except for those that don't... - elif 'latin' in tag_text.lower(): - language_code = 'la' - elif 'la biblia' in tag_text.lower() or 'nueva' in tag_text.lower(): - language_code = 'es' - elif 'chinese' in tag_text.lower(): - language_code = 'zh' - elif 'greek' in tag_text.lower(): - language_code = 'el' - elif 'nova' in tag_text.lower(): - language_code = 'pt' - else: - language_code = 'en' - bibles.append((tag_text, short_name, language_code)) - return bibles - - class HTTPBible(BibleImport, RegistryProperties): log.info('{name} HTTPBible loaded'.format(name=__name__)) diff --git a/openlp/plugins/bibles/lib/importers/osis.py b/openlp/plugins/bibles/lib/importers/osis.py index 99a138acd..c833277fe 100644 --- a/openlp/plugins/bibles/lib/importers/osis.py +++ b/openlp/plugins/bibles/lib/importers/osis.py @@ -108,7 +108,7 @@ class OSISBible(BibleImport): if self.stop_import_flag: break # Remove div-tags in the book - etree.strip_tags(book, ('{http://www.bibletechnologies.net/2003/OSIS/namespace}div')) + etree.strip_tags(book, '{http://www.bibletechnologies.net/2003/OSIS/namespace}div') book_ref_id = self.get_book_ref_id_by_name(book.get('osisID'), num_books, language_id) if not book_ref_id: log.error('Importing books from "{name}" failed'.format(name=self.filename)) diff --git a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py index 084bfa476..fd557eece 100644 --- a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py +++ b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py @@ -25,7 +25,9 @@ from unittest import TestCase, skip from openlp.core.common import Registry -from openlp.plugins.bibles.lib.importers.http import BGExtract, CWExtract, BSExtract +from openlp.plugins.bibles.lib.importers.biblegateway import BGExtract +from openlp.plugins.bibles.lib.importers.bibleserver import BSExtract +from openlp.plugins.bibles.lib.importers.crosswalk import CWExtract from tests.interfaces import MagicMock From f5480640f687d8ed44700243eab02135b42df4fe Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Thu, 11 Aug 2016 20:07:21 +0100 Subject: [PATCH 05/65] more files --- .../bibles/lib/importers/biblegateway.py | 313 ++++++++++++++++++ .../bibles/lib/importers/bibleserver.py | 162 +++++++++ .../plugins/bibles/lib/importers/crosswalk.py | 171 ++++++++++ 3 files changed, 646 insertions(+) create mode 100644 openlp/plugins/bibles/lib/importers/biblegateway.py create mode 100644 openlp/plugins/bibles/lib/importers/bibleserver.py create mode 100644 openlp/plugins/bibles/lib/importers/crosswalk.py diff --git a/openlp/plugins/bibles/lib/importers/biblegateway.py b/openlp/plugins/bibles/lib/importers/biblegateway.py new file mode 100644 index 000000000..c6a8074bf --- /dev/null +++ b/openlp/plugins/bibles/lib/importers/biblegateway.py @@ -0,0 +1,313 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`http` module enables OpenLP to retrieve scripture from bible websites. +""" +import logging +import socket +import urllib.parse +import urllib.error + +from bs4 import BeautifulSoup, NavigableString, Tag + +from openlp.core.common import RegistryProperties +from openlp.core.lib.webpagereader import get_web_page +from openlp.plugins.bibles.lib import SearchResults +from openlp.plugins.bibles.lib.importers.http import get_soup_for_bible_ref, send_error_message + +UGLY_CHARS = { + '\u2014': ' - ', + '\u2018': '\'', + '\u2019': '\'', + '\u201c': '"', + '\u201d': '"', + ' ': ' ' +} + +log = logging.getLogger(__name__) + + +class BGExtract(RegistryProperties): + """ + Extract verses from BibleGateway + """ + def __init__(self, proxy_url=None): + log.debug('BGExtract.init("{url}")'.format(url=proxy_url)) + self.proxy_url = proxy_url + socket.setdefaulttimeout(30) + + def _remove_elements(self, parent, tag, class_=None): + """ + Remove a particular element from the BeautifulSoup tree. + + :param parent: The element from which items need to be removed. + :param tag: A string of the tab type, e.g. "div" + :param class_: An HTML class attribute for further qualification. + """ + if class_: + all_tags = parent.find_all(tag, class_) + else: + all_tags = parent.find_all(tag) + for element in all_tags: + element.extract() + + def _extract_verse(self, tag): + """ + Extract a verse (or part of a verse) from a tag. + + :param tag: The BeautifulSoup Tag element with the stuff we want. + """ + if isinstance(tag, NavigableString): + return None, str(tag) + elif tag.get('class') and (tag.get('class')[0] == 'versenum' or tag.get('class')[0] == 'versenum mid-line'): + verse = str(tag.string).replace('[', '').replace(']', '').strip() + return verse, None + elif tag.get('class') and tag.get('class')[0] == 'chapternum': + verse = '1' + return verse, None + else: + verse = None + text = '' + for child in tag.contents: + c_verse, c_text = self._extract_verse(child) + if c_verse: + verse = c_verse + if text and c_text: + text += c_text + elif c_text is not None: + text = c_text + return verse, text + + def _clean_soup(self, tag): + """ + Remove all the rubbish from the HTML page. + + :param tag: The base tag within which we want to remove stuff. + """ + self._remove_elements(tag, 'sup', 'crossreference') + self._remove_elements(tag, 'sup', 'footnote') + self._remove_elements(tag, 'div', 'footnotes') + self._remove_elements(tag, 'div', 'crossrefs') + self._remove_elements(tag, 'h3') + self._remove_elements(tag, 'h4') + self._remove_elements(tag, 'h5') + + def _extract_verses(self, tags): + """ + Extract all the verses from a pre-prepared list of HTML tags. + + :param tags: A list of BeautifulSoup Tag elements. + """ + verses = [] + tags = tags[::-1] + current_text = '' + for tag in tags: + verse = None + text = '' + for child in tag.contents: + c_verse, c_text = self._extract_verse(child) + if c_verse: + verse = c_verse + if text and c_text: + text += c_text + elif c_text is not None: + text = c_text + if not verse: + current_text = text + ' ' + current_text + else: + text += ' ' + current_text + current_text = '' + if text: + for old, new in UGLY_CHARS.items(): + text = text.replace(old, new) + text = ' '.join(text.split()) + if verse and text: + verse = verse.strip() + try: + verse = int(verse) + except ValueError: + verse_parts = verse.split('-') + if len(verse_parts) > 1: + verse = int(verse_parts[0]) + except TypeError: + log.warning('Illegal verse number: {verse:d}'.format(verse=verse)) + verses.append((verse, text)) + verse_list = {} + for verse, text in verses[::-1]: + verse_list[verse] = text + return verse_list + + def _extract_verses_old(self, div): + """ + Use the old style of parsing for those Bibles on BG who mysteriously have not been migrated to the new (still + broken) HTML. + + :param div: The parent div. + """ + verse_list = {} + # Cater for inconsistent mark up in the first verse of a chapter. + first_verse = div.find('versenum') + if first_verse and first_verse.contents: + verse_list[1] = str(first_verse.contents[0]) + for verse in div('sup', 'versenum'): + raw_verse_num = verse.next_element + clean_verse_num = 0 + # Not all verses exist in all translations and may or may not be represented by a verse number. If they are + # not fine, if they are it will probably be in a format that breaks int(). We will then have no idea what + # garbage may be sucked in to the verse text so if we do not get a clean int() then ignore the verse + # completely. + try: + clean_verse_num = int(str(raw_verse_num)) + except ValueError: + verse_parts = str(raw_verse_num).split('-') + if len(verse_parts) > 1: + clean_verse_num = int(verse_parts[0]) + except TypeError: + log.warning('Illegal verse number: {verse:d}'.format(verse=raw_verse_num)) + if clean_verse_num: + verse_text = raw_verse_num.next_element + part = raw_verse_num.next_element.next_element + while not (isinstance(part, Tag) and part.get('class')[0] == 'versenum'): + # While we are still in the same verse grab all the text. + if isinstance(part, NavigableString): + verse_text += part + if isinstance(part.next_element, Tag) and part.next_element.name == 'div': + # Run out of verses so stop. + break + part = part.next_element + verse_list[clean_verse_num] = str(verse_text) + return verse_list + + def get_bible_chapter(self, version, book_name, chapter): + """ + Access and decode Bibles via the BibleGateway website. + + :param version: The version of the Bible like 31 for New International version. + :param book_name: Name of the Book. + :param chapter: Chapter number. + """ + log.debug('BGExtract.get_bible_chapter("{version}", "{name}", "{chapter}")'.format(version=version, + name=book_name, + chapter=chapter)) + url_book_name = urllib.parse.quote(book_name.encode("utf-8")) + url_params = 'search={name}+{chapter}&version={version}'.format(name=url_book_name, + chapter=chapter, + version=version) + soup = get_soup_for_bible_ref( + 'http://biblegateway.com/passage/?{url}'.format(url=url_params), + pre_parse_regex=r'', pre_parse_substitute='') + if not soup: + return None + div = soup.find('div', 'result-text-style-normal') + if not div: + return None + self._clean_soup(div) + span_list = div.find_all('span', 'text') + log.debug('Span list: {span}'.format(span=span_list)) + if not span_list: + # If we don't get any spans then we must have the old HTML format + verse_list = self._extract_verses_old(div) + else: + verse_list = self._extract_verses(span_list) + if not verse_list: + log.debug('No content found in the BibleGateway response.') + send_error_message('parse') + return None + return SearchResults(book_name, chapter, verse_list) + + def get_books_from_http(self, version): + """ + Load a list of all books a Bible contains from BibleGateway website. + + :param version: The version of the Bible like NIV for New International Version + """ + log.debug('BGExtract.get_books_from_http("{version}")'.format(version=version)) + url_params = urllib.parse.urlencode({'action': 'getVersionInfo', 'vid': '{version}'.format(version=version)}) + reference_url = 'http://biblegateway.com/versions/?{url}#books'.format(url=url_params) + page = get_web_page(reference_url) + if not page: + send_error_message('download') + return None + page_source = page.read() + try: + page_source = str(page_source, 'utf8') + except UnicodeDecodeError: + page_source = str(page_source, 'cp1251') + try: + soup = BeautifulSoup(page_source, 'lxml') + except Exception: + log.error('BeautifulSoup could not parse the Bible page.') + send_error_message('parse') + return None + if not soup: + send_error_message('parse') + return None + self.application.process_events() + content = soup.find('table', 'infotable') + if content: + content = content.find_all('tr') + if not content: + log.error('No books found in the Biblegateway response.') + send_error_message('parse') + return None + books = [] + for book in content: + book = book.find('td') + if book: + books.append(book.contents[1]) + return books + + def get_bibles_from_http(self): + """ + Load a list of bibles from BibleGateway website. + + returns a list in the form [(biblename, biblekey, language_code)] + """ + log.debug('BGExtract.get_bibles_from_http') + bible_url = 'https://biblegateway.com/versions/' + soup = get_soup_for_bible_ref(bible_url) + if not soup: + return None + bible_select = soup.find('select', {'class': 'search-translation-select'}) + if not bible_select: + log.debug('No select tags found - did site change?') + return None + option_tags = bible_select.find_all('option') + if not option_tags: + log.debug('No option tags found - did site change?') + return None + current_lang = '' + bibles = [] + for ot in option_tags: + tag_class = '' + try: + tag_class = ot['class'][0] + except KeyError: + tag_class = '' + tag_text = ot.get_text() + if tag_class == 'lang': + current_lang = tag_text[tag_text.find('(') + 1:tag_text.find(')')].lower() + elif tag_class == 'spacer': + continue + else: + bibles.append((tag_text, ot['value'], current_lang)) + return bibles diff --git a/openlp/plugins/bibles/lib/importers/bibleserver.py b/openlp/plugins/bibles/lib/importers/bibleserver.py new file mode 100644 index 000000000..e651b84ab --- /dev/null +++ b/openlp/plugins/bibles/lib/importers/bibleserver.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`http` module enables OpenLP to retrieve scripture from bible websites. +""" +import logging +import re +import socket +import urllib.parse +import urllib.error + +from openlp.core.common import RegistryProperties +from openlp.plugins.bibles.lib import SearchResults +from openlp.plugins.bibles.lib.http import get_soup_for_bible_ref, send_error_message + +VERSE_NUMBER_REGEX = re.compile(r'v(\d{1,2})(\d{3})(\d{3}) verse.*') + +BIBLESERVER_LANGUAGE_CODE = { + 'fl_1': 'de', + 'fl_2': 'en', + 'fl_3': 'fr', + 'fl_4': 'it', + 'fl_5': 'es', + 'fl_6': 'pt', + 'fl_7': 'ru', + 'fl_8': 'sv', + 'fl_9': 'no', + 'fl_10': 'nl', + 'fl_11': 'cs', + 'fl_12': 'sk', + 'fl_13': 'ro', + 'fl_14': 'hr', + 'fl_15': 'hu', + 'fl_16': 'bg', + 'fl_17': 'ar', + 'fl_18': 'tr', + 'fl_19': 'pl', + 'fl_20': 'da', + 'fl_21': 'zh' +} + +log = logging.getLogger(__name__) + + +class BSExtract(RegistryProperties): + """ + Extract verses from Bibleserver.com + """ + def __init__(self, proxy_url=None): + log.debug('BSExtract.init("{url}")'.format(url=proxy_url)) + self.proxy_url = proxy_url + socket.setdefaulttimeout(30) + + def get_bible_chapter(self, version, book_name, chapter): + """ + Access and decode bibles via Bibleserver mobile website + + :param version: The version of the bible like NIV for New International Version + :param book_name: Text name of bible book e.g. Genesis, 1. John, 1John or Offenbarung + :param chapter: Chapter number + """ + log.debug('BSExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, + book=book_name, + chapter=chapter)) + url_version = urllib.parse.quote(version.encode("utf-8")) + url_book_name = urllib.parse.quote(book_name.encode("utf-8")) + chapter_url = 'http://m.bibleserver.com/text/{version}/{name}{chapter:d}'.format(version=url_version, + name=url_book_name, + chapter=chapter) + header = ('Accept-Language', 'en') + soup = get_soup_for_bible_ref(chapter_url, header) + if not soup: + return None + self.application.process_events() + content = soup.find('div', 'content') + if not content: + log.error('No verses found in the Bibleserver response.') + send_error_message('parse') + return None + content = content.find('div').find_all('div') + verses = {} + for verse in content: + self.application.process_events() + versenumber = int(VERSE_NUMBER_REGEX.sub(r'\3', ' '.join(verse['class']))) + verses[versenumber] = verse.contents[1].rstrip('\n') + return SearchResults(book_name, chapter, verses) + + def get_books_from_http(self, version): + """ + Load a list of all books a Bible contains from Bibleserver mobile website. + + :param version: The version of the Bible like NIV for New International Version + """ + log.debug('BSExtract.get_books_from_http("{version}")'.format(version=version)) + url_version = urllib.parse.quote(version.encode("utf-8")) + chapter_url = 'http://m.bibleserver.com/overlay/selectBook?translation={version}'.format(version=url_version) + soup = get_soup_for_bible_ref(chapter_url) + if not soup: + return None + content = soup.find('ul') + if not content: + log.error('No books found in the Bibleserver response.') + send_error_message('parse') + return None + content = content.find_all('li') + return [book.contents[0].contents[0] for book in content if len(book.contents[0].contents)] + + def get_bibles_from_http(self): + """ + Load a list of bibles from Bibleserver website. + + returns a list in the form [(biblename, biblekey, language_code)] + """ + log.debug('BSExtract.get_bibles_from_http') + bible_url = 'http://www.bibleserver.com/index.php?language=2' + soup = get_soup_for_bible_ref(bible_url) + if not soup: + return None + bible_links = soup.find_all('a', {'class': 'trlCell'}) + if not bible_links: + log.debug('No a tags found - did site change?') + return None + bibles = [] + for link in bible_links: + bible_name = link.get_text() + # Skip any audio + if 'audio' in bible_name.lower(): + continue + try: + bible_link = link['href'] + bible_key = bible_link[bible_link.rfind('/') + 1:] + css_classes = link['class'] + except KeyError: + log.debug('No href/class attribute found - did site change?') + language_code = '' + for css_class in css_classes: + if css_class.startswith('fl_'): + try: + language_code = BIBLESERVER_LANGUAGE_CODE[css_class] + except KeyError: + language_code = '' + bibles.append((bible_name, bible_key, language_code)) + return bibles diff --git a/openlp/plugins/bibles/lib/importers/crosswalk.py b/openlp/plugins/bibles/lib/importers/crosswalk.py new file mode 100644 index 000000000..6c75209d1 --- /dev/null +++ b/openlp/plugins/bibles/lib/importers/crosswalk.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`http` module enables OpenLP to retrieve scripture from bible websites. +""" +import logging +import re +import socket +import urllib.parse +import urllib.error + +from openlp.core.common import RegistryProperties +from openlp.plugins.bibles.lib import SearchResults +from openlp.plugins.bibles.lib.importers.http import get_soup_for_bible_ref, send_error_message + +FIX_PUNKCTUATION_REGEX = re.compile(r'[ ]+([.,;])') +REDUCE_SPACES_REGEX = re.compile(r'[ ]{2,}') + + +CROSSWALK_LANGUAGES = { + 'Portuguese': 'pt', + 'German': 'de', + 'Italian': 'it', + 'Español': 'es', + 'French': 'fr', + 'Dutch': 'nl' +} + +log = logging.getLogger(__name__) + + +class CWExtract(RegistryProperties): + """ + Extract verses from CrossWalk/BibleStudyTools + """ + def __init__(self, proxy_url=None): + log.debug('CWExtract.init("{url}")'.format(url=proxy_url)) + self.proxy_url = proxy_url + socket.setdefaulttimeout(30) + + def get_bible_chapter(self, version, book_name, chapter): + """ + Access and decode bibles via the Crosswalk website + + :param version: The version of the Bible like niv for New International Version + :param book_name: Text name of in english e.g. 'gen' for Genesis + :param chapter: Chapter number + """ + log.debug('CWExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, + book=book_name, + chapter=chapter)) + url_book_name = book_name.replace(' ', '-') + url_book_name = url_book_name.lower() + url_book_name = urllib.parse.quote(url_book_name.encode("utf-8")) + chapter_url = 'http://www.biblestudytools.com/{version}/{book}/{chapter}.html'.format(version=version, + book=url_book_name, + chapter=chapter) + soup = get_soup_for_bible_ref(chapter_url) + if not soup: + return None + self.application.process_events() + verses_div = soup.find_all('div', 'verse') + if not verses_div: + log.error('No verses found in the CrossWalk response.') + send_error_message('parse') + return None + verses = {} + for verse in verses_div: + self.application.process_events() + verse_number = int(verse.find('strong').contents[0]) + verse_span = verse.find('span') + tags_to_remove = verse_span.find_all(['a', 'sup']) + for tag in tags_to_remove: + tag.decompose() + verse_text = verse_span.get_text() + self.application.process_events() + # Fix up leading and trailing spaces, multiple spaces, and spaces between text and , and . + verse_text = verse_text.strip('\n\r\t ') + verse_text = REDUCE_SPACES_REGEX.sub(' ', verse_text) + verse_text = FIX_PUNKCTUATION_REGEX.sub(r'\1', verse_text) + verses[verse_number] = verse_text + return SearchResults(book_name, chapter, verses) + + def get_books_from_http(self, version): + """ + Load a list of all books a Bible contain from the Crosswalk website. + + :param version: The version of the bible like NIV for New International Version + """ + log.debug('CWExtract.get_books_from_http("{version}")'.format(version=version)) + chapter_url = 'http://www.biblestudytools.com/{version}/'.format(version=version) + soup = get_soup_for_bible_ref(chapter_url) + if not soup: + return None + content = soup.find_all('h4', {'class': 'small-header'}) + if not content: + log.error('No books found in the Crosswalk response.') + send_error_message('parse') + return None + books = [] + for book in content: + books.append(book.contents[0]) + return books + + def get_bibles_from_http(self): + """ + Load a list of bibles from Crosswalk website. + returns a list in the form [(biblename, biblekey, language_code)] + """ + log.debug('CWExtract.get_bibles_from_http') + bible_url = 'http://www.biblestudytools.com/bible-versions/' + soup = get_soup_for_bible_ref(bible_url) + if not soup: + return None + h4_tags = soup.find_all('h4', {'class': 'small-header'}) + if not h4_tags: + log.debug('No h4 tags found - did site change?') + return None + bibles = [] + for h4t in h4_tags: + short_name = None + if h4t.span: + short_name = h4t.span.get_text().strip().lower() + else: + log.error('No span tag found - did site change?') + return None + if not short_name: + continue + h4t.span.extract() + tag_text = h4t.get_text().strip() + # The names of non-english bibles has their language in parentheses at the end + if tag_text.endswith(')'): + language = tag_text[tag_text.rfind('(') + 1:-1] + if language in CROSSWALK_LANGUAGES: + language_code = CROSSWALK_LANGUAGES[language] + else: + language_code = '' + # ... except for those that don't... + elif 'latin' in tag_text.lower(): + language_code = 'la' + elif 'la biblia' in tag_text.lower() or 'nueva' in tag_text.lower(): + language_code = 'es' + elif 'chinese' in tag_text.lower(): + language_code = 'zh' + elif 'greek' in tag_text.lower(): + language_code = 'el' + elif 'nova' in tag_text.lower(): + language_code = 'pt' + else: + language_code = 'en' + bibles.append((tag_text, short_name, language_code)) + return bibles From 83b30799ca41ca3e444f4896ea96b67102f1bc4f Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Thu, 11 Aug 2016 20:34:55 +0100 Subject: [PATCH 06/65] finished up with tests --- .../bibles/lib/{importers => }/http.py | 0 .../bibles/lib/importers/biblegateway.py | 4 +- .../bibles/lib/importers/bibleserver.py | 2 +- .../plugins/bibles/lib/importers/crosswalk.py | 4 +- openlp/plugins/bibles/lib/manager.py | 2 +- .../{test_http.py => test_bibleserver.py} | 43 ++++--------------- 6 files changed, 14 insertions(+), 41 deletions(-) rename openlp/plugins/bibles/lib/{importers => }/http.py (100%) rename tests/functional/openlp_plugins/bibles/{test_http.py => test_bibleserver.py} (88%) diff --git a/openlp/plugins/bibles/lib/importers/http.py b/openlp/plugins/bibles/lib/http.py similarity index 100% rename from openlp/plugins/bibles/lib/importers/http.py rename to openlp/plugins/bibles/lib/http.py diff --git a/openlp/plugins/bibles/lib/importers/biblegateway.py b/openlp/plugins/bibles/lib/importers/biblegateway.py index c6a8074bf..f3caa2204 100644 --- a/openlp/plugins/bibles/lib/importers/biblegateway.py +++ b/openlp/plugins/bibles/lib/importers/biblegateway.py @@ -20,7 +20,7 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ -The :mod:`http` module enables OpenLP to retrieve scripture from bible websites. +The :mod:`biblegateway` module enables OpenLP to retrieve scripture from http://biblegateway.com. """ import logging import socket @@ -32,7 +32,7 @@ from bs4 import BeautifulSoup, NavigableString, Tag from openlp.core.common import RegistryProperties from openlp.core.lib.webpagereader import get_web_page from openlp.plugins.bibles.lib import SearchResults -from openlp.plugins.bibles.lib.importers.http import get_soup_for_bible_ref, send_error_message +from openlp.plugins.bibles.lib.http import get_soup_for_bible_ref, send_error_message UGLY_CHARS = { '\u2014': ' - ', diff --git a/openlp/plugins/bibles/lib/importers/bibleserver.py b/openlp/plugins/bibles/lib/importers/bibleserver.py index e651b84ab..16924d84a 100644 --- a/openlp/plugins/bibles/lib/importers/bibleserver.py +++ b/openlp/plugins/bibles/lib/importers/bibleserver.py @@ -20,7 +20,7 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ -The :mod:`http` module enables OpenLP to retrieve scripture from bible websites. +The :mod:`bibleserver` module enables OpenLP to retrieve scripture from http://bibleserver.com. """ import logging import re diff --git a/openlp/plugins/bibles/lib/importers/crosswalk.py b/openlp/plugins/bibles/lib/importers/crosswalk.py index 6c75209d1..fb354dd29 100644 --- a/openlp/plugins/bibles/lib/importers/crosswalk.py +++ b/openlp/plugins/bibles/lib/importers/crosswalk.py @@ -20,7 +20,7 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ -The :mod:`http` module enables OpenLP to retrieve scripture from bible websites. +The :mod:`crosswalk` module enables OpenLP to retrieve scripture from www.biblestudytools.com. """ import logging import re @@ -30,7 +30,7 @@ import urllib.error from openlp.core.common import RegistryProperties from openlp.plugins.bibles.lib import SearchResults -from openlp.plugins.bibles.lib.importers.http import get_soup_for_bible_ref, send_error_message +from openlp.plugins.bibles.lib.http import get_soup_for_bible_ref, send_error_message FIX_PUNKCTUATION_REGEX = re.compile(r'[ ]+([.,;])') REDUCE_SPACES_REGEX = re.compile(r'[ ]{2,}') diff --git a/openlp/plugins/bibles/lib/manager.py b/openlp/plugins/bibles/lib/manager.py index d2286bed2..2734411f5 100644 --- a/openlp/plugins/bibles/lib/manager.py +++ b/openlp/plugins/bibles/lib/manager.py @@ -27,7 +27,7 @@ from openlp.core.common import RegistryProperties, AppLocation, Settings, transl from openlp.plugins.bibles.lib import parse_reference, LanguageSelection from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta from .importers.csvbible import CSVBible -from .importers.http import HTTPBible +from .http import HTTPBible from .importers.opensong import OpenSongBible from .importers.osis import OSISBible from .importers.zefania import ZefaniaBible diff --git a/tests/functional/openlp_plugins/bibles/test_http.py b/tests/functional/openlp_plugins/bibles/test_bibleserver.py similarity index 88% rename from tests/functional/openlp_plugins/bibles/test_http.py rename to tests/functional/openlp_plugins/bibles/test_bibleserver.py index 839c81008..0849a63e3 100644 --- a/tests/functional/openlp_plugins/bibles/test_http.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleserver.py @@ -20,41 +20,13 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ -This module contains tests for the http module of the Bibles plugin. +This module contains tests for the bibleserver module of the Bibles plugin. """ from unittest import TestCase from bs4 import BeautifulSoup from tests.functional import patch, MagicMock -from openlp.plugins.bibles.lib.importers.http import BSExtract - -# TODO: Items left to test -# BGExtract -# __init__ -# _remove_elements -# _extract_verse -# _clean_soup -# _extract_verses -# _extract_verses_old -# get_bible_chapter -# get_books_from_http -# _get_application -# CWExtract -# __init__ -# get_bible_chapter -# get_books_from_http -# _get_application -# HTTPBible -# __init__ -# do_import -# get_verses -# get_chapter -# get_books -# get_chapter_count -# get_verse_count -# _get_application -# get_soup_for_bible_ref -# send_error_message +from openlp.plugins.bibles.lib.importers.bibleserver import BSExtract class TestBSExtract(TestCase): @@ -68,11 +40,12 @@ class TestBSExtract(TestCase): # get_books_from_http # _get_application def setUp(self): - self.get_soup_for_bible_ref_patcher = patch('openlp.plugins.bibles.lib.importers.http.get_soup_for_bible_ref') - self.log_patcher = patch('openlp.plugins.bibles.lib.importers.http.log') - self.send_error_message_patcher = patch('openlp.plugins.bibles.lib.importers.http.send_error_message') - self.socket_patcher = patch('openlp.plugins.bibles.lib.importers.http.socket') - self.urllib_patcher = patch('openlp.plugins.bibles.lib.importers.http.urllib') + self.get_soup_for_bible_ref_patcher = patch( + 'openlp.plugins.bibles.lib.importers.bibleserver.get_soup_for_bible_ref') + self.log_patcher = patch('openlp.plugins.bibles.lib.importers.bibleserver.log') + self.send_error_message_patcher = patch('openlp.plugins.bibles.lib.importers.bibleserver.send_error_message') + self.socket_patcher = patch('openlp.plugins.bibles.lib.http.socket') + self.urllib_patcher = patch('openlp.plugins.bibles.lib.importers.bibleserver.urllib') self.mock_get_soup_for_bible_ref = self.get_soup_for_bible_ref_patcher.start() self.mock_log = self.log_patcher.start() From f590294b4adf9838e1f6ffeedcb654014103bd53 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Thu, 11 Aug 2016 23:54:56 +0300 Subject: [PATCH 07/65] - Coded a combined help button, which opens the Local Help if using Mac or Win and Online help if using other OS's. (still need to change the help menu items) --- openlp/core/common/settings.py | 6 +++--- openlp/core/ui/mainwindow.py | 24 +++++++++++------------- openlp/core/ui/slidecontroller.py | 1 + 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/openlp/core/common/settings.py b/openlp/core/common/settings.py index 0be8d819a..11aaeae83 100644 --- a/openlp/core/common/settings.py +++ b/openlp/core/common/settings.py @@ -254,7 +254,7 @@ class Settings(QtCore.QSettings): QtCore.QSettings.__init__(self, *args) # Add shortcuts here so QKeySequence has a QApplication instance to use. Settings.__default_settings__.update({ - 'shortcuts/aboutItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], + 'shortcuts/aboutItem': [QtGui.QKeySequence(QtCore.Qt.SHIFT + QtCore.Qt.Key_F1)], 'shortcuts/addToService': [], 'shortcuts/audioPauseItem': [], 'shortcuts/displayTagItem': [], @@ -334,8 +334,8 @@ class Settings(QtCore.QSettings): QtGui.QKeySequence(QtCore.Qt.Key_PageDown)], 'shortcuts/nextService': [QtGui.QKeySequence(QtCore.Qt.Key_Right)], 'shortcuts/newService': [], - 'shortcuts/offlineHelpItem': [QtGui.QKeySequence(QtGui.QKeySequence.HelpContents)], - 'shortcuts/onlineHelpItem': [QtGui.QKeySequence(QtGui.QKeySequence.HelpContents)], + 'shortcuts/offlineHelpItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], + 'shortcuts/onlineHelpItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], 'shortcuts/openService': [], 'shortcuts/saveService': [], 'shortcuts/previousItem_live': [QtGui.QKeySequence(QtCore.Qt.Key_Up), diff --git a/openlp/core/ui/mainwindow.py b/openlp/core/ui/mainwindow.py index 32b1c4db2..2f51dd997 100644 --- a/openlp/core/ui/mainwindow.py +++ b/openlp/core/ui/mainwindow.py @@ -312,18 +312,18 @@ class Ui_MainWindow(object): self.offline_help_item = create_action(main_window, 'offlineHelpItem', icon=':/system/system_help_contents.png', can_shortcuts=True, - category=UiStrings().Help, triggers=self.on_offline_help_clicked) + category=UiStrings().Help, triggers=self.on_help_button_clicked) elif is_macosx(): self.local_help_file = os.path.join(AppLocation.get_directory(AppLocation.AppDir), '..', 'Resources', 'OpenLP.help') self.offline_help_item = create_action(main_window, 'offlineHelpItem', icon=':/system/system_help_contents.png', can_shortcuts=True, - category=UiStrings().Help, triggers=self.on_offline_help_clicked) + category=UiStrings().Help, triggers=self.on_help_button_clicked) self.on_line_help_item = create_action(main_window, 'onlineHelpItem', icon=':/system/system_online_help.png', can_shortcuts=True, - category=UiStrings().Help, triggers=self.on_online_help_clicked) + category=UiStrings().Help, triggers=self.on_help_button_clicked) self.web_site_item = create_action(main_window, 'webSiteItem', can_shortcuts=True, category=UiStrings().Help) # Shortcuts not connected to buttons or menu entries. self.search_shortcut_action = create_action(main_window, @@ -778,18 +778,16 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow, RegistryProperties): import webbrowser webbrowser.open_new('http://openlp.org/') - def on_offline_help_clicked(self): + def on_help_button_clicked(self): """ - Load the local OpenLP help file + If is_macosx or is_win, open the local OpenLP help file. + Use the Online manual in other cases. (Linux) """ - QtGui.QDesktopServices.openUrl(QtCore.QUrl("file:///" + self.local_help_file)) - - def on_online_help_clicked(self): - """ - Load the online OpenLP manual - """ - import webbrowser - webbrowser.open_new('http://manual.openlp.org/') + if is_macosx() or is_win(): + QtGui.QDesktopServices.openUrl(QtCore.QUrl("file:///" + self.local_help_file)) + else: + import webbrowser + webbrowser.open_new('http://manual.openlp.org/') def on_about_item_clicked(self): """ diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index 43348a980..f56e8247e 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -234,6 +234,7 @@ class SlideController(DisplayController, RegistryProperties): self.hide_menu.setPopupMode(QtWidgets.QToolButton.MenuButtonPopup) self.hide_menu.setMenu(QtWidgets.QMenu(translate('OpenLP.SlideController', 'Hide'), self.toolbar)) self.toolbar.add_toolbar_widget(self.hide_menu) + # The order of the blank to modes in Shortcuts list comes from here. self.desktop_screen = create_action(self, 'desktopScreen', text=translate('OpenLP.SlideController', 'Show Desktop'), icon=':/slides/slide_desktop.png', From 851a35a8227f75fb0e2782181b168ed51fcd5ca6 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 12 Aug 2016 00:11:34 +0300 Subject: [PATCH 08/65] - Removed Local & Online help buttons, created "User manual" button. --- openlp/core/ui/mainwindow.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/openlp/core/ui/mainwindow.py b/openlp/core/ui/mainwindow.py index 2f51dd997..ef3c8d06d 100644 --- a/openlp/core/ui/mainwindow.py +++ b/openlp/core/ui/mainwindow.py @@ -309,21 +309,13 @@ class Ui_MainWindow(object): self.about_item.setMenuRole(QtWidgets.QAction.AboutRole) if is_win(): self.local_help_file = os.path.join(AppLocation.get_directory(AppLocation.AppDir), 'OpenLP.chm') - self.offline_help_item = create_action(main_window, 'offlineHelpItem', - icon=':/system/system_help_contents.png', - can_shortcuts=True, - category=UiStrings().Help, triggers=self.on_help_button_clicked) elif is_macosx(): self.local_help_file = os.path.join(AppLocation.get_directory(AppLocation.AppDir), '..', 'Resources', 'OpenLP.help') - self.offline_help_item = create_action(main_window, 'offlineHelpItem', - icon=':/system/system_help_contents.png', - can_shortcuts=True, - category=UiStrings().Help, triggers=self.on_help_button_clicked) - self.on_line_help_item = create_action(main_window, 'onlineHelpItem', - icon=':/system/system_online_help.png', + self.on_help_item = create_action(main_window, 'onlineHelpItem', + icon=':/system/system_help_contents.png', can_shortcuts=True, - category=UiStrings().Help, triggers=self.on_help_button_clicked) + category=UiStrings().Help, triggers=self.on_help_clicked) self.web_site_item = create_action(main_window, 'webSiteItem', can_shortcuts=True, category=UiStrings().Help) # Shortcuts not connected to buttons or menu entries. self.search_shortcut_action = create_action(main_window, @@ -362,11 +354,7 @@ class Ui_MainWindow(object): add_actions(self.tools_menu, (self.tools_open_data_folder, None)) add_actions(self.tools_menu, (self.tools_first_time_wizard, None)) add_actions(self.tools_menu, [self.update_theme_images]) - if (is_win() or is_macosx()) and (hasattr(sys, 'frozen') and sys.frozen == 1): - add_actions(self.help_menu, (self.offline_help_item, self.on_line_help_item, None, self.web_site_item, - self.about_item)) - else: - add_actions(self.help_menu, (self.on_line_help_item, None, self.web_site_item, self.about_item)) + add_actions(self.help_menu, (self.on_help_item, None, self.web_site_item, self.about_item)) add_actions(self.menu_bar, (self.file_menu.menuAction(), self.view_menu.menuAction(), self.tools_menu.menuAction(), self.settings_menu.menuAction(), self.help_menu.menuAction())) add_actions(self, [self.search_shortcut_action]) @@ -462,9 +450,7 @@ class Ui_MainWindow(object): 'from here.')) self.about_item.setText(translate('OpenLP.MainWindow', '&About')) self.about_item.setStatusTip(translate('OpenLP.MainWindow', 'More information about OpenLP.')) - if is_win() or is_macosx(): - self.offline_help_item.setText(translate('OpenLP.MainWindow', '&User Guide')) - self.on_line_help_item.setText(translate('OpenLP.MainWindow', '&Online Help')) + self.on_help_item.setText(translate('OpenLP.MainWindow', '&User Manual')) self.search_shortcut_action.setText(UiStrings().Search) self.search_shortcut_action.setToolTip( translate('OpenLP.MainWindow', 'Jump to the search box of the current active plugin.')) @@ -778,7 +764,7 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow, RegistryProperties): import webbrowser webbrowser.open_new('http://openlp.org/') - def on_help_button_clicked(self): + def on_help_clicked(self): """ If is_macosx or is_win, open the local OpenLP help file. Use the Online manual in other cases. (Linux) From 4944a98a1de4810cfb99bbdd5a1b46156f2458c8 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 12 Aug 2016 00:15:03 +0300 Subject: [PATCH 09/65] - Removed resources/images/system_online_help.png as it is no longer needed. --- resources/images/openlp-2.qrc | 1 - resources/images/system_online_help.png | Bin 953 -> 0 bytes 2 files changed, 1 deletion(-) delete mode 100644 resources/images/system_online_help.png diff --git a/resources/images/openlp-2.qrc b/resources/images/openlp-2.qrc index b45cc745d..867e4dd7a 100644 --- a/resources/images/openlp-2.qrc +++ b/resources/images/openlp-2.qrc @@ -130,7 +130,6 @@ clear_shortcut.png system_about.png system_help_contents.png - system_online_help.png system_mediamanager.png system_volunteer.png system_servicemanager.png diff --git a/resources/images/system_online_help.png b/resources/images/system_online_help.png deleted file mode 100644 index 670c0716f6b7ad8188916f43c4ce6607eb76dcbd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 953 zcmV;q14jIbP)@AEu!U{b6o>#4 z1W{1Yzz+x%G$0{Cnp9w02ni{Y1uYVPfr1u>B{`q0CI3_WlmbwM+?u5is{)=3PX9q z)X?ar%*;>Hies9wA`p^CZN=7sN06jA+1&esqsvuU=(9hZt_(XG$ul!!#2QirFJG_t(cLKJ`T=8Mn?*7TTU8VnT7FZw!L99r1LShU8fi0xi zNt79iQbX-5;JaUUs617sk%BKRRK_AZATd&)O-i35S=*=+)lSfD>3uyCmemPM(Mqg=>xspwLB)TY@8NE$xgh8Kqod0ocw(z;USmR}2W*0yMOx@^`v z?6*RO^EP9HS*jNY@DpeU5w_8^wpu1=#a}74v?c*sp7&*=*16l>IMq4VAy6sxcFcHx zhLN5mPEt~nAf&~~y}D^_H-C@(&SwBNfHA9)5I0`ao7`+4-Me}D+SB=?;Xbx{&8Kn7 zgZp*XZ$IRp@7JR6)L)G{?G~2?SfW3;00000NkvXXu0mjf7 Date: Fri, 12 Aug 2016 00:32:20 +0300 Subject: [PATCH 10/65] - Created a shortcut for the new help button, removed the old two shortcuts and created obsolete settings transfer. --- openlp/core/common/settings.py | 7 ++++--- openlp/core/ui/mainwindow.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/openlp/core/common/settings.py b/openlp/core/common/settings.py index 11aaeae83..35d2dfad7 100644 --- a/openlp/core/common/settings.py +++ b/openlp/core/common/settings.py @@ -214,7 +214,9 @@ class Settings(QtCore.QSettings): ('media/players', 'media/players_temp', [(media_players_conv, None)]), # Convert phonon to system ('media/players_temp', 'media/players', []), # Move temp setting from above to correct setting ('advanced/default color', 'core/logo background color', []), # Default image renamed + moved to general > 2.4. - ('advanced/default image', '/core/logo file', []) # Default image renamed + moved to general after 2.4. + ('advanced/default image', 'core/logo file', []), # Default image renamed + moved to general after 2.4. + ('shortcuts/offlineHelpItem', 'shortcuts/HelpItem', []), # There used to be separated buttons for local and + ('shortcuts/onlineHelpItem', 'shortcuts/HelpItem', []) # online help buttons. Now combined into one since 2.6. ] @staticmethod @@ -274,6 +276,7 @@ class Settings(QtCore.QSettings): 'shortcuts/fileSaveItem': [QtGui.QKeySequence(QtGui.QKeySequence.Save)], 'shortcuts/fileOpenItem': [QtGui.QKeySequence(QtGui.QKeySequence.Open)], 'shortcuts/goLive': [], + 'shortcuts/HelpItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], 'shortcuts/importThemeItem': [], 'shortcuts/importBibleItem': [], 'shortcuts/listViewBiblesDeleteItem': [QtGui.QKeySequence(QtGui.QKeySequence.Delete)], @@ -334,8 +337,6 @@ class Settings(QtCore.QSettings): QtGui.QKeySequence(QtCore.Qt.Key_PageDown)], 'shortcuts/nextService': [QtGui.QKeySequence(QtCore.Qt.Key_Right)], 'shortcuts/newService': [], - 'shortcuts/offlineHelpItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], - 'shortcuts/onlineHelpItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], 'shortcuts/openService': [], 'shortcuts/saveService': [], 'shortcuts/previousItem_live': [QtGui.QKeySequence(QtCore.Qt.Key_Up), diff --git a/openlp/core/ui/mainwindow.py b/openlp/core/ui/mainwindow.py index ef3c8d06d..5096ad431 100644 --- a/openlp/core/ui/mainwindow.py +++ b/openlp/core/ui/mainwindow.py @@ -312,7 +312,7 @@ class Ui_MainWindow(object): elif is_macosx(): self.local_help_file = os.path.join(AppLocation.get_directory(AppLocation.AppDir), '..', 'Resources', 'OpenLP.help') - self.on_help_item = create_action(main_window, 'onlineHelpItem', + self.on_help_item = create_action(main_window, 'HelpItem', icon=':/system/system_help_contents.png', can_shortcuts=True, category=UiStrings().Help, triggers=self.on_help_clicked) From c2b05e34545c9409b5c2d33588f24c82caec7610 Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Sun, 14 Aug 2016 10:24:17 +0200 Subject: [PATCH 11/65] Fix bug #1547964 by ignoring the exception (it's harmless) Fixes: https://launchpad.net/bugs/1547964 --- openlp/core/ui/media/systemplayer.py | 11 ++++++++++ .../openlp_core_ui_media/test_systemplayer.py | 20 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/openlp/core/ui/media/systemplayer.py b/openlp/core/ui/media/systemplayer.py index a36ce445b..e632f9b7b 100644 --- a/openlp/core/ui/media/systemplayer.py +++ b/openlp/core/ui/media/systemplayer.py @@ -95,6 +95,16 @@ class SystemPlayer(MediaPlayer): mime_type_list.append(ext) log.info('MediaPlugin: %s extensions: %s', mime_type, ' '.join(extensions)) + def disconnect_slots(self, signal): + """ + Safely disconnect the slots from `signal` + """ + try: + signal.disconnect() + except TypeError: + # If disconnect() is called on a signal without slots, it throws a TypeError + pass + def setup(self, display): """ Set up the player widgets @@ -160,6 +170,7 @@ class SystemPlayer(MediaPlayer): if start_time > 0: self.seek(display, controller.media_info.start_time * 1000) self.volume(display, controller.media_info.volume) + self.disconnect_slots(display.media_player.durationChanged) display.media_player.durationChanged.connect(functools.partial(self.set_duration, controller)) self.set_state(MediaState.Playing, display) display.video_widget.raise_() diff --git a/tests/functional/openlp_core_ui_media/test_systemplayer.py b/tests/functional/openlp_core_ui_media/test_systemplayer.py index d1873d2df..2bac0089b 100644 --- a/tests/functional/openlp_core_ui_media/test_systemplayer.py +++ b/tests/functional/openlp_core_ui_media/test_systemplayer.py @@ -107,6 +107,22 @@ class TestSystemPlayer(TestCase): mocked_video_widget.hide.assert_called_once_with() self.assertTrue(player.has_own_widget) + def test_disconnect_slots(self): + """ + Test that we the disconnect slots method catches the TypeError + """ + # GIVEN: A SystemPlayer class and a signal that throws a TypeError + player = SystemPlayer(self) + mocked_signal = MagicMock() + mocked_signal.disconnect.side_effect = \ + TypeError('disconnect() failed between \'durationChanged\' and all its connections') + + # WHEN: disconnect_slots() is called + player.disconnect_slots(mocked_signal) + + # THEN: disconnect should have been called and the exception should have been ignored + mocked_signal.disconnect.assert_called_once_with() + def test_check_available(self): """ Test the check_available() method on SystemPlayer @@ -198,7 +214,8 @@ class TestSystemPlayer(TestCase): with patch.object(player, 'get_live_state') as mocked_get_live_state, \ patch.object(player, 'seek') as mocked_seek, \ patch.object(player, 'volume') as mocked_volume, \ - patch.object(player, 'set_state') as mocked_set_state: + patch.object(player, 'set_state') as mocked_set_state, \ + patch.object(player, 'disconnect_slots') as mocked_disconnect_slots: mocked_get_live_state.return_value = QtMultimedia.QMediaPlayer.PlayingState result = player.play(mocked_display) @@ -207,6 +224,7 @@ class TestSystemPlayer(TestCase): mocked_display.media_player.play.assert_called_once_with() mocked_seek.assert_called_once_with(mocked_display, 1000) mocked_volume.assert_called_once_with(mocked_display, 1) + mocked_disconnect_slots.assert_called_once_with(mocked_display.media_player.durationChanged) mocked_display.media_player.durationChanged.connect.assert_called_once_with('function') mocked_set_state.assert_called_once_with(MediaState.Playing, mocked_display) mocked_display.video_widget.raise_.assert_called_once_with() From f08d0c28a58f4885a56c8dc52bbff376082a04b3 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sun, 14 Aug 2016 11:00:27 +0100 Subject: [PATCH 12/65] further bible refactors --- openlp/plugins/bibles/bibleplugin.py | 8 +- openlp/plugins/bibles/lib/__init__.py | 4 +- openlp/plugins/bibles/lib/bibleimport.py | 1 - .../plugins/bibles/lib/importers/opensong.py | 7 +- openlp/plugins/bibles/lib/importers/osis.py | 9 +- .../plugins/bibles/lib/importers/zefania.py | 4 +- .../openlp_plugins/bibles/test_bibleimport.py | 85 ++++++++++++++++--- .../openlp_plugins/bibles/test_csvimport.py | 4 +- .../bibles/test_zefaniaimport.py | 6 +- 9 files changed, 86 insertions(+), 42 deletions(-) diff --git a/openlp/plugins/bibles/bibleplugin.py b/openlp/plugins/bibles/bibleplugin.py index f63b85a92..e9168d695 100644 --- a/openlp/plugins/bibles/bibleplugin.py +++ b/openlp/plugins/bibles/bibleplugin.py @@ -140,10 +140,10 @@ class BiblePlugin(Plugin): def uses_theme(self, theme): """ - Called to find out if the bible plugin is currently using a theme. Returns ``1`` if the theme is being used, - otherwise returns ``0``. + Called to find out if the bible plugin is currently using a theme. :param theme: The theme + :return: 1 if the theme is being used, otherwise returns 0 """ if str(self.settings_tab.bible_theme) == theme: return 1 @@ -151,11 +151,11 @@ class BiblePlugin(Plugin): def rename_theme(self, old_theme, new_theme): """ - Rename the theme the bible plugin is using making the plugin use the - new name. + Rename the theme the bible plugin is using, making the plugin use the new name. :param old_theme: The name of the theme the plugin should stop using. Unused for this particular plugin. :param new_theme: The new name the plugin should now use. + :return: None """ self.settings_tab.bible_theme = new_theme self.settings_tab.save() diff --git a/openlp/plugins/bibles/lib/__init__.py b/openlp/plugins/bibles/lib/__init__.py index 804755d18..e730009e7 100644 --- a/openlp/plugins/bibles/lib/__init__.py +++ b/openlp/plugins/bibles/lib/__init__.py @@ -173,7 +173,7 @@ class BibleStrings(object): def update_reference_separators(): """ - Updates separators and matches for parsing and formating scripture references. + Updates separators and matches for parsing and formatting scripture references. """ default_separators = [ '|'.join([ @@ -215,7 +215,7 @@ def update_reference_separators(): # escape reserved characters for character in '\\.^$*+?{}[]()': source_string = source_string.replace(character, '\\' + character) - # add various unicode alternatives + # add various Unicode alternatives source_string = source_string.replace('-', '(?:[-\u00AD\u2010\u2011\u2012\u2014\u2014\u2212\uFE63\uFF0D])') source_string = source_string.replace(',', '(?:[,\u201A])') REFERENCE_SEPARATORS['sep_{role}'.format(role=role)] = '\s*(?:{source})\s*'.format(source=source_string) diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index 4d015223b..7ebdcb170 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -35,7 +35,6 @@ class BibleImport(OpenLPMixin, BibleDB): """ Helper class to import bibles from a third party source into OpenLP """ - # TODO: Test def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.filename = kwargs['filename'] if 'filename' in kwargs else None diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 43c1cf8ca..10c0ed87e 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -73,12 +73,7 @@ class OpenSongBible(BibleImport): for book in bible.b: if self.stop_import_flag: break - book_ref_id = self.get_book_ref_id_by_name(str(book.attrib['n']), len(bible.b), language_id) - if not book_ref_id: - log.error('Importing books from "{name}" failed'.format(name=self.filename)) - return False - book_details = BiblesResourcesDB.get_book_by_id(book_ref_id) - db_book = self.create_book(book.attrib['n'], book_ref_id, book_details['testament_id']) + db_book = self.find_and_create_book(str(book.attrib['n']), len(bible.b), language_id) chapter_number = 0 for chapter in book.c: if self.stop_import_flag: diff --git a/openlp/plugins/bibles/lib/importers/osis.py b/openlp/plugins/bibles/lib/importers/osis.py index c833277fe..db12bb7e9 100644 --- a/openlp/plugins/bibles/lib/importers/osis.py +++ b/openlp/plugins/bibles/lib/importers/osis.py @@ -98,7 +98,7 @@ class OSISBible(BibleImport): language_id = self.get_language_id(language[0] if language else None, bible_name=self.filename) if not language_id: return False - num_books = int(osis_bible_tree.xpath("count(//ns:div[@type='book'])", namespaces=NS)) + no_of_books = int(osis_bible_tree.xpath("count(//ns:div[@type='book'])", namespaces=NS)) # Precompile a few xpath-querys verse_in_chapter = etree.XPath('count(//ns:chapter[1]/ns:verse)', namespaces=NS) text_in_verse = etree.XPath('count(//ns:verse[1]/text())', namespaces=NS) @@ -109,12 +109,7 @@ class OSISBible(BibleImport): break # Remove div-tags in the book etree.strip_tags(book, '{http://www.bibletechnologies.net/2003/OSIS/namespace}div') - book_ref_id = self.get_book_ref_id_by_name(book.get('osisID'), num_books, language_id) - if not book_ref_id: - log.error('Importing books from "{name}" failed'.format(name=self.filename)) - return False - book_details = BiblesResourcesDB.get_book_by_id(book_ref_id) - db_book = self.create_book(book_details['name'], book_ref_id, book_details['testament_id']) + db_book = self.find_and_create_book(book.get('osisID'), no_of_books, language_id) # Find out if chapter-tags contains the verses, or if it is used as milestone/anchor if int(verse_in_chapter(book)) > 0: # The chapter tags contains the verses diff --git a/openlp/plugins/bibles/lib/importers/zefania.py b/openlp/plugins/bibles/lib/importers/zefania.py index 61ee41166..bc31a1664 100644 --- a/openlp/plugins/bibles/lib/importers/zefania.py +++ b/openlp/plugins/bibles/lib/importers/zefania.py @@ -54,7 +54,7 @@ class ZefaniaBible(BibleImport): language_id = self.get_language_id(language[0] if language else None, bible_name=self.filename) if not language_id: return False - num_books = int(xmlbible.xpath('count(//BIBLEBOOK)')) + no_of_books = int(xmlbible.xpath('count(//BIBLEBOOK)')) self.wizard.progress_bar.setMaximum(int(xmlbible.xpath('count(//CHAPTER)'))) for BIBLEBOOK in xmlbible: if self.stop_import_flag: @@ -64,7 +64,7 @@ class ZefaniaBible(BibleImport): if not bname and not bnumber: continue if bname: - book_ref_id = self.get_book_ref_id_by_name(bname, num_books, language_id) + book_ref_id = self.get_book_ref_id_by_name(bname, no_of_books, language_id) else: log.debug('Could not find a name, will use number, basically a guess.') book_ref_id = int(bnumber) diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index e2076df55..127c6fd16 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -29,8 +29,10 @@ from lxml import etree, objectify from unittest import TestCase from openlp.core.common.languages import Language +from openlp.core.lib.exceptions import ValidationError from openlp.plugins.bibles.lib.bibleimport import BibleImport -from tests.functional import MagicMock, patch +from openlp.plugins.bibles.lib.db import BibleDB +from tests.functional import ANY, MagicMock, patch class TestBibleImport(TestCase): @@ -39,23 +41,79 @@ class TestBibleImport(TestCase): """ def setUp(self): - test_file = BytesIO(b'\n' - b'\n' - b'
Test

data

tokeep
\n' - b' Testdatatodiscard\n' - b'') + test_file = BytesIO( + b'\n' + b'\n' + b'
Test

data

tokeep
\n' + b' Testdatatodiscard\n' + b'' + ) self.file_patcher = patch('builtins.open', return_value=test_file) - self.log_patcher = patch('openlp.plugins.bibles.lib.bibleimport.log') - self.setup_patcher = patch('openlp.plugins.bibles.lib.db.BibleDB._setup') - self.addCleanup(self.file_patcher.stop) - self.addCleanup(self.log_patcher.stop) - self.addCleanup(self.setup_patcher.stop) - self.file_patcher.start() + self.log_patcher = patch('openlp.plugins.bibles.lib.bibleimport.log') + self.addCleanup(self.log_patcher.stop) self.mock_log = self.log_patcher.start() + self.setup_patcher = patch('openlp.plugins.bibles.lib.db.BibleDB._setup') + self.addCleanup(self.setup_patcher.stop) self.setup_patcher.start() + def init_kwargs_none_test(self): + """ + Test the initialisation of the BibleImport Class when no key word arguments are supplied + """ + # GIVEN: A patched BibleDB._setup, BibleImport class and mocked parent + # WHEN: Creating an instance of BibleImport with no key word arguments + instance = BibleImport(MagicMock()) + + # THEN: The filename attribute should be None + self.assertIsNone(instance.filename) + self.assertIsInstance(instance, BibleDB) + + def init_kwargs_set_test(self): + """ + Test the initialisation of the BibleImport Class when supplied with select keyword arguments + """ + # GIVEN: A patched BibleDB._setup, BibleImport class and mocked parent + # WHEN: Creating an instance of BibleImport with selected key word arguments + kwargs = {'filename': 'bible.xml'} + instance = BibleImport(MagicMock(), **kwargs) + + # THEN: The filename keyword should be set to bible.xml + self.assertEqual(instance.filename, 'bible.xml') + self.assertIsInstance(instance, BibleDB) + + def check_for_compression_test(self): + """ + Test the check_for_compression method when called with a path to an uncompressed file + """ + # GIVEN: A mocked is_zipfile which returns False and an instance of BibleImport + with patch('openlp.plugins.bibles.lib.bibleimport.is_zipfile', return_value=False) as mocked_is_zip: + instance = BibleImport(MagicMock()) + + # WHEN: Calling check_for_compression + result = instance.check_for_compression('filename.tst') + + # THEN: None should be returned + self.assertIsNone(result) + mocked_is_zip.assert_called_once_with('filename.tst') + + def check_for_compression_zip_file_test(self): + """ + Test the check_for_compression method when called with a path to a compressed file + """ + # GIVEN: A patched is_zipfile which returns True and an instance of BibleImport + with patch('openlp.plugins.bibles.lib.bibleimport.is_zipfile', return_value=True),\ + patch('openlp.plugins.bibles.lib.bibleimport.critical_error_message_box') as mocked_message_box: + instance = BibleImport(MagicMock()) + + # WHEN: Calling check_for_compression + # THEN: A Validation error should be raised and the user should be notified. + with self.assertRaises(ValidationError) as context: + instance.check_for_compression('filename.tst') + self.assertTrue(mocked_message_box.called) + self.assertEqual(context.exception.msg, '"filename.tst" is compressed') + def get_language_id_language_found_test(self): """ Test get_language_id() when called with a name found in the languages list @@ -81,8 +139,7 @@ class TestBibleImport(TestCase): Test get_language_id() when called with a name not found in the languages list """ # GIVEN: A mocked languages.get_language which returns language and an instance of BibleImport - with patch('openlp.core.common.languages.get_language', return_value=None) \ - as mocked_languages_get_language, \ + with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \ patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=20) as mocked_db_get_language: instance = BibleImport(MagicMock()) instance.save_meta = MagicMock() diff --git a/tests/functional/openlp_plugins/bibles/test_csvimport.py b/tests/functional/openlp_plugins/bibles/test_csvimport.py index ada03a07d..7a56c1b85 100644 --- a/tests/functional/openlp_plugins/bibles/test_csvimport.py +++ b/tests/functional/openlp_plugins/bibles/test_csvimport.py @@ -46,10 +46,10 @@ class TestCSVImport(TestCase): def setUp(self): self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') self.addCleanup(self.manager_patcher.stop) - self.addCleanup(self.registry_patcher.stop) self.manager_patcher.start() + self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() def test_create_importer(self): diff --git a/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py b/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py index 200a36f45..5294b7f5c 100644 --- a/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py +++ b/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py @@ -42,14 +42,12 @@ class TestZefaniaImport(TestCase): def setUp(self): self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') + self.addCleanup(self.manager_patcher.stop) self.manager_patcher.start() - def tearDown(self): - self.registry_patcher.stop() - self.manager_patcher.stop() - def test_create_importer(self): """ Test creating an instance of the Zefania file importer From c77dd470c5c165e7a92d2639cb23f3f11f91eed1 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sun, 14 Aug 2016 23:19:54 +0300 Subject: [PATCH 13/65] - Added the missing "clicked" to make the code easier to understand. --- openlp/plugins/bibles/lib/mediaitem.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openlp/plugins/bibles/lib/mediaitem.py b/openlp/plugins/bibles/lib/mediaitem.py index bdcda9942..2b35c9115 100644 --- a/openlp/plugins/bibles/lib/mediaitem.py +++ b/openlp/plugins/bibles/lib/mediaitem.py @@ -254,8 +254,8 @@ class BibleMediaItem(MediaManagerItem): self.quickStyleComboBox.activated.connect(self.on_quick_style_combo_box_changed) self.advancedStyleComboBox.activated.connect(self.on_advanced_style_combo_box_changed) # Buttons - self.advancedClearButton.clicked.connect(self.on_clear_button_advanced) - self.quickClearButton.clicked.connect(self.on_clear_button) + self.advancedClearButton.clicked.connect(self.on_advanced_clear_button_clicked) + self.quickClearButton.clicked.connect(self.on_clear_button_clicked) self.advancedSearchButton.clicked.connect(self.on_advanced_search_button) self.quickSearchButton.clicked.connect(self.on_quick_search_button) # Other stuff @@ -548,15 +548,15 @@ class BibleMediaItem(MediaManagerItem): self.advancedTab.setVisible(True) self.advanced_book_combo_box.setFocus() - def on_clear_button(self): + def on_clear_button_clicked(self): # Clear the list, then set the "No search Results" message, then clear the text field and give it focus. self.list_view.clear() self.check_search_result() self.quick_search_edit.clear() self.quick_search_edit.setFocus() - def on_clear_button_advanced(self): - # The same as the on_clear_button, but does not give focus to Quick search field. + def on_advanced_clear_button_clicked(self): + # The same as the on_clear_button_clicked, but does not give focus to Quick search field. self.list_view.clear() self.check_search_result() From 693c18a23acc1989301cb642122500d810349eeb Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Sun, 14 Aug 2016 22:48:36 +0200 Subject: [PATCH 14/65] Add some tests for the MediaShout importer --- .../openlp_plugins/songs/test_mediashout.py | 224 ++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 tests/functional/openlp_plugins/songs/test_mediashout.py diff --git a/tests/functional/openlp_plugins/songs/test_mediashout.py b/tests/functional/openlp_plugins/songs/test_mediashout.py new file mode 100644 index 000000000..ae575ae1e --- /dev/null +++ b/tests/functional/openlp_plugins/songs/test_mediashout.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +Test the MediaShout importer +""" +from unittest import TestCase +from collections import namedtuple + +from openlp.core.common import Registry +from openlp.plugins.songs.lib.importers.mediashout import MediaShoutImport + +from tests.functional import MagicMock, patch, call + + +class TestMediaShoutImport(TestCase): + """ + Test the MediaShout importer + """ + def setUp(self): + """ + Set the tests up + """ + Registry().create() + + def test_constructor(self): + """ + Test the MediaShoutImport constructor + """ + # GIVEN: A MediaShoutImport class + # WHEN: It is created + importer = MediaShoutImport(MagicMock(), filename='mediashout.db') + + # THEN: It should not be None + self.assertIsNotNone(importer) + + @patch('openlp.plugins.songs.lib.importers.mediashout.pyodbc') + def test_do_import_fails_to_connect(self, mocked_pyodbc): + """ + Test that do_import exits early when unable to connect to the database + """ + # GIVEN: A MediaShoutImport instance + importer = MediaShoutImport(MagicMock(), filename='mediashout.db') + mocked_pyodbc.connect.side_effect = Exception('Unable to connect') + + # WHEN: do_import is called + with patch.object(importer, 'log_error') as mocked_log_error: + importer.do_import() + + # THEN: The songs should have been imported + mocked_log_error.assert_called_once_with('mediashout.db', 'Unable to open the MediaShout database.') + + @patch('openlp.plugins.songs.lib.importers.mediashout.pyodbc') + def test_do_import(self, mocked_pyodbc): + """ + Test the MediaShoutImport do_import method + """ + SongRecord = namedtuple('SongRecord', 'Record, Title, Author, Copyright, SongID, CCLI, Notes') + VerseRecord = namedtuple('VerseRecord', 'Type, Number, Text') + PlayOrderRecord = namedtuple('PlayOrderRecord', 'Type, Number, POrder') + ThemeRecord = namedtuple('ThemeRecord', 'Name') + GroupRecord = namedtuple('GroupRecord', 'Name') + song = SongRecord(1, 'Amazing Grace', 'William Wilberforce', 'Public Domain', 1, '654321', '') + verse = VerseRecord('Verse', 1, 'Amazing grace, how sweet the sound\nThat saved a wretch like me') + play_order = PlayOrderRecord('Verse', 1, 1) + theme = ThemeRecord('Grace') + group = GroupRecord('Hymns') + + # GIVEN: A MediaShoutImport instance and a bunch of stuff mocked out + importer = MediaShoutImport(MagicMock(), filename='mediashout.db') + mocked_cursor = MagicMock() + mocked_cursor.fetchall.side_effect = [[song], [verse], [play_order], [theme], [group]] + mocked_cursor.tables.fetchone.return_value = True + mocked_connection = MagicMock() + mocked_connection.cursor.return_value = mocked_cursor + mocked_pyodbc.connect.return_value = mocked_connection + + # WHEN: do_import is called + with patch.object(importer, 'import_wizard') as mocked_import_wizard, \ + patch.object(importer, 'process_song') as mocked_process_song: + importer.do_import() + + # THEN: The songs should have been imported + expected_execute_calls = [ + call('SELECT Record, Title, Author, Copyright, SongID, CCLI, Notes FROM Songs ORDER BY Title'), + call('SELECT Type, Number, Text FROM Verses WHERE Record = ? ORDER BY Type, Number', 1.0), + call('SELECT Type, Number, POrder FROM PlayOrder WHERE Record = ? ORDER BY POrder', 1.0), + call('SELECT Name FROM Themes INNER JOIN SongThemes ON SongThemes.ThemeId = Themes.ThemeId ' + 'WHERE SongThemes.Record = ?', 1.0), + call('SELECT Name FROM Groups INNER JOIN SongGroups ON SongGroups.GroupId = Groups.GroupId ' + 'WHERE SongGroups.Record = ?', 1.0) + ] + self.assertEqual(expected_execute_calls, mocked_cursor.execute.call_args_list) + mocked_process_song.assert_called_once_with(song, [verse], [play_order], [theme, group]) + + @patch('openlp.plugins.songs.lib.importers.mediashout.pyodbc') + def test_do_import_breaks_on_stop(self, mocked_pyodbc): + """ + Test the MediaShoutImport do_import stops when the user presses the cancel button + """ + SongRecord = namedtuple('SongRecord', 'Record, Title, Author, Copyright, SongID, CCLI, Notes') + song = SongRecord(1, 'Amazing Grace', 'William Wilberforce', 'Public Domain', 1, '654321', '') + + # GIVEN: A MediaShoutImport instance and a bunch of stuff mocked out + importer = MediaShoutImport(MagicMock(), filename='mediashout.db') + mocked_cursor = MagicMock() + mocked_cursor.fetchall.return_value = [song] + mocked_connection = MagicMock() + mocked_connection.cursor.return_value = mocked_cursor + mocked_pyodbc.connect.return_value = mocked_connection + + # WHEN: do_import is called, but cancelled + with patch.object(importer, 'import_wizard') as mocked_import_wizard, \ + patch.object(importer, 'process_song') as mocked_process_song: + importer.stop_import_flag = True + importer.do_import() + + # THEN: The songs should have been imported + mocked_cursor.execute.assert_called_once_with( + 'SELECT Record, Title, Author, Copyright, SongID, CCLI, Notes FROM Songs ORDER BY Title') + mocked_process_song.assert_not_called() + + def test_process_song(self): + """ + Test the process_song method of the MediaShoutImport + """ + # GIVEN: An importer and a song + SongRecord = namedtuple('SongRecord', 'Record, Title, Author, Copyright, SongID, CCLI, Notes') + VerseRecord = namedtuple('VerseRecord', 'Type, Number, Text') + PlayOrderRecord = namedtuple('PlayOrderRecord', 'Type, Number, POrder') + ThemeRecord = namedtuple('ThemeRecord', 'Name') + GroupRecord = namedtuple('GroupRecord', 'Name') + song = SongRecord(1, 'Amazing Grace', 'William Wilberforce', 'Public Domain', 'Hymns', '654321', + 'Great old hymn') + verse = VerseRecord(0, 1, 'Amazing grace, how sweet the sound\nThat saved a wretch like me') + play_order = PlayOrderRecord(0, 1, 1) + theme = ThemeRecord('Grace') + group = GroupRecord('Hymns') + importer = MediaShoutImport(MagicMock(), filename='mediashout.db') + + # WHEN: A song is processed + with patch.object(importer, 'set_defaults') as mocked_set_defaults, \ + patch.object(importer, 'parse_author') as mocked_parse_author, \ + patch.object(importer, 'add_copyright') as mocked_add_copyright, \ + patch.object(importer, 'add_verse') as mocked_add_verse, \ + patch.object(importer, 'finish') as mocked_finish: + importer.topics = [] + importer.verse_order_list = [] + importer.process_song(song, [verse], [play_order], [theme, group]) + + # THEN: It should be added to the database + mocked_set_defaults.assert_called_once_with() + self.assertEqual('Amazing Grace', importer.title) + mocked_parse_author.assert_called_once_with('William Wilberforce') + mocked_add_copyright.assert_called_once_with('Public Domain') + self.assertEqual('Great old hymn', importer.comments) + self.assertEqual(['Grace', 'Hymns'], importer.topics) + self.assertEqual('Hymns', importer.song_book_name) + self.assertEqual('', importer.song_number) + mocked_add_verse.assert_called_once_with( + 'Amazing grace, how sweet the sound\nThat saved a wretch like me', 'V1') + self.assertEqual(['V1'], importer.verse_order_list) + mocked_finish.assert_called_once_with() + + def test_process_song_with_song_number(self): + """ + Test the process_song method with a song that has a song number + """ + # GIVEN: An importer and a song + SongRecord = namedtuple('SongRecord', 'Record, Title, Author, Copyright, SongID, CCLI, Notes') + VerseRecord = namedtuple('VerseRecord', 'Type, Number, Text') + PlayOrderRecord = namedtuple('PlayOrderRecord', 'Type, Number, POrder') + ThemeRecord = namedtuple('ThemeRecord', 'Name') + GroupRecord = namedtuple('GroupRecord', 'Name') + song = SongRecord(1, 'Amazing Grace', 'William Wilberforce', 'Public Domain', 'Hymns-2', '654321', + 'Great old hymn') + verse = VerseRecord(0, 1, 'Amazing grace, how sweet the sound\nThat saved a wretch like me') + play_order = PlayOrderRecord(0, 1, 1) + theme = ThemeRecord('Grace') + group = GroupRecord('Hymns') + importer = MediaShoutImport(MagicMock(), filename='mediashout.db') + + # WHEN: A song is processed + with patch.object(importer, 'set_defaults') as mocked_set_defaults, \ + patch.object(importer, 'parse_author') as mocked_parse_author, \ + patch.object(importer, 'add_copyright') as mocked_add_copyright, \ + patch.object(importer, 'add_verse') as mocked_add_verse, \ + patch.object(importer, 'finish') as mocked_finish: + importer.topics = [] + importer.verse_order_list = [] + importer.process_song(song, [verse], [play_order], [theme, group]) + + # THEN: It should be added to the database + mocked_set_defaults.assert_called_once_with() + self.assertEqual('Amazing Grace', importer.title) + mocked_parse_author.assert_called_once_with('William Wilberforce') + mocked_add_copyright.assert_called_once_with('Public Domain') + self.assertEqual('Great old hymn', importer.comments) + self.assertEqual(['Grace', 'Hymns'], importer.topics) + self.assertEqual('Hymns', importer.song_book_name) + self.assertEqual('2', importer.song_number) + mocked_add_verse.assert_called_once_with( + 'Amazing grace, how sweet the sound\nThat saved a wretch like me', 'V1') + self.assertEqual(['V1'], importer.verse_order_list) + mocked_finish.assert_called_once_with() + From b657d255cc060bbf1bdabf25f80b76ab475a8be6 Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Sun, 14 Aug 2016 22:58:27 +0200 Subject: [PATCH 15/65] Remove blank line Fixes: https://launchpad.net/bugs/1547964 --- tests/functional/openlp_plugins/songs/test_mediashout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/openlp_plugins/songs/test_mediashout.py b/tests/functional/openlp_plugins/songs/test_mediashout.py index ae575ae1e..4f03c4f94 100644 --- a/tests/functional/openlp_plugins/songs/test_mediashout.py +++ b/tests/functional/openlp_plugins/songs/test_mediashout.py @@ -221,4 +221,3 @@ class TestMediaShoutImport(TestCase): 'Amazing grace, how sweet the sound\nThat saved a wretch like me', 'V1') self.assertEqual(['V1'], importer.verse_order_list) mocked_finish.assert_called_once_with() - From 35758b306a90fca04ca4fb9c49977ecadbfcbbca Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Mon, 15 Aug 2016 00:01:48 +0300 Subject: [PATCH 16/65] - Restored the default Help shortcut. --- openlp/core/common/settings.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openlp/core/common/settings.py b/openlp/core/common/settings.py index 35d2dfad7..1f727c290 100644 --- a/openlp/core/common/settings.py +++ b/openlp/core/common/settings.py @@ -215,8 +215,9 @@ class Settings(QtCore.QSettings): ('media/players_temp', 'media/players', []), # Move temp setting from above to correct setting ('advanced/default color', 'core/logo background color', []), # Default image renamed + moved to general > 2.4. ('advanced/default image', 'core/logo file', []), # Default image renamed + moved to general after 2.4. - ('shortcuts/offlineHelpItem', 'shortcuts/HelpItem', []), # There used to be separated buttons for local and - ('shortcuts/onlineHelpItem', 'shortcuts/HelpItem', []) # online help buttons. Now combined into one since 2.6. + ('shortcuts/escapeItem', 'shortcuts/desktopScreen', []), # Default image renamed + moved to general after 2.4. + ('shortcuts/offlineHelpItem', 'shortcuts/HelpItem', []), # Online and Offline help were combined in 2.6. + ('shortcuts/onlineHelpItem', 'shortcuts/HelpItem', []) # Online and Offline help were combined in 2.6. ] @staticmethod @@ -256,7 +257,7 @@ class Settings(QtCore.QSettings): QtCore.QSettings.__init__(self, *args) # Add shortcuts here so QKeySequence has a QApplication instance to use. Settings.__default_settings__.update({ - 'shortcuts/aboutItem': [QtGui.QKeySequence(QtCore.Qt.SHIFT + QtCore.Qt.Key_F1)], + 'shortcuts/aboutItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], 'shortcuts/addToService': [], 'shortcuts/audioPauseItem': [], 'shortcuts/displayTagItem': [], @@ -276,7 +277,7 @@ class Settings(QtCore.QSettings): 'shortcuts/fileSaveItem': [QtGui.QKeySequence(QtGui.QKeySequence.Save)], 'shortcuts/fileOpenItem': [QtGui.QKeySequence(QtGui.QKeySequence.Open)], 'shortcuts/goLive': [], - 'shortcuts/HelpItem': [QtGui.QKeySequence(QtCore.Qt.CTRL + QtCore.Qt.Key_F1)], + 'shortcuts/HelpItem': [QtGui.QKeySequence(QtGui.QKeySequence.HelpContents)], 'shortcuts/importThemeItem': [], 'shortcuts/importBibleItem': [], 'shortcuts/listViewBiblesDeleteItem': [QtGui.QKeySequence(QtGui.QKeySequence.Delete)], From 33aaf53c421900d861d6a89b9e39f54f498d153f Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Mon, 15 Aug 2016 00:14:35 +0300 Subject: [PATCH 17/65] - Pep8 (Over ident fixed) --- openlp/core/ui/mainwindow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/core/ui/mainwindow.py b/openlp/core/ui/mainwindow.py index 70bd7e440..b8bd126dd 100644 --- a/openlp/core/ui/mainwindow.py +++ b/openlp/core/ui/mainwindow.py @@ -313,9 +313,9 @@ class Ui_MainWindow(object): self.local_help_file = os.path.join(AppLocation.get_directory(AppLocation.AppDir), '..', 'Resources', 'OpenLP.help') self.on_help_item = create_action(main_window, 'HelpItem', - icon=':/system/system_help_contents.png', - can_shortcuts=True, - category=UiStrings().Help, triggers=self.on_help_clicked) + icon=':/system/system_help_contents.png', + can_shortcuts=True, + category=UiStrings().Help, triggers=self.on_help_clicked) self.web_site_item = create_action(main_window, 'webSiteItem', can_shortcuts=True, category=UiStrings().Help) # Shortcuts not connected to buttons or menu entries. self.search_shortcut_action = create_action(main_window, From 4f900dbffbf0f4ac9abd64eaf979aca9c09bb173 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Mon, 15 Aug 2016 19:37:53 +0300 Subject: [PATCH 18/65] - Removed for_display=True which was causing traceback since it's apparently no longer used. - Added manual row split for the duplicated key error message. (Looks bad to have 12 words in one row and then one word in the row after that) --- openlp/core/ui/shortcutlistform.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openlp/core/ui/shortcutlistform.py b/openlp/core/ui/shortcutlistform.py index e23fa0a27..eb91313c9 100644 --- a/openlp/core/ui/shortcutlistform.py +++ b/openlp/core/ui/shortcutlistform.py @@ -426,12 +426,11 @@ class ShortcutListForm(QtWidgets.QDialog, Ui_ShortcutListDialog, RegistryPropert is_valid = False if not is_valid: text = translate('OpenLP.ShortcutListDialog', - 'The shortcut "{key}" is already assigned to another action, please' - ' use a different shortcut.' + 'The shortcut "{key}" is already assigned to another action,\n' + 'please use a different shortcut.' ).format(key=self.get_shortcut_string(key_sequence)) self.main_window.warning_message(translate('OpenLP.ShortcutListDialog', 'Duplicate Shortcut'), text) - for_display = True self.dialog_was_shown = True return is_valid From 6ab2686b0971989e3d79e356f19c18065b5c2c79 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Tue, 16 Aug 2016 21:36:21 +0100 Subject: [PATCH 19/65] Modify CSV Importer and test to give 100% coverage! --- .../plugins/bibles/lib/importers/csvbible.py | 2 +- .../openlp_plugins/bibles/test_bibleimport.py | 31 ------------------- .../openlp_plugins/bibles/test_csvimport.py | 2 +- 3 files changed, 2 insertions(+), 33 deletions(-) diff --git a/openlp/plugins/bibles/lib/importers/csvbible.py b/openlp/plugins/bibles/lib/importers/csvbible.py index 549cec581..3733145b6 100644 --- a/openlp/plugins/bibles/lib/importers/csvbible.py +++ b/openlp/plugins/bibles/lib/importers/csvbible.py @@ -142,7 +142,7 @@ class CSVBible(BibleImport): book_ptr = None for verse in verses: if self.stop_import_flag: - return None + break verse_book = self.get_book_name(verse.book_id_name, books) if book_ptr != verse_book: book = self.get_book(verse_book) diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index 127c6fd16..37c6c3fda 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -83,37 +83,6 @@ class TestBibleImport(TestCase): self.assertEqual(instance.filename, 'bible.xml') self.assertIsInstance(instance, BibleDB) - def check_for_compression_test(self): - """ - Test the check_for_compression method when called with a path to an uncompressed file - """ - # GIVEN: A mocked is_zipfile which returns False and an instance of BibleImport - with patch('openlp.plugins.bibles.lib.bibleimport.is_zipfile', return_value=False) as mocked_is_zip: - instance = BibleImport(MagicMock()) - - # WHEN: Calling check_for_compression - result = instance.check_for_compression('filename.tst') - - # THEN: None should be returned - self.assertIsNone(result) - mocked_is_zip.assert_called_once_with('filename.tst') - - def check_for_compression_zip_file_test(self): - """ - Test the check_for_compression method when called with a path to a compressed file - """ - # GIVEN: A patched is_zipfile which returns True and an instance of BibleImport - with patch('openlp.plugins.bibles.lib.bibleimport.is_zipfile', return_value=True),\ - patch('openlp.plugins.bibles.lib.bibleimport.critical_error_message_box') as mocked_message_box: - instance = BibleImport(MagicMock()) - - # WHEN: Calling check_for_compression - # THEN: A Validation error should be raised and the user should be notified. - with self.assertRaises(ValidationError) as context: - instance.check_for_compression('filename.tst') - self.assertTrue(mocked_message_box.called) - self.assertEqual(context.exception.msg, '"filename.tst" is compressed') - def get_language_id_language_found_test(self): """ Test get_language_id() when called with a name found in the languages list diff --git a/tests/functional/openlp_plugins/bibles/test_csvimport.py b/tests/functional/openlp_plugins/bibles/test_csvimport.py index 7a56c1b85..f6d3697af 100644 --- a/tests/functional/openlp_plugins/bibles/test_csvimport.py +++ b/tests/functional/openlp_plugins/bibles/test_csvimport.py @@ -240,7 +240,7 @@ class TestCSVImport(TestCase): importer.wizard = MagicMock() # WHEN: Calling process_verses - result = importer.process_verses([], []) + result = importer.process_verses(['Dummy Verse'], []) # THEN: get_book_name should not be called and the return value should be None self.assertFalse(importer.get_book_name.called) From 46b6d041cd50bea9f5dfea711be4f2168d7655fb Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Thu, 18 Aug 2016 07:31:36 +0100 Subject: [PATCH 20/65] Opensong refactors and tests --- openlp/plugins/bibles/lib/bibleimport.py | 19 +- .../plugins/bibles/lib/importers/opensong.py | 98 +++++++---- .../bibles/test_opensongimport.py | 164 +++++++++++++++--- 3 files changed, 220 insertions(+), 61 deletions(-) diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index 7ebdcb170..d6cfb83fa 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -23,9 +23,11 @@ import logging from lxml import etree, objectify +from zipfile import is_zipfile from openlp.core.common import OpenLPMixin, languages -from openlp.core.lib import ValidationError +from openlp.core.lib import ValidationError, translate +from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB log = logging.getLogger(__name__) @@ -39,6 +41,21 @@ class BibleImport(OpenLPMixin, BibleDB): super().__init__(*args, **kwargs) self.filename = kwargs['filename'] if 'filename' in kwargs else None + @staticmethod + def is_compressed(file): + """ + Check if the supplied file is compressed + + :param file: A path to the file to check + """ + if is_zipfile(file): + critical_error_message_box( + message=translate('BiblesPlugin.BibleImport', + 'The file "{file}" you supplied is compressed. You must decompress it before import.' + ).format(file=file)) + return True + return False + def get_language_id(self, file_language=None, bible_name=None): """ Get the language_id for the language of the bible. Fallback to user input if we cannot do this pragmatically. diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 10c0ed87e..66c127408 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -21,12 +21,12 @@ ############################################################################### import logging -from lxml import etree, objectify +from lxml import etree from openlp.core.common import translate, trace_error_handler +from openlp.core.lib.exceptions import ValidationError from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.bibleimport import BibleImport -from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB log = logging.getLogger(__name__) @@ -51,12 +51,69 @@ class OpenSongBible(BibleImport): verse_text += element.tail return verse_text + @staticmethod + def process_chapter_no(number, previous_number): + """ + Process the chapter number + + :param number: The raw data from the xml + :param previous_number: The previous chapter number + :return: Number of current chapter. (Int) + """ + if number: + return int(number.split()[-1]) + return previous_number + 1 + + @staticmethod + def process_verse_no(number, previous_number): + """ + Process the verse number retrieved from the xml + + :param number: The raw data from the xml + :param previous_number: The previous verse number + :return: Number of current verse. (Int) + """ + if not number: + return previous_number + 1 + try: + return int(number) + except ValueError: + verse_parts = number.split('-') + if len(verse_parts) > 1: + number = int(verse_parts[0]) + return number + except TypeError: + log.warning('Illegal verse number: {verse_no}'.format(verse_no=str(number))) + return previous_number + 1 + + @staticmethod + def validate_file(filename): + """ + Validate the supplied file + + :param filename: The supplied file + :return: True if valid. ValidationError is raised otherwise. + """ + if BibleImport.is_compressed(): + raise ValidationError(msg='Compressed file') + bible = BibleImport.parse_xml(filename, use_objectify=True) + root_tag = bible.tag.lower() + if root_tag != 'bible': + if root_tag == 'xmlbible': + # Zefania bibles have a root tag of XMLBIBLE". Sometimes these bibles are referred to as 'OpenSong' + critical_error_message_box( + message=translate('BiblesPlugin.OpenSongImport', + 'Incorrect Bible file type supplied. This looks like a Zefania XML bible, ' + 'please use the Zefania import option.')) + raise ValidationError(msg='Invalid xml.') + return True + def do_import(self, bible_name=None): """ Loads a Bible from file. """ + self.validate_file(self.filename) log.debug('Starting OpenSong import from "{name}"'.format(name=self.filename)) - success = True try: bible = self.parse_xml(self.filename, use_objectify=True) # Check that we're not trying to import a Zefania XML bible, it is sometimes refered to as 'OpenSong' @@ -78,46 +135,21 @@ class OpenSongBible(BibleImport): for chapter in book.c: if self.stop_import_flag: break - number = chapter.attrib['n'] - if number: - chapter_number = int(number.split()[-1]) - else: - chapter_number += 1 + chapter_number = self.process_chapter_no(chapter.attrib['n'], chapter_number) verse_number = 0 for verse in chapter.v: if self.stop_import_flag: break - number = verse.attrib['n'] - if number: - try: - number = int(number) - except ValueError: - verse_parts = number.split('-') - if len(verse_parts) > 1: - number = int(verse_parts[0]) - except TypeError: - log.warning('Illegal verse number: {verse:d}'.format(verse=verse.attrib['n'])) - verse_number = number - else: - verse_number += 1 + verse_number = self.process_verse_no(verse.attrib['n'], verse_number) self.create_verse(db_book.id, chapter_number, verse_number, self.get_text(verse)) self.wizard.increment_progress_bar(translate('BiblesPlugin.Opensong', 'Importing {name} {chapter}...' ).format(name=db_book.name, chapter=chapter_number)) self.session.commit() self.application.process_events() - except etree.XMLSyntaxError as inst: - trace_error_handler(log) - critical_error_message_box( - message=translate('BiblesPlugin.OpenSongImport', - 'Incorrect Bible file type supplied. OpenSong Bibles may be ' - 'compressed. You must decompress them before import.')) - log.exception(inst) - success = False - except (IOError, AttributeError): + except (AttributeError, ValidationError, etree.XMLSyntaxError): log.exception('Loading Bible from OpenSong file failed') - success = False + trace_error_handler(log) + return False if self.stop_import_flag: return False - else: - return success diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index d6997135b..af6215c45 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -27,9 +27,13 @@ import os import json from unittest import TestCase + +from lxml import objectify + from tests.functional import MagicMock, patch +from openlp.core.lib.exceptions import ValidationError from openlp.plugins.bibles.lib.importers.opensong import OpenSongBible -from openlp.plugins.bibles.lib.db import BibleDB +from openlp.plugins.bibles.lib.bibleimport import BibleImport TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'bibles')) @@ -41,14 +45,12 @@ class TestOpenSongImport(TestCase): """ def setUp(self): - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') - self.registry_patcher.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') + self.addCleanup(self.manager_patcher.stop) self.manager_patcher.start() - - def tearDown(self): - self.registry_patcher.stop() - self.manager_patcher.stop() + self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.addCleanup(self.registry_patcher.stop) + self.registry_patcher.start() def test_create_importer(self): """ @@ -61,7 +63,134 @@ class TestOpenSongImport(TestCase): importer = OpenSongBible(mocked_manager, path='.', name='.', filename='') # THEN: The importer should be an instance of BibleDB - self.assertIsInstance(importer, BibleDB) + self.assertIsInstance(importer, BibleImport) + + def process_chapter_no_test(self): + """ + Test process_chapter_no when supplied with chapter number and an instance of OpenSongBible + """ + # GIVEN: The number 10 represented as a string + # WHEN: Calling process_chapter_no + result = OpenSongBible.process_chapter_no('10', 0) + + # THEN: The 10 should be returned as an Int + self.assertEqual(result, 10) + + def process_chapter_no_empty_attribute_test(self): + """ + Test process_chapter_no when the chapter number is an empty string. (Bug #1074727) + """ + # GIVEN: An empty string, and the previous chapter number set as 12 and an instance of OpenSongBible + # WHEN: Calling process_chapter_no + result = OpenSongBible.process_chapter_no('', 12) + + # THEN: process_chapter_no should increment the previous verse number + self.assertEqual(result, 13) + + def process_verse_no_valid_verse_no_test(self): + """ + Test process_verse_no when supplied with a valid verse number + """ + # GIVEN: The number 15 represented as a string and an instance of OpenSongBible + # WHEN: Calling process_verse_no + result = OpenSongBible.process_verse_no('15', 0) + + # THEN: process_verse_no should return the verse number + self.assertEqual(result, 15) + + def process_verse_no_verse_range_test(self): + """ + Test process_verse_no when supplied with a verse range + """ + # GIVEN: The range 24-26 represented as a string + # WHEN: Calling process_verse_no + result = OpenSongBible.process_verse_no('24-26', 0) + + # THEN: process_verse_no should return the first verse number in the range + self.assertEqual(result, 24) + + def process_verse_no_invalid_verse_no_test(self): + """ + Test process_verse_no when supplied with a invalid verse number + """ + # GIVEN: An non numeric string represented as a string + # WHEN: Calling process_verse_no + result = OpenSongBible.process_verse_no('invalid', 41) + + # THEN: process_verse_no should increment the previous verse number + self.assertEqual(result, 42) + + def process_verse_no_empty_attribute_test(self): + """ + Test process_verse_no when the verse number is an empty string. (Bug #1074727) + """ + # GIVEN: An empty string, and the previous verse number set as 14 + # WHEN: Calling process_verse_no + result = OpenSongBible.process_verse_no('', 14) + + # THEN: process_verse_no should increment the previous verse number + self.assertEqual(result, 15) + + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + def process_verse_no_invalid_type_test(self, mocked_log): + """ + Test process_verse_no when the verse number is an invalid type) + """ + # GIVEN: A mocked out log, a Tuple, and the previous verse number set as 12 + # WHEN: Calling process_verse_no + result = OpenSongBible.process_verse_no((1,2,3), 12) + + # THEN: process_verse_no should log the verse number it was called with increment the previous verse number + mocked_log.warning.assert_called_once_with('Illegal verse number: (1, 2, 3)') + self.assertEqual(result, 13) + + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport') + def validate_xml_bible_test(self, mocked_bible_import): + """ + Test that validate_xml returns True with valid XML + """ + # GIVEN: Some test data with an OpenSong Bible "bible" root tag + mocked_bible_import.parse_xml.return_value = objectify.fromstring('') + + # WHEN: Calling validate_xml + result = OpenSongBible.validate_file('file.name') + + # THEN: A True should be returned + self.assertTrue(result) + + @patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport') + def validate_xml_zefania_root_test(self, mocked_bible_import, mocked_message_box): + """ + Test that validate_xml raises a ValidationError with a Zefinia root tag + """ + # GIVEN: Some test data with a Zefinia "XMLBIBLE" root tag + mocked_bible_import.parse_xml.return_value = objectify.fromstring('') + + # WHEN: Calling validate_xml + # THEN: critical_error_message_box should be called and an ValidationError should be raised + with self.assertRaises(ValidationError) as context: + OpenSongBible.validate_file('file.name') + self.assertEqual(context.exception.msg, 'Invalid xml.') + mocked_message_box.assert_called_once_with( + message='Incorrect Bible file type supplied. This looks like a Zefania XML bible, please use the ' + 'Zefania import option.') + + @patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport') + def validate_xml_invalid_root_test(self, mocked_bible_import, mocked_message_box): + """ + Test that validate_xml raises a ValidationError with an invalid root tag + """ + # GIVEN: Some test data with an invalid root tag and an instance of OpenSongBible + mocked_bible_import.parse_xml.return_value = objectify.fromstring('') + + # WHEN: Calling validate_xml + # THEN: ValidationError should be raised, and the critical error message box should not have been called + with self.assertRaises(ValidationError) as context: + OpenSongBible.validate_file('file.name') + self.assertEqual(context.exception.msg, 'Invalid xml.') + self.assertFalse(mocked_message_box.called) def test_file_import(self): """ @@ -92,22 +221,3 @@ class TestOpenSongImport(TestCase): self.assertTrue(importer.create_verse.called) for verse_tag, verse_text in test_data['verses']: importer.create_verse.assert_any_call(importer.create_book().id, 1, int(verse_tag), verse_text) - - def test_zefania_import_error(self): - """ - Test that we give an error message if trying to import a zefania bible - """ - # GIVEN: A mocked out "manager" and mocked out critical_error_message_box and an import - with patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') as \ - mocked_critical_error_message_box: - mocked_manager = MagicMock() - importer = OpenSongBible(mocked_manager, path='.', name='.', filename='') - - # WHEN: An trying to import a zefania bible - importer.filename = os.path.join(TEST_PATH, 'zefania-dk1933.xml') - importer.do_import() - - # THEN: The importer should have "shown" an error message - mocked_critical_error_message_box.assert_called_with(message='Incorrect Bible file type supplied. ' - 'This looks like a Zefania XML bible, ' - 'please use the Zefania import option.') From 345978286be1450b0958f64a3cc549d57aa550e1 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 20 Aug 2016 01:49:51 +0300 Subject: [PATCH 21/65] - Added a test. --- .../openlp_plugins/bibles/test_mediaitem.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 05418f177..1d88aa6fb 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -150,3 +150,23 @@ class TestMediaItem(TestCase, TestMixin): self.assertEqual(2, self.media_item.quickSearchButton.setEnabled.call_count, 'Disable and Enable the button') self.assertEqual(1, self.media_item.check_search_result.call_count, 'Check results Should had been called once') self.assertEqual(1, self.app.set_normal_cursor.call_count, 'Normal cursor should had been called once') + + def on_clear_button_clicked_test(self): + """ + Test that the on_clear_button_clicked works properly. (Used by Bible search tab) + """ + + # GIVEN: Mocked list_view, check_search_results & quick_search_edit. + self.media_item.list_view = MagicMock() + self.media_item.check_search_result = MagicMock() + self.media_item.quick_search_edit = MagicMock() + + + # WHEN: on_clear_button_clicked is called + self.media_item.on_clear_button_clicked() + + # THEN: Search result should be reset and search field should receive focus. + self.assertEqual(1, self.media_item.list_view.clear.call_count, 'List_view.clear should had been called once.') + self.assertEqual(1, self.media_item.check_search_result.call_count, 'Check results Should had been called once') + self.assertEqual(1, self.media_item.quick_search_edit.clear.call_count, 'Should had been called once') + self.assertEqual(1, self.media_item.quick_search_edit.setFocus.call_count, 'Should had been called once') From b5c4cb1f851eb92d98ac43fd9c8d97c093653978 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 20 Aug 2016 01:53:20 +0300 Subject: [PATCH 22/65] - pep8 fix (2x blank line) --- tests/functional/openlp_plugins/bibles/test_mediaitem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 1d88aa6fb..606c65a63 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -161,7 +161,6 @@ class TestMediaItem(TestCase, TestMixin): self.media_item.check_search_result = MagicMock() self.media_item.quick_search_edit = MagicMock() - # WHEN: on_clear_button_clicked is called self.media_item.on_clear_button_clicked() From 890b617fc77306269727df22cc6e939e2ad5809e Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 20 Aug 2016 02:42:48 +0300 Subject: [PATCH 23/65] - Fixed the same issue for the lock button. --- openlp/plugins/bibles/lib/mediaitem.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/bibles/lib/mediaitem.py b/openlp/plugins/bibles/lib/mediaitem.py index 2b35c9115..979eb5e89 100644 --- a/openlp/plugins/bibles/lib/mediaitem.py +++ b/openlp/plugins/bibles/lib/mediaitem.py @@ -556,16 +556,25 @@ class BibleMediaItem(MediaManagerItem): self.quick_search_edit.setFocus() def on_advanced_clear_button_clicked(self): - # The same as the on_clear_button_clicked, but does not give focus to Quick search field. + # The same as the on_clear_button_clicked, but gives focus to Book name field in "Select" (advanced). self.list_view.clear() self.check_search_result() + self.advanced_book_combo_box.setFocus() def on_lock_button_toggled(self, checked): - self.quick_search_edit.setFocus() + """ + Toggle the lock button, if Search tab is used, set focus to search field, if Select tab is used, + give focus to Bible book name field. + """ if checked: self.sender().setIcon(self.lock_icon) else: self.sender().setIcon(self.unlock_icon) + if self.quickTab.isVisible(): + self.quick_search_edit.setFocus() + else: + self.advanced_book_combo_box.setFocus() + def on_quick_style_combo_box_changed(self): self.settings.layout_style = self.quickStyleComboBox.currentIndex() From cf65d6921e5514d378abe703b1fa50134cde4c21 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 20 Aug 2016 02:46:20 +0300 Subject: [PATCH 24/65] - Pep8 --- openlp/plugins/bibles/lib/mediaitem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openlp/plugins/bibles/lib/mediaitem.py b/openlp/plugins/bibles/lib/mediaitem.py index 979eb5e89..e20abe797 100644 --- a/openlp/plugins/bibles/lib/mediaitem.py +++ b/openlp/plugins/bibles/lib/mediaitem.py @@ -575,7 +575,6 @@ class BibleMediaItem(MediaManagerItem): else: self.advanced_book_combo_box.setFocus() - def on_quick_style_combo_box_changed(self): self.settings.layout_style = self.quickStyleComboBox.currentIndex() self.advancedStyleComboBox.setCurrentIndex(self.settings.layout_style) From 7c77d7e8bd0c6b73df164a3d772d7faf4ef1dbe9 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sat, 20 Aug 2016 19:12:42 +0100 Subject: [PATCH 25/65] Refactor of OpenSong Bible importer + 100% test coverage --- .../plugins/bibles/lib/importers/opensong.py | 77 ++-- .../openlp_core_ui/test_exceptionform.py | 7 +- .../bibles/test_opensongimport.py | 431 +++++++++++++++--- 3 files changed, 414 insertions(+), 101 deletions(-) diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 66c127408..c0a82a4ff 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -36,7 +36,8 @@ class OpenSongBible(BibleImport): """ OpenSong Bible format importer class. This class is used to import Bibles from OpenSong's XML format. """ - def get_text(self, element): + @staticmethod + def get_text(element): """ Recursively get all text in an objectify element and its child elements. @@ -46,15 +47,15 @@ class OpenSongBible(BibleImport): if element.text: verse_text = element.text for sub_element in element.iterchildren(): - verse_text += self.get_text(sub_element) + verse_text += OpenSongBible.get_text(sub_element) if element.tail: verse_text += element.tail return verse_text @staticmethod - def process_chapter_no(number, previous_number): + def parse_chapter_number(number, previous_number): """ - Process the chapter number + Parse the chapter number :param number: The raw data from the xml :param previous_number: The previous chapter number @@ -65,9 +66,9 @@ class OpenSongBible(BibleImport): return previous_number + 1 @staticmethod - def process_verse_no(number, previous_number): + def parse_verse_number(number, previous_number): """ - Process the verse number retrieved from the xml + Parse the verse number retrieved from the xml :param number: The raw data from the xml :param previous_number: The previous verse number @@ -94,7 +95,7 @@ class OpenSongBible(BibleImport): :param filename: The supplied file :return: True if valid. ValidationError is raised otherwise. """ - if BibleImport.is_compressed(): + if BibleImport.is_compressed(filename): raise ValidationError(msg='Compressed file') bible = BibleImport.parse_xml(filename, use_objectify=True) root_tag = bible.tag.lower() @@ -108,44 +109,47 @@ class OpenSongBible(BibleImport): raise ValidationError(msg='Invalid xml.') return True + def process_books(self, books): + for book in books: + if self.stop_import_flag: + break + db_book = self.find_and_create_book(str(book.attrib['n']), len(books), self.language_id) + self.process_chapters(db_book, book.c) + self.session.commit() + + def process_chapters(self, book, chapters): + chapter_number = 0 + for chapter in chapters: + if self.stop_import_flag: + break + chapter_number = self.parse_chapter_number(chapter.attrib['n'], chapter_number) + self.process_verses(book, chapter_number, chapter.v) + self.wizard.increment_progress_bar(translate('BiblesPlugin.Opensong', + 'Importing {name} {chapter}...' + ).format(name=book.name, chapter=chapter_number)) + + def process_verses(self, book, chapter_number, verses): + verse_number = 0 + for verse in verses: + if self.stop_import_flag: + break + verse_number = self.parse_verse_number(verse.attrib['n'], verse_number) + self.create_verse(book.id, chapter_number, verse_number, self.get_text(verse)) + def do_import(self, bible_name=None): """ - Loads a Bible from file. + Loads an Open Song Bible from a file. """ - self.validate_file(self.filename) log.debug('Starting OpenSong import from "{name}"'.format(name=self.filename)) try: + self.validate_file(self.filename) bible = self.parse_xml(self.filename, use_objectify=True) # Check that we're not trying to import a Zefania XML bible, it is sometimes refered to as 'OpenSong' - if bible.tag.upper() == 'XMLBIBLE': - critical_error_message_box( - message=translate('BiblesPlugin.OpenSongImport', - 'Incorrect Bible file type supplied. This looks like a Zefania XML bible, ' - 'please use the Zefania import option.')) - return False # No language info in the opensong format, so ask the user - language_id = self.get_language_id(bible_name=self.filename) - if not language_id: + self.language_id = self.get_language_id(bible_name=self.filename) + if not self.language_id: return False - for book in bible.b: - if self.stop_import_flag: - break - db_book = self.find_and_create_book(str(book.attrib['n']), len(bible.b), language_id) - chapter_number = 0 - for chapter in book.c: - if self.stop_import_flag: - break - chapter_number = self.process_chapter_no(chapter.attrib['n'], chapter_number) - verse_number = 0 - for verse in chapter.v: - if self.stop_import_flag: - break - verse_number = self.process_verse_no(verse.attrib['n'], verse_number) - self.create_verse(db_book.id, chapter_number, verse_number, self.get_text(verse)) - self.wizard.increment_progress_bar(translate('BiblesPlugin.Opensong', - 'Importing {name} {chapter}...' - ).format(name=db_book.name, chapter=chapter_number)) - self.session.commit() + self.process_books(bible.b) self.application.process_events() except (AttributeError, ValidationError, etree.XMLSyntaxError): log.exception('Loading Bible from OpenSong file failed') @@ -153,3 +157,4 @@ class OpenSongBible(BibleImport): return False if self.stop_import_flag: return False + return True diff --git a/tests/functional/openlp_core_ui/test_exceptionform.py b/tests/functional/openlp_core_ui/test_exceptionform.py index 452a8dee9..493b2baeb 100644 --- a/tests/functional/openlp_core_ui/test_exceptionform.py +++ b/tests/functional/openlp_core_ui/test_exceptionform.py @@ -24,18 +24,13 @@ Package to test the openlp.core.ui.exeptionform package. """ import os -import socket import tempfile -import urllib from unittest import TestCase from unittest.mock import mock_open -from PyQt5.QtCore import QUrlQuery - from openlp.core.common import Registry -from openlp.core.ui.firsttimeform import FirstTimeForm -from tests.functional import MagicMock, patch +from tests.functional import patch from tests.helpers.testmixin import TestMixin from openlp.core.ui import exceptionform diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index af6215c45..ee4e794c0 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -23,14 +23,15 @@ This module contains tests for the OpenSong Bible importer. """ -import os import json +import os from unittest import TestCase +from lxml import etree, objectify -from lxml import objectify - -from tests.functional import MagicMock, patch +from tests.functional import MagicMock, patch, call +from tests.helpers.testmixin import TestMixin +from openlp.core.common import Registry from openlp.core.lib.exceptions import ValidationError from openlp.plugins.bibles.lib.importers.opensong import OpenSongBible from openlp.plugins.bibles.lib.bibleimport import BibleImport @@ -39,7 +40,7 @@ TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'bibles')) -class TestOpenSongImport(TestCase): +class TestOpenSongImport(TestCase, TestMixin): """ Test the functions in the :mod:`opensongimport` module. """ @@ -48,9 +49,10 @@ class TestOpenSongImport(TestCase): self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') self.addCleanup(self.manager_patcher.stop) self.manager_patcher.start() - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') - self.addCleanup(self.registry_patcher.stop) - self.registry_patcher.start() + self.setup_application() + self.app.process_events = MagicMock() + Registry.create() + Registry().register('application', self.app) def test_create_importer(self): """ @@ -65,133 +67,444 @@ class TestOpenSongImport(TestCase): # THEN: The importer should be an instance of BibleDB self.assertIsInstance(importer, BibleImport) - def process_chapter_no_test(self): + def get_text_no_text_test(self): """ - Test process_chapter_no when supplied with chapter number and an instance of OpenSongBible + Test that get_text handles elements containing text in a combination of text and tail attributes + """ + # GIVEN: Some test data which contains an empty element and an instance of OpenSongBible + test_data = objectify.fromstring('') + + # WHEN: Calling get_text + result = OpenSongBible.get_text(test_data) + + # THEN: A blank string should be returned + self.assertEqual(result, '') + + def get_text_text_test(self): + """ + Test that get_text handles elements containing text in a combination of text and tail attributes + """ + # GIVEN: Some test data which contains all possible permutation of text and tail text possible and an instance + # of OpenSongBible + test_data = objectify.fromstring('Element text ' + 'sub_text_tail text sub_text_tail tail ' + 'sub_text text ' + 'sub_tail tail') + + # WHEN: Calling get_text + result = OpenSongBible.get_text(test_data) + + # THEN: The text returned should be as expected + self.assertEqual(result, 'Element text sub_text_tail text sub_text_tail tail sub_text text sub_tail tail') + + def parse_chapter_number_test(self): + """ + Test parse_chapter_number when supplied with chapter number and an instance of OpenSongBible """ # GIVEN: The number 10 represented as a string - # WHEN: Calling process_chapter_no - result = OpenSongBible.process_chapter_no('10', 0) + # WHEN: Calling parse_chapter_nnumber + result = OpenSongBible.parse_chapter_number('10', 0) # THEN: The 10 should be returned as an Int self.assertEqual(result, 10) - def process_chapter_no_empty_attribute_test(self): + def parse_chapter_number_empty_attribute_test(self): """ - Test process_chapter_no when the chapter number is an empty string. (Bug #1074727) + Testparse_chapter_number when the chapter number is an empty string. (Bug #1074727) """ # GIVEN: An empty string, and the previous chapter number set as 12 and an instance of OpenSongBible - # WHEN: Calling process_chapter_no - result = OpenSongBible.process_chapter_no('', 12) + # WHEN: Calling parse_chapter_number + result = OpenSongBible.parse_chapter_number('', 12) - # THEN: process_chapter_no should increment the previous verse number + # THEN: parse_chapter_number should increment the previous verse number self.assertEqual(result, 13) - def process_verse_no_valid_verse_no_test(self): + def parse_verse_number_valid_verse_no_test(self): """ - Test process_verse_no when supplied with a valid verse number + Test parse_verse_number when supplied with a valid verse number """ # GIVEN: The number 15 represented as a string and an instance of OpenSongBible - # WHEN: Calling process_verse_no - result = OpenSongBible.process_verse_no('15', 0) + # WHEN: Calling parse_verse_number + result = OpenSongBible.parse_verse_number('15', 0) - # THEN: process_verse_no should return the verse number + # THEN: parse_verse_number should return the verse number self.assertEqual(result, 15) - def process_verse_no_verse_range_test(self): + def parse_verse_number_verse_range_test(self): """ - Test process_verse_no when supplied with a verse range + Test parse_verse_number when supplied with a verse range """ # GIVEN: The range 24-26 represented as a string - # WHEN: Calling process_verse_no - result = OpenSongBible.process_verse_no('24-26', 0) + # WHEN: Calling parse_verse_number + result = OpenSongBible.parse_verse_number('24-26', 0) - # THEN: process_verse_no should return the first verse number in the range + # THEN: parse_verse_number should return the first verse number in the range self.assertEqual(result, 24) - def process_verse_no_invalid_verse_no_test(self): + def parse_verse_number_invalid_verse_no_test(self): """ - Test process_verse_no when supplied with a invalid verse number + Test parse_verse_number when supplied with a invalid verse number """ # GIVEN: An non numeric string represented as a string - # WHEN: Calling process_verse_no - result = OpenSongBible.process_verse_no('invalid', 41) + # WHEN: Calling parse_verse_number + result = OpenSongBible.parse_verse_number('invalid', 41) - # THEN: process_verse_no should increment the previous verse number + # THEN: parse_verse_number should increment the previous verse number self.assertEqual(result, 42) - def process_verse_no_empty_attribute_test(self): + def parse_verse_number_empty_attribute_test(self): """ - Test process_verse_no when the verse number is an empty string. (Bug #1074727) + Test parse_verse_number when the verse number is an empty string. (Bug #1074727) """ # GIVEN: An empty string, and the previous verse number set as 14 - # WHEN: Calling process_verse_no - result = OpenSongBible.process_verse_no('', 14) + # WHEN: Calling parse_verse_number + result = OpenSongBible.parse_verse_number('', 14) - # THEN: process_verse_no should increment the previous verse number + # THEN: parse_verse_number should increment the previous verse number self.assertEqual(result, 15) @patch('openlp.plugins.bibles.lib.importers.opensong.log') - def process_verse_no_invalid_type_test(self, mocked_log): + def parse_verse_number_invalid_type_test(self, mocked_log): """ - Test process_verse_no when the verse number is an invalid type) + Test parse_verse_number when the verse number is an invalid type) """ # GIVEN: A mocked out log, a Tuple, and the previous verse number set as 12 - # WHEN: Calling process_verse_no - result = OpenSongBible.process_verse_no((1,2,3), 12) + # WHEN: Calling parse_verse_number + result = OpenSongBible.parse_verse_number((1, 2, 3), 12) - # THEN: process_verse_no should log the verse number it was called with increment the previous verse number + # THEN: parse_verse_number should log the verse number it was called with increment the previous verse number mocked_log.warning.assert_called_once_with('Illegal verse number: (1, 2, 3)') self.assertEqual(result, 13) - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport') - def validate_xml_bible_test(self, mocked_bible_import): + @patch('openlp.plugins.bibles.lib.bibleimport.BibleImport.find_and_create_book') + def process_books_stop_import_test(self, mocked_find_and_create_book): """ - Test that validate_xml returns True with valid XML + Test process_books when stop_import is set to True + """ + # GIVEN: An isntance of OpenSongBible + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: stop_import_flag is set to True + importer.stop_import_flag = True + importer.process_books(['Book']) + + # THEN: find_and_create_book should not have been called + self.assertFalse(mocked_find_and_create_book.called) + + @patch('openlp.plugins.bibles.lib.bibleimport.BibleImport.find_and_create_book', + **{'side_effect': ['db_book1', 'db_book2']}) + def process_books_completes_test(self, mocked_find_and_create_book): + """ + Test process_books when it processes all books + """ + # GIVEN: An instance of OpenSongBible Importer and two mocked books + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + + book1 = MagicMock() + book1.attrib = {'n': 'Name1'} + book1.c = 'Chapter1' + book2 = MagicMock() + book2.attrib = {'n': 'Name2'} + book2.c = 'Chapter2' + importer.language_id = 10 + importer.process_chapters = MagicMock() + importer.session = MagicMock() + importer.stop_import_flag = False + + # WHEN: Calling process_books with the two books + importer.process_books([book1, book2]) + + # THEN: find_and_create_book and process_books should be called with the details from the mocked books + self.assertEqual(mocked_find_and_create_book.call_args_list, [call('Name1', 2, 10), call('Name2', 2, 10)]) + self.assertEqual(importer.process_chapters.call_args_list, + [call('db_book1', 'Chapter1'), call('db_book2', 'Chapter2')]) + self.assertEqual(importer.session.commit.call_count, 2) + + def process_chapters_stop_import_test(self): + """ + Test process_chapters when stop_import is set to True + """ + # GIVEN: An isntance of OpenSongBible + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.parse_chapter_number = MagicMock() + + # WHEN: stop_import_flag is set to True + importer.stop_import_flag = True + importer.process_chapters('Book', ['Chapter1']) + + # THEN: importer.parse_chapter_number not have been called + self.assertFalse(importer.parse_chapter_number.called) + + @patch('openlp.plugins.bibles.lib.importers.opensong.translate', **{'side_effect': lambda x, y: y}) + def process_chapters_completes_test(self, mocked_translate): + """ + Test process_chapters when it completes + """ + # GIVEN: An instance of OpenSongBible + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.parse_chapter_number = MagicMock() + importer.parse_chapter_number.side_effect = [1, 2] + importer.wizard = MagicMock() + + # WHEN: called with some valid data + book = MagicMock() + book.name = "Book" + chapter1 = MagicMock() + chapter1.attrib = {'n': '1'} + chapter1.c = 'Chapter1' + chapter1.v = ['Chapter1 Verses'] + chapter2 = MagicMock() + chapter2.attrib = {'n': '2'} + chapter2.c = 'Chapter2' + chapter2.v = ['Chapter2 Verses'] + + importer.process_verses = MagicMock() + importer.stop_import_flag = False + importer.process_chapters(book, [chapter1, chapter2]) + + # THEN: parse_chapter_number, process_verses and increment_process_bar should have been called + self.assertEqual(importer.parse_chapter_number.call_args_list, [call('1', 0), call('2', 1)]) + self.assertEqual( + importer.process_verses.call_args_list, + [call(book, 1, ['Chapter1 Verses']), call(book, 2, ['Chapter2 Verses'])]) + self.assertEqual(importer.wizard.increment_progress_bar.call_args_list, + [call('Importing Book 1...'), call('Importing Book 2...')]) + + def process_verses_stop_import_test(self): + """ + Test process_verses when stop_import is set to True + """ + # GIVEN: An isntance of OpenSongBible + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.parse_verse_number = MagicMock() + + # WHEN: stop_import_flag is set to True + importer.stop_import_flag = True + importer.process_verses('Book', 1, 'Verses') + + # THEN: importer.parse_verse_number not have been called + self.assertFalse(importer.parse_verse_number.called) + + def process_verses_completes_test(self): + """ + Test process_verses when it completes + """ + # GIVEN: An instance of OpenSongBible + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.get_text = MagicMock() + importer.get_text.side_effect = ['Verse1 Text', 'Verse2 Text'] + importer.parse_verse_number = MagicMock() + importer.parse_verse_number.side_effect = [1, 2] + importer.wizard = MagicMock() + + # WHEN: called with some valid data + book = MagicMock() + book.id = 1 + verse1 = MagicMock() + verse1.attrib = {'n': '1'} + verse1.c = 'Chapter1' + verse1.v = ['Chapter1 Verses'] + verse2 = MagicMock() + verse2.attrib = {'n': '2'} + verse2.c = 'Chapter2' + verse2.v = ['Chapter2 Verses'] + + importer.create_verse = MagicMock() + importer.stop_import_flag = False + importer.process_verses(book, 1, [verse1, verse2]) + + # THEN: parse_chapter_number, process_verses and increment_process_bar should have been called + self.assertEqual(importer.parse_verse_number.call_args_list, [call('1', 0), call('2', 1)]) + self.assertEqual(importer.get_text.call_args_list, [call(verse1), call(verse2)]) + self.assertEqual( + importer.create_verse.call_args_list, + [call(1, 1, 1, 'Verse1 Text'), call(1, 1, 2, 'Verse2 Text')]) + + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed') + def validate_file_compressed_test(self, mocked_is_compressed): + """ + Test that validate_file raises a ValidationError when supplied with a compressed file + """ + # GIVEN: A mocked is_compressed method which returns True + mocked_is_compressed.return_value = True + + # WHEN: Calling validate_file + # THEN: ValidationError should be raised + with self.assertRaises(ValidationError) as context: + OpenSongBible.validate_file('file.name') + self.assertEqual(context.exception.msg, 'Compressed file') + + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.parse_xml') + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed', **{'return_value': False}) + def validate_file_bible_test(self, mocked_is_compressed, mocked_parse_xml): + """ + Test that validate_file returns True with valid XML """ # GIVEN: Some test data with an OpenSong Bible "bible" root tag - mocked_bible_import.parse_xml.return_value = objectify.fromstring('') + mocked_parse_xml.return_value = objectify.fromstring('') - # WHEN: Calling validate_xml + # WHEN: Calling validate_file result = OpenSongBible.validate_file('file.name') # THEN: A True should be returned self.assertTrue(result) @patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport') - def validate_xml_zefania_root_test(self, mocked_bible_import, mocked_message_box): + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.parse_xml') + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed', **{'return_value': False}) + def validate_file_zefania_root_test(self, mocked_is_compressed, mocked_parse_xml, mocked_message_box): """ - Test that validate_xml raises a ValidationError with a Zefinia root tag + Test that validate_file raises a ValidationError with a Zefinia root tag """ # GIVEN: Some test data with a Zefinia "XMLBIBLE" root tag - mocked_bible_import.parse_xml.return_value = objectify.fromstring('') + mocked_parse_xml.return_value = objectify.fromstring('') - # WHEN: Calling validate_xml + # WHEN: Calling validate_file # THEN: critical_error_message_box should be called and an ValidationError should be raised with self.assertRaises(ValidationError) as context: OpenSongBible.validate_file('file.name') - self.assertEqual(context.exception.msg, 'Invalid xml.') + self.assertEqual(context.exception.msg, 'Invalid xml.') mocked_message_box.assert_called_once_with( message='Incorrect Bible file type supplied. This looks like a Zefania XML bible, please use the ' 'Zefania import option.') @patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport') - def validate_xml_invalid_root_test(self, mocked_bible_import, mocked_message_box): + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.parse_xml') + @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed', **{'return_value': False}) + def validate_file_invalid_root_test(self, mocked_is_compressed, mocked_parse_xml, mocked_message_box): """ - Test that validate_xml raises a ValidationError with an invalid root tag + Test that validate_file raises a ValidationError with an invalid root tag """ # GIVEN: Some test data with an invalid root tag and an instance of OpenSongBible - mocked_bible_import.parse_xml.return_value = objectify.fromstring('') + mocked_parse_xml.return_value = objectify.fromstring('') - # WHEN: Calling validate_xml + # WHEN: Calling validate_file # THEN: ValidationError should be raised, and the critical error message box should not have been called with self.assertRaises(ValidationError) as context: OpenSongBible.validate_file('file.name') - self.assertEqual(context.exception.msg, 'Invalid xml.') + self.assertEqual(context.exception.msg, 'Invalid xml.') self.assertFalse(mocked_message_box.called) + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + @patch('openlp.plugins.bibles.lib.importers.opensong.trace_error_handler') + def do_import_attribute_error_test(self, mocked_trace_error_handler, mocked_log): + """ + Test do_import when an AttributeError exception is raised + """ + # GIVEN: An instance of OpenSongBible and a mocked validate_file which raises an AttributeError + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.validate_file = MagicMock(**{'side_effect': AttributeError()}) + importer.parse_xml = MagicMock() + + # WHEN: Calling do_import + result = importer.do_import() + + # THEN: do_import should return False after logging the exception + mocked_log.exception.assert_called_once_with('Loading Bible from OpenSong file failed') + mocked_trace_error_handler.assert_called_once_with(mocked_log) + self.assertFalse(result) + self.assertFalse(importer.parse_xml.called) + + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + @patch('openlp.plugins.bibles.lib.importers.opensong.trace_error_handler') + def do_import_validation_error_test(self, mocked_trace_error_handler, mocked_log): + """ + Test do_import when an ValidationError exception is raised + """ + # GIVEN: An instance of OpenSongBible and a mocked validate_file which raises an ValidationError + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.validate_file = MagicMock(**{'side_effect': ValidationError()}) + importer.parse_xml = MagicMock() + + # WHEN: Calling do_import + result = importer.do_import() + + # THEN: do_import should return False after logging the exception. parse_xml should not be called. + mocked_log.exception.assert_called_once_with('Loading Bible from OpenSong file failed') + mocked_trace_error_handler.assert_called_once_with(mocked_log) + self.assertFalse(result) + self.assertFalse(importer.parse_xml.called) + + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + @patch('openlp.plugins.bibles.lib.importers.opensong.trace_error_handler') + def do_import_xml_syntax_error_test(self, mocked_trace_error_handler, mocked_log): + """ + Test do_import when an etree.XMLSyntaxError exception is raised + """ + # GIVEN: An instance of OpenSongBible and a mocked validate_file which raises an etree.XMLSyntaxError + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.validate_file = MagicMock(**{'side_effect': etree.XMLSyntaxError(None, None, None, None)}) + importer.parse_xml = MagicMock() + + # WHEN: Calling do_import + result = importer.do_import() + + # THEN: do_import should return False after logging the exception. parse_xml should not be called. + mocked_log.exception.assert_called_once_with('Loading Bible from OpenSong file failed') + mocked_trace_error_handler.assert_called_once_with(mocked_log) + self.assertFalse(result) + self.assertFalse(importer.parse_xml.called) + + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + def do_import_no_language_test(self, mocked_log): + """ + Test do_import when the user cancels the language selection dialog + """ + # GIVEN: An instance of OpenSongBible and a mocked get_language which returns False + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.validate_file = MagicMock() + importer.parse_xml = MagicMock() + importer.get_language_id = MagicMock(**{'return_value': False}) + importer.process_books = MagicMock() + + # WHEN: Calling do_import + result = importer.do_import() + + # THEN: do_import should return False and process_books should have not been called + self.assertFalse(result) + self.assertFalse(importer.process_books.called) + + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + def do_import_stop_import_test(self, mocked_log): + """ + Test do_import when the stop_import_flag is set to True + """ + # GIVEN: An instance of OpenSongBible and stop_import_flag set to True + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.validate_file = MagicMock() + importer.parse_xml = MagicMock() + importer.get_language_id = MagicMock(**{'return_value': 10}) + importer.process_books = MagicMock() + importer.stop_import_flag = True + + # WHEN: Calling do_import + result = importer.do_import() + + # THEN: do_import should return False and process_books should have not been called + self.assertFalse(result) + self.assertTrue(importer.application.process_events.called) + + self.assertTrue(importer.application.process_events.called) + + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + def do_import_completes_test(self, mocked_log): + """ + Test do_import when it completes successfully + """ + # GIVEN: An instance of OpenSongBible and stop_import_flag set to True + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.validate_file = MagicMock() + importer.parse_xml = MagicMock() + importer.get_language_id = MagicMock(**{'return_value': 10}) + importer.process_books = MagicMock() + importer.stop_import_flag = False + + # WHEN: Calling do_import + result = importer.do_import() + + # THEN: do_import should return True + self.assertTrue(result) + def test_file_import(self): """ Test the actual import of OpenSong Bible file From 685c01ec25fb821bdccb2c006d6fb11cecccbdaa Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 20 Aug 2016 21:59:31 +0300 Subject: [PATCH 26/65] - Renamed test title from x_test to test_x --- tests/functional/openlp_plugins/bibles/test_mediaitem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 606c65a63..bacafe009 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -114,7 +114,7 @@ class TestMediaItem(TestCase, TestMixin): self.assertEqual(self.media_item.search_results, {}) self.assertEqual(self.media_item.second_search_results, {}) - def on_quick_search_button_general_test(self): + def test_on_quick_search_button_general(self): """ Test that general things, which should be called on all Quick searches are called. """ @@ -151,7 +151,7 @@ class TestMediaItem(TestCase, TestMixin): self.assertEqual(1, self.media_item.check_search_result.call_count, 'Check results Should had been called once') self.assertEqual(1, self.app.set_normal_cursor.call_count, 'Normal cursor should had been called once') - def on_clear_button_clicked_test(self): + def test_on_clear_button_clicked(self): """ Test that the on_clear_button_clicked works properly. (Used by Bible search tab) """ From 8224e39a2cf2f54c85d53ea257b7157ba55c2860 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 20 Aug 2016 22:14:15 +0300 Subject: [PATCH 27/65] - Added test --- .../openlp_core_ui/test_mainwindow.py | 3 ++- .../openlp_plugins/bibles/test_mediaitem.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/functional/openlp_core_ui/test_mainwindow.py b/tests/functional/openlp_core_ui/test_mainwindow.py index 341e74eb8..09c423304 100644 --- a/tests/functional/openlp_core_ui/test_mainwindow.py +++ b/tests/functional/openlp_core_ui/test_mainwindow.py @@ -26,11 +26,12 @@ import os from unittest import TestCase -from PyQt5 import QtWidgets +from PyQt5 import QtWidgets, QtGui, QtCore from openlp.core.ui.mainwindow import MainWindow from openlp.core.lib.ui import UiStrings from openlp.core.common.registry import Registry +from openlp.core.common import is_win, is_macosx, is_linux from tests.functional import MagicMock, patch from tests.helpers.testmixin import TestMixin diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 05418f177..f4b356d2c 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -114,6 +114,27 @@ class TestMediaItem(TestCase, TestMixin): self.assertEqual(self.media_item.search_results, {}) self.assertEqual(self.media_item.second_search_results, {}) + def test_required_icons(self): + """ + Test that all the required icons are set properly. + """ + # GIVEN: Mocked icons that need to be called. + self.media_item.has_import_icon = MagicMock() + self.media_item.has_new_icon = MagicMock() + self.media_item.has_edit_icon = MagicMock() + self.media_item.has_delete_icon = MagicMock() + self.media_item.add_to_service_item = MagicMock() + + # WHEN: self.media_item.required_icons is called + self.media_item.required_icons() + + # THEN: On windows it should return True, on other platforms False + self.assertTrue(self.media_item.has_import_icon, 'Check that the icon is as True.') + self.assertFalse(self.media_item.has_new_icon, 'Check that the icon is called as False.') + self.assertTrue(self.media_item.has_edit_icon, 'Check that the icon is called as True.') + self.assertTrue(self.media_item.has_delete_icon, 'Check that the icon is called as True.') + self.assertFalse(self.media_item.add_to_service_item, 'Check that the icon is called as False') + def on_quick_search_button_general_test(self): """ Test that general things, which should be called on all Quick searches are called. From 4733aa2169c10c955dcdcda44019101aaa978b48 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 20 Aug 2016 22:17:00 +0300 Subject: [PATCH 28/65] - Removed modifications to test_mainwindow. --- tests/functional/openlp_core_ui/test_mainwindow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/functional/openlp_core_ui/test_mainwindow.py b/tests/functional/openlp_core_ui/test_mainwindow.py index 09c423304..341e74eb8 100644 --- a/tests/functional/openlp_core_ui/test_mainwindow.py +++ b/tests/functional/openlp_core_ui/test_mainwindow.py @@ -26,12 +26,11 @@ import os from unittest import TestCase -from PyQt5 import QtWidgets, QtGui, QtCore +from PyQt5 import QtWidgets from openlp.core.ui.mainwindow import MainWindow from openlp.core.lib.ui import UiStrings from openlp.core.common.registry import Registry -from openlp.core.common import is_win, is_macosx, is_linux from tests.functional import MagicMock, patch from tests.helpers.testmixin import TestMixin From 894b4fbf10c08f9f0489bab6086199c8728a3809 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sat, 20 Aug 2016 21:00:50 +0100 Subject: [PATCH 29/65] revert changes to http.py due to circular references --- .../plugins/bibles/forms/bibleimportform.py | 4 +- openlp/plugins/bibles/lib/http.py | 535 ++++++++++++++++++ .../bibles/lib/importers/biblegateway.py | 313 ---------- .../bibles/lib/importers/bibleserver.py | 162 ------ .../plugins/bibles/lib/importers/crosswalk.py | 171 ------ .../openlp_plugins/bibles/test_bibleserver.py | 43 +- .../openlp_plugins/bibles/test_lib_http.py | 4 +- 7 files changed, 572 insertions(+), 660 deletions(-) delete mode 100644 openlp/plugins/bibles/lib/importers/biblegateway.py delete mode 100644 openlp/plugins/bibles/lib/importers/bibleserver.py delete mode 100644 openlp/plugins/bibles/lib/importers/crosswalk.py diff --git a/openlp/plugins/bibles/forms/bibleimportform.py b/openlp/plugins/bibles/forms/bibleimportform.py index e9eee88d5..3d02228ca 100644 --- a/openlp/plugins/bibles/forms/bibleimportform.py +++ b/openlp/plugins/bibles/forms/bibleimportform.py @@ -40,9 +40,7 @@ from openlp.core.ui.lib.wizard import OpenLPWizard, WizardStrings from openlp.core.common.languagemanager import get_locale_key from openlp.plugins.bibles.lib.manager import BibleFormat from openlp.plugins.bibles.lib.db import clean_filename -from openlp.plugins.bibles.lib.importers.biblegateway import BGExtract -from openlp.plugins.bibles.lib.importers.bibleserver import BSExtract -from openlp.plugins.bibles.lib.importers.crosswalk import CWExtract +from openlp.plugins.bibles.lib.importers.http import CWExtract, BGExtract, BSExtract log = logging.getLogger(__name__) diff --git a/openlp/plugins/bibles/lib/http.py b/openlp/plugins/bibles/lib/http.py index 5afd107f6..6921c9005 100644 --- a/openlp/plugins/bibles/lib/http.py +++ b/openlp/plugins/bibles/lib/http.py @@ -38,10 +38,545 @@ from openlp.plugins.bibles.lib.bibleimport import BibleImport from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB, Book CLEANER_REGEX = re.compile(r' |
|\'\+\'') +FIX_PUNKCTUATION_REGEX = re.compile(r'[ ]+([.,;])') +REDUCE_SPACES_REGEX = re.compile(r'[ ]{2,}') +UGLY_CHARS = { + '\u2014': ' - ', + '\u2018': '\'', + '\u2019': '\'', + '\u201c': '"', + '\u201d': '"', + ' ': ' ' +} +VERSE_NUMBER_REGEX = re.compile(r'v(\d{1,2})(\d{3})(\d{3}) verse.*') + +BIBLESERVER_LANGUAGE_CODE = { + 'fl_1': 'de', + 'fl_2': 'en', + 'fl_3': 'fr', + 'fl_4': 'it', + 'fl_5': 'es', + 'fl_6': 'pt', + 'fl_7': 'ru', + 'fl_8': 'sv', + 'fl_9': 'no', + 'fl_10': 'nl', + 'fl_11': 'cs', + 'fl_12': 'sk', + 'fl_13': 'ro', + 'fl_14': 'hr', + 'fl_15': 'hu', + 'fl_16': 'bg', + 'fl_17': 'ar', + 'fl_18': 'tr', + 'fl_19': 'pl', + 'fl_20': 'da', + 'fl_21': 'zh' +} + +CROSSWALK_LANGUAGES = { + 'Portuguese': 'pt', + 'German': 'de', + 'Italian': 'it', + 'Español': 'es', + 'French': 'fr', + 'Dutch': 'nl' +} log = logging.getLogger(__name__) +class BGExtract(RegistryProperties): + """ + Extract verses from BibleGateway + """ + def __init__(self, proxy_url=None): + log.debug('BGExtract.init("{url}")'.format(url=proxy_url)) + self.proxy_url = proxy_url + socket.setdefaulttimeout(30) + + def _remove_elements(self, parent, tag, class_=None): + """ + Remove a particular element from the BeautifulSoup tree. + + :param parent: The element from which items need to be removed. + :param tag: A string of the tab type, e.g. "div" + :param class_: An HTML class attribute for further qualification. + """ + if class_: + all_tags = parent.find_all(tag, class_) + else: + all_tags = parent.find_all(tag) + for element in all_tags: + element.extract() + + def _extract_verse(self, tag): + """ + Extract a verse (or part of a verse) from a tag. + + :param tag: The BeautifulSoup Tag element with the stuff we want. + """ + if isinstance(tag, NavigableString): + return None, str(tag) + elif tag.get('class') and (tag.get('class')[0] == 'versenum' or tag.get('class')[0] == 'versenum mid-line'): + verse = str(tag.string).replace('[', '').replace(']', '').strip() + return verse, None + elif tag.get('class') and tag.get('class')[0] == 'chapternum': + verse = '1' + return verse, None + else: + verse = None + text = '' + for child in tag.contents: + c_verse, c_text = self._extract_verse(child) + if c_verse: + verse = c_verse + if text and c_text: + text += c_text + elif c_text is not None: + text = c_text + return verse, text + + def _clean_soup(self, tag): + """ + Remove all the rubbish from the HTML page. + + :param tag: The base tag within which we want to remove stuff. + """ + self._remove_elements(tag, 'sup', 'crossreference') + self._remove_elements(tag, 'sup', 'footnote') + self._remove_elements(tag, 'div', 'footnotes') + self._remove_elements(tag, 'div', 'crossrefs') + self._remove_elements(tag, 'h3') + self._remove_elements(tag, 'h4') + self._remove_elements(tag, 'h5') + + def _extract_verses(self, tags): + """ + Extract all the verses from a pre-prepared list of HTML tags. + + :param tags: A list of BeautifulSoup Tag elements. + """ + verses = [] + tags = tags[::-1] + current_text = '' + for tag in tags: + verse = None + text = '' + for child in tag.contents: + c_verse, c_text = self._extract_verse(child) + if c_verse: + verse = c_verse + if text and c_text: + text += c_text + elif c_text is not None: + text = c_text + if not verse: + current_text = text + ' ' + current_text + else: + text += ' ' + current_text + current_text = '' + if text: + for old, new in UGLY_CHARS.items(): + text = text.replace(old, new) + text = ' '.join(text.split()) + if verse and text: + verse = verse.strip() + try: + verse = int(verse) + except ValueError: + verse_parts = verse.split('-') + if len(verse_parts) > 1: + verse = int(verse_parts[0]) + except TypeError: + log.warning('Illegal verse number: {verse:d}'.format(verse=verse)) + verses.append((verse, text)) + verse_list = {} + for verse, text in verses[::-1]: + verse_list[verse] = text + return verse_list + + def _extract_verses_old(self, div): + """ + Use the old style of parsing for those Bibles on BG who mysteriously have not been migrated to the new (still + broken) HTML. + + :param div: The parent div. + """ + verse_list = {} + # Cater for inconsistent mark up in the first verse of a chapter. + first_verse = div.find('versenum') + if first_verse and first_verse.contents: + verse_list[1] = str(first_verse.contents[0]) + for verse in div('sup', 'versenum'): + raw_verse_num = verse.next_element + clean_verse_num = 0 + # Not all verses exist in all translations and may or may not be represented by a verse number. If they are + # not fine, if they are it will probably be in a format that breaks int(). We will then have no idea what + # garbage may be sucked in to the verse text so if we do not get a clean int() then ignore the verse + # completely. + try: + clean_verse_num = int(str(raw_verse_num)) + except ValueError: + verse_parts = str(raw_verse_num).split('-') + if len(verse_parts) > 1: + clean_verse_num = int(verse_parts[0]) + except TypeError: + log.warning('Illegal verse number: {verse:d}'.format(verse=raw_verse_num)) + if clean_verse_num: + verse_text = raw_verse_num.next_element + part = raw_verse_num.next_element.next_element + while not (isinstance(part, Tag) and part.get('class')[0] == 'versenum'): + # While we are still in the same verse grab all the text. + if isinstance(part, NavigableString): + verse_text += part + if isinstance(part.next_element, Tag) and part.next_element.name == 'div': + # Run out of verses so stop. + break + part = part.next_element + verse_list[clean_verse_num] = str(verse_text) + return verse_list + + def get_bible_chapter(self, version, book_name, chapter): + """ + Access and decode Bibles via the BibleGateway website. + + :param version: The version of the Bible like 31 for New International version. + :param book_name: Name of the Book. + :param chapter: Chapter number. + """ + log.debug('BGExtract.get_bible_chapter("{version}", "{name}", "{chapter}")'.format(version=version, + name=book_name, + chapter=chapter)) + url_book_name = urllib.parse.quote(book_name.encode("utf-8")) + url_params = 'search={name}+{chapter}&version={version}'.format(name=url_book_name, + chapter=chapter, + version=version) + soup = get_soup_for_bible_ref( + 'http://biblegateway.com/passage/?{url}'.format(url=url_params), + pre_parse_regex=r'', pre_parse_substitute='') + if not soup: + return None + div = soup.find('div', 'result-text-style-normal') + if not div: + return None + self._clean_soup(div) + span_list = div.find_all('span', 'text') + log.debug('Span list: {span}'.format(span=span_list)) + if not span_list: + # If we don't get any spans then we must have the old HTML format + verse_list = self._extract_verses_old(div) + else: + verse_list = self._extract_verses(span_list) + if not verse_list: + log.debug('No content found in the BibleGateway response.') + send_error_message('parse') + return None + return SearchResults(book_name, chapter, verse_list) + + def get_books_from_http(self, version): + """ + Load a list of all books a Bible contains from BibleGateway website. + + :param version: The version of the Bible like NIV for New International Version + """ + log.debug('BGExtract.get_books_from_http("{version}")'.format(version=version)) + url_params = urllib.parse.urlencode({'action': 'getVersionInfo', 'vid': '{version}'.format(version=version)}) + reference_url = 'http://biblegateway.com/versions/?{url}#books'.format(url=url_params) + page = get_web_page(reference_url) + if not page: + send_error_message('download') + return None + page_source = page.read() + try: + page_source = str(page_source, 'utf8') + except UnicodeDecodeError: + page_source = str(page_source, 'cp1251') + try: + soup = BeautifulSoup(page_source, 'lxml') + except Exception: + log.error('BeautifulSoup could not parse the Bible page.') + send_error_message('parse') + return None + if not soup: + send_error_message('parse') + return None + self.application.process_events() + content = soup.find('table', 'infotable') + if content: + content = content.find_all('tr') + if not content: + log.error('No books found in the Biblegateway response.') + send_error_message('parse') + return None + books = [] + for book in content: + book = book.find('td') + if book: + books.append(book.contents[1]) + return books + + def get_bibles_from_http(self): + """ + Load a list of bibles from BibleGateway website. + + returns a list in the form [(biblename, biblekey, language_code)] + """ + log.debug('BGExtract.get_bibles_from_http') + bible_url = 'https://biblegateway.com/versions/' + soup = get_soup_for_bible_ref(bible_url) + if not soup: + return None + bible_select = soup.find('select', {'class': 'search-translation-select'}) + if not bible_select: + log.debug('No select tags found - did site change?') + return None + option_tags = bible_select.find_all('option') + if not option_tags: + log.debug('No option tags found - did site change?') + return None + current_lang = '' + bibles = [] + for ot in option_tags: + tag_class = '' + try: + tag_class = ot['class'][0] + except KeyError: + tag_class = '' + tag_text = ot.get_text() + if tag_class == 'lang': + current_lang = tag_text[tag_text.find('(') + 1:tag_text.find(')')].lower() + elif tag_class == 'spacer': + continue + else: + bibles.append((tag_text, ot['value'], current_lang)) + return bibles + + +class BSExtract(RegistryProperties): + """ + Extract verses from Bibleserver.com + """ + def __init__(self, proxy_url=None): + log.debug('BSExtract.init("{url}")'.format(url=proxy_url)) + self.proxy_url = proxy_url + socket.setdefaulttimeout(30) + + def get_bible_chapter(self, version, book_name, chapter): + """ + Access and decode bibles via Bibleserver mobile website + + :param version: The version of the bible like NIV for New International Version + :param book_name: Text name of bible book e.g. Genesis, 1. John, 1John or Offenbarung + :param chapter: Chapter number + """ + log.debug('BSExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, + book=book_name, + chapter=chapter)) + url_version = urllib.parse.quote(version.encode("utf-8")) + url_book_name = urllib.parse.quote(book_name.encode("utf-8")) + chapter_url = 'http://m.bibleserver.com/text/{version}/{name}{chapter:d}'.format(version=url_version, + name=url_book_name, + chapter=chapter) + header = ('Accept-Language', 'en') + soup = get_soup_for_bible_ref(chapter_url, header) + if not soup: + return None + self.application.process_events() + content = soup.find('div', 'content') + if not content: + log.error('No verses found in the Bibleserver response.') + send_error_message('parse') + return None + content = content.find('div').find_all('div') + verses = {} + for verse in content: + self.application.process_events() + versenumber = int(VERSE_NUMBER_REGEX.sub(r'\3', ' '.join(verse['class']))) + verses[versenumber] = verse.contents[1].rstrip('\n') + return SearchResults(book_name, chapter, verses) + + def get_books_from_http(self, version): + """ + Load a list of all books a Bible contains from Bibleserver mobile website. + + :param version: The version of the Bible like NIV for New International Version + """ + log.debug('BSExtract.get_books_from_http("{version}")'.format(version=version)) + url_version = urllib.parse.quote(version.encode("utf-8")) + chapter_url = 'http://m.bibleserver.com/overlay/selectBook?translation={version}'.format(version=url_version) + soup = get_soup_for_bible_ref(chapter_url) + if not soup: + return None + content = soup.find('ul') + if not content: + log.error('No books found in the Bibleserver response.') + send_error_message('parse') + return None + content = content.find_all('li') + return [book.contents[0].contents[0] for book in content if len(book.contents[0].contents)] + + def get_bibles_from_http(self): + """ + Load a list of bibles from Bibleserver website. + + returns a list in the form [(biblename, biblekey, language_code)] + """ + log.debug('BSExtract.get_bibles_from_http') + bible_url = 'http://www.bibleserver.com/index.php?language=2' + soup = get_soup_for_bible_ref(bible_url) + if not soup: + return None + bible_links = soup.find_all('a', {'class': 'trlCell'}) + if not bible_links: + log.debug('No a tags found - did site change?') + return None + bibles = [] + for link in bible_links: + bible_name = link.get_text() + # Skip any audio + if 'audio' in bible_name.lower(): + continue + try: + bible_link = link['href'] + bible_key = bible_link[bible_link.rfind('/') + 1:] + css_classes = link['class'] + except KeyError: + log.debug('No href/class attribute found - did site change?') + language_code = '' + for css_class in css_classes: + if css_class.startswith('fl_'): + try: + language_code = BIBLESERVER_LANGUAGE_CODE[css_class] + except KeyError: + language_code = '' + bibles.append((bible_name, bible_key, language_code)) + return bibles + + +class CWExtract(RegistryProperties): + """ + Extract verses from CrossWalk/BibleStudyTools + """ + def __init__(self, proxy_url=None): + log.debug('CWExtract.init("{url}")'.format(url=proxy_url)) + self.proxy_url = proxy_url + socket.setdefaulttimeout(30) + + def get_bible_chapter(self, version, book_name, chapter): + """ + Access and decode bibles via the Crosswalk website + + :param version: The version of the Bible like niv for New International Version + :param book_name: Text name of in english e.g. 'gen' for Genesis + :param chapter: Chapter number + """ + log.debug('CWExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, + book=book_name, + chapter=chapter)) + url_book_name = book_name.replace(' ', '-') + url_book_name = url_book_name.lower() + url_book_name = urllib.parse.quote(url_book_name.encode("utf-8")) + chapter_url = 'http://www.biblestudytools.com/{version}/{book}/{chapter}.html'.format(version=version, + book=url_book_name, + chapter=chapter) + soup = get_soup_for_bible_ref(chapter_url) + if not soup: + return None + self.application.process_events() + verses_div = soup.find_all('div', 'verse') + if not verses_div: + log.error('No verses found in the CrossWalk response.') + send_error_message('parse') + return None + verses = {} + for verse in verses_div: + self.application.process_events() + verse_number = int(verse.find('strong').contents[0]) + verse_span = verse.find('span') + tags_to_remove = verse_span.find_all(['a', 'sup']) + for tag in tags_to_remove: + tag.decompose() + verse_text = verse_span.get_text() + self.application.process_events() + # Fix up leading and trailing spaces, multiple spaces, and spaces between text and , and . + verse_text = verse_text.strip('\n\r\t ') + verse_text = REDUCE_SPACES_REGEX.sub(' ', verse_text) + verse_text = FIX_PUNKCTUATION_REGEX.sub(r'\1', verse_text) + verses[verse_number] = verse_text + return SearchResults(book_name, chapter, verses) + + def get_books_from_http(self, version): + """ + Load a list of all books a Bible contain from the Crosswalk website. + + :param version: The version of the bible like NIV for New International Version + """ + log.debug('CWExtract.get_books_from_http("{version}")'.format(version=version)) + chapter_url = 'http://www.biblestudytools.com/{version}/'.format(version=version) + soup = get_soup_for_bible_ref(chapter_url) + if not soup: + return None + content = soup.find_all('h4', {'class': 'small-header'}) + if not content: + log.error('No books found in the Crosswalk response.') + send_error_message('parse') + return None + books = [] + for book in content: + books.append(book.contents[0]) + return books + + def get_bibles_from_http(self): + """ + Load a list of bibles from Crosswalk website. + returns a list in the form [(biblename, biblekey, language_code)] + """ + log.debug('CWExtract.get_bibles_from_http') + bible_url = 'http://www.biblestudytools.com/bible-versions/' + soup = get_soup_for_bible_ref(bible_url) + if not soup: + return None + h4_tags = soup.find_all('h4', {'class': 'small-header'}) + if not h4_tags: + log.debug('No h4 tags found - did site change?') + return None + bibles = [] + for h4t in h4_tags: + short_name = None + if h4t.span: + short_name = h4t.span.get_text().strip().lower() + else: + log.error('No span tag found - did site change?') + return None + if not short_name: + continue + h4t.span.extract() + tag_text = h4t.get_text().strip() + # The names of non-english bibles has their language in parentheses at the end + if tag_text.endswith(')'): + language = tag_text[tag_text.rfind('(') + 1:-1] + if language in CROSSWALK_LANGUAGES: + language_code = CROSSWALK_LANGUAGES[language] + else: + language_code = '' + # ... except for those that don't... + elif 'latin' in tag_text.lower(): + language_code = 'la' + elif 'la biblia' in tag_text.lower() or 'nueva' in tag_text.lower(): + language_code = 'es' + elif 'chinese' in tag_text.lower(): + language_code = 'zh' + elif 'greek' in tag_text.lower(): + language_code = 'el' + elif 'nova' in tag_text.lower(): + language_code = 'pt' + else: + language_code = 'en' + bibles.append((tag_text, short_name, language_code)) + return bibles + + class HTTPBible(BibleImport, RegistryProperties): log.info('{name} HTTPBible loaded'.format(name=__name__)) diff --git a/openlp/plugins/bibles/lib/importers/biblegateway.py b/openlp/plugins/bibles/lib/importers/biblegateway.py deleted file mode 100644 index f3caa2204..000000000 --- a/openlp/plugins/bibles/lib/importers/biblegateway.py +++ /dev/null @@ -1,313 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2016 OpenLP Developers # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### -""" -The :mod:`biblegateway` module enables OpenLP to retrieve scripture from http://biblegateway.com. -""" -import logging -import socket -import urllib.parse -import urllib.error - -from bs4 import BeautifulSoup, NavigableString, Tag - -from openlp.core.common import RegistryProperties -from openlp.core.lib.webpagereader import get_web_page -from openlp.plugins.bibles.lib import SearchResults -from openlp.plugins.bibles.lib.http import get_soup_for_bible_ref, send_error_message - -UGLY_CHARS = { - '\u2014': ' - ', - '\u2018': '\'', - '\u2019': '\'', - '\u201c': '"', - '\u201d': '"', - ' ': ' ' -} - -log = logging.getLogger(__name__) - - -class BGExtract(RegistryProperties): - """ - Extract verses from BibleGateway - """ - def __init__(self, proxy_url=None): - log.debug('BGExtract.init("{url}")'.format(url=proxy_url)) - self.proxy_url = proxy_url - socket.setdefaulttimeout(30) - - def _remove_elements(self, parent, tag, class_=None): - """ - Remove a particular element from the BeautifulSoup tree. - - :param parent: The element from which items need to be removed. - :param tag: A string of the tab type, e.g. "div" - :param class_: An HTML class attribute for further qualification. - """ - if class_: - all_tags = parent.find_all(tag, class_) - else: - all_tags = parent.find_all(tag) - for element in all_tags: - element.extract() - - def _extract_verse(self, tag): - """ - Extract a verse (or part of a verse) from a tag. - - :param tag: The BeautifulSoup Tag element with the stuff we want. - """ - if isinstance(tag, NavigableString): - return None, str(tag) - elif tag.get('class') and (tag.get('class')[0] == 'versenum' or tag.get('class')[0] == 'versenum mid-line'): - verse = str(tag.string).replace('[', '').replace(']', '').strip() - return verse, None - elif tag.get('class') and tag.get('class')[0] == 'chapternum': - verse = '1' - return verse, None - else: - verse = None - text = '' - for child in tag.contents: - c_verse, c_text = self._extract_verse(child) - if c_verse: - verse = c_verse - if text and c_text: - text += c_text - elif c_text is not None: - text = c_text - return verse, text - - def _clean_soup(self, tag): - """ - Remove all the rubbish from the HTML page. - - :param tag: The base tag within which we want to remove stuff. - """ - self._remove_elements(tag, 'sup', 'crossreference') - self._remove_elements(tag, 'sup', 'footnote') - self._remove_elements(tag, 'div', 'footnotes') - self._remove_elements(tag, 'div', 'crossrefs') - self._remove_elements(tag, 'h3') - self._remove_elements(tag, 'h4') - self._remove_elements(tag, 'h5') - - def _extract_verses(self, tags): - """ - Extract all the verses from a pre-prepared list of HTML tags. - - :param tags: A list of BeautifulSoup Tag elements. - """ - verses = [] - tags = tags[::-1] - current_text = '' - for tag in tags: - verse = None - text = '' - for child in tag.contents: - c_verse, c_text = self._extract_verse(child) - if c_verse: - verse = c_verse - if text and c_text: - text += c_text - elif c_text is not None: - text = c_text - if not verse: - current_text = text + ' ' + current_text - else: - text += ' ' + current_text - current_text = '' - if text: - for old, new in UGLY_CHARS.items(): - text = text.replace(old, new) - text = ' '.join(text.split()) - if verse and text: - verse = verse.strip() - try: - verse = int(verse) - except ValueError: - verse_parts = verse.split('-') - if len(verse_parts) > 1: - verse = int(verse_parts[0]) - except TypeError: - log.warning('Illegal verse number: {verse:d}'.format(verse=verse)) - verses.append((verse, text)) - verse_list = {} - for verse, text in verses[::-1]: - verse_list[verse] = text - return verse_list - - def _extract_verses_old(self, div): - """ - Use the old style of parsing for those Bibles on BG who mysteriously have not been migrated to the new (still - broken) HTML. - - :param div: The parent div. - """ - verse_list = {} - # Cater for inconsistent mark up in the first verse of a chapter. - first_verse = div.find('versenum') - if first_verse and first_verse.contents: - verse_list[1] = str(first_verse.contents[0]) - for verse in div('sup', 'versenum'): - raw_verse_num = verse.next_element - clean_verse_num = 0 - # Not all verses exist in all translations and may or may not be represented by a verse number. If they are - # not fine, if they are it will probably be in a format that breaks int(). We will then have no idea what - # garbage may be sucked in to the verse text so if we do not get a clean int() then ignore the verse - # completely. - try: - clean_verse_num = int(str(raw_verse_num)) - except ValueError: - verse_parts = str(raw_verse_num).split('-') - if len(verse_parts) > 1: - clean_verse_num = int(verse_parts[0]) - except TypeError: - log.warning('Illegal verse number: {verse:d}'.format(verse=raw_verse_num)) - if clean_verse_num: - verse_text = raw_verse_num.next_element - part = raw_verse_num.next_element.next_element - while not (isinstance(part, Tag) and part.get('class')[0] == 'versenum'): - # While we are still in the same verse grab all the text. - if isinstance(part, NavigableString): - verse_text += part - if isinstance(part.next_element, Tag) and part.next_element.name == 'div': - # Run out of verses so stop. - break - part = part.next_element - verse_list[clean_verse_num] = str(verse_text) - return verse_list - - def get_bible_chapter(self, version, book_name, chapter): - """ - Access and decode Bibles via the BibleGateway website. - - :param version: The version of the Bible like 31 for New International version. - :param book_name: Name of the Book. - :param chapter: Chapter number. - """ - log.debug('BGExtract.get_bible_chapter("{version}", "{name}", "{chapter}")'.format(version=version, - name=book_name, - chapter=chapter)) - url_book_name = urllib.parse.quote(book_name.encode("utf-8")) - url_params = 'search={name}+{chapter}&version={version}'.format(name=url_book_name, - chapter=chapter, - version=version) - soup = get_soup_for_bible_ref( - 'http://biblegateway.com/passage/?{url}'.format(url=url_params), - pre_parse_regex=r'', pre_parse_substitute='') - if not soup: - return None - div = soup.find('div', 'result-text-style-normal') - if not div: - return None - self._clean_soup(div) - span_list = div.find_all('span', 'text') - log.debug('Span list: {span}'.format(span=span_list)) - if not span_list: - # If we don't get any spans then we must have the old HTML format - verse_list = self._extract_verses_old(div) - else: - verse_list = self._extract_verses(span_list) - if not verse_list: - log.debug('No content found in the BibleGateway response.') - send_error_message('parse') - return None - return SearchResults(book_name, chapter, verse_list) - - def get_books_from_http(self, version): - """ - Load a list of all books a Bible contains from BibleGateway website. - - :param version: The version of the Bible like NIV for New International Version - """ - log.debug('BGExtract.get_books_from_http("{version}")'.format(version=version)) - url_params = urllib.parse.urlencode({'action': 'getVersionInfo', 'vid': '{version}'.format(version=version)}) - reference_url = 'http://biblegateway.com/versions/?{url}#books'.format(url=url_params) - page = get_web_page(reference_url) - if not page: - send_error_message('download') - return None - page_source = page.read() - try: - page_source = str(page_source, 'utf8') - except UnicodeDecodeError: - page_source = str(page_source, 'cp1251') - try: - soup = BeautifulSoup(page_source, 'lxml') - except Exception: - log.error('BeautifulSoup could not parse the Bible page.') - send_error_message('parse') - return None - if not soup: - send_error_message('parse') - return None - self.application.process_events() - content = soup.find('table', 'infotable') - if content: - content = content.find_all('tr') - if not content: - log.error('No books found in the Biblegateway response.') - send_error_message('parse') - return None - books = [] - for book in content: - book = book.find('td') - if book: - books.append(book.contents[1]) - return books - - def get_bibles_from_http(self): - """ - Load a list of bibles from BibleGateway website. - - returns a list in the form [(biblename, biblekey, language_code)] - """ - log.debug('BGExtract.get_bibles_from_http') - bible_url = 'https://biblegateway.com/versions/' - soup = get_soup_for_bible_ref(bible_url) - if not soup: - return None - bible_select = soup.find('select', {'class': 'search-translation-select'}) - if not bible_select: - log.debug('No select tags found - did site change?') - return None - option_tags = bible_select.find_all('option') - if not option_tags: - log.debug('No option tags found - did site change?') - return None - current_lang = '' - bibles = [] - for ot in option_tags: - tag_class = '' - try: - tag_class = ot['class'][0] - except KeyError: - tag_class = '' - tag_text = ot.get_text() - if tag_class == 'lang': - current_lang = tag_text[tag_text.find('(') + 1:tag_text.find(')')].lower() - elif tag_class == 'spacer': - continue - else: - bibles.append((tag_text, ot['value'], current_lang)) - return bibles diff --git a/openlp/plugins/bibles/lib/importers/bibleserver.py b/openlp/plugins/bibles/lib/importers/bibleserver.py deleted file mode 100644 index 16924d84a..000000000 --- a/openlp/plugins/bibles/lib/importers/bibleserver.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2016 OpenLP Developers # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### -""" -The :mod:`bibleserver` module enables OpenLP to retrieve scripture from http://bibleserver.com. -""" -import logging -import re -import socket -import urllib.parse -import urllib.error - -from openlp.core.common import RegistryProperties -from openlp.plugins.bibles.lib import SearchResults -from openlp.plugins.bibles.lib.http import get_soup_for_bible_ref, send_error_message - -VERSE_NUMBER_REGEX = re.compile(r'v(\d{1,2})(\d{3})(\d{3}) verse.*') - -BIBLESERVER_LANGUAGE_CODE = { - 'fl_1': 'de', - 'fl_2': 'en', - 'fl_3': 'fr', - 'fl_4': 'it', - 'fl_5': 'es', - 'fl_6': 'pt', - 'fl_7': 'ru', - 'fl_8': 'sv', - 'fl_9': 'no', - 'fl_10': 'nl', - 'fl_11': 'cs', - 'fl_12': 'sk', - 'fl_13': 'ro', - 'fl_14': 'hr', - 'fl_15': 'hu', - 'fl_16': 'bg', - 'fl_17': 'ar', - 'fl_18': 'tr', - 'fl_19': 'pl', - 'fl_20': 'da', - 'fl_21': 'zh' -} - -log = logging.getLogger(__name__) - - -class BSExtract(RegistryProperties): - """ - Extract verses from Bibleserver.com - """ - def __init__(self, proxy_url=None): - log.debug('BSExtract.init("{url}")'.format(url=proxy_url)) - self.proxy_url = proxy_url - socket.setdefaulttimeout(30) - - def get_bible_chapter(self, version, book_name, chapter): - """ - Access and decode bibles via Bibleserver mobile website - - :param version: The version of the bible like NIV for New International Version - :param book_name: Text name of bible book e.g. Genesis, 1. John, 1John or Offenbarung - :param chapter: Chapter number - """ - log.debug('BSExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, - book=book_name, - chapter=chapter)) - url_version = urllib.parse.quote(version.encode("utf-8")) - url_book_name = urllib.parse.quote(book_name.encode("utf-8")) - chapter_url = 'http://m.bibleserver.com/text/{version}/{name}{chapter:d}'.format(version=url_version, - name=url_book_name, - chapter=chapter) - header = ('Accept-Language', 'en') - soup = get_soup_for_bible_ref(chapter_url, header) - if not soup: - return None - self.application.process_events() - content = soup.find('div', 'content') - if not content: - log.error('No verses found in the Bibleserver response.') - send_error_message('parse') - return None - content = content.find('div').find_all('div') - verses = {} - for verse in content: - self.application.process_events() - versenumber = int(VERSE_NUMBER_REGEX.sub(r'\3', ' '.join(verse['class']))) - verses[versenumber] = verse.contents[1].rstrip('\n') - return SearchResults(book_name, chapter, verses) - - def get_books_from_http(self, version): - """ - Load a list of all books a Bible contains from Bibleserver mobile website. - - :param version: The version of the Bible like NIV for New International Version - """ - log.debug('BSExtract.get_books_from_http("{version}")'.format(version=version)) - url_version = urllib.parse.quote(version.encode("utf-8")) - chapter_url = 'http://m.bibleserver.com/overlay/selectBook?translation={version}'.format(version=url_version) - soup = get_soup_for_bible_ref(chapter_url) - if not soup: - return None - content = soup.find('ul') - if not content: - log.error('No books found in the Bibleserver response.') - send_error_message('parse') - return None - content = content.find_all('li') - return [book.contents[0].contents[0] for book in content if len(book.contents[0].contents)] - - def get_bibles_from_http(self): - """ - Load a list of bibles from Bibleserver website. - - returns a list in the form [(biblename, biblekey, language_code)] - """ - log.debug('BSExtract.get_bibles_from_http') - bible_url = 'http://www.bibleserver.com/index.php?language=2' - soup = get_soup_for_bible_ref(bible_url) - if not soup: - return None - bible_links = soup.find_all('a', {'class': 'trlCell'}) - if not bible_links: - log.debug('No a tags found - did site change?') - return None - bibles = [] - for link in bible_links: - bible_name = link.get_text() - # Skip any audio - if 'audio' in bible_name.lower(): - continue - try: - bible_link = link['href'] - bible_key = bible_link[bible_link.rfind('/') + 1:] - css_classes = link['class'] - except KeyError: - log.debug('No href/class attribute found - did site change?') - language_code = '' - for css_class in css_classes: - if css_class.startswith('fl_'): - try: - language_code = BIBLESERVER_LANGUAGE_CODE[css_class] - except KeyError: - language_code = '' - bibles.append((bible_name, bible_key, language_code)) - return bibles diff --git a/openlp/plugins/bibles/lib/importers/crosswalk.py b/openlp/plugins/bibles/lib/importers/crosswalk.py deleted file mode 100644 index fb354dd29..000000000 --- a/openlp/plugins/bibles/lib/importers/crosswalk.py +++ /dev/null @@ -1,171 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2016 OpenLP Developers # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### -""" -The :mod:`crosswalk` module enables OpenLP to retrieve scripture from www.biblestudytools.com. -""" -import logging -import re -import socket -import urllib.parse -import urllib.error - -from openlp.core.common import RegistryProperties -from openlp.plugins.bibles.lib import SearchResults -from openlp.plugins.bibles.lib.http import get_soup_for_bible_ref, send_error_message - -FIX_PUNKCTUATION_REGEX = re.compile(r'[ ]+([.,;])') -REDUCE_SPACES_REGEX = re.compile(r'[ ]{2,}') - - -CROSSWALK_LANGUAGES = { - 'Portuguese': 'pt', - 'German': 'de', - 'Italian': 'it', - 'Español': 'es', - 'French': 'fr', - 'Dutch': 'nl' -} - -log = logging.getLogger(__name__) - - -class CWExtract(RegistryProperties): - """ - Extract verses from CrossWalk/BibleStudyTools - """ - def __init__(self, proxy_url=None): - log.debug('CWExtract.init("{url}")'.format(url=proxy_url)) - self.proxy_url = proxy_url - socket.setdefaulttimeout(30) - - def get_bible_chapter(self, version, book_name, chapter): - """ - Access and decode bibles via the Crosswalk website - - :param version: The version of the Bible like niv for New International Version - :param book_name: Text name of in english e.g. 'gen' for Genesis - :param chapter: Chapter number - """ - log.debug('CWExtract.get_bible_chapter("{version}", "{book}", "{chapter}")'.format(version=version, - book=book_name, - chapter=chapter)) - url_book_name = book_name.replace(' ', '-') - url_book_name = url_book_name.lower() - url_book_name = urllib.parse.quote(url_book_name.encode("utf-8")) - chapter_url = 'http://www.biblestudytools.com/{version}/{book}/{chapter}.html'.format(version=version, - book=url_book_name, - chapter=chapter) - soup = get_soup_for_bible_ref(chapter_url) - if not soup: - return None - self.application.process_events() - verses_div = soup.find_all('div', 'verse') - if not verses_div: - log.error('No verses found in the CrossWalk response.') - send_error_message('parse') - return None - verses = {} - for verse in verses_div: - self.application.process_events() - verse_number = int(verse.find('strong').contents[0]) - verse_span = verse.find('span') - tags_to_remove = verse_span.find_all(['a', 'sup']) - for tag in tags_to_remove: - tag.decompose() - verse_text = verse_span.get_text() - self.application.process_events() - # Fix up leading and trailing spaces, multiple spaces, and spaces between text and , and . - verse_text = verse_text.strip('\n\r\t ') - verse_text = REDUCE_SPACES_REGEX.sub(' ', verse_text) - verse_text = FIX_PUNKCTUATION_REGEX.sub(r'\1', verse_text) - verses[verse_number] = verse_text - return SearchResults(book_name, chapter, verses) - - def get_books_from_http(self, version): - """ - Load a list of all books a Bible contain from the Crosswalk website. - - :param version: The version of the bible like NIV for New International Version - """ - log.debug('CWExtract.get_books_from_http("{version}")'.format(version=version)) - chapter_url = 'http://www.biblestudytools.com/{version}/'.format(version=version) - soup = get_soup_for_bible_ref(chapter_url) - if not soup: - return None - content = soup.find_all('h4', {'class': 'small-header'}) - if not content: - log.error('No books found in the Crosswalk response.') - send_error_message('parse') - return None - books = [] - for book in content: - books.append(book.contents[0]) - return books - - def get_bibles_from_http(self): - """ - Load a list of bibles from Crosswalk website. - returns a list in the form [(biblename, biblekey, language_code)] - """ - log.debug('CWExtract.get_bibles_from_http') - bible_url = 'http://www.biblestudytools.com/bible-versions/' - soup = get_soup_for_bible_ref(bible_url) - if not soup: - return None - h4_tags = soup.find_all('h4', {'class': 'small-header'}) - if not h4_tags: - log.debug('No h4 tags found - did site change?') - return None - bibles = [] - for h4t in h4_tags: - short_name = None - if h4t.span: - short_name = h4t.span.get_text().strip().lower() - else: - log.error('No span tag found - did site change?') - return None - if not short_name: - continue - h4t.span.extract() - tag_text = h4t.get_text().strip() - # The names of non-english bibles has their language in parentheses at the end - if tag_text.endswith(')'): - language = tag_text[tag_text.rfind('(') + 1:-1] - if language in CROSSWALK_LANGUAGES: - language_code = CROSSWALK_LANGUAGES[language] - else: - language_code = '' - # ... except for those that don't... - elif 'latin' in tag_text.lower(): - language_code = 'la' - elif 'la biblia' in tag_text.lower() or 'nueva' in tag_text.lower(): - language_code = 'es' - elif 'chinese' in tag_text.lower(): - language_code = 'zh' - elif 'greek' in tag_text.lower(): - language_code = 'el' - elif 'nova' in tag_text.lower(): - language_code = 'pt' - else: - language_code = 'en' - bibles.append((tag_text, short_name, language_code)) - return bibles diff --git a/tests/functional/openlp_plugins/bibles/test_bibleserver.py b/tests/functional/openlp_plugins/bibles/test_bibleserver.py index 0849a63e3..839c81008 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleserver.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleserver.py @@ -20,13 +20,41 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ -This module contains tests for the bibleserver module of the Bibles plugin. +This module contains tests for the http module of the Bibles plugin. """ from unittest import TestCase from bs4 import BeautifulSoup from tests.functional import patch, MagicMock -from openlp.plugins.bibles.lib.importers.bibleserver import BSExtract +from openlp.plugins.bibles.lib.importers.http import BSExtract + +# TODO: Items left to test +# BGExtract +# __init__ +# _remove_elements +# _extract_verse +# _clean_soup +# _extract_verses +# _extract_verses_old +# get_bible_chapter +# get_books_from_http +# _get_application +# CWExtract +# __init__ +# get_bible_chapter +# get_books_from_http +# _get_application +# HTTPBible +# __init__ +# do_import +# get_verses +# get_chapter +# get_books +# get_chapter_count +# get_verse_count +# _get_application +# get_soup_for_bible_ref +# send_error_message class TestBSExtract(TestCase): @@ -40,12 +68,11 @@ class TestBSExtract(TestCase): # get_books_from_http # _get_application def setUp(self): - self.get_soup_for_bible_ref_patcher = patch( - 'openlp.plugins.bibles.lib.importers.bibleserver.get_soup_for_bible_ref') - self.log_patcher = patch('openlp.plugins.bibles.lib.importers.bibleserver.log') - self.send_error_message_patcher = patch('openlp.plugins.bibles.lib.importers.bibleserver.send_error_message') - self.socket_patcher = patch('openlp.plugins.bibles.lib.http.socket') - self.urllib_patcher = patch('openlp.plugins.bibles.lib.importers.bibleserver.urllib') + self.get_soup_for_bible_ref_patcher = patch('openlp.plugins.bibles.lib.importers.http.get_soup_for_bible_ref') + self.log_patcher = patch('openlp.plugins.bibles.lib.importers.http.log') + self.send_error_message_patcher = patch('openlp.plugins.bibles.lib.importers.http.send_error_message') + self.socket_patcher = patch('openlp.plugins.bibles.lib.importers.http.socket') + self.urllib_patcher = patch('openlp.plugins.bibles.lib.importers.http.urllib') self.mock_get_soup_for_bible_ref = self.get_soup_for_bible_ref_patcher.start() self.mock_log = self.log_patcher.start() diff --git a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py index fd557eece..084bfa476 100644 --- a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py +++ b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py @@ -25,9 +25,7 @@ from unittest import TestCase, skip from openlp.core.common import Registry -from openlp.plugins.bibles.lib.importers.biblegateway import BGExtract -from openlp.plugins.bibles.lib.importers.bibleserver import BSExtract -from openlp.plugins.bibles.lib.importers.crosswalk import CWExtract +from openlp.plugins.bibles.lib.importers.http import BGExtract, CWExtract, BSExtract from tests.interfaces import MagicMock From aefcd48cc3d92a1149ee0f1228111379c15d8fcd Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sat, 20 Aug 2016 21:32:25 +0100 Subject: [PATCH 30/65] reverted changes to bibles http.py --- openlp/plugins/bibles/lib/{ => importers}/http.py | 0 openlp/plugins/bibles/lib/manager.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename openlp/plugins/bibles/lib/{ => importers}/http.py (100%) diff --git a/openlp/plugins/bibles/lib/http.py b/openlp/plugins/bibles/lib/importers/http.py similarity index 100% rename from openlp/plugins/bibles/lib/http.py rename to openlp/plugins/bibles/lib/importers/http.py diff --git a/openlp/plugins/bibles/lib/manager.py b/openlp/plugins/bibles/lib/manager.py index 2734411f5..d2286bed2 100644 --- a/openlp/plugins/bibles/lib/manager.py +++ b/openlp/plugins/bibles/lib/manager.py @@ -27,7 +27,7 @@ from openlp.core.common import RegistryProperties, AppLocation, Settings, transl from openlp.plugins.bibles.lib import parse_reference, LanguageSelection from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta from .importers.csvbible import CSVBible -from .http import HTTPBible +from .importers.http import HTTPBible from .importers.opensong import OpenSongBible from .importers.osis import OSISBible from .importers.zefania import ZefaniaBible From 3c29427866a37877c21c809318464bcada9df9e0 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sun, 21 Aug 2016 08:39:40 +0100 Subject: [PATCH 31/65] Added some doc strings which I missed --- .../plugins/bibles/lib/importers/opensong.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index c0a82a4ff..201bea1da 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -42,6 +42,7 @@ class OpenSongBible(BibleImport): Recursively get all text in an objectify element and its child elements. :param element: An objectify element to get the text from + :return: The text content of the element (str) """ verse_text = '' if element.text: @@ -110,6 +111,12 @@ class OpenSongBible(BibleImport): return True def process_books(self, books): + """ + Extract and create the books from the objectified xml + + :param books: Objectified xml + :return: None + """ for book in books: if self.stop_import_flag: break @@ -118,6 +125,13 @@ class OpenSongBible(BibleImport): self.session.commit() def process_chapters(self, book, chapters): + """ + Extract and create the chapters from the objectified xml for the book `book` + + :param book: A database Book object to add the chapters to + :param chapters: Objectified xml containing chapters + :return: None + """ chapter_number = 0 for chapter in chapters: if self.stop_import_flag: @@ -129,6 +143,14 @@ class OpenSongBible(BibleImport): ).format(name=book.name, chapter=chapter_number)) def process_verses(self, book, chapter_number, verses): + """ + Extract and create the verses from the objectified xml + + :param book: A database Book object + :param chapter_number: The chapter number to add the verses to (int) + :param verses: Objectified xml containing verses + :return: None + """ verse_number = 0 for verse in verses: if self.stop_import_flag: @@ -139,6 +161,9 @@ class OpenSongBible(BibleImport): def do_import(self, bible_name=None): """ Loads an Open Song Bible from a file. + + :param bible_name: The name of the bible being imported + :return: True if import completed, False if import was unsuccessful """ log.debug('Starting OpenSong import from "{name}"'.format(name=self.filename)) try: From dcd8d6f6b51c6acc8603bd9764497d56b44d31a1 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sun, 21 Aug 2016 12:04:18 +0100 Subject: [PATCH 32/65] Remove upgrade code not needed since 2.2 --- openlp/plugins/bibles/lib/upgrade.py | 166 +-------------------------- 1 file changed, 2 insertions(+), 164 deletions(-) diff --git a/openlp/plugins/bibles/lib/upgrade.py b/openlp/plugins/bibles/lib/upgrade.py index aebd088e8..9907c71cc 100644 --- a/openlp/plugins/bibles/lib/upgrade.py +++ b/openlp/plugins/bibles/lib/upgrade.py @@ -24,8 +24,6 @@ The :mod:`upgrade` module provides a way for the database and schema that is the """ import logging -from sqlalchemy import delete, func, insert, select - log = logging.getLogger(__name__) __version__ = 1 @@ -35,166 +33,6 @@ def upgrade_1(session, metadata): """ Version 1 upgrade. - This upgrade renames a number of keys to a single naming convention. + This upgrade renamed a number of keys to a single naming convention. """ - metadata_table = metadata.tables['metadata'] - # Copy "Version" to "name" ("version" used by upgrade system) - try: - session.execute(insert(metadata_table).values( - key='name', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'Version' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'Version')) - except: - log.exception('Exception when upgrading Version') - # Copy "Copyright" to "copyright" - try: - session.execute(insert(metadata_table).values( - key='copyright', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'Copyright' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'Copyright')) - except: - log.exception('Exception when upgrading Copyright') - # Copy "Permissions" to "permissions" - try: - session.execute(insert(metadata_table).values( - key='permissions', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'Permissions' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'Permissions')) - except: - log.exception('Exception when upgrading Permissions') - # Copy "Bookname language" to "book_name_language" - try: - value_count = session.execute( - select( - [func.count(metadata_table.c.value)], - metadata_table.c.key == 'Bookname language' - ) - ).scalar() - if value_count > 0: - session.execute(insert(metadata_table).values( - key='book_name_language', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'Bookname language' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'Bookname language')) - except: - log.exception('Exception when upgrading Bookname language') - # Copy "download source" to "download_source" - try: - value_count = session.execute( - select( - [func.count(metadata_table.c.value)], - metadata_table.c.key == 'download source' - ) - ).scalar() - log.debug('download source: {count}'.format(count=value_count)) - if value_count > 0: - session.execute(insert(metadata_table).values( - key='download_source', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'download source' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'download source')) - except: - log.exception('Exception when upgrading download source') - # Copy "download name" to "download_name" - try: - value_count = session.execute( - select( - [func.count(metadata_table.c.value)], - metadata_table.c.key == 'download name' - ) - ).scalar() - log.debug('download name: {count}'.format(count=value_count)) - if value_count > 0: - session.execute(insert(metadata_table).values( - key='download_name', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'download name' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'download name')) - except: - log.exception('Exception when upgrading download name') - # Copy "proxy server" to "proxy_server" - try: - value_count = session.execute( - select( - [func.count(metadata_table.c.value)], - metadata_table.c.key == 'proxy server' - ) - ).scalar() - log.debug('proxy server: {count}'.format(count=value_count)) - if value_count > 0: - session.execute(insert(metadata_table).values( - key='proxy_server', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'proxy server' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'proxy server')) - except: - log.exception('Exception when upgrading proxy server') - # Copy "proxy username" to "proxy_username" - try: - value_count = session.execute( - select( - [func.count(metadata_table.c.value)], - metadata_table.c.key == 'proxy username' - ) - ).scalar() - log.debug('proxy username: {count}'.format(count=value_count)) - if value_count > 0: - session.execute(insert(metadata_table).values( - key='proxy_username', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'proxy username' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'proxy username')) - except: - log.exception('Exception when upgrading proxy username') - # Copy "proxy password" to "proxy_password" - try: - value_count = session.execute( - select( - [func.count(metadata_table.c.value)], - metadata_table.c.key == 'proxy password' - ) - ).scalar() - log.debug('proxy password: {count}'.format(count=value_count)) - if value_count > 0: - session.execute(insert(metadata_table).values( - key='proxy_password', - value=select( - [metadata_table.c.value], - metadata_table.c.key == 'proxy password' - ).as_scalar() - )) - session.execute(delete(metadata_table).where(metadata_table.c.key == 'proxy password')) - except: - log.exception('Exception when upgrading proxy password') - try: - session.execute(delete(metadata_table).where(metadata_table.c.key == 'dbversion')) - except: - log.exception('Exception when deleting dbversion') - session.commit() + log.info('No upgrades to perform') From 28d94b7d10162a9c6f4a2f6830b47529ed015f49 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sun, 21 Aug 2016 21:36:59 +0100 Subject: [PATCH 33/65] moved some static methods to module level --- .../plugins/bibles/lib/importers/opensong.py | 153 +++++++++--------- .../bibles/test_opensongimport.py | 52 +++--- 2 files changed, 103 insertions(+), 102 deletions(-) diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 201bea1da..01be8407e 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -32,84 +32,62 @@ from openlp.plugins.bibles.lib.bibleimport import BibleImport log = logging.getLogger(__name__) +def get_text(element): + """ + Recursively get all text in an objectify element and its child elements. + + :param element: An objectify element to get the text from + :return: The text content of the element (str) + """ + verse_text = '' + if element.text: + verse_text = element.text + for sub_element in element.iterchildren(): + verse_text += get_text(sub_element) + if element.tail: + verse_text += element.tail + return verse_text + + +def parse_chapter_number(number, previous_number): + """ + Parse the chapter number + + :param number: The raw data from the xml + :param previous_number: The previous chapter number + :return: Number of current chapter. (Int) + """ + if number: + return int(number.split()[-1]) + return previous_number + 1 + + +def parse_verse_number(number, previous_number): + """ + Parse the verse number retrieved from the xml + + :param number: The raw data from the xml + :param previous_number: The previous verse number + :return: Number of current verse. (Int) + """ + if not number: + return previous_number + 1 + try: + return int(number) + except ValueError: + verse_parts = number.split('-') + if len(verse_parts) > 1: + number = int(verse_parts[0]) + return number + except TypeError: + log.warning('Illegal verse number: {verse_no}'.format(verse_no=str(number))) + return previous_number + 1 + + class OpenSongBible(BibleImport): """ OpenSong Bible format importer class. This class is used to import Bibles from OpenSong's XML format. """ - @staticmethod - def get_text(element): - """ - Recursively get all text in an objectify element and its child elements. - - :param element: An objectify element to get the text from - :return: The text content of the element (str) - """ - verse_text = '' - if element.text: - verse_text = element.text - for sub_element in element.iterchildren(): - verse_text += OpenSongBible.get_text(sub_element) - if element.tail: - verse_text += element.tail - return verse_text - - @staticmethod - def parse_chapter_number(number, previous_number): - """ - Parse the chapter number - - :param number: The raw data from the xml - :param previous_number: The previous chapter number - :return: Number of current chapter. (Int) - """ - if number: - return int(number.split()[-1]) - return previous_number + 1 - - @staticmethod - def parse_verse_number(number, previous_number): - """ - Parse the verse number retrieved from the xml - - :param number: The raw data from the xml - :param previous_number: The previous verse number - :return: Number of current verse. (Int) - """ - if not number: - return previous_number + 1 - try: - return int(number) - except ValueError: - verse_parts = number.split('-') - if len(verse_parts) > 1: - number = int(verse_parts[0]) - return number - except TypeError: - log.warning('Illegal verse number: {verse_no}'.format(verse_no=str(number))) - return previous_number + 1 - - @staticmethod - def validate_file(filename): - """ - Validate the supplied file - - :param filename: The supplied file - :return: True if valid. ValidationError is raised otherwise. - """ - if BibleImport.is_compressed(filename): - raise ValidationError(msg='Compressed file') - bible = BibleImport.parse_xml(filename, use_objectify=True) - root_tag = bible.tag.lower() - if root_tag != 'bible': - if root_tag == 'xmlbible': - # Zefania bibles have a root tag of XMLBIBLE". Sometimes these bibles are referred to as 'OpenSong' - critical_error_message_box( - message=translate('BiblesPlugin.OpenSongImport', - 'Incorrect Bible file type supplied. This looks like a Zefania XML bible, ' - 'please use the Zefania import option.')) - raise ValidationError(msg='Invalid xml.') - return True - def process_books(self, books): """ Extract and create the books from the objectified xml @@ -136,7 +114,7 @@ class OpenSongBible(BibleImport): for chapter in chapters: if self.stop_import_flag: break - chapter_number = self.parse_chapter_number(chapter.attrib['n'], chapter_number) + chapter_number = parse_chapter_number(chapter.attrib['n'], chapter_number) self.process_verses(book, chapter_number, chapter.v) self.wizard.increment_progress_bar(translate('BiblesPlugin.Opensong', 'Importing {name} {chapter}...' @@ -155,8 +133,29 @@ class OpenSongBible(BibleImport): for verse in verses: if self.stop_import_flag: break - verse_number = self.parse_verse_number(verse.attrib['n'], verse_number) - self.create_verse(book.id, chapter_number, verse_number, self.get_text(verse)) + verse_number = parse_verse_number(verse.attrib['n'], verse_number) + self.create_verse(book.id, chapter_number, verse_number, get_text(verse)) + + def validate_file(self, filename): + """ + Validate the supplied file + + :param filename: The supplied file + :return: True if valid. ValidationError is raised otherwise. + """ + if BibleImport.is_compressed(filename): + raise ValidationError(msg='Compressed file') + bible = self.parse_xml(filename, use_objectify=True) + root_tag = bible.tag.lower() + if root_tag != 'bible': + if root_tag == 'xmlbible': + # Zefania bibles have a root tag of XMLBIBLE". Sometimes these bibles are referred to as 'OpenSong' + critical_error_message_box( + message=translate('BiblesPlugin.OpenSongImport', + 'Incorrect Bible file type supplied. This looks like a Zefania XML bible, ' + 'please use the Zefania import option.')) + raise ValidationError(msg='Invalid xml.') + return True def do_import(self, bible_name=None): """ diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index ee4e794c0..0f5c404ac 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -33,7 +33,8 @@ from tests.functional import MagicMock, patch, call from tests.helpers.testmixin import TestMixin from openlp.core.common import Registry from openlp.core.lib.exceptions import ValidationError -from openlp.plugins.bibles.lib.importers.opensong import OpenSongBible +from openlp.plugins.bibles.lib.importers.opensong import OpenSongBible, get_text, parse_chapter_number,\ + parse_verse_number from openlp.plugins.bibles.lib.bibleimport import BibleImport TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -75,7 +76,7 @@ class TestOpenSongImport(TestCase, TestMixin): test_data = objectify.fromstring('') # WHEN: Calling get_text - result = OpenSongBible.get_text(test_data) + result = get_text(test_data) # THEN: A blank string should be returned self.assertEqual(result, '') @@ -92,7 +93,7 @@ class TestOpenSongImport(TestCase, TestMixin): 'sub_tail tail') # WHEN: Calling get_text - result = OpenSongBible.get_text(test_data) + result = get_text(test_data) # THEN: The text returned should be as expected self.assertEqual(result, 'Element text sub_text_tail text sub_text_tail tail sub_text text sub_tail tail') @@ -103,7 +104,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: The number 10 represented as a string # WHEN: Calling parse_chapter_nnumber - result = OpenSongBible.parse_chapter_number('10', 0) + result = parse_chapter_number('10', 0) # THEN: The 10 should be returned as an Int self.assertEqual(result, 10) @@ -114,7 +115,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: An empty string, and the previous chapter number set as 12 and an instance of OpenSongBible # WHEN: Calling parse_chapter_number - result = OpenSongBible.parse_chapter_number('', 12) + result = parse_chapter_number('', 12) # THEN: parse_chapter_number should increment the previous verse number self.assertEqual(result, 13) @@ -125,7 +126,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: The number 15 represented as a string and an instance of OpenSongBible # WHEN: Calling parse_verse_number - result = OpenSongBible.parse_verse_number('15', 0) + result = parse_verse_number('15', 0) # THEN: parse_verse_number should return the verse number self.assertEqual(result, 15) @@ -136,7 +137,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: The range 24-26 represented as a string # WHEN: Calling parse_verse_number - result = OpenSongBible.parse_verse_number('24-26', 0) + result = parse_verse_number('24-26', 0) # THEN: parse_verse_number should return the first verse number in the range self.assertEqual(result, 24) @@ -147,7 +148,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: An non numeric string represented as a string # WHEN: Calling parse_verse_number - result = OpenSongBible.parse_verse_number('invalid', 41) + result = parse_verse_number('invalid', 41) # THEN: parse_verse_number should increment the previous verse number self.assertEqual(result, 42) @@ -158,7 +159,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: An empty string, and the previous verse number set as 14 # WHEN: Calling parse_verse_number - result = OpenSongBible.parse_verse_number('', 14) + result = parse_verse_number('', 14) # THEN: parse_verse_number should increment the previous verse number self.assertEqual(result, 15) @@ -170,7 +171,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: A mocked out log, a Tuple, and the previous verse number set as 12 # WHEN: Calling parse_verse_number - result = OpenSongBible.parse_verse_number((1, 2, 3), 12) + result = parse_verse_number((1, 2, 3), 12) # THEN: parse_verse_number should log the verse number it was called with increment the previous verse number mocked_log.warning.assert_called_once_with('Illegal verse number: (1, 2, 3)') @@ -236,14 +237,13 @@ class TestOpenSongImport(TestCase, TestMixin): self.assertFalse(importer.parse_chapter_number.called) @patch('openlp.plugins.bibles.lib.importers.opensong.translate', **{'side_effect': lambda x, y: y}) - def process_chapters_completes_test(self, mocked_translate): + @patch('openlp.plugins.bibles.lib.importers.opensong.parse_chapter_number', **{'side_effect': [1, 2]}) + def process_chapters_completes_test(self, mocked_parse_chapter_number, mocked_translate): """ Test process_chapters when it completes """ # GIVEN: An instance of OpenSongBible importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.parse_chapter_number = MagicMock() - importer.parse_chapter_number.side_effect = [1, 2] importer.wizard = MagicMock() # WHEN: called with some valid data @@ -263,7 +263,7 @@ class TestOpenSongImport(TestCase, TestMixin): importer.process_chapters(book, [chapter1, chapter2]) # THEN: parse_chapter_number, process_verses and increment_process_bar should have been called - self.assertEqual(importer.parse_chapter_number.call_args_list, [call('1', 0), call('2', 1)]) + self.assertEqual(mocked_parse_chapter_number.call_args_list, [call('1', 0), call('2', 1)]) self.assertEqual( importer.process_verses.call_args_list, [call(book, 1, ['Chapter1 Verses']), call(book, 2, ['Chapter2 Verses'])]) @@ -285,16 +285,14 @@ class TestOpenSongImport(TestCase, TestMixin): # THEN: importer.parse_verse_number not have been called self.assertFalse(importer.parse_verse_number.called) - def process_verses_completes_test(self): + @patch('openlp.plugins.bibles.lib.importers.opensong.parse_verse_number', **{'side_effect': [1, 2]}) + @patch('openlp.plugins.bibles.lib.importers.opensong.get_text', **{'side_effect': ['Verse1 Text', 'Verse2 Text']}) + def process_verses_completes_test(self, mocked_get_text, mocked_parse_verse_number): """ Test process_verses when it completes """ # GIVEN: An instance of OpenSongBible importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.get_text = MagicMock() - importer.get_text.side_effect = ['Verse1 Text', 'Verse2 Text'] - importer.parse_verse_number = MagicMock() - importer.parse_verse_number.side_effect = [1, 2] importer.wizard = MagicMock() # WHEN: called with some valid data @@ -314,8 +312,8 @@ class TestOpenSongImport(TestCase, TestMixin): importer.process_verses(book, 1, [verse1, verse2]) # THEN: parse_chapter_number, process_verses and increment_process_bar should have been called - self.assertEqual(importer.parse_verse_number.call_args_list, [call('1', 0), call('2', 1)]) - self.assertEqual(importer.get_text.call_args_list, [call(verse1), call(verse2)]) + self.assertEqual(mocked_parse_verse_number.call_args_list, [call('1', 0), call('2', 1)]) + self.assertEqual(mocked_get_text.call_args_list, [call(verse1), call(verse2)]) self.assertEqual( importer.create_verse.call_args_list, [call(1, 1, 1, 'Verse1 Text'), call(1, 1, 2, 'Verse2 Text')]) @@ -327,11 +325,12 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: A mocked is_compressed method which returns True mocked_is_compressed.return_value = True + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') # WHEN: Calling validate_file # THEN: ValidationError should be raised with self.assertRaises(ValidationError) as context: - OpenSongBible.validate_file('file.name') + importer.validate_file('file.name') self.assertEqual(context.exception.msg, 'Compressed file') @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.parse_xml') @@ -342,9 +341,10 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: Some test data with an OpenSong Bible "bible" root tag mocked_parse_xml.return_value = objectify.fromstring('') + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') # WHEN: Calling validate_file - result = OpenSongBible.validate_file('file.name') + result = importer.validate_file('file.name') # THEN: A True should be returned self.assertTrue(result) @@ -358,11 +358,12 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: Some test data with a Zefinia "XMLBIBLE" root tag mocked_parse_xml.return_value = objectify.fromstring('') + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') # WHEN: Calling validate_file # THEN: critical_error_message_box should be called and an ValidationError should be raised with self.assertRaises(ValidationError) as context: - OpenSongBible.validate_file('file.name') + importer.validate_file('file.name') self.assertEqual(context.exception.msg, 'Invalid xml.') mocked_message_box.assert_called_once_with( message='Incorrect Bible file type supplied. This looks like a Zefania XML bible, please use the ' @@ -377,11 +378,12 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: Some test data with an invalid root tag and an instance of OpenSongBible mocked_parse_xml.return_value = objectify.fromstring('') + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') # WHEN: Calling validate_file # THEN: ValidationError should be raised, and the critical error message box should not have been called with self.assertRaises(ValidationError) as context: - OpenSongBible.validate_file('file.name') + importer.validate_file('file.name') self.assertEqual(context.exception.msg, 'Invalid xml.') self.assertFalse(mocked_message_box.called) From 7fba73843b8ac7cb73ef2721356d07bd8d6a4e2c Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Thu, 25 Aug 2016 22:24:59 +0300 Subject: [PATCH 34/65] - Added the missing tag to expection form. (bug 1616441) --- openlp/core/ui/exceptionform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/core/ui/exceptionform.py b/openlp/core/ui/exceptionform.py index 2122acfaa..9e58ac8b2 100644 --- a/openlp/core/ui/exceptionform.py +++ b/openlp/core/ui/exceptionform.py @@ -208,7 +208,7 @@ class ExceptionForm(QtWidgets.QDialog, Ui_ExceptionDialog, RegistryProperties): self.__button_state(False) self.description_word_count.setText( translate('OpenLP.ExceptionDialog', 'Please enter a more detailed description of the situation' - )) + '')) def on_attach_file_button_clicked(self): """ From e91b87da8ceccceb0824cf6820257a2f43e9898b Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sun, 28 Aug 2016 22:03:04 +0100 Subject: [PATCH 35/65] Start on osis importer. Some further improvements of other importers --- .../plugins/bibles/forms/bibleimportform.py | 40 ++- openlp/plugins/bibles/lib/bibleimport.py | 46 ++- .../plugins/bibles/lib/importers/csvbible.py | 36 +- .../plugins/bibles/lib/importers/opensong.py | 27 +- openlp/plugins/bibles/lib/importers/osis.py | 189 ++++++----- .../openlp_plugins/bibles/test_bibleimport.py | 33 +- .../openlp_plugins/bibles/test_csvimport.py | 7 - .../bibles/test_opensongimport.py | 145 +++----- .../openlp_plugins/bibles/test_osisimport.py | 311 +++++++++++------- 9 files changed, 451 insertions(+), 383 deletions(-) diff --git a/openlp/plugins/bibles/forms/bibleimportform.py b/openlp/plugins/bibles/forms/bibleimportform.py index 3d02228ca..85daa786e 100644 --- a/openlp/plugins/bibles/forms/bibleimportform.py +++ b/openlp/plugins/bibles/forms/bibleimportform.py @@ -25,6 +25,7 @@ The bible import functions for OpenLP import logging import os import urllib.error +from lxml import etree from PyQt5 import QtWidgets try: @@ -33,14 +34,15 @@ try: except: PYSWORD_AVAILABLE = False -from openlp.core.common import AppLocation, Settings, UiStrings, translate, clean_filename +from openlp.core.common import AppLocation, Settings, UiStrings, trace_error_handler, translate +from openlp.core.common.languagemanager import get_locale_key from openlp.core.lib.db import delete_database +from openlp.core.lib.exceptions import ValidationError from openlp.core.lib.ui import critical_error_message_box from openlp.core.ui.lib.wizard import OpenLPWizard, WizardStrings -from openlp.core.common.languagemanager import get_locale_key -from openlp.plugins.bibles.lib.manager import BibleFormat from openlp.plugins.bibles.lib.db import clean_filename from openlp.plugins.bibles.lib.importers.http import CWExtract, BGExtract, BSExtract +from openlp.plugins.bibles.lib.manager import BibleFormat log = logging.getLogger(__name__) @@ -809,16 +811,22 @@ class BibleImportForm(OpenLPWizard): sword_path=self.field('sword_zip_path'), sword_key=self.sword_zipbible_combo_box.itemData( self.sword_zipbible_combo_box.currentIndex())) - if importer.do_import(license_version): - self.manager.save_meta_data(license_version, license_version, license_copyright, license_permissions) - self.manager.reload_bibles() - if bible_type == BibleFormat.WebDownload: - self.progress_label.setText( - translate('BiblesPlugin.ImportWizardForm', 'Registered Bible. Please note, that verses will be ' - 'downloaded on demand and thus an internet connection is required.')) - else: - self.progress_label.setText(WizardStrings.FinishedImport) - else: - self.progress_label.setText(translate('BiblesPlugin.ImportWizardForm', 'Your Bible import failed.')) - del self.manager.db_cache[importer.name] - delete_database(self.plugin.settings_section, importer.file) + + try: + if importer.do_import(license_version): + self.manager.save_meta_data(license_version, license_version, license_copyright, license_permissions) + self.manager.reload_bibles() + if bible_type == BibleFormat.WebDownload: + self.progress_label.setText( + translate('BiblesPlugin.ImportWizardForm', 'Registered Bible. Please note, that verses will be ' + 'downloaded on demand and thus an internet connection is required.')) + else: + self.progress_label.setText(WizardStrings.FinishedImport) + return + except (AttributeError, ValidationError, etree.XMLSyntaxError): + log.exception('Importing bible failed') + trace_error_handler(log) + + self.progress_label.setText(translate('BiblesPlugin.ImportWizardForm', 'Your Bible import failed.')) + del self.manager.db_cache[importer.name] + delete_database(self.plugin.settings_section, importer.file) diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index d6cfb83fa..ff95ac287 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -25,8 +25,8 @@ import logging from lxml import etree, objectify from zipfile import is_zipfile -from openlp.core.common import OpenLPMixin, languages -from openlp.core.lib import ValidationError, translate +from openlp.core.common import OpenLPMixin, languages, trace_error_handler, translate +from openlp.core.lib import ValidationError from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB @@ -103,8 +103,7 @@ class BibleImport(OpenLPMixin, BibleDB): 'importing {file}'.format(book_ref=book_ref_id, file=self.filename)) return self.create_book(name, book_ref_id, book_details['testament_id']) - @staticmethod - def parse_xml(filename, use_objectify=False, elements=None, tags=None): + def parse_xml(self, filename, use_objectify=False, elements=None, tags=None): """ Parse and clean the supplied file by removing any elements or tags we don't use. :param filename: The filename of the xml file to parse. Str @@ -113,17 +112,28 @@ class BibleImport(OpenLPMixin, BibleDB): :param tags: A tuple of element names (Str) to remove, preserving their content. :return: The root element of the xml document """ - with open(filename, 'rb') as import_file: - # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding - # detection, and the two mechanisms together interfere with each other. - if not use_objectify: - tree = etree.parse(import_file, parser=etree.XMLParser(recover=True)) - else: - tree = objectify.parse(import_file, parser=objectify.makeparser(recover=True)) - if elements: - # Strip tags we don't use - remove content - etree.strip_elements(tree, elements, with_tail=False) - if tags: - # Strip tags we don't use - keep content - etree.strip_tags(tree, tags) - return tree.getroot() + try: + with open(filename, 'rb') as import_file: + # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding + # detection, and the two mechanisms together interfere with each other. + if not use_objectify: + tree = etree.parse(import_file, parser=etree.XMLParser(recover=True)) + else: + tree = objectify.parse(import_file, parser=objectify.makeparser(recover=True)) + if elements or tags: + self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport', + 'Removing unused tags (this may take a few minutes)...')) + if elements: + # Strip tags we don't use - remove content + etree.strip_elements(tree, elements, with_tail=False) + if tags: + # Strip tags we don't use - keep content + etree.strip_tags(tree, tags) + return tree.getroot() + except OSError as e: + log.exception('Opening {file_name} failed.'.format(file_name=e.filename)) + trace_error_handler(log) + critical_error_message_box( title='An Error Occured When Opening A File', + message='The following error occurred when trying to open\n{file_name}:\n\n{error}' + .format(file_name=e.filename, error=e.strerror)) + return None diff --git a/openlp/plugins/bibles/lib/importers/csvbible.py b/openlp/plugins/bibles/lib/importers/csvbible.py index 3733145b6..4ca546232 100644 --- a/openlp/plugins/bibles/lib/importers/csvbible.py +++ b/openlp/plugins/bibles/lib/importers/csvbible.py @@ -128,7 +128,6 @@ class CSVBible(BibleImport): translate('BiblesPlugin.CSVBible', 'Importing books... {book}').format(book=book.name)) self.find_and_create_book(book.name, number_of_books, self.language_id) book_list.update({int(book.id): book.name}) - self.application.process_events() return book_list def process_verses(self, verses, books): @@ -153,7 +152,6 @@ class CSVBible(BibleImport): self.session.commit() self.create_verse(book.id, verse.chapter_number, verse.number, verse.text) self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible', 'Importing verses... done.')) - self.application.process_events() self.session.commit() def do_import(self, bible_name=None): @@ -163,24 +161,18 @@ class CSVBible(BibleImport): :param bible_name: Optional name of the bible being imported. Str or None :return: True if the import was successful, False if it failed or was cancelled """ - try: - self.language_id = self.get_language(bible_name) - if not self.language_id: - raise ValidationError(msg='Invalid language selected') - books = self.parse_csv_file(self.books_file, Book) - self.wizard.progress_bar.setValue(0) - self.wizard.progress_bar.setMinimum(0) - self.wizard.progress_bar.setMaximum(len(books)) - book_list = self.process_books(books) - if self.stop_import_flag: - return False - verses = self.parse_csv_file(self.verses_file, Verse) - self.wizard.progress_bar.setValue(0) - self.wizard.progress_bar.setMaximum(len(books) + 1) - self.process_verses(verses, book_list) - if self.stop_import_flag: - return False - except ValidationError: - log.exception('Could not import CSV bible') + self.language_id = self.get_language(bible_name) + if not self.language_id: return False - return True + books = self.parse_csv_file(self.books_file, Book) + self.wizard.progress_bar.setValue(0) + self.wizard.progress_bar.setMinimum(0) + self.wizard.progress_bar.setMaximum(len(books)) + book_list = self.process_books(books) + if self.stop_import_flag: + return False + verses = self.parse_csv_file(self.verses_file, Verse) + self.wizard.progress_bar.setValue(0) + self.wizard.progress_bar.setMaximum(len(books) + 1) + self.process_verses(verses, book_list) + return not self.stop_import_flag diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 01be8407e..42b058e48 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -23,7 +23,7 @@ import logging from lxml import etree -from openlp.core.common import translate, trace_error_handler +from openlp.core.common import trace_error_handler, translate from openlp.core.lib.exceptions import ValidationError from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.bibleimport import BibleImport @@ -146,6 +146,8 @@ class OpenSongBible(BibleImport): if BibleImport.is_compressed(filename): raise ValidationError(msg='Compressed file') bible = self.parse_xml(filename, use_objectify=True) + if bible is None: + raise ValidationError(msg='Error when opening file') root_tag = bible.tag.lower() if root_tag != 'bible': if root_tag == 'xmlbible': @@ -165,20 +167,13 @@ class OpenSongBible(BibleImport): :return: True if import completed, False if import was unsuccessful """ log.debug('Starting OpenSong import from "{name}"'.format(name=self.filename)) - try: - self.validate_file(self.filename) - bible = self.parse_xml(self.filename, use_objectify=True) - # Check that we're not trying to import a Zefania XML bible, it is sometimes refered to as 'OpenSong' - # No language info in the opensong format, so ask the user - self.language_id = self.get_language_id(bible_name=self.filename) - if not self.language_id: - return False - self.process_books(bible.b) - self.application.process_events() - except (AttributeError, ValidationError, etree.XMLSyntaxError): - log.exception('Loading Bible from OpenSong file failed') - trace_error_handler(log) + self.validate_file(self.filename) + bible = self.parse_xml(self.filename, use_objectify=True) + if bible is None: return False - if self.stop_import_flag: + # No language info in the opensong format, so ask the user + self.language_id = self.get_language_id(bible_name=self.filename) + if not self.language_id: return False - return True + self.process_books(bible.b) + return not self.stop_import_flag diff --git a/openlp/plugins/bibles/lib/importers/osis.py b/openlp/plugins/bibles/lib/importers/osis.py index db12bb7e9..2a874076d 100644 --- a/openlp/plugins/bibles/lib/importers/osis.py +++ b/openlp/plugins/bibles/lib/importers/osis.py @@ -24,9 +24,9 @@ import logging from lxml import etree from openlp.core.common import translate, trace_error_handler +from openlp.core.lib.exceptions import ValidationError from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.bibleimport import BibleImport -from openlp.plugins.bibles.lib.db import BiblesResourcesDB log = logging.getLogger(__name__) @@ -79,94 +79,119 @@ def replacement(match): return match.group(2).upper() +# Precompile a few xpath-querys +verse_in_chapter = etree.XPath('count(//ns:chapter[1]/ns:verse)', namespaces=NS) +text_in_verse = etree.XPath('count(//ns:verse[1]/text())', namespaces=NS) + + class OSISBible(BibleImport): """ `OSIS `_ Bible format importer class. """ + def process_books(self, bible_data): + """ + + :param bible_data: + :return: + """ + no_of_books = int(bible_data.xpath("count(//ns:div[@type='book'])", namespaces=NS)) + # Find books in the bible + bible_books = bible_data.xpath("//ns:div[@type='book']", namespaces=NS) + for book in bible_books: + if self.stop_import_flag: + break + # Remove div-tags in the book + etree.strip_tags(book, '{http://www.bibletechnologies.net/2003/OSIS/namespace}div') + db_book = self.find_and_create_book(book.get('osisID'), no_of_books, self.language_id) + self.process_chapters_and_verses(db_book, book) + self.session.commit() + + def process_chapters_and_verses(self, book, chapters): + """ + + :param book: + :param chapters: + :return: + """ + # Find out if chapter-tags contains the verses, or if it is used as milestone/anchor + if int(verse_in_chapter(chapters)) > 0: + # The chapter tags contains the verses + for chapter in chapters: + chapter_number = chapter.get("osisID").split('.')[1] + # Find out if verse-tags contains the text, or if it is used as milestone/anchor + if int(text_in_verse(chapter)) == 0: + # verse-tags are used as milestone + for verse in chapter: + # If this tag marks the start of a verse, the verse text is between this tag and + # the next tag, which the "tail" attribute gives us. + if verse.get('sID'): + verse_number = verse.get("osisID").split('.')[2] + verse_text = verse.tail + if verse_text: + self.create_verse(book.id, chapter_number, verse_number, verse_text.strip()) + else: + # Verse-tags contains the text + for verse in chapter: + verse_number = verse.get("osisID").split('.')[2] + if verse.text: + self.create_verse(book.id, chapter_number, verse_number, verse.text.strip()) + self.wizard.increment_progress_bar( + translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...') % + {'bookname': book.name, 'chapter': chapter_number}) + else: + # The chapter tags is used as milestones. For now we assume verses is also milestones + chapter_number = 0 + for element in chapters: + if element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' \ + and element.get('sID'): + chapter_number = element.get("osisID").split('.')[1] + self.wizard.increment_progress_bar( + translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...') % + {'bookname': book.name, 'chapter': chapter_number}) + elif element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}verse' \ + and element.get('sID'): + # If this tag marks the start of a verse, the verse text is between this tag and + # the next tag, which the "tail" attribute gives us. + verse_number = element.get("osisID").split('.')[2] + verse_text = element.tail + if verse_text: + self.create_verse(book.id, chapter_number, verse_number, verse_text.strip()) + + def validate_file(self, filename): + """ + Validate the supplied file + + :param filename: The supplied file + :return: True if valid. ValidationError is raised otherwise. + """ + if BibleImport.is_compressed(filename): + raise ValidationError(msg='Compressed file') + bible = self.parse_xml(filename, use_objectify=True) + if bible is None: + raise ValidationError(msg='Error when opening file') + root_tag = bible.tag + tag_str = '{{{name_space}}}osis'.format(name_space=NS['ns']) + if root_tag != tag_str: + critical_error_message_box( + message=translate('BiblesPlugin.OpenSongImport', + 'Incorrect Bible file type supplied. This looks like a Zefania XML bible, ' + 'please use the Zefania import option.')) + raise ValidationError(msg='Invalid xml.') + return True + def do_import(self, bible_name=None): """ Loads a Bible from file. """ log.debug('Starting OSIS import from "{name}"'.format(name=self.filename)) - success = True - try: - self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport', - 'Removing unused tags (this may take a few minutes)...')) - osis_bible_tree = self.parse_xml(self.filename, elements=REMOVABLE_ELEMENTS, tags=REMOVABLE_TAGS) - # Find bible language] - language = osis_bible_tree.xpath("//ns:osisText/@xml:lang", namespaces=NS) - language_id = self.get_language_id(language[0] if language else None, bible_name=self.filename) - if not language_id: - return False - no_of_books = int(osis_bible_tree.xpath("count(//ns:div[@type='book'])", namespaces=NS)) - # Precompile a few xpath-querys - verse_in_chapter = etree.XPath('count(//ns:chapter[1]/ns:verse)', namespaces=NS) - text_in_verse = etree.XPath('count(//ns:verse[1]/text())', namespaces=NS) - # Find books in the bible - bible_books = osis_bible_tree.xpath("//ns:div[@type='book']", namespaces=NS) - for book in bible_books: - if self.stop_import_flag: - break - # Remove div-tags in the book - etree.strip_tags(book, '{http://www.bibletechnologies.net/2003/OSIS/namespace}div') - db_book = self.find_and_create_book(book.get('osisID'), no_of_books, language_id) - # Find out if chapter-tags contains the verses, or if it is used as milestone/anchor - if int(verse_in_chapter(book)) > 0: - # The chapter tags contains the verses - for chapter in book: - chapter_number = chapter.get("osisID").split('.')[1] - # Find out if verse-tags contains the text, or if it is used as milestone/anchor - if int(text_in_verse(chapter)) == 0: - # verse-tags are used as milestone - for verse in chapter: - # If this tag marks the start of a verse, the verse text is between this tag and - # the next tag, which the "tail" attribute gives us. - if verse.get('sID'): - verse_number = verse.get("osisID").split('.')[2] - verse_text = verse.tail - if verse_text: - self.create_verse(db_book.id, chapter_number, verse_number, verse_text.strip()) - else: - # Verse-tags contains the text - for verse in chapter: - verse_number = verse.get("osisID").split('.')[2] - if verse.text: - self.create_verse(db_book.id, chapter_number, verse_number, verse.text.strip()) - self.wizard.increment_progress_bar( - translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...') % - {'bookname': db_book.name, 'chapter': chapter_number}) - else: - # The chapter tags is used as milestones. For now we assume verses is also milestones - chapter_number = 0 - for element in book: - if element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' \ - and element.get('sID'): - chapter_number = element.get("osisID").split('.')[1] - self.wizard.increment_progress_bar( - translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...') % - {'bookname': db_book.name, 'chapter': chapter_number}) - elif element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}verse' \ - and element.get('sID'): - # If this tag marks the start of a verse, the verse text is between this tag and - # the next tag, which the "tail" attribute gives us. - verse_number = element.get("osisID").split('.')[2] - verse_text = element.tail - if verse_text: - self.create_verse(db_book.id, chapter_number, verse_number, verse_text.strip()) - self.session.commit() - self.application.process_events() - except (ValueError, IOError): - log.exception('Loading bible from OSIS file failed') - trace_error_handler(log) - success = False - except etree.XMLSyntaxError as e: - log.exception('Loading bible from OSIS file failed') - trace_error_handler(log) - success = False - critical_error_message_box(message=translate('BiblesPlugin.OsisImport', - 'The file is not a valid OSIS-XML file:' - '\n{text}').format(text=e.msg)) - if self.stop_import_flag: + self.validate_file(self.filename) + bible = self.parse_xml(self.filename, elements=REMOVABLE_ELEMENTS, tags=REMOVABLE_TAGS) + if bible is None: return False - else: - return success + # Find bible language + language = bible.xpath("//ns:osisText/@xml:lang", namespaces=NS) + self.language_id = self.get_language_id(language[0] if language else None, bible_name=self.filename) + if not self.language_id: + return False + self.process_books(bible) + return not self.stop_import_flag diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index 37c6c3fda..97cfd9efa 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -171,9 +171,12 @@ class TestBibleImport(TestCase): """ Test BibleImport.parse_xml() when called with the use_objectify default value """ - # GIVEN: A sample "file" to parse + # GIVEN: A sample "file" to parse and an instance of BibleImport + instance = BibleImport(MagicMock()) + instance.wizard = MagicMock() + # WHEN: Calling parse_xml - result = BibleImport.parse_xml('file.tst') + result = instance.parse_xml('file.tst') # THEN: The result returned should contain the correct data, and should be an instance of eetree_Element self.assertEqual(etree.tostring(result), @@ -185,9 +188,12 @@ class TestBibleImport(TestCase): """ Test BibleImport.parse_xml() when called with use_objectify set to True """ - # GIVEN: A sample "file" to parse + # GIVEN: A sample "file" to parse and an instance of BibleImport + instance = BibleImport(MagicMock()) + instance.wizard = MagicMock() + # WHEN: Calling parse_xml - result = BibleImport.parse_xml('file.tst', use_objectify=True) + result = instance.parse_xml('file.tst', use_objectify=True) # THEN: The result returned should contain the correct data, and should be an instance of ObjectifiedElement self.assertEqual(etree.tostring(result), @@ -199,11 +205,13 @@ class TestBibleImport(TestCase): """ Test BibleImport.parse_xml() when given a tuple of elements to remove """ - # GIVEN: A tuple of elements to remove + # GIVEN: A tuple of elements to remove and an instance of BibleImport elements = ('unsupported', 'x', 'y') + instance = BibleImport(MagicMock()) + instance.wizard = MagicMock() # WHEN: Calling parse_xml, with a test file - result = BibleImport.parse_xml('file.tst', elements=elements) + result = instance.parse_xml('file.tst', elements=elements) # THEN: The result returned should contain the correct data self.assertEqual(etree.tostring(result), @@ -213,11 +221,14 @@ class TestBibleImport(TestCase): """ Test BibleImport.parse_xml() when given a tuple of tags to remove """ - # GIVEN: A tuple of tags to remove + # GIVEN: A tuple of tags to remove and an instance of BibleImport tags = ('div', 'p', 'a') + instance = BibleImport(MagicMock()) + instance.wizard = MagicMock() + # WHEN: Calling parse_xml, with a test file - result = BibleImport.parse_xml('file.tst', tags=tags) + result = instance.parse_xml('file.tst', tags=tags) # THEN: The result returned should contain the correct data self.assertEqual(etree.tostring(result), b'\n Testdatatokeep\n Test' @@ -227,12 +238,14 @@ class TestBibleImport(TestCase): """ Test BibleImport.parse_xml() when given a tuple of elements and of tags to remove """ - # GIVEN: A tuple of elements and of tags to remove + # GIVEN: A tuple of elements and of tags to remove and an instacne of BibleImport elements = ('unsupported', 'x', 'y') tags = ('div', 'p', 'a') + instance = BibleImport(MagicMock()) + instance.wizard = MagicMock() # WHEN: Calling parse_xml, with a test file - result = BibleImport.parse_xml('file.tst', elements=elements, tags=tags) + result = instance.parse_xml('file.tst', elements=elements, tags=tags) # THEN: The result returned should contain the correct data self.assertEqual(etree.tostring(result), b'\n Testdatatokeep\n \n') diff --git a/tests/functional/openlp_plugins/bibles/test_csvimport.py b/tests/functional/openlp_plugins/bibles/test_csvimport.py index f6d3697af..50fe17884 100644 --- a/tests/functional/openlp_plugins/bibles/test_csvimport.py +++ b/tests/functional/openlp_plugins/bibles/test_csvimport.py @@ -207,7 +207,6 @@ class TestCSVImport(TestCase): with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\ patch('openlp.plugins.bibles.lib.importers.csvbible.translate'): importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv') - type(importer).application = PropertyMock() importer.find_and_create_book = MagicMock() importer.language_id = 10 importer.stop_import_flag = False @@ -222,7 +221,6 @@ class TestCSVImport(TestCase): # The returned data should be a dictionary with both song's id and names. self.assertEqual(importer.find_and_create_book.mock_calls, [call('1. Mosebog', 2, 10), call('2. Mosebog', 2, 10)]) - importer.application.process_events.assert_called_once_with() self.assertDictEqual(result, {1: '1. Mosebog', 2: '2. Mosebog'}) def process_verses_stopped_import_test(self): @@ -233,7 +231,6 @@ class TestCSVImport(TestCase): mocked_manager = MagicMock() with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'): importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv') - type(importer).application = PropertyMock() importer.get_book_name = MagicMock() importer.session = MagicMock() importer.stop_import_flag = True @@ -245,7 +242,6 @@ class TestCSVImport(TestCase): # THEN: get_book_name should not be called and the return value should be None self.assertFalse(importer.get_book_name.called) importer.wizard.increment_progress_bar.assert_called_once_with('Importing verses... done.') - importer.application.process_events.assert_called_once_with() self.assertIsNone(result) def process_verses_successful_test(self): @@ -257,7 +253,6 @@ class TestCSVImport(TestCase): with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\ patch('openlp.plugins.bibles.lib.importers.csvbible.translate'): importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv') - type(importer).application = PropertyMock() importer.create_verse = MagicMock() importer.get_book = MagicMock(return_value=Book('1', '1', '1. Mosebog', '1Mos')) importer.get_book_name = MagicMock(return_value='1. Mosebog') @@ -280,7 +275,6 @@ class TestCSVImport(TestCase): [call('1', 1, 1, 'I Begyndelsen skabte Gud Himmelen og Jorden.'), call('1', 1, 2, 'Og Jorden var øde og tom, og der var Mørke over Verdensdybet. ' 'Men Guds Ånd svævede over Vandene.')]) - importer.application.process_events.assert_called_once_with() def do_import_invalid_language_id_test(self): """ @@ -299,7 +293,6 @@ class TestCSVImport(TestCase): # THEN: The log.exception method should have been called to show that it reached the except clause. # False should be returned. importer.get_language.assert_called_once_with('Bible Name') - mocked_log.exception.assert_called_once_with('Could not import CSV bible') self.assertFalse(result) def do_import_stop_import_test(self): diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index 0f5c404ac..ead118fd9 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -387,125 +387,82 @@ class TestOpenSongImport(TestCase, TestMixin): self.assertEqual(context.exception.msg, 'Invalid xml.') self.assertFalse(mocked_message_box.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - @patch('openlp.plugins.bibles.lib.importers.opensong.trace_error_handler') - def do_import_attribute_error_test(self, mocked_trace_error_handler, mocked_log): + def do_import_parse_xml_fails_test(self): """ - Test do_import when an AttributeError exception is raised + Test do_import when parse_xml fails (returns None) """ - # GIVEN: An instance of OpenSongBible and a mocked validate_file which raises an AttributeError - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.validate_file = MagicMock(**{'side_effect': AttributeError()}) - importer.parse_xml = MagicMock() + # GIVEN: An instance of OpenSongBible and a mocked parse_xml which returns False + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OpenSongBible, 'validate_file'), \ + patch.object(OpenSongBible, 'parse_xml', return_value=None), \ + patch.object(OpenSongBible, 'get_language_id') as mocked_language_id: + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - # WHEN: Calling do_import - result = importer.do_import() + # WHEN: Calling do_import + result = importer.do_import() - # THEN: do_import should return False after logging the exception - mocked_log.exception.assert_called_once_with('Loading Bible from OpenSong file failed') - mocked_trace_error_handler.assert_called_once_with(mocked_log) - self.assertFalse(result) - self.assertFalse(importer.parse_xml.called) + # THEN: do_import should return False and get_language_id should have not been called + self.assertFalse(result) + self.assertFalse(mocked_language_id.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - @patch('openlp.plugins.bibles.lib.importers.opensong.trace_error_handler') - def do_import_validation_error_test(self, mocked_trace_error_handler, mocked_log): - """ - Test do_import when an ValidationError exception is raised - """ - # GIVEN: An instance of OpenSongBible and a mocked validate_file which raises an ValidationError - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.validate_file = MagicMock(**{'side_effect': ValidationError()}) - importer.parse_xml = MagicMock() - - # WHEN: Calling do_import - result = importer.do_import() - - # THEN: do_import should return False after logging the exception. parse_xml should not be called. - mocked_log.exception.assert_called_once_with('Loading Bible from OpenSong file failed') - mocked_trace_error_handler.assert_called_once_with(mocked_log) - self.assertFalse(result) - self.assertFalse(importer.parse_xml.called) - - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - @patch('openlp.plugins.bibles.lib.importers.opensong.trace_error_handler') - def do_import_xml_syntax_error_test(self, mocked_trace_error_handler, mocked_log): - """ - Test do_import when an etree.XMLSyntaxError exception is raised - """ - # GIVEN: An instance of OpenSongBible and a mocked validate_file which raises an etree.XMLSyntaxError - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.validate_file = MagicMock(**{'side_effect': etree.XMLSyntaxError(None, None, None, None)}) - importer.parse_xml = MagicMock() - - # WHEN: Calling do_import - result = importer.do_import() - - # THEN: do_import should return False after logging the exception. parse_xml should not be called. - mocked_log.exception.assert_called_once_with('Loading Bible from OpenSong file failed') - mocked_trace_error_handler.assert_called_once_with(mocked_log) - self.assertFalse(result) - self.assertFalse(importer.parse_xml.called) - - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - def do_import_no_language_test(self, mocked_log): + def do_import_no_language_test(self): """ Test do_import when the user cancels the language selection dialog """ # GIVEN: An instance of OpenSongBible and a mocked get_language which returns False - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.validate_file = MagicMock() - importer.parse_xml = MagicMock() - importer.get_language_id = MagicMock(**{'return_value': False}) - importer.process_books = MagicMock() + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OpenSongBible, 'validate_file'), \ + patch.object(OpenSongBible, 'parse_xml'), \ + patch.object(OpenSongBible, 'get_language_id', return_value=False), \ + patch.object(OpenSongBible, 'process_books') as mocked_process_books: + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - # WHEN: Calling do_import - result = importer.do_import() + # WHEN: Calling do_import + result = importer.do_import() - # THEN: do_import should return False and process_books should have not been called - self.assertFalse(result) - self.assertFalse(importer.process_books.called) + # THEN: do_import should return False and process_books should have not been called + self.assertFalse(result) + self.assertFalse(mocked_process_books.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - def do_import_stop_import_test(self, mocked_log): + def do_import_stop_import_test(self): """ Test do_import when the stop_import_flag is set to True """ # GIVEN: An instance of OpenSongBible and stop_import_flag set to True - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.validate_file = MagicMock() - importer.parse_xml = MagicMock() - importer.get_language_id = MagicMock(**{'return_value': 10}) - importer.process_books = MagicMock() - importer.stop_import_flag = True + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OpenSongBible, 'validate_file'), \ + patch.object(OpenSongBible, 'parse_xml'), \ + patch.object(OpenSongBible, 'get_language_id', return_value=10), \ + patch.object(OpenSongBible, 'process_books') as mocked_process_books: - # WHEN: Calling do_import - result = importer.do_import() + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.stop_import_flag = True - # THEN: do_import should return False and process_books should have not been called - self.assertFalse(result) - self.assertTrue(importer.application.process_events.called) + # WHEN: Calling do_import + result = importer.do_import() - self.assertTrue(importer.application.process_events.called) + # THEN: do_import should return False and process_books should have not been called + self.assertFalse(result) + self.assertTrue(mocked_process_books.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - def do_import_completes_test(self, mocked_log): + def do_import_completes_test(self): """ Test do_import when it completes successfully """ - # GIVEN: An instance of OpenSongBible and stop_import_flag set to True - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.validate_file = MagicMock() - importer.parse_xml = MagicMock() - importer.get_language_id = MagicMock(**{'return_value': 10}) - importer.process_books = MagicMock() - importer.stop_import_flag = False + # GIVEN: An instance of OpenSongBible and stop_import_flag set to False + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OpenSongBible, 'validate_file'), \ + patch.object(OpenSongBible, 'parse_xml'), \ + patch.object(OpenSongBible, 'get_language_id', return_value=10), \ + patch.object(OpenSongBible, 'process_books'): + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.stop_import_flag = False - # WHEN: Calling do_import - result = importer.do_import() + # WHEN: Calling do_import + result = importer.do_import() - # THEN: do_import should return True - self.assertTrue(result) + # THEN: do_import should return True + self.assertTrue(result) def test_file_import(self): """ diff --git a/tests/functional/openlp_plugins/bibles/test_osisimport.py b/tests/functional/openlp_plugins/bibles/test_osisimport.py index e1700fbb7..612a63b52 100644 --- a/tests/functional/openlp_plugins/bibles/test_osisimport.py +++ b/tests/functional/openlp_plugins/bibles/test_osisimport.py @@ -42,143 +42,218 @@ class TestOsisImport(TestCase): def setUp(self): self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') + self.addCleanup(self.manager_patcher.stop) self.manager_patcher.start() - def tearDown(self): - self.registry_patcher.stop() - self.manager_patcher.stop() - - def test_create_importer(self): - """ - Test creating an instance of the OSIS file importer - """ - # GIVEN: A mocked out "manager" - mocked_manager = MagicMock() - - # WHEN: An importer object is created - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - - # THEN: The importer should be an instance of BibleDB - self.assertIsInstance(importer, BibleDB) - - def test_file_import_nested_tags(self): - """ - Test the actual import of OSIS Bible file, with nested chapter and verse tags - """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-dk1933.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + def test_create_importer(self): + """ + Test creating an instance of the OSIS file importer + """ + # GIVEN: A mocked out "manager" mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() + + # WHEN: An importer object is created importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'Danish' - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # THEN: The importer should be an instance of BibleDB + self.assertIsInstance(importer, BibleDB) - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) - - def test_file_import_mixed_tags(self): + def do_import_parse_xml_fails_test(self): """ - Test the actual import of OSIS Bible file, with chapter tags containing milestone verse tags. + Test do_import when parse_xml fails (returns None) """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'kjv.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-kjv.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'English' + # GIVEN: An instance of OpenSongBible and a mocked parse_xml which returns False + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OSISBible, 'validate_file'), \ + patch.object(OSISBible, 'parse_xml', return_value=None), \ + patch.object(OSISBible, 'get_language_id') as mocked_language_id: + importer = OSISBible(MagicMock(), path='.', name='.', filename='') - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # WHEN: Calling do_import + result = importer.do_import() - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + # THEN: do_import should return False and get_language_id should have not been called + self.assertFalse(result) + self.assertFalse(mocked_language_id.called) - def test_file_import_milestone_tags(self): + def do_import_no_language_test(self): """ - Test the actual import of OSIS Bible file, with milestone chapter and verse tags. + Test do_import when the user cancels the language selection dialog """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'web.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-web.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'English' + # GIVEN: An instance of OpenSongBible and a mocked get_language which returns False + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OSISBible, 'validate_file'), \ + patch.object(OSISBible, 'parse_xml'), \ + patch.object(OSISBible, 'get_language_id', **{'return_value': False}), \ + patch.object(OSISBible, 'process_books') as mocked_process_books: + importer = OSISBible(MagicMock(), path='.', name='.', filename='') - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # WHEN: Calling do_import + result = importer.do_import() - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + # THEN: do_import should return False and process_books should have not been called + self.assertFalse(result) + self.assertFalse(mocked_process_books.called) - def test_file_import_empty_verse_tags(self): + def do_import_stop_import_test(self): """ - Test the actual import of OSIS Bible file, with an empty verse tags. + Test do_import when the stop_import_flag is set to True """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-dk1933-empty-verse.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'Danish' + # GIVEN: An instance of OpenSongBible and stop_import_flag set to True + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OSISBible, 'validate_file'), \ + patch.object(OSISBible, 'parse_xml'), \ + patch.object(OSISBible, 'get_language_id', **{'return_value': 10}), \ + patch.object(OSISBible, 'process_books'): + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + importer.stop_import_flag = True - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # WHEN: Calling do_import + result = importer.do_import() - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + # THEN: do_import should return False and process_books should have not been called + self.assertFalse(result) + self.assertTrue(importer.process_books.called) + + @patch('openlp.plugins.bibles.lib.importers.opensong.log') + def do_import_completes_test(self, mocked_log): + """ + Test do_import when it completes successfully + """ + # GIVEN: An instance of OpenSongBible and stop_import_flag set to True + with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + patch.object(OSISBible, 'validate_file'), \ + patch.object(OSISBible, 'parse_xml'), \ + patch.object(OSISBible, 'get_language_id', **{'return_value': 10}), \ + patch.object(OSISBible, 'process_books'): + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + importer.stop_import_flag = False + + # WHEN: Calling do_import + result = importer.do_import() + + # THEN: do_import should return True + self.assertTrue(result) + + def test_file_import_nested_tags(self): + """ + Test the actual import of OSIS Bible file, with nested chapter and verse tags + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-dk1933.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'Danish' + + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() + + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + + def test_file_import_mixed_tags(self): + """ + Test the actual import of OSIS Bible file, with chapter tags containing milestone verse tags. + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'kjv.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-kjv.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'English' + + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() + + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + + def test_file_import_milestone_tags(self): + """ + Test the actual import of OSIS Bible file, with milestone chapter and verse tags. + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'web.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-web.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'English' + + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() + + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + + def test_file_import_empty_verse_tags(self): + """ + Test the actual import of OSIS Bible file, with an empty verse tags. + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-dk1933-empty-verse.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'Danish' + + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() + + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) From 628027ffc8f52f8e6f396c7598b34ba3c20fdf20 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Mon, 29 Aug 2016 17:11:09 +0100 Subject: [PATCH 36/65] Moved validate_xml in to bible import and improved operation. More tests --- .../plugins/bibles/forms/bibleimportform.py | 2 +- openlp/plugins/bibles/lib/bibleimport.py | 33 ++- .../plugins/bibles/lib/importers/csvbible.py | 6 +- .../plugins/bibles/lib/importers/opensong.py | 27 +-- openlp/plugins/bibles/lib/importers/osis.py | 26 +-- .../openlp_plugins/bibles/test_bibleimport.py | 221 +++++++++++++++++- .../openlp_plugins/bibles/test_csvimport.py | 58 +---- .../bibles/test_opensongimport.py | 153 +++--------- .../openlp_plugins/bibles/test_osisimport.py | 29 +-- 9 files changed, 286 insertions(+), 269 deletions(-) diff --git a/openlp/plugins/bibles/forms/bibleimportform.py b/openlp/plugins/bibles/forms/bibleimportform.py index 85daa786e..e1e062155 100644 --- a/openlp/plugins/bibles/forms/bibleimportform.py +++ b/openlp/plugins/bibles/forms/bibleimportform.py @@ -813,7 +813,7 @@ class BibleImportForm(OpenLPWizard): self.sword_zipbible_combo_box.currentIndex())) try: - if importer.do_import(license_version): + if importer.do_import(license_version) and not importer.stop_import_flag: self.manager.save_meta_data(license_version, license_version, license_copyright, license_permissions) self.manager.reload_bibles() if bible_type == BibleFormat.WebDownload: diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index ff95ac287..cd9f4d37a 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -134,6 +134,35 @@ class BibleImport(OpenLPMixin, BibleDB): log.exception('Opening {file_name} failed.'.format(file_name=e.filename)) trace_error_handler(log) critical_error_message_box( title='An Error Occured When Opening A File', - message='The following error occurred when trying to open\n{file_name}:\n\n{error}' - .format(file_name=e.filename, error=e.strerror)) + message='The following error occurred when trying to open\n{file_name}:\n\n{error}'.format(file_name=e.filename, error=e.strerror)) return None + + def validate_xml_file(self, filename, tag): + """ + Validate the supplied file + + :param filename: The supplied file + :return: True if valid. ValidationError is raised otherwise. + """ + if BibleImport.is_compressed(filename): + raise ValidationError(msg='Compressed file') + bible = self.parse_xml(filename, use_objectify=True) + if bible is None: + raise ValidationError(msg='Error when opening file') + root_tag = bible.tag.lower() + bible_type = translate('BiblesPlugin.BibleImport', 'unknown type of', + 'This looks like an unknown type of XML bible.') + if root_tag == tag: + return True + elif root_tag == 'bible': + bible_type = "OpenSong" + elif root_tag == '{http://www.bibletechnologies.net/2003/osis/namespace}osis': + bible_type = 'OSIS' + elif root_tag == 'xmlbible': + bible_type = 'Zefania' + critical_error_message_box( + message=translate('BiblesPlugin.BibleImport', + 'Incorrect Bible file type supplied. This looks like an {bible_type} XML bible.' + .format(bible_type=bible_type))) + raise ValidationError(msg='Invalid xml.') + diff --git a/openlp/plugins/bibles/lib/importers/csvbible.py b/openlp/plugins/bibles/lib/importers/csvbible.py index 4ca546232..bdb903ddb 100644 --- a/openlp/plugins/bibles/lib/importers/csvbible.py +++ b/openlp/plugins/bibles/lib/importers/csvbible.py @@ -123,7 +123,7 @@ class CSVBible(BibleImport): number_of_books = len(books) for book in books: if self.stop_import_flag: - return None + break self.wizard.increment_progress_bar( translate('BiblesPlugin.CSVBible', 'Importing books... {book}').format(book=book.name)) self.find_and_create_book(book.name, number_of_books, self.language_id) @@ -169,10 +169,8 @@ class CSVBible(BibleImport): self.wizard.progress_bar.setMinimum(0) self.wizard.progress_bar.setMaximum(len(books)) book_list = self.process_books(books) - if self.stop_import_flag: - return False verses = self.parse_csv_file(self.verses_file, Verse) self.wizard.progress_bar.setValue(0) self.wizard.progress_bar.setMaximum(len(books) + 1) self.process_verses(verses, book_list) - return not self.stop_import_flag + return True diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 42b058e48..3afa160fa 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -136,29 +136,6 @@ class OpenSongBible(BibleImport): verse_number = parse_verse_number(verse.attrib['n'], verse_number) self.create_verse(book.id, chapter_number, verse_number, get_text(verse)) - def validate_file(self, filename): - """ - Validate the supplied file - - :param filename: The supplied file - :return: True if valid. ValidationError is raised otherwise. - """ - if BibleImport.is_compressed(filename): - raise ValidationError(msg='Compressed file') - bible = self.parse_xml(filename, use_objectify=True) - if bible is None: - raise ValidationError(msg='Error when opening file') - root_tag = bible.tag.lower() - if root_tag != 'bible': - if root_tag == 'xmlbible': - # Zefania bibles have a root tag of XMLBIBLE". Sometimes these bibles are referred to as 'OpenSong' - critical_error_message_box( - message=translate('BiblesPlugin.OpenSongImport', - 'Incorrect Bible file type supplied. This looks like a Zefania XML bible, ' - 'please use the Zefania import option.')) - raise ValidationError(msg='Invalid xml.') - return True - def do_import(self, bible_name=None): """ Loads an Open Song Bible from a file. @@ -167,7 +144,7 @@ class OpenSongBible(BibleImport): :return: True if import completed, False if import was unsuccessful """ log.debug('Starting OpenSong import from "{name}"'.format(name=self.filename)) - self.validate_file(self.filename) + self.validate_xml_file(self.filename, 'bible') bible = self.parse_xml(self.filename, use_objectify=True) if bible is None: return False @@ -176,4 +153,4 @@ class OpenSongBible(BibleImport): if not self.language_id: return False self.process_books(bible.b) - return not self.stop_import_flag + return True diff --git a/openlp/plugins/bibles/lib/importers/osis.py b/openlp/plugins/bibles/lib/importers/osis.py index 2a874076d..b4bb18abd 100644 --- a/openlp/plugins/bibles/lib/importers/osis.py +++ b/openlp/plugins/bibles/lib/importers/osis.py @@ -157,34 +157,12 @@ class OSISBible(BibleImport): if verse_text: self.create_verse(book.id, chapter_number, verse_number, verse_text.strip()) - def validate_file(self, filename): - """ - Validate the supplied file - - :param filename: The supplied file - :return: True if valid. ValidationError is raised otherwise. - """ - if BibleImport.is_compressed(filename): - raise ValidationError(msg='Compressed file') - bible = self.parse_xml(filename, use_objectify=True) - if bible is None: - raise ValidationError(msg='Error when opening file') - root_tag = bible.tag - tag_str = '{{{name_space}}}osis'.format(name_space=NS['ns']) - if root_tag != tag_str: - critical_error_message_box( - message=translate('BiblesPlugin.OpenSongImport', - 'Incorrect Bible file type supplied. This looks like a Zefania XML bible, ' - 'please use the Zefania import option.')) - raise ValidationError(msg='Invalid xml.') - return True - def do_import(self, bible_name=None): """ Loads a Bible from file. """ log.debug('Starting OSIS import from "{name}"'.format(name=self.filename)) - self.validate_file(self.filename) + self.validate_xml_file(self.filename, '{http://www.bibletechnologies.net/2003/OSIS/namespace}osis') bible = self.parse_xml(self.filename, elements=REMOVABLE_ELEMENTS, tags=REMOVABLE_TAGS) if bible is None: return False @@ -194,4 +172,4 @@ class OSISBible(BibleImport): if not self.language_id: return False self.process_books(bible) - return not self.stop_import_flag + return True diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index 97cfd9efa..de7c935a0 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -41,22 +41,33 @@ class TestBibleImport(TestCase): """ def setUp(self): - test_file = BytesIO( + self.test_file = BytesIO( b'\n' b'\n' b'
Test

data

tokeep
\n' b' Testdatatodiscard\n' b'' ) - self.file_patcher = patch('builtins.open', return_value=test_file) - self.addCleanup(self.file_patcher.stop) - self.file_patcher.start() + self.open_patcher = patch('builtins.open') + self.addCleanup(self.open_patcher.stop) + self.mocked_open = self.open_patcher.start() self.log_patcher = patch('openlp.plugins.bibles.lib.bibleimport.log') self.addCleanup(self.log_patcher.stop) - self.mock_log = self.log_patcher.start() + self.mocked_log = self.log_patcher.start() + self.critical_error_message_box_patcher = \ + patch('openlp.plugins.bibles.lib.bibleimport.critical_error_message_box') + self.addCleanup(self.critical_error_message_box_patcher.stop) + self.mocked_critical_error_message_box = self.critical_error_message_box_patcher.start() self.setup_patcher = patch('openlp.plugins.bibles.lib.db.BibleDB._setup') self.addCleanup(self.setup_patcher.stop) self.setup_patcher.start() + self.trace_error_handler_patcher = patch('openlp.plugins.bibles.lib.bibleimport.trace_error_handler') + self.addCleanup(self.trace_error_handler_patcher.stop) + self.mocked_trace_error_handler = self.trace_error_handler_patcher.start() + self.translate_patcher = patch('openlp.plugins.bibles.lib.bibleimport.translate', + side_effect=lambda module, string_to_translate, *args: string_to_translate) + self.addCleanup(self.translate_patcher.stop) + self.mocked_translate = self.translate_patcher.start() def init_kwargs_none_test(self): """ @@ -130,7 +141,7 @@ class TestBibleImport(TestCase): # language id. with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \ patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=40) as mocked_db_get_language: - self.mock_log.error.reset_mock() + self.mocked_log.error.reset_mock() instance = BibleImport(MagicMock()) instance.save_meta = MagicMock() @@ -140,7 +151,7 @@ class TestBibleImport(TestCase): # THEN: The id of the language returned from BibleDB.get_language should be returned mocked_languages_get_language.assert_called_once_with('English') mocked_db_get_language.assert_called_once_with('KJV') - self.assertFalse(self.mock_log.error.called) + self.assertFalse(self.mocked_log.error.called) instance.save_meta.assert_called_once_with('language_id', 40) self.assertEqual(result, 40) @@ -152,7 +163,7 @@ class TestBibleImport(TestCase): # language id. with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \ patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=None) as mocked_db_get_language: - self.mock_log.error.reset_mock() + self.mocked_log.error.reset_mock() instance = BibleImport(MagicMock()) instance.save_meta = MagicMock() @@ -162,7 +173,7 @@ class TestBibleImport(TestCase): # THEN: None should be returned and an error should be logged mocked_languages_get_language.assert_called_once_with('Qwerty') mocked_db_get_language.assert_called_once_with('KJV') - self.mock_log.error.assert_called_once_with('Language detection failed when importing from "KJV". ' + self.mocked_log.error.assert_called_once_with('Language detection failed when importing from "KJV". ' 'User aborted language selection.') self.assertFalse(instance.save_meta.called) self.assertIsNone(result) @@ -172,6 +183,7 @@ class TestBibleImport(TestCase): Test BibleImport.parse_xml() when called with the use_objectify default value """ # GIVEN: A sample "file" to parse and an instance of BibleImport + self.mocked_open.return_value = self.test_file instance = BibleImport(MagicMock()) instance.wizard = MagicMock() @@ -189,6 +201,7 @@ class TestBibleImport(TestCase): Test BibleImport.parse_xml() when called with use_objectify set to True """ # GIVEN: A sample "file" to parse and an instance of BibleImport + self.mocked_open.return_value = self.test_file instance = BibleImport(MagicMock()) instance.wizard = MagicMock() @@ -206,6 +219,7 @@ class TestBibleImport(TestCase): Test BibleImport.parse_xml() when given a tuple of elements to remove """ # GIVEN: A tuple of elements to remove and an instance of BibleImport + self.mocked_open.return_value = self.test_file elements = ('unsupported', 'x', 'y') instance = BibleImport(MagicMock()) instance.wizard = MagicMock() @@ -222,11 +236,11 @@ class TestBibleImport(TestCase): Test BibleImport.parse_xml() when given a tuple of tags to remove """ # GIVEN: A tuple of tags to remove and an instance of BibleImport + self.mocked_open.return_value = self.test_file tags = ('div', 'p', 'a') instance = BibleImport(MagicMock()) instance.wizard = MagicMock() - # WHEN: Calling parse_xml, with a test file result = instance.parse_xml('file.tst', tags=tags) @@ -239,6 +253,7 @@ class TestBibleImport(TestCase): Test BibleImport.parse_xml() when given a tuple of elements and of tags to remove """ # GIVEN: A tuple of elements and of tags to remove and an instacne of BibleImport + self.mocked_open.return_value = self.test_file elements = ('unsupported', 'x', 'y') tags = ('div', 'p', 'a') instance = BibleImport(MagicMock()) @@ -249,3 +264,189 @@ class TestBibleImport(TestCase): # THEN: The result returned should contain the correct data self.assertEqual(etree.tostring(result), b'\n Testdatatokeep\n \n') + + def parse_xml_file_file_not_found_exception_test(self): + """ + Test that validate_xml_file raises a ValidationError with an OpenSong root tag + """ + # GIVEN: A mocked open which raises a FileNotFoundError and an instance of BibleImporter + exception = FileNotFoundError() + exception.filename = 'file.tst' + exception.strerror = 'No such file or directory' + self.mocked_open.side_effect = exception + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling parse_xml + result = importer.parse_xml('file.tst') + + # THEN: parse_xml should have caught the error, informed the user and returned None + self.mocked_log.exception.assert_called_once_with('Opening file.tst failed.') + self.mocked_trace_error_handler.assert_called_once_with(self.mocked_log) + self.mocked_critical_error_message_box.assert_called_once_with( + title='An Error Occured When Opening A File', + message='The following error occurred when trying to open\nfile.tst:\n\nNo such file or directory') + self.assertIsNone(result) + + def parse_xml_file_file_not_found_exception_test(self): + """ + Test that parse_xml handles a FileNotFoundError exception correctly + """ + # GIVEN: A mocked open which raises a FileNotFoundError and an instance of BibleImporter + exception = FileNotFoundError() + exception.filename = 'file.tst' + exception.strerror = 'No such file or directory' + self.mocked_open.side_effect = exception + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling parse_xml + result = importer.parse_xml('file.tst') + + # THEN: parse_xml should have caught the error, informed the user and returned None + self.mocked_log.exception.assert_called_once_with('Opening file.tst failed.') + self.mocked_trace_error_handler.assert_called_once_with(self.mocked_log) + self.mocked_critical_error_message_box.assert_called_once_with( + title='An Error Occured When Opening A File', + message='The following error occurred when trying to open\nfile.tst:\n\nNo such file or directory') + self.assertIsNone(result) + + def parse_xml_file_permission_error_exception_test(self): + """ + Test that parse_xml handles a PermissionError exception correctly + """ + # GIVEN: A mocked open which raises a PermissionError and an instance of BibleImporter + exception = PermissionError() + exception.filename = 'file.tst' + exception.strerror = 'Permission denied' + self.mocked_open.side_effect = exception + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling parse_xml + result = importer.parse_xml('file.tst') + + # THEN: parse_xml should have caught the error, informed the user and returned None + self.mocked_log.exception.assert_called_once_with('Opening file.tst failed.') + self.mocked_trace_error_handler.assert_called_once_with(self.mocked_log) + self.mocked_critical_error_message_box.assert_called_once_with( + title='An Error Occured When Opening A File', + message='The following error occurred when trying to open\nfile.tst:\n\nPermission denied') + self.assertIsNone(result) + + def validate_xml_file_compressed_file_test(self): + """ + Test that validate_xml_file raises a ValidationError when is_compressed returns True + """ + # GIVEN: A mocked parse_xml which returns None + with patch.object(BibleImport, 'is_compressed', return_value=True): + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling is_compressed + # THEN: ValidationError should be raised, with the message 'Compressed file' + with self.assertRaises(ValidationError) as context: + importer.validate_xml_file('file.name', 'xbible') + self.assertEqual(context.exception.msg, 'Compressed file') + + def validate_xml_file_parse_xml_fails_test(self): + """ + Test that validate_xml_file raises a ValidationError when parse_xml returns None + """ + # GIVEN: A mocked parse_xml which returns None + with patch.object(BibleImport, 'parse_xml', return_value=None), \ + patch.object(BibleImport, 'is_compressed', return_value=False): + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling validate_xml_file + # THEN: ValidationError should be raised, with the message 'Error when opening file' + # the user that an OpenSong bible was found + with self.assertRaises(ValidationError) as context: + importer.validate_xml_file('file.name', 'xbible') + self.assertEqual(context.exception.msg, 'Error when opening file') + + def validate_xml_file_success_test(self): + """ + Test that validate_xml_file returns True with valid XML + """ + # GIVEN: Some test data with an OpenSong Bible "bible" root tag + with patch.object(BibleImport, 'parse_xml', return_value=objectify.fromstring('')), \ + patch.object(BibleImport, 'is_compressed', return_value=False): + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling validate_xml_file + result = importer.validate_xml_file('file.name', 'bible') + + # THEN: True should be returned + self.assertTrue(result) + + def validate_xml_file_opensong_root_test(self): + """ + Test that validate_xml_file raises a ValidationError with an OpenSong root tag + """ + # GIVEN: Some test data with an Zefania root tag and an instance of BibleImport + with patch.object(BibleImport, 'parse_xml', return_value=objectify.fromstring('')), \ + patch.object(BibleImport, 'is_compressed', return_value=False): + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling validate_xml_file + # THEN: ValidationError should be raised, and the critical error message box should was called informing + # the user that an OpenSong bible was found + with self.assertRaises(ValidationError) as context: + importer.validate_xml_file('file.name', 'xbible') + self.assertEqual(context.exception.msg, 'Invalid xml.') + self.mocked_critical_error_message_box.assert_called_once_with( + message='Incorrect Bible file type supplied. This looks like an OpenSong XML bible.') + + def validate_xml_file_osis_root_test(self): + """ + Test that validate_xml_file raises a ValidationError with an OSIS root tag + """ + # GIVEN: Some test data with an Zefania root tag and an instance of BibleImport + with patch.object(BibleImport, 'parse_xml', return_value=objectify.fromstring( + '')), \ + patch.object(BibleImport, 'is_compressed', return_value=False): + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling validate_xml_file + # THEN: ValidationError should be raised, and the critical error message box should was called informing + # the user that an OSIS bible was found + with self.assertRaises(ValidationError) as context: + importer.validate_xml_file('file.name', 'xbible') + self.assertEqual(context.exception.msg, 'Invalid xml.') + self.mocked_critical_error_message_box.assert_called_once_with( + message='Incorrect Bible file type supplied. This looks like an OSIS XML bible.') + + def validate_xml_file_zefania_root_test(self): + """ + Test that validate_xml_file raises a ValidationError with an Zefania root tag + """ + # GIVEN: Some test data with an Zefania root tag and an instance of BibleImport + with patch.object(BibleImport, 'parse_xml', + return_value=objectify.fromstring('')), \ + patch.object(BibleImport, 'is_compressed', return_value=False): + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling validate_xml_file + # THEN: ValidationError should be raised, and the critical error message box should was called informing + # the user that an Zefania bible was found + with self.assertRaises(ValidationError) as context: + importer.validate_xml_file('file.name', 'xbible') + self.assertEqual(context.exception.msg, 'Invalid xml.') + self.mocked_critical_error_message_box.assert_called_once_with( + message='Incorrect Bible file type supplied. This looks like an Zefania XML bible.') + + def validate_xml_file_unknown_root_test(self): + """ + Test that validate_xml_file raises a ValidationError with an unknown root tag + """ + # GIVEN: Some test data with an unknown root tag and an instance of BibleImport + with patch.object(BibleImport, 'parse_xml', + return_value=objectify.fromstring('')), \ + patch.object(BibleImport, 'is_compressed', return_value=False): + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling validate_xml_file + # THEN: ValidationError should be raised, and the critical error message box should was called informing + # the user that a unknown xml bible was found + with self.assertRaises(ValidationError) as context: + importer.validate_xml_file('file.name', 'xbible') + self.assertEqual(context.exception.msg, 'Invalid xml.') + self.mocked_critical_error_message_box.assert_called_once_with( + message='Incorrect Bible file type supplied. This looks like an unknown type of XML bible.') diff --git a/tests/functional/openlp_plugins/bibles/test_csvimport.py b/tests/functional/openlp_plugins/bibles/test_csvimport.py index 50fe17884..7fa0cbf8c 100644 --- a/tests/functional/openlp_plugins/bibles/test_csvimport.py +++ b/tests/functional/openlp_plugins/bibles/test_csvimport.py @@ -194,9 +194,9 @@ class TestCSVImport(TestCase): # WHEN: Calling process_books result = importer.process_books(['Book 1']) - # THEN: increment_progress_bar should not be called and the return value should be None + # THEN: increment_progress_bar should not be called and the return value should be an empty dictionary self.assertFalse(importer.wizard.increment_progress_bar.called) - self.assertIsNone(result) + self.assertEqual(result, {}) def process_books_test(self): """ @@ -295,60 +295,6 @@ class TestCSVImport(TestCase): importer.get_language.assert_called_once_with('Bible Name') self.assertFalse(result) - def do_import_stop_import_test(self): - """ - Test do_import when the import is stopped - """ - # GIVEN: An instance of CSVBible with stop_import set to True - mocked_manager = MagicMock() - with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\ - patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log: - importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv') - importer.get_language = MagicMock(return_value=10) - importer.parse_csv_file = MagicMock(return_value=['Book 1', 'Book 2', 'Book 3']) - importer.process_books = MagicMock() - importer.stop_import_flag = True - importer.wizard = MagicMock() - - # WHEN: Calling do_import - result = importer.do_import('Bible Name') - - # THEN: log.exception should not be called, parse_csv_file should only be called once, - # and False should be returned. - self.assertFalse(mocked_log.exception.called) - importer.parse_csv_file.assert_called_once_with('books.csv', Book) - importer.process_books.assert_called_once_with(['Book 1', 'Book 2', 'Book 3']) - self.assertFalse(result) - - def do_import_stop_import_2_test(self): - """ - Test do_import when the import is stopped - """ - # GIVEN: An instance of CSVBible with stop_import which is True the second time of calling - mocked_manager = MagicMock() - with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\ - patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log: - CSVBible.stop_import_flag = PropertyMock(side_effect=[False, True]) - importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verses.csv') - importer.get_language = MagicMock(return_value=10) - importer.parse_csv_file = MagicMock(side_effect=[['Book 1'], ['Verse 1']]) - importer.process_books = MagicMock(return_value=['Book 1']) - importer.process_verses = MagicMock(return_value=['Verse 1']) - importer.wizard = MagicMock() - - # WHEN: Calling do_import - result = importer.do_import('Bible Name') - - # THEN: log.exception should not be called, parse_csv_file should be called twice, - # and False should be returned. - self.assertFalse(mocked_log.exception.called) - self.assertEqual(importer.parse_csv_file.mock_calls, [call('books.csv', Book), call('verses.csv', Verse)]) - importer.process_verses.assert_called_once_with(['Verse 1'], ['Book 1']) - self.assertFalse(result) - - # Cleanup - del CSVBible.stop_import_flag - def do_import_success_test(self): """ Test do_import when the import succeeds diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index ead118fd9..6f3b9c42b 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -318,82 +318,13 @@ class TestOpenSongImport(TestCase, TestMixin): importer.create_verse.call_args_list, [call(1, 1, 1, 'Verse1 Text'), call(1, 1, 2, 'Verse2 Text')]) - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed') - def validate_file_compressed_test(self, mocked_is_compressed): - """ - Test that validate_file raises a ValidationError when supplied with a compressed file - """ - # GIVEN: A mocked is_compressed method which returns True - mocked_is_compressed.return_value = True - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - - # WHEN: Calling validate_file - # THEN: ValidationError should be raised - with self.assertRaises(ValidationError) as context: - importer.validate_file('file.name') - self.assertEqual(context.exception.msg, 'Compressed file') - - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.parse_xml') - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed', **{'return_value': False}) - def validate_file_bible_test(self, mocked_is_compressed, mocked_parse_xml): - """ - Test that validate_file returns True with valid XML - """ - # GIVEN: Some test data with an OpenSong Bible "bible" root tag - mocked_parse_xml.return_value = objectify.fromstring('') - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - - # WHEN: Calling validate_file - result = importer.validate_file('file.name') - - # THEN: A True should be returned - self.assertTrue(result) - - @patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.parse_xml') - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed', **{'return_value': False}) - def validate_file_zefania_root_test(self, mocked_is_compressed, mocked_parse_xml, mocked_message_box): - """ - Test that validate_file raises a ValidationError with a Zefinia root tag - """ - # GIVEN: Some test data with a Zefinia "XMLBIBLE" root tag - mocked_parse_xml.return_value = objectify.fromstring('') - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - - # WHEN: Calling validate_file - # THEN: critical_error_message_box should be called and an ValidationError should be raised - with self.assertRaises(ValidationError) as context: - importer.validate_file('file.name') - self.assertEqual(context.exception.msg, 'Invalid xml.') - mocked_message_box.assert_called_once_with( - message='Incorrect Bible file type supplied. This looks like a Zefania XML bible, please use the ' - 'Zefania import option.') - - @patch('openlp.plugins.bibles.lib.importers.opensong.critical_error_message_box') - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.parse_xml') - @patch('openlp.plugins.bibles.lib.importers.opensong.BibleImport.is_compressed', **{'return_value': False}) - def validate_file_invalid_root_test(self, mocked_is_compressed, mocked_parse_xml, mocked_message_box): - """ - Test that validate_file raises a ValidationError with an invalid root tag - """ - # GIVEN: Some test data with an invalid root tag and an instance of OpenSongBible - mocked_parse_xml.return_value = objectify.fromstring('') - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - - # WHEN: Calling validate_file - # THEN: ValidationError should be raised, and the critical error message box should not have been called - with self.assertRaises(ValidationError) as context: - importer.validate_file('file.name') - self.assertEqual(context.exception.msg, 'Invalid xml.') - self.assertFalse(mocked_message_box.called) - def do_import_parse_xml_fails_test(self): """ Test do_import when parse_xml fails (returns None) """ # GIVEN: An instance of OpenSongBible and a mocked parse_xml which returns False with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OpenSongBible, 'validate_file'), \ + patch.object(OpenSongBible, 'validate_xml_file'), \ patch.object(OpenSongBible, 'parse_xml', return_value=None), \ patch.object(OpenSongBible, 'get_language_id') as mocked_language_id: importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') @@ -411,7 +342,7 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: An instance of OpenSongBible and a mocked get_language which returns False with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OpenSongBible, 'validate_file'), \ + patch.object(OpenSongBible, 'validate_xml_file'), \ patch.object(OpenSongBible, 'parse_xml'), \ patch.object(OpenSongBible, 'get_language_id', return_value=False), \ patch.object(OpenSongBible, 'process_books') as mocked_process_books: @@ -424,39 +355,17 @@ class TestOpenSongImport(TestCase, TestMixin): self.assertFalse(result) self.assertFalse(mocked_process_books.called) - def do_import_stop_import_test(self): - """ - Test do_import when the stop_import_flag is set to True - """ - # GIVEN: An instance of OpenSongBible and stop_import_flag set to True - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OpenSongBible, 'validate_file'), \ - patch.object(OpenSongBible, 'parse_xml'), \ - patch.object(OpenSongBible, 'get_language_id', return_value=10), \ - patch.object(OpenSongBible, 'process_books') as mocked_process_books: - - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.stop_import_flag = True - - # WHEN: Calling do_import - result = importer.do_import() - - # THEN: do_import should return False and process_books should have not been called - self.assertFalse(result) - self.assertTrue(mocked_process_books.called) - def do_import_completes_test(self): """ Test do_import when it completes successfully """ - # GIVEN: An instance of OpenSongBible and stop_import_flag set to False + # GIVEN: An instance of OpenSongBible with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OpenSongBible, 'validate_file'), \ + patch.object(OpenSongBible, 'validate_xml_file'), \ patch.object(OpenSongBible, 'parse_xml'), \ patch.object(OpenSongBible, 'get_language_id', return_value=10), \ patch.object(OpenSongBible, 'process_books'): importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.stop_import_flag = False # WHEN: Calling do_import result = importer.do_import() @@ -464,32 +373,32 @@ class TestOpenSongImport(TestCase, TestMixin): # THEN: do_import should return True self.assertTrue(result) - def test_file_import(self): - """ - Test the actual import of OpenSong Bible file - """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'opensong-dk1933.xml' - with patch('openlp.plugins.bibles.lib.importers.opensong.OpenSongBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OpenSongBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'Danish' + def test_file_import(self): + """ + Test the actual import of OpenSong Bible file + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'opensong-dk1933.xml' + with patch('openlp.plugins.bibles.lib.importers.opensong.OpenSongBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OpenSongBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'Danish' - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, 1, int(verse_tag), verse_text) + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, 1, int(verse_tag), verse_text) diff --git a/tests/functional/openlp_plugins/bibles/test_osisimport.py b/tests/functional/openlp_plugins/bibles/test_osisimport.py index 612a63b52..048123662 100644 --- a/tests/functional/openlp_plugins/bibles/test_osisimport.py +++ b/tests/functional/openlp_plugins/bibles/test_osisimport.py @@ -67,7 +67,7 @@ class TestOsisImport(TestCase): """ # GIVEN: An instance of OpenSongBible and a mocked parse_xml which returns False with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OSISBible, 'validate_file'), \ + patch.object(OSISBible, 'validate_xml_file'), \ patch.object(OSISBible, 'parse_xml', return_value=None), \ patch.object(OSISBible, 'get_language_id') as mocked_language_id: importer = OSISBible(MagicMock(), path='.', name='.', filename='') @@ -85,7 +85,7 @@ class TestOsisImport(TestCase): """ # GIVEN: An instance of OpenSongBible and a mocked get_language which returns False with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OSISBible, 'validate_file'), \ + patch.object(OSISBible, 'validate_xml_file'), \ patch.object(OSISBible, 'parse_xml'), \ patch.object(OSISBible, 'get_language_id', **{'return_value': False}), \ patch.object(OSISBible, 'process_books') as mocked_process_books: @@ -98,39 +98,18 @@ class TestOsisImport(TestCase): self.assertFalse(result) self.assertFalse(mocked_process_books.called) - def do_import_stop_import_test(self): - """ - Test do_import when the stop_import_flag is set to True - """ - # GIVEN: An instance of OpenSongBible and stop_import_flag set to True - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OSISBible, 'validate_file'), \ - patch.object(OSISBible, 'parse_xml'), \ - patch.object(OSISBible, 'get_language_id', **{'return_value': 10}), \ - patch.object(OSISBible, 'process_books'): - importer = OSISBible(MagicMock(), path='.', name='.', filename='') - importer.stop_import_flag = True - - # WHEN: Calling do_import - result = importer.do_import() - - # THEN: do_import should return False and process_books should have not been called - self.assertFalse(result) - self.assertTrue(importer.process_books.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.log') def do_import_completes_test(self, mocked_log): """ Test do_import when it completes successfully """ - # GIVEN: An instance of OpenSongBible and stop_import_flag set to True + # GIVEN: An instance of OpenSongBible with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OSISBible, 'validate_file'), \ + patch.object(OSISBible, 'validate_xml_file'), \ patch.object(OSISBible, 'parse_xml'), \ patch.object(OSISBible, 'get_language_id', **{'return_value': 10}), \ patch.object(OSISBible, 'process_books'): importer = OSISBible(MagicMock(), path='.', name='.', filename='') - importer.stop_import_flag = False # WHEN: Calling do_import result = importer.do_import() From 14187f48843b094975ceaa720070b154200945be Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Fri, 2 Sep 2016 15:23:20 +0100 Subject: [PATCH 37/65] Mostly refactors of OSIS + tests --- openlp/plugins/bibles/lib/bibleimport.py | 20 +- .../plugins/bibles/lib/importers/csvbible.py | 3 +- .../plugins/bibles/lib/importers/opensong.py | 4 +- openlp/plugins/bibles/lib/importers/osis.py | 87 +-- .../plugins/bibles/lib/importers/zefania.py | 2 +- .../openlp_plugins/bibles/test_csvimport.py | 3 +- .../bibles/test_opensongimport.py | 114 ++-- .../openlp_plugins/bibles/test_osisimport.py | 546 ++++++++++++++---- .../bibles/test_zefaniaimport.py | 4 +- tests/resources/bibles/dk1933.json | 20 +- tests/resources/bibles/kjv.json | 20 +- tests/resources/bibles/rst.json | 20 +- tests/resources/bibles/web.json | 20 +- 13 files changed, 588 insertions(+), 275 deletions(-) diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index cd9f4d37a..34a1f04b7 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -41,6 +41,10 @@ class BibleImport(OpenLPMixin, BibleDB): super().__init__(*args, **kwargs) self.filename = kwargs['filename'] if 'filename' in kwargs else None + def set_current_chapter(self, book_name, chapter_name): + self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport', 'Importing {book} {chapter}...') + .format(book=book_name, chapter=chapter_name)) + @staticmethod def is_compressed(file): """ @@ -114,15 +118,15 @@ class BibleImport(OpenLPMixin, BibleDB): """ try: with open(filename, 'rb') as import_file: - # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own encoding - # detection, and the two mechanisms together interfere with each other. + # NOTE: We don't need to do any of the normal encoding detection here, because lxml does it's own + # encoding detection, and the two mechanisms together interfere with each other. if not use_objectify: tree = etree.parse(import_file, parser=etree.XMLParser(recover=True)) else: tree = objectify.parse(import_file, parser=objectify.makeparser(recover=True)) if elements or tags: - self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport', - 'Removing unused tags (this may take a few minutes)...')) + self.wizard.increment_progress_bar( + translate('BiblesPlugin.OsisImport', 'Removing unused tags (this may take a few minutes)...')) if elements: # Strip tags we don't use - remove content etree.strip_elements(tree, elements, with_tail=False) @@ -133,8 +137,10 @@ class BibleImport(OpenLPMixin, BibleDB): except OSError as e: log.exception('Opening {file_name} failed.'.format(file_name=e.filename)) trace_error_handler(log) - critical_error_message_box( title='An Error Occured When Opening A File', - message='The following error occurred when trying to open\n{file_name}:\n\n{error}'.format(file_name=e.filename, error=e.strerror)) + critical_error_message_box( + title='An Error Occured When Opening A File', + message='The following error occurred when trying to open\n{file_name}:\n\n{error}' + .format(file_name=e.filename, error=e.strerror)) return None def validate_xml_file(self, filename, tag): @@ -142,6 +148,7 @@ class BibleImport(OpenLPMixin, BibleDB): Validate the supplied file :param filename: The supplied file + :param tag: The expected root tag type :return: True if valid. ValidationError is raised otherwise. """ if BibleImport.is_compressed(filename): @@ -165,4 +172,3 @@ class BibleImport(OpenLPMixin, BibleDB): 'Incorrect Bible file type supplied. This looks like an {bible_type} XML bible.' .format(bible_type=bible_type))) raise ValidationError(msg='Invalid xml.') - diff --git a/openlp/plugins/bibles/lib/importers/csvbible.py b/openlp/plugins/bibles/lib/importers/csvbible.py index bdb903ddb..d0beedf58 100644 --- a/openlp/plugins/bibles/lib/importers/csvbible.py +++ b/openlp/plugins/bibles/lib/importers/csvbible.py @@ -150,8 +150,7 @@ class CSVBible(BibleImport): translate('BiblesPlugin.CSVBible', 'Importing verses from {book}...', 'Importing verses from ...').format(book=book.name)) self.session.commit() - self.create_verse(book.id, verse.chapter_number, verse.number, verse.text) - self.wizard.increment_progress_bar(translate('BiblesPlugin.CSVBible', 'Importing verses... done.')) + self.create_verse(book.id, int(verse.chapter_number), int(verse.number), verse.text) self.session.commit() def do_import(self, bible_name=None): diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 3afa160fa..5219a4d12 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -115,10 +115,8 @@ class OpenSongBible(BibleImport): if self.stop_import_flag: break chapter_number = parse_chapter_number(chapter.attrib['n'], chapter_number) + self.set_current_chapter(book.name, chapter_number) self.process_verses(book, chapter_number, chapter.v) - self.wizard.increment_progress_bar(translate('BiblesPlugin.Opensong', - 'Importing {name} {chapter}...' - ).format(name=book.name, chapter=chapter_number)) def process_verses(self, book, chapter_number, verses): """ diff --git a/openlp/plugins/bibles/lib/importers/osis.py b/openlp/plugins/bibles/lib/importers/osis.py index b4bb18abd..549854804 100644 --- a/openlp/plugins/bibles/lib/importers/osis.py +++ b/openlp/plugins/bibles/lib/importers/osis.py @@ -23,9 +23,6 @@ import logging from lxml import etree -from openlp.core.common import translate, trace_error_handler -from openlp.core.lib.exceptions import ValidationError -from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.bibleimport import BibleImport log = logging.getLogger(__name__) @@ -74,14 +71,9 @@ REMOVABLE_TAGS = ( '{http://www.bibletechnologies.net/2003/OSIS/namespace}caption' ) - -def replacement(match): - return match.group(2).upper() - - # Precompile a few xpath-querys -verse_in_chapter = etree.XPath('count(//ns:chapter[1]/ns:verse)', namespaces=NS) -text_in_verse = etree.XPath('count(//ns:verse[1]/text())', namespaces=NS) +verse_in_chapter = etree.XPath('//ns:chapter[1]/ns:verse', namespaces=NS) +text_in_verse = etree.XPath('//ns:verse[1]/text()', namespaces=NS) class OSISBible(BibleImport): @@ -90,79 +82,88 @@ class OSISBible(BibleImport): """ def process_books(self, bible_data): """ + Extract and create the bible books from the parsed xml - :param bible_data: - :return: + :param bible_data: parsed xml + :return: None """ - no_of_books = int(bible_data.xpath("count(//ns:div[@type='book'])", namespaces=NS)) # Find books in the bible bible_books = bible_data.xpath("//ns:div[@type='book']", namespaces=NS) + no_of_books = len(bible_books) for book in bible_books: if self.stop_import_flag: break # Remove div-tags in the book etree.strip_tags(book, '{http://www.bibletechnologies.net/2003/OSIS/namespace}div') db_book = self.find_and_create_book(book.get('osisID'), no_of_books, self.language_id) - self.process_chapters_and_verses(db_book, book) + self.process_chapters(db_book, book) self.session.commit() - def process_chapters_and_verses(self, book, chapters): + def process_chapters(self, book, chapters): """ + Extract the chapters, and do some initial processing of the verses - :param book: - :param chapters: - :return: + :param book: An OpenLP bible database book object + :param chapters: parsed chapters + :return: None """ # Find out if chapter-tags contains the verses, or if it is used as milestone/anchor - if int(verse_in_chapter(chapters)) > 0: + if verse_in_chapter(chapters): # The chapter tags contains the verses for chapter in chapters: - chapter_number = chapter.get("osisID").split('.')[1] + chapter_number = int(chapter.get("osisID").split('.')[1]) + self.set_current_chapter(book.name, chapter_number) # Find out if verse-tags contains the text, or if it is used as milestone/anchor - if int(text_in_verse(chapter)) == 0: + if not text_in_verse(chapter): # verse-tags are used as milestone for verse in chapter: # If this tag marks the start of a verse, the verse text is between this tag and # the next tag, which the "tail" attribute gives us. - if verse.get('sID'): - verse_number = verse.get("osisID").split('.')[2] - verse_text = verse.tail - if verse_text: - self.create_verse(book.id, chapter_number, verse_number, verse_text.strip()) + self.process_verse(book, chapter_number, verse, use_milestones=True) else: # Verse-tags contains the text for verse in chapter: - verse_number = verse.get("osisID").split('.')[2] - if verse.text: - self.create_verse(book.id, chapter_number, verse_number, verse.text.strip()) - self.wizard.increment_progress_bar( - translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...') % - {'bookname': book.name, 'chapter': chapter_number}) + self.process_verse(book, chapter_number, verse) else: # The chapter tags is used as milestones. For now we assume verses is also milestones chapter_number = 0 for element in chapters: if element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' \ and element.get('sID'): - chapter_number = element.get("osisID").split('.')[1] - self.wizard.increment_progress_bar( - translate('BiblesPlugin.OsisImport', 'Importing %(bookname)s %(chapter)s...') % - {'bookname': book.name, 'chapter': chapter_number}) - elif element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}verse' \ - and element.get('sID'): + chapter_number = int(element.get("osisID").split('.')[1]) + self.set_current_chapter(book.name, chapter_number) + elif element.tag == '{http://www.bibletechnologies.net/2003/OSIS/namespace}verse': # If this tag marks the start of a verse, the verse text is between this tag and # the next tag, which the "tail" attribute gives us. - verse_number = element.get("osisID").split('.')[2] - verse_text = element.tail - if verse_text: - self.create_verse(book.id, chapter_number, verse_number, verse_text.strip()) + self.process_verse(book, chapter_number, element, use_milestones=True) + + def process_verse(self, book, chapter_number, element, use_milestones=False): + """ + Process a verse element + :param book: A database Book object + :param chapter_number: The chapter number to add the verses to (int) + :param element: The verse element to process. (etree element type) + :param use_milestones: set to True to process a 'milestone' verse. Defaults to False + :return: None + """ + osis_id = element.get("osisID") + if not osis_id: + return None + verse_number = int(osis_id.split('.')[2]) + verse_text = '' + if use_milestones and element.get('sID'): + verse_text = element.tail + elif not use_milestones: + verse_text = element.text + if verse_text: + self.create_verse(book.id, chapter_number, verse_number, verse_text.strip()) def do_import(self, bible_name=None): """ Loads a Bible from file. """ log.debug('Starting OSIS import from "{name}"'.format(name=self.filename)) - self.validate_xml_file(self.filename, '{http://www.bibletechnologies.net/2003/OSIS/namespace}osis') + self.validate_xml_file(self.filename, '{http://www.bibletechnologies.net/2003/osis/namespace}osis') bible = self.parse_xml(self.filename, elements=REMOVABLE_ELEMENTS, tags=REMOVABLE_TAGS) if bible is None: return False diff --git a/openlp/plugins/bibles/lib/importers/zefania.py b/openlp/plugins/bibles/lib/importers/zefania.py index bc31a1664..7265c32ac 100644 --- a/openlp/plugins/bibles/lib/importers/zefania.py +++ b/openlp/plugins/bibles/lib/importers/zefania.py @@ -79,7 +79,7 @@ class ZefaniaBible(BibleImport): chapter_number = CHAPTER.get("cnumber") for VERS in CHAPTER: verse_number = VERS.get("vnumber") - self.create_verse(db_book.id, chapter_number, verse_number, VERS.text.replace('
', '\n')) + self.create_verse(db_book.id, int(chapter_number), int(verse_number), VERS.text.replace('
', '\n')) self.wizard.increment_progress_bar( translate('BiblesPlugin.Zefnia', 'Importing {book} {chapter}...').format(book=db_book.name, diff --git a/tests/functional/openlp_plugins/bibles/test_csvimport.py b/tests/functional/openlp_plugins/bibles/test_csvimport.py index 7fa0cbf8c..e72c85de6 100644 --- a/tests/functional/openlp_plugins/bibles/test_csvimport.py +++ b/tests/functional/openlp_plugins/bibles/test_csvimport.py @@ -241,7 +241,6 @@ class TestCSVImport(TestCase): # THEN: get_book_name should not be called and the return value should be None self.assertFalse(importer.get_book_name.called) - importer.wizard.increment_progress_bar.assert_called_once_with('Importing verses... done.') self.assertIsNone(result) def process_verses_successful_test(self): @@ -352,6 +351,6 @@ class TestCSVImport(TestCase): # THEN: The create_verse() method should have been called with each verse in the file. self.assertTrue(importer.create_verse.called) for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.get_book().id, '1', verse_tag, verse_text) + importer.create_verse.assert_any_call(importer.get_book().id, 1, verse_tag, verse_text) importer.create_book.assert_any_call('1. Mosebog', importer.get_book_ref_id_by_name(), 1) importer.create_book.assert_any_call('1. Krønikebog', importer.get_book_ref_id_by_name(), 1) diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index 6f3b9c42b..1ebf7dd73 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -47,6 +47,9 @@ class TestOpenSongImport(TestCase, TestMixin): """ def setUp(self): + self.find_and_create_book_patch = patch.object(BibleImport, 'find_and_create_book') + self.addCleanup(self.find_and_create_book_patch.stop) + self.mocked_find_and_create_book = self.find_and_create_book_patch.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') self.addCleanup(self.manager_patcher.stop) self.manager_patcher.start() @@ -177,12 +180,11 @@ class TestOpenSongImport(TestCase, TestMixin): mocked_log.warning.assert_called_once_with('Illegal verse number: (1, 2, 3)') self.assertEqual(result, 13) - @patch('openlp.plugins.bibles.lib.bibleimport.BibleImport.find_and_create_book') - def process_books_stop_import_test(self, mocked_find_and_create_book): + def process_books_stop_import_test(self): """ Test process_books when stop_import is set to True """ - # GIVEN: An isntance of OpenSongBible + # GIVEN: An instance of OpenSongBible importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') # WHEN: stop_import_flag is set to True @@ -190,36 +192,35 @@ class TestOpenSongImport(TestCase, TestMixin): importer.process_books(['Book']) # THEN: find_and_create_book should not have been called - self.assertFalse(mocked_find_and_create_book.called) + self.assertFalse(self.mocked_find_and_create_book.called) - @patch('openlp.plugins.bibles.lib.bibleimport.BibleImport.find_and_create_book', - **{'side_effect': ['db_book1', 'db_book2']}) - def process_books_completes_test(self, mocked_find_and_create_book): + def process_books_completes_test(self): """ Test process_books when it processes all books """ # GIVEN: An instance of OpenSongBible Importer and two mocked books - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + self.mocked_find_and_create_book.side_effect = ['db_book1', 'db_book2'] + with patch.object(OpenSongBible, 'process_chapters') as mocked_process_chapters: + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - book1 = MagicMock() - book1.attrib = {'n': 'Name1'} - book1.c = 'Chapter1' - book2 = MagicMock() - book2.attrib = {'n': 'Name2'} - book2.c = 'Chapter2' - importer.language_id = 10 - importer.process_chapters = MagicMock() - importer.session = MagicMock() - importer.stop_import_flag = False + book1 = MagicMock() + book1.attrib = {'n': 'Name1'} + book1.c = 'Chapter1' + book2 = MagicMock() + book2.attrib = {'n': 'Name2'} + book2.c = 'Chapter2' + importer.language_id = 10 + importer.session = MagicMock() + importer.stop_import_flag = False - # WHEN: Calling process_books with the two books - importer.process_books([book1, book2]) + # WHEN: Calling process_books with the two books + importer.process_books([book1, book2]) - # THEN: find_and_create_book and process_books should be called with the details from the mocked books - self.assertEqual(mocked_find_and_create_book.call_args_list, [call('Name1', 2, 10), call('Name2', 2, 10)]) - self.assertEqual(importer.process_chapters.call_args_list, - [call('db_book1', 'Chapter1'), call('db_book2', 'Chapter2')]) - self.assertEqual(importer.session.commit.call_count, 2) + # THEN: find_and_create_book and process_books should be called with the details from the mocked books + self.assertEqual(self.mocked_find_and_create_book.call_args_list, [call('Name1', 2, 10), call('Name2', 2, 10)]) + self.assertEqual(mocked_process_chapters.call_args_list, + [call('db_book1', 'Chapter1'), call('db_book2', 'Chapter2')]) + self.assertEqual(importer.session.commit.call_count, 2) def process_chapters_stop_import_test(self): """ @@ -373,32 +374,41 @@ class TestOpenSongImport(TestCase, TestMixin): # THEN: do_import should return True self.assertTrue(result) - def test_file_import(self): - """ - Test the actual import of OpenSong Bible file - """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'opensong-dk1933.xml' - with patch('openlp.plugins.bibles.lib.importers.opensong.OpenSongBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OpenSongBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'Danish' +class TestOpenSongImportFileImports(TestCase, TestMixin): + """ + Test the functions in the :mod:`opensongimport` module. + """ + def setUp(self): + self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') + self.addCleanup(self.manager_patcher.stop) + self.manager_patcher.start() - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + def test_file_import(self): + """ + Test the actual import of OpenSong Bible file + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'opensong-dk1933.xml' + with patch('openlp.plugins.bibles.lib.importers.opensong.OpenSongBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OpenSongBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'Danish' - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, 1, int(verse_tag), verse_text) + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() + + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, 1, int(verse_tag), verse_text) diff --git a/tests/functional/openlp_plugins/bibles/test_osisimport.py b/tests/functional/openlp_plugins/bibles/test_osisimport.py index 048123662..6cf157711 100644 --- a/tests/functional/openlp_plugins/bibles/test_osisimport.py +++ b/tests/functional/openlp_plugins/bibles/test_osisimport.py @@ -27,9 +27,10 @@ import os import json from unittest import TestCase -from tests.functional import MagicMock, patch -from openlp.plugins.bibles.lib.importers.osis import OSISBible +from tests.functional import MagicMock, call, patch +from openlp.plugins.bibles.lib.bibleimport import BibleImport from openlp.plugins.bibles.lib.db import BibleDB +from openlp.plugins.bibles.lib.importers.osis import OSISBible TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'bibles')) @@ -39,8 +40,16 @@ class TestOsisImport(TestCase): """ Test the functions in the :mod:`osisimport` module. """ - def setUp(self): + self.etree_patcher = patch('openlp.plugins.bibles.lib.importers.osis.etree') + self.addCleanup(self.etree_patcher.stop) + self.mocked_etree = self.etree_patcher.start() + self.create_verse_patcher = patch('openlp.plugins.bibles.lib.db.BibleDB.create_verse') + self.addCleanup(self.create_verse_patcher.stop) + self.mocked_create_verse = self.create_verse_patcher.start() + self.find_and_create_book_patch = patch.object(BibleImport, 'find_and_create_book') + self.addCleanup(self.find_and_create_book_patch.stop) + self.mocked_find_and_create_book = self.find_and_create_book_patch.start() self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() @@ -48,18 +57,297 @@ class TestOsisImport(TestCase): self.addCleanup(self.manager_patcher.stop) self.manager_patcher.start() - def test_create_importer(self): - """ - Test creating an instance of the OSIS file importer - """ - # GIVEN: A mocked out "manager" - mocked_manager = MagicMock() + def test_create_importer(self): + """ + Test creating an instance of the OSIS file importer + """ + # GIVEN: A mocked out "manager" + mocked_manager = MagicMock() - # WHEN: An importer object is created - importer = OSISBible(mocked_manager, path='.', name='.', filename='') + # WHEN: An importer object is created + importer = OSISBible(mocked_manager, path='.', name='.', filename='') - # THEN: The importer should be an instance of BibleDB - self.assertIsInstance(importer, BibleDB) + # THEN: The importer should be an instance of BibleDB + self.assertIsInstance(importer, BibleDB) + + def process_books_stop_import_test(self): + """ + Test process_books when stop_import is set to True + """ + # GIVEN: An instance of OSISBible adn some mocked data + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + mocked_data = MagicMock(**{'xpath.return_value': ['Book']}) + + # WHEN: stop_import_flag is set to True and process_books is called + importer.stop_import_flag = True + importer.process_books(mocked_data) + + # THEN: find_and_create_book should not have been called + self.assertFalse(self.mocked_find_and_create_book.called) + + def process_books_completes_test(self): + """ + Test process_books when it processes all books + """ + # GIVEN: An instance of OSISBible Importer and two mocked books + self.mocked_find_and_create_book.side_effect = ['db_book1', 'db_book2'] + with patch.object(OSISBible, 'process_chapters') as mocked_process_chapters: + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + book1 = MagicMock() + book1.get.return_value = 'Name1' + book2 = MagicMock() + book2.get.return_value = 'Name2' + mocked_data = MagicMock(**{'xpath.return_value': [book1, book2]}) + importer.language_id = 10 + importer.session = MagicMock() + importer.stop_import_flag = False + + # WHEN: Calling process_books with the two books + importer.process_books(mocked_data) + + # THEN: find_and_create_book and process_books should be called with the details from the mocked books + self.assertEqual(self.mocked_find_and_create_book.call_args_list, + [call('Name1', 2, 10), call('Name2', 2, 10)]) + self.assertEqual(mocked_process_chapters.call_args_list, + [call('db_book1', book1), call('db_book2', book2)]) + self.assertEqual(importer.session.commit.call_count, 2) + + def process_chapters_verse_in_chapter_verse_text_test(self): + """ + Test process_chapters when supplied with an etree element with a verse element nested in it + """ + with patch('openlp.plugins.bibles.lib.importers.osis.verse_in_chapter', return_value=True), \ + patch('openlp.plugins.bibles.lib.importers.osis.text_in_verse', return_value=True), \ + patch.object(OSISBible, 'set_current_chapter') as mocked_set_current_chapter, \ + patch.object(OSISBible, 'process_verse') as mocked_process_verse: + + # GIVEN: Some test data and an instance of OSISBible + test_book = MagicMock() + test_verse = MagicMock() + test_verse.tail = '\n ' # Whitespace + test_verse.text = 'Verse Text' + test_chapter = MagicMock() + test_chapter.__iter__.return_value = [test_verse] + test_chapter.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_chapters + importer.process_chapters(test_book, [test_chapter]) + + # THEN: set_current_chapter and process_verse should have been called with the test data + mocked_set_current_chapter.assert_called_once_with(test_book.name, 2) + mocked_process_verse.assert_called_once_with(test_book, 2, test_verse) + + def process_chapters_verse_in_chapter_verse_milestone_test(self): + """ + Test process_chapters when supplied with an etree element with a verse element nested, when the verse system is + based on milestones + """ + with patch('openlp.plugins.bibles.lib.importers.osis.verse_in_chapter', return_value=True), \ + patch('openlp.plugins.bibles.lib.importers.osis.text_in_verse', return_value=False), \ + patch.object(OSISBible, 'set_current_chapter') as mocked_set_current_chapter, \ + patch.object(OSISBible, 'process_verse') as mocked_process_verse: + + # GIVEN: Some test data and an instance of OSISBible + test_book = MagicMock() + test_verse = MagicMock() + test_verse.tail = '\n ' # Whitespace + test_verse.text = 'Verse Text' + test_chapter = MagicMock() + test_chapter.__iter__.return_value = [test_verse] + test_chapter.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_chapters + importer.process_chapters(test_book, [test_chapter]) + + # THEN: set_current_chapter and process_verse should have been called with the test data + mocked_set_current_chapter.assert_called_once_with(test_book.name, 2) + mocked_process_verse.assert_called_once_with(test_book, 2, test_verse, use_milestones=True) + + def process_chapters_milestones_chapter_no_sid_test(self): + """ + Test process_chapters when supplied with an etree element with a chapter and verse element in the milestone + configuration, where the chapter is the "closing" milestone. (Missing the sID attribute) + """ + with patch('openlp.plugins.bibles.lib.importers.osis.verse_in_chapter', return_value=False), \ + patch.object(OSISBible, 'set_current_chapter') as mocked_set_current_chapter, \ + patch.object(OSISBible, 'process_verse') as mocked_process_verse: + + # GIVEN: Some test data and an instance of OSISBible + test_book = MagicMock() + test_chapter = MagicMock() + test_chapter.tag = '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' + test_chapter.get.side_effect = lambda x: {'osisID': '1.2.4'}.get(x) + + # WHEN: Calling process_chapters + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + importer.process_chapters(test_book, [test_chapter]) + + # THEN: neither set_current_chapter or process_verse should have been called + self.assertFalse(mocked_set_current_chapter.called) + self.assertFalse(mocked_process_verse.called) + + def process_chapters_milestones_chapter_sid_test(self): + """ + Test process_chapters when supplied with an etree element with a chapter and verse element in the milestone + configuration, where the chapter is the "opening" milestone. (Has the sID attribute) + """ + with patch('openlp.plugins.bibles.lib.importers.osis.verse_in_chapter', return_value=False), \ + patch.object(OSISBible, 'set_current_chapter') as mocked_set_current_chapter, \ + patch.object(OSISBible, 'process_verse') as mocked_process_verse: + + # GIVEN: Some test data and an instance of OSISBible + test_book = MagicMock() + test_chapter = MagicMock() + test_chapter.tag = '{http://www.bibletechnologies.net/2003/OSIS/namespace}chapter' + test_chapter.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_chapters + importer.process_chapters(test_book, [test_chapter]) + + # THEN: set_current_chapter should have been called with the test data + mocked_set_current_chapter.assert_called_once_with(test_book.name, 2) + self.assertFalse(mocked_process_verse.called) + + def process_chapters_milestones_verse_tag_test(self): + """ + Test process_chapters when supplied with an etree element with a chapter and verse element in the milestone + configuration, where the verse is the "opening" milestone. (Has the sID attribute) + """ + with patch('openlp.plugins.bibles.lib.importers.osis.verse_in_chapter', return_value=False), \ + patch.object(OSISBible, 'set_current_chapter') as mocked_set_current_chapter, \ + patch.object(OSISBible, 'process_verse') as mocked_process_verse: + + # GIVEN: Some test data and an instance of OSISBible + test_book = MagicMock() + test_verse = MagicMock() + test_verse.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + test_verse.tag = '{http://www.bibletechnologies.net/2003/OSIS/namespace}verse' + test_verse.tail = '\n ' # Whitespace + test_verse.text = 'Verse Text' + + # WHEN: Calling process_chapters + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + importer.process_chapters(test_book, [test_verse]) + + # THEN: process_verse should have been called with the test data + self.assertFalse(mocked_set_current_chapter.called) + mocked_process_verse.assert_called_once_with(test_book, 0, test_verse, use_milestones=True) + + def process_verse_no_osis_id_test(self): + """ + Test process_verse when the element supplied does not have and osisID attribute + """ + # GIVEN: An instance of OSISBible, and some mocked test data + test_book = MagicMock() + test_verse = MagicMock() + test_verse.get.side_effect = lambda x: {}.get(x) + test_verse.tail = 'Verse Text' + test_verse.text = None + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_verse with the test data + importer.process_verse(test_book, 2, test_verse) + + # THEN: create_verse should not have been called + self.assertFalse(self.mocked_create_verse.called) + + def process_verse_use_milestones_no_s_id_test(self): + """ + Test process_verse when called with use_milestones set to True, but the element supplied does not have and sID + attribute + """ + # GIVEN: An instance of OSISBible, and some mocked test data + test_book = MagicMock() + test_verse = MagicMock() + test_verse.get.side_effect = lambda x: {}.get(x) + test_verse.tail = 'Verse Text' + test_verse.text = None + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_verse with the test data + importer.process_verse(test_book, 2, test_verse) + + # THEN: create_verse should not have been called + self.assertFalse(self.mocked_create_verse.called) + + def process_verse_use_milestones_no_tail_test(self): + """ + Test process_verse when called with use_milestones set to True, but the element supplied does not have a 'tail' + """ + # GIVEN: An instance of OSISBible, and some mocked test data + test_book = MagicMock() + test_verse = MagicMock() + test_verse.tail = None + test_verse.text = None + test_verse.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_verse with the test data + importer.process_verse(test_book, 2, test_verse, use_milestones=True) + + # THEN: create_verse should not have been called + self.assertFalse(self.mocked_create_verse.called) + + def process_verse_use_milestones_success_test(self): + """ + Test process_verse when called with use_milestones set to True, and the verse element successfully imports + """ + # GIVEN: An instance of OSISBible, and some mocked test data + test_book = MagicMock() + test_book.id = 1 + test_verse = MagicMock() + test_verse.tail = 'Verse Text' + test_verse.text = None + test_verse.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_verse with the test data + importer.process_verse(test_book, 2, test_verse, use_milestones=True) + + # THEN: create_verse should have been called with the test data + self.mocked_create_verse.assert_called_once_with(1, 2, 4, 'Verse Text') + + def process_verse_no_text_test(self): + """ + Test process_verse when called with an empty verse element + """ + # GIVEN: An instance of OSISBible, and some mocked test data + test_book = MagicMock() + test_book.id = 1 + test_verse = MagicMock() + test_verse.tail = '\n ' # Whitespace + test_verse.text = None + test_verse.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_verse with the test data + importer.process_verse(test_book, 2, test_verse) + + # THEN: create_verse should not have been called + self.assertFalse(self.mocked_create_verse.called) + + def process_verse_success_test(self): + """ + Test process_verse when called with an element with text set + """ + # GIVEN: An instance of OSISBible, and some mocked test data + test_book = MagicMock() + test_book.id = 1 + test_verse = MagicMock() + test_verse.tail = '\n ' # Whitespace + test_verse.text = 'Verse Text' + test_verse.get.side_effect = lambda x: {'osisID': '1.2.4', 'sID': '999'}.get(x) + importer = OSISBible(MagicMock(), path='.', name='.', filename='') + + # WHEN: Calling process_verse with the test data + importer.process_verse(test_book, 2, test_verse) + + # THEN: create_verse should have been called with the test data + self.mocked_create_verse.assert_called_once_with(1, 2, 4, 'Verse Text') def do_import_parse_xml_fails_test(self): """ @@ -98,8 +386,7 @@ class TestOsisImport(TestCase): self.assertFalse(result) self.assertFalse(mocked_process_books.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - def do_import_completes_test(self, mocked_log): + def do_import_completes_test(self): """ Test do_import when it completes successfully """ @@ -117,122 +404,135 @@ class TestOsisImport(TestCase): # THEN: do_import should return True self.assertTrue(result) - def test_file_import_nested_tags(self): - """ - Test the actual import of OSIS Bible file, with nested chapter and verse tags - """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-dk1933.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'Danish' - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() +class TestOsisImportFileImports(TestCase): + """ + Test the functions in the :mod:`osisimport` module. + """ + def setUp(self): + self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.addCleanup(self.registry_patcher.stop) + self.registry_patcher.start() + self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') + self.addCleanup(self.manager_patcher.stop) + self.manager_patcher.start() - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + def test_file_import_nested_tags(self): + """ + Test the actual import of OSIS Bible file, with nested chapter and verse tags + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-dk1933.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'Danish' - def test_file_import_mixed_tags(self): - """ - Test the actual import of OSIS Bible file, with chapter tags containing milestone verse tags. - """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'kjv.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-kjv.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'English' + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, 1, verse_tag, verse_text) - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + def test_file_import_mixed_tags(self): + """ + Test the actual import of OSIS Bible file, with chapter tags containing milestone verse tags. + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'kjv.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-kjv.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'English' - def test_file_import_milestone_tags(self): - """ - Test the actual import of OSIS Bible file, with milestone chapter and verse tags. - """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'web.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-web.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'English' + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, 1, verse_tag, verse_text) - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + def test_file_import_milestone_tags(self): + """ + Test the actual import of OSIS Bible file, with milestone chapter and verse tags. + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'web.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-web.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'English' - def test_file_import_empty_verse_tags(self): - """ - Test the actual import of OSIS Bible file, with an empty verse tags. - """ - # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions - # get_book_ref_id_by_name, create_verse, create_book, session and get_language. - result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') - test_data = json.loads(result_file.read().decode()) - bible_file = 'osis-dk1933-empty-verse.xml' - with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): - mocked_manager = MagicMock() - mocked_import_wizard = MagicMock() - importer = OSISBible(mocked_manager, path='.', name='.', filename='') - importer.wizard = mocked_import_wizard - importer.get_book_ref_id_by_name = MagicMock() - importer.create_verse = MagicMock() - importer.create_book = MagicMock() - importer.session = MagicMock() - importer.get_language = MagicMock() - importer.get_language.return_value = 'Danish' + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() - # WHEN: Importing bible file - importer.filename = os.path.join(TEST_PATH, bible_file) - importer.do_import() + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, 1, verse_tag, verse_text) - # THEN: The create_verse() method should have been called with each verse in the file. - self.assertTrue(importer.create_verse.called) - for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + def test_file_import_empty_verse_tags(self): + """ + Test the actual import of OSIS Bible file, with an empty verse tags. + """ + # GIVEN: Test files with a mocked out "manager", "import_wizard", and mocked functions + # get_book_ref_id_by_name, create_verse, create_book, session and get_language. + result_file = open(os.path.join(TEST_PATH, 'dk1933.json'), 'rb') + test_data = json.loads(result_file.read().decode()) + bible_file = 'osis-dk1933-empty-verse.xml' + with patch('openlp.plugins.bibles.lib.importers.osis.OSISBible.application'): + mocked_manager = MagicMock() + mocked_import_wizard = MagicMock() + importer = OSISBible(mocked_manager, path='.', name='.', filename='') + importer.wizard = mocked_import_wizard + importer.get_book_ref_id_by_name = MagicMock() + importer.create_verse = MagicMock() + importer.create_book = MagicMock() + importer.session = MagicMock() + importer.get_language = MagicMock() + importer.get_language.return_value = 'Danish' + + # WHEN: Importing bible file + importer.filename = os.path.join(TEST_PATH, bible_file) + importer.do_import() + + # THEN: The create_verse() method should have been called with each verse in the file. + self.assertTrue(importer.create_verse.called) + for verse_tag, verse_text in test_data['verses']: + importer.create_verse.assert_any_call(importer.create_book().id, 1, verse_tag, verse_text) diff --git a/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py b/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py index 5294b7f5c..11fcfb7e3 100644 --- a/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py +++ b/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py @@ -88,7 +88,7 @@ class TestZefaniaImport(TestCase): # THEN: The create_verse() method should have been called with each verse in the file. self.assertTrue(importer.create_verse.called) for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + importer.create_verse.assert_any_call(importer.create_book().id, 1, verse_tag, verse_text) importer.create_book.assert_any_call('Genesis', 1, 1) def test_file_import_no_book_name(self): @@ -118,5 +118,5 @@ class TestZefaniaImport(TestCase): # THEN: The create_verse() method should have been called with each verse in the file. self.assertTrue(importer.create_verse.called) for verse_tag, verse_text in test_data['verses']: - importer.create_verse.assert_any_call(importer.create_book().id, '1', verse_tag, verse_text) + importer.create_verse.assert_any_call(importer.create_book().id, 1, verse_tag, verse_text) importer.create_book.assert_any_call('Exodus', 2, 1) diff --git a/tests/resources/bibles/dk1933.json b/tests/resources/bibles/dk1933.json index f364cb47e..2b792eeb3 100644 --- a/tests/resources/bibles/dk1933.json +++ b/tests/resources/bibles/dk1933.json @@ -2,15 +2,15 @@ "book": "Genesis", "chapter": 1, "verses": [ - [ "1", "I Begyndelsen skabte Gud Himmelen og Jorden."], - [ "2", "Og Jorden var øde og tom, og der var Mørke over Verdensdybet. Men Guds Ånd svævede over Vandene." ], - [ "3", "Og Gud sagde: \"Der blive Lys!\" Og der blev Lys." ], - [ "4", "Og Gud så, at Lyset var godt, og Gud satte Skel mellem Lyset og Mørket," ], - [ "5", "og Gud kaldte Lyset Dag, og Mørket kaldte han Nat. Og det blev Aften, og det blev Morgen, første Dag." ], - [ "6", "Derpå sagde Gud: \"Der blive en Hvælving midt i Vandene til at skille Vandene ad!\"" ], - [ "7", "Og således skete det: Gud gjorde Hvælvingen og skilte Vandet under Hvælvingen fra Vandet over Hvælvingen;" ], - [ "8", "og Gud kaldte Hvælvingen Himmel. Og det blev Aften, og det blev Morgen, anden Dag." ], - [ "9", "Derpå sagde Gud: \"Vandet under Himmelen samle sig på eet Sted, så det faste Land kommer til Syne!\" Og således skete det;" ], - [ "10", "og Gud kaldte det faste Land Jord, og Stedet, hvor Vandet samlede sig, kaldte han Hav. Og Gud så, at det var godt." ] + [ 1, "I Begyndelsen skabte Gud Himmelen og Jorden."], + [ 2, "Og Jorden var øde og tom, og der var Mørke over Verdensdybet. Men Guds Ånd svævede over Vandene." ], + [ 3, "Og Gud sagde: \"Der blive Lys!\" Og der blev Lys." ], + [ 4, "Og Gud så, at Lyset var godt, og Gud satte Skel mellem Lyset og Mørket," ], + [ 5, "og Gud kaldte Lyset Dag, og Mørket kaldte han Nat. Og det blev Aften, og det blev Morgen, første Dag." ], + [ 6, "Derpå sagde Gud: \"Der blive en Hvælving midt i Vandene til at skille Vandene ad!\"" ], + [ 7, "Og således skete det: Gud gjorde Hvælvingen og skilte Vandet under Hvælvingen fra Vandet over Hvælvingen;" ], + [ 8, "og Gud kaldte Hvælvingen Himmel. Og det blev Aften, og det blev Morgen, anden Dag." ], + [ 9, "Derpå sagde Gud: \"Vandet under Himmelen samle sig på eet Sted, så det faste Land kommer til Syne!\" Og således skete det;" ], + [ 10, "og Gud kaldte det faste Land Jord, og Stedet, hvor Vandet samlede sig, kaldte han Hav. Og Gud så, at det var godt." ] ] } \ No newline at end of file diff --git a/tests/resources/bibles/kjv.json b/tests/resources/bibles/kjv.json index a375a1b40..dacabab44 100644 --- a/tests/resources/bibles/kjv.json +++ b/tests/resources/bibles/kjv.json @@ -2,15 +2,15 @@ "book": "Genesis", "chapter": 1, "verses": [ - [ "1", "In the beginning God created the heaven and the earth."], - [ "2", "And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters." ], - [ "3", "And God said, Let there be light: and there was light." ], - [ "4", "And God saw the light, that it was good: and God divided the light from the darkness." ], - [ "5", "And God called the light Day, and the darkness he called Night. And the evening and the morning were the first day." ], - [ "6", "And God said, Let there be a firmament in the midst of the waters, and let it divide the waters from the waters." ], - [ "7", "And God made the firmament, and divided the waters which were under the firmament from the waters which were above the firmament: and it was so." ], - [ "8", "And God called the firmament Heaven. And the evening and the morning were the second day." ], - [ "9", "And God said, Let the waters under the heaven be gathered together unto one place, and let the dry land appear: and it was so." ], - [ "10", "And God called the dry land Earth; and the gathering together of the waters called he Seas: and God saw that it was good." ] + [ 1, "In the beginning God created the heaven and the earth."], + [ 2, "And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters." ], + [ 3, "And God said, Let there be light: and there was light." ], + [ 4, "And God saw the light, that it was good: and God divided the light from the darkness." ], + [ 5, "And God called the light Day, and the darkness he called Night. And the evening and the morning were the first day." ], + [ 6, "And God said, Let there be a firmament in the midst of the waters, and let it divide the waters from the waters." ], + [ 7, "And God made the firmament, and divided the waters which were under the firmament from the waters which were above the firmament: and it was so." ], + [ 8, "And God called the firmament Heaven. And the evening and the morning were the second day." ], + [ 9, "And God said, Let the waters under the heaven be gathered together unto one place, and let the dry land appear: and it was so." ], + [ 10, "And God called the dry land Earth; and the gathering together of the waters called he Seas: and God saw that it was good." ] ] } diff --git a/tests/resources/bibles/rst.json b/tests/resources/bibles/rst.json index d8aca09ac..1fb612b11 100644 --- a/tests/resources/bibles/rst.json +++ b/tests/resources/bibles/rst.json @@ -2,15 +2,15 @@ "book": "Exodus", "chapter": 1, "verses": [ - [ "1", "Вот имена сынов Израилевых, которые вошли в Египет с Иаковом, вошли каждый с домом своим:" ], - [ "2", "Рувим, Симеон, Левий и Иуда," ], - [ "3", "Иссахар, Завулон и Вениамин," ], - [ "4", "Дан и Неффалим, Гад и Асир." ], - [ "5", "Всех же душ, происшедших от чресл Иакова, было семьдесят, а Иосиф был [уже] в Египте." ], - [ "6", "И умер Иосиф и все братья его и весь род их;" ], - [ "7", "а сыны Израилевы расплодились и размножились, и возросли и усилились чрезвычайно, и наполнилась ими земля та." ], - [ "8", "И восстал в Египте новый царь, который не знал Иосифа," ], - [ "9", "и сказал народу своему: вот, народ сынов Израилевых многочислен и сильнее нас;" ], - [ "10", "перехитрим же его, чтобы он не размножался; иначе, когда случится война, соединится и он с нашими неприятелями, и вооружится против нас, и выйдет из земли [нашей]." ] + [ 1, "Вот имена сынов Израилевых, которые вошли в Египет с Иаковом, вошли каждый с домом своим:" ], + [ 2, "Рувим, Симеон, Левий и Иуда," ], + [ 3, "Иссахар, Завулон и Вениамин," ], + [ 4, "Дан и Неффалим, Гад и Асир." ], + [ 5, "Всех же душ, происшедших от чресл Иакова, было семьдесят, а Иосиф был [уже] в Египте." ], + [ 6, "И умер Иосиф и все братья его и весь род их;" ], + [ 7, "а сыны Израилевы расплодились и размножились, и возросли и усилились чрезвычайно, и наполнилась ими земля та." ], + [ 8, "И восстал в Египте новый царь, который не знал Иосифа," ], + [ 9, "и сказал народу своему: вот, народ сынов Израилевых многочислен и сильнее нас;" ], + [ 10, "перехитрим же его, чтобы он не размножался; иначе, когда случится война, соединится и он с нашими неприятелями, и вооружится против нас, и выйдет из земли [нашей]." ] ] } diff --git a/tests/resources/bibles/web.json b/tests/resources/bibles/web.json index 0fbc95669..069d5e276 100644 --- a/tests/resources/bibles/web.json +++ b/tests/resources/bibles/web.json @@ -2,15 +2,15 @@ "book": "Genesis", "chapter": "1", "verses": [ - [ "1", "In the beginning God created the heavens and the earth."], - [ "2", "Now the earth was formless and empty. Darkness was on the surface of the deep. God’s Spirit was hovering over the surface of the waters." ], - [ "3", "God said, “Let there be light,” and there was light." ], - [ "4", "God saw the light, and saw that it was good. God divided the light from the darkness." ], - [ "5", "God called the light “day,” and the darkness he called “night.” There was evening and there was morning, one day." ], - [ "6", "God said, “Let there be an expanse in the middle of the waters, and let it divide the waters from the waters.”" ], - [ "7", "God made the expanse, and divided the waters which were under the expanse from the waters which were above the expanse; and it was so." ], - [ "8", "God called the expanse “sky.” There was evening and there was morning, a second day." ], - [ "9", "God said, “Let the waters under the sky be gathered together to one place, and let the dry land appear;” and it was so." ], - [ "10", "God called the dry land “earth,” and the gathering together of the waters he called “seas.” God saw that it was good." ] + [ 1, "In the beginning God created the heavens and the earth."], + [ 2, "Now the earth was formless and empty. Darkness was on the surface of the deep. God’s Spirit was hovering over the surface of the waters." ], + [ 3, "God said, “Let there be light,” and there was light." ], + [ 4, "God saw the light, and saw that it was good. God divided the light from the darkness." ], + [ 5, "God called the light “day,” and the darkness he called “night.” There was evening and there was morning, one day." ], + [ 6, "God said, “Let there be an expanse in the middle of the waters, and let it divide the waters from the waters.”" ], + [ 7, "God made the expanse, and divided the waters which were under the expanse from the waters which were above the expanse; and it was so." ], + [ 8, "God called the expanse “sky.” There was evening and there was morning, a second day." ], + [ 9, "God said, “Let the waters under the sky be gathered together to one place, and let the dry land appear;” and it was so." ], + [ 10, "God called the dry land “earth,” and the gathering together of the waters he called “seas.” God saw that it was good." ] ] } From 6fc1467c5a15e0aff3648d6cc8e2e9df92929fa4 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Fri, 2 Sep 2016 15:32:14 +0100 Subject: [PATCH 38/65] PEP fixes --- openlp/plugins/bibles/lib/importers/zefania.py | 3 ++- .../openlp_plugins/bibles/test_bibleimport.py | 11 +++++------ .../openlp_plugins/bibles/test_opensongimport.py | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/openlp/plugins/bibles/lib/importers/zefania.py b/openlp/plugins/bibles/lib/importers/zefania.py index 7265c32ac..5d8bddf4e 100644 --- a/openlp/plugins/bibles/lib/importers/zefania.py +++ b/openlp/plugins/bibles/lib/importers/zefania.py @@ -79,7 +79,8 @@ class ZefaniaBible(BibleImport): chapter_number = CHAPTER.get("cnumber") for VERS in CHAPTER: verse_number = VERS.get("vnumber") - self.create_verse(db_book.id, int(chapter_number), int(verse_number), VERS.text.replace('
', '\n')) + self.create_verse( + db_book.id, int(chapter_number), int(verse_number), VERS.text.replace('
', '\n')) self.wizard.increment_progress_bar( translate('BiblesPlugin.Zefnia', 'Importing {book} {chapter}...').format(book=db_book.name, diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index de7c935a0..d293150b5 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -173,8 +173,8 @@ class TestBibleImport(TestCase): # THEN: None should be returned and an error should be logged mocked_languages_get_language.assert_called_once_with('Qwerty') mocked_db_get_language.assert_called_once_with('KJV') - self.mocked_log.error.assert_called_once_with('Language detection failed when importing from "KJV". ' - 'User aborted language selection.') + self.mocked_log.error.assert_called_once_with( + 'Language detection failed when importing from "KJV". User aborted language selection.') self.assertFalse(instance.save_meta.called) self.assertIsNone(result) @@ -418,8 +418,7 @@ class TestBibleImport(TestCase): Test that validate_xml_file raises a ValidationError with an Zefania root tag """ # GIVEN: Some test data with an Zefania root tag and an instance of BibleImport - with patch.object(BibleImport, 'parse_xml', - return_value=objectify.fromstring('')), \ + with patch.object(BibleImport, 'parse_xml', return_value=objectify.fromstring('')), \ patch.object(BibleImport, 'is_compressed', return_value=False): importer = BibleImport(MagicMock(), path='.', name='.', filename='') @@ -437,8 +436,8 @@ class TestBibleImport(TestCase): Test that validate_xml_file raises a ValidationError with an unknown root tag """ # GIVEN: Some test data with an unknown root tag and an instance of BibleImport - with patch.object(BibleImport, 'parse_xml', - return_value=objectify.fromstring('')), \ + with patch.object( + BibleImport, 'parse_xml', return_value=objectify.fromstring('')), \ patch.object(BibleImport, 'is_compressed', return_value=False): importer = BibleImport(MagicMock(), path='.', name='.', filename='') diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index 1ebf7dd73..c902440b0 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -217,7 +217,8 @@ class TestOpenSongImport(TestCase, TestMixin): importer.process_books([book1, book2]) # THEN: find_and_create_book and process_books should be called with the details from the mocked books - self.assertEqual(self.mocked_find_and_create_book.call_args_list, [call('Name1', 2, 10), call('Name2', 2, 10)]) + self.assertEqual(self.mocked_find_and_create_book.call_args_list, + [call('Name1', 2, 10), call('Name2', 2, 10)]) self.assertEqual(mocked_process_chapters.call_args_list, [call('db_book1', 'Chapter1'), call('db_book2', 'Chapter2')]) self.assertEqual(importer.session.commit.call_count, 2) @@ -325,9 +326,9 @@ class TestOpenSongImport(TestCase, TestMixin): """ # GIVEN: An instance of OpenSongBible and a mocked parse_xml which returns False with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ - patch.object(OpenSongBible, 'validate_xml_file'), \ - patch.object(OpenSongBible, 'parse_xml', return_value=None), \ - patch.object(OpenSongBible, 'get_language_id') as mocked_language_id: + patch.object(OpenSongBible, 'validate_xml_file'), \ + patch.object(OpenSongBible, 'parse_xml', return_value=None), \ + patch.object(OpenSongBible, 'get_language_id') as mocked_language_id: importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') # WHEN: Calling do_import @@ -374,6 +375,7 @@ class TestOpenSongImport(TestCase, TestMixin): # THEN: do_import should return True self.assertTrue(result) + class TestOpenSongImportFileImports(TestCase, TestMixin): """ Test the functions in the :mod:`opensongimport` module. From 875481d2d958a931341f94fca4156f99cff0f144 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 2 Sep 2016 18:22:29 +0300 Subject: [PATCH 39/65] - A working only blank to desktop --- openlp/core/common/settings.py | 5 ++-- openlp/core/ui/slidecontroller.py | 44 ++++++++++++++++++------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/openlp/core/common/settings.py b/openlp/core/common/settings.py index 1f727c290..9bff77163 100644 --- a/openlp/core/common/settings.py +++ b/openlp/core/common/settings.py @@ -215,7 +215,6 @@ class Settings(QtCore.QSettings): ('media/players_temp', 'media/players', []), # Move temp setting from above to correct setting ('advanced/default color', 'core/logo background color', []), # Default image renamed + moved to general > 2.4. ('advanced/default image', 'core/logo file', []), # Default image renamed + moved to general after 2.4. - ('shortcuts/escapeItem', 'shortcuts/desktopScreen', []), # Default image renamed + moved to general after 2.4. ('shortcuts/offlineHelpItem', 'shortcuts/HelpItem', []), # Online and Offline help were combined in 2.6. ('shortcuts/onlineHelpItem', 'shortcuts/HelpItem', []) # Online and Offline help were combined in 2.6. ] @@ -263,8 +262,8 @@ class Settings(QtCore.QSettings): 'shortcuts/displayTagItem': [], 'shortcuts/blankScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Period)], 'shortcuts/collapse': [QtGui.QKeySequence(QtCore.Qt.Key_Minus)], - 'shortcuts/desktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Escape), - QtGui.QKeySequence(QtCore.Qt.Key_D)], + 'shortcuts/showDesktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Escape)], + 'shortcuts/desktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_D)], 'shortcuts/delete': [QtGui.QKeySequence(QtGui.QKeySequence.Delete)], 'shortcuts/down': [QtGui.QKeySequence(QtCore.Qt.Key_Down)], 'shortcuts/editSong': [], diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index f56e8247e..75ff4e4fb 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -236,7 +236,7 @@ class SlideController(DisplayController, RegistryProperties): self.toolbar.add_toolbar_widget(self.hide_menu) # The order of the blank to modes in Shortcuts list comes from here. self.desktop_screen = create_action(self, 'desktopScreen', - text=translate('OpenLP.SlideController', 'Show Desktop'), + text=translate('OpenLP.SlideController', 'Show or hide Desktop'), icon=':/slides/slide_desktop.png', checked=False, can_shortcuts=True, category=self.category, triggers=self.on_hide_display) @@ -250,10 +250,17 @@ class SlideController(DisplayController, RegistryProperties): icon=':/slides/slide_blank.png', checked=False, can_shortcuts=True, category=self.category, triggers=self.on_blank_display) + self.escape_item = create_action(self, 'escapeItem', + text=translate('OpenLP.SlideController', 'Escape Item'), + can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, + category=self.category, + triggers=self.live_escape) + self.hide_menu.setDefaultAction(self.blank_screen) self.hide_menu.menu().addAction(self.blank_screen) self.hide_menu.menu().addAction(self.theme_screen) self.hide_menu.menu().addAction(self.desktop_screen) + self.hide_menu.menu().addAction(self.escape_item) # Wide menu of display control buttons. self.blank_screen_button = QtWidgets.QToolButton(self.toolbar) self.blank_screen_button.setObjectName('blank_screen_button') @@ -267,6 +274,13 @@ class SlideController(DisplayController, RegistryProperties): self.desktop_screen_button.setObjectName('desktop_screen_button') self.toolbar.add_toolbar_widget(self.desktop_screen_button) self.desktop_screen_button.setDefaultAction(self.desktop_screen) + + self.escape_item_button = QtWidgets.QToolButton(self.toolbar) + self.escape_item_button.setObjectName('escape_item_button') + self.toolbar.add_toolbar_widget(self.escape_item_button) + self.escape_item_button.setDefaultAction(self.escape_item) + + self.toolbar.add_toolbar_action('loop_separator', separator=True) # Play Slides Menu self.play_slides_menu = QtWidgets.QToolButton(self.toolbar) @@ -513,23 +527,6 @@ class SlideController(DisplayController, RegistryProperties): can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, category=self.category, triggers=self.service_next) - self.escape_item = create_action(parent, 'escapeItem', - text=translate('OpenLP.SlideController', 'Escape Item'), - can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, - category=self.category, - triggers=self.live_escape) - - def live_escape(self, field=None): - """ - If you press ESC on the live screen it should close the display temporarily. - """ - self.display.setVisible(False) - self.media_controller.media_stop(self) - # Stop looping if active - if self.play_slides_loop.isChecked(): - self.on_play_slides_loop(False) - elif self.play_slides_once.isChecked(): - self.on_play_slides_once(False) def toggle_display(self, action): """ @@ -1044,6 +1041,17 @@ class SlideController(DisplayController, RegistryProperties): self.update_preview() self.on_toggle_loop() + def live_escape(self, checked=None): + """ + If you press ESC on the live screen it should close the display temporarily. + """ + self.blank_screen.setChecked(False) + self.theme_screen.setChecked(False) + # Not sure if this line is required. + self.desktop_screen.setChecked(checked) + Registry().execute('live_display_hide', HideMode.Screen) + self.desktop_screen.setChecked(True) + def blank_plugin(self): """ Blank/Hide the display screen within a plugin if required. From 4f48cb5df2ebd14a71c89408a08ba4268f9f29e0 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 2 Sep 2016 18:34:39 +0300 Subject: [PATCH 40/65] - fixed the "blank only" button visibility --- openlp/core/ui/slidecontroller.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index 75ff4e4fb..e59fdfffd 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -235,6 +235,11 @@ class SlideController(DisplayController, RegistryProperties): self.hide_menu.setMenu(QtWidgets.QMenu(translate('OpenLP.SlideController', 'Hide'), self.toolbar)) self.toolbar.add_toolbar_widget(self.hide_menu) # The order of the blank to modes in Shortcuts list comes from here. + self.escape_item = create_action(self, 'escapeItem', + text=translate('OpenLP.SlideController', 'Show Desktop'), + can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, + category=self.category, + triggers=self.live_escape) self.desktop_screen = create_action(self, 'desktopScreen', text=translate('OpenLP.SlideController', 'Show or hide Desktop'), icon=':/slides/slide_desktop.png', @@ -250,12 +255,6 @@ class SlideController(DisplayController, RegistryProperties): icon=':/slides/slide_blank.png', checked=False, can_shortcuts=True, category=self.category, triggers=self.on_blank_display) - self.escape_item = create_action(self, 'escapeItem', - text=translate('OpenLP.SlideController', 'Escape Item'), - can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, - category=self.category, - triggers=self.live_escape) - self.hide_menu.setDefaultAction(self.blank_screen) self.hide_menu.menu().addAction(self.blank_screen) self.hide_menu.menu().addAction(self.theme_screen) @@ -274,13 +273,6 @@ class SlideController(DisplayController, RegistryProperties): self.desktop_screen_button.setObjectName('desktop_screen_button') self.toolbar.add_toolbar_widget(self.desktop_screen_button) self.desktop_screen_button.setDefaultAction(self.desktop_screen) - - self.escape_item_button = QtWidgets.QToolButton(self.toolbar) - self.escape_item_button.setObjectName('escape_item_button') - self.toolbar.add_toolbar_widget(self.escape_item_button) - self.escape_item_button.setDefaultAction(self.escape_item) - - self.toolbar.add_toolbar_action('loop_separator', separator=True) # Play Slides Menu self.play_slides_menu = QtWidgets.QToolButton(self.toolbar) From 2c4dbda42fbeef8e43296b621a22bcc8e7257595 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 2 Sep 2016 18:41:16 +0300 Subject: [PATCH 41/65] - The new method now also stops loops. --- openlp/core/ui/slidecontroller.py | 8 +++++--- tests/functional/openlp_core_ui/test_slidecontroller.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index e59fdfffd..e5260bd80 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -235,7 +235,7 @@ class SlideController(DisplayController, RegistryProperties): self.hide_menu.setMenu(QtWidgets.QMenu(translate('OpenLP.SlideController', 'Hide'), self.toolbar)) self.toolbar.add_toolbar_widget(self.hide_menu) # The order of the blank to modes in Shortcuts list comes from here. - self.escape_item = create_action(self, 'escapeItem', + self.desktop_screen_enable = create_action(self, 'escapeItem', text=translate('OpenLP.SlideController', 'Show Desktop'), can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, category=self.category, @@ -259,7 +259,7 @@ class SlideController(DisplayController, RegistryProperties): self.hide_menu.menu().addAction(self.blank_screen) self.hide_menu.menu().addAction(self.theme_screen) self.hide_menu.menu().addAction(self.desktop_screen) - self.hide_menu.menu().addAction(self.escape_item) + self.hide_menu.menu().addAction(self.desktop_screen_enable) # Wide menu of display control buttons. self.blank_screen_button = QtWidgets.QToolButton(self.toolbar) self.blank_screen_button.setObjectName('blank_screen_button') @@ -612,7 +612,7 @@ class SlideController(DisplayController, RegistryProperties): widget.addActions([ self.previous_item, self.next_item, self.previous_service, self.next_service, - self.escape_item, + self.desktop_screen_enable, self.desktop_screen, self.theme_screen, self.blank_screen]) @@ -1043,6 +1043,8 @@ class SlideController(DisplayController, RegistryProperties): self.desktop_screen.setChecked(checked) Registry().execute('live_display_hide', HideMode.Screen) self.desktop_screen.setChecked(True) + self.update_preview() + self.on_toggle_loop() def blank_plugin(self): """ diff --git a/tests/functional/openlp_core_ui/test_slidecontroller.py b/tests/functional/openlp_core_ui/test_slidecontroller.py index ef1ce5793..7a171535b 100644 --- a/tests/functional/openlp_core_ui/test_slidecontroller.py +++ b/tests/functional/openlp_core_ui/test_slidecontroller.py @@ -697,7 +697,7 @@ class TestSlideController(TestCase): slide_controller.next_item = MagicMock() slide_controller.previous_service = MagicMock() slide_controller.next_service = MagicMock() - slide_controller.escape_item = MagicMock() + slide_controller.desktop_screen_enable = MagicMock() slide_controller.desktop_screen = MagicMock() slide_controller.blank_screen = MagicMock() slide_controller.theme_screen = MagicMock() @@ -709,7 +709,7 @@ class TestSlideController(TestCase): mocked_widget.addActions.assert_called_with([ slide_controller.previous_item, slide_controller.next_item, slide_controller.previous_service, slide_controller.next_service, - slide_controller.escape_item, slide_controller.desktop_screen, + slide_controller.desktop_screen_enable, slide_controller.desktop_screen, slide_controller.theme_screen, slide_controller.blank_screen ]) From 2fcd29296937ca6b44bf4ca28eba04d8c5e79857 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 2 Sep 2016 18:52:44 +0300 Subject: [PATCH 42/65] - Renamed some code --- openlp/core/ui/slidecontroller.py | 12 ++++++++---- .../openlp_core_ui/test_slidecontroller.py | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index e5260bd80..b97fd3f2b 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -239,7 +239,7 @@ class SlideController(DisplayController, RegistryProperties): text=translate('OpenLP.SlideController', 'Show Desktop'), can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, category=self.category, - triggers=self.live_escape) + triggers=self.on_hide_display_enable) self.desktop_screen = create_action(self, 'desktopScreen', text=translate('OpenLP.SlideController', 'Show or hide Desktop'), icon=':/slides/slide_desktop.png', @@ -955,7 +955,7 @@ class SlideController(DisplayController, RegistryProperties): else: Registry().execute('live_display_show') else: - self.live_escape() + self.on_hide_display_enable() def on_slide_blank(self): """ @@ -1015,6 +1015,7 @@ class SlideController(DisplayController, RegistryProperties): def on_hide_display(self, checked=None): """ Handle the Hide screen button + This toggles the desktop screen. :param checked: the new state of the of the widget """ @@ -1033,9 +1034,12 @@ class SlideController(DisplayController, RegistryProperties): self.update_preview() self.on_toggle_loop() - def live_escape(self, checked=None): + def on_hide_display_enable(self, checked=None): """ - If you press ESC on the live screen it should close the display temporarily. + Handle the on_hide_display_enable + This only enables the desktop screen. + + :param checked: the new state of the of the widget """ self.blank_screen.setChecked(False) self.theme_screen.setChecked(False) diff --git a/tests/functional/openlp_core_ui/test_slidecontroller.py b/tests/functional/openlp_core_ui/test_slidecontroller.py index 7a171535b..5346c4537 100644 --- a/tests/functional/openlp_core_ui/test_slidecontroller.py +++ b/tests/functional/openlp_core_ui/test_slidecontroller.py @@ -208,9 +208,9 @@ class TestSlideController(TestCase): mocked_on_theme_display.assert_called_once_with(False) mocked_on_hide_display.assert_called_once_with(False) - def test_live_escape(self): + def test_on_hide_display_enable(self): """ - Test that when the live_escape() method is called, the display is set to invisible and any media is stopped + Test that when the on_hide_display_enable() method is called, the display is set to invisible and any media is stopped """ # GIVEN: A new SlideController instance and mocked out display and media_controller mocked_display = MagicMock() @@ -224,8 +224,8 @@ class TestSlideController(TestCase): slide_controller.play_slides_loop = play_slides slide_controller.play_slides_once = play_slides - # WHEN: live_escape() is called - slide_controller.live_escape() + # WHEN: on_hide_display_enable() is called + slide_controller.on_hide_display_enable() # THEN: the display should be set to invisible and the media controller stopped mocked_display.setVisible.assert_called_once_with(False) From e4e46d40f57b41d2d66d084e0c5a8b6ad6de3e20 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 2 Sep 2016 19:19:28 +0300 Subject: [PATCH 43/65] - Setting migration for escape item - Renamed shortcut descriptions for blanks --- openlp/core/common/settings.py | 4 ++-- openlp/core/ui/slidecontroller.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/openlp/core/common/settings.py b/openlp/core/common/settings.py index 9bff77163..7695e2a7f 100644 --- a/openlp/core/common/settings.py +++ b/openlp/core/common/settings.py @@ -215,6 +215,7 @@ class Settings(QtCore.QSettings): ('media/players_temp', 'media/players', []), # Move temp setting from above to correct setting ('advanced/default color', 'core/logo background color', []), # Default image renamed + moved to general > 2.4. ('advanced/default image', 'core/logo file', []), # Default image renamed + moved to general after 2.4. + ('shortcuts/escapeItem', 'shortcuts/desktopScreenEnable', []), # Escape item was removed in 2.6. ('shortcuts/offlineHelpItem', 'shortcuts/HelpItem', []), # Online and Offline help were combined in 2.6. ('shortcuts/onlineHelpItem', 'shortcuts/HelpItem', []) # Online and Offline help were combined in 2.6. ] @@ -262,12 +263,11 @@ class Settings(QtCore.QSettings): 'shortcuts/displayTagItem': [], 'shortcuts/blankScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Period)], 'shortcuts/collapse': [QtGui.QKeySequence(QtCore.Qt.Key_Minus)], - 'shortcuts/showDesktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_Escape)], 'shortcuts/desktopScreen': [QtGui.QKeySequence(QtCore.Qt.Key_D)], + 'shortcuts/desktopScreenEnable': [QtGui.QKeySequence(QtCore.Qt.Key_Escape)], 'shortcuts/delete': [QtGui.QKeySequence(QtGui.QKeySequence.Delete)], 'shortcuts/down': [QtGui.QKeySequence(QtCore.Qt.Key_Down)], 'shortcuts/editSong': [], - 'shortcuts/escapeItem': [], 'shortcuts/expand': [QtGui.QKeySequence(QtCore.Qt.Key_Plus)], 'shortcuts/exportThemeItem': [], 'shortcuts/fileNewItem': [QtGui.QKeySequence(QtGui.QKeySequence.New)], diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index b97fd3f2b..51794cdd3 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -235,23 +235,23 @@ class SlideController(DisplayController, RegistryProperties): self.hide_menu.setMenu(QtWidgets.QMenu(translate('OpenLP.SlideController', 'Hide'), self.toolbar)) self.toolbar.add_toolbar_widget(self.hide_menu) # The order of the blank to modes in Shortcuts list comes from here. - self.desktop_screen_enable = create_action(self, 'escapeItem', - text=translate('OpenLP.SlideController', 'Show Desktop'), - can_shortcuts=True, context=QtCore.Qt.WidgetWithChildrenShortcut, - category=self.category, - triggers=self.on_hide_display_enable) + self.desktop_screen_enable = create_action(self, 'desktopScreenEnable', + text=translate('OpenLP.SlideController', 'Show Desktop'), + icon=':/slides/slide_desktop.png', can_shortcuts=True, + context=QtCore.Qt.WidgetWithChildrenShortcut, + category=self.category, triggers=self.on_hide_display_enable) self.desktop_screen = create_action(self, 'desktopScreen', - text=translate('OpenLP.SlideController', 'Show or hide Desktop'), + text=translate('OpenLP.SlideController', 'Toggle Desktop'), icon=':/slides/slide_desktop.png', checked=False, can_shortcuts=True, category=self.category, triggers=self.on_hide_display) self.theme_screen = create_action(self, 'themeScreen', - text=translate('OpenLP.SlideController', 'Blank to Theme'), + text=translate('OpenLP.SlideController', 'Toggle Blank to Theme'), icon=':/slides/slide_theme.png', checked=False, can_shortcuts=True, category=self.category, triggers=self.on_theme_display) self.blank_screen = create_action(self, 'blankScreen', - text=translate('OpenLP.SlideController', 'Blank Screen'), + text=translate('OpenLP.SlideController', 'Toggle Blank Screen'), icon=':/slides/slide_blank.png', checked=False, can_shortcuts=True, category=self.category, triggers=self.on_blank_display) From 4a57caf084b7a3a150b2e71285777c4ffd7d3e96 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 2 Sep 2016 19:24:57 +0300 Subject: [PATCH 44/65] - Removed the old "escape item" test. --- .../openlp_core_ui/test_slidecontroller.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/tests/functional/openlp_core_ui/test_slidecontroller.py b/tests/functional/openlp_core_ui/test_slidecontroller.py index 5346c4537..b93952097 100644 --- a/tests/functional/openlp_core_ui/test_slidecontroller.py +++ b/tests/functional/openlp_core_ui/test_slidecontroller.py @@ -208,29 +208,6 @@ class TestSlideController(TestCase): mocked_on_theme_display.assert_called_once_with(False) mocked_on_hide_display.assert_called_once_with(False) - def test_on_hide_display_enable(self): - """ - Test that when the on_hide_display_enable() method is called, the display is set to invisible and any media is stopped - """ - # GIVEN: A new SlideController instance and mocked out display and media_controller - mocked_display = MagicMock() - mocked_media_controller = MagicMock() - Registry.create() - Registry().register('media_controller', mocked_media_controller) - slide_controller = SlideController(None) - slide_controller.display = mocked_display - play_slides = MagicMock() - play_slides.isChecked.return_value = False - slide_controller.play_slides_loop = play_slides - slide_controller.play_slides_once = play_slides - - # WHEN: on_hide_display_enable() is called - slide_controller.on_hide_display_enable() - - # THEN: the display should be set to invisible and the media controller stopped - mocked_display.setVisible.assert_called_once_with(False) - mocked_media_controller.media_stop.assert_called_once_with(slide_controller) - def test_on_go_live_live_controller(self): """ Test that when the on_go_live() method is called the message is sent to the live controller and focus is From aecd6b885873beadab1c6fc44f5008c4db4b3d82 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 2 Sep 2016 19:54:00 +0300 Subject: [PATCH 45/65] Fixed 1 strong tag --- openlp/core/ui/exceptiondialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/core/ui/exceptiondialog.py b/openlp/core/ui/exceptiondialog.py index 7341287c7..819e3faaa 100644 --- a/openlp/core/ui/exceptiondialog.py +++ b/openlp/core/ui/exceptiondialog.py @@ -107,7 +107,7 @@ class Ui_ExceptionDialog(object): 'No email app? You can save this ' 'information to a file and
' 'send it from your mail on browser via an attachment.

' - 'Thank you for being part of making OpenLP better!
' + 'Thank you for being part of making OpenLP better!
' ).format(first_part=exception_part1)) self.send_report_button.setText(translate('OpenLP.ExceptionDialog', 'Send E-Mail')) self.save_report_button.setText(translate('OpenLP.ExceptionDialog', 'Save to File')) From 80cdc35433d76348a6fa9c2c7a25ff3788fa170f Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sun, 4 Sep 2016 17:15:57 +0100 Subject: [PATCH 46/65] Move already refactored imorters over to using the log methods in OpenLPMixin --- openlp/core/common/openlpmixin.py | 6 + openlp/plugins/bibles/lib/bibleimport.py | 15 +-- .../plugins/bibles/lib/importers/csvbible.py | 7 +- .../plugins/bibles/lib/importers/opensong.py | 57 ++++----- openlp/plugins/bibles/lib/importers/osis.py | 5 +- openlp/plugins/bibles/lib/manager.py | 6 +- .../openlp_plugins/bibles/test_bibleimport.py | 85 ++++++-------- .../openlp_plugins/bibles/test_csvimport.py | 12 +- .../bibles/test_opensongimport.py | 111 ++++++++++-------- .../openlp_plugins/bibles/test_osisimport.py | 9 +- 10 files changed, 147 insertions(+), 166 deletions(-) diff --git a/openlp/core/common/openlpmixin.py b/openlp/core/common/openlpmixin.py index 94505b86b..2f1d5bbba 100644 --- a/openlp/core/common/openlpmixin.py +++ b/openlp/core/common/openlpmixin.py @@ -71,6 +71,12 @@ class OpenLPMixin(object): """ self.logger.info(message) + def log_warning(self, message): + """ + Common log warning handler + """ + self.logger.warning(message) + def log_error(self, message): """ Common log error handler which prints the calling path diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index 34a1f04b7..eaef01aed 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -20,18 +20,14 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### -import logging - from lxml import etree, objectify from zipfile import is_zipfile -from openlp.core.common import OpenLPMixin, languages, trace_error_handler, translate +from openlp.core.common import OpenLPMixin, languages, translate from openlp.core.lib import ValidationError from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB -log = logging.getLogger(__name__) - class BibleImport(OpenLPMixin, BibleDB): """ @@ -78,8 +74,8 @@ class BibleImport(OpenLPMixin, BibleDB): language_id = self.get_language(bible_name) if not language_id: # User cancelled get_language dialog - log.error('Language detection failed when importing from "{name}". User aborted language selection.' - .format(name=bible_name)) + self.log_error('Language detection failed when importing from "{name}". User aborted language selection.' + .format(name=bible_name)) return None self.save_meta('language_id', language_id) return language_id @@ -97,7 +93,7 @@ class BibleImport(OpenLPMixin, BibleDB): if name: book_ref_id = self.get_book_ref_id_by_name(name, no_of_books, language_id) else: - log.debug('No book name supplied. Falling back to guess_id') + self.log_debug('No book name supplied. Falling back to guess_id') book_ref_id = guess_id if not book_ref_id: raise ValidationError(msg='Could not resolve book_ref_id in "{}"'.format(self.filename)) @@ -135,8 +131,7 @@ class BibleImport(OpenLPMixin, BibleDB): etree.strip_tags(tree, tags) return tree.getroot() except OSError as e: - log.exception('Opening {file_name} failed.'.format(file_name=e.filename)) - trace_error_handler(log) + self.log_exception('Opening {file_name} failed.'.format(file_name=e.filename)) critical_error_message_box( title='An Error Occured When Opening A File', message='The following error occurred when trying to open\n{file_name}:\n\n{error}' diff --git a/openlp/plugins/bibles/lib/importers/csvbible.py b/openlp/plugins/bibles/lib/importers/csvbible.py index d0beedf58..d9edd6cc7 100644 --- a/openlp/plugins/bibles/lib/importers/csvbible.py +++ b/openlp/plugins/bibles/lib/importers/csvbible.py @@ -50,7 +50,6 @@ There are two acceptable formats of the verses file. They are: All CSV files are expected to use a comma (',') as the delimiter and double quotes ('"') as the quote symbol. """ import csv -import logging from collections import namedtuple from openlp.core.common import get_file_encoding, translate @@ -58,8 +57,6 @@ from openlp.core.lib.exceptions import ValidationError from openlp.plugins.bibles.lib.bibleimport import BibleImport -log = logging.getLogger(__name__) - Book = namedtuple('Book', 'id, testament_id, name, abbreviation') Verse = namedtuple('Verse', 'book_id_name, chapter_number, number, text') @@ -68,15 +65,13 @@ class CSVBible(BibleImport): """ This class provides a specialisation for importing of CSV Bibles. """ - log.info('CSVBible loaded') - def __init__(self, *args, **kwargs): """ Loads a Bible from a set of CSV files. This class assumes the files contain all the information and a clean bible is being loaded. """ - log.info(self.__class__.__name__) super().__init__(*args, **kwargs) + self.log_info(self.__class__.__name__) self.books_file = kwargs['booksfile'] self.verses_file = kwargs['versefile'] diff --git a/openlp/plugins/bibles/lib/importers/opensong.py b/openlp/plugins/bibles/lib/importers/opensong.py index 5219a4d12..d75f37e03 100644 --- a/openlp/plugins/bibles/lib/importers/opensong.py +++ b/openlp/plugins/bibles/lib/importers/opensong.py @@ -20,18 +20,9 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### -import logging -from lxml import etree - -from openlp.core.common import trace_error_handler, translate -from openlp.core.lib.exceptions import ValidationError -from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib.bibleimport import BibleImport -log = logging.getLogger(__name__) - - def get_text(element): """ Recursively get all text in an objectify element and its child elements. @@ -62,32 +53,32 @@ def parse_chapter_number(number, previous_number): return previous_number + 1 -def parse_verse_number(number, previous_number): - """ - Parse the verse number retrieved from the xml - - :param number: The raw data from the xml - :param previous_number: The previous verse number - :return: Number of current verse. (Int) - """ - if not number: - return previous_number + 1 - try: - return int(number) - except ValueError: - verse_parts = number.split('-') - if len(verse_parts) > 1: - number = int(verse_parts[0]) - return number - except TypeError: - log.warning('Illegal verse number: {verse_no}'.format(verse_no=str(number))) - return previous_number + 1 - - class OpenSongBible(BibleImport): """ OpenSong Bible format importer class. This class is used to import Bibles from OpenSong's XML format. """ + + def parse_verse_number(self, number, previous_number): + """ + Parse the verse number retrieved from the xml + + :param number: The raw data from the xml + :param previous_number: The previous verse number + :return: Number of current verse. (Int) + """ + if not number: + return previous_number + 1 + try: + return int(number) + except ValueError: + verse_parts = number.split('-') + if len(verse_parts) > 1: + number = int(verse_parts[0]) + return number + except TypeError: + self.log_warning('Illegal verse number: {verse_no}'.format(verse_no=str(number))) + return previous_number + 1 + def process_books(self, books): """ Extract and create the books from the objectified xml @@ -131,7 +122,7 @@ class OpenSongBible(BibleImport): for verse in verses: if self.stop_import_flag: break - verse_number = parse_verse_number(verse.attrib['n'], verse_number) + verse_number = self.parse_verse_number(verse.attrib['n'], verse_number) self.create_verse(book.id, chapter_number, verse_number, get_text(verse)) def do_import(self, bible_name=None): @@ -141,7 +132,7 @@ class OpenSongBible(BibleImport): :param bible_name: The name of the bible being imported :return: True if import completed, False if import was unsuccessful """ - log.debug('Starting OpenSong import from "{name}"'.format(name=self.filename)) + self.log_debug('Starting OpenSong import from "{name}"'.format(name=self.filename)) self.validate_xml_file(self.filename, 'bible') bible = self.parse_xml(self.filename, use_objectify=True) if bible is None: diff --git a/openlp/plugins/bibles/lib/importers/osis.py b/openlp/plugins/bibles/lib/importers/osis.py index 549854804..3c799daa3 100644 --- a/openlp/plugins/bibles/lib/importers/osis.py +++ b/openlp/plugins/bibles/lib/importers/osis.py @@ -20,13 +20,10 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### -import logging from lxml import etree from openlp.plugins.bibles.lib.bibleimport import BibleImport -log = logging.getLogger(__name__) - NS = {'ns': 'http://www.bibletechnologies.net/2003/OSIS/namespace'} # Tags we don't use and can remove the content REMOVABLE_ELEMENTS = ( @@ -162,7 +159,7 @@ class OSISBible(BibleImport): """ Loads a Bible from file. """ - log.debug('Starting OSIS import from "{name}"'.format(name=self.filename)) + self.log_debug('Starting OSIS import from "{name}"'.format(name=self.filename)) self.validate_xml_file(self.filename, '{http://www.bibletechnologies.net/2003/osis/namespace}osis') bible = self.parse_xml(self.filename, elements=REMOVABLE_ELEMENTS, tags=REMOVABLE_TAGS) if bible is None: diff --git a/openlp/plugins/bibles/lib/manager.py b/openlp/plugins/bibles/lib/manager.py index d2286bed2..dbd9ae0dd 100644 --- a/openlp/plugins/bibles/lib/manager.py +++ b/openlp/plugins/bibles/lib/manager.py @@ -23,8 +23,8 @@ import logging import os -from openlp.core.common import RegistryProperties, AppLocation, Settings, translate, delete_file, UiStrings -from openlp.plugins.bibles.lib import parse_reference, LanguageSelection +from openlp.core.common import AppLocation, OpenLPMixin, RegistryProperties, Settings, translate, delete_file, UiStrings +from openlp.plugins.bibles.lib import LanguageSelection, parse_reference from openlp.plugins.bibles.lib.db import BibleDB, BibleMeta from .importers.csvbible import CSVBible from .importers.http import HTTPBible @@ -88,7 +88,7 @@ class BibleFormat(object): ] -class BibleManager(RegistryProperties): +class BibleManager(OpenLPMixin, RegistryProperties): """ The Bible manager which holds and manages all the Bibles. """ diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index d293150b5..50baa655b 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -32,7 +32,7 @@ from openlp.core.common.languages import Language from openlp.core.lib.exceptions import ValidationError from openlp.plugins.bibles.lib.bibleimport import BibleImport from openlp.plugins.bibles.lib.db import BibleDB -from tests.functional import ANY, MagicMock, patch +from tests.functional import MagicMock, patch class TestBibleImport(TestCase): @@ -51,9 +51,6 @@ class TestBibleImport(TestCase): self.open_patcher = patch('builtins.open') self.addCleanup(self.open_patcher.stop) self.mocked_open = self.open_patcher.start() - self.log_patcher = patch('openlp.plugins.bibles.lib.bibleimport.log') - self.addCleanup(self.log_patcher.stop) - self.mocked_log = self.log_patcher.start() self.critical_error_message_box_patcher = \ patch('openlp.plugins.bibles.lib.bibleimport.critical_error_message_box') self.addCleanup(self.critical_error_message_box_patcher.stop) @@ -61,9 +58,6 @@ class TestBibleImport(TestCase): self.setup_patcher = patch('openlp.plugins.bibles.lib.db.BibleDB._setup') self.addCleanup(self.setup_patcher.stop) self.setup_patcher.start() - self.trace_error_handler_patcher = patch('openlp.plugins.bibles.lib.bibleimport.trace_error_handler') - self.addCleanup(self.trace_error_handler_patcher.stop) - self.mocked_trace_error_handler = self.trace_error_handler_patcher.start() self.translate_patcher = patch('openlp.plugins.bibles.lib.bibleimport.translate', side_effect=lambda module, string_to_translate, *args: string_to_translate) self.addCleanup(self.translate_patcher.stop) @@ -101,7 +95,7 @@ class TestBibleImport(TestCase): # GIVEN: A mocked languages.get_language which returns language and an instance of BibleImport with patch('openlp.core.common.languages.get_language', return_value=Language(30, 'English', 'en')) \ as mocked_languages_get_language, \ - patch('openlp.plugins.bibles.lib.db.BibleDB.get_language') as mocked_db_get_language: + patch.object(BibleImport, 'get_language') as mocked_db_get_language: instance = BibleImport(MagicMock()) instance.save_meta = MagicMock() @@ -120,7 +114,7 @@ class TestBibleImport(TestCase): """ # GIVEN: A mocked languages.get_language which returns language and an instance of BibleImport with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \ - patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=20) as mocked_db_get_language: + patch.object(BibleImport, 'get_language', return_value=20) as mocked_db_get_language: instance = BibleImport(MagicMock()) instance.save_meta = MagicMock() @@ -140,8 +134,8 @@ class TestBibleImport(TestCase): # GIVEN: A mocked languages.get_language which returns None a mocked BibleDB.get_language which returns a # language id. with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \ - patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=40) as mocked_db_get_language: - self.mocked_log.error.reset_mock() + patch.object(BibleImport, 'get_language', return_value=40) as mocked_db_get_language, \ + patch.object(BibleImport, 'log_error') as mocked_log_error: instance = BibleImport(MagicMock()) instance.save_meta = MagicMock() @@ -151,7 +145,7 @@ class TestBibleImport(TestCase): # THEN: The id of the language returned from BibleDB.get_language should be returned mocked_languages_get_language.assert_called_once_with('English') mocked_db_get_language.assert_called_once_with('KJV') - self.assertFalse(self.mocked_log.error.called) + self.assertFalse(mocked_log_error.error.called) instance.save_meta.assert_called_once_with('language_id', 40) self.assertEqual(result, 40) @@ -162,8 +156,8 @@ class TestBibleImport(TestCase): # GIVEN: A mocked languages.get_language which returns None a mocked BibleDB.get_language which returns a # language id. with patch('openlp.core.common.languages.get_language', return_value=None) as mocked_languages_get_language, \ - patch('openlp.plugins.bibles.lib.db.BibleDB.get_language', return_value=None) as mocked_db_get_language: - self.mocked_log.error.reset_mock() + patch.object(BibleImport, 'get_language', return_value=None) as mocked_db_get_language, \ + patch.object(BibleImport, 'log_error') as mocked_log_error: instance = BibleImport(MagicMock()) instance.save_meta = MagicMock() @@ -173,7 +167,7 @@ class TestBibleImport(TestCase): # THEN: None should be returned and an error should be logged mocked_languages_get_language.assert_called_once_with('Qwerty') mocked_db_get_language.assert_called_once_with('KJV') - self.mocked_log.error.assert_called_once_with( + mocked_log_error.assert_called_once_with( 'Language detection failed when importing from "KJV". User aborted language selection.') self.assertFalse(instance.save_meta.called) self.assertIsNone(result) @@ -281,7 +275,6 @@ class TestBibleImport(TestCase): # THEN: parse_xml should have caught the error, informed the user and returned None self.mocked_log.exception.assert_called_once_with('Opening file.tst failed.') - self.mocked_trace_error_handler.assert_called_once_with(self.mocked_log) self.mocked_critical_error_message_box.assert_called_once_with( title='An Error Occured When Opening A File', message='The following error occurred when trying to open\nfile.tst:\n\nNo such file or directory') @@ -291,45 +284,45 @@ class TestBibleImport(TestCase): """ Test that parse_xml handles a FileNotFoundError exception correctly """ - # GIVEN: A mocked open which raises a FileNotFoundError and an instance of BibleImporter - exception = FileNotFoundError() - exception.filename = 'file.tst' - exception.strerror = 'No such file or directory' - self.mocked_open.side_effect = exception - importer = BibleImport(MagicMock(), path='.', name='.', filename='') + with patch.object(BibleImport, 'log_exception') as mocked_log_exception: + # GIVEN: A mocked open which raises a FileNotFoundError and an instance of BibleImporter + exception = FileNotFoundError() + exception.filename = 'file.tst' + exception.strerror = 'No such file or directory' + self.mocked_open.side_effect = exception + importer = BibleImport(MagicMock(), path='.', name='.', filename='') - # WHEN: Calling parse_xml - result = importer.parse_xml('file.tst') + # WHEN: Calling parse_xml + result = importer.parse_xml('file.tst') - # THEN: parse_xml should have caught the error, informed the user and returned None - self.mocked_log.exception.assert_called_once_with('Opening file.tst failed.') - self.mocked_trace_error_handler.assert_called_once_with(self.mocked_log) - self.mocked_critical_error_message_box.assert_called_once_with( - title='An Error Occured When Opening A File', - message='The following error occurred when trying to open\nfile.tst:\n\nNo such file or directory') - self.assertIsNone(result) + # THEN: parse_xml should have caught the error, informed the user and returned None + mocked_log_exception.assert_called_once_with('Opening file.tst failed.') + self.mocked_critical_error_message_box.assert_called_once_with( + title='An Error Occured When Opening A File', + message='The following error occurred when trying to open\nfile.tst:\n\nNo such file or directory') + self.assertIsNone(result) def parse_xml_file_permission_error_exception_test(self): """ Test that parse_xml handles a PermissionError exception correctly """ - # GIVEN: A mocked open which raises a PermissionError and an instance of BibleImporter - exception = PermissionError() - exception.filename = 'file.tst' - exception.strerror = 'Permission denied' - self.mocked_open.side_effect = exception - importer = BibleImport(MagicMock(), path='.', name='.', filename='') + with patch.object(BibleImport, 'log_exception') as mocked_log_exception: + # GIVEN: A mocked open which raises a PermissionError and an instance of BibleImporter + exception = PermissionError() + exception.filename = 'file.tst' + exception.strerror = 'Permission denied' + self.mocked_open.side_effect = exception + importer = BibleImport(MagicMock(), path='.', name='.', filename='') - # WHEN: Calling parse_xml - result = importer.parse_xml('file.tst') + # WHEN: Calling parse_xml + result = importer.parse_xml('file.tst') - # THEN: parse_xml should have caught the error, informed the user and returned None - self.mocked_log.exception.assert_called_once_with('Opening file.tst failed.') - self.mocked_trace_error_handler.assert_called_once_with(self.mocked_log) - self.mocked_critical_error_message_box.assert_called_once_with( - title='An Error Occured When Opening A File', - message='The following error occurred when trying to open\nfile.tst:\n\nPermission denied') - self.assertIsNone(result) + # THEN: parse_xml should have caught the error, informed the user and returned None + mocked_log_exception.assert_called_once_with('Opening file.tst failed.') + self.mocked_critical_error_message_box.assert_called_once_with( + title='An Error Occured When Opening A File', + message='The following error occurred when trying to open\nfile.tst:\n\nPermission denied') + self.assertIsNone(result) def validate_xml_file_compressed_file_test(self): """ diff --git a/tests/functional/openlp_plugins/bibles/test_csvimport.py b/tests/functional/openlp_plugins/bibles/test_csvimport.py index e72c85de6..a4a19279e 100644 --- a/tests/functional/openlp_plugins/bibles/test_csvimport.py +++ b/tests/functional/openlp_plugins/bibles/test_csvimport.py @@ -281,16 +281,14 @@ class TestCSVImport(TestCase): """ # GIVEN: An instance of CSVBible and a mocked get_language which simulates the user cancelling the language box mocked_manager = MagicMock() - with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\ - patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log: + with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'): importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verse.csv') importer.get_language = MagicMock(return_value=None) # WHEN: Calling do_import result = importer.do_import('Bible Name') - # THEN: The log.exception method should have been called to show that it reached the except clause. - # False should be returned. + # THEN: The False should be returned. importer.get_language.assert_called_once_with('Bible Name') self.assertFalse(result) @@ -300,8 +298,7 @@ class TestCSVImport(TestCase): """ # GIVEN: An instance of CSVBible mocked_manager = MagicMock() - with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\ - patch('openlp.plugins.bibles.lib.importers.csvbible.log') as mocked_log: + with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'): importer = CSVBible(mocked_manager, path='.', name='.', booksfile='books.csv', versefile='verses.csv') importer.get_language = MagicMock(return_value=10) importer.parse_csv_file = MagicMock(side_effect=[['Book 1'], ['Verse 1']]) @@ -314,9 +311,8 @@ class TestCSVImport(TestCase): # WHEN: Calling do_import result = importer.do_import('Bible Name') - # THEN: log.exception should not be called, parse_csv_file should be called twice, + # THEN: parse_csv_file should be called twice, # and True should be returned. - self.assertFalse(mocked_log.exception.called) self.assertEqual(importer.parse_csv_file.mock_calls, [call('books.csv', Book), call('verses.csv', Verse)]) importer.process_books.assert_called_once_with(['Book 1']) importer.process_verses.assert_called_once_with(['Verse 1'], ['Book 1']) diff --git a/tests/functional/openlp_plugins/bibles/test_opensongimport.py b/tests/functional/openlp_plugins/bibles/test_opensongimport.py index c902440b0..68ebdd37c 100644 --- a/tests/functional/openlp_plugins/bibles/test_opensongimport.py +++ b/tests/functional/openlp_plugins/bibles/test_opensongimport.py @@ -27,14 +27,12 @@ import json import os from unittest import TestCase -from lxml import etree, objectify +from lxml import objectify from tests.functional import MagicMock, patch, call from tests.helpers.testmixin import TestMixin from openlp.core.common import Registry -from openlp.core.lib.exceptions import ValidationError -from openlp.plugins.bibles.lib.importers.opensong import OpenSongBible, get_text, parse_chapter_number,\ - parse_verse_number +from openlp.plugins.bibles.lib.importers.opensong import OpenSongBible, get_text, parse_chapter_number from openlp.plugins.bibles.lib.bibleimport import BibleImport TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -127,9 +125,11 @@ class TestOpenSongImport(TestCase, TestMixin): """ Test parse_verse_number when supplied with a valid verse number """ - # GIVEN: The number 15 represented as a string and an instance of OpenSongBible + # GIVEN: An instance of OpenSongBible, the number 15 represented as a string and an instance of OpenSongBible + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + # WHEN: Calling parse_verse_number - result = parse_verse_number('15', 0) + result = importer.parse_verse_number('15', 0) # THEN: parse_verse_number should return the verse number self.assertEqual(result, 15) @@ -138,9 +138,11 @@ class TestOpenSongImport(TestCase, TestMixin): """ Test parse_verse_number when supplied with a verse range """ - # GIVEN: The range 24-26 represented as a string + # GIVEN: An instance of OpenSongBible, and the range 24-26 represented as a string + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + # WHEN: Calling parse_verse_number - result = parse_verse_number('24-26', 0) + result = importer.parse_verse_number('24-26', 0) # THEN: parse_verse_number should return the first verse number in the range self.assertEqual(result, 24) @@ -149,9 +151,11 @@ class TestOpenSongImport(TestCase, TestMixin): """ Test parse_verse_number when supplied with a invalid verse number """ - # GIVEN: An non numeric string represented as a string + # GIVEN: An instance of OpenSongBible, a non numeric string represented as a string + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + # WHEN: Calling parse_verse_number - result = parse_verse_number('invalid', 41) + result = importer.parse_verse_number('invalid', 41) # THEN: parse_verse_number should increment the previous verse number self.assertEqual(result, 42) @@ -160,25 +164,29 @@ class TestOpenSongImport(TestCase, TestMixin): """ Test parse_verse_number when the verse number is an empty string. (Bug #1074727) """ - # GIVEN: An empty string, and the previous verse number set as 14 + # GIVEN: An instance of OpenSongBible, an empty string, and the previous verse number set as 14 + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') # WHEN: Calling parse_verse_number - result = parse_verse_number('', 14) + result = importer.parse_verse_number('', 14) # THEN: parse_verse_number should increment the previous verse number self.assertEqual(result, 15) - @patch('openlp.plugins.bibles.lib.importers.opensong.log') - def parse_verse_number_invalid_type_test(self, mocked_log): + def parse_verse_number_invalid_type_test(self): """ Test parse_verse_number when the verse number is an invalid type) """ - # GIVEN: A mocked out log, a Tuple, and the previous verse number set as 12 - # WHEN: Calling parse_verse_number - result = parse_verse_number((1, 2, 3), 12) + with patch.object(OpenSongBible, 'log_warning')as mocked_log_warning: + # GIVEN: An instanceofOpenSongBible, a Tuple, and the previous verse number set as 12 + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - # THEN: parse_verse_number should log the verse number it was called with increment the previous verse number - mocked_log.warning.assert_called_once_with('Illegal verse number: (1, 2, 3)') - self.assertEqual(result, 13) + # WHEN: Calling parse_verse_number + result = importer.parse_verse_number((1, 2, 3), 12) + + # THEN: parse_verse_number should log the verse number it was called with increment the previous verse + # number + mocked_log_warning.assert_called_once_with('Illegal verse number: (1, 2, 3)') + self.assertEqual(result, 13) def process_books_stop_import_test(self): """ @@ -238,9 +246,8 @@ class TestOpenSongImport(TestCase, TestMixin): # THEN: importer.parse_chapter_number not have been called self.assertFalse(importer.parse_chapter_number.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.translate', **{'side_effect': lambda x, y: y}) @patch('openlp.plugins.bibles.lib.importers.opensong.parse_chapter_number', **{'side_effect': [1, 2]}) - def process_chapters_completes_test(self, mocked_parse_chapter_number, mocked_translate): + def process_chapters_completes_test(self, mocked_parse_chapter_number): """ Test process_chapters when it completes """ @@ -287,45 +294,47 @@ class TestOpenSongImport(TestCase, TestMixin): # THEN: importer.parse_verse_number not have been called self.assertFalse(importer.parse_verse_number.called) - @patch('openlp.plugins.bibles.lib.importers.opensong.parse_verse_number', **{'side_effect': [1, 2]}) - @patch('openlp.plugins.bibles.lib.importers.opensong.get_text', **{'side_effect': ['Verse1 Text', 'Verse2 Text']}) - def process_verses_completes_test(self, mocked_get_text, mocked_parse_verse_number): + def process_verses_completes_test(self): """ Test process_verses when it completes """ - # GIVEN: An instance of OpenSongBible - importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') - importer.wizard = MagicMock() + with patch('openlp.plugins.bibles.lib.importers.opensong.get_text', + **{'side_effect': ['Verse1 Text', 'Verse2 Text']}) as mocked_get_text, \ + patch.object(OpenSongBible, 'parse_verse_number', + **{'side_effect': [1, 2]}) as mocked_parse_verse_number: + # GIVEN: An instance of OpenSongBible + importer = OpenSongBible(MagicMock(), path='.', name='.', filename='') + importer.wizard = MagicMock() - # WHEN: called with some valid data - book = MagicMock() - book.id = 1 - verse1 = MagicMock() - verse1.attrib = {'n': '1'} - verse1.c = 'Chapter1' - verse1.v = ['Chapter1 Verses'] - verse2 = MagicMock() - verse2.attrib = {'n': '2'} - verse2.c = 'Chapter2' - verse2.v = ['Chapter2 Verses'] + # WHEN: called with some valid data + book = MagicMock() + book.id = 1 + verse1 = MagicMock() + verse1.attrib = {'n': '1'} + verse1.c = 'Chapter1' + verse1.v = ['Chapter1 Verses'] + verse2 = MagicMock() + verse2.attrib = {'n': '2'} + verse2.c = 'Chapter2' + verse2.v = ['Chapter2 Verses'] - importer.create_verse = MagicMock() - importer.stop_import_flag = False - importer.process_verses(book, 1, [verse1, verse2]) + importer.create_verse = MagicMock() + importer.stop_import_flag = False + importer.process_verses(book, 1, [verse1, verse2]) - # THEN: parse_chapter_number, process_verses and increment_process_bar should have been called - self.assertEqual(mocked_parse_verse_number.call_args_list, [call('1', 0), call('2', 1)]) - self.assertEqual(mocked_get_text.call_args_list, [call(verse1), call(verse2)]) - self.assertEqual( - importer.create_verse.call_args_list, - [call(1, 1, 1, 'Verse1 Text'), call(1, 1, 2, 'Verse2 Text')]) + # THEN: parse_chapter_number, process_verses and increment_process_bar should have been called + self.assertEqual(mocked_parse_verse_number.call_args_list, [call('1', 0), call('2', 1)]) + self.assertEqual(mocked_get_text.call_args_list, [call(verse1), call(verse2)]) + self.assertEqual( + importer.create_verse.call_args_list, + [call(1, 1, 1, 'Verse1 Text'), call(1, 1, 2, 'Verse2 Text')]) def do_import_parse_xml_fails_test(self): """ Test do_import when parse_xml fails (returns None) """ # GIVEN: An instance of OpenSongBible and a mocked parse_xml which returns False - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + with patch.object(OpenSongBible, 'log_debug'), \ patch.object(OpenSongBible, 'validate_xml_file'), \ patch.object(OpenSongBible, 'parse_xml', return_value=None), \ patch.object(OpenSongBible, 'get_language_id') as mocked_language_id: @@ -343,7 +352,7 @@ class TestOpenSongImport(TestCase, TestMixin): Test do_import when the user cancels the language selection dialog """ # GIVEN: An instance of OpenSongBible and a mocked get_language which returns False - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + with patch.object(OpenSongBible, 'log_debug'), \ patch.object(OpenSongBible, 'validate_xml_file'), \ patch.object(OpenSongBible, 'parse_xml'), \ patch.object(OpenSongBible, 'get_language_id', return_value=False), \ @@ -362,7 +371,7 @@ class TestOpenSongImport(TestCase, TestMixin): Test do_import when it completes successfully """ # GIVEN: An instance of OpenSongBible - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + with patch.object(OpenSongBible, 'log_debug'), \ patch.object(OpenSongBible, 'validate_xml_file'), \ patch.object(OpenSongBible, 'parse_xml'), \ patch.object(OpenSongBible, 'get_language_id', return_value=10), \ diff --git a/tests/functional/openlp_plugins/bibles/test_osisimport.py b/tests/functional/openlp_plugins/bibles/test_osisimport.py index 6cf157711..dd0f661a0 100644 --- a/tests/functional/openlp_plugins/bibles/test_osisimport.py +++ b/tests/functional/openlp_plugins/bibles/test_osisimport.py @@ -32,8 +32,7 @@ from openlp.plugins.bibles.lib.bibleimport import BibleImport from openlp.plugins.bibles.lib.db import BibleDB from openlp.plugins.bibles.lib.importers.osis import OSISBible -TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), - '..', '..', '..', 'resources', 'bibles')) +TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'bibles')) class TestOsisImport(TestCase): @@ -354,7 +353,7 @@ class TestOsisImport(TestCase): Test do_import when parse_xml fails (returns None) """ # GIVEN: An instance of OpenSongBible and a mocked parse_xml which returns False - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + with patch.object(OSISBible, 'log_debug'), \ patch.object(OSISBible, 'validate_xml_file'), \ patch.object(OSISBible, 'parse_xml', return_value=None), \ patch.object(OSISBible, 'get_language_id') as mocked_language_id: @@ -372,7 +371,7 @@ class TestOsisImport(TestCase): Test do_import when the user cancels the language selection dialog """ # GIVEN: An instance of OpenSongBible and a mocked get_language which returns False - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + with patch.object(OSISBible, 'log_debug'), \ patch.object(OSISBible, 'validate_xml_file'), \ patch.object(OSISBible, 'parse_xml'), \ patch.object(OSISBible, 'get_language_id', **{'return_value': False}), \ @@ -391,7 +390,7 @@ class TestOsisImport(TestCase): Test do_import when it completes successfully """ # GIVEN: An instance of OpenSongBible - with patch('openlp.plugins.bibles.lib.importers.opensong.log'), \ + with patch.object(OSISBible, 'log_debug'), \ patch.object(OSISBible, 'validate_xml_file'), \ patch.object(OSISBible, 'parse_xml'), \ patch.object(OSISBible, 'get_language_id', **{'return_value': 10}), \ From 223b198dc8f50bd8b4baa568cea61aa3078d8c7b Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Sun, 4 Sep 2016 22:16:43 +0100 Subject: [PATCH 47/65] Moved some import related stuff out of the db module --- openlp/plugins/bibles/lib/bibleimport.py | 73 +++++++++-- openlp/plugins/bibles/lib/db.py | 63 +--------- .../openlp_plugins/bibles/test_bibleimport.py | 118 ++++++++++++++---- .../openlp_plugins/bibles/test_csvimport.py | 2 +- .../openlp_plugins/bibles/test_db.py | 56 +-------- .../openlp_plugins/bibles/test_osisimport.py | 4 +- .../openlp_plugins/bibles/test_swordimport.py | 2 +- .../bibles/test_zefaniaimport.py | 2 +- 8 files changed, 171 insertions(+), 149 deletions(-) diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index eaef01aed..53f3a55e3 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -23,23 +23,21 @@ from lxml import etree, objectify from zipfile import is_zipfile -from openlp.core.common import OpenLPMixin, languages, translate +from openlp.core.common import OpenLPMixin, Registry, RegistryProperties, languages, translate from openlp.core.lib import ValidationError from openlp.core.lib.ui import critical_error_message_box -from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB +from openlp.plugins.bibles.lib.db import AlternativeBookNamesDB, BibleDB, BiblesResourcesDB -class BibleImport(OpenLPMixin, BibleDB): +class BibleImport(OpenLPMixin, RegistryProperties, BibleDB): """ Helper class to import bibles from a third party source into OpenLP """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.filename = kwargs['filename'] if 'filename' in kwargs else None - - def set_current_chapter(self, book_name, chapter_name): - self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport', 'Importing {book} {chapter}...') - .format(book=book_name, chapter=chapter_name)) + self.wizard = None + Registry().register_function('openlp_stop_wizard', self.stop_import) @staticmethod def is_compressed(file): @@ -56,6 +54,45 @@ class BibleImport(OpenLPMixin, BibleDB): return True return False + def get_book_ref_id_by_name(self, book, maxbooks, language_id=None): + self.log_debug('BibleDB.get_book_ref_id_by_name:("{book}", "{lang}")'.format(book=book, lang=language_id)) + book_id = None + if BiblesResourcesDB.get_book(book, True): + book_temp = BiblesResourcesDB.get_book(book, True) + book_id = book_temp['id'] + elif BiblesResourcesDB.get_alternative_book_name(book): + book_id = BiblesResourcesDB.get_alternative_book_name(book) + elif AlternativeBookNamesDB.get_book_reference_id(book): + book_id = AlternativeBookNamesDB.get_book_reference_id(book) + else: + from openlp.plugins.bibles.forms import BookNameForm + book_name = BookNameForm(self.wizard) + if book_name.exec(book, self.get_books(), maxbooks): + book_id = book_name.book_id + if book_id: + AlternativeBookNamesDB.create_alternative_book_name( + book, book_id, language_id) + return book_id + + def get_language(self, bible_name=None): + """ + If no language is given it calls a dialog window where the user could select the bible language. + Return the language id of a bible. + + :param bible_name: The language the bible is. + """ + self.log_debug('BibleImpoer.get_language()') + from openlp.plugins.bibles.forms import LanguageForm + language_id = None + language_form = LanguageForm(self.wizard) + if language_form.exec(bible_name): + combo_box = language_form.language_combo_box + language_id = combo_box.itemData(combo_box.currentIndex()) + if not language_id: + return None + self.save_meta('language_id', language_id) + return language_id + def get_language_id(self, file_language=None, bible_name=None): """ Get the language_id for the language of the bible. Fallback to user input if we cannot do this pragmatically. @@ -138,6 +175,28 @@ class BibleImport(OpenLPMixin, BibleDB): .format(file_name=e.filename, error=e.strerror)) return None + def register(self, wizard): + """ + This method basically just initialises the database. It is called from the Bible Manager when a Bible is + imported. Descendant classes may want to override this method to supply their own custom + initialisation as well. + + :param wizard: The actual Qt wizard form. + """ + self.wizard = wizard + return self.name + + def set_current_chapter(self, book_name, chapter_name): + self.wizard.increment_progress_bar(translate('BiblesPlugin.OsisImport', 'Importing {book} {chapter}...') + .format(book=book_name, chapter=chapter_name)) + + def stop_import(self): + """ + Stops the import of the Bible. + """ + self.log_debug('Stopping import') + self.stop_import_flag = True + def validate_xml_file(self, filename, tag): """ Validate the supplied file diff --git a/openlp/plugins/bibles/lib/db.py b/openlp/plugins/bibles/lib/db.py index e5142ae98..e7d29d016 100644 --- a/openlp/plugins/bibles/lib/db.py +++ b/openlp/plugins/bibles/lib/db.py @@ -33,7 +33,7 @@ from sqlalchemy.exc import OperationalError from sqlalchemy.orm import class_mapper, mapper, relation from sqlalchemy.orm.exc import UnmappedClassError -from openlp.core.common import Registry, RegistryProperties, AppLocation, translate, clean_filename +from openlp.core.common import AppLocation, translate, clean_filename from openlp.core.lib.db import BaseModel, init_db, Manager from openlp.core.lib.ui import critical_error_message_box from openlp.plugins.bibles.lib import upgrade @@ -106,7 +106,7 @@ def init_schema(url): return session -class BibleDB(Manager, RegistryProperties): +class BibleDB(Manager): """ This class represents a database-bound Bible. It is used as a base class for all the custom importers, so that the can implement their own import methods, but benefit from the database methods in here via inheritance, @@ -153,15 +153,6 @@ class BibleDB(Manager, RegistryProperties): self.get_name() if 'path' in kwargs: self.path = kwargs['path'] - self.wizard = None - Registry().register_function('openlp_stop_wizard', self.stop_import) - - def stop_import(self): - """ - Stops the import of the Bible. - """ - log.debug('Stopping import') - self.stop_import_flag = True def get_name(self): """ @@ -171,17 +162,6 @@ class BibleDB(Manager, RegistryProperties): self.name = version_name.value if version_name else None return self.name - def register(self, wizard): - """ - This method basically just initialises the database. It is called from the Bible Manager when a Bible is - imported. Descendant classes may want to override this method to supply their own custom - initialisation as well. - - :param wizard: The actual Qt wizard form. - """ - self.wizard = wizard - return self.name - def create_book(self, name, bk_ref_id, testament=1): """ Add a book to the database. @@ -306,26 +286,6 @@ class BibleDB(Manager, RegistryProperties): log.debug('BibleDB.get_book_by_book_ref_id("{ref}")'.format(ref=ref_id)) return self.get_object_filtered(Book, Book.book_reference_id.like(ref_id)) - def get_book_ref_id_by_name(self, book, maxbooks, language_id=None): - log.debug('BibleDB.get_book_ref_id_by_name:("{book}", "{lang}")'.format(book=book, lang=language_id)) - book_id = None - if BiblesResourcesDB.get_book(book, True): - book_temp = BiblesResourcesDB.get_book(book, True) - book_id = book_temp['id'] - elif BiblesResourcesDB.get_alternative_book_name(book): - book_id = BiblesResourcesDB.get_alternative_book_name(book) - elif AlternativeBookNamesDB.get_book_reference_id(book): - book_id = AlternativeBookNamesDB.get_book_reference_id(book) - else: - from openlp.plugins.bibles.forms import BookNameForm - book_name = BookNameForm(self.wizard) - if book_name.exec(book, self.get_books(), maxbooks): - book_id = book_name.book_id - if book_id: - AlternativeBookNamesDB.create_alternative_book_name( - book, book_id, language_id) - return book_id - def get_book_ref_id_by_localised_name(self, book, language_selection): """ Return the id of a named book. @@ -462,25 +422,6 @@ class BibleDB(Manager, RegistryProperties): return 0 return count - def get_language(self, bible_name=None): - """ - If no language is given it calls a dialog window where the user could select the bible language. - Return the language id of a bible. - - :param bible_name: The language the bible is. - """ - log.debug('BibleDB.get_language()') - from openlp.plugins.bibles.forms import LanguageForm - language_id = None - language_form = LanguageForm(self.wizard) - if language_form.exec(bible_name): - combo_box = language_form.language_combo_box - language_id = combo_box.itemData(combo_box.currentIndex()) - if not language_id: - return None - self.save_meta('language_id', language_id) - return language_id - def dump_bible(self): """ Utility debugging method to dump the contents of a bible. diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index 50baa655b..30ab45352 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -62,6 +62,9 @@ class TestBibleImport(TestCase): side_effect=lambda module, string_to_translate, *args: string_to_translate) self.addCleanup(self.translate_patcher.stop) self.mocked_translate = self.translate_patcher.start() + self.registry_patcher = patch('openlp.plugins.bibles.lib.bibleimport.Registry') + self.addCleanup(self.registry_patcher.stop) + self.registry_patcher.start() def init_kwargs_none_test(self): """ @@ -88,6 +91,54 @@ class TestBibleImport(TestCase): self.assertEqual(instance.filename, 'bible.xml') self.assertIsInstance(instance, BibleDB) + def get_language_canceled_test(self): + """ + Test the BibleImport.get_language method when the user rejects the dialog box + """ + # GIVEN: A mocked LanguageForm with an exec method which returns QtDialog.Rejected and an instance of BibleDB + with patch.object(BibleDB, '_setup'), patch('openlp.plugins.bibles.forms.LanguageForm') as mocked_language_form: + + # The integer value of QtDialog.Rejected is 0. Using the enumeration causes a seg fault for some reason + mocked_language_form_instance = MagicMock(**{'exec.return_value': 0}) + mocked_language_form.return_value = mocked_language_form_instance + instance = BibleImport(MagicMock()) + mocked_wizard = MagicMock() + instance.wizard = mocked_wizard + + # WHEN: Calling get_language() + result = instance.get_language() + + # THEN: get_language() should return False + mocked_language_form.assert_called_once_with(mocked_wizard) + mocked_language_form_instance.exec.assert_called_once_with(None) + self.assertFalse(result, 'get_language() should return False if the user rejects the dialog box') + + def get_language_accepted_test(self): + """ + Test the BibleImport.get_language method when the user accepts the dialog box + """ + # GIVEN: A mocked LanguageForm with an exec method which returns QtDialog.Accepted an instance of BibleDB and + # a combobox with the selected item data as 10 + with patch.object(BibleDB, 'save_meta'), patch.object(BibleDB, '_setup'), \ + patch('openlp.plugins.bibles.forms.LanguageForm') as mocked_language_form: + + # The integer value of QtDialog.Accepted is 1. Using the enumeration causes a seg fault for some reason + mocked_language_form_instance = MagicMock(**{'exec.return_value': 1, + 'language_combo_box.itemData.return_value': 10}) + mocked_language_form.return_value = mocked_language_form_instance + instance = BibleImport(MagicMock()) + mocked_wizard = MagicMock() + instance.wizard = mocked_wizard + + # WHEN: Calling get_language() + result = instance.get_language('Bible Name') + + # THEN: get_language() should return the id of the selected language in the combo box + mocked_language_form.assert_called_once_with(mocked_wizard) + mocked_language_form_instance.exec.assert_called_once_with('Bible Name') + self.assertEqual(result, 10, 'get_language() should return the id of the language the user has chosen when ' + 'they accept the dialog box') + def get_language_id_language_found_test(self): """ Test get_language_id() when called with a name found in the languages list @@ -172,6 +223,38 @@ class TestBibleImport(TestCase): self.assertFalse(instance.save_meta.called) self.assertIsNone(result) + def is_compressed_compressed_test(self): + """ + Test is_compressed when the 'file' being tested is compressed + """ + # GIVEN: An instance of BibleImport and a mocked is_zipfile which returns True + with patch('openlp.plugins.bibles.lib.bibleimport.is_zipfile', return_value=True): + instance = BibleImport(MagicMock()) + + # WHEN: Calling is_compressed + result = instance.is_compressed('file.ext') + + # THEN: Then critical_error_message_box should be called informing the user that the file is compressed and + # True should be returned + self.mocked_critical_error_message_box.assert_called_once_with( + message='The file "file.ext" you supplied is compressed. You must decompress it before import.') + self.assertTrue(result) + + def is_compressed_not_compressed_test(self): + """ + Test is_compressed when the 'file' being tested is compressed + """ + # GIVEN: An instance of BibleImport and a mocked is_zipfile which returns True + with patch('openlp.plugins.bibles.lib.bibleimport.is_zipfile', return_value=False): + instance = BibleImport(MagicMock()) + + # WHEN: Calling is_compressed + result = instance.is_compressed('file.ext') + + # THEN: False should be returned and critical_error_message_box should not have been called + self.assertFalse(result) + self.assertFalse(self.mocked_critical_error_message_box.called) + def parse_xml_etree_test(self): """ Test BibleImport.parse_xml() when called with the use_objectify default value @@ -259,27 +342,6 @@ class TestBibleImport(TestCase): # THEN: The result returned should contain the correct data self.assertEqual(etree.tostring(result), b'\n Testdatatokeep\n \n') - def parse_xml_file_file_not_found_exception_test(self): - """ - Test that validate_xml_file raises a ValidationError with an OpenSong root tag - """ - # GIVEN: A mocked open which raises a FileNotFoundError and an instance of BibleImporter - exception = FileNotFoundError() - exception.filename = 'file.tst' - exception.strerror = 'No such file or directory' - self.mocked_open.side_effect = exception - importer = BibleImport(MagicMock(), path='.', name='.', filename='') - - # WHEN: Calling parse_xml - result = importer.parse_xml('file.tst') - - # THEN: parse_xml should have caught the error, informed the user and returned None - self.mocked_log.exception.assert_called_once_with('Opening file.tst failed.') - self.mocked_critical_error_message_box.assert_called_once_with( - title='An Error Occured When Opening A File', - message='The following error occurred when trying to open\nfile.tst:\n\nNo such file or directory') - self.assertIsNone(result) - def parse_xml_file_file_not_found_exception_test(self): """ Test that parse_xml handles a FileNotFoundError exception correctly @@ -324,6 +386,20 @@ class TestBibleImport(TestCase): message='The following error occurred when trying to open\nfile.tst:\n\nPermission denied') self.assertIsNone(result) + def set_current_chapter_test(self): + """ + Test set_current_chapter + """ + # GIVEN: An instance of BibleImport and a mocked wizard + importer = BibleImport(MagicMock(), path='.', name='.', filename='') + importer.wizard = MagicMock() + + # WHEN: Calling set_current_chapter + importer.set_current_chapter('Book_Name', 'Chapter') + + # THEN: Increment_progress_bar should have been called with a text string + importer.wizard.increment_progress_bar.assert_called_once_with('Importing Book_Name Chapter...') + def validate_xml_file_compressed_file_test(self): """ Test that validate_xml_file raises a ValidationError when is_compressed returns True diff --git a/tests/functional/openlp_plugins/bibles/test_csvimport.py b/tests/functional/openlp_plugins/bibles/test_csvimport.py index a4a19279e..8eff7274e 100644 --- a/tests/functional/openlp_plugins/bibles/test_csvimport.py +++ b/tests/functional/openlp_plugins/bibles/test_csvimport.py @@ -48,7 +48,7 @@ class TestCSVImport(TestCase): self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') self.addCleanup(self.manager_patcher.stop) self.manager_patcher.start() - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.registry_patcher = patch('openlp.plugins.bibles.lib.bibleimport.Registry') self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() diff --git a/tests/functional/openlp_plugins/bibles/test_db.py b/tests/functional/openlp_plugins/bibles/test_db.py index 2807a8a3e..75e008953 100644 --- a/tests/functional/openlp_plugins/bibles/test_db.py +++ b/tests/functional/openlp_plugins/bibles/test_db.py @@ -25,63 +25,9 @@ This module contains tests for the db submodule of the Bibles plugin. from unittest import TestCase -from openlp.plugins.bibles.lib.db import BibleDB -from tests.functional import MagicMock, patch - class TestBibleDB(TestCase): """ Test the functions in the BibleDB class. """ - - def test_get_language_canceled(self): - """ - Test the BibleDB.get_language method when the user rejects the dialog box - """ - # GIVEN: A mocked LanguageForm with an exec method which returns QtDialog.Rejected and an instance of BibleDB - with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'),\ - patch('openlp.plugins.bibles.forms.LanguageForm') as mocked_language_form: - - # The integer value of QtDialog.Rejected is 0. Using the enumeration causes a seg fault for some reason - mocked_language_form_instance = MagicMock(**{'exec.return_value': 0}) - mocked_language_form.return_value = mocked_language_form_instance - mocked_parent = MagicMock() - instance = BibleDB(mocked_parent) - mocked_wizard = MagicMock() - instance.wizard = mocked_wizard - - # WHEN: Calling get_language() - result = instance.get_language() - - # THEN: get_language() should return False - mocked_language_form.assert_called_once_with(mocked_wizard) - mocked_language_form_instance.exec.assert_called_once_with(None) - self.assertFalse(result, 'get_language() should return False if the user rejects the dialog box') - - def test_get_language_accepted(self): - """ - Test the BibleDB.get_language method when the user accepts the dialog box - """ - # GIVEN: A mocked LanguageForm with an exec method which returns QtDialog.Accepted an instance of BibleDB and - # a combobox with the selected item data as 10 - with patch('openlp.plugins.bibles.lib.db.BibleDB._setup'), \ - patch('openlp.plugins.bibles.lib.db.BibleDB.save_meta'), \ - patch('openlp.plugins.bibles.forms.LanguageForm') as mocked_language_form: - - # The integer value of QtDialog.Accepted is 1. Using the enumeration causes a seg fault for some reason - mocked_language_form_instance = MagicMock(**{'exec.return_value': 1, - 'language_combo_box.itemData.return_value': 10}) - mocked_language_form.return_value = mocked_language_form_instance - mocked_parent = MagicMock() - instance = BibleDB(mocked_parent) - mocked_wizard = MagicMock() - instance.wizard = mocked_wizard - - # WHEN: Calling get_language() - result = instance.get_language('Bible Name') - - # THEN: get_language() should return the id of the selected language in the combo box - mocked_language_form.assert_called_once_with(mocked_wizard) - mocked_language_form_instance.exec.assert_called_once_with('Bible Name') - self.assertEqual(result, 10, 'get_language() should return the id of the language the user has chosen when ' - 'they accept the dialog box') + pass diff --git a/tests/functional/openlp_plugins/bibles/test_osisimport.py b/tests/functional/openlp_plugins/bibles/test_osisimport.py index dd0f661a0..eaaea7206 100644 --- a/tests/functional/openlp_plugins/bibles/test_osisimport.py +++ b/tests/functional/openlp_plugins/bibles/test_osisimport.py @@ -49,7 +49,7 @@ class TestOsisImport(TestCase): self.find_and_create_book_patch = patch.object(BibleImport, 'find_and_create_book') self.addCleanup(self.find_and_create_book_patch.stop) self.mocked_find_and_create_book = self.find_and_create_book_patch.start() - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.registry_patcher = patch('openlp.plugins.bibles.lib.bibleimport.Registry') self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') @@ -409,7 +409,7 @@ class TestOsisImportFileImports(TestCase): Test the functions in the :mod:`osisimport` module. """ def setUp(self): - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.registry_patcher = patch('openlp.plugins.bibles.lib.bibleimport.Registry') self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') diff --git a/tests/functional/openlp_plugins/bibles/test_swordimport.py b/tests/functional/openlp_plugins/bibles/test_swordimport.py index 51e629f53..261df1c0e 100644 --- a/tests/functional/openlp_plugins/bibles/test_swordimport.py +++ b/tests/functional/openlp_plugins/bibles/test_swordimport.py @@ -46,7 +46,7 @@ class TestSwordImport(TestCase): """ def setUp(self): - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.registry_patcher = patch('openlp.plugins.bibles.lib.bibleimport.Registry') self.registry_patcher.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') self.manager_patcher.start() diff --git a/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py b/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py index 11fcfb7e3..510961a65 100644 --- a/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py +++ b/tests/functional/openlp_plugins/bibles/test_zefaniaimport.py @@ -41,7 +41,7 @@ class TestZefaniaImport(TestCase): """ def setUp(self): - self.registry_patcher = patch('openlp.plugins.bibles.lib.db.Registry') + self.registry_patcher = patch('openlp.plugins.bibles.lib.bibleimport.Registry') self.addCleanup(self.registry_patcher.stop) self.registry_patcher.start() self.manager_patcher = patch('openlp.plugins.bibles.lib.db.Manager') From 9a049929fc71bfb739afb4bd01520a265cdc7cea Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Mon, 5 Sep 2016 19:02:47 +0100 Subject: [PATCH 48/65] Missed an attribute --- openlp/plugins/bibles/lib/bibleimport.py | 1 + openlp/plugins/bibles/lib/db.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index 53f3a55e3..6761d84e8 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -37,6 +37,7 @@ class BibleImport(OpenLPMixin, RegistryProperties, BibleDB): super().__init__(*args, **kwargs) self.filename = kwargs['filename'] if 'filename' in kwargs else None self.wizard = None + self.stop_import_flag = False Registry().register_function('openlp_stop_wizard', self.stop_import) @staticmethod diff --git a/openlp/plugins/bibles/lib/db.py b/openlp/plugins/bibles/lib/db.py index e7d29d016..3a0a89757 100644 --- a/openlp/plugins/bibles/lib/db.py +++ b/openlp/plugins/bibles/lib/db.py @@ -140,7 +140,6 @@ class BibleDB(Manager): raise KeyError('Missing keyword argument "path".') if 'name' not in kwargs and 'file' not in kwargs: raise KeyError('Missing keyword argument "name" or "file".') - self.stop_import_flag = False if 'name' in kwargs: self.name = kwargs['name'] if not isinstance(self.name, str): From 4a71bda0d1741f068bf551f6c680ebbff179014c Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Mon, 5 Sep 2016 21:27:14 +0100 Subject: [PATCH 49/65] More BibleImport tests --- .../openlp_plugins/bibles/test_bibleimport.py | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index 30ab45352..6de470d87 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -27,6 +27,7 @@ from io import BytesIO from lxml import etree, objectify from unittest import TestCase +from PyQt5.QtWidgets import QDialog from openlp.core.common.languages import Language from openlp.core.lib.exceptions import ValidationError @@ -223,6 +224,100 @@ class TestBibleImport(TestCase): self.assertFalse(instance.save_meta.called) self.assertIsNone(result) + def get_book_ref_id_by_name_get_book_test(self): + """ + Test get_book_ref_id_by_name when the book is found as a book in BiblesResourcesDB + """ + # GIVEN: An instance of BibleImport and a mocked BiblesResourcesDB which returns a book id when get_book is + # called + with patch.object(BibleImport, 'log_debug'), \ + patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', + **{'get_book.return_value':{'id': 20}}): + instance = BibleImport(MagicMock()) + + # WHEN: Calling get_book_ref_id_by_name + result = instance.get_book_ref_id_by_name('Gen', 66, 4) + + # THEN: The bible id should be returned + self.assertEqual(result,20) + + def get_book_ref_id_by_name_get_alternative_book_name_test(self): + """ + Test get_book_ref_id_by_name when the book is found as an alternative book in BiblesResourcesDB + """ + # GIVEN: An instance of BibleImport and a mocked BiblesResourcesDB which returns a book id when + # get_alternative_book_name is called + with patch.object(BibleImport, 'log_debug'), \ + patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', + **{'get_book.return_value': None, 'get_alternative_book_name.return_value': 30}): + instance = BibleImport(MagicMock()) + + # WHEN: Calling get_book_ref_id_by_name + result = instance.get_book_ref_id_by_name('Gen', 66, 4) + + # THEN: The bible id should be returned + self.assertEqual(result, 30) + + def get_book_ref_id_by_name_get_book_reference_id_test(self): + """ + Test get_book_ref_id_by_name when the book is found as a book in AlternativeBookNamesDB + """ + # GIVEN: An instance of BibleImport and a mocked AlternativeBookNamesDB which returns a book id when + # get_book_reference_id is called + with patch.object(BibleImport, 'log_debug'), \ + patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', + **{'get_book.return_value': None, 'get_alternative_book_name.return_value': None}), \ + patch('openlp.plugins.bibles.lib.bibleimport.AlternativeBookNamesDB', + **{'get_book_reference_id.return_value': 40}): + instance = BibleImport(MagicMock()) + + # WHEN: Calling get_book_ref_id_by_name + result = instance.get_book_ref_id_by_name('Gen', 66, 4) + + # THEN: The bible id should be returned + self.assertEqual(result, 40) + + def get_book_ref_id_by_name_book_name_form_rejected_test(self): + """ + Test get_book_ref_id_by_name when the user rejects the BookNameForm + """ + # GIVEN: An instance of BibleImport and a mocked BookNameForm which simulates a user rejecting the dialog + with patch.object(BibleImport, 'log_debug'), patch.object(BibleImport, 'get_books'), \ + patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', + **{'get_book.return_value': None, 'get_alternative_book_name.return_value': None}), \ + patch('openlp.plugins.bibles.lib.bibleimport.AlternativeBookNamesDB', + **{'get_book_reference_id.return_value': None}), \ + patch('openlp.plugins.bibles.forms.BookNameForm', + return_value=MagicMock(**{'exec.return_value': QDialog.Rejected})): + instance = BibleImport(MagicMock()) + + # WHEN: Calling get_book_ref_id_by_name + result = instance.get_book_ref_id_by_name('Gen', 66, 4) + + # THEN: None should be returned + self.assertIsNone(result) + + def get_book_ref_id_by_name_book_name_form_accepted_test(self): + """ + Test get_book_ref_id_by_name when the user accepts the BookNameForm + """ + # GIVEN: An instance of BibleImport and a mocked BookNameForm which simulates a user accepting the dialog + with patch.object(BibleImport, 'log_debug'), patch.object(BibleImport, 'get_books'), \ + patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', + **{'get_book.return_value': None, 'get_alternative_book_name.return_value': None}), \ + patch('openlp.plugins.bibles.lib.bibleimport.AlternativeBookNamesDB', + **{'get_book_reference_id.return_value': None}) as mocked_alternative_book_names_db, \ + patch('openlp.plugins.bibles.forms.BookNameForm', + return_value=MagicMock(**{'exec.return_value': QDialog.Accepted, 'book_id':50})): + instance = BibleImport(MagicMock()) + + # WHEN: Calling get_book_ref_id_by_name + result = instance.get_book_ref_id_by_name('Gen', 66, 4) + + # THEN: An alternative book name should be created and a bible id should be returned + mocked_alternative_book_names_db.create_alternative_book_name.assert_called_once_with('Gen', 50, 4) + self.assertEqual(result, 50) + def is_compressed_compressed_test(self): """ Test is_compressed when the 'file' being tested is compressed @@ -242,9 +337,9 @@ class TestBibleImport(TestCase): def is_compressed_not_compressed_test(self): """ - Test is_compressed when the 'file' being tested is compressed + Test is_compressed when the 'file' being tested is not compressed """ - # GIVEN: An instance of BibleImport and a mocked is_zipfile which returns True + # GIVEN: An instance of BibleImport and a mocked is_zipfile which returns False with patch('openlp.plugins.bibles.lib.bibleimport.is_zipfile', return_value=False): instance = BibleImport(MagicMock()) From 359dded22a35f8e5f36cde218f553fb2813d8677 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 9 Sep 2016 10:51:19 +0300 Subject: [PATCH 50/65] - Fixed the test --- tests/functional/openlp_plugins/bibles/test_mediaitem.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index bacafe009..3aea7d825 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -155,7 +155,6 @@ class TestMediaItem(TestCase, TestMixin): """ Test that the on_clear_button_clicked works properly. (Used by Bible search tab) """ - # GIVEN: Mocked list_view, check_search_results & quick_search_edit. self.media_item.list_view = MagicMock() self.media_item.check_search_result = MagicMock() @@ -165,7 +164,7 @@ class TestMediaItem(TestCase, TestMixin): self.media_item.on_clear_button_clicked() # THEN: Search result should be reset and search field should receive focus. - self.assertEqual(1, self.media_item.list_view.clear.call_count, 'List_view.clear should had been called once.') - self.assertEqual(1, self.media_item.check_search_result.call_count, 'Check results Should had been called once') - self.assertEqual(1, self.media_item.quick_search_edit.clear.call_count, 'Should had been called once') - self.assertEqual(1, self.media_item.quick_search_edit.setFocus.call_count, 'Should had been called once') + self.media_item.list_view.clear.assert_called_once_with(), + self.media_item.check_search_result.assert_called_once_with(), + self.media_item.quick_search_edit.clear.assert_called_once_with(), + self.media_item.quick_search_edit.setFocus.assert_called_once_with() \ No newline at end of file From a99bd8603f4ab101a213ff581eb3ca88b6a30b74 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Fri, 9 Sep 2016 10:57:11 +0300 Subject: [PATCH 51/65] - Combined some tags in expection dialogue. - Added new line to the end of the test. --- openlp/core/ui/exceptiondialog.py | 6 +++--- tests/functional/openlp_plugins/bibles/test_mediaitem.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openlp/core/ui/exceptiondialog.py b/openlp/core/ui/exceptiondialog.py index 7341287c7..bcbc67552 100644 --- a/openlp/core/ui/exceptiondialog.py +++ b/openlp/core/ui/exceptiondialog.py @@ -97,9 +97,9 @@ class Ui_ExceptionDialog(object): translate('OpenLP.ExceptionDialog', 'Please describe what you were trying to do. ' ' If possible, write in English.')) exception_part1 = (translate('OpenLP.ExceptionDialog', - 'Oops, OpenLP hit a problem and couldn\'t recover!

' - 'You can help the OpenLP developers to fix this' - ' by
sending them a bug report to {email}{newlines}' + 'Oops, OpenLP hit a problem and couldn\'t recover!

' + 'You can help the OpenLP developers to fix this' + ' by
sending them a bug report to {email}{newlines}' ).format(email=' bugs@openlp.org', newlines='

')) self.message_label.setText( diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 3aea7d825..6d468275b 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -167,4 +167,4 @@ class TestMediaItem(TestCase, TestMixin): self.media_item.list_view.clear.assert_called_once_with(), self.media_item.check_search_result.assert_called_once_with(), self.media_item.quick_search_edit.clear.assert_called_once_with(), - self.media_item.quick_search_edit.setFocus.assert_called_once_with() \ No newline at end of file + self.media_item.quick_search_edit.setFocus.assert_called_once_with() From 29bc21d807931eab68e585b89eee1ac28efa9fdb Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Fri, 9 Sep 2016 22:36:10 +0100 Subject: [PATCH 52/65] Doc string --- openlp/plugins/bibles/lib/bibleimport.py | 41 +++++++++++++----------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/openlp/plugins/bibles/lib/bibleimport.py b/openlp/plugins/bibles/lib/bibleimport.py index 6761d84e8..5eb22e47d 100644 --- a/openlp/plugins/bibles/lib/bibleimport.py +++ b/openlp/plugins/bibles/lib/bibleimport.py @@ -55,25 +55,30 @@ class BibleImport(OpenLPMixin, RegistryProperties, BibleDB): return True return False - def get_book_ref_id_by_name(self, book, maxbooks, language_id=None): + def get_book_ref_id_by_name(self, book, maxbooks=66, language_id=None): + """ + Find the book id from the name or abbreviation of the book. If it doesn't currently exist, ask the user. + + :param book: The name or abbreviation of the book + :param maxbooks: The number of books in the bible + :param language_id: The language_id of the bible + :return: The id of the bible, or None + """ self.log_debug('BibleDB.get_book_ref_id_by_name:("{book}", "{lang}")'.format(book=book, lang=language_id)) - book_id = None - if BiblesResourcesDB.get_book(book, True): - book_temp = BiblesResourcesDB.get_book(book, True) - book_id = book_temp['id'] - elif BiblesResourcesDB.get_alternative_book_name(book): - book_id = BiblesResourcesDB.get_alternative_book_name(book) - elif AlternativeBookNamesDB.get_book_reference_id(book): - book_id = AlternativeBookNamesDB.get_book_reference_id(book) - else: - from openlp.plugins.bibles.forms import BookNameForm - book_name = BookNameForm(self.wizard) - if book_name.exec(book, self.get_books(), maxbooks): - book_id = book_name.book_id - if book_id: - AlternativeBookNamesDB.create_alternative_book_name( - book, book_id, language_id) - return book_id + book_temp = BiblesResourcesDB.get_book(book, True) + if book_temp: + return book_temp['id'] + book_id = BiblesResourcesDB.get_alternative_book_name(book) + if book_id: + return book_id + book_id = AlternativeBookNamesDB.get_book_reference_id(book) + if book_id: + return book_id + from openlp.plugins.bibles.forms import BookNameForm + book_name = BookNameForm(self.wizard) + if book_name.exec(book, self.get_books(), maxbooks) and book_name.book_id: + AlternativeBookNamesDB.create_alternative_book_name(book, book_name.book_id, language_id) + return book_name.book_id def get_language(self, bible_name=None): """ From 2dd770ea119d77e3b9d3edea1c91a33cd519d180 Mon Sep 17 00:00:00 2001 From: Philip Ridout Date: Fri, 9 Sep 2016 22:47:29 +0100 Subject: [PATCH 53/65] PEP fixes --- .../openlp_plugins/bibles/test_bibleimport.py | 18 +++++++++--------- .../openlp_plugins/bibles/test_osisimport.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/functional/openlp_plugins/bibles/test_bibleimport.py b/tests/functional/openlp_plugins/bibles/test_bibleimport.py index 6de470d87..a2244dbf7 100644 --- a/tests/functional/openlp_plugins/bibles/test_bibleimport.py +++ b/tests/functional/openlp_plugins/bibles/test_bibleimport.py @@ -232,14 +232,14 @@ class TestBibleImport(TestCase): # called with patch.object(BibleImport, 'log_debug'), \ patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', - **{'get_book.return_value':{'id': 20}}): + **{'get_book.return_value': {'id': 20}}): instance = BibleImport(MagicMock()) # WHEN: Calling get_book_ref_id_by_name result = instance.get_book_ref_id_by_name('Gen', 66, 4) # THEN: The bible id should be returned - self.assertEqual(result,20) + self.assertEqual(result, 20) def get_book_ref_id_by_name_get_alternative_book_name_test(self): """ @@ -266,7 +266,7 @@ class TestBibleImport(TestCase): # get_book_reference_id is called with patch.object(BibleImport, 'log_debug'), \ patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', - **{'get_book.return_value': None, 'get_alternative_book_name.return_value': None}), \ + **{'get_book.return_value': None, 'get_alternative_book_name.return_value': None}), \ patch('openlp.plugins.bibles.lib.bibleimport.AlternativeBookNamesDB', **{'get_book_reference_id.return_value': 40}): instance = BibleImport(MagicMock()) @@ -303,12 +303,12 @@ class TestBibleImport(TestCase): """ # GIVEN: An instance of BibleImport and a mocked BookNameForm which simulates a user accepting the dialog with patch.object(BibleImport, 'log_debug'), patch.object(BibleImport, 'get_books'), \ - patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', - **{'get_book.return_value': None, 'get_alternative_book_name.return_value': None}), \ - patch('openlp.plugins.bibles.lib.bibleimport.AlternativeBookNamesDB', - **{'get_book_reference_id.return_value': None}) as mocked_alternative_book_names_db, \ - patch('openlp.plugins.bibles.forms.BookNameForm', - return_value=MagicMock(**{'exec.return_value': QDialog.Accepted, 'book_id':50})): + patch('openlp.plugins.bibles.lib.bibleimport.BiblesResourcesDB', + **{'get_book.return_value': None, 'get_alternative_book_name.return_value': None}), \ + patch('openlp.plugins.bibles.lib.bibleimport.AlternativeBookNamesDB', + **{'get_book_reference_id.return_value': None}) as mocked_alternative_book_names_db, \ + patch('openlp.plugins.bibles.forms.BookNameForm', + return_value=MagicMock(**{'exec.return_value': QDialog.Accepted, 'book_id': 50})): instance = BibleImport(MagicMock()) # WHEN: Calling get_book_ref_id_by_name diff --git a/tests/functional/openlp_plugins/bibles/test_osisimport.py b/tests/functional/openlp_plugins/bibles/test_osisimport.py index eaaea7206..18f591faf 100644 --- a/tests/functional/openlp_plugins/bibles/test_osisimport.py +++ b/tests/functional/openlp_plugins/bibles/test_osisimport.py @@ -32,7 +32,7 @@ from openlp.plugins.bibles.lib.bibleimport import BibleImport from openlp.plugins.bibles.lib.db import BibleDB from openlp.plugins.bibles.lib.importers.osis import OSISBible -TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'bibles')) +TEST_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'bibles')) class TestOsisImport(TestCase): From 7ea310585701cb64c183ab89a3f27454b4c3c0cd Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Mon, 12 Sep 2016 14:41:38 +0300 Subject: [PATCH 54/65] - Fixed some .forma tags to .format --- openlp/plugins/custom/lib/mediaitem.py | 2 +- openlp/plugins/songs/forms/editsongform.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/custom/lib/mediaitem.py b/openlp/plugins/custom/lib/mediaitem.py index e9e26111f..2b999da19 100644 --- a/openlp/plugins/custom/lib/mediaitem.py +++ b/openlp/plugins/custom/lib/mediaitem.py @@ -350,7 +350,7 @@ class CustomMediaItem(MediaManagerItem): :param string: The search string :param show_error: The error string to be show. """ - search = '%{search}%'.forma(search=string.lower()) + search = '%{search}%'.format(search=string.lower()) search_results = self.plugin.db_manager.get_all_objects(CustomSlide, or_(func.lower(CustomSlide.title).like(search), func.lower(CustomSlide.text).like(search)), diff --git a/openlp/plugins/songs/forms/editsongform.py b/openlp/plugins/songs/forms/editsongform.py index 200e1436f..a17c9fb5f 100644 --- a/openlp/plugins/songs/forms/editsongform.py +++ b/openlp/plugins/songs/forms/editsongform.py @@ -317,7 +317,7 @@ class EditSongForm(QtWidgets.QDialog, Ui_EditSongDialog, RegistryProperties): self.song.verse_order = re.sub('([' + verse.upper() + verse.lower() + '])(\W|$)', r'\g<1>1\2', self.song.verse_order) except: - log.exception('Problem processing song Lyrics \n{xml}'.forma(xml=sxml.dump_xml())) + log.exception('Problem processing song Lyrics \n{xml}'.format(xml=sxml.dump_xml())) raise def keyPressEvent(self, event): From 1fae73014ec71982e88df324c2f04badf62c2b42 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Mon, 12 Sep 2016 15:11:59 +0300 Subject: [PATCH 55/65] - Removed one unrequired code line --- openlp/core/ui/slidecontroller.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openlp/core/ui/slidecontroller.py b/openlp/core/ui/slidecontroller.py index 51794cdd3..13a09d5df 100644 --- a/openlp/core/ui/slidecontroller.py +++ b/openlp/core/ui/slidecontroller.py @@ -1043,8 +1043,6 @@ class SlideController(DisplayController, RegistryProperties): """ self.blank_screen.setChecked(False) self.theme_screen.setChecked(False) - # Not sure if this line is required. - self.desktop_screen.setChecked(checked) Registry().execute('live_display_hide', HideMode.Screen) self.desktop_screen.setChecked(True) self.update_preview() From a034532d195841920e21c4f875b742a97f3a881b Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Thu, 15 Sep 2016 02:17:17 +0300 Subject: [PATCH 56/65] - Fixed a bug where web bible's trigger traceback in search while typing. --- openlp/plugins/bibles/lib/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openlp/plugins/bibles/lib/manager.py b/openlp/plugins/bibles/lib/manager.py index d2286bed2..cff979714 100644 --- a/openlp/plugins/bibles/lib/manager.py +++ b/openlp/plugins/bibles/lib/manager.py @@ -367,7 +367,6 @@ class BibleManager(RegistryProperties): second_web_bible = self.db_cache[second_bible].get_object(BibleMeta, 'download_source') if web_bible or second_web_bible: # If either Bible is Web, cursor is reset to normal and search ends w/o any message. - self.check_search_result() self.application.set_normal_cursor() return None # Fetch the results from db. If no results are found, return None, no message is given for this. From 191845b70c32d1dc5dfc219bff8904e64c8d7b02 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Thu, 15 Sep 2016 10:41:54 +0300 Subject: [PATCH 57/65] - Added param: to Lock toggle button --- openlp/plugins/bibles/lib/mediaitem.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openlp/plugins/bibles/lib/mediaitem.py b/openlp/plugins/bibles/lib/mediaitem.py index e20abe797..567f35823 100644 --- a/openlp/plugins/bibles/lib/mediaitem.py +++ b/openlp/plugins/bibles/lib/mediaitem.py @@ -565,6 +565,8 @@ class BibleMediaItem(MediaManagerItem): """ Toggle the lock button, if Search tab is used, set focus to search field, if Select tab is used, give focus to Bible book name field. + :param checked: The state of the toggle button. bool + :return: None """ if checked: self.sender().setIcon(self.lock_icon) From 09da152372d1e1b8bd3b8858cf17742d9665cb5d Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sat, 17 Sep 2016 00:43:30 +0300 Subject: [PATCH 58/65] - Added a test for toggle lock button - Removed give focus to Select bookname on Select tab, may be confusing. --- openlp/plugins/bibles/lib/mediaitem.py | 5 +---- .../openlp_plugins/bibles/test_mediaitem.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/openlp/plugins/bibles/lib/mediaitem.py b/openlp/plugins/bibles/lib/mediaitem.py index 567f35823..fc92b8f7a 100644 --- a/openlp/plugins/bibles/lib/mediaitem.py +++ b/openlp/plugins/bibles/lib/mediaitem.py @@ -563,8 +563,7 @@ class BibleMediaItem(MediaManagerItem): def on_lock_button_toggled(self, checked): """ - Toggle the lock button, if Search tab is used, set focus to search field, if Select tab is used, - give focus to Bible book name field. + Toggle the lock button, if Search tab is used, set focus to search field. :param checked: The state of the toggle button. bool :return: None """ @@ -574,8 +573,6 @@ class BibleMediaItem(MediaManagerItem): self.sender().setIcon(self.unlock_icon) if self.quickTab.isVisible(): self.quick_search_edit.setFocus() - else: - self.advanced_book_combo_box.setFocus() def on_quick_style_combo_box_changed(self): self.settings.layout_style = self.quickStyleComboBox.currentIndex() diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 6d468275b..484e5bd3d 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -168,3 +168,18 @@ class TestMediaItem(TestCase, TestMixin): self.media_item.check_search_result.assert_called_once_with(), self.media_item.quick_search_edit.clear.assert_called_once_with(), self.media_item.quick_search_edit.setFocus.assert_called_once_with() + + def test_on_lock_button_toggled_search_tab(self): + """ + Test that "on_lock_button_toggled" gives focus to the right field. + """ + # GIVEN: Mocked functions + self.media_item.sender = MagicMock() + self.media_item.quickTab = MagicMock() + self.media_item.quick_search_edit = MagicMock() + + # WHEN: on_lock_button_toggled is called and quickTab.isVisible() returns = True. + self.media_item.on_lock_button_toggled(True) + + # THEN: on_quick_search_edit should receive focus. + self.media_item.quick_search_edit.setFocus.assert_called_once_with() From 190baf19d5767a8d9cb4f788b839a15fb6c8c699 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sun, 18 Sep 2016 01:53:47 +0300 Subject: [PATCH 59/65] - Added two new tests for lock button toggle. --- .../openlp_plugins/bibles/test_mediaitem.py | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 484e5bd3d..4543b6974 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -169,17 +169,43 @@ class TestMediaItem(TestCase, TestMixin): self.media_item.quick_search_edit.clear.assert_called_once_with(), self.media_item.quick_search_edit.setFocus.assert_called_once_with() - def test_on_lock_button_toggled_search_tab(self): + def test_on_lock_button_toggled_search_tab_lock_icon(self): """ - Test that "on_lock_button_toggled" gives focus to the right field. + Test that "on_lock_button_toggled" gives focus to the right field and toggles the lock properly. """ - # GIVEN: Mocked functions + # GIVEN: Mocked sender & Search edit, quickTab returning value = True on isVisible. self.media_item.sender = MagicMock() - self.media_item.quickTab = MagicMock() self.media_item.quick_search_edit = MagicMock() + self.media_item.quickTab = MagicMock(**{'isVisible.return_value': True}) - # WHEN: on_lock_button_toggled is called and quickTab.isVisible() returns = True. + self.media_item.lock_icon = 'lock icon' + sender_instance_mock = MagicMock() + self.media_item.sender = MagicMock(return_value=sender_instance_mock) + + # WHEN: on_lock_button_toggled is called and checked returns = True. self.media_item.on_lock_button_toggled(True) - # THEN: on_quick_search_edit should receive focus. + # THEN: on_quick_search_edit should receive focus and Lock icon should be set. self.media_item.quick_search_edit.setFocus.assert_called_once_with() + sender_instance_mock.setIcon.assert_called_once_with('lock icon') + + def test_on_lock_button_toggled_select_tab_unlock_icon(self): + """ + Test that "on_lock_button_toggled" does not give focus to Search field in Select + and lock toggles properly. + """ + # GIVEN: Mocked sender & Search edit, quickTab returning value = False on isVisible. + self.media_item.sender = MagicMock() + self.media_item.quick_search_edit = MagicMock() + self.media_item.quickTab = MagicMock(**{'isVisible.return_value': False}) + + self.media_item.unlock_icon = 'unlock icon' + sender_instance_mock = MagicMock() + self.media_item.sender = MagicMock(return_value=sender_instance_mock) + + # WHEN: on_lock_button_toggled is called and checked returns = False. + self.media_item.on_lock_button_toggled(False) + + # THEN: on_quick_search_edit should not receive focus and Unlock icon should be set. + self.media_item.quick_search_edit.setFocus.assert_not_called_once_with() + sender_instance_mock.setIcon.assert_called_once_with('unlock icon') From ff2e1d1ea781cf5421aa36779d47cc8441221672 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Sun, 18 Sep 2016 02:07:24 +0300 Subject: [PATCH 60/65] - The checking for not_called_once does not work apparently, fails in jenkins but passes in nosetests. --- .../openlp_plugins/bibles/test_mediaitem.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/functional/openlp_plugins/bibles/test_mediaitem.py b/tests/functional/openlp_plugins/bibles/test_mediaitem.py index 4543b6974..d5645cd3f 100644 --- a/tests/functional/openlp_plugins/bibles/test_mediaitem.py +++ b/tests/functional/openlp_plugins/bibles/test_mediaitem.py @@ -189,16 +189,15 @@ class TestMediaItem(TestCase, TestMixin): self.media_item.quick_search_edit.setFocus.assert_called_once_with() sender_instance_mock.setIcon.assert_called_once_with('lock icon') - def test_on_lock_button_toggled_select_tab_unlock_icon(self): + def test_on_lock_button_toggled_unlock_icon(self): """ - Test that "on_lock_button_toggled" does not give focus to Search field in Select - and lock toggles properly. + Test that lock button unlocks properly and lock toggles properly. """ # GIVEN: Mocked sender & Search edit, quickTab returning value = False on isVisible. self.media_item.sender = MagicMock() self.media_item.quick_search_edit = MagicMock() - self.media_item.quickTab = MagicMock(**{'isVisible.return_value': False}) - + self.media_item.quickTab = MagicMock() + self.media_item.quickTab.isVisible = MagicMock() self.media_item.unlock_icon = 'unlock icon' sender_instance_mock = MagicMock() self.media_item.sender = MagicMock(return_value=sender_instance_mock) @@ -206,6 +205,5 @@ class TestMediaItem(TestCase, TestMixin): # WHEN: on_lock_button_toggled is called and checked returns = False. self.media_item.on_lock_button_toggled(False) - # THEN: on_quick_search_edit should not receive focus and Unlock icon should be set. - self.media_item.quick_search_edit.setFocus.assert_not_called_once_with() + # THEN: Unlock icon should be set. sender_instance_mock.setIcon.assert_called_once_with('unlock icon') From 3a3ca8d93322a35062c5b192e081ceac26d1a1c3 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sun, 18 Sep 2016 16:54:55 +0200 Subject: [PATCH 61/65] Fix fetching bible texts from CrossWalk. --- openlp/plugins/bibles/lib/importers/http.py | 2 +- .../openlp_plugins/bibles/test_lib_http.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/openlp/plugins/bibles/lib/importers/http.py b/openlp/plugins/bibles/lib/importers/http.py index 6921c9005..d41187d93 100644 --- a/openlp/plugins/bibles/lib/importers/http.py +++ b/openlp/plugins/bibles/lib/importers/http.py @@ -493,7 +493,7 @@ class CWExtract(RegistryProperties): for verse in verses_div: self.application.process_events() verse_number = int(verse.find('strong').contents[0]) - verse_span = verse.find('span') + verse_span = verse.find('span', class_='verse-%d' % verse_number) tags_to_remove = verse_span.find_all(['a', 'sup']) for tag in tags_to_remove: tag.decompose() diff --git a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py index 084bfa476..7068898fe 100644 --- a/tests/interfaces/openlp_plugins/bibles/test_lib_http.py +++ b/tests/interfaces/openlp_plugins/bibles/test_lib_http.py @@ -163,3 +163,19 @@ class TestBibleHTTP(TestCase): # THEN: The list should not be None, and some known bibles should be there self.assertIsNotNone(bibles) self.assertIn(('Giovanni Diodati 1649 (Italian)', 'gdb', 'it'), bibles) + + def test_crosswalk_get_verse_text(self): + """ + Test verse text from Crosswalk.com + """ + # GIVEN: A new Crosswalk extraction class + handler = CWExtract() + + # WHEN: downloading NIV Genesis from Crosswalk + niv_genesis_chapter_one = handler.get_bible_chapter('niv', 'Genesis', 1) + + # THEN: The verse list should contain the verses + self.assertTrue(niv_genesis_chapter_one.has_verse_list()) + self.assertEquals('In the beginning God created the heavens and the earth.', + niv_genesis_chapter_one.verse_list[1], + 'The first chapter of genesis should have been fetched.') From 4754d1eb8b3c653487c1652b2dae72189dbb376d Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sun, 18 Sep 2016 17:28:55 +0200 Subject: [PATCH 62/65] Try to fix a mediashout import issue by replacing the line tab char with 2 linebreaks. --- openlp/plugins/songs/lib/importers/mediashout.py | 2 +- openlp/plugins/songs/lib/importers/songimport.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openlp/plugins/songs/lib/importers/mediashout.py b/openlp/plugins/songs/lib/importers/mediashout.py index a3bd7bbbc..9b916cd43 100644 --- a/openlp/plugins/songs/lib/importers/mediashout.py +++ b/openlp/plugins/songs/lib/importers/mediashout.py @@ -101,7 +101,7 @@ class MediaShoutImport(SongImport): self.song_book_name = song.SongID for verse in verses: tag = VERSE_TAGS[verse.Type] + str(verse.Number) if verse.Type < len(VERSE_TAGS) else 'O' - self.add_verse(verse.Text, tag) + self.add_verse(self.tidy_text(verse.Text), tag) for order in verse_order: if order.Type < len(VERSE_TAGS): self.verse_order_list.append(VERSE_TAGS[order.Type] + str(order.Number)) diff --git a/openlp/plugins/songs/lib/importers/songimport.py b/openlp/plugins/songs/lib/importers/songimport.py index c9ef382a7..3fea7e9c7 100644 --- a/openlp/plugins/songs/lib/importers/songimport.py +++ b/openlp/plugins/songs/lib/importers/songimport.py @@ -140,6 +140,7 @@ class SongImport(QtCore.QObject): text = text.replace('\u2026', '...') text = text.replace('\u2013', '-') text = text.replace('\u2014', '-') + text = text.replace('\x0b', '\n\n') # Remove surplus blank lines, spaces, trailing/leading spaces text = re.sub(r'[ \t\v]+', ' ', text) text = re.sub(r' ?(\r\n?|\n) ?', '\n', text) From 32cfacff7e05955e04067b373296247c765eb849 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sun, 18 Sep 2016 17:47:59 +0200 Subject: [PATCH 63/65] Skip MediaShout import testing on non-win platforms. --- tests/functional/openlp_plugins/songs/test_mediashout.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/functional/openlp_plugins/songs/test_mediashout.py b/tests/functional/openlp_plugins/songs/test_mediashout.py index 4f03c4f94..c08b04df7 100644 --- a/tests/functional/openlp_plugins/songs/test_mediashout.py +++ b/tests/functional/openlp_plugins/songs/test_mediashout.py @@ -22,15 +22,20 @@ """ Test the MediaShout importer """ -from unittest import TestCase +from unittest import TestCase, skipUnless from collections import namedtuple from openlp.core.common import Registry -from openlp.plugins.songs.lib.importers.mediashout import MediaShoutImport +try: + from openlp.plugins.songs.lib.importers.mediashout import MediaShoutImport + CAN_RUN_TESTS = True +except ImportError: + CAN_RUN_TESTS = False from tests.functional import MagicMock, patch, call +@skipUnless(CAN_RUN_TESTS, 'Not Windows, skipping test') class TestMediaShoutImport(TestCase): """ Test the MediaShout importer From 453bd53657e0b11d600b83225911be65f3ec18a9 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Fri, 23 Sep 2016 23:46:43 +0200 Subject: [PATCH 64/65] Fix the tidy_text method to not insert linefeeds, which is not allowed in XML. --- openlp/plugins/songs/lib/importers/songimport.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/songimport.py b/openlp/plugins/songs/lib/importers/songimport.py index 3fea7e9c7..c8648db85 100644 --- a/openlp/plugins/songs/lib/importers/songimport.py +++ b/openlp/plugins/songs/lib/importers/songimport.py @@ -140,11 +140,13 @@ class SongImport(QtCore.QObject): text = text.replace('\u2026', '...') text = text.replace('\u2013', '-') text = text.replace('\u2014', '-') - text = text.replace('\x0b', '\n\n') + # Replace vertical tab with 2 linebreaks + text = text.replace('\v', '\n\n') + # Replace form feed (page break) with 2 linebreaks + text = text.replace('\f', '\n\n') # Remove surplus blank lines, spaces, trailing/leading spaces - text = re.sub(r'[ \t\v]+', ' ', text) + text = re.sub(r'[ \t]+', ' ', text) text = re.sub(r' ?(\r\n?|\n) ?', '\n', text) - text = re.sub(r' ?(\n{5}|\f)+ ?', '\f', text) return text def process_song_text(self, text): From 97d828af4373d5ba80fb09a0754772f426698b33 Mon Sep 17 00:00:00 2001 From: Olli Suutari Date: Tue, 4 Oct 2016 16:48:54 +0300 Subject: [PATCH 65/65] - Added pip's fix for sword test --- tests/functional/openlp_plugins/bibles/test_swordimport.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/functional/openlp_plugins/bibles/test_swordimport.py b/tests/functional/openlp_plugins/bibles/test_swordimport.py index 261df1c0e..14480bdd1 100644 --- a/tests/functional/openlp_plugins/bibles/test_swordimport.py +++ b/tests/functional/openlp_plugins/bibles/test_swordimport.py @@ -70,8 +70,7 @@ class TestSwordImport(TestCase): @patch('openlp.plugins.bibles.lib.importers.sword.SwordBible.application') @patch('openlp.plugins.bibles.lib.importers.sword.modules') - @patch('openlp.core.common.languages') - def test_simple_import(self, mocked_languages, mocked_pysword_modules, mocked_application): + def test_simple_import(self, mocked_pysword_modules, mocked_application): """ Test that a simple SWORD import works """ @@ -88,7 +87,7 @@ class TestSwordImport(TestCase): importer.create_verse = MagicMock() importer.create_book = MagicMock() importer.session = MagicMock() - mocked_languages.get_language.return_value = 'Danish' + importer.get_language = MagicMock(return_value='Danish') mocked_bible = MagicMock() mocked_genesis = MagicMock() mocked_genesis.name = 'Genesis'