From 8f4ccc542697b88ea3f9c8958756914b2f2b40d0 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Thu, 6 Nov 2014 14:14:05 +0100 Subject: [PATCH 01/10] Close db session after import to unlock file on windows --- openlp/plugins/songs/lib/importers/openlp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openlp/plugins/songs/lib/importers/openlp.py b/openlp/plugins/songs/lib/importers/openlp.py index 1a27e8d69..f3d0a74dc 100644 --- a/openlp/plugins/songs/lib/importers/openlp.py +++ b/openlp/plugins/songs/lib/importers/openlp.py @@ -217,4 +217,5 @@ class OpenLPSongImport(SongImport): self.import_wizard.increment_progress_bar(WizardStrings.ImportingType % new_song.title) if self.stop_import_flag: break + self.source_session.close() engine.dispose() From bc8afc9e4f2e8e7345b850de54467924605a7c32 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Thu, 6 Nov 2014 14:16:14 +0100 Subject: [PATCH 02/10] Update thememanager widget after theme import --- openlp/core/ui/mainwindow.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openlp/core/ui/mainwindow.py b/openlp/core/ui/mainwindow.py index 825a12889..48d00cc1f 100644 --- a/openlp/core/ui/mainwindow.py +++ b/openlp/core/ui/mainwindow.py @@ -706,7 +706,10 @@ class MainWindow(QtGui.QMainWindow, Ui_MainWindow, RegistryProperties): self.active_plugin.toggle_status(PluginStatus.Inactive) # Set global theme and Registry().execute('theme_update_global') + # Load the themes from files self.theme_manager_contents.load_first_time_themes() + # Update the theme widget + self.theme_manager_contents.load_themes() # Check if any Bibles downloaded. If there are, they will be processed. Registry().execute('bibles_load_list', True) self.application.set_normal_cursor() From f685572da244225d384f6074b87b2a3371e83739 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Thu, 6 Nov 2014 14:19:00 +0100 Subject: [PATCH 03/10] Improved FTW handling of connection issues --- openlp/core/ui/firsttimeform.py | 177 +++++++++++++++++++------------- 1 file changed, 105 insertions(+), 72 deletions(-) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 02d9e65f7..8a1c29dba 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -42,8 +42,9 @@ from configparser import ConfigParser from PyQt4 import QtCore, QtGui from openlp.core.common import Registry, RegistryProperties, AppLocation, Settings, check_directory_exists, \ - translate, clean_button_text + translate, clean_button_text, trace_error_handler from openlp.core.lib import PluginStatus, build_icon +from openlp.core.lib.ui import critical_error_message_box from openlp.core.utils import get_web_page from .firsttimewizard import UiFirstTimeWizard, FirstTimePage @@ -275,24 +276,34 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): def url_get_file(self, url, f_path): """" Download a file given a URL. The file is retrieved in chunks, giving the ability to cancel the download at any - point. + point. Returns False on download error. + + :param url: URL to download + :param f_path: Destination file """ block_count = 0 block_size = 4096 - url_file = urllib.request.urlopen(url) - filename = open(f_path, "wb") - # Download until finished or canceled. - while not self.was_download_cancelled: - data = url_file.read(block_size) - if not data: - break - filename.write(data) - block_count += 1 - self._download_progress(block_count, block_size) - filename.close() + try: + url_file = urllib.request.urlopen(url, timeout=30) + filename = open(f_path, "wb") + # Download until finished or canceled. + while not self.was_download_cancelled: + data = url_file.read(block_size) + if not data: + break + filename.write(data) + block_count += 1 + self._download_progress(block_count, block_size) + filename.close() + except (ConnectionError, IOError): + trace_error_handler(log) + filename.close() + os.remove(f_path) + return False # Delete file if cancelled, it may be a partial file. if self.was_download_cancelled: os.remove(f_path) + return True def _build_theme_screenshots(self): """ @@ -311,7 +322,7 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): :param url: The URL of the file we want to download. """ - site = urllib.request.urlopen(url) + site = urllib.request.urlopen(url, timeout=30) meta = site.info() return int(meta.get("Content-Length")) @@ -343,32 +354,41 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): self.max_progress = 0 self.finish_button.setVisible(False) self.application.process_events() - # Loop through the songs list and increase for each selected item - for i in range(self.songs_list_widget.count()): - self.application.process_events() - item = self.songs_list_widget.item(i) - if item.checkState() == QtCore.Qt.Checked: - filename = item.data(QtCore.Qt.UserRole) - size = self._get_file_size('%s%s' % (self.songs_url, filename)) - self.max_progress += size - # Loop through the Bibles list and increase for each selected item - iterator = QtGui.QTreeWidgetItemIterator(self.bibles_tree_widget) - while iterator.value(): - self.application.process_events() - item = iterator.value() - if item.parent() and item.checkState(0) == QtCore.Qt.Checked: - filename = item.data(0, QtCore.Qt.UserRole) - size = self._get_file_size('%s%s' % (self.bibles_url, filename)) - self.max_progress += size - iterator += 1 - # Loop through the themes list and increase for each selected item - for i in range(self.themes_list_widget.count()): - self.application.process_events() - item = self.themes_list_widget.item(i) - if item.checkState() == QtCore.Qt.Checked: - filename = item.data(QtCore.Qt.UserRole) - size = self._get_file_size('%s%s' % (self.themes_url, filename)) - self.max_progress += size + try: + # Loop through the songs list and increase for each selected item + for i in range(self.songs_list_widget.count()): + self.application.process_events() + item = self.songs_list_widget.item(i) + if item.checkState() == QtCore.Qt.Checked: + filename = item.data(QtCore.Qt.UserRole) + size = self._get_file_size('%s%s' % (self.songs_url, filename)) + self.max_progress += size + # Loop through the Bibles list and increase for each selected item + iterator = QtGui.QTreeWidgetItemIterator(self.bibles_tree_widget) + while iterator.value(): + self.application.process_events() + item = iterator.value() + if item.parent() and item.checkState(0) == QtCore.Qt.Checked: + filename = item.data(0, QtCore.Qt.UserRole) + size = self._get_file_size('%s%s' % (self.bibles_url, filename)) + self.max_progress += size + iterator += 1 + # Loop through the themes list and increase for each selected item + for i in range(self.themes_list_widget.count()): + self.application.process_events() + item = self.themes_list_widget.item(i) + if item.checkState() == QtCore.Qt.Checked: + filename = item.data(QtCore.Qt.UserRole) + size = self._get_file_size('%s%s' % (self.themes_url, filename)) + self.max_progress += size + except (ConnectionError, IOError): + trace_error_handler(log) + critical_error_message_box(translate('OpenLP.FirstTimeWizard', 'Download Error'), + translate('OpenLP.FirstTimeWizard', 'There was a connection problem during ' + 'download, so further downloads will be skipped. Try to re-run the ' + 'First Time Wizard later.')) + self.max_progress = 0 + self.web_access = None if self.max_progress: # Add on 2 for plugins status setting plus a "finished" point. self.max_progress += 2 @@ -432,38 +452,11 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): self._set_plugin_status(self.song_usage_check_box, 'songusage/status') self._set_plugin_status(self.alert_check_box, 'alerts/status') if self.web_access: - # Build directories for downloads - songs_destination = os.path.join(gettempdir(), 'openlp') - bibles_destination = AppLocation.get_section_data_path('bibles') - themes_destination = AppLocation.get_section_data_path('themes') - # Download songs - for i in range(self.songs_list_widget.count()): - item = self.songs_list_widget.item(i) - if item.checkState() == QtCore.Qt.Checked: - filename = item.data(QtCore.Qt.UserRole) - self._increment_progress_bar(self.downloading % filename, 0) - self.previous_size = 0 - destination = os.path.join(songs_destination, str(filename)) - self.url_get_file('%s%s' % (self.songs_url, filename), destination) - # Download Bibles - bibles_iterator = QtGui.QTreeWidgetItemIterator( - self.bibles_tree_widget) - while bibles_iterator.value(): - item = bibles_iterator.value() - if item.parent() and item.checkState(0) == QtCore.Qt.Checked: - bible = item.data(0, QtCore.Qt.UserRole) - self._increment_progress_bar(self.downloading % bible, 0) - self.previous_size = 0 - self.url_get_file('%s%s' % (self.bibles_url, bible), os.path.join(bibles_destination, bible)) - bibles_iterator += 1 - # Download themes - for i in range(self.themes_list_widget.count()): - item = self.themes_list_widget.item(i) - if item.checkState() == QtCore.Qt.Checked: - theme = item.data(QtCore.Qt.UserRole) - self._increment_progress_bar(self.downloading % theme, 0) - self.previous_size = 0 - self.url_get_file('%s%s' % (self.themes_url, theme), os.path.join(themes_destination, theme)) + if not self._download_selected(): + critical_error_message_box(translate('OpenLP.FirstTimeWizard', 'Download Error'), + translate('OpenLP.FirstTimeWizard', 'There was a connection problem while ' + 'downloading, so further downloads will be skipped. Try to re-run ' + 'the First Time Wizard later.')) # Set Default Display if self.display_combo_box.currentIndex() != -1: Settings().setValue('core/monitor', self.display_combo_box.currentIndex()) @@ -472,6 +465,46 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): if self.theme_combo_box.currentIndex() != -1: Settings().setValue('themes/global theme', self.theme_combo_box.currentText()) + def _download_selected(self): + """ + Download selected songs, bibles and themes. Returns False on download error + """ + # Build directories for downloads + songs_destination = os.path.join(gettempdir(), 'openlp') + bibles_destination = AppLocation.get_section_data_path('bibles') + themes_destination = AppLocation.get_section_data_path('themes') + # Download songs + for i in range(self.songs_list_widget.count()): + item = self.songs_list_widget.item(i) + if item.checkState() == QtCore.Qt.Checked: + filename = item.data(QtCore.Qt.UserRole) + self._increment_progress_bar(self.downloading % filename, 0) + self.previous_size = 0 + destination = os.path.join(songs_destination, str(filename)) + if not self.url_get_file('%s%s' % (self.songs_url, filename), destination): + return False + # Download Bibles + bibles_iterator = QtGui.QTreeWidgetItemIterator(self.bibles_tree_widget) + while bibles_iterator.value(): + item = bibles_iterator.value() + if item.parent() and item.checkState(0) == QtCore.Qt.Checked: + bible = item.data(0, QtCore.Qt.UserRole) + self._increment_progress_bar(self.downloading % bible, 0) + self.previous_size = 0 + if not self.url_get_file('%s%s' % (self.bibles_url, bible), os.path.join(bibles_destination, bible)): + return False + bibles_iterator += 1 + # Download themes + for i in range(self.themes_list_widget.count()): + item = self.themes_list_widget.item(i) + if item.checkState() == QtCore.Qt.Checked: + theme = item.data(QtCore.Qt.UserRole) + self._increment_progress_bar(self.downloading % theme, 0) + self.previous_size = 0 + if not self.url_get_file('%s%s' % (self.themes_url, theme), os.path.join(themes_destination, theme)): + return False + return True + def _set_plugin_status(self, field, tag): """ Set the status of a plugin. From 529f94c504d84bda5a33e9a1198186bc7caa7162 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Fri, 7 Nov 2014 09:26:16 +0100 Subject: [PATCH 04/10] Remove old pyinstaller hooks which has been moved to the packaging branch --- .../pyinstaller/hook-openlp.core.ui.media.py | 32 ---------------- ...lugins.presentations.presentationplugin.py | 33 ---------------- resources/pyinstaller/hook-openlp.py | 38 ------------------- 3 files changed, 103 deletions(-) delete mode 100644 resources/pyinstaller/hook-openlp.core.ui.media.py delete mode 100644 resources/pyinstaller/hook-openlp.plugins.presentations.presentationplugin.py delete mode 100644 resources/pyinstaller/hook-openlp.py diff --git a/resources/pyinstaller/hook-openlp.core.ui.media.py b/resources/pyinstaller/hook-openlp.core.ui.media.py deleted file mode 100644 index a17856dca..000000000 --- a/resources/pyinstaller/hook-openlp.core.ui.media.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2014 Raoul Snyman # -# Portions copyright (c) 2008-2014 Tim Bentley, Gerald Britton, Jonathan # -# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub, # -# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer. # -# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru, # -# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith, # -# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock, # -# Frode Woldsund, Martin Zibricky, Patrick Zimmermann # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### - -hiddenimports = ['openlp.core.ui.media.phononplayer', - 'openlp.core.ui.media.vlcplayer', - 'openlp.core.ui.media.webkitplayer'] diff --git a/resources/pyinstaller/hook-openlp.plugins.presentations.presentationplugin.py b/resources/pyinstaller/hook-openlp.plugins.presentations.presentationplugin.py deleted file mode 100644 index 6ffb416fa..000000000 --- a/resources/pyinstaller/hook-openlp.plugins.presentations.presentationplugin.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2014 Raoul Snyman # -# Portions copyright (c) 2008-2014 Tim Bentley, Gerald Britton, Jonathan # -# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub, # -# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer. # -# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru, # -# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith, # -# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock, # -# Frode Woldsund, Martin Zibricky, Patrick Zimmermann # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### - -hiddenimports = ['openlp.plugins.presentations.lib.impresscontroller', - 'openlp.plugins.presentations.lib.powerpointcontroller', - 'openlp.plugins.presentations.lib.pptviewcontroller', - 'openlp.plugins.presentations.lib.pdfcontroller'] diff --git a/resources/pyinstaller/hook-openlp.py b/resources/pyinstaller/hook-openlp.py deleted file mode 100644 index c45dec009..000000000 --- a/resources/pyinstaller/hook-openlp.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2014 Raoul Snyman # -# Portions copyright (c) 2008-2014 Tim Bentley, Gerald Britton, Jonathan # -# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub, # -# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer. # -# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru, # -# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith, # -# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock, # -# Frode Woldsund, Martin Zibricky, Patrick Zimmermann # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### - -hiddenimports = ['plugins.songs.songsplugin', - 'plugins.bibles.bibleplugin', - 'plugins.presentations.presentationplugin', - 'plugins.media.mediaplugin', - 'plugins.images.imageplugin', - 'plugins.custom.customplugin', - 'plugins.songusage.songusageplugin', - 'plugins.remotes.remoteplugin', - 'plugins.alerts.alertsplugin'] From 48c561e34bc1bd455161a53b287bc5ceed28d440 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Fri, 7 Nov 2014 10:32:45 +0100 Subject: [PATCH 05/10] Made the FTW able to handle if the downloaded config file is invalid. Fixes: https://launchpad.net/bugs/1222944 --- openlp/core/ui/firsttimeform.py | 28 ++++++----- openlp/core/utils/__init__.py | 4 +- .../openlp_core_ui/test_firsttimeform.py | 46 +++++++++++++++++++ 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 8a1c29dba..20a18cb82 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -37,7 +37,7 @@ import urllib.request import urllib.parse import urllib.error from tempfile import gettempdir -from configparser import ConfigParser +from configparser import ConfigParser, MissingSectionHeaderError, NoSectionError, NoOptionError from PyQt4 import QtCore, QtGui @@ -131,17 +131,23 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): """ self.screens = screens # check to see if we have web access + self.web_access = False self.web = 'http://openlp.org/files/frw/' self.config = ConfigParser() user_agent = 'OpenLP/' + Registry().get('application').applicationVersion() - self.web_access = get_web_page('%s%s' % (self.web, 'download.cfg'), header=('User-Agent', user_agent)) - if self.web_access: - files = self.web_access.read() - self.config.read_string(files.decode()) - self.web = self.config.get('general', 'base url') - self.songs_url = self.web + self.config.get('songs', 'directory') + '/' - self.bibles_url = self.web + self.config.get('bibles', 'directory') + '/' - self.themes_url = self.web + self.config.get('themes', 'directory') + '/' + web_config = get_web_page('%s%s' % (self.web, 'download.cfg'), header=('User-Agent', user_agent)) + if web_config: + files = web_config.read() + try: + self.config.read_string(files.decode()) + self.web = self.config.get('general', 'base url') + self.songs_url = self.web + self.config.get('songs', 'directory') + '/' + self.bibles_url = self.web + self.config.get('bibles', 'directory') + '/' + self.themes_url = self.web + self.config.get('themes', 'directory') + '/' + self.web_access = True + except (NoSectionError, NoOptionError, MissingSectionHeaderError): + log.debug('A problem occured while parsing the downloaded config file') + trace_error_handler(log) self.update_screen_list_combo() self.was_download_cancelled = False self.theme_screenshot_thread = None @@ -295,7 +301,7 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): block_count += 1 self._download_progress(block_count, block_size) filename.close() - except (ConnectionError, IOError): + except ConnectionError: trace_error_handler(log) filename.close() os.remove(f_path) @@ -381,7 +387,7 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): filename = item.data(QtCore.Qt.UserRole) size = self._get_file_size('%s%s' % (self.themes_url, filename)) self.max_progress += size - except (ConnectionError, IOError): + except ConnectionError: trace_error_handler(log) critical_error_message_box(translate('OpenLP.FirstTimeWizard', 'Download Error'), translate('OpenLP.FirstTimeWizard', 'There was a connection problem during ' diff --git a/openlp/core/utils/__init__.py b/openlp/core/utils/__init__.py index 128c09b38..8f032cd7d 100644 --- a/openlp/core/utils/__init__.py +++ b/openlp/core/utils/__init__.py @@ -390,9 +390,9 @@ def get_web_page(url, header=None, update_openlp=False): page = None log.debug('Downloading URL = %s' % url) try: - page = urllib.request.urlopen(req) + page = urllib.request.urlopen(req, timeout=30) log.debug('Downloaded URL = %s' % page.geturl()) - except urllib.error.URLError: + except (urllib.error.URLError, ConnectionError): log.exception('The web page could not be downloaded') if not page: return None diff --git a/tests/functional/openlp_core_ui/test_firsttimeform.py b/tests/functional/openlp_core_ui/test_firsttimeform.py index ed7a7a9e8..ead9ef4cf 100644 --- a/tests/functional/openlp_core_ui/test_firsttimeform.py +++ b/tests/functional/openlp_core_ui/test_firsttimeform.py @@ -49,6 +49,22 @@ directory = bibles directory = themes """ +FAKE_BROKEN_CONFIG = b""" +[general] +base url = http://example.com/frw/ +[songs] +directory = songs +[bibles] +directory = bibles +""" + +FAKE_INVALID_CONFIG = b""" + +This is not a config file +Some text + +""" + class TestFirstTimeForm(TestCase, TestMixin): @@ -104,3 +120,33 @@ class TestFirstTimeForm(TestCase, TestMixin): self.assertEqual(expected_songs_url, first_time_form.songs_url, 'The songs URL should be correct') self.assertEqual(expected_bibles_url, first_time_form.bibles_url, 'The bibles URL should be correct') self.assertEqual(expected_themes_url, first_time_form.themes_url, 'The themes URL should be correct') + + def broken_config_test(self): + """ + Test if we can handle an config file with missing data + """ + # GIVEN: A mocked get_web_page, a First Time Wizard, an expected screen object, and a mocked broken config file + with patch('openlp.core.ui.firsttimeform.get_web_page') as mocked_get_web_page: + first_time_form = FirstTimeForm(None) + mocked_get_web_page.return_value.read.return_value = FAKE_BROKEN_CONFIG + + # WHEN: The First Time Wizard is initialised + first_time_form.initialize(MagicMock()) + + # THEN: The First Time Form should not have web access + self.assertFalse(first_time_form.web_access, 'There should not be web access with a broken config file') + + def invalid_config_test(self): + """ + Test if we can handle an config file in invalid format + """ + # GIVEN: A mocked get_web_page, a First Time Wizard, an expected screen object, and a mocked invalid config file + with patch('openlp.core.ui.firsttimeform.get_web_page') as mocked_get_web_page: + first_time_form = FirstTimeForm(None) + mocked_get_web_page.return_value.read.return_value = FAKE_INVALID_CONFIG + + # WHEN: The First Time Wizard is initialised + first_time_form.initialize(MagicMock()) + + # THEN: The First Time Form should not have web access + self.assertFalse(first_time_form.web_access, 'There should not be web access with an invalid config file') From 519119aecb2810e1efde88241c60b646f3a9c041 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Fri, 7 Nov 2014 10:41:37 +0100 Subject: [PATCH 06/10] Fixed test to reflect 30 sec timeout in get_web_page --- tests/functional/openlp_core_utils/test_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/openlp_core_utils/test_utils.py b/tests/functional/openlp_core_utils/test_utils.py index 075ecb14f..de3e68791 100644 --- a/tests/functional/openlp_core_utils/test_utils.py +++ b/tests/functional/openlp_core_utils/test_utils.py @@ -335,7 +335,7 @@ class TestUtils(TestCase): self.assertEqual(1, mocked_request_object.add_header.call_count, 'There should only be 1 call to add_header') mock_get_user_agent.assert_called_with() - mock_urlopen.assert_called_with(mocked_request_object) + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called') self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') @@ -365,7 +365,7 @@ class TestUtils(TestCase): self.assertEqual(2, mocked_request_object.add_header.call_count, 'There should only be 2 calls to add_header') mock_get_user_agent.assert_called_with() - mock_urlopen.assert_called_with(mocked_request_object) + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') @@ -393,7 +393,7 @@ class TestUtils(TestCase): self.assertEqual(1, mocked_request_object.add_header.call_count, 'There should only be 1 call to add_header') self.assertEqual(0, mock_get_user_agent.call_count, '_get_user_agent should not have been called') - mock_urlopen.assert_called_with(mocked_request_object) + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') @@ -425,7 +425,7 @@ class TestUtils(TestCase): mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent') self.assertEqual(1, mocked_request_object.add_header.call_count, 'There should only be 1 call to add_header') - mock_urlopen.assert_called_with(mocked_request_object) + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() mocked_registry_object.get.assert_called_with('application') mocked_application_object.process_events.assert_called_with() From a8bd5f19964455a4fcfc685cf4393ca05e99f004 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Fri, 7 Nov 2014 13:06:58 +0100 Subject: [PATCH 07/10] Fix for importing old bible DBs, like KJV --- openlp/plugins/bibles/bibleplugin.py | 2 +- openlp/plugins/bibles/lib/db.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/openlp/plugins/bibles/bibleplugin.py b/openlp/plugins/bibles/bibleplugin.py index e7f1fdd56..76dc75d35 100644 --- a/openlp/plugins/bibles/bibleplugin.py +++ b/openlp/plugins/bibles/bibleplugin.py @@ -159,7 +159,7 @@ class BiblePlugin(Plugin): self.upgrade_wizard = BibleUpgradeForm(self.main_window, self.manager, self) # If the import was not cancelled then reload. if self.upgrade_wizard.exec_(): - self.media_item.reloadBibles() + self.media_item.reload_bibles() def on_bible_import_click(self): if self.media_item: diff --git a/openlp/plugins/bibles/lib/db.py b/openlp/plugins/bibles/lib/db.py index b0d8d0e7d..8470c2765 100644 --- a/openlp/plugins/bibles/lib/db.py +++ b/openlp/plugins/bibles/lib/db.py @@ -170,6 +170,9 @@ class BibleDB(QtCore.QObject, Manager, RegistryProperties): Returns the version name of the Bible. """ version_name = self.get_object(BibleMeta, 'name') + # Fallback to old way of naming + if not version_name: + version_name = self.get_object(BibleMeta, 'Version') self.name = version_name.value if version_name else None return self.name @@ -969,11 +972,15 @@ class OldBibleDB(QtCore.QObject, Manager): """ Returns the version name of the Bible. """ + self.name = None version_name = self.run_sql('SELECT value FROM metadata WHERE key = "name"') if version_name: self.name = version_name[0][0] else: - self.name = None + # Fallback to old way of naming + version_name = self.run_sql('SELECT value FROM metadata WHERE key = "Version"') + if version_name: + self.name = version_name[0][0] return self.name def get_metadata(self): From 75960ac8da0b0fa19e0f4cff4c42bfc3c327b6ac Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 11 Nov 2014 14:58:03 +0100 Subject: [PATCH 08/10] When bible or song plugin is disabled in FTW don't show download for those plugins in FTW. Fixes bug 1194622. Fixes: https://launchpad.net/bugs/1194622 --- openlp/core/ui/firsttimeform.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 20a18cb82..0688681a1 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -90,6 +90,21 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): super(FirstTimeForm, self).__init__(parent) self.setup_ui(self) + def get_next_page_id(self): + """ + Returns the id of the next FirstTimePage to go to based on enabled plugins + """ + # The songs plugin is enabled + if FirstTimePage.Welcome < self.currentId() < FirstTimePage.Songs and self.songs_check_box.isChecked(): + return FirstTimePage.Songs + # The Bibles plugin is enabled + elif FirstTimePage.Welcome < self.currentId() < FirstTimePage.Bibles and self.bible_check_box.isChecked(): + return FirstTimePage.Bibles + elif FirstTimePage.Welcome < self.currentId() < FirstTimePage.Themes: + return FirstTimePage.Themes + else: + return self.currentId() + 1 + def nextId(self): """ Determine the next page in the Wizard to go to. @@ -99,7 +114,7 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): if not self.web_access: return FirstTimePage.NoInternet else: - return FirstTimePage.Songs + return self.get_next_page_id() elif self.currentId() == FirstTimePage.Progress: return -1 elif self.currentId() == FirstTimePage.NoInternet: @@ -114,7 +129,7 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): self.application.set_normal_cursor() return FirstTimePage.Defaults else: - return self.currentId() + 1 + return self.get_next_page_id() def exec_(self): """ From 0af6dc8eea5fcff46fd672a3e2bed8fdcc4ba9cc Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Sun, 23 Nov 2014 20:52:16 +0000 Subject: [PATCH 09/10] Always retry on connection failure. --- openlp/core/ui/firsttimeform.py | 58 +++++++++++++++++++++------------ openlp/core/utils/__init__.py | 34 ++++++++++++++----- 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 0688681a1..7713aa948 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -45,7 +45,7 @@ from openlp.core.common import Registry, RegistryProperties, AppLocation, Settin translate, clean_button_text, trace_error_handler from openlp.core.lib import PluginStatus, build_icon from openlp.core.lib.ui import critical_error_message_box -from openlp.core.utils import get_web_page +from openlp.core.utils import get_web_page, CONNECTION_RETRIES, CONNECTION_TIMEOUT from .firsttimewizard import UiFirstTimeWizard, FirstTimePage log = logging.getLogger(__name__) @@ -304,23 +304,30 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): """ block_count = 0 block_size = 4096 - try: - url_file = urllib.request.urlopen(url, timeout=30) - filename = open(f_path, "wb") - # Download until finished or canceled. - while not self.was_download_cancelled: - data = url_file.read(block_size) - if not data: - break - filename.write(data) - block_count += 1 - self._download_progress(block_count, block_size) - filename.close() - except ConnectionError: - trace_error_handler(log) - filename.close() - os.remove(f_path) - return False + retries = 0 + while True: + try: + url_file = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT) + filename = open(f_path, "wb") + # Download until finished or canceled. + while not self.was_download_cancelled: + data = url_file.read(block_size) + if not data: + break + filename.write(data) + block_count += 1 + self._download_progress(block_count, block_size) + filename.close() + except ConnectionError: + trace_error_handler(log) + filename.close() + os.remove(f_path) + if retries > CONNECTION_RETRIES: + return False + else: + retries += 1 + continue + break # Delete file if cancelled, it may be a partial file. if self.was_download_cancelled: os.remove(f_path) @@ -343,9 +350,18 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): :param url: The URL of the file we want to download. """ - site = urllib.request.urlopen(url, timeout=30) - meta = site.info() - return int(meta.get("Content-Length")) + retries = 0 + while True: + try: + site = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT) + meta = site.info() + return int(meta.get("Content-Length")) + except ConnectionException: + if retries > CONNECTION_RETRIES: + raise + else: + retries += 1 + continue def _download_progress(self, count, block_size): """ diff --git a/openlp/core/utils/__init__.py b/openlp/core/utils/__init__.py index 8f032cd7d..87dfc48cb 100644 --- a/openlp/core/utils/__init__.py +++ b/openlp/core/utils/__init__.py @@ -91,6 +91,8 @@ USER_AGENTS = { 'Mozilla/5.0 (X11; NetBSD amd64; rv:18.0) Gecko/20130120 Firefox/18.0' ] } +CONNECTION_TIMEOUT = 30 +CONNECTION_RETRIES = 2 class VersionThread(QtCore.QThread): @@ -250,10 +252,17 @@ def check_latest_version(current_version): req = urllib.request.Request('http://www.openlp.org/files/version.txt') req.add_header('User-Agent', 'OpenLP/%s' % current_version['full']) remote_version = None - try: - remote_version = str(urllib.request.urlopen(req, None).read().decode()).strip() - except IOError: - log.exception('Failed to download the latest OpenLP version file') + retries = 0 + while True: + try: + remote_version = str(urllib.request.urlopen(req, None, timeout=CONNECTION_TIMEOUT).read().decode()).strip() + except ConnectionException: + if retries > CONNECTION_RETRIES: + log.exception('Failed to download the latest OpenLP version file') + else: + retries += 1 + continue + break if remote_version: version_string = remote_version return version_string @@ -389,11 +398,18 @@ def get_web_page(url, header=None, update_openlp=False): req.add_header(header[0], header[1]) page = None log.debug('Downloading URL = %s' % url) - try: - page = urllib.request.urlopen(req, timeout=30) - log.debug('Downloaded URL = %s' % page.geturl()) - except (urllib.error.URLError, ConnectionError): - log.exception('The web page could not be downloaded') + retries = 0 + while True: + try: + page = urllib.request.urlopen(req, timeout=CONNECTION_TIMEOUT) + log.debug('Downloaded URL = %s' % page.geturl()) + except (urllib.error.URLError, ConnectionError): + if retries > CONNECTION_RETRIES: + log.exception('The web page could not be downloaded') + raise + else: + continue + break if not page: return None if update_openlp: From 5b794e57623ec4f872a700d4d23336dcf60fc7ed Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Mon, 24 Nov 2014 19:38:15 +0000 Subject: [PATCH 10/10] Added small pause before connection retry --- openlp/core/ui/firsttimeform.py | 2 ++ openlp/core/utils/__init__.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 7713aa948..b291ba249 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -326,6 +326,7 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): return False else: retries += 1 + time.sleep(0.1) continue break # Delete file if cancelled, it may be a partial file. @@ -361,6 +362,7 @@ class FirstTimeForm(QtGui.QWizard, UiFirstTimeWizard, RegistryProperties): raise else: retries += 1 + time.sleep(0.1) continue def _download_progress(self, count, block_size): diff --git a/openlp/core/utils/__init__.py b/openlp/core/utils/__init__.py index 87dfc48cb..df1fd2a9c 100644 --- a/openlp/core/utils/__init__.py +++ b/openlp/core/utils/__init__.py @@ -35,6 +35,7 @@ import logging import locale import os import re +import time from subprocess import Popen, PIPE import sys import urllib.request @@ -261,6 +262,7 @@ def check_latest_version(current_version): log.exception('Failed to download the latest OpenLP version file') else: retries += 1 + time.sleep(0.1) continue break if remote_version: @@ -408,6 +410,7 @@ def get_web_page(url, header=None, update_openlp=False): log.exception('The web page could not be downloaded') raise else: + time.sleep(0.1) continue break if not page: