From 4008ed008feabd4ab6c8d785bbf39f58c47cb79b Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Tue, 20 Dec 2016 21:20:54 +0000 Subject: [PATCH 1/9] Move url size --- openlp/core/lib/webpagereader.py | 182 -------------- openlp/core/ui/firsttimeform.py | 28 +-- openlp/plugins/bibles/lib/importers/http.py | 2 +- .../openlp_core_lib/test_webpagereader.py | 229 ------------------ .../openlp_core_ui/test_first_time.py | 2 +- 5 files changed, 6 insertions(+), 437 deletions(-) delete mode 100644 openlp/core/lib/webpagereader.py delete mode 100644 tests/functional/openlp_core_lib/test_webpagereader.py diff --git a/openlp/core/lib/webpagereader.py b/openlp/core/lib/webpagereader.py deleted file mode 100644 index 52c98bbaf..000000000 --- a/openlp/core/lib/webpagereader.py +++ /dev/null @@ -1,182 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2016 OpenLP Developers # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### -""" -The :mod:`openlp.core.utils` module provides the utility libraries for OpenLP. -""" -import logging -import socket -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from http.client import HTTPException -from random import randint - -from openlp.core.common import Registry - -log = logging.getLogger(__name__ + '.__init__') - -USER_AGENTS = { - 'win32': [ - 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36' - ], - 'darwin': [ - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) ' - 'Chrome/26.0.1410.43 Safari/537.31', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.11 (KHTML, like Gecko) ' - 'Chrome/20.0.1132.57 Safari/536.11', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/536.11 (KHTML, like Gecko) ' - 'Chrome/20.0.1132.47 Safari/536.11', - ], - 'linux2': [ - 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 ' - 'Chrome/25.0.1364.160 Safari/537.22', - 'Mozilla/5.0 (X11; CrOS armv7l 2913.260.0) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.99 ' - 'Safari/537.11', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.27 (KHTML, like Gecko) Chrome/26.0.1389.0 Safari/537.27' - ], - 'default': [ - 'Mozilla/5.0 (X11; NetBSD amd64; rv:18.0) Gecko/20130120 Firefox/18.0' - ] -} -CONNECTION_TIMEOUT = 30 -CONNECTION_RETRIES = 2 - - -class HTTPRedirectHandlerFixed(urllib.request.HTTPRedirectHandler): - """ - Special HTTPRedirectHandler used to work around http://bugs.python.org/issue22248 - (Redirecting to urls with special chars) - """ - def redirect_request(self, req, fp, code, msg, headers, new_url): - # - """ - Test if the new_url can be decoded to ascii - - :param req: - :param fp: - :param code: - :param msg: - :param headers: - :param new_url: - :return: - """ - try: - new_url.encode('latin1').decode('ascii') - fixed_url = new_url - except Exception: - # The url could not be decoded to ascii, so we do some url encoding - fixed_url = urllib.parse.quote(new_url.encode('latin1').decode('utf-8', 'replace'), safe='/:') - return super(HTTPRedirectHandlerFixed, self).redirect_request(req, fp, code, msg, headers, fixed_url) - - -def _get_user_agent(): - """ - Return a user agent customised for the platform the user is on. - """ - browser_list = USER_AGENTS.get(sys.platform, None) - if not browser_list: - browser_list = USER_AGENTS['default'] - random_index = randint(0, len(browser_list) - 1) - return browser_list[random_index] - - -def get_web_page(url, header=None, update_openlp=False): - """ - Attempts to download the webpage at url and returns that page or None. - - :param url: The URL to be downloaded. - :param header: An optional HTTP header to pass in the request to the web server. - :param update_openlp: Tells OpenLP to update itself if the page is successfully downloaded. - Defaults to False. - """ - # TODO: Add proxy usage. Get proxy info from OpenLP settings, add to a - # proxy_handler, build into an opener and install the opener into urllib2. - # http://docs.python.org/library/urllib2.html - if not url: - return None - # This is needed to work around http://bugs.python.org/issue22248 and https://bugs.launchpad.net/openlp/+bug/1251437 - opener = urllib.request.build_opener(HTTPRedirectHandlerFixed()) - urllib.request.install_opener(opener) - req = urllib.request.Request(url) - if not header or header[0].lower() != 'user-agent': - user_agent = _get_user_agent() - req.add_header('User-Agent', user_agent) - if header: - req.add_header(header[0], header[1]) - log.debug('Downloading URL = %s' % url) - retries = 0 - while retries <= CONNECTION_RETRIES: - retries += 1 - time.sleep(0.1) - try: - page = urllib.request.urlopen(req, timeout=CONNECTION_TIMEOUT) - log.debug('Downloaded page {text}'.format(text=page.geturl())) - break - except urllib.error.URLError as err: - log.exception('URLError on {text}'.format(text=url)) - log.exception('URLError: {text}'.format(text=err.reason)) - page = None - if retries > CONNECTION_RETRIES: - raise - except socket.timeout: - log.exception('Socket timeout: {text}'.format(text=url)) - page = None - if retries > CONNECTION_RETRIES: - raise - except socket.gaierror: - log.exception('Socket gaierror: {text}'.format(text=url)) - page = None - if retries > CONNECTION_RETRIES: - raise - except ConnectionRefusedError: - log.exception('ConnectionRefused: {text}'.format(text=url)) - page = None - if retries > CONNECTION_RETRIES: - raise - break - except ConnectionError: - log.exception('Connection error: {text}'.format(text=url)) - page = None - if retries > CONNECTION_RETRIES: - raise - except HTTPException: - log.exception('HTTPException error: {text}'.format(text=url)) - page = None - if retries > CONNECTION_RETRIES: - raise - except: - # Don't know what's happening, so reraise the original - raise - if update_openlp: - Registry().get('application').process_events() - if not page: - log.exception('{text} could not be downloaded'.format(text=url)) - return None - log.debug(page) - return page - - -__all__ = ['get_web_page'] diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 974ef90db..b59f31211 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -39,7 +39,7 @@ from openlp.core.common import Registry, RegistryProperties, AppLocation, Settin translate, clean_button_text, trace_error_handler from openlp.core.lib import PluginStatus, build_icon from openlp.core.lib.ui import critical_error_message_box -from openlp.core.lib.webpagereader import get_web_page, CONNECTION_RETRIES, CONNECTION_TIMEOUT +from openlp.core.common.httputils import get_web_page, get_url_file_size, CONNECTION_RETRIES, CONNECTION_TIMEOUT from .firsttimewizard import UiFirstTimeWizard, FirstTimePage log = logging.getLogger(__name__) @@ -455,26 +455,6 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): if item: item.setIcon(build_icon(os.path.join(gettempdir(), 'openlp', screenshot))) - def _get_file_size(self, url): - """ - Get the size of a file. - - :param url: The URL of the file we want to download. - """ - retries = 0 - while True: - try: - site = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT) - meta = site.info() - return int(meta.get("Content-Length")) - except urllib.error.URLError: - if retries > CONNECTION_RETRIES: - raise - else: - retries += 1 - time.sleep(0.1) - continue - def _download_progress(self, count, block_size): """ Calculate and display the download progress. @@ -510,7 +490,7 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): item = self.songs_list_widget.item(i) if item.checkState() == QtCore.Qt.Checked: filename, sha256 = item.data(QtCore.Qt.UserRole) - size = self._get_file_size('{path}{name}'.format(path=self.songs_url, name=filename)) + size = get_url_file_size('{path}{name}'.format(path=self.songs_url, name=filename)) self.max_progress += size # Loop through the Bibles list and increase for each selected item iterator = QtWidgets.QTreeWidgetItemIterator(self.bibles_tree_widget) @@ -519,7 +499,7 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): item = iterator.value() if item.parent() and item.checkState(0) == QtCore.Qt.Checked: filename, sha256 = item.data(0, QtCore.Qt.UserRole) - size = self._get_file_size('{path}{name}'.format(path=self.bibles_url, name=filename)) + size = get_url_file_size('{path}{name}'.format(path=self.bibles_url, name=filename)) self.max_progress += size iterator += 1 # Loop through the themes list and increase for each selected item @@ -528,7 +508,7 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): item = self.themes_list_widget.item(i) if item.checkState() == QtCore.Qt.Checked: filename, sha256 = item.data(QtCore.Qt.UserRole) - size = self._get_file_size('{path}{name}'.format(path=self.themes_url, name=filename)) + size = get_url_file_size('{path}{name}'.format(path=self.themes_url, name=filename)) self.max_progress += size except urllib.error.URLError: trace_error_handler(log) diff --git a/openlp/plugins/bibles/lib/importers/http.py b/openlp/plugins/bibles/lib/importers/http.py index d41187d93..071ab0119 100644 --- a/openlp/plugins/bibles/lib/importers/http.py +++ b/openlp/plugins/bibles/lib/importers/http.py @@ -32,7 +32,7 @@ from bs4 import BeautifulSoup, NavigableString, Tag from openlp.core.common import Registry, RegistryProperties, translate from openlp.core.lib.ui import critical_error_message_box -from openlp.core.lib.webpagereader import get_web_page +from openlp.core.common.httputils import get_web_page from openlp.plugins.bibles.lib import SearchResults from openlp.plugins.bibles.lib.bibleimport import BibleImport from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB, Book diff --git a/tests/functional/openlp_core_lib/test_webpagereader.py b/tests/functional/openlp_core_lib/test_webpagereader.py deleted file mode 100644 index 6e33fca51..000000000 --- a/tests/functional/openlp_core_lib/test_webpagereader.py +++ /dev/null @@ -1,229 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -############################################################################### -# OpenLP - Open Source Lyrics Projection # -# --------------------------------------------------------------------------- # -# Copyright (c) 2008-2016 OpenLP Developers # -# --------------------------------------------------------------------------- # -# This program is free software; you can redistribute it and/or modify it # -# under the terms of the GNU General Public License as published by the Free # -# Software Foundation; version 2 of the License. # -# # -# This program is distributed in the hope that it will be useful, but WITHOUT # -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # -# more details. # -# # -# You should have received a copy of the GNU General Public License along # -# with this program; if not, write to the Free Software Foundation, Inc., 59 # -# Temple Place, Suite 330, Boston, MA 02111-1307 USA # -############################################################################### -""" -Functional tests to test the AppLocation class and related methods. -""" -from unittest import TestCase - -from openlp.core.lib.webpagereader import _get_user_agent, get_web_page - -from tests.functional import MagicMock, patch - - -class TestUtils(TestCase): - """ - A test suite to test out various methods around the AppLocation class. - """ - def test_get_user_agent_linux(self): - """ - Test that getting a user agent on Linux returns a user agent suitable for Linux - """ - with patch('openlp.core.lib.webpagereader.sys') as mocked_sys: - - # GIVEN: The system is Linux - mocked_sys.platform = 'linux2' - - # WHEN: We call _get_user_agent() - user_agent = _get_user_agent() - - # THEN: The user agent is a Linux (or ChromeOS) user agent - result = 'Linux' in user_agent or 'CrOS' in user_agent - self.assertTrue(result, 'The user agent should be a valid Linux user agent') - - def test_get_user_agent_windows(self): - """ - Test that getting a user agent on Windows returns a user agent suitable for Windows - """ - with patch('openlp.core.lib.webpagereader.sys') as mocked_sys: - - # GIVEN: The system is Linux - mocked_sys.platform = 'win32' - - # WHEN: We call _get_user_agent() - user_agent = _get_user_agent() - - # THEN: The user agent is a Linux (or ChromeOS) user agent - self.assertIn('Windows', user_agent, 'The user agent should be a valid Windows user agent') - - def test_get_user_agent_macos(self): - """ - Test that getting a user agent on OS X returns a user agent suitable for OS X - """ - with patch('openlp.core.lib.webpagereader.sys') as mocked_sys: - - # GIVEN: The system is Linux - mocked_sys.platform = 'darwin' - - # WHEN: We call _get_user_agent() - user_agent = _get_user_agent() - - # THEN: The user agent is a Linux (or ChromeOS) user agent - self.assertIn('Mac OS X', user_agent, 'The user agent should be a valid OS X user agent') - - def test_get_user_agent_default(self): - """ - Test that getting a user agent on a non-Linux/Windows/OS X platform returns the default user agent - """ - with patch('openlp.core.lib.webpagereader.sys') as mocked_sys: - - # GIVEN: The system is Linux - mocked_sys.platform = 'freebsd' - - # WHEN: We call _get_user_agent() - user_agent = _get_user_agent() - - # THEN: The user agent is a Linux (or ChromeOS) user agent - self.assertIn('NetBSD', user_agent, 'The user agent should be the default user agent') - - def test_get_web_page_no_url(self): - """ - Test that sending a URL of None to the get_web_page method returns None - """ - # GIVEN: A None url - test_url = None - - # WHEN: We try to get the test URL - result = get_web_page(test_url) - - # THEN: None should be returned - self.assertIsNone(result, 'The return value of get_web_page should be None') - - def test_get_web_page(self): - """ - Test that the get_web_page method works correctly - """ - with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \ - patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \ - patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \ - patch('openlp.core.common.Registry') as MockRegistry: - # GIVEN: Mocked out objects and a fake URL - mocked_request_object = MagicMock() - MockRequest.return_value = mocked_request_object - mocked_page_object = MagicMock() - mock_urlopen.return_value = mocked_page_object - mock_get_user_agent.return_value = 'user_agent' - fake_url = 'this://is.a.fake/url' - - # WHEN: The get_web_page() method is called - returned_page = get_web_page(fake_url) - - # THEN: The correct methods are called with the correct arguments and a web page is returned - MockRequest.assert_called_with(fake_url) - mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent') - self.assertEqual(1, mocked_request_object.add_header.call_count, - 'There should only be 1 call to add_header') - mock_get_user_agent.assert_called_with() - mock_urlopen.assert_called_with(mocked_request_object, timeout=30) - mocked_page_object.geturl.assert_called_with() - self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called') - self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') - - def test_get_web_page_with_header(self): - """ - Test that adding a header to the call to get_web_page() adds the header to the request - """ - with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \ - patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \ - patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent: - # GIVEN: Mocked out objects, a fake URL and a fake header - mocked_request_object = MagicMock() - MockRequest.return_value = mocked_request_object - mocked_page_object = MagicMock() - mock_urlopen.return_value = mocked_page_object - mock_get_user_agent.return_value = 'user_agent' - fake_url = 'this://is.a.fake/url' - fake_header = ('Fake-Header', 'fake value') - - # WHEN: The get_web_page() method is called - returned_page = get_web_page(fake_url, header=fake_header) - - # THEN: The correct methods are called with the correct arguments and a web page is returned - MockRequest.assert_called_with(fake_url) - mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1]) - self.assertEqual(2, mocked_request_object.add_header.call_count, - 'There should only be 2 calls to add_header') - mock_get_user_agent.assert_called_with() - mock_urlopen.assert_called_with(mocked_request_object, timeout=30) - mocked_page_object.geturl.assert_called_with() - self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') - - def test_get_web_page_with_user_agent_in_headers(self): - """ - Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request - """ - with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \ - patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \ - patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent: - # GIVEN: Mocked out objects, a fake URL and a fake header - mocked_request_object = MagicMock() - MockRequest.return_value = mocked_request_object - mocked_page_object = MagicMock() - mock_urlopen.return_value = mocked_page_object - fake_url = 'this://is.a.fake/url' - user_agent_header = ('User-Agent', 'OpenLP/2.2.0') - - # WHEN: The get_web_page() method is called - returned_page = get_web_page(fake_url, header=user_agent_header) - - # THEN: The correct methods are called with the correct arguments and a web page is returned - MockRequest.assert_called_with(fake_url) - mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1]) - self.assertEqual(1, mocked_request_object.add_header.call_count, - 'There should only be 1 call to add_header') - self.assertEqual(0, mock_get_user_agent.call_count, '_get_user_agent should not have been called') - mock_urlopen.assert_called_with(mocked_request_object, timeout=30) - mocked_page_object.geturl.assert_called_with() - self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') - - def test_get_web_page_update_openlp(self): - """ - Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events() - """ - with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \ - patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \ - patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \ - patch('openlp.core.lib.webpagereader.Registry') as MockRegistry: - # GIVEN: Mocked out objects, a fake URL - mocked_request_object = MagicMock() - MockRequest.return_value = mocked_request_object - mocked_page_object = MagicMock() - mock_urlopen.return_value = mocked_page_object - mock_get_user_agent.return_value = 'user_agent' - mocked_registry_object = MagicMock() - mocked_application_object = MagicMock() - mocked_registry_object.get.return_value = mocked_application_object - MockRegistry.return_value = mocked_registry_object - fake_url = 'this://is.a.fake/url' - - # WHEN: The get_web_page() method is called - returned_page = get_web_page(fake_url, update_openlp=True) - - # THEN: The correct methods are called with the correct arguments and a web page is returned - MockRequest.assert_called_with(fake_url) - mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent') - self.assertEqual(1, mocked_request_object.add_header.call_count, - 'There should only be 1 call to add_header') - mock_urlopen.assert_called_with(mocked_request_object, timeout=30) - mocked_page_object.geturl.assert_called_with() - mocked_registry_object.get.assert_called_with('application') - mocked_application_object.process_events.assert_called_with() - self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') diff --git a/tests/functional/openlp_core_ui/test_first_time.py b/tests/functional/openlp_core_ui/test_first_time.py index d8067dfbe..f23bf4db6 100644 --- a/tests/functional/openlp_core_ui/test_first_time.py +++ b/tests/functional/openlp_core_ui/test_first_time.py @@ -31,7 +31,7 @@ import urllib.parse from tests.functional import patch from tests.helpers.testmixin import TestMixin -from openlp.core.lib.webpagereader import CONNECTION_RETRIES, get_web_page +from openlp.core.common.httputils import CONNECTION_RETRIES, get_web_page class TestFirstTimeWizard(TestMixin, TestCase): From 95e70465de6737ddd4377b871874187bdb67c25f Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Tue, 20 Dec 2016 21:21:17 +0000 Subject: [PATCH 2/9] Add files --- openlp/core/common/httputils.py | 203 ++++++++++++++ .../openlp_core_common/test_httputils.py | 247 ++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 openlp/core/common/httputils.py create mode 100644 tests/functional/openlp_core_common/test_httputils.py diff --git a/openlp/core/common/httputils.py b/openlp/core/common/httputils.py new file mode 100644 index 000000000..cab3ffb49 --- /dev/null +++ b/openlp/core/common/httputils.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +The :mod:`openlp.core.utils` module provides the utility libraries for OpenLP. +""" +import logging +import socket +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from http.client import HTTPException +from random import randint + +from openlp.core.common import Registry + +log = logging.getLogger(__name__ + '.__init__') + +USER_AGENTS = { + 'win32': [ + 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36' + ], + 'darwin': [ + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) ' + 'Chrome/26.0.1410.43 Safari/537.31', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.11 (KHTML, like Gecko) ' + 'Chrome/20.0.1132.57 Safari/536.11', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/536.11 (KHTML, like Gecko) ' + 'Chrome/20.0.1132.47 Safari/536.11', + ], + 'linux2': [ + 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 ' + 'Chrome/25.0.1364.160 Safari/537.22', + 'Mozilla/5.0 (X11; CrOS armv7l 2913.260.0) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.99 ' + 'Safari/537.11', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.27 (KHTML, like Gecko) Chrome/26.0.1389.0 Safari/537.27' + ], + 'default': [ + 'Mozilla/5.0 (X11; NetBSD amd64; rv:18.0) Gecko/20130120 Firefox/18.0' + ] +} +CONNECTION_TIMEOUT = 30 +CONNECTION_RETRIES = 2 + + +class HTTPRedirectHandlerFixed(urllib.request.HTTPRedirectHandler): + """ + Special HTTPRedirectHandler used to work around http://bugs.python.org/issue22248 + (Redirecting to urls with special chars) + """ + def redirect_request(self, req, fp, code, msg, headers, new_url): + # + """ + Test if the new_url can be decoded to ascii + + :param req: + :param fp: + :param code: + :param msg: + :param headers: + :param new_url: + :return: + """ + try: + new_url.encode('latin1').decode('ascii') + fixed_url = new_url + except Exception: + # The url could not be decoded to ascii, so we do some url encoding + fixed_url = urllib.parse.quote(new_url.encode('latin1').decode('utf-8', 'replace'), safe='/:') + return super(HTTPRedirectHandlerFixed, self).redirect_request(req, fp, code, msg, headers, fixed_url) + + +def get_user_agent(): + """ + Return a user agent customised for the platform the user is on. + """ + browser_list = USER_AGENTS.get(sys.platform, None) + if not browser_list: + browser_list = USER_AGENTS['default'] + random_index = randint(0, len(browser_list) - 1) + return browser_list[random_index] + + +def get_web_page(url, header=None, update_openlp=False): + """ + Attempts to download the webpage at url and returns that page or None. + + :param url: The URL to be downloaded. + :param header: An optional HTTP header to pass in the request to the web server. + :param update_openlp: Tells OpenLP to update itself if the page is successfully downloaded. + Defaults to False. + """ + # TODO: Add proxy usage. Get proxy info from OpenLP settings, add to a + # proxy_handler, build into an opener and install the opener into urllib2. + # http://docs.python.org/library/urllib2.html + if not url: + return None + # This is needed to work around http://bugs.python.org/issue22248 and https://bugs.launchpad.net/openlp/+bug/1251437 + opener = urllib.request.build_opener(HTTPRedirectHandlerFixed()) + urllib.request.install_opener(opener) + req = urllib.request.Request(url) + if not header or header[0].lower() != 'user-agent': + user_agent = get_user_agent() + req.add_header('User-Agent', user_agent) + if header: + req.add_header(header[0], header[1]) + log.debug('Downloading URL = %s' % url) + retries = 0 + while retries <= CONNECTION_RETRIES: + retries += 1 + time.sleep(0.1) + try: + page = urllib.request.urlopen(req, timeout=CONNECTION_TIMEOUT) + log.debug('Downloaded page {text}'.format(text=page.geturl())) + break + except urllib.error.URLError as err: + log.exception('URLError on {text}'.format(text=url)) + log.exception('URLError: {text}'.format(text=err.reason)) + page = None + if retries > CONNECTION_RETRIES: + raise + except socket.timeout: + log.exception('Socket timeout: {text}'.format(text=url)) + page = None + if retries > CONNECTION_RETRIES: + raise + except socket.gaierror: + log.exception('Socket gaierror: {text}'.format(text=url)) + page = None + if retries > CONNECTION_RETRIES: + raise + except ConnectionRefusedError: + log.exception('ConnectionRefused: {text}'.format(text=url)) + page = None + if retries > CONNECTION_RETRIES: + raise + break + except ConnectionError: + log.exception('Connection error: {text}'.format(text=url)) + page = None + if retries > CONNECTION_RETRIES: + raise + except HTTPException: + log.exception('HTTPException error: {text}'.format(text=url)) + page = None + if retries > CONNECTION_RETRIES: + raise + except: + # Don't know what's happening, so reraise the original + raise + if update_openlp: + Registry().get('application').process_events() + if not page: + log.exception('{text} could not be downloaded'.format(text=url)) + return None + log.debug(page) + return page + + +def get_url_file_size(url): + """ + Get the size of a file. + + :param url: The URL of the file we want to download. + """ + retries = 0 + while True: + try: + site = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT) + meta = site.info() + return int(meta.get("Content-Length")) + except urllib.error.URLError: + if retries > CONNECTION_RETRIES: + raise + else: + retries += 1 + time.sleep(0.1) + continue + + +__all__ = ['get_web_page'] diff --git a/tests/functional/openlp_core_common/test_httputils.py b/tests/functional/openlp_core_common/test_httputils.py new file mode 100644 index 000000000..7a8d478c5 --- /dev/null +++ b/tests/functional/openlp_core_common/test_httputils.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2016 OpenLP Developers # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +Functional tests to test the AppLocation class and related methods. +""" +from unittest import TestCase + +from openlp.core.common.httputils import get_user_agent, get_web_page, get_url_file_size + +from tests.functional import MagicMock, patch + + +class TestHttpUtils(TestCase): + """ + A test suite to test out various methods around the AppLocation class. + """ + def test_get_user_agent_linux(self): + """ + Test that getting a user agent on Linux returns a user agent suitable for Linux + """ + with patch('openlp.core.common.httputils.sys') as mocked_sys: + + # GIVEN: The system is Linux + mocked_sys.platform = 'linux2' + + # WHEN: We call get_user_agent() + user_agent = get_user_agent() + + # THEN: The user agent is a Linux (or ChromeOS) user agent + result = 'Linux' in user_agent or 'CrOS' in user_agent + self.assertTrue(result, 'The user agent should be a valid Linux user agent') + + def test_get_user_agent_windows(self): + """ + Test that getting a user agent on Windows returns a user agent suitable for Windows + """ + with patch('openlp.core.common.httputils.sys') as mocked_sys: + + # GIVEN: The system is Linux + mocked_sys.platform = 'win32' + + # WHEN: We call get_user_agent() + user_agent = get_user_agent() + + # THEN: The user agent is a Linux (or ChromeOS) user agent + self.assertIn('Windows', user_agent, 'The user agent should be a valid Windows user agent') + + def test_get_user_agent_macos(self): + """ + Test that getting a user agent on OS X returns a user agent suitable for OS X + """ + with patch('openlp.core.common.httputils.sys') as mocked_sys: + + # GIVEN: The system is Linux + mocked_sys.platform = 'darwin' + + # WHEN: We call get_user_agent() + user_agent = get_user_agent() + + # THEN: The user agent is a Linux (or ChromeOS) user agent + self.assertIn('Mac OS X', user_agent, 'The user agent should be a valid OS X user agent') + + def test_get_user_agent_default(self): + """ + Test that getting a user agent on a non-Linux/Windows/OS X platform returns the default user agent + """ + with patch('openlp.core.common.httputils.sys') as mocked_sys: + + # GIVEN: The system is Linux + mocked_sys.platform = 'freebsd' + + # WHEN: We call get_user_agent() + user_agent = get_user_agent() + + # THEN: The user agent is a Linux (or ChromeOS) user agent + self.assertIn('NetBSD', user_agent, 'The user agent should be the default user agent') + + def test_get_web_page_no_url(self): + """ + Test that sending a URL of None to the get_web_page method returns None + """ + # GIVEN: A None url + test_url = None + + # WHEN: We try to get the test URL + result = get_web_page(test_url) + + # THEN: None should be returned + self.assertIsNone(result, 'The return value of get_web_page should be None') + + def test_get_web_page(self): + """ + Test that the get_web_page method works correctly + """ + with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ + patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ + patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \ + patch('openlp.core.common.Registry') as MockRegistry: + # GIVEN: Mocked out objects and a fake URL + mocked_request_object = MagicMock() + MockRequest.return_value = mocked_request_object + mocked_page_object = MagicMock() + mock_urlopen.return_value = mocked_page_object + mock_get_user_agent.return_value = 'user_agent' + fake_url = 'this://is.a.fake/url' + + # WHEN: The get_web_page() method is called + returned_page = get_web_page(fake_url) + + # THEN: The correct methods are called with the correct arguments and a web page is returned + MockRequest.assert_called_with(fake_url) + mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent') + self.assertEqual(1, mocked_request_object.add_header.call_count, + 'There should only be 1 call to add_header') + mock_get_user_agent.assert_called_with() + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) + mocked_page_object.geturl.assert_called_with() + self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called') + self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') + + def test_get_web_page_with_header(self): + """ + Test that adding a header to the call to get_web_page() adds the header to the request + """ + with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ + patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ + patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent: + # GIVEN: Mocked out objects, a fake URL and a fake header + mocked_request_object = MagicMock() + MockRequest.return_value = mocked_request_object + mocked_page_object = MagicMock() + mock_urlopen.return_value = mocked_page_object + mock_get_user_agent.return_value = 'user_agent' + fake_url = 'this://is.a.fake/url' + fake_header = ('Fake-Header', 'fake value') + + # WHEN: The get_web_page() method is called + returned_page = get_web_page(fake_url, header=fake_header) + + # THEN: The correct methods are called with the correct arguments and a web page is returned + MockRequest.assert_called_with(fake_url) + mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1]) + self.assertEqual(2, mocked_request_object.add_header.call_count, + 'There should only be 2 calls to add_header') + mock_get_user_agent.assert_called_with() + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) + mocked_page_object.geturl.assert_called_with() + self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') + + def test_get_web_page_with_user_agent_in_headers(self): + """ + Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request + """ + with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ + patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ + patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent: + # GIVEN: Mocked out objects, a fake URL and a fake header + mocked_request_object = MagicMock() + MockRequest.return_value = mocked_request_object + mocked_page_object = MagicMock() + mock_urlopen.return_value = mocked_page_object + fake_url = 'this://is.a.fake/url' + user_agent_header = ('User-Agent', 'OpenLP/2.2.0') + + # WHEN: The get_web_page() method is called + returned_page = get_web_page(fake_url, header=user_agent_header) + + # THEN: The correct methods are called with the correct arguments and a web page is returned + MockRequest.assert_called_with(fake_url) + mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1]) + self.assertEqual(1, mocked_request_object.add_header.call_count, + 'There should only be 1 call to add_header') + self.assertEqual(0, mock_get_user_agent.call_count, 'get_user_agent should not have been called') + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) + mocked_page_object.geturl.assert_called_with() + self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') + + def test_get_web_page_update_openlp(self): + """ + Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events() + """ + with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ + patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ + patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \ + patch('openlp.core.common.httputils.Registry') as MockRegistry: + # GIVEN: Mocked out objects, a fake URL + mocked_request_object = MagicMock() + MockRequest.return_value = mocked_request_object + mocked_page_object = MagicMock() + mock_urlopen.return_value = mocked_page_object + mock_get_user_agent.return_value = 'user_agent' + mocked_registry_object = MagicMock() + mocked_application_object = MagicMock() + mocked_registry_object.get.return_value = mocked_application_object + MockRegistry.return_value = mocked_registry_object + fake_url = 'this://is.a.fake/url' + + # WHEN: The get_web_page() method is called + returned_page = get_web_page(fake_url, update_openlp=True) + + # THEN: The correct methods are called with the correct arguments and a web page is returned + MockRequest.assert_called_with(fake_url) + mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent') + self.assertEqual(1, mocked_request_object.add_header.call_count, + 'There should only be 1 call to add_header') + mock_urlopen.assert_called_with(mocked_request_object, timeout=30) + mocked_page_object.geturl.assert_called_with() + mocked_registry_object.get.assert_called_with('application') + mocked_application_object.process_events.assert_called_with() + self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object') + + def test_get_url_file_size(self): + """ + Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events() + """ + with patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ + patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent: + # GIVEN: Mocked out objects, a fake URL + mocked_page_object = MagicMock() + mock_urlopen.return_value = mocked_page_object + mock_get_user_agent.return_value = 'user_agent' + fake_url = 'this://is.a.fake/url' + + # WHEN: The get_url_file_size() method is called + size = get_url_file_size(fake_url) + + # THEN: The correct methods are called with the correct arguments and a web page is returned + mock_urlopen.assert_called_with(fake_url, timeout=30) From a368a1d69566ce94988333534ed27088120762b2 Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Tue, 20 Dec 2016 21:59:40 +0000 Subject: [PATCH 3/9] Move urg_get_file --- openlp/core/common/httputils.py | 52 +++++++++++++++++++++++++++++- openlp/core/ui/firsttimeform.py | 57 +++------------------------------ 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/openlp/core/common/httputils.py b/openlp/core/common/httputils.py index cab3ffb49..638e8a98a 100644 --- a/openlp/core/common/httputils.py +++ b/openlp/core/common/httputils.py @@ -22,7 +22,9 @@ """ The :mod:`openlp.core.utils` module provides the utility libraries for OpenLP. """ +import hashlib import logging +import os import socket import sys import time @@ -32,7 +34,7 @@ import urllib.request from http.client import HTTPException from random import randint -from openlp.core.common import Registry +from openlp.core.common import Registry, trace_error_handler log = logging.getLogger(__name__ + '.__init__') @@ -200,4 +202,52 @@ def get_url_file_size(url): continue +def url_get_file(callback, url, f_path, sha256=None): + """" + Download a file given a URL. The file is retrieved in chunks, giving the ability to cancel the download at any + point. Returns False on download error. + + :param url: URL to download + :param f_path: Destination file + """ + block_count = 0 + block_size = 4096 + retries = 0 + while True: + try: + filename = open(f_path, "wb") + url_file = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT) + if sha256: + hasher = hashlib.sha256() + # Download until finished or canceled. + while not callback.was_cancelled: + data = url_file.read(block_size) + if not data: + break + filename.write(data) + if sha256: + hasher.update(data) + block_count += 1 + callback._download_progress(block_count, block_size) + filename.close() + if sha256 and hasher.hexdigest() != sha256: + log.error('sha256 sums did not match for file: {file}'.format(file=f_path)) + os.remove(f_path) + return False + except (urllib.error.URLError, socket.timeout) as err: + trace_error_handler(log) + filename.close() + os.remove(f_path) + if retries > CONNECTION_RETRIES: + return False + else: + retries += 1 + time.sleep(0.1) + continue + break + # Delete file if cancelled, it may be a partial file. + if callback.was_cancelled: + os.remove(f_path) + return True + __all__ = ['get_web_page'] diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index b59f31211..d331bf843 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -22,7 +22,6 @@ """ This module contains the first time wizard. """ -import hashlib import logging import os import socket @@ -39,7 +38,7 @@ from openlp.core.common import Registry, RegistryProperties, AppLocation, Settin translate, clean_button_text, trace_error_handler from openlp.core.lib import PluginStatus, build_icon from openlp.core.lib.ui import critical_error_message_box -from openlp.core.common.httputils import get_web_page, get_url_file_size, CONNECTION_RETRIES, CONNECTION_TIMEOUT +from openlp.core.common.httputils import get_web_page, get_url_file_size, url_get_file, CONNECTION_TIMEOUT from .firsttimewizard import UiFirstTimeWizard, FirstTimePage log = logging.getLogger(__name__) @@ -395,54 +394,6 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): self.was_cancelled = True self.close() - def url_get_file(self, url, f_path, sha256=None): - """" - Download a file given a URL. The file is retrieved in chunks, giving the ability to cancel the download at any - point. Returns False on download error. - - :param url: URL to download - :param f_path: Destination file - """ - block_count = 0 - block_size = 4096 - retries = 0 - while True: - try: - filename = open(f_path, "wb") - url_file = urllib.request.urlopen(url, timeout=CONNECTION_TIMEOUT) - if sha256: - hasher = hashlib.sha256() - # Download until finished or canceled. - while not self.was_cancelled: - data = url_file.read(block_size) - if not data: - break - filename.write(data) - if sha256: - hasher.update(data) - block_count += 1 - self._download_progress(block_count, block_size) - filename.close() - if sha256 and hasher.hexdigest() != sha256: - log.error('sha256 sums did not match for file: {file}'.format(file=f_path)) - os.remove(f_path) - return False - except (urllib.error.URLError, socket.timeout) as err: - trace_error_handler(log) - filename.close() - os.remove(f_path) - if retries > CONNECTION_RETRIES: - return False - else: - retries += 1 - time.sleep(0.1) - continue - break - # Delete file if cancelled, it may be a partial file. - if self.was_cancelled: - os.remove(f_path) - return True - def _build_theme_screenshots(self): """ This method builds the theme screenshots' icons for all items in the ``self.themes_list_widget``. @@ -616,7 +567,7 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): self._increment_progress_bar(self.downloading.format(name=filename), 0) self.previous_size = 0 destination = os.path.join(songs_destination, str(filename)) - if not self.url_get_file('{path}{name}'.format(path=self.songs_url, name=filename), + if not url_get_file(self, '{path}{name}'.format(path=self.songs_url, name=filename), destination, sha256): missed_files.append('Song: {name}'.format(name=filename)) # Download Bibles @@ -628,7 +579,7 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): # TODO: Tested at home self._increment_progress_bar(self.downloading.format(name=bible), 0) self.previous_size = 0 - if not self.url_get_file('{path}{name}'.format(path=self.bibles_url, name=bible), + if not url_get_file(self, '{path}{name}'.format(path=self.bibles_url, name=bible), os.path.join(bibles_destination, bible), sha256): missed_files.append('Bible: {name}'.format(name=bible)) @@ -643,7 +594,7 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): self.previous_size = 0 if not self.url_get_file('{path}{name}'.format(path=self.themes_url, name=theme), os.path.join(themes_destination, theme), - sha256): + sha256, self): missed_files.append('Theme: {name}'.format(name=theme)) if missed_files: file_list = '' From 8570fb5e8cf50aeb37a63870af7bdc6abed03246 Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Wed, 21 Dec 2016 09:41:57 +0000 Subject: [PATCH 4/9] Fix tests --- openlp/core/common/httputils.py | 2 ++ .../openlp_core_common/test_httputils.py | 19 ++++++++++++++++++- .../openlp_core_ui/test_firsttimeform.py | 18 ------------------ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/openlp/core/common/httputils.py b/openlp/core/common/httputils.py index 638e8a98a..b3cb50ef3 100644 --- a/openlp/core/common/httputils.py +++ b/openlp/core/common/httputils.py @@ -207,8 +207,10 @@ def url_get_file(callback, url, f_path, sha256=None): Download a file given a URL. The file is retrieved in chunks, giving the ability to cancel the download at any point. Returns False on download error. + :param callback: the class which needs to be updated :param url: URL to download :param f_path: Destination file + :param sha256: The check sum value to be checked against the download value """ block_count = 0 block_size = 4096 diff --git a/tests/functional/openlp_core_common/test_httputils.py b/tests/functional/openlp_core_common/test_httputils.py index 7a8d478c5..8709e9cfb 100644 --- a/tests/functional/openlp_core_common/test_httputils.py +++ b/tests/functional/openlp_core_common/test_httputils.py @@ -22,9 +22,11 @@ """ Functional tests to test the AppLocation class and related methods. """ +import socket +import os from unittest import TestCase -from openlp.core.common.httputils import get_user_agent, get_web_page, get_url_file_size +from openlp.core.common.httputils import get_user_agent, get_web_page, get_url_file_size, url_get_file from tests.functional import MagicMock, patch @@ -245,3 +247,18 @@ class TestHttpUtils(TestCase): # THEN: The correct methods are called with the correct arguments and a web page is returned mock_urlopen.assert_called_with(fake_url, timeout=30) + + @patch('openlp.core.ui.firsttimeform.urllib.request.urlopen') + def test_socket_timeout(self, mocked_urlopen): + """ + Test socket timeout gets caught + """ + # GIVEN: Mocked urlopen to fake a network disconnect in the middle of a download + mocked_urlopen.side_effect = socket.timeout() + + # WHEN: Attempt to retrieve a file + url_get_file(url='http://localhost/test', f_path=self.tempfile) + + # THEN: socket.timeout should have been caught + # NOTE: Test is if $tmpdir/tempfile is still there, then test fails since ftw deletes bad downloaded files + self.assertFalse(os.path.exists(self.tempfile), 'FTW url_get_file should have caught socket.timeout') \ No newline at end of file diff --git a/tests/functional/openlp_core_ui/test_firsttimeform.py b/tests/functional/openlp_core_ui/test_firsttimeform.py index 5dd1430cd..ec26f60fe 100644 --- a/tests/functional/openlp_core_ui/test_firsttimeform.py +++ b/tests/functional/openlp_core_ui/test_firsttimeform.py @@ -23,7 +23,6 @@ Package to test the openlp.core.ui.firsttimeform package. """ import os -import socket import tempfile import urllib from unittest import TestCase @@ -236,20 +235,3 @@ class TestFirstTimeForm(TestCase, TestMixin): # THEN: the critical_error_message_box should have been called self.assertEquals(mocked_message_box.mock_calls[1][1][0], 'Network Error 407', 'first_time_form should have caught Network Error') - - @patch('openlp.core.ui.firsttimeform.urllib.request.urlopen') - def test_socket_timeout(self, mocked_urlopen): - """ - Test socket timeout gets caught - """ - # GIVEN: Mocked urlopen to fake a network disconnect in the middle of a download - first_time_form = FirstTimeForm(None) - first_time_form.initialize(MagicMock()) - mocked_urlopen.side_effect = socket.timeout() - - # WHEN: Attempt to retrieve a file - first_time_form.url_get_file(url='http://localhost/test', f_path=self.tempfile) - - # THEN: socket.timeout should have been caught - # NOTE: Test is if $tmpdir/tempfile is still there, then test fails since ftw deletes bad downloaded files - self.assertFalse(os.path.exists(self.tempfile), 'FTW url_get_file should have caught socket.timeout') From a69df60978644d47365f7d98e0db059377c288ea Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Wed, 21 Dec 2016 09:47:33 +0000 Subject: [PATCH 5/9] Fix tests again --- .../openlp_core_common/test_httputils.py | 18 ++++++++++++++---- .../openlp_core_ui/test_firsttimeform.py | 3 ++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/functional/openlp_core_common/test_httputils.py b/tests/functional/openlp_core_common/test_httputils.py index 8709e9cfb..6bf7aaa4c 100644 --- a/tests/functional/openlp_core_common/test_httputils.py +++ b/tests/functional/openlp_core_common/test_httputils.py @@ -22,19 +22,29 @@ """ Functional tests to test the AppLocation class and related methods. """ -import socket import os +import tempfile +import socket from unittest import TestCase from openlp.core.common.httputils import get_user_agent, get_web_page, get_url_file_size, url_get_file from tests.functional import MagicMock, patch +from tests.helpers.testmixin import TestMixin -class TestHttpUtils(TestCase): +class TestHttpUtils(TestCase, TestMixin): + """ - A test suite to test out various methods around the AppLocation class. + A test suite to test out various http helper functions. """ + def setUp(self): + self.tempfile = os.path.join(tempfile.gettempdir(), 'testfile') + + def tearDown(self): + if os.path.isfile(self.tempfile): + os.remove(self.tempfile) + def test_get_user_agent_linux(self): """ Test that getting a user agent on Linux returns a user agent suitable for Linux @@ -257,7 +267,7 @@ class TestHttpUtils(TestCase): mocked_urlopen.side_effect = socket.timeout() # WHEN: Attempt to retrieve a file - url_get_file(url='http://localhost/test', f_path=self.tempfile) + url_get_file(MagicMock(), url='http://localhost/test', f_path=self.tempfile) # THEN: socket.timeout should have been caught # NOTE: Test is if $tmpdir/tempfile is still there, then test fails since ftw deletes bad downloaded files diff --git a/tests/functional/openlp_core_ui/test_firsttimeform.py b/tests/functional/openlp_core_ui/test_firsttimeform.py index ec26f60fe..ec77a3134 100644 --- a/tests/functional/openlp_core_ui/test_firsttimeform.py +++ b/tests/functional/openlp_core_ui/test_firsttimeform.py @@ -224,7 +224,8 @@ class TestFirstTimeForm(TestCase, TestMixin): # GIVEN: Initial setup and mocks first_time_form = FirstTimeForm(None) first_time_form.initialize(MagicMock()) - mocked_get_web_page.side_effect = urllib.error.HTTPError(url='http//localhost', + mocked_get_web_page.side_effect = urllib.error.HTTPError(MagicMock(), + url='http//localhost', code=407, msg='Network proxy error', hdrs=None, From 5896e053bc5c5705fff8b58e6058052f10f027fa Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Wed, 21 Dec 2016 09:52:05 +0000 Subject: [PATCH 6/9] Fix tests again --- tests/functional/openlp_core_ui/test_firsttimeform.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/functional/openlp_core_ui/test_firsttimeform.py b/tests/functional/openlp_core_ui/test_firsttimeform.py index ec77a3134..ec26f60fe 100644 --- a/tests/functional/openlp_core_ui/test_firsttimeform.py +++ b/tests/functional/openlp_core_ui/test_firsttimeform.py @@ -224,8 +224,7 @@ class TestFirstTimeForm(TestCase, TestMixin): # GIVEN: Initial setup and mocks first_time_form = FirstTimeForm(None) first_time_form.initialize(MagicMock()) - mocked_get_web_page.side_effect = urllib.error.HTTPError(MagicMock(), - url='http//localhost', + mocked_get_web_page.side_effect = urllib.error.HTTPError(url='http//localhost', code=407, msg='Network proxy error', hdrs=None, From 7bd645bc7212fc83dfe9781eda289cd2126864c3 Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Wed, 21 Dec 2016 10:00:14 +0000 Subject: [PATCH 7/9] pep8 --- openlp/core/ui/firsttimeform.py | 6 +++--- tests/functional/openlp_core_common/test_httputils.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index d331bf843..10456b979 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -568,7 +568,7 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): self.previous_size = 0 destination = os.path.join(songs_destination, str(filename)) if not url_get_file(self, '{path}{name}'.format(path=self.songs_url, name=filename), - destination, sha256): + destination, sha256): missed_files.append('Song: {name}'.format(name=filename)) # Download Bibles bibles_iterator = QtWidgets.QTreeWidgetItemIterator(self.bibles_tree_widget) @@ -580,8 +580,8 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): self._increment_progress_bar(self.downloading.format(name=bible), 0) self.previous_size = 0 if not url_get_file(self, '{path}{name}'.format(path=self.bibles_url, name=bible), - os.path.join(bibles_destination, bible), - sha256): + os.path.join(bibles_destination, bible), + sha256): missed_files.append('Bible: {name}'.format(name=bible)) bibles_iterator += 1 # Download themes diff --git a/tests/functional/openlp_core_common/test_httputils.py b/tests/functional/openlp_core_common/test_httputils.py index 6bf7aaa4c..98b24a994 100644 --- a/tests/functional/openlp_core_common/test_httputils.py +++ b/tests/functional/openlp_core_common/test_httputils.py @@ -271,4 +271,4 @@ class TestHttpUtils(TestCase, TestMixin): # THEN: socket.timeout should have been caught # NOTE: Test is if $tmpdir/tempfile is still there, then test fails since ftw deletes bad downloaded files - self.assertFalse(os.path.exists(self.tempfile), 'FTW url_get_file should have caught socket.timeout') \ No newline at end of file + self.assertFalse(os.path.exists(self.tempfile), 'FTW url_get_file should have caught socket.timeout') From 7323b3797d222391b0ac3b6b022f3ee82fc6f944 Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Wed, 21 Dec 2016 10:20:35 +0000 Subject: [PATCH 8/9] missed one --- openlp/core/ui/firsttimeform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 10456b979..3f56f4089 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -592,9 +592,9 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): # TODO: Tested at home self._increment_progress_bar(self.downloading.format(name=theme), 0) self.previous_size = 0 - if not self.url_get_file('{path}{name}'.format(path=self.themes_url, name=theme), + if not self.url_get_file(self, '{path}{name}'.format(path=self.themes_url, name=theme), os.path.join(themes_destination, theme), - sha256, self): + sha256): missed_files.append('Theme: {name}'.format(name=theme)) if missed_files: file_list = '' From 0643ecdd103e3a13a7506399a663008e947eef0d Mon Sep 17 00:00:00 2001 From: Tim Bentley Date: Wed, 21 Dec 2016 12:46:35 +0000 Subject: [PATCH 9/9] Remove extra self --- openlp/core/ui/firsttimeform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlp/core/ui/firsttimeform.py b/openlp/core/ui/firsttimeform.py index 3f56f4089..8fd7a4f52 100644 --- a/openlp/core/ui/firsttimeform.py +++ b/openlp/core/ui/firsttimeform.py @@ -592,9 +592,9 @@ class FirstTimeForm(QtWidgets.QWizard, UiFirstTimeWizard, RegistryProperties): # TODO: Tested at home self._increment_progress_bar(self.downloading.format(name=theme), 0) self.previous_size = 0 - if not self.url_get_file(self, '{path}{name}'.format(path=self.themes_url, name=theme), - os.path.join(themes_destination, theme), - sha256): + if not url_get_file(self, '{path}{name}'.format(path=self.themes_url, name=theme), + os.path.join(themes_destination, theme), + sha256): missed_files.append('Theme: {name}'.format(name=theme)) if missed_files: file_list = ''