Fix of tracback during SongPro import. Fixes bug 1582152.

Backport of song-search-fix from trunk
Added support for using the new mutool in mudraw mode
Fix crosswalk bible list download
Fix traceback if bible list download fails.

bzr-revno: 2635
Fixes: https://launchpad.net/bugs/1579648, https://launchpad.net/bugs/1582152

openlp/core/utils/__init__.py

@@ -529,7 +529,7 @@ def get_natural_key(string):
     key = [int(part) if part.isdigit() else get_locale_key(part) for part in key]
     # Python 3 does not support comparison of different types anymore. So make sure, that we do not compare str
     # and int.
-    if string[0].isdigit():
+    if string and string[0].isdigit():
         return [b''] + key
     return key
 

openlp/plugins/bibles/forms/bibleimportform.py

@@ -454,7 +454,7 @@ class BibleImportForm(OpenLPWizard):
         :param index: The index of the combo box.
         """
         self.web_translation_combo_box.clear()
-        if self.web_bible_list:
+        if self.web_bible_list and index in self.web_bible_list:
             bibles = list(self.web_bible_list[index].keys())
             bibles.sort(key=get_locale_key)
             self.web_translation_combo_box.addItems(bibles)

openlp/plugins/bibles/lib/http.py

@@ -520,7 +520,7 @@ class CWExtract(RegistryProperties):
         returns a list in the form [(biblename, biblekey, language_code)]
         """
         log.debug('CWExtract.get_bibles_from_http')
-        bible_url = 'http://www.biblestudytools.com/search/bible-search.part/'
+        bible_url = 'http://www.biblestudytools.com/'
         soup = get_soup_for_bible_ref(bible_url)
         if not soup:
             return None
@@ -528,7 +528,7 @@ class CWExtract(RegistryProperties):
         if not bible_select:
             log.debug('No select tags found - did site change?')
             return None
-        option_tags = bible_select.find_all('option')
+        option_tags = bible_select.find_all('option', {'class': 'log-translation'})
         if not option_tags:
             log.debug('No option tags found - did site change?')
             return None

openlp/plugins/presentations/lib/pdfcontroller.py

@@ -93,6 +93,12 @@ class PdfController(PresentationController):
             if found_mudraw:
                 program_type = 'mudraw'
                 break
+            found_mutool = re.search('usage: mutool.*', decoded_line, re.IGNORECASE)
+            if found_mutool:
+                # Test that mutool contains mudraw
+                if re.search('draw\s+--\s+convert document.*', runlog.decode(), re.IGNORECASE | re.MULTILINE):
+                    program_type = 'mutool'
+                    break
             found_gs = re.search('GPL Ghostscript.*', decoded_line, re.IGNORECASE)
             if found_gs:
                 program_type = 'gs'
@@ -117,6 +123,7 @@ class PdfController(PresentationController):
         """
         log.debug('check_installed Pdf')
         self.mudrawbin = ''
+        self.mutoolbin = ''
         self.gsbin = ''
         self.also_supports = []
         # Use the user defined program if given
@@ -127,27 +134,36 @@ class PdfController(PresentationController):
                 self.gsbin = pdf_program
             elif program_type == 'mudraw':
                 self.mudrawbin = pdf_program
+            elif program_type == 'mutool':
+                self.mutoolbin = pdf_program
         else:
             # Fallback to autodetection
             application_path = AppLocation.get_directory(AppLocation.AppDir)
             if is_win():
-                # for windows we only accept mudraw.exe in the base folder
+                # for windows we only accept mudraw.exe or mutool.exe in the base folder
                 application_path = AppLocation.get_directory(AppLocation.AppDir)
                 if os.path.isfile(os.path.join(application_path, 'mudraw.exe')):
                     self.mudrawbin = os.path.join(application_path, 'mudraw.exe')
+                elif os.path.isfile(os.path.join(application_path, 'mutool.exe')):
+                    self.mutoolbin = os.path.join(application_path, 'mutool.exe')
             else:
                 DEVNULL = open(os.devnull, 'wb')
-                # First try to find mupdf
+                # First try to find mudraw
                 self.mudrawbin = which('mudraw')
-                # if mupdf isn't installed, fallback to ghostscript
+                # if mudraw isn't installed, try mutool
                 if not self.mudrawbin:
+                    self.mutoolbin = which('mutool')
+                    # Check we got a working mutool
+                    if not self.mutoolbin or self.check_binary(self.mutoolbin) != 'mutool':
                         self.gsbin = which('gs')
-                # Last option: check if mudraw is placed in OpenLP base folder
+                # Last option: check if mudraw or mutool is placed in OpenLP base folder
-                if not self.mudrawbin and not self.gsbin:
+                if not self.mudrawbin and not self.mutoolbin and not self.gsbin:
                     application_path = AppLocation.get_directory(AppLocation.AppDir)
                     if os.path.isfile(os.path.join(application_path, 'mudraw')):
                         self.mudrawbin = os.path.join(application_path, 'mudraw')
-        if self.mudrawbin:
+                    elif os.path.isfile(os.path.join(application_path, 'mutool')):
+                        self.mutoolbin = os.path.join(application_path, 'mutool')
+        if self.mudrawbin or self.mutoolbin:
             self.also_supports = ['xps', 'oxps']
             return True
         elif self.gsbin:
@@ -254,10 +270,18 @@ class PdfDocument(PresentationDocument):
             if not os.path.isdir(self.get_temp_folder()):
                 os.makedirs(self.get_temp_folder())
             if self.controller.mudrawbin:
+                log.debug('loading presentation using mudraw')
                 runlog = check_output([self.controller.mudrawbin, '-w', str(size.width()), '-h', str(size.height()),
                                        '-o', os.path.join(self.get_temp_folder(), 'mainslide%03d.png'), self.file_path],
                                       startupinfo=self.startupinfo)
+            elif self.controller.mutoolbin:
+                log.debug('loading presentation using mutool')
+                runlog = check_output([self.controller.mutoolbin, 'draw', '-w', str(size.width()), '-h',
+                                       str(size.height()),
+                                       '-o', os.path.join(self.get_temp_folder(), 'mainslide%03d.png'), self.file_path],
+                                      startupinfo=self.startupinfo)
             elif self.controller.gsbin:
+                log.debug('loading presentation using gs')
                 resolution = self.gs_get_resolution(size)
                 runlog = check_output([self.controller.gsbin, '-dSAFER', '-dNOPAUSE', '-dBATCH', '-sDEVICE=png16m',
                                        '-r' + str(resolution), '-dTextAlphaBits=4', '-dGraphicsAlphaBits=4',

openlp/plugins/songs/lib/importers/songpro.py

@@ -72,7 +72,7 @@ class SongProImport(SongImport):
         Receive a single file or a list of files to import.
         """
         self.encoding = None
-        with open(self.import_source, 'rt') as songs_file:
+        with open(self.import_source, 'rt', errors='ignore') as songs_file:
             self.import_wizard.progress_bar.setMaximum(0)
             tag = ''
             text = ''

tests/functional/openlp_plugins/presentations/test_pdfcontroller.py

@@ -29,7 +29,7 @@ from tempfile import mkdtemp
 from PyQt5 import QtCore, QtGui
 
 from openlp.plugins.presentations.lib.pdfcontroller import PdfController, PdfDocument
-from tests.functional import MagicMock
+from tests.functional import MagicMock, patch
 from openlp.core.common import Settings
 from openlp.core.lib import ScreenList
 from tests.utils.constants import TEST_RESOURCES_PATH
@@ -137,3 +137,74 @@ class TestPdfController(TestCase, TestMixin):
         else:
             self.assertEqual(768, image.height(), 'The height should be 768')
             self.assertEqual(543, image.width(), 'The width should be 543')
+
+    @patch('openlp.plugins.presentations.lib.pdfcontroller.check_output')
+    def process_check_binary_mudraw_test(self, mocked_check_binary_exists):
+        """
+        Test that the correct output from mudraw is detected
+        """
+        # GIVEN: A mocked check_binary_exists that returns mudraw output
+        mudraw_output = (b'usage: mudraw [options] input [pages]\n\t-o -\toutput filename (%d for page number)n\t\tsupp'
+                         b'orted formats: pgm, ppm, pam, png, pbmn\t-p -\tpasswordn\t-r -\tresolution in dpi (default: '
+                         b'72)n\t-w -\twidth (in pixels) (maximum width if -r is specified)n\t-h -\theight (in pixels) '
+                         b'(maximum height if -r is specified)')
+        mocked_check_binary_exists.return_value = mudraw_output
+
+        # WHEN: Calling process_check_binary
+        ret = PdfController.check_binary('test')
+
+        # THEN: mudraw should be detected
+        self.assertEqual('mudraw', ret, 'mudraw should have been detected')
+
+    @patch('openlp.plugins.presentations.lib.pdfcontroller.check_output')
+    def process_check_binary_new_motool_test(self, mocked_check_binary_exists):
+        """
+        Test that the correct output from the new mutool is detected
+        """
+        # GIVEN: A mocked check_binary_exists that returns new mutool output
+        new_mutool_output = (b'usage: mutool <command> [options]\n\tdraw\t-- convert document\n\trun\t-- run javascript'
+                             b'\n\tclean\t-- rewrite pdf file\n\textract\t-- extract font and image resources\n\tinfo\t'
+                             b'-- show information about pdf resources\n\tpages\t-- show information about pdf pages\n'
+                             b'\tposter\t-- split large page into many tiles\n\tshow\t-- show internal pdf objects\n\t'
+                             b'create\t-- create pdf document\n\tmerge\t-- merge pages from multiple pdf sources into a'
+                             b'new pdf\n')
+        mocked_check_binary_exists.return_value = new_mutool_output
+
+        # WHEN: Calling process_check_binary
+        ret = PdfController.check_binary('test')
+
+        # THEN: mutool should be detected
+        self.assertEqual('mutool', ret, 'mutool should have been detected')
+
+    @patch('openlp.plugins.presentations.lib.pdfcontroller.check_output')
+    def process_check_binary_old_motool_test(self, mocked_check_binary_exists):
+        """
+        Test that the output from the old mutool is not accepted
+        """
+        # GIVEN: A mocked check_binary_exists that returns old mutool output
+        old_mutool_output = (b'usage: mutool <command> [options]\n\tclean\t-- rewrite pdf file\n\textract\t-- extract '
+                             b'font and image resources\n\tinfo\t-- show information about pdf resources\n\tposter\t-- '
+                             b'split large page into many tiles\n\tshow\t-- show internal pdf objects')
+        mocked_check_binary_exists.return_value = old_mutool_output
+
+        # WHEN: Calling process_check_binary
+        ret = PdfController.check_binary('test')
+
+        # THEN: mutool should be detected
+        self.assertIsNone(ret, 'old mutool should not be accepted!')
+
+    @patch('openlp.plugins.presentations.lib.pdfcontroller.check_output')
+    def process_check_binary_gs_test(self, mocked_check_binary_exists):
+        """
+        Test that the correct output from gs is detected
+        """
+        # GIVEN: A mocked check_binary_exists that returns gs output
+        gs_output = (b'GPL Ghostscript 9.19 (2016-03-23)\nCopyright (C) 2016 Artifex Software, Inc.  All rights reserv'
+                     b'ed.\nUsage: gs [switches] [file1.ps file2.ps ...]')
+        mocked_check_binary_exists.return_value = gs_output
+
+        # WHEN: Calling process_check_binary
+        ret = PdfController.check_binary('test')
+
+        # THEN: mutool should be detected
+        self.assertEqual('gs', ret, 'mutool should have been detected')