Merge branch 'appveyor-macos' into 'master'

Fixes for MacOS build and enable using appveyor for macos

See merge request openlp/packaging!8

builders/builder.py

@@ -24,6 +24,8 @@ Base class for the Windows and macOS builders.
 """
 import os
 import sys
+import runpy
+from io import StringIO
 from argparse import ArgumentParser
 from configparser import ConfigParser
 from shutil import copy, rmtree
@@ -99,6 +101,53 @@ class Builder(object):
             raise Exception(err_msg)
         return output, error
 
+    def _run_module(self, module, argv, err_msg, run_name=None):
+        """
+        Run a python module as if python -m <module>
+        """
+        self._run_runpy('module', module, argv, err_msg, run_name)
+
+    def _run_path(self, path, argv, err_msg, run_name=None):
+        """
+        Run a python script as if python <path>
+        """
+        self._run_runpy('path', path, argv, err_msg, run_name)
+
+    def _run_runpy(self, run_type, exe_arg, argv, err_msg, run_name=None):
+        """
+        Run a python script or module
+        """
+        # Capture stdout and stderr
+        stdout_back = sys.stdout
+        stderr_back = sys.stderr
+        new_stdout = StringIO()
+        new_stderr = StringIO()
+        sys.stdout = new_stdout
+        sys.stderr = new_stderr
+        # Set args
+        sys.argv = argv
+        exit_code = 0
+        try:
+            self._print_verbose('... {}'.format(' '.join(argv)))
+            if run_type == 'module':
+                runpy.run_module(exe_arg, run_name=run_name)
+            else:
+                runpy.run_path(exe_arg, run_name=run_name)
+        except SystemExit as se:
+            if se.code and se.code != 0:
+                exit_code = se.code
+        finally:
+            # Set stdout and stderr back to standard
+            sys.stdout = stdout_back
+            sys.stderr = stderr_back
+        if exit_code != 0:
+            self._print(new_stdout.getvalue())
+            self._print(new_stderr.getvalue())
+            raise Exception(err_msg)
+        else:
+            self._print_verbose(new_stdout.getvalue())
+            self._print_verbose(new_stderr.getvalue())
+
     def _git(self, command, work_path, args=[], err_msg='There was an error running git'):
         """
         Update the code in the branch.
@@ -284,32 +333,28 @@ class Builder(object):
         """
         self._print('Running PyInstaller...')
         os.chdir(self.work_path)
-        if self.pyinstaller_exe.endswith('.py'):
+        cmd = ['pyinstaller'
-            cmd = [self.python, self.pyinstaller_exe]
-        else:
-            cmd = [self.pyinstaller_exe]
-        cmd.extend([
                '--clean',
                '--noconfirm',
                '--windowed',
                '--noupx',
                '--additional-hooks-dir', self.hooks_path,
                '--runtime-hook', os.path.join(self.hooks_path, 'rthook_ssl.py'),
+               # Import to make sqlalchemy work.
+               # Can't be in the custom hook folder because it will conflict with PyInstallers hook
+               '--hidden-import', 'sqlalchemy.ext.baked',
                '-i', self.icon_path,
                '-n', 'OpenLP',
                *self.get_extra_parameters(),  # Adds any extra parameters we wish to use
                self.openlp_script
-        ])
+               ]
         if self.args.verbose:
             cmd.append('--log-level=DEBUG')
         else:
             cmd.append('--log-level=ERROR')
         if self.args.debug:
             cmd.append('-d')
-        self._print_verbose('... {}'.format(' '.join(cmd)))
+        self._run_module('PyInstaller', cmd, 'Error running PyInstaller', run_name='__main__')
-        output, error = self._run_command(cmd, 'Error running PyInstaller')
-        self._print_verbose(output)
-        self._print_verbose(error)
 
     def write_version_file(self):
         """
@@ -327,6 +372,10 @@ class Builder(object):
                 else:
                     self.version = '+'.join(git_version.strip().rsplit('-g', 1))
                     self.version = '.dev'.join(self.version.rsplit('-', 1))
+        try:
+            os.makedirs(self.dist_path)
+        except FileExistsError:
+            pass
         # Write the version to the version file
         with open(os.path.join(self.dist_path, '.version'), 'w') as version_file:
             version_file.write(str(self.version))
@@ -388,7 +437,10 @@ class Builder(object):
         src_dir = os.path.join(self.source_path, 'core', 'ui', 'fonts')
         dst_dir = os.path.join(self.dist_path, 'core', 'ui', 'fonts')
         font_files = ['OpenLP.ttf', 'openlp-charmap.json']
-        os.makedirs(dst_dir)
+        try:
+            os.makedirs(dst_dir)
+        except FileExistsError:
+            pass
         for font_file in font_files:
             src = os.path.join(src_dir, font_file)
             dst = os.path.join(dst_dir, font_file)
@@ -401,7 +453,10 @@ class Builder(object):
         self._print('Copying OpenLP HTML display files...')
         src_dir = os.path.join(self.source_path, 'core', 'display', 'html')
         dst_dir = os.path.join(self.dist_path, 'core', 'display', 'html')
-        os.makedirs(dst_dir)
+        try:
+            os.makedirs(dst_dir)
+        except FileExistsError:
+            pass
         for display_file in os.listdir(src_dir):
             src = os.path.join(src_dir, display_file)
             dst = os.path.join(dst_dir, display_file)
@@ -469,9 +524,8 @@ class Builder(object):
             rmtree(self.manual_build_path)
         os.chdir(self.manual_path)
         sphinx_build = self.get_sphinx_build()
-        command = [self.sphinx_exe, '-b', sphinx_build, '-d', 'build/doctrees', 'source',
+        args = ['sphinx', '-b', sphinx_build, '-d', 'build/doctrees', 'source', 'build/{}'.format(sphinx_build)]
-                   'build/{}'.format(sphinx_build)]
+        self._run_module('sphinx', args, 'Error running sphinx', run_name='__main__')
-        self._run_command(command, 'Error running Sphinx')
         self.after_run_sphinx()
 
     def after_run_sphinx(self):

builders/macosx-builder.py

@@ -27,7 +27,7 @@ Mac OS X Build Script
 This script is used to build the Mac OS X app bundle and pack it into dmg file.
 For this script to work out of the box, it depends on a number of things:
 
-Python 3.4
+Python 3.7
 
 PyQt5
     You should already have this installed, OpenLP doesn't work without it. The
@@ -43,9 +43,7 @@ Sphinx
     the same directory level as OpenLP trunk and named "documentation".
 
 PyInstaller
-    PyInstaller should be a git clone of either
+    PyInstaller can be installed with pip
-    https://github.com/matysek/pyinstaller branch python3 or
-    https://github.com/pyinstaller/pyinstaller branch python3
 
 Git
     You need the command line "git" client installed.
@@ -66,9 +64,8 @@ Mako
 Alembic
     Required for upgrading the databases used in OpenLP.
 
-MuPDF
+PyMuPDF
-    Required for PDF support in OpenLP. Install using macports, or use the
+    Required for PDF support in OpenLP. Install using pip.
-    mudrawbin option in the config file to point to the mudraw binary.
 
 MachOLib
     Python library to analyze and edit Mach-O headers, the executable format
@@ -80,21 +77,22 @@ config.ini.default
     in the bundle as well as directory and file settings for different
     purposes (e.g. PyInstaller location or installer background image)
 
-To install everything you need to install MacPorts. Once MacPorts is installed
+To install everything you should install latest python 3.7 from python.org. It
-and up-to-date, run the following command::
+is recommended to create virtual environment. You can install all dependencies
+like this:
 
-    $ sudo port install python34 py34-pyqt4 py34-sphinx py34-sqlalchemy \
+    $ python -m pip install sqlalchemy alembic appdirs chardet beautifulsoup4 \
-                        py34-macholib py34-mako py34-alembic py34-enchant \
+          lxml Mako mysql-connector-python pytest mock psycopg2-binary \
-                        py34-beautifulsoup4 py34-lxml py34-nose
+          websockets asyncio waitress six webob requests QtAwesome PyQt5 \
-
+          PyQtWebEngine pymediainfo PyMuPDF==1.16.7 QDarkStyle python-vlc \
-You may need to install chardet via pip::
+          Pyro4 zeroconf flask-cors pytest-qt pyenchant pysword pyobjc-core \
-
+          pyobjc-framework-Cocoa dmgbuild sphinx PyInstaller
-    $ sudo pip install chardet
 
 """
 
 import glob
 import os
+import distutils
 from pathlib import Path
 from shutil import copy, copytree, move, rmtree
 
@@ -264,28 +262,16 @@ class MacOSXBuilder(Builder):
         except Exception:
             pass
 
-    def _relink_mudraw(self):
-        """
-        Relink mudraw to bundled libraries
-        """
-        self._relink_binary('mudraw')
-
-    def _relink_mutool(self):
-        """
-        Relink mudraw to bundled libraries
-        """
-        self._relink_binary('mutool')
-
     def _install_pyro4(self):
         """
         Install Pyro4 into the vendor directory
         """
         self._print('Installing Pyro4 for LibreOffice')
         target = os.path.join(self.dist_path, 'plugins', 'presentations', 'lib', 'vendor')
-        self._run_command([self.python, '-m', 'pip', 'install', 'Pyro4', '-t', target, '--disable-pip-version-check',
+        argv = ['pip', 'install', 'Pyro4', '-t', target, '--disable-pip-version-check', '--no-compile']
-                           '--no-compile'], err_msg='Error installing Pyro4')
+        self._run_module('pip', argv, 'Error installing pyro4 with pip', run_name='__main__')
-        egg_info_glob = glob(os.path.join(target, '*.egg-info'))
+        egg_info_glob = glob.glob(os.path.join(target, '*.egg-info'))
-        egg_info_glob.extend(glob(os.path.join(target, '*.dist-info')))
+        egg_info_glob.extend(glob.glob(os.path.join(target, '*.dist-info')))
         self._print_verbose('... glob: {}'.format(egg_info_glob))
         for path in egg_info_glob:
             rmtree(path, True)
@@ -312,6 +298,10 @@ class MacOSXBuilder(Builder):
         Copy Info.plist and OpenLP.icns to app bundle.
         """
         self._print_verbose('... OpenLP.icns')
+        try:
+            os.makedirs(os.path.join(self.dist_app_path, 'Contents', 'Resources'))
+        except FileExistsError:
+            pass
         copy(self.icon_path, os.path.join(self.dist_app_path, 'Contents', 'Resources',
                                           os.path.basename(self.icon_path)))
         self._print_verbose('... Info.plist')
@@ -328,20 +318,13 @@ class MacOSXBuilder(Builder):
         """
         self._print_verbose('... LICENSE.txt')
         copy(self.license_path, os.path.join(self.dist_path, 'LICENSE.txt'))
-        self._print_verbose('... mudraw')
-        if hasattr(self, 'mudraw_exe') and self.mudraw_exe and os.path.isfile(self.mudraw_exe):
-            copy(self.mudraw_exe, os.path.join(self.dist_path, 'mudraw'))
-            self._relink_mudraw()
-        elif hasattr(self, 'mutool_exe') and self.mutool_exe and os.path.isfile(self.mutool_exe):
-            copy(self.mutool_exe, os.path.join(self.dist_path, 'mutool'))
-            self._relink_mutool()
-            copy(self.mutool_lib, os.path.join(self.dist_path, 'libjbig2dec.0.dylib'))
-        else:
-            self._print('... WARNING: mudraw and mutool not found')
 
     def _code_sign(self):
         certificate = self.config.get('codesigning', 'certificate')
         self._print('Checking for certificate...')
+        if not certificate:
+            self._print('Certificate not set, skipping code signing!')
+            return
         self._run_command(['security', 'find-certificate', '-c', certificate],
                           'Could not find certificate "{certificate}" in keychain, '.format(certificate=certificate) +
                           'codesigning will not work without a certificate')
@@ -366,12 +349,11 @@ class MacOSXBuilder(Builder):
         size += 10
 
         os.chdir(os.path.dirname(self.dmg_settings_path))
-        self._run_command([self.dmgbuild_exe, '-s', self.dmg_settings_path, '-D', 'size={size}M'.format(size=size),
+        dmgbuild_exe = os.path.join(distutils.sys.exec_prefix, 'bin', 'dmgbuild')
-                           '-D', 'icon={icon_path}'.format(icon_path=self.icon_path),
+        argv = [dmgbuild_exe, '-s', self.dmg_settings_path, '-D', 'size={size}M'.format(size=size),
-                           '-D', 'app={dist_app_path}'.format(dist_app_path=self.dist_app_path), dmg_title,
+                '-D', 'icon={icon_path}'.format(icon_path=self.icon_path),
-                           self.dmg_file],
+                '-D', 'app={dist_app_path}'.format(dist_app_path=self.dist_app_path), dmg_title, self.dmg_file]
-                          'Unable to run dmgbuild')
+        self._run_path(dmgbuild_exe, argv, 'Error running dmgbuild')
-
         # Dmg done.
         self._print('Finished creating dmg file, resulting file: %s' % self.dmg_file)
 
@@ -414,6 +396,7 @@ class MacOSXBuilder(Builder):
         # self._copy_vlc_files()
         self._copy_bundle_files()
         self._copy_macosx_files()
+        self._install_pyro4()
 
     def after_run_sphinx(self):
         """

osx/Info.plist

@@ -142,5 +142,7 @@
 	<string>OpenLP.help</string>
 	<key>CFBundleHelpBookName</key>
 	<string>org.openlp.OpenLP.help</string>
+	<key>NSRequiresAquaSystemAppearance</key>
+	<false/>
 </dict>
 </plist>

osx/config-appveyor.ini

@@ -0,0 +1,18 @@
+[executables]
+lrelease = /usr/local/opt/qt/bin/lrelease
+
+[paths]
+branch = path/to/openlp/trunk
+documentation = path/to/openlp/documentation
+icon = %(here)s/OpenLP.icns
+bundle_info = %(here)s/Info.plist
+hooks = %(here)s/../pyinstaller-hooks
+dmg_settings = %(here)s/settings.py
+license = %(here)s/LICENSE.txt
+
+[transifex]
+username =
+password =
+
+[codesigning]
+certificate =

osx/config.ini.default

@@ -1,10 +1,5 @@
 [executables]
-sphinx = sphinx-build-3.5
-pyinstaller = /opt/local/Library/Frameworks/Python.framework/Versions/3.5/bin/pyinstaller
 lrelease = /opt/local/libexec/qt5/bin/lrelease
-dmgbuild = dmgbuild
-mudrawbin = mudraw
-mutoolbin = mutool
 
 [paths]
 branch = path/to/openlp/trunk

pyinstaller-hooks/hook-sqlalchemy.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
-
-##########################################################################
-# OpenLP - Open Source Lyrics Projection                                 #
-# ---------------------------------------------------------------------- #
-# Copyright (c) 2008-2019 OpenLP Developers                              #
-# ---------------------------------------------------------------------- #
-# This program is free software: you can redistribute it and/or modify   #
-# it under the terms of the GNU General Public License as published by   #
-# the Free Software Foundation, either version 3 of the License, or      #
-# (at your option) any later version.                                    #
-#                                                                        #
-# This program is distributed in the hope that it will be useful,        #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of         #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          #
-# GNU General Public License for more details.                           #
-#                                                                        #
-# You should have received a copy of the GNU General Public License      #
-# along with this program.  If not, see <https://www.gnu.org/licenses/>. #
-##########################################################################
-
-hiddenimports = ['sqlalchemy.ext.baked']

windows/config-appveyor.ini

@@ -1,11 +1,8 @@
 [executables]
-sphinx = %(pyroot)s\Scripts\sphinx-build.exe
-pyinstaller = %(pyroot)s\Scripts\pyinstaller.exe
 htmlhelp = %(progfilesx86)s\HTML Help Workshop\hhc.exe
 lrelease = C:\Qt\5.12\msvc2017\bin\lrelease.exe
 portablelauncher = %(here)s\..\..\PortableApps.comLauncher\PortableApps.comLauncherGenerator.exe
 portableinstaller = %(here)s\..\..\PortableApps.comInstaller\PortableApps.comInstaller.exe
-mutool = %(here)s\..\..\mupdf-1.14.0-windows\mutool.exe
 candle = %(progfilesx86)s\WiX Toolset v3.11\bin\candle.exe
 light = %(progfilesx86)s\WiX Toolset v3.11\bin\light.exe
 

windows/config.ini.default

@@ -1,11 +1,8 @@
 [executables]
-sphinx = %(pyroot)s\Scripts\sphinx-build.exe
-pyinstaller = %(here)s\..\pyinstaller\pyinstaller.py
 htmlhelp = %(progfiles)s\HTML Help Workshop\hhc.exe
 lrelease = %(sitepackages)s\PyQt5\bin\lrelease.exe
 portablelauncher = %(progfiles)s\PortableApps.comLauncher\PortableApps.comLauncherGenerator.exe
 portableinstaller = %(progfiles)s\PortableApps.comInstaller\PortableApps.comInstaller.exe
-mutool = %(here)s\..\mupdf-1.9a-windows\mutool.exe
 candle = %(progfiles)s\WiX Toolset v3.11\bin\candle.exe
 light = %(progfiles)s\WiX Toolset v3.11\bin\light.exe