diff --git a/builders/builder.py b/builders/builder.py index 596de87..6c07166 100644 --- a/builders/builder.py +++ b/builders/builder.py @@ -24,6 +24,8 @@ Base class for the Windows and macOS builders. """ import os import sys +import runpy +from io import StringIO from argparse import ArgumentParser from configparser import ConfigParser from shutil import copy, rmtree @@ -99,6 +101,53 @@ class Builder(object): raise Exception(err_msg) return output, error + def _run_module(self, module, argv, err_msg, run_name=None): + """ + Run a python module as if python -m + """ + self._run_runpy('module', module, argv, err_msg, run_name) + + def _run_path(self, path, argv, err_msg, run_name=None): + """ + Run a python script as if python + """ + self._run_runpy('path', path, argv, err_msg, run_name) + + def _run_runpy(self, run_type, exe_arg, argv, err_msg, run_name=None): + """ + Run a python script or module + """ + # Capture stdout and stderr + stdout_back = sys.stdout + stderr_back = sys.stderr + new_stdout = StringIO() + new_stderr = StringIO() + sys.stdout = new_stdout + sys.stderr = new_stderr + # Set args + sys.argv = argv + exit_code = 0 + try: + self._print_verbose('... {}'.format(' '.join(argv))) + if run_type == 'module': + runpy.run_module(exe_arg, run_name=run_name) + else: + runpy.run_path(exe_arg, run_name=run_name) + except SystemExit as se: + if se.code and se.code != 0: + exit_code = se.code + finally: + # Set stdout and stderr back to standard + sys.stdout = stdout_back + sys.stderr = stderr_back + if exit_code != 0: + self._print(new_stdout.getvalue()) + self._print(new_stderr.getvalue()) + raise Exception(err_msg) + else: + self._print_verbose(new_stdout.getvalue()) + self._print_verbose(new_stderr.getvalue()) + def _git(self, command, work_path, args=[], err_msg='There was an error running git'): """ Update the code in the branch. @@ -284,32 +333,28 @@ class Builder(object): """ self._print('Running PyInstaller...') os.chdir(self.work_path) - if self.pyinstaller_exe.endswith('.py'): - cmd = [self.python, self.pyinstaller_exe] - else: - cmd = [self.pyinstaller_exe] - cmd.extend([ + cmd = ['pyinstaller' '--clean', '--noconfirm', '--windowed', '--noupx', '--additional-hooks-dir', self.hooks_path, '--runtime-hook', os.path.join(self.hooks_path, 'rthook_ssl.py'), + # Import to make sqlalchemy work. + # Can't be in the custom hook folder because it will conflict with PyInstallers hook + '--hidden-import', 'sqlalchemy.ext.baked', '-i', self.icon_path, '-n', 'OpenLP', *self.get_extra_parameters(), # Adds any extra parameters we wish to use self.openlp_script - ]) + ] if self.args.verbose: cmd.append('--log-level=DEBUG') else: cmd.append('--log-level=ERROR') if self.args.debug: cmd.append('-d') - self._print_verbose('... {}'.format(' '.join(cmd))) - output, error = self._run_command(cmd, 'Error running PyInstaller') - self._print_verbose(output) - self._print_verbose(error) + self._run_module('PyInstaller', cmd, 'Error running PyInstaller', run_name='__main__') def write_version_file(self): """ @@ -327,6 +372,10 @@ class Builder(object): else: self.version = '+'.join(git_version.strip().rsplit('-g', 1)) self.version = '.dev'.join(self.version.rsplit('-', 1)) + try: + os.makedirs(self.dist_path) + except FileExistsError: + pass # Write the version to the version file with open(os.path.join(self.dist_path, '.version'), 'w') as version_file: version_file.write(str(self.version)) @@ -388,7 +437,10 @@ class Builder(object): src_dir = os.path.join(self.source_path, 'core', 'ui', 'fonts') dst_dir = os.path.join(self.dist_path, 'core', 'ui', 'fonts') font_files = ['OpenLP.ttf', 'openlp-charmap.json'] - os.makedirs(dst_dir) + try: + os.makedirs(dst_dir) + except FileExistsError: + pass for font_file in font_files: src = os.path.join(src_dir, font_file) dst = os.path.join(dst_dir, font_file) @@ -401,7 +453,10 @@ class Builder(object): self._print('Copying OpenLP HTML display files...') src_dir = os.path.join(self.source_path, 'core', 'display', 'html') dst_dir = os.path.join(self.dist_path, 'core', 'display', 'html') - os.makedirs(dst_dir) + try: + os.makedirs(dst_dir) + except FileExistsError: + pass for display_file in os.listdir(src_dir): src = os.path.join(src_dir, display_file) dst = os.path.join(dst_dir, display_file) @@ -469,9 +524,8 @@ class Builder(object): rmtree(self.manual_build_path) os.chdir(self.manual_path) sphinx_build = self.get_sphinx_build() - command = [self.sphinx_exe, '-b', sphinx_build, '-d', 'build/doctrees', 'source', - 'build/{}'.format(sphinx_build)] - self._run_command(command, 'Error running Sphinx') + args = ['sphinx', '-b', sphinx_build, '-d', 'build/doctrees', 'source', 'build/{}'.format(sphinx_build)] + self._run_module('sphinx', args, 'Error running sphinx', run_name='__main__') self.after_run_sphinx() def after_run_sphinx(self): diff --git a/builders/macosx-builder.py b/builders/macosx-builder.py index 77a7c0f..6157a2c 100644 --- a/builders/macosx-builder.py +++ b/builders/macosx-builder.py @@ -27,7 +27,7 @@ Mac OS X Build Script This script is used to build the Mac OS X app bundle and pack it into dmg file. For this script to work out of the box, it depends on a number of things: -Python 3.4 +Python 3.7 PyQt5 You should already have this installed, OpenLP doesn't work without it. The @@ -43,9 +43,7 @@ Sphinx the same directory level as OpenLP trunk and named "documentation". PyInstaller - PyInstaller should be a git clone of either - https://github.com/matysek/pyinstaller branch python3 or - https://github.com/pyinstaller/pyinstaller branch python3 + PyInstaller can be installed with pip Git You need the command line "git" client installed. @@ -66,9 +64,8 @@ Mako Alembic Required for upgrading the databases used in OpenLP. -MuPDF - Required for PDF support in OpenLP. Install using macports, or use the - mudrawbin option in the config file to point to the mudraw binary. +PyMuPDF + Required for PDF support in OpenLP. Install using pip. MachOLib Python library to analyze and edit Mach-O headers, the executable format @@ -80,21 +77,22 @@ config.ini.default in the bundle as well as directory and file settings for different purposes (e.g. PyInstaller location or installer background image) -To install everything you need to install MacPorts. Once MacPorts is installed -and up-to-date, run the following command:: +To install everything you should install latest python 3.7 from python.org. It +is recommended to create virtual environment. You can install all dependencies +like this: - $ sudo port install python34 py34-pyqt4 py34-sphinx py34-sqlalchemy \ - py34-macholib py34-mako py34-alembic py34-enchant \ - py34-beautifulsoup4 py34-lxml py34-nose - -You may need to install chardet via pip:: - - $ sudo pip install chardet + $ python -m pip install sqlalchemy alembic appdirs chardet beautifulsoup4 \ + lxml Mako mysql-connector-python pytest mock psycopg2-binary \ + websockets asyncio waitress six webob requests QtAwesome PyQt5 \ + PyQtWebEngine pymediainfo PyMuPDF==1.16.7 QDarkStyle python-vlc \ + Pyro4 zeroconf flask-cors pytest-qt pyenchant pysword pyobjc-core \ + pyobjc-framework-Cocoa dmgbuild sphinx PyInstaller """ import glob import os +import distutils from pathlib import Path from shutil import copy, copytree, move, rmtree @@ -264,28 +262,16 @@ class MacOSXBuilder(Builder): except Exception: pass - def _relink_mudraw(self): - """ - Relink mudraw to bundled libraries - """ - self._relink_binary('mudraw') - - def _relink_mutool(self): - """ - Relink mudraw to bundled libraries - """ - self._relink_binary('mutool') - def _install_pyro4(self): """ Install Pyro4 into the vendor directory """ self._print('Installing Pyro4 for LibreOffice') target = os.path.join(self.dist_path, 'plugins', 'presentations', 'lib', 'vendor') - self._run_command([self.python, '-m', 'pip', 'install', 'Pyro4', '-t', target, '--disable-pip-version-check', - '--no-compile'], err_msg='Error installing Pyro4') - egg_info_glob = glob(os.path.join(target, '*.egg-info')) - egg_info_glob.extend(glob(os.path.join(target, '*.dist-info'))) + argv = ['pip', 'install', 'Pyro4', '-t', target, '--disable-pip-version-check', '--no-compile'] + self._run_module('pip', argv, 'Error installing pyro4 with pip', run_name='__main__') + egg_info_glob = glob.glob(os.path.join(target, '*.egg-info')) + egg_info_glob.extend(glob.glob(os.path.join(target, '*.dist-info'))) self._print_verbose('... glob: {}'.format(egg_info_glob)) for path in egg_info_glob: rmtree(path, True) @@ -312,6 +298,10 @@ class MacOSXBuilder(Builder): Copy Info.plist and OpenLP.icns to app bundle. """ self._print_verbose('... OpenLP.icns') + try: + os.makedirs(os.path.join(self.dist_app_path, 'Contents', 'Resources')) + except FileExistsError: + pass copy(self.icon_path, os.path.join(self.dist_app_path, 'Contents', 'Resources', os.path.basename(self.icon_path))) self._print_verbose('... Info.plist') @@ -328,20 +318,13 @@ class MacOSXBuilder(Builder): """ self._print_verbose('... LICENSE.txt') copy(self.license_path, os.path.join(self.dist_path, 'LICENSE.txt')) - self._print_verbose('... mudraw') - if hasattr(self, 'mudraw_exe') and self.mudraw_exe and os.path.isfile(self.mudraw_exe): - copy(self.mudraw_exe, os.path.join(self.dist_path, 'mudraw')) - self._relink_mudraw() - elif hasattr(self, 'mutool_exe') and self.mutool_exe and os.path.isfile(self.mutool_exe): - copy(self.mutool_exe, os.path.join(self.dist_path, 'mutool')) - self._relink_mutool() - copy(self.mutool_lib, os.path.join(self.dist_path, 'libjbig2dec.0.dylib')) - else: - self._print('... WARNING: mudraw and mutool not found') def _code_sign(self): certificate = self.config.get('codesigning', 'certificate') self._print('Checking for certificate...') + if not certificate: + self._print('Certificate not set, skipping code signing!') + return self._run_command(['security', 'find-certificate', '-c', certificate], 'Could not find certificate "{certificate}" in keychain, '.format(certificate=certificate) + 'codesigning will not work without a certificate') @@ -366,12 +349,11 @@ class MacOSXBuilder(Builder): size += 10 os.chdir(os.path.dirname(self.dmg_settings_path)) - self._run_command([self.dmgbuild_exe, '-s', self.dmg_settings_path, '-D', 'size={size}M'.format(size=size), - '-D', 'icon={icon_path}'.format(icon_path=self.icon_path), - '-D', 'app={dist_app_path}'.format(dist_app_path=self.dist_app_path), dmg_title, - self.dmg_file], - 'Unable to run dmgbuild') - + dmgbuild_exe = os.path.join(distutils.sys.exec_prefix, 'bin', 'dmgbuild') + argv = [dmgbuild_exe, '-s', self.dmg_settings_path, '-D', 'size={size}M'.format(size=size), + '-D', 'icon={icon_path}'.format(icon_path=self.icon_path), + '-D', 'app={dist_app_path}'.format(dist_app_path=self.dist_app_path), dmg_title, self.dmg_file] + self._run_path(dmgbuild_exe, argv, 'Error running dmgbuild') # Dmg done. self._print('Finished creating dmg file, resulting file: %s' % self.dmg_file) @@ -414,6 +396,7 @@ class MacOSXBuilder(Builder): # self._copy_vlc_files() self._copy_bundle_files() self._copy_macosx_files() + self._install_pyro4() def after_run_sphinx(self): """ diff --git a/osx/Info.plist b/osx/Info.plist index 4435075..6fdc63c 100755 --- a/osx/Info.plist +++ b/osx/Info.plist @@ -142,5 +142,7 @@ OpenLP.help CFBundleHelpBookName org.openlp.OpenLP.help + NSRequiresAquaSystemAppearance + diff --git a/osx/config-appveyor.ini b/osx/config-appveyor.ini new file mode 100644 index 0000000..e3d3362 --- /dev/null +++ b/osx/config-appveyor.ini @@ -0,0 +1,18 @@ +[executables] +lrelease = /usr/local/opt/qt/bin/lrelease + +[paths] +branch = path/to/openlp/trunk +documentation = path/to/openlp/documentation +icon = %(here)s/OpenLP.icns +bundle_info = %(here)s/Info.plist +hooks = %(here)s/../pyinstaller-hooks +dmg_settings = %(here)s/settings.py +license = %(here)s/LICENSE.txt + +[transifex] +username = +password = + +[codesigning] +certificate = diff --git a/osx/config.ini.default b/osx/config.ini.default index 71828a7..9cab404 100644 --- a/osx/config.ini.default +++ b/osx/config.ini.default @@ -1,10 +1,5 @@ [executables] -sphinx = sphinx-build-3.5 -pyinstaller = /opt/local/Library/Frameworks/Python.framework/Versions/3.5/bin/pyinstaller lrelease = /opt/local/libexec/qt5/bin/lrelease -dmgbuild = dmgbuild -mudrawbin = mudraw -mutoolbin = mutool [paths] branch = path/to/openlp/trunk diff --git a/pyinstaller-hooks/hook-sqlalchemy.py b/pyinstaller-hooks/hook-sqlalchemy.py deleted file mode 100644 index 3b67f40..0000000 --- a/pyinstaller-hooks/hook-sqlalchemy.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 - -########################################################################## -# OpenLP - Open Source Lyrics Projection # -# ---------------------------------------------------------------------- # -# Copyright (c) 2008-2019 OpenLP Developers # -# ---------------------------------------------------------------------- # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -########################################################################## - -hiddenimports = ['sqlalchemy.ext.baked'] diff --git a/windows/config-appveyor.ini b/windows/config-appveyor.ini index 6bd4aa5..8dacd83 100644 --- a/windows/config-appveyor.ini +++ b/windows/config-appveyor.ini @@ -1,11 +1,8 @@ [executables] -sphinx = %(pyroot)s\Scripts\sphinx-build.exe -pyinstaller = %(pyroot)s\Scripts\pyinstaller.exe htmlhelp = %(progfilesx86)s\HTML Help Workshop\hhc.exe lrelease = C:\Qt\5.12\msvc2017\bin\lrelease.exe portablelauncher = %(here)s\..\..\PortableApps.comLauncher\PortableApps.comLauncherGenerator.exe portableinstaller = %(here)s\..\..\PortableApps.comInstaller\PortableApps.comInstaller.exe -mutool = %(here)s\..\..\mupdf-1.14.0-windows\mutool.exe candle = %(progfilesx86)s\WiX Toolset v3.11\bin\candle.exe light = %(progfilesx86)s\WiX Toolset v3.11\bin\light.exe diff --git a/windows/config.ini.default b/windows/config.ini.default index 0929567..a8201ee 100644 --- a/windows/config.ini.default +++ b/windows/config.ini.default @@ -1,11 +1,8 @@ [executables] -sphinx = %(pyroot)s\Scripts\sphinx-build.exe -pyinstaller = %(here)s\..\pyinstaller\pyinstaller.py htmlhelp = %(progfiles)s\HTML Help Workshop\hhc.exe lrelease = %(sitepackages)s\PyQt5\bin\lrelease.exe portablelauncher = %(progfiles)s\PortableApps.comLauncher\PortableApps.comLauncherGenerator.exe portableinstaller = %(progfiles)s\PortableApps.comInstaller\PortableApps.comInstaller.exe -mutool = %(here)s\..\mupdf-1.9a-windows\mutool.exe candle = %(progfiles)s\WiX Toolset v3.11\bin\candle.exe light = %(progfiles)s\WiX Toolset v3.11\bin\light.exe