HEAD

builders/builder.py

@@ -65,8 +65,8 @@ class Builder(object):
         self.setup_args()
         self.setup_system_paths()
         self.read_config()
-        self.setup_executables()
         self.setup_paths()
+        self.setup_executables()
         self.setup_extra()
 
     def _print(self, text, *args):
@@ -160,6 +160,8 @@ class Builder(object):
         parser.add_argument('--skip-translations', action='store_true', default=False,
                             help='Do NOT update the language translation files')
         parser.add_argument('--debug', action='store_true', default=False, help='Create a debug build')
+        parser.add_argument('--tag-override', metavar='<tag>-bzr<revision>', default=None,
+                            help='Override tag and revision, should be in format <tag>-bzr<revision>')
         self.add_extra_args(parser)
         self.args = parser.parse_args()
 
@@ -262,6 +264,12 @@ class Builder(object):
         self._bzr('export', self.branch_path, ['-r', 'tag:' + self.version, self.work_path],
                   'Error exporting the code')
 
+    def get_extra_parameters(self):
+        """
+        Return a list of any extra parameters we wish to use
+        """
+        return []
+
     def run_pyinstaller(self):
         """
         Run PyInstaller on the branch to build an executable.
@@ -281,6 +289,7 @@ class Builder(object):
                '--runtime-hook', os.path.join(self.hooks_path, 'rthook_ssl.py'),
                '-i', self.icon_path,
                '-n', 'OpenLP',
+               *self.get_extra_parameters(),  # Adds any extra parameters we wish to use
                self.openlp_script
         ])
         if self.args.verbose:
@@ -300,17 +309,20 @@ class Builder(object):
         """
         self._print('Writing version file...')
         if not self.args.release:
-            # This is a development build, get the tag and revision
+            if self.args.tag_override:
-            output = self._bzr('tags', self.branch_path, err_msg='Error running bzr tags')
+                self.version = self.args.tag_override
-            lines = output.splitlines()
-            if len(lines) == 0:
-                tag = '0.0.0'
-                revision = '0'
             else:
-                tag, revision = lines[-1].split()
+                # This is a development build, get the tag and revision
-            output = self._bzr('log', self.branch_path, ['--line', '-r', '-1'], 'Error running bzr log')
+                output = self._bzr('tags', self.branch_path, err_msg='Error running bzr tags')
-            revision = output.split(':')[0]
+                lines = output.splitlines()
-            self.version = '{tag}-bzr{revision}'.format(tag=tag, revision=revision)
+                if len(lines) == 0:
+                    tag = '0.0.0'
+                    revision = '0'
+                else:
+                    tag, revision = lines[-1].split()
+                output = self._bzr('log', self.branch_path, ['--line', '-r', '-1'], 'Error running bzr log')
+                revision = output.split(':')[0]
+                self.version = '{tag}.dev{revision}'.format(tag=tag, revision=revision)
         # Write the version to the version file
         with open(os.path.join(self.dist_path, '.version'), 'w') as version_file:
             version_file.write(str(self.version))
@@ -364,6 +376,33 @@ class Builder(object):
                     self._print_verbose('... %s', filename)
                     copy(os.path.join(root, filename), os.path.join(dest_path, filename))
 
+    def copy_font_files(self):
+        """
+        Copy OpenLP font files
+        """
+        self._print('Copying OpenLP fonts files...')
+        src_dir = os.path.join(self.source_path, 'core', 'ui', 'fonts')
+        dst_dir = os.path.join(self.dist_path, 'core', 'ui', 'fonts')
+        font_files = ['OpenLP.ttf', 'openlp-charmap.json']
+        os.makedirs(dst_dir)
+        for font_file in font_files:
+            src = os.path.join(src_dir, font_file)
+            dst = os.path.join(dst_dir, font_file)
+            copy(src, dst)
+
+    def copy_display_files(self):
+        """
+        Copy OpenLP display HTML files
+        """
+        self._print('Copying OpenLP HTML display files...')
+        src_dir = os.path.join(self.source_path, 'core', 'display', 'html')
+        dst_dir = os.path.join(self.dist_path, 'core', 'display', 'html')
+        os.makedirs(dst_dir)
+        for display_file in os.listdir(src_dir):
+            src = os.path.join(src_dir, display_file)
+            dst = os.path.join(dst_dir, display_file)
+            copy(src, dst)
+
     def copy_extra_files(self):
         """
         Copy any extra files which are particular to a platform
@@ -460,6 +499,8 @@ class Builder(object):
         self.copy_default_theme()
         self.copy_plugins()
         self.copy_media_player()
+        self.copy_font_files()
+        self.copy_display_files()
         if os.path.exists(self.manual_path):
             self.run_sphinx()
         else:

builders/macosx-builder.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4
+# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4
 
 ###############################################################################
 # OpenLP - Open Source Lyrics Projection                                      #
@@ -94,15 +94,15 @@ You may need to install chardet via pip::
 """
 
 import os
-import plistlib
+from pathlib import Path
-import signal
+from shutil import copy, copytree, move, rmtree
-from shutil import copy, copytree
 
 from macholib.MachO import MachO
-from macholib.util import flipwritable, in_system_path
+from macholib.util import in_system_path
 
 from builder import Builder
 
+
 class MacOSXBuilder(Builder):
     """
     The :class:`MacosxBuilder` class encapsulates everything that is needed
@@ -119,6 +119,99 @@ class MacOSXBuilder(Builder):
                 dir_size += os.path.getsize(filename)
         return dir_size
 
+    def _create_symlink(self, folder):
+        """
+        Create the appropriate symlink in the MacOS folder pointing to the Resources folder.
+        """
+        sibling = Path(str(folder).replace('MacOS', ''))
+
+        # PyQt5/Qt/qml/QtQml/Models.2
+        root = str(sibling).partition('Contents')[2].lstrip('/')
+        # ../../../../
+        backward = '../' * len(root.split('/'))
+        # ../../../../Resources/PyQt5/Qt/qml/QtQml/Models.2
+        good_path = f'{backward}Resources/{root}'
+
+        folder.symlink_to(good_path)
+
+    def _fix_qt_dll(self, dll):
+        """
+        Fix the DLL lookup paths to use relative ones for Qt dependencies.
+        Inspiration: PyInstaller/depend/dylib.py:mac_set_relative_dylib_deps()
+        Currently one header is pointing to (we are in the Resources folder):
+            @loader_path/../../../../QtCore (it is referencing to the old MacOS folder)
+        It will be converted to:
+            @loader_path/../../../../../../MacOS/QtCore
+        """
+
+        def match_func(pth):
+            """
+            Callback function for MachO.rewriteLoadCommands() that is
+            called on every lookup path setted in the DLL headers.
+            By returning None for system libraries, it changes nothing.
+            Else we return a relative path pointing to the good file
+            in the MacOS folder.
+            """
+            basename = os.path.basename(pth)
+            if not basename.startswith('Qt'):
+                return None
+            return f'@loader_path{good_path}/{basename}'
+
+        # Resources/PyQt5/Qt/qml/QtQuick/Controls.2/Fusion
+        root = str(dll.parent).partition('Contents')[2][1:]
+        # /../../../../../../..
+        backward = '/..' * len(root.split('/'))
+        # /../../../../../../../MacOS
+        good_path = f'{backward}/MacOS'
+
+        # Rewrite Mach headers with corrected @loader_path
+        dll = MachO(dll)
+        dll.rewriteLoadCommands(match_func)
+        with open(dll.filename, 'rb+') as f:
+            for header in dll.headers:
+                f.seek(0)
+                dll.write(f)
+            f.seek(0, 2)
+            f.flush()
+
+    def _find_problematic_qt_folders(self, folder):
+        """
+        Recursively yields problematic folders (containing a dot in their name).
+        """
+        for path in folder.iterdir():
+            if not path.is_dir() or path.is_symlink():
+                # Skip simlinks as they are allowed (even with a dot)
+                continue
+            if '.' in path.name:
+                yield path
+            else:
+                yield from self._find_problematic_qt_folders(path)
+
+    def _move_contents_to_resources(self, folder):
+        """
+        Recursively move any non symlink file from a problematic folder to the sibling one in Resources.
+        """
+        for path in folder.iterdir():
+            if path.is_symlink():
+                continue
+            if path.is_dir():
+                yield from self._move_contents_to_resources(path)
+            else:
+                sibling = Path(str(path).replace('MacOS', 'Resources'))
+                move(path, sibling)
+                yield sibling
+
+    def _fix_qt_paths(self):
+        """
+        Fix the Qt paths
+        """
+        app_path = Path(self.dist_app_path) / 'Contents' / 'MacOS'
+        for folder in self._find_problematic_qt_folders(app_path):
+            for problematic_file in self._move_contents_to_resources(folder):
+                self._fix_qt_dll(problematic_file)
+            rmtree(folder)
+            self._create_symlink(folder)
+
     def _relink_mupdf(self, bin_name):
         """
         Relink mupdf to bundled libraries
@@ -181,7 +274,8 @@ class MacOSXBuilder(Builder):
         """
         Copy Info.plist and OpenLP.icns to app bundle.
         """
-        copy(self.icon_path, os.path.join(self.dist_app_path, 'Contents', 'Resources', os.path.basename(self.icon_path)))
+        copy(self.icon_path, os.path.join(self.dist_app_path, 'Contents', 'Resources',
+                                          os.path.basename(self.icon_path)))
         # Add OpenLP version to Info.plist and put it to app bundle.
         fr = open(self.bundle_info_path, 'r')
         fw = open(os.path.join(self.dist_app_path, 'Contents', os.path.basename(self.bundle_info_path)), 'w')
@@ -237,9 +331,10 @@ class MacOSXBuilder(Builder):
 
         os.chdir(os.path.dirname(self.dmg_settings_path))
         self._run_command([self.dmgbuild_exe, '-s', self.dmg_settings_path, '-D', 'size={size}M'.format(size=size),
-            '-D', 'icon={icon_path}'.format(icon_path=self.icon_path),
+                           '-D', 'icon={icon_path}'.format(icon_path=self.icon_path),
-            '-D', 'app={dist_app_path}'.format(dist_app_path=self.dist_app_path), dmg_title, self.dmg_file],
+                           '-D', 'app={dist_app_path}'.format(dist_app_path=self.dist_app_path), dmg_title,
-            'Unable to run dmgbuild')
+                           self.dmg_file],
+                          'Unable to run dmgbuild')
 
         # Dmg done.
         self._print('Finished creating dmg file, resulting file: %s' % self.dmg_file)
@@ -264,11 +359,10 @@ class MacOSXBuilder(Builder):
         qt_library_path = QCoreApplication.libraryPaths()[0]
         return os.path.join(os.path.dirname(qt_library_path), 'translations')
 
-    def setup_paths(self):
+    def setup_extra(self):
         """
         Extra setup to run
         """
-        super().setup_paths()
         if hasattr(self, 'mutool_exe'):
             self.mutool_lib = os.path.abspath(
                 os.path.join(os.path.dirname(self.mutool_exe), '..', 'lib', 'libjbig2dec.0.dylib'))
@@ -299,6 +393,7 @@ class MacOSXBuilder(Builder):
         """
         Build the actual DMG
         """
+        self._fix_qt_paths()
         self._code_sign()
         self._create_dmg()
 

builders/windows-builder.py

@@ -26,12 +26,11 @@ Windows Build Script
 This script is used to build the Windows binary and the accompanying installer.
 For this script to work out of the box, it depends on a number of things:
 
-Python 3.4
+Python 3.7
 
 PyQt5
     You should already have this installed, OpenLP doesn't work without it. The
-    version the script expects is the packaged one available from River Bank
+    version the script expects is the packaged one available from pypi.
-    Computing.
 
 PyEnchant
     This script expects the precompiled, installable version of PyEnchant to be
@@ -48,8 +47,7 @@ HTML Help Workshop
     This is used to create the help file.
 
 PyInstaller
-    PyInstaller should be a git clone of
+    PyInstaller can be installed from pypi.
-    https://github.com/matysek/pyinstaller branch develop
 
 Bazaar
     You need the command line "bzr" client installed.
@@ -59,10 +57,6 @@ OpenLP
     shared repository directory. This means your code should be in a directory
     structure like this: "openlp\\branch-name".
 
-Visual C++ 2008 Express Edition
-    This is to build pptviewlib.dll, the library for controlling the
-    PowerPointViewer.
-
 windows-builder.py
     This script, of course. It should be in the "windows-installer" directory
     at the same level as OpenLP trunk.
@@ -115,6 +109,8 @@ Portable App Builds
 """
 
 import os
+import glob
+import sys
 from distutils import dir_util
 from shutil import copy, move, rmtree
 
@@ -133,17 +129,6 @@ class WindowsBuilder(Builder):
     The :class:`WindowsBuilder` class encapsulates everything that is needed
     to build a Windows installer.
     """
-    def _build_pptviewlib(self):
-        """
-        Build the PowerPoint Viewer DLL using Visual Studio.
-        """
-        self._print('Building PPTVIEWLIB.DLL...')
-        if not os.path.exists(self.vcbuild_exe):
-            self._print('... WARNING: vcbuild.exe was not found, skipping building pptviewlib.dll')
-            return
-        self._run_command([self.vcbuild_exe, '/rebuild', os.path.join(self.pptviewlib_path, 'pptviewlib.vcproj'),
-                           'Release|Win32'], 'Error building pptviewlib.dll')
-        copy(os.path.join(self.pptviewlib_path, 'Release', 'pptviewlib.dll'), self.pptviewlib_path)
 
     def _walk_dirs(self, dir_dict, path):
         """
@@ -265,6 +250,7 @@ class WindowsBuilder(Builder):
             content = input_file.read()
             content = content.replace('%(display_version)s', self.portable_version)
             content = content.replace('%(package_version)s', self.portable_version)
+            content = content.replace('%(arch)s', self.arch)
             output_file.write(content)
 
     def _run_portableapp_builder(self):
@@ -303,7 +289,7 @@ class WindowsBuilder(Builder):
         self._print_verbose('... Building PortableApps Installer')
         self._run_command([self.portableinstaller_exe, self.portable_dest_path],
                           'Error running PortableApps Installer')
-        portable_exe_name = 'OpenLPPortable_%s.paf.exe' % self.portable_version
+        portable_exe_name = 'OpenLPPortable_{ver}-{arch}.paf.exe'.format(ver=self.portable_version, arch=self.arch)
         portable_exe_path = os.path.abspath(os.path.join(self.portable_dest_path, '..', portable_exe_name))
         self._print_verbose('... Portable Build: {}'.format(portable_exe_path))
         if os.path.exists(portable_exe_path):
@@ -341,7 +327,7 @@ class WindowsBuilder(Builder):
         """
         Return the path to Qt translation files on macOS
         """
-        return os.path.join(self.site_packages, 'PyQt5', 'translations')
+        return os.path.join(self.site_packages, 'PyQt5', 'Qt', 'translations')
 
     def add_extra_args(self, parser):
         """
@@ -357,7 +343,13 @@ class WindowsBuilder(Builder):
         super().setup_system_paths()
         self.python_root = os.path.dirname(self.python)
         self.site_packages = os.path.join(self.python_root, 'Lib', 'site-packages')
-        self.program_files = os.getenv('PROGRAMFILES')
+        # Default program_files to 'Program Files (x86)' - the folder for 32-bit programs on 64-bit systems, if that
+        # does not exists the host system is 32-bit so fallback to 'Program Files'.
+        self.program_files = os.getenv('PROGRAMFILES(x86)')
+        if not self.program_files:
+            self.program_files = os.getenv('PROGRAMFILES')
+        self._print_verbose('   {:.<20}: {}'.format('site packages: ', self.site_packages))
+        self._print_verbose('   {:.<20}: {}'.format('program files: ', self.program_files))
 
     def setup_paths(self):
         """
@@ -367,7 +359,13 @@ class WindowsBuilder(Builder):
         self.dist_path = os.path.join(self.work_path, 'dist', 'OpenLP')
         self.helpfile_path = os.path.join(self.manual_build_path, 'htmlhelp')
         self.winres_path = os.path.join(self.branch_path, 'resources', 'windows')
-        self.pptviewlib_path = os.path.join(self.source_path, 'plugins', 'presentations', 'lib', 'pptviewlib')
+
+    def setup_extra(self):
+        """
+        Extra setup to run
+        """
+        # Detect python instance bit size
+        self.arch = 'x86' if sys.maxsize == 0x7fffffff else 'x64'
 
     def copy_extra_files(self):
         """
@@ -408,12 +406,24 @@ class WindowsBuilder(Builder):
         """
         Build the installer
         """
-        self._build_pptviewlib()
         self._create_wix_file()
         self._run_wix_tools()
         if self.args.portable:
             self._run_portableapp_builder()
 
+    def get_extra_parameters(self):
+        """
+        Return a list of any extra parameters we wish to use
+        """
+        parameters = []
+        dll_path = '{pf}\\Windows Kits\\10\\Redist\\ucrt\\DLLs\\{arch}\\*.dll'.format(pf=self.program_files,
+                                                                                      arch=self.arch)
+        # Finds the UCRT DDLs available from the Windows 10 SDK
+        for binary in glob.glob(dll_path):
+            parameters.append('--add-binary')
+            parameters.append(binary + ";.")
+        return parameters
+
 
 if __name__ == '__main__':
     WindowsBuilder().main()

windows/OpenLP.iss.default

@@ -7,6 +7,7 @@
 #define AppPublisher "OpenLP Developers"
 #define AppURL "http://openlp.org/"
 #define AppExeName "OpenLP.exe"
+#define Arch "%(arch)s"
 
 #define FileHandle FileOpen("%(branch)s\dist\OpenLP\.version")
 #define FileLine FileRead(FileHandle)
@@ -30,7 +31,7 @@ DefaultGroupName={#AppName}
 AllowNoIcons=true
 LicenseFile=LICENSE.txt
 OutputDir=%(branch)s\dist\
-OutputBaseFilename=OpenLP-{#RealVersion}-setup
+OutputBaseFilename=OpenLP-{#RealVersion}-{#Arch}-setup
 Compression=lzma/Max
 SolidCompression=true
 SetupIconFile=OpenLP.ico
@@ -86,6 +87,7 @@ Filename: {app}\{#AppExeName}; Description: {cm:LaunchProgram,{#AppName}}; Flags
 
 [Registry]
 Root: HKCR; Subkey: .osz; ValueType: string; ValueName: ; ValueData: OpenLP; Flags: uninsdeletevalue
+Root: HKCR; Subkey: .oszl; ValueType: string; ValueName: ; ValueData: OpenLP; Flags: uninsdeletevalue
 Root: HKCR; Subkey: OpenLP; ValueType: string; ValueName: ; ValueData: OpenLP Service; Flags: uninsdeletekey
 Root: HKCR; Subkey: OpenLP\DefaultIcon; ValueType: string; ValueName: ; ValueData: {app}\OpenLP.exe,0
 Root: HKCR; Subkey: OpenLP\shell\open\command; ValueType: string; ValueName: ; ValueData: """{app}\OpenLP.exe"" ""%1"""

windows/appinfo.ini.default

@@ -18,7 +18,7 @@ Freeware=true
 CommercialUse=true
 
 [Version]
-DisplayVersion=%(display_version)s
+DisplayVersion=%(display_version)s-%(arch)s
 PackageVersion=%(package_version)s
 
 [Control]
@@ -27,6 +27,8 @@ Start=OpenLPPortable.exe
 
 [Associations]
 FileType=osz
+FileType=oszl
 
 [FileTypeIcons]
 osz=app
+oszl=app

windows/config-appveyor.ini

@@ -1,18 +1,17 @@
 [executables]
 innosetup = %(progfiles)s\Inno Setup 5\ISCC.exe
 sphinx = %(pyroot)s\Scripts\sphinx-build.exe
-pyinstaller = %(here)s\..\..\PyInstaller-3.2\pyinstaller.py
+pyinstaller = %(pyroot)s\Scripts\pyinstaller-script.py
-vcbuild = %(progfiles)s\Microsoft Visual Studio 9.0\VC\vcpackages\vcbuild.exe
 htmlhelp = %(progfiles)s\HTML Help Workshop\hhc.exe
 psvince = %(here)s\psvince.dll
-lrelease = C:\Qt\5.5\msvc2013\bin\lrelease.exe
+lrelease = C:\Qt\5.12\msvc2017\bin\lrelease.exe
 portablelauncher = %(here)s\..\..\PortableApps.comLauncher\PortableApps.comLauncherGenerator.exe
 portableinstaller = %(here)s\..\..\PortableApps.comInstaller\PortableApps.comInstaller.exe
-mutool = %(here)s\..\..\mupdf-1.9a-windows\mutool.exe
+mutool = %(here)s\..\..\mupdf-1.14.0-windows\mutool.exe
 mediainfo = %(here)s\..\..\MediaInfo\MediaInfo.exe
 
 [paths]
-branch = %(projects)s\trunk
+branch = %(projects)s\openlp-branch
 documentation = %(projects)s\documentation
 icon = %(here)s\OpenLP.ico
 hooks = %(here)s\..\pyinstaller-hooks

windows/config.ini.default

@@ -2,7 +2,6 @@
 innosetup = %(progfiles)s\Inno Setup 5\ISCC.exe
 sphinx = %(pyroot)s\Scripts\sphinx-build.exe
 pyinstaller = %(here)s\..\pyinstaller\pyinstaller.py
-vcbuild = %(progfiles)s\Microsoft Visual Studio 9.0\VC\vcpackages\vcbuild.exe
 htmlhelp = %(progfiles)s\HTML Help Workshop\hhc.exe
 psvince = %(here)s\psvince.dll
 lrelease = %(sitepackages)s\PyQt5\bin\lrelease.exe