# -*- coding: utf-8 -*- # vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4 ############################################################################### # OpenLP - Open Source Lyrics Projection # # --------------------------------------------------------------------------- # # Copyright (c) 2008-2015 OpenLP Developers # # --------------------------------------------------------------------------- # # This program is free software; you can redistribute it and/or modify it # # under the terms of the GNU General Public License as published by the Free # # Software Foundation; version 2 of the License. # # # # This program is distributed in the hope that it will be useful, but WITHOUT # # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # # more details. # # # # You should have received a copy of the GNU General Public License along # # with this program; if not, write to the Free Software Foundation, Inc., 59 # # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ Mac OS X Build Script -------------------- This script is used to build the Mac OS X app bundle and pack it into dmg file. For this script to work out of the box, it depends on a number of things: Python 3.4 PyQt5 You should already have this installed, OpenLP doesn't work without it. The version the script expects is the packaged one available from River Bank Computing. PyEnchant This script expects the precompiled, installable version of PyEnchant to be installed. You can find this on the PyEnchant site. Sphinx This is used to build the documentation. The documentation trunk must be at the same directory level as OpenLP trunk and named "documentation". PyInstaller PyInstaller should be a git clone of either https://github.com/matysek/pyinstaller branch python3 or https://github.com/pyinstaller/pyinstaller branch python3 Bazaar You need the command line "bzr" client installed. OpenLP A checkout of the latest code, in a branch directory, which is in a Bazaar shared repository directory. This means your code should be in a directory structure like this: "openlp\branch-name". macosx-builder.py This script, of course. It should be in the "osx-package" directory at the same level as OpenLP trunk. Mako Mako Templates for Python. This package is required for building the remote plugin. Alembic Required for upgrading the databases used in OpenLP. MuPDF Required for PDF support in OpenLP. Install using macports, or use the mudrawbin option in the config file to point to the mudraw binary. MachOLib Python library to analyze and edit Mach-O headers, the executable format used by Mac OS X. Used to relink the mudraw binary from MuPDF to the bundled libraries. Install using macports or pip. config.ini.default The configuration file contains settings of the version string to include in the bundle as well as directory and file settings for different purposes (e.g. PyInstaller location or installer background image) To install everything you need to install MacPorts. Once MacPorts is installed and up-to-date, run the following command:: $ sudo port install python34 py34-pyqt4 py34-sphinx py34-sqlalchemy \ py34-macholib py34-mako py34-alembic py34-enchant \ py34-beautifulsoup4 py34-lxml py34-nose You may need to install chardet via pip:: $ sudo pip install chardet """ import os import plistlib import signal import subprocess import sys from shutil import copy, copytree, rmtree from subprocess import Popen, PIPE from configparser import ConfigParser from argparse import ArgumentParser from macholib.MachO import MachO from macholib.util import flipwritable, in_system_path def _which(command): """ Return absolute path to a command found on system PATH. """ if command.startswith('/'): return command for path in os.environ["PATH"].split(os.pathsep): if os.access(os.path.join(path, command), os.X_OK): return "%s/%s" % (path, command) class MacosxBuilder(object): """ The :class:`MacosxBuilder` class encapsulates everything that is needed to build a Mac OS X .dmg file. """ def __init__(self): self.setup_args() self.setup_system_paths() self.read_config() self.setup_executables() self.setup_paths() def _print(self, text, *args): """ Print stuff out. Later we might want to use a log file. """ if len(args) > 0: text = text % tuple(args) print(text) def _print_verbose(self, text, *args): """ Print output, obeying "verbose" mode. """ if self.args.verbose: self._print(text, *args) def _run_command(self, cmd, err_msg): """ Run command in subprocess and print error message in case of Exception. Return text from stdout. """ proc = Popen(cmd, stdout=PIPE, stderr=PIPE) output, error = proc.communicate() code = proc.wait() if code != 0: self._print(output) self._print(error) raise Exception(err_msg) return output def _get_directory_size(self, directory): """ Return directory size - size of everything in the dir. """ dir_size = 0 for (path, dirs, files) in os.walk(directory): for file in files: filename = os.path.join(path, file) dir_size += os.path.getsize(filename) return dir_size def _get_mountpoints(self): """ Return list of mounted disks on Mac. """ # Get the output in plist format. paths = [] output = self._run_command([self.hdiutil, 'info', '-plist'], 'Detecting mount points failed.') pl = plistlib.readPlistFromBytes(output) for image in pl['images']: for se in image['system-entities']: if se.get('mount-point'): paths.append(se.get('mount-point')) return paths def setup_args(self): """ Set up an argument parser and parse the command line arguments. """ parser = ArgumentParser() parser.add_argument('-b', '--branch', metavar='BRANCH', dest='branch', help='Specify the path to the branch you wish to build.') parser.add_argument('--devel', dest='devel', action='store_true', default=False, help='Development build does not have set icons for .dmg file ' 'and .dmg filename contains bzr revision number.') parser.add_argument('--release', dest='release', metavar='VERSION', help='Build a release version of OpenLP with the version specified') parser.add_argument('-d', '--documentation', metavar='DOCS', dest='docs', help='Specify the path to the documentation branch.') parser.add_argument('-c', '--config', metavar='CONFIG', dest='config', help='Specify the path to the configuration file.', default=os.path.abspath(os.path.join('.', 'config.ini.default'))) parser.add_argument('-u', '--skip-update', dest='skip_update', action='store_true', default=False, help='Do NOT update the branch before building.') parser.add_argument('-t', '--skip-translations', dest='skip_translations', action='store_true', default=False, help='Do NOT update the language translation files.') parser.add_argument('--transifex', dest='update_translations', action='store_true', default=False, help='Update the language translation from Transifex.') parser.add_argument('--transifex-user', dest='transifex_user', help='Transifex username.') parser.add_argument('--transifex-pass', dest='transifex_pass', help='Transifex password.') parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='Print out additional information.') self.args = parser.parse_args() def read_config(self): """ Read the configuration from the configuration file. """ self.config = ConfigParser(defaults={ 'here': self.script_path, 'projects': os.path.abspath(os.path.join(self.script_path, '..', '..')), }) self.config.read(os.path.abspath(self.args.config)) def setup_system_paths(self): """ Set up some system paths. """ self.script_path = os.path.dirname(os.path.abspath(__file__)) self.python = sys.executable def setup_executables(self): """ Set up the paths to the executables we use. """ self.sphinx = _which(self.config.get('executables', 'sphinx')) self.pyinstaller = os.path.abspath(self.config.get('executables', 'pyinstaller')) self.lrelease = self.config.get('executables', 'lrelease') self.dmgbuild = _which(self.config.get('executables', 'dmgbuild')) self.mudraw_bin = _which(self.config.get('executables', 'mudrawbin')) self.mutool_bin = _which(self.config.get('executables', 'mutoolbin')) def setup_paths(self): """ Set up a variety of paths that we use throughout the build process. """ if self.args.branch: self.branch_path = os.path.abspath(self.args.branch) else: self.branch_path = self.config.get('paths', 'branch') if self.args.docs: self.docs_path = os.path.abspath(self.args.docs) else: self.docs_path = self.config.get('paths', 'documentation') if self.args.release: self.version_number = self.args.release self.work_path = os.path.abspath(os.path.join(self.branch_path, '..', 'OpenLP-' + self.version_number)) else: self.version_number = None self.work_path = self.branch_path self.openlp_script = os.path.abspath(os.path.join(self.work_path, 'openlp.py')) self.hooks_path = os.path.abspath(os.path.join(self.work_path, self.config.get('paths', 'hooks'))) self.app_icon = os.path.abspath(self.config.get('paths', 'app_icon')) self.bundle_info = os.path.abspath(self.config.get('paths', 'bundle_info')) self.i18n_utils = os.path.join(self.work_path, 'scripts', 'translation_utils.py') self.source_path = os.path.join(self.work_path, 'openlp') self.manual_path = os.path.join(self.docs_path, 'manual') self.manual_build_path = os.path.join(self.manual_path, 'build') self.i18n_path = os.path.join(self.work_path, 'resources', 'i18n') self.build_path = os.path.join(self.work_path, 'build') self.dist_app_path = os.path.join(self.work_path, 'dist', 'OpenLP.app') self.dist_path = os.path.join(self.work_path, 'dist', 'OpenLP.app', 'Contents', 'MacOS') self.dmg_settings = os.path.abspath(self.config.get('paths', 'dmg_settings')) # Path to Qt translation files. from PyQt5.QtCore import QCoreApplication qt_plug_dir = str(list(QCoreApplication.libraryPaths())[0]) self.qt_translations_path = os.path.join(os.path.dirname(qt_plug_dir), 'translations') def update_code(self): """ Update the code in the branch. """ os.chdir(self.branch_path) self._print('Reverting any changes to the code...') bzr = Popen(('bzr', 'revert'), stdout=PIPE) output = bzr.communicate()[0] code = bzr.wait() if code != 0: self._print(output) raise Exception('Error reverting the code') self._print('Updating the code...') bzr = Popen(('bzr', 'update'), stdout=PIPE) output = bzr.communicate()[0] code = bzr.wait() if code != 0: self._print(output) raise Exception('Error updating the code') def export_release(self): """ Export a particular release """ if os.path.exists(self.work_path): rmtree(self.work_path) os.chdir(self.branch_path) self._print('Exporting the release version...') bzr = Popen(('bzr', 'export', '-r', 'tag:' + self.version_number, self.work_path), stdout=PIPE) output = bzr.communicate()[0] code = bzr.wait() if code != 0: self._print(output) raise Exception('Error exporting the code') def run_pyinstaller(self): """ Run PyInstaller on the branch to build an executable. """ self._print('Running PyInstaller...') os.chdir(self.work_path) cmd = [self.python, self.pyinstaller, '--clean', '--noconfirm', '--windowed', '--noupx', '--additional-hooks-dir', self.hooks_path, '--runtime-hook', os.path.join(self.hooks_path, 'rthook_ssl.py'), '-i', self.app_icon, '-n', 'OpenLP', self.openlp_script] if not self.args.verbose: cmd.append('--log-level=ERROR') else: cmd.append('--log-level=DEBUG') if self.args.devel: cmd.append('-d') pyinstaller = Popen(cmd) code = pyinstaller.wait() if code != 0: raise Exception('Error running PyInstaller') def write_version_file(self): """ Write the version number to a file for reading once installed. """ self._print('Writing version file...') os.chdir(self.branch_path) bzr = Popen(('bzr', 'tags'), stdout=PIPE) output = bzr.communicate()[0] code = bzr.wait() if code != 0: raise Exception('Error running bzr tags') lines = output.splitlines() if len(lines) == 0: tag = '0.0.0' revision = '0' else: tag, revision = lines[-1].decode('utf-8').split() bzr = Popen(('bzr', 'log', '--line', '-r', '-1'), stdout=PIPE) output, error = bzr.communicate() code = bzr.wait() if code != 0: raise Exception('Error running bzr log') latest = output.decode('utf-8').split(':')[0] self.version_string = '%s-bzr%s' % (tag, latest) self.version_tag = tag version_file = open(os.path.join(self.dist_path, '.version'), 'w') # Release version does not contain revision in .dmg name. if self.args.devel: version_file.write(str(self.version_string)) else: version_file.write(str(self.version_tag)) version_file.close() def copy_default_theme(self): """ Copy the default theme to the correct directory for OpenLP. """ self._print('Copying default theme...') source = os.path.join(self.source_path, 'core', 'lib', 'json') dest = os.path.join(self.dist_path, 'core', 'lib', 'json') for root, dirs, files in os.walk(source): for filename in files: if filename.endswith('.json'): dest_path = os.path.join(dest, root[len(source) + 1:]) if not os.path.exists(dest_path): os.makedirs(dest_path) self._print_verbose('... %s', filename) copy(os.path.join(root, filename), os.path.join(dest_path, filename)) def copy_plugins(self): """ Copy all the plugins to the correct directory so that OpenLP sees that it has plugins. """ self._print('Copying plugins...') source = os.path.join(self.source_path, 'plugins') dest = os.path.join(self.dist_path, 'plugins') for root, dirs, files in os.walk(source): for filename in files: if not filename.endswith('.pyc'): dest_path = os.path.join(dest, root[len(source) + 1:]) if not os.path.exists(dest_path): os.makedirs(dest_path) self._print_verbose('... %s', filename) copy(os.path.join(root, filename), os.path.join(dest_path, filename)) def copy_media_player(self): """ Copy the media players to the correct directory for OpenLP. """ self._print('Copying media player...') source = os.path.join(self.source_path, 'core', 'ui', 'media') dest = os.path.join(self.dist_path, 'core', 'ui', 'media') for root, dirs, files in os.walk(source): for filename in files: if not filename.endswith('.pyc'): dest_path = os.path.join(dest, root[len(source) + 1:]) if not os.path.exists(dest_path): os.makedirs(dest_path) self._print_verbose('... %s', filename) copy(os.path.join(root, filename), os.path.join(dest_path, filename)) def copy_mac_bundle_files(self): """ Copy Info.plist and OpenLP.icns to app bundle. """ copy(self.app_icon, os.path.join(self.dist_app_path, 'Contents', 'Resources', os.path.basename(self.app_icon))) # Add OpenLP version to Info.plist and put it to app bundle. fr = open(self.bundle_info, 'r') fw = open(os.path.join(self.dist_app_path, 'Contents', os.path.basename(self.bundle_info)), 'w') text = fr.read() if self.args.devel: text = text % {'openlp_version': self.version_string} else: text = text % {'openlp_version': self.version_tag} fw.write(text) fr.close() fw.close() def copy_macosx_files(self): """ Copy all the OSX-specific files. """ self._print('Copying extra files for Mac OS X...') self._print_verbose('... LICENSE.txt') copy(os.path.join(self.script_path, 'LICENSE.txt'), os.path.join(self.dist_path, 'LICENSE.txt')) self._print_verbose('... mudraw') if self.mudraw_bin and os.path.isfile(self.mudraw_bin): copy(os.path.join(self.mudraw_bin), os.path.join(self.dist_path, 'mudraw')) self.relink_mudraw() elif self.mutool_bin and os.path.isfile(self.mutool_bin): copy(os.path.join(self.mutool_bin), os.path.join(self.dist_path, 'mutool')) self.relink_mutool() else: self._print('... WARNING: mudraw and mutool not found') def relink_mudraw(self): """ Relink mudraw to bundled libraries """ self.relink_mupdf('mudraw') def relink_mutool(self): """ Relink mudraw to bundled libraries """ self.relink_mupdf('mutool') def relink_mupdf(self, bin_name): """ Relink mupdf to bundled libraries """ self._print('Linking {bin_name} with bundled libraries...'.format(bin_name=bin_name)) libname = os.path.join(self.dist_path, bin_name) distname = os.path.relpath(self.dist_path, libname) self._print_verbose('... {bin_name} path {path}'.format(bin_name=bin_name, path=libname)) # Determine how many directories up is the directory with shared # dynamic libraries. '../' # E.g. ./qt4_plugins/images/ -> ./../../ parent_dir = '' # Check if distname is not only base filename. if os.path.dirname(distname): parent_level = len(os.path.dirname(distname).split(os.sep)) parent_dir = parent_level * (os.pardir + os.sep) def match_func(pth): """ For system libraries leave path unchanged. """ # Match non system dynamic libraries. if not in_system_path(pth): # Use relative path to dependend dynamic libraries bases on # location of the executable. pth = os.path.join('@loader_path', parent_dir, os.path.basename(pth)) self._print_verbose('... %s', pth) return pth # Rewrite mach headers with @loader_path. dll = MachO(libname) dll.rewriteLoadCommands(match_func) # Write changes into file. # Write code is based on macholib example. try: self._print_verbose('... writing new library paths') f = open(dll.filename, 'rb+') for header in dll.headers: f.seek(0) dll.write(f) f.seek(0, 2) f.flush() f.close() except Exception: pass def update_translations(self): """ Update the translations. """ self._print('Updating translations...') if not self.config.has_section('transifex'): raise Exception('No section named "transifex" found.') if not self.config.has_option('transifex', 'username'): raise Exception('No option named "username" found.') if not self.config.has_option('transifex', 'password'): raise Exception('No option named "password" found.') if self.args.transifex_user: username = self.args.transifex_user else: username = self.config.get('transifex', 'username') if self.args.transifex_pass: password = self.args.transifex_pass else: password = self.config.get('transifex', 'password') os.chdir(os.path.split(self.i18n_utils)[0]) translation_utils = Popen([self.python, self.i18n_utils, '-qdpu', '-U', username, '-P', password]) code = translation_utils.wait() if code != 0: raise Exception('Error running translation_utils.py') def compile_translations(self): """ Compile the translations for Qt. """ self._print('Compiling translations...') files = os.listdir(self.i18n_path) if not os.path.exists(os.path.join(self.dist_path, 'i18n')): os.makedirs(os.path.join(self.dist_path, 'i18n')) for file in files: if file.endswith('.ts'): self._print_verbose('... %s', file) source_path = os.path.join(self.i18n_path, file) dest_path = os.path.join(self.dist_path, 'i18n', file.replace('.ts', '.qm')) lconvert = Popen((self.lrelease, '-compress', '-silent', source_path, '-qm', dest_path)) code = lconvert.wait() if code != 0: raise Exception('Error running lconvert on %s' % source_path) self._print('Copying qm files...') source = self.qt_translations_path files = os.listdir(source) for filename in files: if filename.startswith('qt_') and filename.endswith('.qm'): self._print_verbose('... %s', filename) copy(os.path.join(source, filename), os.path.join(self.dist_path, 'i18n', filename)) def run_sphinx(self): """ Run Sphinx to build an HTML Help project. """ self._print('Deleting previous manual build... %s', self.manual_build_path) if os.path.exists(self.manual_build_path): rmtree(self.manual_build_path) self._print('Running Sphinx...') os.chdir(self.manual_path) sphinx = Popen((self.sphinx, '-b', 'applehelp', '-d', 'build/doctrees', 'source', 'build/applehelp'), stdout=PIPE) output, error = sphinx.communicate() code = sphinx.wait() if code != 0: self._print(output) raise Exception('Error running Sphinx') self._print('Copying help file...') source = os.path.join(self.manual_build_path, 'applehelp') files = os.listdir(source) for filename in files: if filename.endswith('.help'): self._print_verbose('... %s', filename) copytree(os.path.join(source, filename), os.path.join(self.dist_app_path, 'Contents', 'Resources', filename)) def code_sign(self): certificate = self.config.get('codesigning', 'certificate') self._print('Checking for certificate...') security = Popen(('security', 'find-certificate', '-c', certificate), stdout=PIPE) output, error = security.communicate() code = security.wait() if code != 0: self._print('Could not find certificate \"%s\" in Keychain...', certificate) self._print('Codesigning will not work without a certificate!!') self._print(output) else: self._print('Codesigning app...') codesign = Popen(('codesign', '--deep', '-s', certificate, self.dist_app_path)) output, error = codesign.communicate() code = codesign.wait() if code != 0: self._print(output) raise Exception('Error running codesign') def create_dmg_file(self): """ Create .dmg file. """ self._print('Creating dmg file...') # Release version does not contain revision in .dmg name. if self.args.devel: dmg_name = 'OpenLP-' + str(self.version_string) + '.dmg' dmg_title = 'OpenLP {version}'.format(version=self.version_string) else: dmg_name = 'OpenLP-' + str(self.version_tag) + '.dmg' dmg_title = 'OpenLP {version}'.format(version=self.version_tag) self.dmg_file = os.path.join(self.work_path, 'dist', dmg_name) # Remove dmg if it exists. if os.path.exists(self.dmg_file): os.remove(self.dmg_file) # Create empty dmg file. size = self._get_directory_size(self.dist_app_path) # in bytes. size = size / (1000 * 1000) # Convert to megabytes. size += 10 # Additional space in .dmg for other files. self._print('... %s' % self.script_path) os.chdir(self.script_path) self._run_command([self.dmgbuild, '-s', self.dmg_settings, '-D', 'size={size}M'.format(size=size), '-D', 'icon={icon_path}'.format(icon_path=self.app_icon), '-D', 'app={dist_app_path}'.format(dist_app_path=self.dist_app_path), dmg_title, self.dmg_file], 'Unable to run dmgbuild') # Jenkins integration. # Continuous integration server needs to know the filename of dmg. # Write java property file. For uploading dmg to openlp. if self.args.devel: fpath = os.path.join(self.branch_path, 'openlp.properties') self._print('... writing property file for jenkins: %s' % fpath) f = open(fpath, 'w') f.write('OPENLP_DMGNAME=' + os.path.basename(self.dmg_file) + '\n') f.close() # Dmg done. self._print('Finished creating dmg file, resulting file: %s' % self.dmg_file) def main(self): """ The main function to run the Mac OS X builder. """ self._print_verbose('OpenLP main script: ......%s', self.openlp_script) self._print_verbose('Script path: .............%s', self.script_path) self._print_verbose('Branch path: .............%s', self.branch_path) self._print_verbose('Source path: .............%s', self.source_path) self._print_verbose('"dist.app" path: .........%s', self.dist_app_path) self._print_verbose('"dist" path: .............%s', self.dist_path) self._print_verbose('"hooks" path: ............%s', self.hooks_path) self._print_verbose('PyInstaller: .............%s', self.pyinstaller) self._print_verbose('dmgbuild: ................%s', self.dmgbuild) self._print_verbose('Documentation branch path:%s', self.docs_path) if self.mudraw_bin: self._print_verbose('mudraw binary ............%s', self.mudraw_bin) elif self.mutool_bin: self._print_verbose('mutool binary ............%s', self.mutool_bin) else: self._print_verbose('mutool/mudraw ............Not found') self._print_verbose('') if not self.args.skip_update: self.update_code() if self.args.release: self.export_release() self.run_pyinstaller() self.write_version_file() self.copy_mac_bundle_files() self.copy_default_theme() self.copy_plugins() self.copy_media_player() # TODO creating help on Mac if os.path.exists(self.manual_path): self.run_sphinx() else: self._print('') self._print('WARNING: Documentation trunk not found. Mac OS X') self._print(' Help file will not be included in build') self._print('') self.copy_macosx_files() if not self.args.skip_translations: if self.args.update_translations: self.update_translations() self.compile_translations() self.code_sign() self.create_dmg_file() self._print('Done.') raise SystemExit() if __name__ == '__main__': MacosxBuilder().main()