packaging/osx/macosx-builder.py

# -*- coding: utf-8 -*-
# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4

###############################################################################
# OpenLP - Open Source Lyrics Projection                                      #
# --------------------------------------------------------------------------- #
# Copyright (c) 2008-2011 Raoul Snyman                                        #
# Portions copyright (c) 2008-2011 Tim Bentley, Jonathan Corwin, Michael      #
# Gorven, Scott Guerrieri, Matthias Hub, Meinert Jordan, Armin Köhler,        #
# Andreas Preikschat, Mattias Põldaru, Christian Richter, Philip Ridout,      #
# Jeffrey Smith, Maikel Stuivenberg, Martin Thompson, Jon Tibble, Frode       #
# Woldsund                                                                    #
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it     #
# under the terms of the GNU General Public License as published by the Free  #
# Software Foundation; version 2 of the License.                              #
#                                                                             #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
# more details.                                                               #
#                                                                             #
# You should have received a copy of the GNU General Public License along     #
# with this program; if not, write to the Free Software Foundation, Inc., 59  #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
###############################################################################

"""
Mac OS X Build Script
--------------------

This script is used to build the Mac OS X app bundle and pack it into dmg file.
For this script to work out of the box, it depends on a number of things:

Python 2.6/2.7

PyQt4
    You should already have this installed, OpenLP doesn't work without it. The
    version the script expects is the packaged one available from River Bank
    Computing.

PyEnchant
    This script expects the precompiled, installable version of PyEnchant to be
    installed. You can find this on the PyEnchant site.

Sphinx
    This is used to build the documentation.  The documentation trunk must be at
    the same directory level as OpenLP trunk and named "documentation".

PyInstaller
    PyInstaller should be a checkout of revision 1355 of trunk, and in a
    directory  which is configured in the openlp.cfg. The revision is very
    important as there is just included a fix for builds on OS X.

    To install PyInstaller, first checkout trunk from Subversion. The
    easiest way is to do a

        svn co http://svn.pyinstaller.org/trunk

    Then you need to copy the two hook-*.py files from the "pyinstaller"
    subdirectory in OpenLP's "resources" directory into PyInstaller's
    "hooks" directory.

Bazaar
    You need the command line "bzr" client installed.

OpenLP
    A checkout of the latest code, in a branch directory, which is in a Bazaar
    shared repository directory. This means your code should be in a directory
    structure like this: "openlp\branch-name".

macosx-builder.py
    This script, of course. It should be in the "osx-package" directory
    at the same level as OpenLP trunk.

Mako
    Mako Templates for Python.  This package is required for building the
    remote plugin.  It can be installed by going to your
    python_directory\scripts\.. and running "easy_install Mako".  If you do not
    have easy_install, the Mako package can be obtained here:

        http://www.makotemplates.org/download.html

SQLAlchemy Migrate
    Required for the databases used in OpenLP.  The package can be
    obtained here:

        http://code.google.com/p/sqlalchemy-migrate/


config.ini.default
    The configuration file contains settings of the version string to include
    in the bundle as well as directory and file settings for different
    purposes (e.g. PyInstaller location or installer background image)

To start the build process do a

   make

The result should be a {openlp_dmgname}.dmg
file in the same directory. If something went wrong - this sometimes happen
with the graphical commands in the Apple script - do a

   make clean

and start the build process again. If you want to execute only parts of the
build process you can specify different make targets

    make view -- runs the Apple scripts to set the icons
    make package -- creates the dmg file and copies the application files
    make bundle -- compresses the dmg file and sets the dmg file icon
"""

import os
import plistlib
import sys
from shutil import copy, rmtree
from subprocess import Popen, PIPE
from ConfigParser import SafeConfigParser as ConfigParser
from argparse import ArgumentParser


def _which(command):
    """
    Return absolute path to a command found on system PATH.
    """
    for path in os.environ["PATH"].split(os.pathsep):
        if os.access(os.path.join(path, command), os.X_OK):
            print "%s/%s" % (path, command)
            return "%s/%s" % (path, command)


class MacosxBuilder(object):
    """
    The :class:`MacosxBuilder` class encapsulates everything that is needed
    to build a Mac OS X .dmg file.
    """
    def __init__(self):
        self.setup_args()
        self.setup_system_paths()
        self.read_config()
        self.setup_executables()
        self.setup_paths()

    def _print(self, text, *args):
        """
        Print stuff out. Later we might want to use a log file.
        """
        if len(args) > 0:
            text = text % tuple(args)
        print text

    def _print_verbose(self, text, *args):
        """
        Print output, obeying "verbose" mode.
        """
        if self.args.verbose:
            self._print(text, *args)

    def _run_command(self, cmd, err_msg):
        """
        Run command in subprocess and print error message in case of Exception.

        Return text from stdout.
        """
        proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
        output, error = proc.communicate()
        code = proc.wait()
        if code != 0:
            self._print(output)
            self._print(error)
            raise Exception(err_msg)
        return output
 
    def _get_directory_size(self, directory):
        """
        Return directory size - size of everything in the dir.
        """
        dir_size = 0
        for (path, dirs, files) in os.walk(directory):
            for file in files:
                filename = os.path.join(path, file)
                dir_size += os.path.getsize(filename)
        return dir_size
     
    def _get_mountpoints(self):
        """
        Return list of mounted disks on Mac.
        """
        # Get the output in plist format.
        paths = []
        output = self._run_command([self.hdiutil, 'info', '-plist'],
                u'Detecting mount points failed.')
        pl = plistlib.readPlistFromString(output)
        for image in pl['images']:
            for se in image['system-entities']:
                if se.get('mount-point'):
                    paths.append(se.get('mount-point'))

        return paths

    def setup_args(self):
        """
        Set up an argument parser and parse the command line arguments.
        """
        parser = ArgumentParser()
        parser.add_argument('-b', '--branch', metavar='BRANCH', dest='branch',
            help='Specify the path to the branch you wish to build.')
        parser.add_argument('--devel', dest='devel',
            action='store_true', default=False,
            help='Development build does not have set icons for .dmg file '
                'and .dmg filename contains bzr revision number.')
        parser.add_argument('-d', '--documentation', metavar='DOCS',
            dest='docs',
            help='Specify the path to the documentation branch.')
        parser.add_argument('-c', '--config', metavar='CONFIG', dest='config',
            help='Specify the path to the configuration file.',
            default=os.path.abspath(os.path.join('.', 'config.ini.default')))
        parser.add_argument('-u', '--skip-update', dest='skip_update',
            action='store_true', default=False,
            help='Do NOT update the branch before building.')
        parser.add_argument('-t', '--skip-translations',
            dest='skip_translations', action='store_true', default=False,
            help='Do NOT update the language translation files.')
        parser.add_argument('--transifex',
            dest='update_translations', action='store_true', default=False,
            help='Update the language translation from Transifex.')
        parser.add_argument('--transifex-user',
            dest='transifex_user', help='Transifex username.')
        parser.add_argument('--transifex-pass',
            dest='transifex_pass', help='Transifex password.')
        parser.add_argument('-v', '--verbose', dest='verbose',
            action='store_true', default=False,
            help='Print out additional information.')
        self.args = parser.parse_args()

    def read_config(self):
        """
        Read the configuration from the configuration file.
        """
        self.config = ConfigParser(defaults={
            u'here': self.script_path,
            u'projects': os.path.abspath(os.path.join(self.script_path,
                '..', '..')),
        })
        self.config.read(os.path.abspath(self.args.config))

    def setup_system_paths(self):
        """
        Set up some system paths.
        """
        self.script_path = os.path.dirname(os.path.abspath(__file__))
        self.python = sys.executable

    def setup_executables(self):
        """
        Set up the paths to the executables we use.
        """
        self.sphinx = _which(self.config.get(u'executables', u'sphinx'))
        self.pyinstaller = os.path.abspath(
            self.config.get(u'executables', u'pyinstaller'))
        self.lrelease = _which(self.config.get(u'executables', u'lrelease'))
        self.diskutil = _which(self.config.get(u'executables', u'diskutil'))
        self.hdiutil = _which(self.config.get(u'executables', u'hdiutil'))
        self.osascript = _which(self.config.get(u'executables', u'osascript'))
        
    def setup_paths(self):
        """
        Set up a variety of paths that we use throughout the build process.
        """
        if self.args.branch:
            self.branch_path = os.path.abspath(self.args.branch)
        else:
            self.branch_path = self.config.get(u'paths', u'branch')
        if self.args.docs:
            self.docs_path = os.path.abspath(self.args.docs)
        else:
            self.docs_path = self.config.get(u'paths', u'documentation')

        self.openlp_script = os.path.abspath(
            os.path.join(self.branch_path, u'openlp.pyw'))
        self.hooks_path = os.path.abspath(os.path.join(
            self.branch_path, self.config.get(u'paths', u'hooks')))
        self.mac_icon = os.path.abspath(
            self.config.get(u'paths', u'macicon'))
        self.bundle_info = os.path.abspath(
            self.config.get(u'paths', u'bundleinfo'))
        self.dmg_background_img = os.path.abspath(
            self.config.get(u'paths', u'dmg_background'))
        self.i18n_utils = os.path.join(self.branch_path, u'scripts',
            u'translation_utils.py')
        self.source_path = os.path.join(self.branch_path, u'openlp')
        self.manual_path = os.path.join(self.docs_path, u'manual')
        self.manual_build_path = os.path.join(self.manual_path, u'build')
        self.i18n_path = os.path.join(self.branch_path, u'resources', u'i18n')
        self.build_path = os.path.join(self.branch_path, u'build')
        self.dist_app_path = os.path.join(self.branch_path, u'dist', u'OpenLP.app')
        self.dist_path = os.path.join(self.branch_path, u'dist', u'OpenLP.app',
                'Contents', 'MacOS')

        # Path to Qt translation files.
        from PyQt4.QtCore import QCoreApplication
        qt_plug_dir = str(list(QCoreApplication.libraryPaths())[0])
        self.qt_translat_path = os.path.join(os.path.dirname(qt_plug_dir),
            'translations')

    def update_code(self):
        """
        Update the code in the branch.
        """
        os.chdir(self.branch_path)
        self._print(u'Reverting any changes to the code...')
        bzr = Popen((u'bzr', u'revert'), stdout=PIPE)
        output = bzr.communicate()[0]
        code = bzr.wait()
        if code != 0:
            self._print(output)
            raise Exception(u'Error reverting the code')
        self._print(u'Updating the code...')
        bzr = Popen((u'bzr', u'update'), stdout=PIPE)
        output = bzr.communicate()[0]
        code = bzr.wait()
        if code != 0:
            self._print(output)
            raise Exception(u'Error updating the code')

    def run_pyinstaller(self):
        """
        Run PyInstaller on the branch to build an executable.
        """
        self._print(u'Running PyInstaller...')
        os.chdir(self.branch_path)
        # arch -i386 ensures 32bit build is created.
        pyinstaller = Popen(('arch', '-i386', self.python,
            self.pyinstaller,
            u'--noconfirm',
            u'--windowed',
            u'--noupx',
            u'--additional-hooks-dir', self.hooks_path,
            u'--log-level=ERROR',
            u'-o', self.branch_path,
            #u'-i', self.mac_icon,
            u'-p', self.branch_path,
            u'-n', u'OpenLP',
            self.openlp_script),
            stdout=PIPE)
        output = pyinstaller.communicate()[0]
        code = pyinstaller.wait()
        if code != 0:
            self._print(output)
            raise Exception(u'Error running PyInstaller')

    def write_version_file(self):
        """
        Write the version number to a file for reading once installed.
        """
        self._print(u'Writing version file...')
        os.chdir(self.branch_path)
        bzr = Popen((u'bzr', u'tags', u'--sort', u'time'), stdout=PIPE)
        output = bzr.communicate()[0]
        code = bzr.wait()
        if code != 0:
            raise Exception(u'Error running bzr tags')
        lines = output.splitlines()
        if len(lines) == 0:
            tag = u'0.0.0'
            revision = u'0'
        else:
            tag, revision = lines[-1].split()
        bzr = Popen((u'bzr', u'log', u'--line', u'-r', u'-1'), stdout=PIPE)
        output, error = bzr.communicate()
        code = bzr.wait()
        if code != 0:
            raise Exception(u'Error running bzr log')
        output_ascii = unicode(output, errors=u'ignore')
        latest = output_ascii.split(u':')[0]
        self.version_string = u'%s-bzr%s' % (tag, latest)
        self.version_tag = tag
        version_file = open(os.path.join(self.dist_path, u'.version'), u'w')
        # Release version does not contain revision in .dmg name.
        if self.args.devel:
            version_file.write(self.version_string)
        else:
            version_file.write(self.version_tag)
        version_file.close()

    def copy_plugins(self):
        """
        Copy all the plugins to the correct directory so that OpenLP sees that
        it has plugins.
        """
        self._print(u'Copying plugins...')
        source = os.path.join(self.source_path, u'plugins')
        dest = os.path.join(self.dist_path, u'plugins')
        for root, dirs, files in os.walk(source):
            for filename in files:
                if not filename.endswith(u'.pyc'):
                    dest_path = os.path.join(dest, root[len(source)+1:])
                    if not os.path.exists(dest_path):
                        os.makedirs(dest_path)
                    self._print_verbose(u'... %s', filename)
                    copy(os.path.join(root, filename),
                        os.path.join(dest_path, filename))

    def copy_media_player(self):
        """
        Copy the media players to the correct directory for OpenLP.
        """
        self._print(u'Copying media player...')
        source = os.path.join(self.source_path, u'core', u'ui', u'media')
        dest = os.path.join(self.dist_path, u'core', u'ui', u'media')
        for root, dirs, files in os.walk(source):
            for filename in files:
                if not filename.endswith(u'.pyc'):
                    dest_path = os.path.join(dest, root[len(source)+1:])
                    if not os.path.exists(dest_path):
                        os.makedirs(dest_path)
                    self._print_verbose(u'... %s', filename)
                    copy(os.path.join(root, filename),
                        os.path.join(dest_path, filename))

    def copy_mac_bundle_files(self):
        """
        Copy Info.plist and OpenLP.icns to app bundle.
        """
        copy(self.mac_icon, os.path.join(self.dist_app_path, 
            'Contents', 'Resources', os.path.basename(self.mac_icon)))
        # Add OpenLP version to Info.plist and put it to app bundle.
        fr = open(self.bundle_info, u'r')
        fw = open(os.path.join(self.dist_app_path, 
            'Contents', os.path.basename(self.bundle_info)), 'w')
        text = fr.read()
        text = text % {'openlp_version': self.version_string}
        fw.write(text)
        
        fr.close()
        fw.close()

    def copy_macosx_files(self):
        """
        Copy all the OSX-specific files.
        """
        self._print(u'Copying extra files for Mac OS X...')
        self._print_verbose(u'... LICENSE.txt')
        copy(os.path.join(self.script_path, u'LICENSE.txt'),
            os.path.join(self.dist_path, u'LICENSE.txt'))

    def update_translations(self):
        """
        Update the translations.
        """
        self._print(u'Updating translations...')
        if not self.config.has_section('transifex'):
            raise Exception(u'No section named "transifex" found.')
        if not self.config.has_option('transifex', 'username'):
            raise Exception(u'No option named "username" found.')
        if not self.config.has_option('transifex', 'password'):
            raise Exception(u'No option named "password" found.')
        if self.args.transifex_user:
            username = self.args.transifex_user
        else:
            username = self.config.get(u'transifex', u'username')
        if self.args.transifex_pass:
            password = self.args.transifex_pass
        else:
            password = self.config.get(u'transifex', u'password')
        os.chdir(os.path.split(self.i18n_utils)[0])
        translation_utils = Popen([self.python, self.i18n_utils, u'-qdpu',
            u'-U', username, u'-P', password])
        code = translation_utils.wait()
        if code != 0:
            raise Exception(u'Error running translation_utils.py')

    def compile_translations(self):
        """
        Compile the translations for Qt.
        """
        self._print(u'Compiling translations...')
        files = os.listdir(self.i18n_path)
        if not os.path.exists(os.path.join(self.dist_path, u'i18n')):
            os.makedirs(os.path.join(self.dist_path, u'i18n'))
        for file in files:
            if file.endswith(u'.ts'):
                self._print_verbose(u'... %s', file)
                source_path = os.path.join(self.i18n_path, file)
                dest_path = os.path.join(self.dist_path, u'i18n',
                    file.replace(u'.ts', u'.qm'))
                lconvert = Popen((self.lrelease, u'-compress', u'-silent',
                    source_path, u'-qm', dest_path))
                code = lconvert.wait()
                if code != 0:
                    raise Exception(u'Error running lconvert on %s' % \
                        source_path)
        self._print(u'Copying qm files...')
        source = self.qt_translat_path
        files = os.listdir(source)
        for filename in files:
            if filename.startswith(u'qt_') and filename.endswith(u'.qm') and \
                len(filename) == 8:
                self._print_verbose(u'... %s', filename)
                copy(os.path.join(source, filename),
                    os.path.join(self.dist_path, u'i18n', filename))

    def run_sphinx(self):
        """
        Run Sphinx to build an HTML Help project.
        """
        self._print(u'Deleting previous manual build... %s',
            self.manual_build_path)
        if os.path.exists(self.manual_build_path):
            rmtree(self.manual_build_path)
        self._print(u'Running Sphinx...')
        os.chdir(self.manual_path)
        sphinx = Popen((self.sphinx, u'-b', u'htmlhelp', u'-d',
            u'build/doctrees', u'source', u'build/htmlhelp'), stdout=PIPE)
        output, error = sphinx.communicate()
        code = sphinx.wait()
        if code != 0:
            self._print(output)
            raise Exception(u'Error running Sphinx')

    def create_dmg_file(self):
        """
        Create .dmg file.
        """
        self._print(u'Creating dmg file...')

        # Release version does not contain revision in .dmg name.
        if self.args.devel:
            dmg_name = 'OpenLP-' + self.version_string + '.dmg'
        else:
            dmg_name = 'OpenLP-' + self.version_tag + '.dmg'

        dmg_file = os.path.join(self.branch_path, 'build', dmg_name)
        # Remove dmg if it exists.
        if os.path.exists(dmg_file):
            os.remove(dmg_file)
        # Create empty dmg file. 
        size = self._get_directory_size(self.dist_app_path)  # in bytes.
        size = size / (1024 * 1024)  # Convert to megabytes.
        size += 10  # Additional space in .dmg for other files.
        self._print(u'... dmg disk size: %s' % size)
        self._run_command([self.hdiutil, 'create', dmg_file, 
            '-ov', '-megabytes', str(size), 
            '-fs', 'HFS+', '-volname', 'OpenLP'],
            u'Could not create dmg file.'
        )

        # Mount empty dmg file.
        old_mounts = self._get_mountpoints()
        self._print(u'... mounting the dmg file: %s' % dmg_file)
        self._run_command([self.hdiutil, 'attach', dmg_file],
            u'Could not mount dmg file, cannot continue.'
        )
        new_mounts = self._get_mountpoints()
        # Get the mount point from difference between paths
        # after mounting and before mounting the dmg file.
        dmg_volume_path = list(set(new_mounts) - set(old_mounts))[0]

        # Copy OpenLP.app and other files to .dmg
        # TODO more reliable way to determine dmg_volume_path
        self._print(u'... Copying the app to the dmg: ' + dmg_volume_path)
        self._run_command(['cp', '-r', self.dist_app_path,
            dmg_volume_path],
            u'Could not copy app bundle, dmg creation failed.'
        )

        # Set icon for dmg file.
        # http://endrift.com/blog/2010/06/14/dmg-files-volume-icons-cli/
        self._print('... Setting the dmg icon.')
        dmg_icon = os.path.join(dmg_volume_path, '.VolumeIcon.icns')
        self._run_command(['cp', self.mac_icon, dmg_icon],
            u'Could not copy the dmg icon file, dmg creation failed.'
        )
        # Set proper dmg icon attributes.
        self._run_command(['SetFile', '-c', 'icnC', dmg_icon],
                'Could not set dmg icon attributes.')
        # Ensures dmg icon will be used while mounted.
        self._run_command(['SetFile', '-a', 'C', dmg_volume_path],
                'Could not set dmg icon attributes.')

        # Set dmg background. Requires running Mac OS X gui.
        # TODO: better formatting and code refactoring
        if not self.args.devel:
            self._print('... Setting the background image.')

            os.mkdir(os.path.join(dmg_volume_path, '.background'))
            self._run_command(['cp', self.dmg_background_img,
                os.path.join(dmg_volume_path,
                '.background/installer-background.png')],
                u'Could not copy the background image, dmg creation failed.'
            )

            self.adjust_dmg_view(os.path.basename(dmg_volume_path))

        # Unmount dmg file.
        self._print('... unmounting the dmg.')
        self._run_command([self.hdiutil, 'detach', dmg_volume_path],
            'Could not unmount the dmg file, dmg creation failed.'
        )

        # Compress dmg file.
        self._print('... compressing the dmg file')
        compressed_dmg = os.path.join(self.branch_path, 'dist',
            os.path.basename(dmg_file))  # Put dmg to 'dist' dir.
        # Remove dmg if it exists.
        if os.path.exists(compressed_dmg):
            os.remove(compressed_dmg)
        self._run_command([self.hdiutil, 
            'convert', dmg_file, '-format', 'UDZO',
            '-imagekey', 'zlib-level=9', '-o', compressed_dmg],
            u'Could not compress the dmg file, dmg creation failed.'
        )

        # Jenkins integration. 
        # Continuous integration server needs to know the filename of dmg.
        # Write java property file. For uploading dmg to openlp.
        if self.args.devel:
            fpath = os.path.join(self.branch_path, 'openlp.properties')
            self._print('... writing property file for jenkins: %s' %
                fpath)
            f = open(fpath, 'w')
            f.write('OPENLP_DMGNAME=' + os.path.basename(dmg_file) + '\n')
            f.close()

        # Dmg done.
        self._print('Finished creating dmg file, resulting file: %s' %
            compressed_dmg)

        self.dmg_file = compressed_dmg

    def adjust_dmg_view(self, dmg_volume_name):
        try:
            # TODO: Use only one applescript file. Remove one for osx 10.5.
            f = open(os.path.join(self.script_path,
                'applescript-adjust-dmg-view.master'))
            p = Popen([self.osascript], stdin=PIPE)
            p.communicate(f.read() % (dmg_volume_name, 'OpenLP'))
            f.close()
            result = p.returncode
            if (result != 0):
                self._print('Adjusting dmg view failed.')
                sys.exit(1)
        except (IOError, OSError):
            self._print('Adjusting dmg view failed.')
            sys.exit(1)

    def main(self):
        """
        The main function to run the Mac OS X builder.
        """
        self._print_verbose(u'OpenLP main script: ......%s',
            self.openlp_script)
        self._print_verbose(u'Script path: .............%s',
            os.path.split(os.path.abspath(__file__))[0])
        self._print_verbose(u'Branch path: .............%s', self.branch_path)
        self._print_verbose(u'Source path: .............%s', self.source_path)
        self._print_verbose(u'"dist.app" path: .........%s', self.dist_app_path)
        self._print_verbose(u'"dist" path: .............%s', self.dist_path)
        self._print_verbose(u'"hooks" path: ............%s', self.hooks_path)
        self._print_verbose(u'PyInstaller: .............%s', self.pyinstaller)
        self._print_verbose(u'Documentation branch path:%s', self.docs_path)
        self._print_verbose(u'')
        if not self.args.skip_update:
            self.update_code()
        self.run_pyinstaller()
        self.write_version_file()
        self.copy_mac_bundle_files()
        self.copy_plugins()
        self.copy_media_player()
        # TODO creating help on Mac
        if os.path.exists(self.manual_path):
            self.run_sphinx()
        else:
            self._print(u'')
            self._print(u'WARNING: Documentation trunk not found. Mac OS X')
            self._print(u'         Help file will not be included in build')
            self._print(u'')
        self.copy_macosx_files()
        if not self.args.skip_translations:
            if self.args.update_translations:
                self.update_translations()
            self.compile_translations()
        self.create_dmg_file()

        self._print(u'Done.')
        raise SystemExit()


if __name__ == u'__main__':
    MacosxBuilder().main()