packaging/builders/builder.py

# -*- coding: utf-8 -*-
# vim: autoindent shiftwidth=4 expandtab textwidth=80 tabstop=4 softtabstop=4

###############################################################################
# OpenLP - Open Source Lyrics Projection                                      #
# --------------------------------------------------------------------------- #
# Copyright (c) 2004-2016 OpenLP Developers                                   #
# --------------------------------------------------------------------------- #
# This program is free software; you can redistribute it and/or modify it     #
# under the terms of the GNU General Public License as published by the Free  #
# Software Foundation; version 2 of the License.                              #
#                                                                             #
# This program is distributed in the hope that it will be useful, but WITHOUT #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or       #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for    #
# more details.                                                               #
#                                                                             #
# You should have received a copy of the GNU General Public License along     #
# with this program; if not, write to the Free Software Foundation, Inc., 59  #
# Temple Place, Suite 330, Boston, MA 02111-1307 USA                          #
###############################################################################
"""
Base class for the Windows and macOS builders.
"""
import os
import sys
from argparse import ArgumentParser
from configparser import ConfigParser
from shutil import copy, rmtree
from subprocess import Popen, PIPE

BUILDER_DESCRIPTION = 'Build OpenLP for {platform}. Options are provided on both the command line and a ' \
    'configuration file. Options in the configuration file are overridden by the command line options.\n\n' \
    'This build system can produce either development or release builds. A development release uses the ' \
    'code as-is in the specified branch directory. The release build exports a tag from bzr and uses the ' \
    'exported code for building. The two modes are invoked by the presence or absence of the --release ' \
    'option. If this option is omitted, a development build is built, while including the --release ' \
    'option with a version number will produce a build of that exact version.'


def _which(program):
    """
    Return absolute path to a command found on system PATH.
    """
    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath and is_exe(os.path.abspath(program)):
        return os.path.abspath(program)
    else:
        for path in os.environ['PATH'].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file
    return None


class Builder(object):
    """
    A Generic class to base other operating system specific builders on
    """
    def __init__(self):
        self.setup_args()
        self.setup_system_paths()
        self.read_config()
        self.setup_paths()
        self.setup_executables()
        self.setup_extra()

    def _print(self, text, *args):
        """
        Print stuff out. Later we might want to use a log file.
        """
        if len(args) > 0:
            text = text % tuple(args)
        print(text)

    def _print_verbose(self, text, *args):
        """
        Print output, obeying "verbose" mode.
        """
        if self.args.verbose:
            self._print(text, *args)

    def _run_command(self, cmd, err_msg, exit_code=0):
        """
        Run command in subprocess and print error message in case of Exception.

        Return text from stdout.
        """
        proc = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
        output, error = proc.communicate()
        code = proc.wait()
        if code != exit_code:
            self._print(output)
            self._print(error)
            raise Exception(err_msg)
        return output, error

    def _bzr(self, command, work_path, args=[], err_msg='There was an error running bzr'):
        """
        Update the code in the branch.
        """
        os.chdir(work_path)
        output, _ = self._run_command(['bzr', command] + args, err_msg)
        return output

    def get_platform(self):
        """
        Return the platform we're building for
        """
        return 'unspecified'

    def get_config_defaults(self):
        """
        Build some default values for the config file
        """
        return {'here': os.path.dirname(self.config_path)}

    def get_sphinx_build(self):
        """
        Get the type of build we should be running for Sphinx. Defaults to html.
        """
        return 'html'

    def get_qt_translations_path(self):
        """
        Return the path to Qt's translation files
        """
        return ''

    def add_extra_args(self, parser):
        """
        Add extra arguments to the argument parser
        """
        pass

    def setup_args(self):
        """
        Set up an argument parser and parse the command line arguments.
        """
        parser = ArgumentParser(description=BUILDER_DESCRIPTION.format(platform=self.get_platform()))
        parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False,
                            help='Print out additional information')
        parser.add_argument('-c', '--config', metavar='FILENAME', required=True,
                            help='Specify the path to the configuration file')
        parser.add_argument('-b', '--branch', metavar='PATH', help='Specify the path to the branch you wish to build')
        parser.add_argument('-r', '--release', metavar='VERSION', default=None,
                            help='Build a release version of OpenLP with the version specified')
        parser.add_argument('-d', '--documentation', metavar='PATH', default=None,
                            help='Specify the path to the documentation branch')
        parser.add_argument('-t', '--update-translations', action='store_true', default=False,
                            help='Update the translations from Transifex')
        parser.add_argument('-u', '--transifex-user', metavar='USERNAME', default=None, help='Transifex username')
        parser.add_argument('-p', '--transifex-pass', metavar='PASSWORD', default=None, help='Transifex password')
        parser.add_argument('--skip-update', action='store_true', default=False,
                            help='Do NOT update the branch before building')
        parser.add_argument('--skip-translations', action='store_true', default=False,
                            help='Do NOT update the language translation files')
        parser.add_argument('--debug', action='store_true', default=False, help='Create a debug build')
        parser.add_argument('--tag-override', metavar='<tag>-bzr<revision>', default=None,
                            help='Override tag and revision, should be in format <tag>-bzr<revision>')
        self.add_extra_args(parser)
        self.args = parser.parse_args()

    def read_config(self):
        """
        Read the configuration from the configuration file.
        """
        self.config = ConfigParser(defaults=self.get_config_defaults())
        self.config.read(self.config_path)

    def setup_system_paths(self):
        """
        Set up some system paths.
        """
        self.python = sys.executable
        self.script_path = os.path.dirname(os.path.abspath(__file__))
        self.config_path = os.path.abspath(self.args.config)
        self._print_verbose('System paths:')
        self._print_verbose('   {:.<20}: {}'.format('python: ', self.python))
        self._print_verbose('   {:.<20}: {}'.format('script: ', self.script_path))
        self._print_verbose('   {:.<20}: {}'.format('config: ', self.config_path))

    def setup_executables(self):
        """
        Set up the paths to the executables we use.
        """
        self._print_verbose('Executables:')
        for executable in self.config.options('executables'):
            path = self.config.get('executables', executable)
            if not path.strip():
                path = None
            else:
                path = _which(path)
            setattr(self, '{exe}_exe'.format(exe=executable), path)
            self._print_verbose('   {exe:.<20} {path}'.format(exe=executable + ': ', path=path))

    def setup_paths(self):
        """
        Set up a variety of paths that we use throughout the build process.
        """
        self._print_verbose('Paths:')
        for name in self.config.options('paths'):
            path = os.path.abspath(self.config.get('paths', name))
            setattr(self, '{name}_path'.format(name=name), path)
            self._print_verbose('   {name:.<20} {path}'.format(name=name + ': ', path=path))
        # Make any command line options override the config file
        if self.args.branch:
            self.branch_path = os.path.abspath(self.args.branch)
        if self.args.documentation:
            self.documentation_path = os.path.abspath(self.args.documentation)
        if self.args.release:
            self.version = self.args.release
            self.work_path = os.path.abspath(os.path.join(self.branch_path, '..', 'OpenLP-' + self.version))
        else:
            self.version = None
            self.work_path = self.branch_path
        self.openlp_script = os.path.abspath(os.path.join(self.work_path, 'run_openlp.py'))
        self.source_path = os.path.join(self.work_path, 'openlp')
        self.manual_path = os.path.join(self.documentation_path, 'manual')
        self.manual_build_path = os.path.join(self.manual_path, 'build')
        self.i18n_utils = os.path.join(self.work_path, 'scripts', 'translation_utils.py')
        self.i18n_path = os.path.join(self.work_path, 'resources', 'i18n')
        self.build_path = os.path.join(self.work_path, 'build')
        # Print out all the values
        self._print_verbose('   {:.<20} {}'.format('openlp script: ', self.openlp_script))
        self._print_verbose('   {:.<20} {}'.format('source: ', self.source_path))
        self._print_verbose('   {:.<20} {}'.format('manual path: ', self.manual_path))
        self._print_verbose('   {:.<20} {}'.format('manual build path: ', self.manual_build_path))
        self._print_verbose('   {:.<20} {}'.format('i18n utils: ', self.i18n_utils))
        self._print_verbose('   {:.<20} {}'.format('i18n path: ', self.i18n_path))
        self._print_verbose('   {:.<20} {}'.format('build path: ', self.build_path))
        self._print_verbose('Overrides:')
        self._print_verbose('   {:.<20} {}'.format('branch **: ', self.branch_path))
        self._print_verbose('   {:.<20} {}'.format('documentation **: ', self.documentation_path))
        self._print_verbose('   {:.<20} {}'.format('version: ', self.version))
        self._print_verbose('   {:.<20} {}'.format('work path: ', self.work_path))

    def setup_extra(self):
        """
        Extra setup to run
        """
        pass

    def update_code(self):
        """
        Update the code in the branch.
        """
        self._print('Reverting any changes to the code...')
        self._bzr('revert', self.branch_path, err_msg='Error reverting the code')
        self._print('Updating the code...')
        self._bzr('update', self.branch_path, err_msg='Error updating the code')

    def export_release(self):
        """
        Export a particular release
        """
        if os.path.exists(self.work_path):
            rmtree(self.work_path)
        self._print('Exporting the release version...')
        self._bzr('export', self.branch_path, ['-r', 'tag:' + self.version, self.work_path],
                  'Error exporting the code')

    def get_extra_parameters(self):
        """
        Return a list of any extra parameters we wish to use
        """
        return []

    def run_pyinstaller(self):
        """
        Run PyInstaller on the branch to build an executable.
        """
        self._print('Running PyInstaller...')
        os.chdir(self.work_path)
        cmd = [self.python,
               self.pyinstaller_exe,
               '--clean',
               '--noconfirm',
               '--windowed',
               '--noupx',
               '--additional-hooks-dir', self.hooks_path,
               '--runtime-hook', os.path.join(self.hooks_path, 'rthook_ssl.py'),
               '-i', self.icon_path,
               '-n', 'OpenLP',
               *self.get_extra_parameters(),  # Adds any extra parameters we wish to use
               self.openlp_script]
        if self.args.verbose:
            cmd.append('--log-level=DEBUG')
        else:
            cmd.append('--log-level=ERROR')
        if self.args.debug:
            cmd.append('-d')
        self._print_verbose('... {}'.format(' '.join(cmd)))
        output, error = self._run_command(cmd, 'Error running PyInstaller')
        self._print_verbose(output)
        self._print_verbose(error)

    def write_version_file(self):
        """
        Write the version number to a file for reading once installed.
        """
        self._print('Writing version file...')
        if not self.args.release:
            if self.args.tag_override:
                self.version = self.args.tag_override
            else:
                # This is a development build, get the tag and revision
                output = self._bzr('tags', self.branch_path, err_msg='Error running bzr tags')
                lines = output.splitlines()
                if len(lines) == 0:
                    tag = '0.0.0'
                    revision = '0'
                else:
                    tag, revision = lines[-1].split()
                output = self._bzr('log', self.branch_path, ['--line', '-r', '-1'], 'Error running bzr log')
                revision = output.split(':')[0]
                self.version = '{tag}.dev{revision}'.format(tag=tag, revision=revision)
        # Write the version to the version file
        with open(os.path.join(self.dist_path, '.version'), 'w') as version_file:
            version_file.write(str(self.version))

    def copy_default_theme(self):
        """
        Copy the default theme to the correct directory for OpenLP.
        """
        self._print('Copying default theme...')
        source = os.path.join(self.source_path, 'core', 'lib', 'json')
        dest = os.path.join(self.dist_path, 'core', 'lib', 'json')
        for root, _, files in os.walk(source):
            for filename in files:
                if filename.endswith('.json'):
                    dest_path = os.path.join(dest, root[len(source) + 1:])
                    if not os.path.exists(dest_path):
                        os.makedirs(dest_path)
                    self._print_verbose('... %s', filename)
                    copy(os.path.join(root, filename), os.path.join(dest_path, filename))

    def copy_plugins(self):
        """
        Copy all the plugins to the correct directory so that OpenLP sees that
        it has plugins.
        """
        self._print('Copying plugins...')
        source = os.path.join(self.source_path, 'plugins')
        dest = os.path.join(self.dist_path, 'plugins')
        for root, _, files in os.walk(source):
            for filename in files:
                if not filename.endswith('.pyc'):
                    dest_path = os.path.join(dest, root[len(source) + 1:])
                    if not os.path.exists(dest_path):
                        os.makedirs(dest_path)
                    self._print_verbose('... %s', filename)
                    copy(os.path.join(root, filename), os.path.join(dest_path, filename))

    def copy_media_player(self):
        """
        Copy the media players to the correct directory for OpenLP.
        """
        self._print('Copying media player...')
        source = os.path.join(self.source_path, 'core', 'ui', 'media')
        dest = os.path.join(self.dist_path, 'core', 'ui', 'media')
        for root, _, files in os.walk(source):
            for filename in files:
                if not filename.endswith('.pyc'):
                    dest_path = os.path.join(dest, root[len(source) + 1:])
                    if not os.path.exists(dest_path):
                        os.makedirs(dest_path)
                    self._print_verbose('... %s', filename)
                    copy(os.path.join(root, filename), os.path.join(dest_path, filename))

    def copy_font_files(self):
        """
        Copy OpenLP font files
        """
        self._print('Copying OpenLP fonts files...')
        src_dir = os.path.join(self.source_path, 'core', 'ui', 'fonts')
        dst_dir = os.path.join(self.dist_path, 'core', 'ui', 'fonts')
        font_files = ['OpenLP.ttf', 'openlp-charmap.json']
        os.makedirs(dst_dir)
        for font_file in font_files:
            src = os.path.join(src_dir, font_file)
            dst = os.path.join(dst_dir, font_file)
            copy(src, dst)

    def copy_display_files(self):
        """
        Copy OpenLP display HTML files
        """
        self._print('Copying OpenLP HTML display files...')
        src_dir = os.path.join(self.source_path, 'core', 'display', 'html')
        dst_dir = os.path.join(self.dist_path, 'core', 'display', 'html')
        os.makedirs(dst_dir)
        for display_file in os.listdir(src_dir):
            src = os.path.join(src_dir, display_file)
            dst = os.path.join(dst_dir, display_file)
            copy(src, dst)

    def copy_extra_files(self):
        """
        Copy any extra files which are particular to a platform
        """
        pass

    def update_translations(self):
        """
        Update the translations.
        """
        self._print('Updating translations...')
        username = None
        password = None
        if self.args.transifex_user:
            username = self.args.transifex_user
        if self.args.transifex_password:
            password = self.args.transifex_pass
        if (not username or not password) and not self.config.has_section('transifex'):
            raise Exception('No section named "transifex" found.')
        elif not username and not self.config.has_option('transifex', 'username'):
            raise Exception('No option named "username" found.')
        elif not password and not self.config.has_option('transifex', 'password'):
            raise Exception('No option named "password" found.')
        if not username:
            username = self.config.get('transifex', 'username')
        if not password:
            password = self.config.get('transifex', 'password')
        os.chdir(os.path.split(self.i18n_utils)[0])
        self._run_command([self.python, self.i18n_utils, '-qdpu', '-U', username, '-P', password],
                          err_msg='Error running translation_utils.py')

    def compile_translations(self):
        """
        Compile the translations for Qt.
        """
        self._print('Compiling translations...')
        if not os.path.exists(os.path.join(self.dist_path, 'i18n')):
            os.makedirs(os.path.join(self.dist_path, 'i18n'))
        for filename in os.listdir(self.i18n_path):
            if filename.endswith('.ts'):
                self._print_verbose('... %s', filename)
                source_path = os.path.join(self.i18n_path, filename)
                dest_path = os.path.join(self.dist_path, 'i18n', filename.replace('.ts', '.qm'))
                self._run_command((self.lrelease_exe, '-compress', '-silent', source_path, '-qm', dest_path),
                                  err_msg='Error running lconvert on %s' % source_path)
        self._print('Copying Qt translation files...')
        source = self.get_qt_translations_path()
        for filename in os.listdir(source):
            if filename.startswith('qt') and filename.endswith('.qm'):
                self._print_verbose('... %s', filename)
                copy(os.path.join(source, filename), os.path.join(self.dist_path, 'i18n', filename))

    def run_sphinx(self):
        """
        Run Sphinx to build the manual
        """
        self._print('Running Sphinx...')
        self._print_verbose('... Deleting previous help manual build... %s', self.manual_build_path)
        if os.path.exists(self.manual_build_path):
            rmtree(self.manual_build_path)
        os.chdir(self.manual_path)
        sphinx_build = self.get_sphinx_build()
        command = [self.sphinx_exe, '-b', sphinx_build, '-d', 'build/doctrees', 'source',
                   'build/{}'.format(sphinx_build)]
        self._run_command(command, 'Error running Sphinx')
        self.after_run_sphinx()

    def after_run_sphinx(self):
        """
        Run some extra commands after sphinx.
        """
        pass

    def build_package(self):
        """
        Actually package the resultant build
        """
        pass

    def main(self):
        """
        The main function to run the builder.
        """
        self._print_verbose('OpenLP main script: ......%s', self.openlp_script)
        self._print_verbose('Script path: .............%s', self.script_path)
        self._print_verbose('Branch path: .............%s', self.branch_path)
        self._print_verbose('')
        if not self.args.skip_update:
            self.update_code()
        if self.args.release:
            self.export_release()
        self.run_pyinstaller()
        self.write_version_file()
        self.copy_default_theme()
        self.copy_plugins()
        self.copy_media_player()
        self.copy_font_files()
        self.copy_display_files()
        if os.path.exists(self.manual_path):
            self.run_sphinx()
        else:
            self._print('')
            self._print('WARNING: Documentation trunk not found')
            self._print('         Help file will not be included in build')
            self._print('')
        self.copy_extra_files()
        if not self.args.skip_translations:
            if self.args.update_translations:
                self.update_translations()
            self.compile_translations()
        self.build_package()

        self._print('Done.')
        raise SystemExit()