From 93e0a35489c5443317d37ace58a0135c7ddcb675 Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Thu, 26 Oct 2017 13:33:22 -0700 Subject: [PATCH 1/6] Update Jenkins script to use python-jenkins --- scripts/jenkins_script.py | 118 ++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 56 deletions(-) diff --git a/scripts/jenkins_script.py b/scripts/jenkins_script.py index 9a0e9a4ec..8ce6af90d 100755 --- a/scripts/jenkins_script.py +++ b/scripts/jenkins_script.py @@ -31,25 +31,16 @@ You probably want to create an alias. Add this to your ~/.bashrc file and then l You can look up the token in the Branch-01-Pull job configuration or ask in IRC. """ - +import os import re -import sys import time -from optparse import OptionParser +from argparse import ArgumentParser from subprocess import Popen, PIPE -import warnings -from requests.exceptions import HTTPError from jenkins import Jenkins - JENKINS_URL = 'https://ci.openlp.io/' REPO_REGEX = r'(.*/+)(~.*)' -# Allows us to black list token. So when we change the token, we can display a proper message to the user. -OLD_TOKENS = [] - -# Disable the InsecureRequestWarning we get from urllib3, because we're not verifying our own self-signed certificate -warnings.simplefilter('ignore') class OpenLPJobs(object): @@ -85,13 +76,22 @@ class JenkinsTrigger(object): :param token: The token we need to trigger the build. If you do not have this token, ask in IRC. """ - def __init__(self, token): + def __init__(self, username, password, can_use_colour): """ Create the JenkinsTrigger instance. """ - self.token = token + self.build_numbers = {} + self.can_use_colour = can_use_colour and not os.name.startswith('nt') self.repo_name = get_repo_name() - self.jenkins_instance = Jenkins(JENKINS_URL) + self.server = Jenkins(JENKINS_URL, username=username, password=password) + + def fetch_build_numbers(self): + """ + Get the next build number from all the jobs + """ + for job_name in OpenLPJobs.Jobs: + job_info = self.server.get_job_info(job_name) + self.build_number[job_name] = job_info['nextBuildNumber'] def trigger_build(self): """ @@ -102,8 +102,7 @@ class JenkinsTrigger(object): # We just want the name (not the email). name = ' '.join(raw_output.decode().split()[:-1]) cause = 'Build triggered by %s (%s)' % (name, self.repo_name) - self.jenkins_instance.job(OpenLPJobs.Branch_Pull).build({'BRANCH_NAME': self.repo_name, 'cause': cause}, - token=self.token) + self.server.build_job(OpenLPJobs.Branch_Pull, {'BRANCH_NAME': self.repo_name, 'cause': cause}) def print_output(self): """ @@ -129,6 +128,20 @@ class JenkinsTrigger(object): # Open the url Popen(('xdg-open', url), stderr=PIPE) + def _get_build_info(self, job_name, build_number): + """ + Get the build info from the server. This method will check the queue and wait for the build. + """ + queue_info = self.server.get_queue_info() + tries = 0 + while queue_info and tries < 50: + tries += 1 + time.sleep(0.5) + queue_info = self.server.get_queue_info() + if tries >= 50: + raise Exception('Build has not started yet, it may be stuck in the queue.') + return self.server.get_build_info(job_name, build_number) + def __print_build_info(self, job_name): """ This helper method prints the job information of the given ``job_name`` @@ -136,21 +149,21 @@ class JenkinsTrigger(object): :param job_name: The name of the job we want the information from. For example *Branch-01-Pull*. Use the class variables from the :class:`OpenLPJobs` class. """ + build_info = self._get_build_info(job_name, self.build_number[job_name]) + print('{} ... '.format(build_info['url']), end='', flush=True) is_success = False - job = self.jenkins_instance.job(job_name) - while job.info['inQueue']: - time.sleep(1) - build = job.last_build - build.wait() - if build.info['result'] == 'SUCCESS': + while build_info['building'] is True: + time.sleep(0.5) + build_info = self.server.get_build_info(job_name, self.build_number[job_name]) + result_string = build_info['result'] + if self.can_use_colour and result_string == 'SUCCESS': # Make 'SUCCESS' green. - result_string = '%s%s%s' % (Colour.GREEN_START, build.info['result'], Colour.GREEN_END) + result_string = '%s%s%s' % (Colour.GREEN_START, result_string, Colour.GREEN_END) is_success = True - else: + elif self.can_use_colour: # Make 'FAILURE' red. - result_string = '%s%s%s' % (Colour.RED_START, build.info['result'], Colour.RED_END) - url = build.info['url'] - print('[%s] %s' % (result_string, url)) + result_string = '%s%s%s' % (Colour.RED_START, result_string, Colour.RED_END) + print('[{}]'.format(result_string)) return is_success @@ -186,36 +199,29 @@ def get_repo_name(): def main(): - usage = 'Usage: python %prog TOKEN [options]' + """ + Run the script + """ + parser = ArgumentParser() + parser.add_argument('-d', '--disable-output', action='store_false', default=True, help='Disable output') + parser.add_argument('-b', '--open-browser', action='store_true', default=False, + help='Opens the jenkins page in your browser') + parser.add_argument('-c', '--enable-colour', action='store_false', default=True, + help='Enable coloured output. Disabled on Windows') + parser.add_argument('-u', '--username', required=True, help='Your Jenkins username') + parser.add_argument('-p', '--password', required=True, help='Your Jenkins password or personal token') + args = parser.parse_args() - parser = OptionParser(usage=usage) - parser.add_option('-d', '--disable-output', dest='enable_output', action='store_false', default=True, - help='Disable output.') - parser.add_option('-b', '--open-browser', dest='open_browser', action='store_true', default=False, - help='Opens the jenkins page in your browser.') - options, args = parser.parse_args(sys.argv) - - if len(args) == 2: - if not get_repo_name(): - print('Not a branch. Have you pushed it to launchpad? Did you cd to the branch?') - return - token = args[-1] - if token in OLD_TOKENS: - print('Your token is not valid anymore. Get the most recent one.') - return - jenkins_trigger = JenkinsTrigger(token) - try: - jenkins_trigger.trigger_build() - except HTTPError: - print('Wrong token.') - return - # Open the browser before printing the output. - if options.open_browser: - jenkins_trigger.open_browser() - if options.enable_output: - jenkins_trigger.print_output() - else: - parser.print_help() + if not get_repo_name(): + print('Not a branch. Have you pushed it to launchpad? Did you cd to the branch?') + return + jenkins_trigger = JenkinsTrigger(username=args.username, password=args.password) + jenkins_trigger.trigger_build() + # Open the browser before printing the output. + if args.open_browser: + jenkins_trigger.open_browser() + if not args.disable_output: + jenkins_trigger.print_output() if __name__ == '__main__': From f6243f6e8828edc534d70be969420f673e41fa3f Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Thu, 26 Oct 2017 15:05:10 -0700 Subject: [PATCH 2/6] Fix some bugs, line up the statuses, provide some failure feedback, flip the colour option --- scripts/jenkins_script.py | 43 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/scripts/jenkins_script.py b/scripts/jenkins_script.py index 8ce6af90d..2d93d53d3 100755 --- a/scripts/jenkins_script.py +++ b/scripts/jenkins_script.py @@ -80,7 +80,7 @@ class JenkinsTrigger(object): """ Create the JenkinsTrigger instance. """ - self.build_numbers = {} + self.build_number = {} self.can_use_colour = can_use_colour and not os.name.startswith('nt') self.repo_name = get_repo_name() self.server = Jenkins(JENKINS_URL, username=username, password=password) @@ -102,6 +102,7 @@ class JenkinsTrigger(object): # We just want the name (not the email). name = ' '.join(raw_output.decode().split()[:-1]) cause = 'Build triggered by %s (%s)' % (name, self.repo_name) + self.fetch_build_numbers() self.server.build_job(OpenLPJobs.Branch_Pull, {'BRANCH_NAME': self.repo_name, 'cause': cause}) def print_output(self): @@ -117,7 +118,10 @@ class JenkinsTrigger(object): for job in OpenLPJobs.Jobs: if not self.__print_build_info(job): - print('Stopping after failure') + if self.current_build: + print('Stopping after failure, see {}console for more details'.format(self.current_build['url'])) + else: + print('Stopping after failure') break def open_browser(self): @@ -149,21 +153,22 @@ class JenkinsTrigger(object): :param job_name: The name of the job we want the information from. For example *Branch-01-Pull*. Use the class variables from the :class:`OpenLPJobs` class. """ - build_info = self._get_build_info(job_name, self.build_number[job_name]) - print('{} ... '.format(build_info['url']), end='', flush=True) + self.current_build = self._get_build_info(job_name, self.build_number[job_name]) + print('{:<60} [RUNNING]'.format(self.current_build['url']), end='', flush=True) is_success = False - while build_info['building'] is True: + while self.current_build['building'] is True: time.sleep(0.5) - build_info = self.server.get_build_info(job_name, self.build_number[job_name]) - result_string = build_info['result'] - if self.can_use_colour and result_string == 'SUCCESS': - # Make 'SUCCESS' green. - result_string = '%s%s%s' % (Colour.GREEN_START, result_string, Colour.GREEN_END) - is_success = True - elif self.can_use_colour: - # Make 'FAILURE' red. - result_string = '%s%s%s' % (Colour.RED_START, result_string, Colour.RED_END) - print('[{}]'.format(result_string)) + self.current_build = self.server.get_build_info(job_name, self.build_number[job_name]) + result_string = self.current_build['result'] + is_success = result_string == 'SUCCESS' + if self.can_use_colour: + if is_success: + # Make 'SUCCESS' green. + result_string = '{}{}{}'.format(Colour.GREEN_START, result_string, Colour.GREEN_END) + else: + # Make 'FAILURE' red. + result_string = '{}{}{}'.format(Colour.RED_START, result_string, Colour.RED_END) + print('\b\b\b\b\b\b\b\b\b[{:>7}]'.format(result_string)) return is_success @@ -203,11 +208,11 @@ def main(): Run the script """ parser = ArgumentParser() - parser.add_argument('-d', '--disable-output', action='store_false', default=True, help='Disable output') + parser.add_argument('-d', '--disable-output', action='store_true', default=False, help='Disable output') parser.add_argument('-b', '--open-browser', action='store_true', default=False, help='Opens the jenkins page in your browser') - parser.add_argument('-c', '--enable-colour', action='store_false', default=True, - help='Enable coloured output. Disabled on Windows') + parser.add_argument('-n', '--no-colour', action='store_true', default=False, + help='Disable coloured output (always disabled on Windows)') parser.add_argument('-u', '--username', required=True, help='Your Jenkins username') parser.add_argument('-p', '--password', required=True, help='Your Jenkins password or personal token') args = parser.parse_args() @@ -215,7 +220,7 @@ def main(): if not get_repo_name(): print('Not a branch. Have you pushed it to launchpad? Did you cd to the branch?') return - jenkins_trigger = JenkinsTrigger(username=args.username, password=args.password) + jenkins_trigger = JenkinsTrigger(args.username, args.password, not args.no_colour) jenkins_trigger.trigger_build() # Open the browser before printing the output. if args.open_browser: From 0d486d051448546c5758ebe2bc678435b5447b7c Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Thu, 26 Oct 2017 15:32:20 -0700 Subject: [PATCH 3/6] Updated the documentation in the file --- scripts/jenkins_script.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/scripts/jenkins_script.py b/scripts/jenkins_script.py index 2d93d53d3..05c589ddd 100755 --- a/scripts/jenkins_script.py +++ b/scripts/jenkins_script.py @@ -21,15 +21,24 @@ # Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################### """ -This script helps to trigger builds of branches. To use it you have to install the jenkins-webapi package: +This script helps to trigger builds of branches. To use it you have to install the python-jenkins module. On Fedora +and Ubuntu/Debian, it is available as the ``python3-jenkins`` package:: - pip3 install jenkins-webapi + $ sudo dnf/apt install python3-jenkins -You probably want to create an alias. Add this to your ~/.bashrc file and then logout and login (to apply the alias): +To make it easier to run you may want to create a shell script or an alias. To create an alias, add this to your +``~/.bashrc`` (or ``~/.zshrc``) file and then log out and log back in again (to apply the alias):: - alias ci="python3 ./scripts/jenkins_script.py TOKEN" + alias ci="python3 /path/to/openlp_root/scripts/jenkins_script.py -u USERNAME -p PASSWORD" -You can look up the token in the Branch-01-Pull job configuration or ask in IRC. +To create a shell script, create the following file in a location in your ``$PATH`` (I called mine ``ci``):: + + #!/bin/bash + python3 /path/to/openlp_root/scripts/jenkins_script.py -u USERNAME -p PASSWORD + +``USERNAME`` is your Jenkins username, and ``PASSWORD`` is your Jenkins password or personal token. + +An older version of this script used to use a shared TOKEN, but this has been replaced with the username and password. """ import os import re @@ -154,7 +163,7 @@ class JenkinsTrigger(object): variables from the :class:`OpenLPJobs` class. """ self.current_build = self._get_build_info(job_name, self.build_number[job_name]) - print('{:<60} [RUNNING]'.format(self.current_build['url']), end='', flush=True) + print('{:<70} [RUNNING]'.format(self.current_build['url']), end='', flush=True) is_success = False while self.current_build['building'] is True: time.sleep(0.5) From ac61fb5a4fff38a30c3bdd4fadff084112ca6d5b Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Fri, 10 Nov 2017 08:57:35 -0700 Subject: [PATCH 4/6] Make the heading line as long as the others --- scripts/jenkins_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/jenkins_script.py b/scripts/jenkins_script.py index 05c589ddd..706d808af 100755 --- a/scripts/jenkins_script.py +++ b/scripts/jenkins_script.py @@ -119,7 +119,7 @@ class JenkinsTrigger(object): Print the status information of the build triggered. """ print('Add this to your merge proposal:') - print('--------------------------------') + print('-' * 80) bzr = Popen(('bzr', 'revno'), stdout=PIPE, stderr=PIPE) raw_output, error = bzr.communicate() revno = raw_output.decode().strip() From a9692f83910b10ba75fbc4fdd0c329f0767580e7 Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Fri, 10 Nov 2017 09:36:06 -0700 Subject: [PATCH 5/6] Print out the build info before the build starts with a WAITING status --- scripts/jenkins_script.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/jenkins_script.py b/scripts/jenkins_script.py index 706d808af..d0ba665b7 100755 --- a/scripts/jenkins_script.py +++ b/scripts/jenkins_script.py @@ -89,18 +89,18 @@ class JenkinsTrigger(object): """ Create the JenkinsTrigger instance. """ - self.build_number = {} + self.jobs = {} self.can_use_colour = can_use_colour and not os.name.startswith('nt') self.repo_name = get_repo_name() self.server = Jenkins(JENKINS_URL, username=username, password=password) - def fetch_build_numbers(self): + def fetch_jobs(self): """ - Get the next build number from all the jobs + Get the job info for all the jobs """ for job_name in OpenLPJobs.Jobs: job_info = self.server.get_job_info(job_name) - self.build_number[job_name] = job_info['nextBuildNumber'] + self.jobs[job_name] = job_info def trigger_build(self): """ @@ -111,7 +111,7 @@ class JenkinsTrigger(object): # We just want the name (not the email). name = ' '.join(raw_output.decode().split()[:-1]) cause = 'Build triggered by %s (%s)' % (name, self.repo_name) - self.fetch_build_numbers() + self.fetch_jobs() self.server.build_job(OpenLPJobs.Branch_Pull, {'BRANCH_NAME': self.repo_name, 'cause': cause}) def print_output(self): @@ -162,8 +162,10 @@ class JenkinsTrigger(object): :param job_name: The name of the job we want the information from. For example *Branch-01-Pull*. Use the class variables from the :class:`OpenLPJobs` class. """ - self.current_build = self._get_build_info(job_name, self.build_number[job_name]) - print('{:<70} [RUNNING]'.format(self.current_build['url']), end='', flush=True) + job = self.jobs[job_name] + print('{:<70} [WAITING]'.format(job['url'] + '/' + job['nextBuildNumber']), end='', flush=True) + self.current_build = self._get_build_info(job_name, job[job_name]['nextBuildNumber']) + print('\b\b\b\b\b\b\b\b\b[RUNNING]', end='', flush=True) is_success = False while self.current_build['building'] is True: time.sleep(0.5) From dd2d1002ce188a3e1cf094a79ddfa9bc3d9e1c4f Mon Sep 17 00:00:00 2001 From: Raoul Snyman Date: Fri, 10 Nov 2017 10:51:42 -0700 Subject: [PATCH 6/6] Fix up a couple of issues, and move URL creation to where the job info is pulled --- scripts/jenkins_script.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/jenkins_script.py b/scripts/jenkins_script.py index d0ba665b7..a669de71e 100755 --- a/scripts/jenkins_script.py +++ b/scripts/jenkins_script.py @@ -101,6 +101,7 @@ class JenkinsTrigger(object): for job_name in OpenLPJobs.Jobs: job_info = self.server.get_job_info(job_name) self.jobs[job_name] = job_info + self.jobs[job_name]['nextBuildUrl'] = '{url}{nextBuildNumber}/'.format(**job_info) def trigger_build(self): """ @@ -163,13 +164,13 @@ class JenkinsTrigger(object): variables from the :class:`OpenLPJobs` class. """ job = self.jobs[job_name] - print('{:<70} [WAITING]'.format(job['url'] + '/' + job['nextBuildNumber']), end='', flush=True) - self.current_build = self._get_build_info(job_name, job[job_name]['nextBuildNumber']) + print('{:<70} [WAITING]'.format(job['nextBuildUrl']), end='', flush=True) + self.current_build = self._get_build_info(job_name, job['nextBuildNumber']) print('\b\b\b\b\b\b\b\b\b[RUNNING]', end='', flush=True) is_success = False while self.current_build['building'] is True: time.sleep(0.5) - self.current_build = self.server.get_build_info(job_name, self.build_number[job_name]) + self.current_build = self.server.get_build_info(job_name, job['nextBuildNumber']) result_string = self.current_build['result'] is_success = result_string == 'SUCCESS' if self.can_use_colour: