This commit is contained in:
Tim Bentley 2017-11-11 08:35:02 +00:00
commit 12b71fc068
48 changed files with 256 additions and 209 deletions

View File

@ -43,9 +43,13 @@ log = logging.getLogger(__name__ + '.__init__')
FIRST_CAMEL_REGEX = re.compile('(.)([A-Z][a-z]+)')
SECOND_CAMEL_REGEX = re.compile('([a-z0-9])([A-Z])')
CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]', re.UNICODE)
INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]', re.UNICODE)
CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]')
INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]')
IMAGES_FILTER = None
REPLACMENT_CHARS_MAP = str.maketrans({'\u2018': '\'', '\u2019': '\'', '\u201c': '"', '\u201d': '"', '\u2026': '...',
'\u2013': '-', '\u2014': '-', '\v': '\n\n', '\f': '\n\n'})
NEW_LINE_REGEX = re.compile(r' ?(\r\n?|\n) ?')
WHITESPACE_REGEX = re.compile(r'[ \t]+')
def trace_error_handler(logger):
@ -339,7 +343,7 @@ def delete_file(file_path):
if file_path.exists():
file_path.unlink()
return True
except (IOError, OSError):
except OSError:
log.exception('Unable to delete file {file_path}'.format(file_path=file_path))
return False
@ -436,3 +440,17 @@ def get_file_encoding(file_path):
return detector.result
except OSError:
log.exception('Error detecting file encoding')
def normalize_str(irreg_str):
"""
Normalize the supplied string. Remove unicode control chars and tidy up white space.
:param str irreg_str: The string to normalize.
:return: The normalized string
:rtype: str
"""
irreg_str = irreg_str.translate(REPLACMENT_CHARS_MAP)
irreg_str = CONTROL_CHARS.sub('', irreg_str)
irreg_str = NEW_LINE_REGEX.sub('\n', irreg_str)
return WHITESPACE_REGEX.sub(' ', irreg_str)

View File

@ -83,7 +83,7 @@ class AppLocation(object):
"""
# Check if we have a different data location.
if Settings().contains('advanced/data path'):
path = Settings().value('advanced/data path')
path = Path(Settings().value('advanced/data path'))
else:
path = AppLocation.get_directory(AppLocation.DataDir)
create_paths(path)

View File

@ -97,8 +97,8 @@ def get_web_page(url, headers=None, update_openlp=False, proxies=None):
response = requests.get(url, headers=headers, proxies=proxies, timeout=float(CONNECTION_TIMEOUT))
log.debug('Downloaded page {url}'.format(url=response.url))
break
except IOError:
# For now, catch IOError. All requests errors inherit from IOError
except OSError:
# For now, catch OSError. All requests errors inherit from OSError
log.exception('Unable to connect to {url}'.format(url=url))
response = None
if retries >= CONNECTION_RETRIES:
@ -127,7 +127,7 @@ def get_url_file_size(url):
try:
response = requests.head(url, timeout=float(CONNECTION_TIMEOUT), allow_redirects=True)
return int(response.headers['Content-Length'])
except IOError:
except OSError:
if retries > CONNECTION_RETRIES:
raise ConnectionError('Unable to download {url}'.format(url=url))
else:
@ -173,7 +173,7 @@ def url_get_file(callback, url, file_path, sha256=None):
file_path.unlink()
return False
break
except IOError:
except OSError:
trace_error_handler(log)
if retries > CONNECTION_RETRIES:
if file_path.exists():

View File

@ -53,7 +53,7 @@ def translate(context, text, comment=None, qt_translate=QtCore.QCoreApplication.
Language = namedtuple('Language', ['id', 'name', 'code'])
ICU_COLLATOR = None
DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+', re.UNICODE)
DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+')
LANGUAGES = sorted([
Language(1, translate('common.languages', '(Afan) Oromo', 'Language code: om'), 'om'),
Language(2, translate('common.languages', 'Abkhazian', 'Language code: ab'), 'ab'),

View File

@ -101,6 +101,20 @@ class RegistryProperties(object):
"""
This adds registry components to classes to use at run time.
"""
_application = None
_plugin_manager = None
_image_manager = None
_media_controller = None
_service_manager = None
_preview_controller = None
_live_controller = None
_main_window = None
_renderer = None
_theme_manager = None
_settings_form = None
_alerts_manager = None
_projector_manager = None
@property
def application(self):
"""

View File

@ -233,7 +233,7 @@ def create_paths(*paths, **kwargs):
try:
if not path.exists():
path.mkdir(parents=True)
except IOError:
except OSError:
if not kwargs.get('do_not_log', False):
log.exception('failed to check if directory exists or create directory')

View File

@ -255,7 +255,10 @@ class Settings(QtCore.QSettings):
('core/logo file', 'core/logo file', [(str_to_path, None)]),
('presentations/last directory', 'presentations/last directory', [(str_to_path, None)]),
('images/last directory', 'images/last directory', [(str_to_path, None)]),
('media/last directory', 'media/last directory', [(str_to_path, None)])
('media/last directory', 'media/last directory', [(str_to_path, None)]),
('songuasge/db password', 'songusage/db password', []),
('songuasge/db hostname', 'songusage/db hostname', []),
('songuasge/db database', 'songusage/db database', [])
]
@staticmethod

View File

@ -104,7 +104,7 @@ def get_text_file_string(text_file_path):
# no BOM was found
file_handle.seek(0)
content = file_handle.read()
except (IOError, UnicodeError):
except (OSError, UnicodeError):
log.exception('Failed to open text file {text}'.format(text=text_file_path))
return content

View File

@ -92,7 +92,7 @@ class MediaManagerItem(QtWidgets.QWidget, RegistryProperties):
Run some initial setup. This method is separate from __init__ in order to mock it out in tests.
"""
self.hide()
self.whitespace = re.compile(r'[\W_]+', re.UNICODE)
self.whitespace = re.compile(r'[\W_]+')
visible_title = self.plugin.get_string(StringContent.VisibleName)
self.title = str(visible_title['title'])
Registry().register(self.plugin.name, self)
@ -344,7 +344,9 @@ class MediaManagerItem(QtWidgets.QWidget, RegistryProperties):
else:
new_files.append(file_name)
if new_files:
self.validate_and_load(new_files, data['target'])
if 'target' in data:
self.validate_and_load(new_files, data['target'])
self.validate_and_load(new_files)
def dnd_move_internal(self, target):
"""

View File

@ -155,7 +155,7 @@ class ExceptionForm(QtWidgets.QDialog, Ui_ExceptionDialog, RegistryProperties):
try:
with file_path.open('w') as report_file:
report_file.write(report_text)
except IOError:
except OSError:
log.exception('Failed to write crash report')
def on_send_report_button_clicked(self):

View File

@ -43,7 +43,7 @@ class FormattingTagController(object):
r'(?P<tag>[^\s/!\?>]+)(?:\s+[^\s=]+="[^"]*")*\s*(?P<empty>/)?'
r'|(?P<cdata>!\[CDATA\[(?:(?!\]\]>).)*\]\])'
r'|(?P<procinst>\?(?:(?!\?>).)*\?)'
r'|(?P<comment>!--(?:(?!-->).)*--))>', re.UNICODE)
r'|(?P<comment>!--(?:(?!-->).)*--))>')
self.html_regex = re.compile(r'^(?:[^<>]*%s)*[^<>]*$' % self.html_tag_regex.pattern)
def pre_save(self):

View File

@ -180,7 +180,7 @@ class Ui_MainWindow(object):
triggers=self.service_manager_contents.on_load_service_clicked)
self.file_save_item = create_action(main_window, 'fileSaveItem', icon=':/general/general_save.png',
can_shortcuts=True, category=UiStrings().File,
triggers=self.service_manager_contents.save_file)
triggers=self.service_manager_contents.decide_save_method)
self.file_save_as_item = create_action(main_window, 'fileSaveAsItem', can_shortcuts=True,
category=UiStrings().File,
triggers=self.service_manager_contents.save_file_as)
@ -1367,7 +1367,7 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow, RegistryProperties):
'- Please wait for copy to finish').format(path=self.new_data_path))
dir_util.copy_tree(str(old_data_path), str(self.new_data_path))
log.info('Copy successful')
except (IOError, os.error, DistutilsFileError) as why:
except (OSError, DistutilsFileError) as why:
self.application.set_normal_cursor()
log.exception('Data copy failed {err}'.format(err=str(why)))
err_text = translate('OpenLP.MainWindow',

View File

@ -193,18 +193,6 @@ class Ui_ServiceManager(object):
text=translate('OpenLP.ServiceManager', 'Move to &bottom'), icon=':/services/service_bottom.png',
tooltip=translate('OpenLP.ServiceManager', 'Move item to the end of the service.'),
can_shortcuts=True, category=UiStrings().Service, triggers=self.on_service_end)
self.down_action = self.order_toolbar.add_toolbar_action(
'down',
text=translate('OpenLP.ServiceManager', 'Move &down'), can_shortcuts=True,
tooltip=translate('OpenLP.ServiceManager', 'Moves the selection down the window.'), visible=False,
triggers=self.on_move_selection_down)
action_list.add_action(self.down_action)
self.up_action = self.order_toolbar.add_toolbar_action(
'up',
text=translate('OpenLP.ServiceManager', 'Move up'), can_shortcuts=True,
tooltip=translate('OpenLP.ServiceManager', 'Moves the selection up the window.'), visible=False,
triggers=self.on_move_selection_up)
action_list.add_action(self.up_action)
self.order_toolbar.addSeparator()
self.delete_action = self.order_toolbar.add_toolbar_action(
'delete', can_shortcuts=True,
@ -300,8 +288,8 @@ class Ui_ServiceManager(object):
self.theme_menu = QtWidgets.QMenu(translate('OpenLP.ServiceManager', '&Change Item Theme'))
self.menu.addMenu(self.theme_menu)
self.service_manager_list.addActions([self.move_down_action, self.move_up_action, self.make_live_action,
self.move_top_action, self.move_bottom_action, self.up_action,
self.down_action, self.expand_action, self.collapse_action])
self.move_top_action, self.move_bottom_action, self.expand_action,
self.collapse_action])
Registry().register_function('theme_update_list', self.update_theme_list)
Registry().register_function('config_screen_changed', self.regenerate_service_items)
Registry().register_function('theme_update_global', self.theme_change)
@ -474,6 +462,12 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
Load a recent file as the service triggered by mainwindow recent service list.
:param field:
"""
if self.is_modified():
result = self.save_modified_service()
if result == QtWidgets.QMessageBox.Cancel:
return False
elif result == QtWidgets.QMessageBox.Save:
self.decide_save_method()
sender = self.sender()
self.load_file(sender.data())
@ -603,7 +597,7 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
if not os.path.exists(save_file):
shutil.copy(audio_from, save_file)
zip_file.write(audio_from, audio_to)
except IOError:
except OSError:
self.log_exception('Failed to save service to disk: {name}'.format(name=temp_file_name))
self.main_window.error_message(translate('OpenLP.ServiceManager', 'Error Saving File'),
translate('OpenLP.ServiceManager', 'There was an error saving your file.'))
@ -664,7 +658,7 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
zip_file = zipfile.ZipFile(temp_file_name, 'w', zipfile.ZIP_STORED, True)
# First we add service contents.
zip_file.writestr(service_file_name, service_content)
except IOError:
except OSError:
self.log_exception('Failed to save service to disk: {name}'.format(name=temp_file_name))
self.main_window.error_message(translate('OpenLP.ServiceManager', 'Error Saving File'),
translate('OpenLP.ServiceManager', 'There was an error saving your file.'))
@ -712,18 +706,23 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
default_file_path = directory_path / default_file_path
# SaveAs from osz to oszl is not valid as the files will be deleted on exit which is not sensible or usable in
# the long term.
lite_filter = translate('OpenLP.ServiceManager', 'OpenLP Service Files - lite (*.oszl)')
packaged_filter = translate('OpenLP.ServiceManager', 'OpenLP Service Files (*.osz)')
if self._file_name.endswith('oszl') or self.service_has_all_original_files:
file_path, filter_used = FileDialog.getSaveFileName(
self.main_window, UiStrings().SaveService, default_file_path,
translate('OpenLP.ServiceManager',
'OpenLP Service Files (*.osz);; OpenLP Service Files - lite (*.oszl)'))
'{packaged};; {lite}'.format(packaged=packaged_filter, lite=lite_filter))
else:
file_path, filter_used = FileDialog.getSaveFileName(
self.main_window, UiStrings().SaveService, file_path,
translate('OpenLP.ServiceManager', 'OpenLP Service Files (*.osz);;'))
self.main_window, UiStrings().SaveService, default_file_path,
'{packaged};;'.format(packaged=packaged_filter))
if not file_path:
return False
file_path.with_suffix('.osz')
if filter_used == lite_filter:
file_path = file_path.with_suffix('.oszl')
else:
file_path = file_path.with_suffix('.osz')
self.set_file_name(file_path)
self.decide_save_method()
@ -791,11 +790,11 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
else:
critical_error_message_box(message=translate('OpenLP.ServiceManager', 'File is not a valid service.'))
self.log_error('File contains no service data')
except (IOError, NameError):
except (OSError, NameError):
self.log_exception('Problem loading service file {name}'.format(name=file_name))
critical_error_message_box(message=translate('OpenLP.ServiceManager',
'File could not be opened because it is corrupt.'))
except zipfile.BadZipfile:
except zipfile.BadZipFile:
if os.path.getsize(file_name) == 0:
self.log_exception('Service file is zero sized: {name}'.format(name=file_name))
QtWidgets.QMessageBox.information(self, translate('OpenLP.ServiceManager', 'Empty File'),
@ -1657,14 +1656,15 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
if start_pos == -1:
return
if item is None:
end_pos = len(self.service_items)
end_pos = len(self.service_items) - 1
else:
end_pos = get_parent_item_data(item) - 1
service_item = self.service_items[start_pos]
self.service_items.remove(service_item)
self.service_items.insert(end_pos, service_item)
self.repaint_service_list(end_pos, child)
self.set_modified()
if start_pos != end_pos:
self.service_items.remove(service_item)
self.service_items.insert(end_pos, service_item)
self.repaint_service_list(end_pos, child)
self.set_modified()
else:
# we are not over anything so drop
replace = False

View File

@ -604,7 +604,7 @@ class ThemeManager(QtWidgets.QWidget, RegistryBase, Ui_ThemeManager, LogMixin, R
else:
with full_name.open('wb') as out_file:
out_file.write(theme_zip.read(zipped_file))
except (IOError, zipfile.BadZipfile):
except (OSError, zipfile.BadZipFile):
self.log_exception('Importing theme from zip failed {name}'.format(name=file_path))
raise ValidationError
except ValidationError:
@ -667,7 +667,7 @@ class ThemeManager(QtWidgets.QWidget, RegistryBase, Ui_ThemeManager, LogMixin, R
theme_path = theme_dir / '{file_name}.json'.format(file_name=name)
try:
theme_path.write_text(theme_pretty)
except IOError:
except OSError:
self.log_exception('Saving theme to file failed')
if image_source_path and image_destination_path:
if self.old_background_image_path and image_destination_path != self.old_background_image_path:
@ -675,7 +675,7 @@ class ThemeManager(QtWidgets.QWidget, RegistryBase, Ui_ThemeManager, LogMixin, R
if image_source_path != image_destination_path:
try:
copyfile(image_source_path, image_destination_path)
except IOError:
except OSError:
self.log_exception('Failed to save theme image')
self.generate_and_save_image(name, theme)

View File

@ -96,7 +96,7 @@ class VersionWorker(QtCore.QObject):
remote_version = response.text
log.debug('New version found: %s', remote_version)
break
except IOError:
except OSError:
log.exception('Unable to connect to OpenLP server to download version file')
retries += 1
else:
@ -182,7 +182,7 @@ def get_version():
try:
version_file = open(file_path, 'r')
full_version = str(version_file.read()).rstrip()
except IOError:
except OSError:
log.exception('Error in version file.')
full_version = '0.0.0-bzr000'
finally:

View File

@ -27,6 +27,7 @@ import re
from PyQt5 import QtCore, QtGui, QtWidgets
from openlp.core.common import CONTROL_CHARS
from openlp.core.common.i18n import UiStrings, translate
from openlp.core.common.path import Path, path_to_str, str_to_path
from openlp.core.common.settings import Settings
@ -241,7 +242,7 @@ class PathEdit(QtWidgets.QWidget):
self.line_edit.editingFinished.connect(self.on_line_edit_editing_finished)
self.update_button_tool_tips()
@property
@QtCore.pyqtProperty('QVariant')
def path(self):
"""
A property getter method to return the selected path.
@ -349,7 +350,7 @@ class PathEdit(QtWidgets.QWidget):
:rtype: None
"""
if self._path != path:
self.path = path
self._path = path
self.pathChanged.emit(path)
@ -470,12 +471,21 @@ class SpellTextEdit(QtWidgets.QPlainTextEdit):
cursor.insertText(html['start tag'])
cursor.insertText(html['end tag'])
def insertFromMimeData(self, source):
"""
Reimplement `insertFromMimeData` so that we can remove any control characters
:param QtCore.QMimeData source: The mime data to insert
:rtype: None
"""
self.insertPlainText(CONTROL_CHARS.sub('', source.text()))
class Highlighter(QtGui.QSyntaxHighlighter):
"""
Provides a text highlighter for pointing out spelling errors in text.
"""
WORDS = r'(?iu)[\w\']+'
WORDS = r'(?i)[\w\']+'
def __init__(self, *args):
"""

View File

@ -336,7 +336,7 @@ class ListWidgetWithDnD(QtWidgets.QListWidget):
for file in listing:
files.append(os.path.join(local_file, file))
Registry().execute('{mime_data}_dnd'.format(mime_data=self.mime_data_text),
{'files': files, 'target': self.itemAt(event.pos())})
{'files': files})
else:
event.ignore()

View File

@ -113,8 +113,7 @@ class BookNameForm(QDialog, Ui_BookNameDialog):
cor_book = self.corresponding_combo_box.currentText()
for character in '\\.^$*+?{}[]()':
cor_book = cor_book.replace(character, '\\' + character)
books = [key for key in list(self.book_names.keys()) if re.match(cor_book, str(self.book_names[key]),
re.UNICODE)]
books = [key for key in list(self.book_names.keys()) if re.match(cor_book, str(self.book_names[key]))]
books = [_f for _f in map(BiblesResourcesDB.get_book, books) if _f]
if books:
self.book_id = books[0]['id']

View File

@ -224,13 +224,13 @@ def update_reference_separators():
range_regex = '(?:(?P<from_chapter>[0-9]+){sep_v})?' \
'(?P<from_verse>[0-9]+)(?P<range_to>{sep_r}(?:(?:(?P<to_chapter>' \
'[0-9]+){sep_v})?(?P<to_verse>[0-9]+)|{sep_e})?)?'.format_map(REFERENCE_SEPARATORS)
REFERENCE_MATCHES['range'] = re.compile(r'^\s*{range}\s*$'.format(range=range_regex), re.UNICODE)
REFERENCE_MATCHES['range_separator'] = re.compile(REFERENCE_SEPARATORS['sep_l'], re.UNICODE)
REFERENCE_MATCHES['range'] = re.compile(r'^\s*{range}\s*$'.format(range=range_regex))
REFERENCE_MATCHES['range_separator'] = re.compile(REFERENCE_SEPARATORS['sep_l'])
# full reference match: <book>(<range>(,(?!$)|(?=$)))+
REFERENCE_MATCHES['full'] = \
re.compile(r'^\s*(?!\s)(?P<book>[\d]*[.]?[^\d\.]+)\.*(?<!\s)\s*'
r'(?P<ranges>(?:{range_regex}(?:{sep_l}(?!\s*$)|(?=\s*$)))+)\s*$'.format(
range_regex=range_regex, sep_l=REFERENCE_SEPARATORS['sep_l']), re.UNICODE)
range_regex=range_regex, sep_l=REFERENCE_SEPARATORS['sep_l']))
def get_reference_separator(separator_type):

View File

@ -307,8 +307,7 @@ class BibleDB(Manager):
book_escaped = book
for character in RESERVED_CHARACTERS:
book_escaped = book_escaped.replace(character, '\\' + character)
regex_book = re.compile('\\s*{book}\\s*'.format(book='\\s*'.join(book_escaped.split())),
re.UNICODE | re.IGNORECASE)
regex_book = re.compile('\\s*{book}\\s*'.format(book='\\s*'.join(book_escaped.split())), re.IGNORECASE)
if language_selection == LanguageSelection.Bible:
db_book = self.get_book(book)
if db_book:

View File

@ -366,7 +366,7 @@ class ImageMediaItem(MediaManagerItem):
if validate_thumb(image.file_path, thumbnail_path):
icon = build_icon(thumbnail_path)
else:
icon = create_thumb(image.file_path, thumbnail_path)
icon = create_thumb(str(image.file_path), str(thumbnail_path))
item_name = QtWidgets.QTreeWidgetItem([file_name])
item_name.setText(0, file_name)
item_name.setIcon(0, icon)
@ -390,6 +390,7 @@ class ImageMediaItem(MediaManagerItem):
:param files: A List of strings containing the filenames of the files to be loaded
:param target_group: The QTreeWidgetItem of the group that will be the parent of the added files
"""
file_paths = [Path(file) for file in file_paths]
self.application.set_normal_cursor()
self.load_list(file_paths, target_group)
last_dir = file_paths[0].parent

View File

@ -70,7 +70,7 @@ class PptviewController(PresentationController):
try:
self.start_process()
return self.process.CheckInstalled()
except WindowsError:
except OSError:
return False
def start_process(self):

View File

@ -105,9 +105,9 @@ class EditSongForm(QtWidgets.QDialog, Ui_EditSongDialog, RegistryProperties):
self.topics_list_view.setSortingEnabled(False)
self.topics_list_view.setAlternatingRowColors(True)
self.audio_list_widget.setAlternatingRowColors(True)
self.find_verse_split = re.compile('---\[\]---\n', re.UNICODE)
self.whitespace = re.compile(r'\W+', re.UNICODE)
self.find_tags = re.compile(u'\{/?\w+\}', re.UNICODE)
self.find_verse_split = re.compile('---\[\]---\n')
self.whitespace = re.compile(r'\W+')
self.find_tags = re.compile(r'\{/?\w+\}')
def _load_objects(self, cls, combo, cache):
"""

View File

@ -24,7 +24,6 @@ The :mod:`~openlp.plugins.songs.lib` module contains a number of library functio
"""
import logging
import os
import re
from PyQt5 import QtWidgets
@ -39,8 +38,8 @@ from openlp.plugins.songs.lib.ui import SongStrings
log = logging.getLogger(__name__)
WHITESPACE = re.compile(r'[\W_]+', re.UNICODE)
APOSTROPHE = re.compile('[\'`ʻ]', re.UNICODE)
WHITESPACE = re.compile(r'[\W_]+')
APOSTROPHE = re.compile(r'[\'`ʻ]')
# PATTERN will look for the next occurence of one of these symbols:
# \controlword - optionally preceded by \*, optionally followed by a number
# \'## - where ## is a pair of hex digits, representing a single character

View File

@ -25,6 +25,7 @@ import re
from lxml import etree, objectify
from openlp.core.common import normalize_str
from openlp.plugins.songs.lib import VerseType
from openlp.plugins.songs.lib.importers.songimport import SongImport
@ -225,7 +226,7 @@ class EasySlidesImport(SongImport):
verses[reg].setdefault(vt, {})
verses[reg][vt].setdefault(vn, {})
verses[reg][vt][vn].setdefault(inst, [])
verses[reg][vt][vn][inst].append(self.tidy_text(line))
verses[reg][vt][vn][inst].append(normalize_str(line))
# done parsing
versetags = []
# we use our_verse_order to ensure, we insert lyrics in the same order

View File

@ -101,7 +101,7 @@ class MediaShoutImport(SongImport):
self.song_book_name = song.SongID
for verse in verses:
tag = VERSE_TAGS[verse.Type] + str(verse.Number) if verse.Type < len(VERSE_TAGS) else 'O'
self.add_verse(self.tidy_text(verse.Text), tag)
self.add_verse(verse.Text, tag)
for order in verse_order:
if order.Type < len(VERSE_TAGS):
self.verse_order_list.append(VERSE_TAGS[order.Type] + str(order.Number))

View File

@ -24,7 +24,7 @@ import time
from PyQt5 import QtCore
from openlp.core.common import is_win, get_uno_command, get_uno_instance
from openlp.core.common import get_uno_command, get_uno_instance, is_win, normalize_str
from openlp.core.common.i18n import translate
from .songimport import SongImport
@ -241,7 +241,7 @@ class OpenOfficeImport(SongImport):
:param text: The text.
"""
song_texts = self.tidy_text(text).split('\f')
song_texts = normalize_str(text).split('\f')
self.set_defaults()
for song_text in song_texts:
if song_text.strip():

View File

@ -25,6 +25,7 @@ import re
from lxml import objectify
from lxml.etree import Error, LxmlError
from openlp.core.common import normalize_str
from openlp.core.common.i18n import translate
from openlp.core.common.settings import Settings
from openlp.plugins.songs.lib import VerseType
@ -262,7 +263,7 @@ class OpenSongImport(SongImport):
post=this_line[offset + column:])
offset += len(chord) + 2
# Tidy text and remove the ____s from extended words
this_line = self.tidy_text(this_line)
this_line = normalize_str(this_line)
this_line = this_line.replace('_', '')
this_line = this_line.replace('||', '\n[---]\n')
this_line = this_line.strip()

View File

@ -25,6 +25,7 @@ import re
from PyQt5 import QtCore
from openlp.core.common import normalize_str
from openlp.core.common.applocation import AppLocation
from openlp.core.common.i18n import translate
from openlp.core.common.path import copyfile, create_paths
@ -130,26 +131,6 @@ class SongImport(QtCore.QObject):
def register(self, import_wizard):
self.import_wizard = import_wizard
def tidy_text(self, text):
"""
Get rid of some dodgy unicode and formatting characters we're not interested in. Some can be converted to ascii.
"""
text = text.replace('\u2018', '\'')
text = text.replace('\u2019', '\'')
text = text.replace('\u201c', '"')
text = text.replace('\u201d', '"')
text = text.replace('\u2026', '...')
text = text.replace('\u2013', '-')
text = text.replace('\u2014', '-')
# Replace vertical tab with 2 linebreaks
text = text.replace('\v', '\n\n')
# Replace form feed (page break) with 2 linebreaks
text = text.replace('\f', '\n\n')
# Remove surplus blank lines, spaces, trailing/leading spaces
text = re.sub(r'[ \t]+', ' ', text)
text = re.sub(r' ?(\r\n?|\n) ?', '\n', text)
return text
def process_song_text(self, text):
"""
Process the song text from import
@ -368,7 +349,7 @@ class SongImport(QtCore.QObject):
verse_tag = VerseType.tags[VerseType.Other]
log.info('Versetype {old} changing to {new}'.format(old=verse_def, new=new_verse_def))
verse_def = new_verse_def
sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], verse_text, lang)
sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], normalize_str(verse_text), lang)
song.lyrics = str(sxml.extract_xml(), 'utf-8')
if not self.verse_order_list and self.verse_order_list_generated_useful:
self.verse_order_list = self.verse_order_list_generated

View File

@ -194,7 +194,6 @@ class SongsOfFellowshipImport(OpenOfficeImport):
:param text_portion: A Piece of text
"""
text = text_portion.getString()
text = self.tidy_text(text)
if text.strip() == '':
return text
if text_portion.CharWeight == BOLD:

View File

@ -30,9 +30,6 @@ from openlp.plugins.songs.lib.importers.songimport import SongImport
log = logging.getLogger(__name__)
# Used to strip control chars (except 10=LF, 13=CR)
CONTROL_CHARS_MAP = dict.fromkeys(list(range(10)) + [11, 12] + list(range(14, 32)) + [127])
class ZionWorxImport(SongImport):
"""
@ -95,12 +92,12 @@ class ZionWorxImport(SongImport):
return
self.set_defaults()
try:
self.title = self._decode(record['Title1'])
self.title = record['Title1']
if record['Title2']:
self.alternate_title = self._decode(record['Title2'])
self.parse_author(self._decode(record['Writer']))
self.add_copyright(self._decode(record['Copyright']))
lyrics = self._decode(record['Lyrics'])
self.alternate_title = record['Title2']
self.parse_author(record['Writer'])
self.add_copyright(record['Copyright'])
lyrics = record['Lyrics']
except UnicodeDecodeError as e:
self.log_error(translate('SongsPlugin.ZionWorxImport', 'Record {index}').format(index=index),
translate('SongsPlugin.ZionWorxImport', 'Decoding error: {error}').format(error=e))
@ -122,10 +119,3 @@ class ZionWorxImport(SongImport):
if not self.finish():
self.log_error(translate('SongsPlugin.ZionWorxImport', 'Record %d') % index +
(': "' + title + '"' if title else ''))
def _decode(self, str):
"""
Strips all control characters (except new lines).
"""
# ZionWorx has no option for setting the encoding for its songs, so we assume encoding is always the same.
return str.translate(CONTROL_CHARS_MAP)

View File

@ -281,7 +281,7 @@ class OpenLyrics(object):
# Process the formatting tags.
# Have we any tags in song lyrics?
tags_element = None
match = re.search('\{/?\w+\}', song.lyrics, re.UNICODE)
match = re.search(r'\{/?\w+\}', song.lyrics)
if match:
# Named 'format_' - 'format' is built-in function in Python.
format_ = etree.SubElement(song_xml, 'format')

View File

@ -54,8 +54,14 @@ class SongUsageDetailForm(QtWidgets.QDialog, Ui_SongUsageDetailDialog, RegistryP
"""
We need to set up the screen
"""
self.from_date_calendar.setSelectedDate(Settings().value(self.plugin.settings_section + '/from date'))
self.to_date_calendar.setSelectedDate(Settings().value(self.plugin.settings_section + '/to date'))
to_date = Settings().value(self.plugin.settings_section + '/to date')
if not (isinstance(to_date, QtCore.QDate) and to_date.isValid()):
to_date = QtCore.QDate.currentDate()
from_date = Settings().value(self.plugin.settings_section + '/from date')
if not (isinstance(from_date, QtCore.QDate) and from_date.isValid()):
from_date = to_date.addYears(-1)
self.from_date_calendar.setSelectedDate(from_date)
self.to_date_calendar.setSelectedDate(to_date)
self.report_path_edit.path = Settings().value(self.plugin.settings_section + '/last directory export')
def on_report_path_edit_path_changed(self, file_path):

View File

@ -38,20 +38,17 @@ from openlp.plugins.songusage.lib.db import init_schema, SongUsageItem
log = logging.getLogger(__name__)
YEAR = QtCore.QDate().currentDate().year()
if QtCore.QDate().currentDate().month() < 9:
YEAR -= 1
TODAY = QtCore.QDate.currentDate()
__default_settings__ = {
'songusage/db type': 'sqlite',
'songusage/db username': '',
'songuasge/db password': '',
'songuasge/db hostname': '',
'songuasge/db database': '',
'songusage/db password': '',
'songusage/db hostname': '',
'songusage/db database': '',
'songusage/active': False,
'songusage/to date': QtCore.QDate(YEAR, 8, 31),
'songusage/from date': QtCore.QDate(YEAR - 1, 9, 1),
'songusage/to date': TODAY,
'songusage/from date': TODAY.addYears(-1),
'songusage/last directory export': None
}

View File

@ -21,35 +21,35 @@
# Temple Place, Suite 330, Boston, MA 02111-1307 USA #
###############################################################################
"""
This script helps to trigger builds of branches. To use it you have to install the jenkins-webapi package:
This script helps to trigger builds of branches. To use it you have to install the python-jenkins module. On Fedora
and Ubuntu/Debian, it is available as the ``python3-jenkins`` package::
pip3 install jenkins-webapi
$ sudo dnf/apt install python3-jenkins
You probably want to create an alias. Add this to your ~/.bashrc file and then logout and login (to apply the alias):
To make it easier to run you may want to create a shell script or an alias. To create an alias, add this to your
``~/.bashrc`` (or ``~/.zshrc``) file and then log out and log back in again (to apply the alias)::
alias ci="python3 ./scripts/jenkins_script.py TOKEN"
alias ci="python3 /path/to/openlp_root/scripts/jenkins_script.py -u USERNAME -p PASSWORD"
You can look up the token in the Branch-01-Pull job configuration or ask in IRC.
To create a shell script, create the following file in a location in your ``$PATH`` (I called mine ``ci``)::
#!/bin/bash
python3 /path/to/openlp_root/scripts/jenkins_script.py -u USERNAME -p PASSWORD
``USERNAME`` is your Jenkins username, and ``PASSWORD`` is your Jenkins password or personal token.
An older version of this script used to use a shared TOKEN, but this has been replaced with the username and password.
"""
import os
import re
import sys
import time
from optparse import OptionParser
from argparse import ArgumentParser
from subprocess import Popen, PIPE
import warnings
from requests.exceptions import HTTPError
from jenkins import Jenkins
JENKINS_URL = 'https://ci.openlp.io/'
REPO_REGEX = r'(.*/+)(~.*)'
# Allows us to black list token. So when we change the token, we can display a proper message to the user.
OLD_TOKENS = []
# Disable the InsecureRequestWarning we get from urllib3, because we're not verifying our own self-signed certificate
warnings.simplefilter('ignore')
class OpenLPJobs(object):
@ -85,13 +85,23 @@ class JenkinsTrigger(object):
:param token: The token we need to trigger the build. If you do not have this token, ask in IRC.
"""
def __init__(self, token):
def __init__(self, username, password, can_use_colour):
"""
Create the JenkinsTrigger instance.
"""
self.token = token
self.jobs = {}
self.can_use_colour = can_use_colour and not os.name.startswith('nt')
self.repo_name = get_repo_name()
self.jenkins_instance = Jenkins(JENKINS_URL)
self.server = Jenkins(JENKINS_URL, username=username, password=password)
def fetch_jobs(self):
"""
Get the job info for all the jobs
"""
for job_name in OpenLPJobs.Jobs:
job_info = self.server.get_job_info(job_name)
self.jobs[job_name] = job_info
self.jobs[job_name]['nextBuildUrl'] = '{url}{nextBuildNumber}/'.format(**job_info)
def trigger_build(self):
"""
@ -102,15 +112,15 @@ class JenkinsTrigger(object):
# We just want the name (not the email).
name = ' '.join(raw_output.decode().split()[:-1])
cause = 'Build triggered by %s (%s)' % (name, self.repo_name)
self.jenkins_instance.job(OpenLPJobs.Branch_Pull).build({'BRANCH_NAME': self.repo_name, 'cause': cause},
token=self.token)
self.fetch_jobs()
self.server.build_job(OpenLPJobs.Branch_Pull, {'BRANCH_NAME': self.repo_name, 'cause': cause})
def print_output(self):
"""
Print the status information of the build triggered.
"""
print('Add this to your merge proposal:')
print('--------------------------------')
print('-' * 80)
bzr = Popen(('bzr', 'revno'), stdout=PIPE, stderr=PIPE)
raw_output, error = bzr.communicate()
revno = raw_output.decode().strip()
@ -118,7 +128,10 @@ class JenkinsTrigger(object):
for job in OpenLPJobs.Jobs:
if not self.__print_build_info(job):
print('Stopping after failure')
if self.current_build:
print('Stopping after failure, see {}console for more details'.format(self.current_build['url']))
else:
print('Stopping after failure')
break
def open_browser(self):
@ -129,6 +142,20 @@ class JenkinsTrigger(object):
# Open the url
Popen(('xdg-open', url), stderr=PIPE)
def _get_build_info(self, job_name, build_number):
"""
Get the build info from the server. This method will check the queue and wait for the build.
"""
queue_info = self.server.get_queue_info()
tries = 0
while queue_info and tries < 50:
tries += 1
time.sleep(0.5)
queue_info = self.server.get_queue_info()
if tries >= 50:
raise Exception('Build has not started yet, it may be stuck in the queue.')
return self.server.get_build_info(job_name, build_number)
def __print_build_info(self, job_name):
"""
This helper method prints the job information of the given ``job_name``
@ -136,21 +163,24 @@ class JenkinsTrigger(object):
:param job_name: The name of the job we want the information from. For example *Branch-01-Pull*. Use the class
variables from the :class:`OpenLPJobs` class.
"""
job = self.jobs[job_name]
print('{:<70} [WAITING]'.format(job['nextBuildUrl']), end='', flush=True)
self.current_build = self._get_build_info(job_name, job['nextBuildNumber'])
print('\b\b\b\b\b\b\b\b\b[RUNNING]', end='', flush=True)
is_success = False
job = self.jenkins_instance.job(job_name)
while job.info['inQueue']:
time.sleep(1)
build = job.last_build
build.wait()
if build.info['result'] == 'SUCCESS':
# Make 'SUCCESS' green.
result_string = '%s%s%s' % (Colour.GREEN_START, build.info['result'], Colour.GREEN_END)
is_success = True
else:
# Make 'FAILURE' red.
result_string = '%s%s%s' % (Colour.RED_START, build.info['result'], Colour.RED_END)
url = build.info['url']
print('[%s] %s' % (result_string, url))
while self.current_build['building'] is True:
time.sleep(0.5)
self.current_build = self.server.get_build_info(job_name, job['nextBuildNumber'])
result_string = self.current_build['result']
is_success = result_string == 'SUCCESS'
if self.can_use_colour:
if is_success:
# Make 'SUCCESS' green.
result_string = '{}{}{}'.format(Colour.GREEN_START, result_string, Colour.GREEN_END)
else:
# Make 'FAILURE' red.
result_string = '{}{}{}'.format(Colour.RED_START, result_string, Colour.RED_END)
print('\b\b\b\b\b\b\b\b\b[{:>7}]'.format(result_string))
return is_success
@ -186,36 +216,29 @@ def get_repo_name():
def main():
usage = 'Usage: python %prog TOKEN [options]'
"""
Run the script
"""
parser = ArgumentParser()
parser.add_argument('-d', '--disable-output', action='store_true', default=False, help='Disable output')
parser.add_argument('-b', '--open-browser', action='store_true', default=False,
help='Opens the jenkins page in your browser')
parser.add_argument('-n', '--no-colour', action='store_true', default=False,
help='Disable coloured output (always disabled on Windows)')
parser.add_argument('-u', '--username', required=True, help='Your Jenkins username')
parser.add_argument('-p', '--password', required=True, help='Your Jenkins password or personal token')
args = parser.parse_args()
parser = OptionParser(usage=usage)
parser.add_option('-d', '--disable-output', dest='enable_output', action='store_false', default=True,
help='Disable output.')
parser.add_option('-b', '--open-browser', dest='open_browser', action='store_true', default=False,
help='Opens the jenkins page in your browser.')
options, args = parser.parse_args(sys.argv)
if len(args) == 2:
if not get_repo_name():
print('Not a branch. Have you pushed it to launchpad? Did you cd to the branch?')
return
token = args[-1]
if token in OLD_TOKENS:
print('Your token is not valid anymore. Get the most recent one.')
return
jenkins_trigger = JenkinsTrigger(token)
try:
jenkins_trigger.trigger_build()
except HTTPError:
print('Wrong token.')
return
# Open the browser before printing the output.
if options.open_browser:
jenkins_trigger.open_browser()
if options.enable_output:
jenkins_trigger.print_output()
else:
parser.print_help()
if not get_repo_name():
print('Not a branch. Have you pushed it to launchpad? Did you cd to the branch?')
return
jenkins_trigger = JenkinsTrigger(args.username, args.password, not args.no_colour)
jenkins_trigger.trigger_build()
# Open the browser before printing the output.
if args.open_browser:
jenkins_trigger.open_browser()
if not args.disable_output:
jenkins_trigger.print_output()
if __name__ == '__main__':

View File

@ -153,6 +153,7 @@ class TestActionList(TestCase, TestMixin):
"""
Prepare the tests
"""
self.setup_application()
self.action_list = ActionList.get_instance()
self.build_settings()
self.settings = Settings()

View File

@ -233,7 +233,7 @@ class TestHttpUtils(TestCase, TestMixin):
Test socket timeout gets caught
"""
# GIVEN: Mocked urlopen to fake a network disconnect in the middle of a download
mocked_requests.get.side_effect = IOError
mocked_requests.get.side_effect = OSError
# WHEN: Attempt to retrieve a file
url_get_file(MagicMock(), url='http://localhost/test', file_path=Path(self.tempfile))

View File

@ -155,7 +155,7 @@ def test_check_same_instance():
assert first_instance is second_instance, 'Two UiStrings objects should be the same instance'
def test_translate(self):
def test_translate():
"""
Test the translate() function
"""

View File

@ -371,13 +371,13 @@ class TestPath(TestCase):
@patch('openlp.core.common.path.log')
def test_create_paths_dir_io_error(self, mocked_logger):
"""
Test the create_paths() when an IOError is raised
Test the create_paths() when an OSError is raised
"""
# GIVEN: A `Path` to check with patched out mkdir and exists methods
mocked_path = MagicMock()
mocked_path.exists.side_effect = IOError('Cannot make directory')
mocked_path.exists.side_effect = OSError('Cannot make directory')
# WHEN: An IOError is raised when checking the if the path exists.
# WHEN: An OSError is raised when checking the if the path exists.
create_paths(mocked_path)
# THEN: The Error should have been logged
@ -385,7 +385,7 @@ class TestPath(TestCase):
def test_create_paths_dir_value_error(self):
"""
Test the create_paths() when an error other than IOError is raised
Test the create_paths() when an error other than OSError is raised
"""
# GIVEN: A `Path` to check with patched out mkdir and exists methods
mocked_path = MagicMock()

View File

@ -168,7 +168,7 @@ class TestLib(TestCase):
patch.object(Path, 'open'):
file_path = Path('testfile.txt')
file_path.is_file.return_value = True
file_path.open.side_effect = IOError()
file_path.open.side_effect = OSError()
# WHEN: get_text_file_string is called
result = get_text_file_string(file_path)

View File

@ -40,7 +40,7 @@ class TestFirstTimeWizard(TestMixin, TestCase):
Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031
"""
# GIVEN: Initial settings and mocks
mocked_requests.get.side_effect = IOError('Unable to connect')
mocked_requests.get.side_effect = OSError('Unable to connect')
# WHEN: A webpage is requested
try:

View File

@ -627,4 +627,3 @@ class TestTreeWidgetWithDnD(TestCase):
assert widget.allow_internal_dnd is False
assert widget.indentation() == 0
assert widget.isAnimated() is True

View File

@ -144,7 +144,7 @@ class TestPresentationController(TestCase):
# GIVEN: A mocked open, get_thumbnail_folder and exists
with patch('openlp.plugins.presentations.lib.presentationcontroller.Path.read_text') as mocked_read_text, \
patch(FOLDER_TO_PATCH) as mocked_get_thumbnail_folder:
mocked_read_text.side_effect = IOError()
mocked_read_text.side_effect = OSError()
mocked_get_thumbnail_folder.return_value = Path('test')
# WHEN: calling get_titles_and_notes

View File

@ -42,8 +42,8 @@ class TestProjectorManager(TestCase, TestMixin):
"""
Create the UI and setup necessary options
"""
self.build_settings()
self.setup_application()
self.build_settings()
Registry.create()
with patch('openlp.core.lib.projector.db.init_url') as mocked_init_url:
if os.path.exists(TEST_DB):

View File

@ -64,8 +64,8 @@ class ProjectorSourceFormTest(TestCase, TestMixin):
Set up anything necessary for all tests
"""
mocked_init_url.return_value = 'sqlite:///{}'.format(TEST_DB)
self.build_settings()
self.setup_application()
self.build_settings()
Registry.create()
# Do not try to recreate if we've already been created from a previous test
if not hasattr(self, 'projectordb'):

View File

@ -41,8 +41,8 @@ class TestThemeManager(TestCase, TestMixin):
"""
Create the UI
"""
self.build_settings()
self.setup_application()
self.build_settings()
Registry.create()
self.theme_manager = ThemeManager()

View File

@ -36,7 +36,7 @@ def convert_file_service_item(test_path, name, row=0):
try:
items = json.load(open_file)
first_line = items[row]
except IOError:
except OSError:
first_line = ''
finally:
open_file.close()

View File

@ -58,17 +58,21 @@ class TestPylint(TestCase):
# GIVEN: Some checks to disable and enable, and the pylint script
disabled_checks = 'import-error,no-member'
enabled_checks = 'missing-format-argument-key,unused-format-string-argument,bad-format-string'
if is_win() or 'arch' in platform.dist()[0].lower():
pylint_script = 'pylint'
else:
pylint_script = 'pylint3'
pylint_kwargs = {
'return_std': True
}
if version < '1.7.0':
if is_win() or 'arch' in platform.dist()[0].lower():
pylint_kwargs.update({'script': 'pylint'})
else:
pylint_kwargs.update({'script': 'pylint3'})
# WHEN: Running pylint
(pylint_stdout, pylint_stderr) = \
lint.py_run('openlp --errors-only --disable={disabled} --enable={enabled} '
'--reports=no --output-format=parseable'.format(disabled=disabled_checks,
enabled=enabled_checks),
return_std=True, script=pylint_script)
**pylint_kwargs)
stdout = pylint_stdout.read()
stderr = pylint_stderr.read()
filtered_stdout = self._filter_tolerated_errors(stdout)