Merge in Phil's changes

openlp/core/common/__init__.py

@@ -43,9 +43,13 @@ log = logging.getLogger(__name__ + '.__init__')
 
 FIRST_CAMEL_REGEX = re.compile('(.)([A-Z][a-z]+)')
 SECOND_CAMEL_REGEX = re.compile('([a-z0-9])([A-Z])')
-CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]', re.UNICODE)
+CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]')
-INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]', re.UNICODE)
+INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]')
 IMAGES_FILTER = None
+REPLACMENT_CHARS_MAP = str.maketrans({'\u2018': '\'', '\u2019': '\'', '\u201c': '"', '\u201d': '"', '\u2026': '...',
+                                      '\u2013': '-', '\u2014': '-', '\v': '\n\n', '\f': '\n\n'})
+NEW_LINE_REGEX = re.compile(r' ?(\r\n?|\n) ?')
+WHITESPACE_REGEX = re.compile(r'[ \t]+')
 
 
 def trace_error_handler(logger):
@@ -339,7 +343,7 @@ def delete_file(file_path):
         if file_path.exists():
             file_path.unlink()
         return True
-    except (IOError, OSError):
+    except OSError:
         log.exception('Unable to delete file {file_path}'.format(file_path=file_path))
         return False
 
@@ -436,3 +440,17 @@ def get_file_encoding(file_path):
         return detector.result
     except OSError:
         log.exception('Error detecting file encoding')
+
+
+def normalize_str(irreg_str):
+    """
+    Normalize the supplied string. Remove unicode control chars and tidy up white space.
+
+    :param str irreg_str: The string to normalize.
+    :return: The normalized string
+    :rtype: str
+    """
+    irreg_str = irreg_str.translate(REPLACMENT_CHARS_MAP)
+    irreg_str = CONTROL_CHARS.sub('', irreg_str)
+    irreg_str = NEW_LINE_REGEX.sub('\n', irreg_str)
+    return WHITESPACE_REGEX.sub(' ', irreg_str)

openlp/core/common/applocation.py

@@ -83,7 +83,7 @@ class AppLocation(object):
         """
         # Check if we have a different data location.
         if Settings().contains('advanced/data path'):
-            path = Settings().value('advanced/data path')
+            path = Path(Settings().value('advanced/data path'))
         else:
             path = AppLocation.get_directory(AppLocation.DataDir)
             create_paths(path)

openlp/core/common/httputils.py

@@ -97,8 +97,8 @@ def get_web_page(url, headers=None, update_openlp=False, proxies=None):
             response = requests.get(url, headers=headers, proxies=proxies, timeout=float(CONNECTION_TIMEOUT))
             log.debug('Downloaded page {url}'.format(url=response.url))
             break
-        except IOError:
+        except OSError:
-            # For now, catch IOError. All requests errors inherit from IOError
+            # For now, catch OSError. All requests errors inherit from OSError
             log.exception('Unable to connect to {url}'.format(url=url))
             response = None
             if retries >= CONNECTION_RETRIES:
@@ -127,7 +127,7 @@ def get_url_file_size(url):
         try:
             response = requests.head(url, timeout=float(CONNECTION_TIMEOUT), allow_redirects=True)
             return int(response.headers['Content-Length'])
-        except IOError:
+        except OSError:
             if retries > CONNECTION_RETRIES:
                 raise ConnectionError('Unable to download {url}'.format(url=url))
             else:
@@ -173,7 +173,7 @@ def url_get_file(callback, url, file_path, sha256=None):
                     file_path.unlink()
                 return False
             break
-        except IOError:
+        except OSError:
             trace_error_handler(log)
             if retries > CONNECTION_RETRIES:
                 if file_path.exists():

openlp/core/common/i18n.py

@@ -53,7 +53,7 @@ def translate(context, text, comment=None, qt_translate=QtCore.QCoreApplication.
 
 Language = namedtuple('Language', ['id', 'name', 'code'])
 ICU_COLLATOR = None
-DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+', re.UNICODE)
+DIGITS_OR_NONDIGITS = re.compile(r'\d+|\D+')
 LANGUAGES = sorted([
     Language(1, translate('common.languages', '(Afan) Oromo', 'Language code: om'), 'om'),
     Language(2, translate('common.languages', 'Abkhazian', 'Language code: ab'), 'ab'),

openlp/core/common/path.py

@@ -233,7 +233,7 @@ def create_paths(*paths, **kwargs):
         try:
             if not path.exists():
                 path.mkdir(parents=True)
-        except IOError:
+        except OSError:
             if not kwargs.get('do_not_log', False):
                 log.exception('failed to check if directory exists or create directory')
 

openlp/core/common/settings.py

@@ -258,6 +258,12 @@ class Settings(QtCore.QSettings):
         ('media/last directory', 'media/last directory', [(str_to_path, None)])
     ]
 
+    __setting_upgrade_3__ = [
+        ('songuasge/db password', 'songusage/db password', []),
+        ('songuasge/db hostname', 'songusage/db hostname', []),
+        ('songuasge/db database', 'songusage/db database', [])
+    ]
+
     @staticmethod
     def extend_default_settings(default_values):
         """

openlp/core/lib/__init__.py

@@ -104,7 +104,7 @@ def get_text_file_string(text_file_path):
                 # no BOM was found
                 file_handle.seek(0)
             content = file_handle.read()
-    except (IOError, UnicodeError):
+    except (OSError, UnicodeError):
         log.exception('Failed to open text file {text}'.format(text=text_file_path))
     return content
 

openlp/core/lib/mediamanageritem.py

@@ -92,7 +92,7 @@ class MediaManagerItem(QtWidgets.QWidget, RegistryProperties):
         Run some initial setup. This method is separate from __init__ in order to mock it out in tests.
         """
         self.hide()
-        self.whitespace = re.compile(r'[\W_]+', re.UNICODE)
+        self.whitespace = re.compile(r'[\W_]+')
         visible_title = self.plugin.get_string(StringContent.VisibleName)
         self.title = str(visible_title['title'])
         Registry().register(self.plugin.name, self)
@@ -344,7 +344,9 @@ class MediaManagerItem(QtWidgets.QWidget, RegistryProperties):
             else:
                 new_files.append(file_name)
         if new_files:
-            self.validate_and_load(new_files, data['target'])
+            if 'target' in data:
+                self.validate_and_load(new_files, data['target'])
+            self.validate_and_load(new_files)
 
     def dnd_move_internal(self, target):
         """

openlp/core/ui/exceptionform.py

@@ -155,7 +155,7 @@ class ExceptionForm(QtWidgets.QDialog, Ui_ExceptionDialog, RegistryProperties):
             try:
                 with file_path.open('w') as report_file:
                     report_file.write(report_text)
-            except IOError:
+            except OSError:
                 log.exception('Failed to write crash report')
 
     def on_send_report_button_clicked(self):

openlp/core/ui/formattingtagcontroller.py

@@ -43,7 +43,7 @@ class FormattingTagController(object):
             r'(?P<tag>[^\s/!\?>]+)(?:\s+[^\s=]+="[^"]*")*\s*(?P<empty>/)?'
             r'|(?P<cdata>!\[CDATA\[(?:(?!\]\]>).)*\]\])'
             r'|(?P<procinst>\?(?:(?!\?>).)*\?)'
-            r'|(?P<comment>!--(?:(?!-->).)*--))>', re.UNICODE)
+            r'|(?P<comment>!--(?:(?!-->).)*--))>')
         self.html_regex = re.compile(r'^(?:[^<>]*%s)*[^<>]*$' % self.html_tag_regex.pattern)
 
     def pre_save(self):

openlp/core/ui/mainwindow.py

@@ -180,7 +180,7 @@ class Ui_MainWindow(object):
                                             triggers=self.service_manager_contents.on_load_service_clicked)
         self.file_save_item = create_action(main_window, 'fileSaveItem', icon=':/general/general_save.png',
                                             can_shortcuts=True, category=UiStrings().File,
-                                            triggers=self.service_manager_contents.save_file)
+                                            triggers=self.service_manager_contents.decide_save_method)
         self.file_save_as_item = create_action(main_window, 'fileSaveAsItem', can_shortcuts=True,
                                                category=UiStrings().File,
                                                triggers=self.service_manager_contents.save_file_as)
@@ -1367,7 +1367,7 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow, RegistryProperties):
                               '- Please wait for copy to finish').format(path=self.new_data_path))
                 dir_util.copy_tree(str(old_data_path), str(self.new_data_path))
                 log.info('Copy successful')
-            except (IOError, os.error, DistutilsFileError) as why:
+            except (OSError, DistutilsFileError) as why:
                 self.application.set_normal_cursor()
                 log.exception('Data copy failed {err}'.format(err=str(why)))
                 err_text = translate('OpenLP.MainWindow',

openlp/core/ui/servicemanager.py

@@ -193,18 +193,6 @@ class Ui_ServiceManager(object):
             text=translate('OpenLP.ServiceManager', 'Move to &bottom'), icon=':/services/service_bottom.png',
             tooltip=translate('OpenLP.ServiceManager', 'Move item to the end of the service.'),
             can_shortcuts=True, category=UiStrings().Service, triggers=self.on_service_end)
-        self.down_action = self.order_toolbar.add_toolbar_action(
-            'down',
-            text=translate('OpenLP.ServiceManager', 'Move &down'), can_shortcuts=True,
-            tooltip=translate('OpenLP.ServiceManager', 'Moves the selection down the window.'), visible=False,
-            triggers=self.on_move_selection_down)
-        action_list.add_action(self.down_action)
-        self.up_action = self.order_toolbar.add_toolbar_action(
-            'up',
-            text=translate('OpenLP.ServiceManager', 'Move up'), can_shortcuts=True,
-            tooltip=translate('OpenLP.ServiceManager', 'Moves the selection up the window.'), visible=False,
-            triggers=self.on_move_selection_up)
-        action_list.add_action(self.up_action)
         self.order_toolbar.addSeparator()
         self.delete_action = self.order_toolbar.add_toolbar_action(
             'delete', can_shortcuts=True,
@@ -300,8 +288,8 @@ class Ui_ServiceManager(object):
         self.theme_menu = QtWidgets.QMenu(translate('OpenLP.ServiceManager', '&Change Item Theme'))
         self.menu.addMenu(self.theme_menu)
         self.service_manager_list.addActions([self.move_down_action, self.move_up_action, self.make_live_action,
-                                              self.move_top_action, self.move_bottom_action, self.up_action,
+                                              self.move_top_action, self.move_bottom_action, self.expand_action,
-                                              self.down_action, self.expand_action, self.collapse_action])
+                                              self.collapse_action])
         Registry().register_function('theme_update_list', self.update_theme_list)
         Registry().register_function('config_screen_changed', self.regenerate_service_items)
         Registry().register_function('theme_update_global', self.theme_change)
@@ -474,6 +462,12 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
         Load a recent file as the service triggered by mainwindow recent service list.
         :param field:
         """
+        if self.is_modified():
+            result = self.save_modified_service()
+            if result == QtWidgets.QMessageBox.Cancel:
+                return False
+            elif result == QtWidgets.QMessageBox.Save:
+                self.decide_save_method()
         sender = self.sender()
         self.load_file(sender.data())
 
@@ -603,7 +597,7 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
                 if not os.path.exists(save_file):
                     shutil.copy(audio_from, save_file)
                 zip_file.write(audio_from, audio_to)
-        except IOError:
+        except OSError:
             self.log_exception('Failed to save service to disk: {name}'.format(name=temp_file_name))
             self.main_window.error_message(translate('OpenLP.ServiceManager', 'Error Saving File'),
                                            translate('OpenLP.ServiceManager', 'There was an error saving your file.'))
@@ -664,7 +658,7 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
             zip_file = zipfile.ZipFile(temp_file_name, 'w', zipfile.ZIP_STORED, True)
             # First we add service contents.
             zip_file.writestr(service_file_name, service_content)
-        except IOError:
+        except OSError:
             self.log_exception('Failed to save service to disk: {name}'.format(name=temp_file_name))
             self.main_window.error_message(translate('OpenLP.ServiceManager', 'Error Saving File'),
                                            translate('OpenLP.ServiceManager', 'There was an error saving your file.'))
@@ -712,18 +706,23 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
             default_file_path = directory_path / default_file_path
         # SaveAs from osz to oszl is not valid as the files will be deleted on exit which is not sensible or usable in
         # the long term.
+        lite_filter = translate('OpenLP.ServiceManager', 'OpenLP Service Files - lite (*.oszl)')
+        packaged_filter = translate('OpenLP.ServiceManager', 'OpenLP Service Files (*.osz)')
+
         if self._file_name.endswith('oszl') or self.service_has_all_original_files:
             file_path, filter_used = FileDialog.getSaveFileName(
                 self.main_window, UiStrings().SaveService, default_file_path,
-                translate('OpenLP.ServiceManager',
+                '{packaged};; {lite}'.format(packaged=packaged_filter, lite=lite_filter))
-                          'OpenLP Service Files (*.osz);; OpenLP Service Files - lite (*.oszl)'))
         else:
             file_path, filter_used = FileDialog.getSaveFileName(
-                self.main_window, UiStrings().SaveService, file_path,
+                self.main_window, UiStrings().SaveService, default_file_path,
-                translate('OpenLP.ServiceManager', 'OpenLP Service Files (*.osz);;'))
+                '{packaged};;'.format(packaged=packaged_filter))
         if not file_path:
             return False
-        file_path.with_suffix('.osz')
+        if filter_used == lite_filter:
+            file_path = file_path.with_suffix('.oszl')
+        else:
+            file_path = file_path.with_suffix('.osz')
         self.set_file_name(file_path)
         self.decide_save_method()
 
@@ -791,11 +790,11 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
             else:
                 critical_error_message_box(message=translate('OpenLP.ServiceManager', 'File is not a valid service.'))
                 self.log_error('File contains no service data')
-        except (IOError, NameError):
+        except (OSError, NameError):
             self.log_exception('Problem loading service file {name}'.format(name=file_name))
             critical_error_message_box(message=translate('OpenLP.ServiceManager',
                                        'File could not be opened because it is corrupt.'))
-        except zipfile.BadZipfile:
+        except zipfile.BadZipFile:
             if os.path.getsize(file_name) == 0:
                 self.log_exception('Service file is zero sized: {name}'.format(name=file_name))
                 QtWidgets.QMessageBox.information(self, translate('OpenLP.ServiceManager', 'Empty File'),
@@ -1657,14 +1656,15 @@ class ServiceManager(QtWidgets.QWidget, RegistryBase, Ui_ServiceManager, LogMixi
                 if start_pos == -1:
                     return
                 if item is None:
-                    end_pos = len(self.service_items)
+                    end_pos = len(self.service_items) - 1
                 else:
                     end_pos = get_parent_item_data(item) - 1
                 service_item = self.service_items[start_pos]
-                self.service_items.remove(service_item)
+                if start_pos != end_pos:
-                self.service_items.insert(end_pos, service_item)
+                    self.service_items.remove(service_item)
-                self.repaint_service_list(end_pos, child)
+                    self.service_items.insert(end_pos, service_item)
-                self.set_modified()
+                    self.repaint_service_list(end_pos, child)
+                    self.set_modified()
             else:
                 # we are not over anything so drop
                 replace = False

openlp/core/ui/thememanager.py

@@ -604,7 +604,7 @@ class ThemeManager(QtWidgets.QWidget, RegistryBase, Ui_ThemeManager, LogMixin, R
                     else:
                         with full_name.open('wb') as out_file:
                             out_file.write(theme_zip.read(zipped_file))
-        except (IOError, zipfile.BadZipfile):
+        except (OSError, zipfile.BadZipFile):
             self.log_exception('Importing theme from zip failed {name}'.format(name=file_path))
             raise ValidationError
         except ValidationError:
@@ -667,7 +667,7 @@ class ThemeManager(QtWidgets.QWidget, RegistryBase, Ui_ThemeManager, LogMixin, R
         theme_path = theme_dir / '{file_name}.json'.format(file_name=name)
         try:
                 theme_path.write_text(theme_pretty)
-        except IOError:
+        except OSError:
             self.log_exception('Saving theme to file failed')
         if image_source_path and image_destination_path:
             if self.old_background_image_path and image_destination_path != self.old_background_image_path:
@@ -675,7 +675,7 @@ class ThemeManager(QtWidgets.QWidget, RegistryBase, Ui_ThemeManager, LogMixin, R
             if image_source_path != image_destination_path:
                 try:
                     copyfile(image_source_path, image_destination_path)
-                except IOError:
+                except OSError:
                     self.log_exception('Failed to save theme image')
         self.generate_and_save_image(name, theme)
 

openlp/core/version.py

@@ -96,7 +96,7 @@ class VersionWorker(QtCore.QObject):
                 remote_version = response.text
                 log.debug('New version found: %s', remote_version)
                 break
-            except IOError:
+            except OSError:
                 log.exception('Unable to connect to OpenLP server to download version file')
                 retries += 1
         else:
@@ -182,7 +182,7 @@ def get_version():
         try:
             version_file = open(file_path, 'r')
             full_version = str(version_file.read()).rstrip()
-        except IOError:
+        except OSError:
             log.exception('Error in version file.')
             full_version = '0.0.0-bzr000'
         finally:

openlp/core/widgets/edits.py

@@ -27,6 +27,7 @@ import re
 
 from PyQt5 import QtCore, QtGui, QtWidgets
 
+from openlp.core.common import CONTROL_CHARS
 from openlp.core.common.i18n import UiStrings, translate
 from openlp.core.common.path import Path, path_to_str, str_to_path
 from openlp.core.common.settings import Settings
@@ -241,7 +242,7 @@ class PathEdit(QtWidgets.QWidget):
         self.line_edit.editingFinished.connect(self.on_line_edit_editing_finished)
         self.update_button_tool_tips()
 
-    @property
+    @QtCore.pyqtProperty('QVariant')
     def path(self):
         """
         A property getter method to return the selected path.
@@ -349,7 +350,7 @@ class PathEdit(QtWidgets.QWidget):
         :rtype: None
         """
         if self._path != path:
-            self.path = path
+            self._path = path
             self.pathChanged.emit(path)
 
 
@@ -470,12 +471,21 @@ class SpellTextEdit(QtWidgets.QPlainTextEdit):
                     cursor.insertText(html['start tag'])
                     cursor.insertText(html['end tag'])
 
+    def insertFromMimeData(self, source):
+        """
+        Reimplement `insertFromMimeData` so that we can remove any control characters
+
+        :param QtCore.QMimeData source: The mime data to insert
+        :rtype: None
+        """
+        self.insertPlainText(CONTROL_CHARS.sub('', source.text()))
+
 
 class Highlighter(QtGui.QSyntaxHighlighter):
     """
     Provides a text highlighter for pointing out spelling errors in text.
     """
-    WORDS = r'(?iu)[\w\']+'
+    WORDS = r'(?i)[\w\']+'
 
     def __init__(self, *args):
         """

openlp/core/widgets/views.py

@@ -336,7 +336,7 @@ class ListWidgetWithDnD(QtWidgets.QListWidget):
                     for file in listing:
                         files.append(os.path.join(local_file, file))
             Registry().execute('{mime_data}_dnd'.format(mime_data=self.mime_data_text),
-                               {'files': files, 'target': self.itemAt(event.pos())})
+                               {'files': files})
         else:
             event.ignore()
 

openlp/plugins/bibles/forms/booknameform.py

@@ -113,8 +113,7 @@ class BookNameForm(QDialog, Ui_BookNameDialog):
             cor_book = self.corresponding_combo_box.currentText()
             for character in '\\.^$*+?{}[]()':
                 cor_book = cor_book.replace(character, '\\' + character)
-            books = [key for key in list(self.book_names.keys()) if re.match(cor_book, str(self.book_names[key]),
+            books = [key for key in list(self.book_names.keys()) if re.match(cor_book, str(self.book_names[key]))]
-                                                                             re.UNICODE)]
             books = [_f for _f in map(BiblesResourcesDB.get_book, books) if _f]
             if books:
                 self.book_id = books[0]['id']

openlp/plugins/bibles/lib/__init__.py

@@ -224,13 +224,13 @@ def update_reference_separators():
     range_regex = '(?:(?P<from_chapter>[0-9]+){sep_v})?' \
         '(?P<from_verse>[0-9]+)(?P<range_to>{sep_r}(?:(?:(?P<to_chapter>' \
         '[0-9]+){sep_v})?(?P<to_verse>[0-9]+)|{sep_e})?)?'.format_map(REFERENCE_SEPARATORS)
-    REFERENCE_MATCHES['range'] = re.compile(r'^\s*{range}\s*$'.format(range=range_regex), re.UNICODE)
+    REFERENCE_MATCHES['range'] = re.compile(r'^\s*{range}\s*$'.format(range=range_regex))
-    REFERENCE_MATCHES['range_separator'] = re.compile(REFERENCE_SEPARATORS['sep_l'], re.UNICODE)
+    REFERENCE_MATCHES['range_separator'] = re.compile(REFERENCE_SEPARATORS['sep_l'])
     # full reference match: <book>(<range>(,(?!$)|(?=$)))+
     REFERENCE_MATCHES['full'] = \
         re.compile(r'^\s*(?!\s)(?P<book>[\d]*[.]?[^\d\.]+)\.*(?<!\s)\s*'
                    r'(?P<ranges>(?:{range_regex}(?:{sep_l}(?!\s*$)|(?=\s*$)))+)\s*$'.format(
-                       range_regex=range_regex, sep_l=REFERENCE_SEPARATORS['sep_l']), re.UNICODE)
+                       range_regex=range_regex, sep_l=REFERENCE_SEPARATORS['sep_l']))
 
 
 def get_reference_separator(separator_type):

openlp/plugins/bibles/lib/db.py

@@ -307,8 +307,7 @@ class BibleDB(Manager):
         book_escaped = book
         for character in RESERVED_CHARACTERS:
             book_escaped = book_escaped.replace(character, '\\' + character)
-        regex_book = re.compile('\\s*{book}\\s*'.format(book='\\s*'.join(book_escaped.split())),
+        regex_book = re.compile('\\s*{book}\\s*'.format(book='\\s*'.join(book_escaped.split())), re.IGNORECASE)
-                                re.UNICODE | re.IGNORECASE)
         if language_selection == LanguageSelection.Bible:
             db_book = self.get_book(book)
             if db_book:

openlp/plugins/images/lib/mediaitem.py

@@ -366,7 +366,7 @@ class ImageMediaItem(MediaManagerItem):
                 if validate_thumb(image.file_path, thumbnail_path):
                     icon = build_icon(thumbnail_path)
                 else:
-                    icon = create_thumb(image.file_path, thumbnail_path)
+                    icon = create_thumb(str(image.file_path), str(thumbnail_path))
             item_name = QtWidgets.QTreeWidgetItem([file_name])
             item_name.setText(0, file_name)
             item_name.setIcon(0, icon)
@@ -390,6 +390,7 @@ class ImageMediaItem(MediaManagerItem):
         :param files: A List of strings containing the filenames of the files to be loaded
         :param target_group: The QTreeWidgetItem of the group that will be the parent of the added files
         """
+        file_paths = [Path(file) for file in file_paths]
         self.application.set_normal_cursor()
         self.load_list(file_paths, target_group)
         last_dir = file_paths[0].parent

openlp/plugins/presentations/lib/pptviewcontroller.py

@@ -70,7 +70,7 @@ class PptviewController(PresentationController):
             try:
                 self.start_process()
                 return self.process.CheckInstalled()
-            except WindowsError:
+            except OSError:
                 return False
 
         def start_process(self):

openlp/plugins/songs/forms/editsongform.py

@@ -105,9 +105,9 @@ class EditSongForm(QtWidgets.QDialog, Ui_EditSongDialog, RegistryProperties):
         self.topics_list_view.setSortingEnabled(False)
         self.topics_list_view.setAlternatingRowColors(True)
         self.audio_list_widget.setAlternatingRowColors(True)
-        self.find_verse_split = re.compile('---\[\]---\n', re.UNICODE)
+        self.find_verse_split = re.compile('---\[\]---\n')
-        self.whitespace = re.compile(r'\W+', re.UNICODE)
+        self.whitespace = re.compile(r'\W+')
-        self.find_tags = re.compile(u'\{/?\w+\}', re.UNICODE)
+        self.find_tags = re.compile(r'\{/?\w+\}')
 
     def _load_objects(self, cls, combo, cache):
         """

openlp/plugins/songs/lib/__init__.py

@@ -24,7 +24,6 @@ The :mod:`~openlp.plugins.songs.lib` module contains a number of library functio
 """
 
 import logging
-import os
 import re
 
 from PyQt5 import QtWidgets
@@ -39,8 +38,8 @@ from openlp.plugins.songs.lib.ui import SongStrings
 
 log = logging.getLogger(__name__)
 
-WHITESPACE = re.compile(r'[\W_]+', re.UNICODE)
+WHITESPACE = re.compile(r'[\W_]+')
-APOSTROPHE = re.compile('[\'`’ʻ′]', re.UNICODE)
+APOSTROPHE = re.compile(r'[\'`’ʻ′]')
 # PATTERN will look for the next occurence of one of these symbols:
 #   \controlword - optionally preceded by \*, optionally followed by a number
 #   \'## - where ## is a pair of hex digits, representing a single character

openlp/plugins/songs/lib/importers/easyslides.py

@@ -25,6 +25,7 @@ import re
 
 from lxml import etree, objectify
 
+from openlp.core.common import normalize_str
 from openlp.plugins.songs.lib import VerseType
 from openlp.plugins.songs.lib.importers.songimport import SongImport
 
@@ -225,7 +226,7 @@ class EasySlidesImport(SongImport):
                 verses[reg].setdefault(vt, {})
                 verses[reg][vt].setdefault(vn, {})
                 verses[reg][vt][vn].setdefault(inst, [])
-                verses[reg][vt][vn][inst].append(self.tidy_text(line))
+                verses[reg][vt][vn][inst].append(normalize_str(line))
         # done parsing
         versetags = []
         # we use our_verse_order to ensure, we insert lyrics in the same order

openlp/plugins/songs/lib/importers/mediashout.py

@@ -101,7 +101,7 @@ class MediaShoutImport(SongImport):
             self.song_book_name = song.SongID
         for verse in verses:
             tag = VERSE_TAGS[verse.Type] + str(verse.Number) if verse.Type < len(VERSE_TAGS) else 'O'
-            self.add_verse(self.tidy_text(verse.Text), tag)
+            self.add_verse(verse.Text, tag)
         for order in verse_order:
             if order.Type < len(VERSE_TAGS):
                 self.verse_order_list.append(VERSE_TAGS[order.Type] + str(order.Number))

openlp/plugins/songs/lib/importers/openoffice.py

@@ -24,7 +24,7 @@ import time
 
 from PyQt5 import QtCore
 
-from openlp.core.common import is_win, get_uno_command, get_uno_instance
+from openlp.core.common import get_uno_command, get_uno_instance, is_win, normalize_str
 from openlp.core.common.i18n import translate
 from .songimport import SongImport
 
@@ -241,7 +241,7 @@ class OpenOfficeImport(SongImport):
 
         :param text: The text.
         """
-        song_texts = self.tidy_text(text).split('\f')
+        song_texts = normalize_str(text).split('\f')
         self.set_defaults()
         for song_text in song_texts:
             if song_text.strip():

openlp/plugins/songs/lib/importers/opensong.py

@@ -25,6 +25,7 @@ import re
 from lxml import objectify
 from lxml.etree import Error, LxmlError
 
+from openlp.core.common import normalize_str
 from openlp.core.common.i18n import translate
 from openlp.core.common.settings import Settings
 from openlp.plugins.songs.lib import VerseType
@@ -262,7 +263,7 @@ class OpenSongImport(SongImport):
                                                               post=this_line[offset + column:])
                     offset += len(chord) + 2
             # Tidy text and remove the ____s from extended words
-            this_line = self.tidy_text(this_line)
+            this_line = normalize_str(this_line)
             this_line = this_line.replace('_', '')
             this_line = this_line.replace('||', '\n[---]\n')
             this_line = this_line.strip()

openlp/plugins/songs/lib/importers/songimport.py

@@ -25,6 +25,7 @@ import re
 
 from PyQt5 import QtCore
 
+from openlp.core.common import normalize_str
 from openlp.core.common.applocation import AppLocation
 from openlp.core.common.i18n import translate
 from openlp.core.common.path import copyfile, create_paths
@@ -130,26 +131,6 @@ class SongImport(QtCore.QObject):
     def register(self, import_wizard):
         self.import_wizard = import_wizard
 
-    def tidy_text(self, text):
-        """
-        Get rid of some dodgy unicode and formatting characters we're not interested in. Some can be converted to ascii.
-        """
-        text = text.replace('\u2018', '\'')
-        text = text.replace('\u2019', '\'')
-        text = text.replace('\u201c', '"')
-        text = text.replace('\u201d', '"')
-        text = text.replace('\u2026', '...')
-        text = text.replace('\u2013', '-')
-        text = text.replace('\u2014', '-')
-        # Replace vertical tab with 2 linebreaks
-        text = text.replace('\v', '\n\n')
-        # Replace form feed (page break) with 2 linebreaks
-        text = text.replace('\f', '\n\n')
-        # Remove surplus blank lines, spaces, trailing/leading spaces
-        text = re.sub(r'[ \t]+', ' ', text)
-        text = re.sub(r' ?(\r\n?|\n) ?', '\n', text)
-        return text
-
     def process_song_text(self, text):
         """
         Process the song text from import
@@ -368,7 +349,7 @@ class SongImport(QtCore.QObject):
                 verse_tag = VerseType.tags[VerseType.Other]
                 log.info('Versetype {old} changing to {new}'.format(old=verse_def, new=new_verse_def))
                 verse_def = new_verse_def
-            sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], verse_text, lang)
+            sxml.add_verse_to_lyrics(verse_tag, verse_def[1:], normalize_str(verse_text), lang)
         song.lyrics = str(sxml.extract_xml(), 'utf-8')
         if not self.verse_order_list and self.verse_order_list_generated_useful:
             self.verse_order_list = self.verse_order_list_generated

openlp/plugins/songs/lib/importers/songsoffellowship.py

@@ -194,7 +194,6 @@ class SongsOfFellowshipImport(OpenOfficeImport):
         :param text_portion: A Piece of text
         """
         text = text_portion.getString()
-        text = self.tidy_text(text)
         if text.strip() == '':
             return text
         if text_portion.CharWeight == BOLD:

openlp/plugins/songs/lib/importers/zionworx.py

@@ -30,9 +30,6 @@ from openlp.plugins.songs.lib.importers.songimport import SongImport
 
 log = logging.getLogger(__name__)
 
-# Used to strip control chars (except 10=LF, 13=CR)
-CONTROL_CHARS_MAP = dict.fromkeys(list(range(10)) + [11, 12] + list(range(14, 32)) + [127])
-
 
 class ZionWorxImport(SongImport):
     """
@@ -95,12 +92,12 @@ class ZionWorxImport(SongImport):
                     return
                 self.set_defaults()
                 try:
-                    self.title = self._decode(record['Title1'])
+                    self.title = record['Title1']
                     if record['Title2']:
-                        self.alternate_title = self._decode(record['Title2'])
+                        self.alternate_title = record['Title2']
-                    self.parse_author(self._decode(record['Writer']))
+                    self.parse_author(record['Writer'])
-                    self.add_copyright(self._decode(record['Copyright']))
+                    self.add_copyright(record['Copyright'])
-                    lyrics = self._decode(record['Lyrics'])
+                    lyrics = record['Lyrics']
                 except UnicodeDecodeError as e:
                     self.log_error(translate('SongsPlugin.ZionWorxImport', 'Record {index}').format(index=index),
                                    translate('SongsPlugin.ZionWorxImport', 'Decoding error: {error}').format(error=e))
@@ -122,10 +119,3 @@ class ZionWorxImport(SongImport):
                 if not self.finish():
                     self.log_error(translate('SongsPlugin.ZionWorxImport', 'Record %d') % index +
                                    (': "' + title + '"' if title else ''))
-
-    def _decode(self, str):
-        """
-        Strips all control characters (except new lines).
-        """
-        # ZionWorx has no option for setting the encoding for its songs, so we assume encoding is always the same.
-        return str.translate(CONTROL_CHARS_MAP)

openlp/plugins/songs/lib/openlyricsxml.py

@@ -281,7 +281,7 @@ class OpenLyrics(object):
         # Process the formatting tags.
         # Have we any tags in song lyrics?
         tags_element = None
-        match = re.search('\{/?\w+\}', song.lyrics, re.UNICODE)
+        match = re.search(r'\{/?\w+\}', song.lyrics)
         if match:
             # Named 'format_' - 'format' is built-in function in Python.
             format_ = etree.SubElement(song_xml, 'format')

openlp/plugins/songusage/forms/songusagedetailform.py

@@ -54,8 +54,14 @@ class SongUsageDetailForm(QtWidgets.QDialog, Ui_SongUsageDetailDialog, RegistryP
         """
         We need to set up the screen
         """
-        self.from_date_calendar.setSelectedDate(Settings().value(self.plugin.settings_section + '/from date'))
+        to_date = Settings().value(self.plugin.settings_section + '/to date')
-        self.to_date_calendar.setSelectedDate(Settings().value(self.plugin.settings_section + '/to date'))
+        if not (isinstance(to_date, QtCore.QDate) and to_date.isValid()):
+            to_date = QtCore.QDate.currentDate()
+        from_date = Settings().value(self.plugin.settings_section + '/from date')
+        if not (isinstance(from_date, QtCore.QDate) and from_date.isValid()):
+            from_date = to_date.addYears(-1)
+        self.from_date_calendar.setSelectedDate(from_date)
+        self.to_date_calendar.setSelectedDate(to_date)
         self.report_path_edit.path = Settings().value(self.plugin.settings_section + '/last directory export')
 
     def on_report_path_edit_path_changed(self, file_path):

openlp/plugins/songusage/songusageplugin.py

@@ -38,20 +38,17 @@ from openlp.plugins.songusage.lib.db import init_schema, SongUsageItem
 
 log = logging.getLogger(__name__)
 
-YEAR = QtCore.QDate().currentDate().year()
+TODAY = QtCore.QDate.currentDate()
-if QtCore.QDate().currentDate().month() < 9:
-    YEAR -= 1
-
 
 __default_settings__ = {
     'songusage/db type': 'sqlite',
     'songusage/db username': '',
-    'songuasge/db password': '',
+    'songusage/db password': '',
-    'songuasge/db hostname': '',
+    'songusage/db hostname': '',
-    'songuasge/db database': '',
+    'songusage/db database': '',
     'songusage/active': False,
-    'songusage/to date': QtCore.QDate(YEAR, 8, 31),
+    'songusage/to date': TODAY,
-    'songusage/from date': QtCore.QDate(YEAR - 1, 9, 1),
+    'songusage/from date': TODAY.addYears(-1),
     'songusage/last directory export': None
 }
 

tests/functional/openlp_core/common/test_actions.py

@@ -153,6 +153,7 @@ class TestActionList(TestCase, TestMixin):
         """
         Prepare the tests
         """
+        self.setup_application()
         self.action_list = ActionList.get_instance()
         self.build_settings()
         self.settings = Settings()

tests/functional/openlp_core/common/test_httputils.py

@@ -233,7 +233,7 @@ class TestHttpUtils(TestCase, TestMixin):
         Test socket timeout gets caught
         """
         # GIVEN: Mocked urlopen to fake a network disconnect in the middle of a download
-        mocked_requests.get.side_effect = IOError
+        mocked_requests.get.side_effect = OSError
 
         # WHEN: Attempt to retrieve a file
         url_get_file(MagicMock(), url='http://localhost/test', file_path=Path(self.tempfile))

tests/functional/openlp_core/common/test_i18n.py

@@ -155,7 +155,7 @@ def test_check_same_instance():
     assert first_instance is second_instance, 'Two UiStrings objects should be the same instance'
 
 
-def test_translate(self):
+def test_translate():
     """
     Test the translate() function
     """

tests/functional/openlp_core/common/test_path.py

@@ -371,13 +371,13 @@ class TestPath(TestCase):
     @patch('openlp.core.common.path.log')
     def test_create_paths_dir_io_error(self, mocked_logger):
         """
-        Test the create_paths() when an IOError is raised
+        Test the create_paths() when an OSError is raised
         """
         # GIVEN: A `Path` to check with patched out mkdir and exists methods
         mocked_path = MagicMock()
-        mocked_path.exists.side_effect = IOError('Cannot make directory')
+        mocked_path.exists.side_effect = OSError('Cannot make directory')
 
-        # WHEN: An IOError is raised when checking the if the path exists.
+        # WHEN: An OSError is raised when checking the if the path exists.
         create_paths(mocked_path)
 
         # THEN: The Error should have been logged
@@ -385,7 +385,7 @@ class TestPath(TestCase):
 
     def test_create_paths_dir_value_error(self):
         """
-        Test the create_paths() when an error other than IOError is raised
+        Test the create_paths() when an error other than OSError is raised
         """
         # GIVEN: A `Path` to check with patched out mkdir and exists methods
         mocked_path = MagicMock()

tests/functional/openlp_core/lib/test_lib.py

@@ -168,7 +168,7 @@ class TestLib(TestCase):
                 patch.object(Path, 'open'):
             file_path = Path('testfile.txt')
             file_path.is_file.return_value = True
-            file_path.open.side_effect = IOError()
+            file_path.open.side_effect = OSError()
 
             # WHEN: get_text_file_string is called
             result = get_text_file_string(file_path)

tests/functional/openlp_core/ui/test_first_time.py

@@ -40,7 +40,7 @@ class TestFirstTimeWizard(TestMixin, TestCase):
         Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031
         """
         # GIVEN: Initial settings and mocks
-        mocked_requests.get.side_effect = IOError('Unable to connect')
+        mocked_requests.get.side_effect = OSError('Unable to connect')
 
         # WHEN: A webpage is requested
         try:

tests/functional/openlp_core/widgets/test_views.py

@@ -627,4 +627,3 @@ class TestTreeWidgetWithDnD(TestCase):
         assert widget.allow_internal_dnd is False
         assert widget.indentation() == 0
         assert widget.isAnimated() is True
-

tests/functional/openlp_plugins/presentations/test_presentationcontroller.py

@@ -144,7 +144,7 @@ class TestPresentationController(TestCase):
         # GIVEN: A mocked open, get_thumbnail_folder and exists
         with patch('openlp.plugins.presentations.lib.presentationcontroller.Path.read_text') as mocked_read_text, \
                 patch(FOLDER_TO_PATCH) as mocked_get_thumbnail_folder:
-            mocked_read_text.side_effect = IOError()
+            mocked_read_text.side_effect = OSError()
             mocked_get_thumbnail_folder.return_value = Path('test')
 
             # WHEN: calling get_titles_and_notes

tests/interfaces/openlp_core/ui/test_projectormanager.py

@@ -42,8 +42,8 @@ class TestProjectorManager(TestCase, TestMixin):
         """
         Create the UI and setup necessary options
         """
-        self.build_settings()
         self.setup_application()
+        self.build_settings()
         Registry.create()
         with patch('openlp.core.lib.projector.db.init_url') as mocked_init_url:
             if os.path.exists(TEST_DB):

tests/interfaces/openlp_core/ui/test_projectorsourceform.py

@@ -64,8 +64,8 @@ class ProjectorSourceFormTest(TestCase, TestMixin):
         Set up anything necessary for all tests
         """
         mocked_init_url.return_value = 'sqlite:///{}'.format(TEST_DB)
-        self.build_settings()
         self.setup_application()
+        self.build_settings()
         Registry.create()
         # Do not try to recreate if we've already been created from a previous test
         if not hasattr(self, 'projectordb'):

tests/interfaces/openlp_core/ui/test_thememanager.py

@@ -41,8 +41,8 @@ class TestThemeManager(TestCase, TestMixin):
         """
         Create the UI
         """
-        self.build_settings()
         self.setup_application()
+        self.build_settings()
         Registry.create()
         self.theme_manager = ThemeManager()
 

tests/utils/__init__.py

@@ -36,7 +36,7 @@ def convert_file_service_item(test_path, name, row=0):
     try:
         items = json.load(open_file)
         first_line = items[row]
-    except IOError:
+    except OSError:
         first_line = ''
     finally:
         open_file.close()