Fix the fix for 1727517

bzr-revno: 2817

openlp/core/common/__init__.py

@@ -44,7 +44,7 @@ log = logging.getLogger(__name__ + '.__init__')
 
 FIRST_CAMEL_REGEX = re.compile('(.)([A-Z][a-z]+)')
 SECOND_CAMEL_REGEX = re.compile('([a-z0-9])([A-Z])')
-CONTROL_CHARS = re.compile(r'[\x00-\x1F\x7F-\x9F]')
+CONTROL_CHARS = re.compile(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]')
 INVALID_FILE_CHARS = re.compile(r'[\\/:\*\?"<>\|\+\[\]%]')
 IMAGES_FILTER = None
 REPLACMENT_CHARS_MAP = str.maketrans({'\u2018': '\'', '\u2019': '\'', '\u201c': '"', '\u201d': '"', '\u2026': '...',
@@ -471,15 +471,15 @@ def get_file_encoding(file_path):
         log.exception('Error detecting file encoding')
 
 
-def normalize_str(irreg_str):
+def normalize_str(irregular_string):
     """
     Normalize the supplied string. Remove unicode control chars and tidy up white space.
 
-    :param str irreg_str: The string to normalize.
+    :param str irregular_string: The string to normalize.
     :return: The normalized string
     :rtype: str
     """
-    irreg_str = irreg_str.translate(REPLACMENT_CHARS_MAP)
-    irreg_str = CONTROL_CHARS.sub('', irreg_str)
-    irreg_str = NEW_LINE_REGEX.sub('\n', irreg_str)
-    return WHITESPACE_REGEX.sub(' ', irreg_str)
+    irregular_string = irregular_string.translate(REPLACMENT_CHARS_MAP)
+    irregular_string = CONTROL_CHARS.sub('', irregular_string)
+    irregular_string = NEW_LINE_REGEX.sub('\n', irregular_string)
+    return WHITESPACE_REGEX.sub(' ', irregular_string)

tests/functional/openlp_core/common/test_common.py

@@ -25,8 +25,8 @@ Functional tests to test the AppLocation class and related methods.
 from unittest import TestCase
 from unittest.mock import MagicMock, call, patch
 
-from openlp.core.common import clean_button_text, de_hump, extension_loader, is_macosx, is_linux, is_win, \
-    path_to_module, trace_error_handler
+from openlp.core.common import clean_button_text, de_hump, extension_loader, is_macosx, is_linux, \
+    is_win, normalize_str, path_to_module, trace_error_handler
 from openlp.core.common.path import Path
 
 
@@ -211,6 +211,30 @@ class TestCommonFunctions(TestCase):
             assert is_win() is False, 'is_win() should return False'
             assert is_macosx() is False, 'is_macosx() should return False'
 
+    def test_normalize_str_leaves_newlines(self):
+        # GIVEN: a string containing newlines
+        string = 'something\nelse'
+        # WHEN: normalize is called
+        normalized_string = normalize_str(string)
+        # THEN: string is unchanged
+        assert normalized_string == string
+
+    def test_normalize_str_removes_null_byte(self):
+        # GIVEN: a string containing a null byte
+        string = 'somet\x00hing'
+        # WHEN: normalize is called
+        normalized_string = normalize_str(string)
+        # THEN: nullbyte is removed
+        assert normalized_string == 'something'
+
+    def test_normalize_str_replaces_crlf_with_lf(self):
+        # GIVEN: a string containing crlf
+        string = 'something\r\nelse'
+        # WHEN: normalize is called
+        normalized_string = normalize_str(string)
+        # THEN: crlf is replaced with lf
+        assert normalized_string == 'something\nelse'
+
     def test_clean_button_text(self):
         """
         Test the clean_button_text() function.