head

openlp/plugins/songs/lib/songshowplusimport.py

@@ -30,13 +30,14 @@
 The :mod:`songshowplusimport` module provides the functionality for importing
 SongShow Plus songs into the OpenLP database.
 """
+import chardet
 import os
 import logging
 import re
 import struct
 
 from openlp.core.ui.wizard import WizardStrings
-from openlp.plugins.songs.lib import VerseType
+from openlp.plugins.songs.lib import VerseType, retrieve_windows_encoding
 from openlp.plugins.songs.lib.songimport import SongImport
 
 TITLE = 1
@@ -132,41 +133,43 @@ class SongShowPlusImport(SongImport):
                 else:
                     length_descriptor, = struct.unpack("B", song_data.read(1))
                 log.debug(length_descriptor_size)
-                data = song_data.read(length_descriptor).decode()
+                data = song_data.read(length_descriptor)
                 if block_key == TITLE:
-                    self.title = data
+                    self.title = self.decode(data)
                 elif block_key == AUTHOR:
-                    authors = data.split(" / ")
+                    authors = self.decode(data).split(" / ")
                     for author in authors:
                         if author.find(",") !=-1:
                             authorParts = author.split(", ")
                             author = authorParts[1] + " " + authorParts[0]
                         self.parse_author(author)
                 elif block_key == COPYRIGHT:
-                    self.addCopyright(data)
+                    self.addCopyright(self.decode(data))
                 elif block_key == CCLI_NO:
                     self.ccliNumber = int(data)
                 elif block_key == VERSE:
-                    self.addVerse(data, "%s%s" % (VerseType.tags[VerseType.Verse], verse_no))
+                    self.addVerse(self.decode(data), "%s%s" % (VerseType.tags[VerseType.Verse], verse_no))
                 elif block_key == CHORUS:
-                    self.addVerse(data, "%s%s" % (VerseType.tags[VerseType.Chorus], verse_no))
+                    self.addVerse(self.decode(data), "%s%s" % (VerseType.tags[VerseType.Chorus], verse_no))
                 elif block_key == BRIDGE:
-                    self.addVerse(data, "%s%s" % (VerseType.tags[VerseType.Bridge], verse_no))
+                    self.addVerse(self.decode(data), "%s%s" % (VerseType.tags[VerseType.Bridge], verse_no))
                 elif block_key == TOPIC:
-                    self.topics.append(data)
+                    self.topics.append(self.decode(data))
                 elif block_key == COMMENTS:
-                    self.comments = data
+                    self.comments = self.decode(data)
                 elif block_key == VERSE_ORDER:
-                    verse_tag = self.to_openlp_verse_tag(data, True)
+                    verse_tag = self.to_openlp_verse_tag(self.decode(data), True)
                     if verse_tag:
+                        if not isinstance(verse_tag, str):
+                            verse_tag = self.decode(verse_tag)
                         self.ssp_verse_order_list.append(verse_tag)
                 elif block_key == SONG_BOOK:
-                    self.songBookName = data
+                    self.songBookName = self.decode(data)
                 elif block_key == SONG_NUMBER:
                     self.songNumber = ord(data)
                 elif block_key == CUSTOM_VERSE:
                     verse_tag = self.to_openlp_verse_tag(verse_name)
-                    self.addVerse(data, verse_tag)
+                    self.addVerse(self.decode(data), verse_tag)
                 else:
                     log.debug("Unrecognised blockKey: %s, data: %s" % (block_key, data))
                     song_data.seek(next_block_starts)
@@ -204,3 +207,9 @@ class SongShowPlusImport(SongImport):
             verse_tag = VerseType.tags[VerseType.Other]
             verse_number = self.other_list[verse_name]
         return verse_tag + verse_number
+
+    def decode(self, data):
+        try:
+            return str(data, chardet.detect(data)['encoding'])
+        except:
+            return str(data, retrieve_windows_encoding())

tests/functional/openlp_core_utils/test_utils.py

@@ -150,35 +150,14 @@ class TestUtils(TestCase):
         # THEN: The file name should be cleaned.
         self.assertEqual(wanted_name, result, 'The file name should not contain any special characters.')
 
-    def get_locale_key_windows_test(self):
+    def get_locale_key_test(self):
         """
         Test the get_locale_key(string) function
         """
-        with patch('openlp.core.utils.languagemanager.LanguageManager.get_language') as mocked_get_language,  \
+        with patch('openlp.core.utils.languagemanager.LanguageManager.get_language') as mocked_get_language:
-                patch('openlp.core.utils.os') as mocked_os:
             # GIVEN: The language is German
             # 0x00C3 (A with diaresis) should be sorted as "A". 0x00DF (sharp s) should be sorted as "ss".
             mocked_get_language.return_value = 'de'
-            mocked_os.name = 'nt'
-            unsorted_list = ['Auszug', 'Aushang', '\u00C4u\u00DFerung']
-
-            # WHEN: We sort the list and use get_locale_key() to generate the sorting keys
-            sorted_list = sorted(unsorted_list, key=get_locale_key)
-
-            # THEN: We get a properly sorted list
-            self.assertEqual(['Aushang', '\u00C4u\u00DFerung', 'Auszug'], sorted_list,
-                'Strings should be sorted properly')
-
-    def get_locale_key_linux_test(self):
-        """
-        Test the get_locale_key(string) function
-        """
-        with patch('openlp.core.utils.languagemanager.LanguageManager.get_language') as mocked_get_language,  \
-                patch('openlp.core.utils.os.name') as mocked_os:
-            # GIVEN: The language is German
-            # 0x00C3 (A with diaresis) should be sorted as "A". 0x00DF (sharp s) should be sorted as "ss".
-            mocked_get_language.return_value = 'de'
-            mocked_os.name = 'linux'
             unsorted_list = ['Auszug', 'Aushang', '\u00C4u\u00DFerung']
 
             # WHEN: We sort the list and use get_locale_key() to generate the sorting keys