Clean search lyrics for formatting tags. Fixes bug #1655988.

Fix an issue with easyslide import not handling verse order correctly. Fixes bug #1655985. Improve the songbeamer encoding detection. Fixes bug #1530597. Handle a few videopsalm quirks. Fixes bug #1652851. bzr-revno: 2718
2017-01-20 21:42:01 +00:00 · 2017-01-20 21:42:01 +00:00 · df668fde84
commit df668fde84
parent d4ddb7a9d5 b12c414732
9 changed files with 130 additions and 29 deletions
--- a/openlp/plugins/songs/lib/init.py
+++ b/openlp/plugins/songs/lib/init.py
@ -30,7 +30,7 @@ import re
 from PyQt5 import QtWidgets

 from openlp.core.common import AppLocation, CONTROL_CHARS
-from openlp.core.lib import translate
+from openlp.core.lib import translate, clean_tags
 from openlp.plugins.songs.lib.db import Author, MediaFile, Song, Topic
 from openlp.plugins.songs.lib.ui import SongStrings

@ -380,7 +380,7 @@ def clean_song(manager, song):
    if isinstance(song.lyrics, bytes):
        song.lyrics = str(song.lyrics, encoding='utf8')
    verses = SongXML().get_verses(song.lyrics)
-    song.search_lyrics = ' '.join([clean_string(verse[1]) for verse in verses])
+    song.search_lyrics = ' '.join([clean_string(clean_tags(verse[1])) for verse in verses])
    # The song does not have any author, add one.
    if not song.authors_songs:
        name = SongStrings.AuthorUnknown
--- a/openlp/plugins/songs/lib/importers/easyslides.py
+++ b/openlp/plugins/songs/lib/importers/easyslides.py
@ -180,7 +180,7 @@ class EasySlidesImport(SongImport):
        reg = default_region
        verses[reg] = {}
        # instance differentiates occurrences of same verse tag
-        vt = 'V'
+        vt = 'v'
        vn = '1'
        inst = 1
        for line in lines:
@ -193,14 +193,14 @@ class EasySlidesImport(SongImport):
                        inst += 1
                else:
                    # separators are not used, so empty line starts a new verse
-                    vt = 'V'
+                    vt = 'v'
                    vn = len(verses[reg].get(vt, {})) + 1
                    inst = 1
            elif line[0:7] == '[region':
                reg = self._extract_region(line)
                verses.setdefault(reg, {})
                if not regions_in_verses:
-                    vt = 'V'
+                    vt = 'v'
                    vn = '1'
                    inst = 1
            elif line[0] == '[':
@ -213,7 +213,7 @@ class EasySlidesImport(SongImport):
                if match:
                    marker = match.group(1).strip()
                    vn = match.group(2)
-                vt = MarkTypes.get(marker, 'O') if marker else 'V'
+                vt = MarkTypes.get(marker, 'o') if marker else 'v'
                if regions_in_verses:
                    region = default_region
                inst = 1
@ -238,13 +238,13 @@ class EasySlidesImport(SongImport):
                lines = '\n'.join(verses[reg][vt][vn][inst])
                self.add_verse(lines, versetag)
        SeqTypes = {
-            'p': 'P1',
-            'q': 'P2',
-            'c': 'C1',
-            't': 'C2',
-            'b': 'B1',
-            'w': 'B2',
-            'e': 'E1'}
+            'p': 'p1',
+            'q': 'p2',
+            'c': 'c1',
+            't': 'c2',
+            'b': 'b1',
+            'w': 'b2',
+            'e': 'e1'}
        # Make use of Sequence data, determining the order of verses
        try:
            order = str(song.Sequence).strip().split(',')
@ -252,7 +252,7 @@ class EasySlidesImport(SongImport):
                if not tag:
                    continue
                elif tag[0].isdigit():
-                    tag = 'V' + tag
+                    tag = 'v' + tag
                elif tag.lower() in SeqTypes:
                    tag = SeqTypes[tag.lower()]
                else:
--- a/openlp/plugins/songs/lib/importers/songbeamer.py
+++ b/openlp/plugins/songs/lib/importers/songbeamer.py
@ -28,6 +28,7 @@ import logging
 import os
 import re

+from openlp.core.common import get_file_encoding
 from openlp.plugins.songs.lib import VerseType
 from openlp.plugins.songs.lib.importers.songimport import SongImport

@ -113,13 +114,15 @@ class SongBeamerImport(SongImport):
            read_verses = False
            file_name = os.path.split(import_file)[1]
            if os.path.isfile(import_file):
-                # First open in binary mode to detect the encoding
-                detect_file = open(import_file, 'rb')
-                details = chardet.detect(detect_file.read())
-                detect_file.close()
-                infile = codecs.open(import_file, 'r', details['encoding'])
+                # Detect the encoding
+                self.input_file_encoding = get_file_encoding(import_file)['encoding']
+                # The encoding should only be ANSI (cp1252), UTF-8, Unicode, Big-Endian-Unicode.
+                # So if it doesn't start with 'u' we default to cp1252. See:
+                # https://forum.songbeamer.com/viewtopic.php?p=419&sid=ca4814924e37c11e4438b7272a98b6f2
+                if self.input_file_encoding.lower().startswith('u'):
+                    self.input_file_encoding = 'cp1252'
+                infile = open(import_file, 'rt', encoding=self.input_file_encoding)
                song_data = infile.readlines()
-                infile.close()
            else:
                continue
            self.title = file_name.split('.sng')[0]
--- a/openlp/plugins/songs/lib/importers/videopsalm.py
+++ b/openlp/plugins/songs/lib/importers/videopsalm.py
@ -65,8 +65,8 @@ class VideoPsalmImport(SongImport):
                if c == '\n':
                    if inside_quotes:
                        processed_content += '\\n'
-                # Put keys in quotes
-                elif c.isalnum() and not inside_quotes:
+                # Put keys in quotes. The '-' is for handling nagative numbers
+                elif (c.isalnum() or c == '-') and not inside_quotes:
                    processed_content += '"' + c
                    c = next(file_content_it)
                    while c.isalnum():
@ -121,6 +121,8 @@ class VideoPsalmImport(SongImport):
                if 'Memo3' in song:
                    self.add_comment(song['Memo3'])
                for verse in song['Verses']:
+                    if 'Text' not in verse:
+                        continue
                    self.add_verse(verse['Text'], 'v')
                if not self.finish():
                    self.log_error('Could not import {title}'.format(title=self.title))
--- a/tests/functional/openlp_plugins/songs/test_easyslidesimport.py
+++ b/tests/functional/openlp_plugins/songs/test_easyslidesimport.py
@ -43,3 +43,5 @@ class TestEasySlidesFileImport(SongImportTestHelper):
        """
        self.file_import(os.path.join(TEST_PATH, 'amazing-grace.xml'),
                         self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace.json')))
+        self.file_import(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.xml'),
+                         self.load_external_result_data(os.path.join(TEST_PATH, 'Export_2017-01-12_BB.json')))
--- a/tests/resources/easyslidessongs/Amazing
+++ b/tests/resources/easyslidessongs/Amazing
@ -6,27 +6,27 @@
    "verses": [
        [
            "Amazing grace! How sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.",
-            "V1"
+            "v1"
        ],
        [
            "'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!",
-            "V2"
+            "v2"
        ],
        [
            "Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.",
-            "V3"
+            "v3"
        ],
        [
            "The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.",
-            "V4"
+            "v4"
        ],
        [
            "Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.",
-            "V5"
+            "v5"
        ],
        [
            "When we've been there a thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.",
-            "V6"
+            "v6"
        ]
    ]
 }
--- a/tests/resources/easyslidessongs/Export_2017-01-12_BB.json
+++ b/tests/resources/easyslidessongs/Export_2017-01-12_BB.json
@ -0,0 +1,44 @@
+{
+    "title": "BBBBBBBBB",
+    "authors": [
+        "John Newton (1725-1807)"
+    ],
+    "verses": [
+        [
+            "V1V1V1V1V1V1\nV1V1V1V1V1V1",
+            "v1"
+        ],
+        [
+            "V2V2V2V2V2V2\nV2V2V2V2V2V2",
+            "v2"
+        ],
+        [
+            "C1C1C1C1C1C1\nC1C1C1C1C1C1",
+            "c1"
+        ],
+        [
+            "C2C2C2C2C2C2\nC2C2C2C2C2C2",
+            "c2"
+        ],
+        [
+            "B1B1B1B1B1B1\nB1B1B1B1B1B1",
+            "b1"
+        ],
+        [
+            "B2B2B2B2B2B2\nB2B2B2B2B2B2",
+            "b2"
+        ],
+        [
+            "PRE1PRE1PRE1\nPRE1PRE1PRE1",
+            "p1"
+        ],
+        [
+            "PRE2PRE2PRE2\nPRE2PRE2PRE2",
+            "p2"
+        ],
+        [
+            "ENDENDENDEND\nENDENDENDEND",
+            "e1"
+        ]
+    ]
+}
--- a/tests/resources/easyslidessongs/Export_2017-01-12_BB.xml
+++ b/tests/resources/easyslidessongs/Export_2017-01-12_BB.xml
@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="utf-8"?>
+<EasiSlides>
+  <Item>
+    <Title1>BBBBBBBBB</Title1>
+    <Title2 />
+    <Folder>NAGY</Folder>
+    <SongNumber>0</SongNumber>
+    <Contents>[1]
+V1V1V1V1V1V1
+V1V1V1V1V1V1
+[2]
+V2V2V2V2V2V2
+V2V2V2V2V2V2
+[chorus]
+C1C1C1C1C1C1
+C1C1C1C1C1C1
+[chorus 2]
+C2C2C2C2C2C2
+C2C2C2C2C2C2
+[bridge]
+B1B1B1B1B1B1
+B1B1B1B1B1B1
+[bridge 2]
+B2B2B2B2B2B2
+B2B2B2B2B2B2
+[prechorus]
+PRE1PRE1PRE1
+PRE1PRE1PRE1
+[prechorus 2]
+PRE2PRE2PRE2
+PRE2PRE2PRE2
+[ending]
+ENDENDENDEND
+ENDENDENDEND</Contents>
+    <Notations />
+    <Sequence>1,2,c,t,b,w,p,q,e</Sequence>
+    <Writer />
+    <Copyright />
+    <Category />
+    <Timing />
+    <MusicKey />
+    <Capo>-1</Capo>
+    <LicenceAdmin1 />
+    <LicenceAdmin2 />
+    <BookReference />
+    <UserReference />
+    <FormatData />
+    <Settings>10=&gt;</Settings>
+  </Item>
+</EasiSlides>
--- a/tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json
+++ b/tests/resources/videopsalmsongs/videopsalm-as-safe-a-stronghold.json
@ -1,4 +1,4 @@
-{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Copyright:"Public
+{Abbreviation:"SB1",Copyright:"Public domain",Songs:[{ID:3,Composer:"Unknown",Author:"Martin Luther",Capo:-1,Copyright:"Public
 Domain",Theme:"tema1
 tema2",CCLI:"12345",Alias:"A safe stronghold",Memo1:"This is
 the first comment