From 2ed5f2bbe48c2d36b644ed6baf42b699e10c5b9f Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Mon, 3 Nov 2014 11:03:53 +0100 Subject: [PATCH 1/9] Fix for import of Words of Worship file --- .../plugins/songs/lib/importers/wordsofworship.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/openlp/plugins/songs/lib/importers/wordsofworship.py b/openlp/plugins/songs/lib/importers/wordsofworship.py index 1b398c604..8fcbc9f84 100644 --- a/openlp/plugins/songs/lib/importers/wordsofworship.py +++ b/openlp/plugins/songs/lib/importers/wordsofworship.py @@ -112,17 +112,17 @@ class WordsOfWorshipImport(SongImport): return self.set_defaults() song_data = open(source, 'rb') - if song_data.read(19) != 'WoW File\nSong Words': + if song_data.read(19).decode() != 'WoW File\nSong Words': self.log_error(source, str(translate('SongsPlugin.WordsofWorshipSongImport', - 'Invalid Words of Worship song file. Missing "Wow File\\nSong ' + 'Invalid Words of Worship song file. Missing "WoW File\\nSong ' 'Words" header.'))) continue # Seek to byte which stores number of blocks in the song song_data.seek(56) no_of_blocks = ord(song_data.read(1)) song_data.seek(66) - if song_data.read(16) != 'CSongDoc::CBlock': + if song_data.read(16).decode() != 'CSongDoc::CBlock': self.log_error(source, str(translate('SongsPlugin.WordsofWorshipSongImport', 'Invalid Words of Worship song file. Missing "CSongDoc::CBlock" ' @@ -131,11 +131,17 @@ class WordsOfWorshipImport(SongImport): # Seek to the beginning of the first block song_data.seek(82) for block in range(no_of_blocks): + skip_char_at_end = True self.lines_to_read = ord(song_data.read(4)[:1]) block_text = '' while self.lines_to_read: self.line_text = str(song_data.read(ord(song_data.read(1))), 'cp1252') - song_data.seek(1, os.SEEK_CUR) + if skip_char_at_end: + skip_char = ord(song_data.read(1)) + # Check if we really should skip a char. In some wsg files we shouldn't + if skip_char != 0: + song_data.seek(-1, os.SEEK_CUR) + skip_char_at_end = False if block_text: block_text += '\n' block_text += self.line_text From 51f4539822b837464cfb62f69d07600cb5d2eaf7 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Mon, 3 Nov 2014 15:36:27 +0100 Subject: [PATCH 2/9] Added test for Word of Worship import --- .../songs/lib/importers/wordsofworship.py | 2 +- .../songs/test_wordsofworshipimport.py | 54 ++++++++++++++++++ .../Amazing Grace (6 Verses).json | 33 +++++++++++ .../Amazing Grace (6 Verses).wow-song | Bin 0 -> 965 bytes 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py create mode 100644 tests/resources/wordsofworshipsongs/Amazing Grace (6 Verses).json create mode 100644 tests/resources/wordsofworshipsongs/Amazing Grace (6 Verses).wow-song diff --git a/openlp/plugins/songs/lib/importers/wordsofworship.py b/openlp/plugins/songs/lib/importers/wordsofworship.py index 8fcbc9f84..29b384085 100644 --- a/openlp/plugins/songs/lib/importers/wordsofworship.py +++ b/openlp/plugins/songs/lib/importers/wordsofworship.py @@ -99,7 +99,7 @@ class WordsOfWorshipImport(SongImport): """ Initialise the Words of Worship importer. """ - SongImport.__init__(self, manager, **kwargs) + super(WordsOfWorshipImport, self).__init__(manager, **kwargs) def do_import(self): """ diff --git a/tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py b/tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py new file mode 100644 index 000000000..952e16971 --- /dev/null +++ b/tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# vim: autoindent shiftwidth=4 expandtab textwidth=120 tabstop=4 softtabstop=4 + +############################################################################### +# OpenLP - Open Source Lyrics Projection # +# --------------------------------------------------------------------------- # +# Copyright (c) 2008-2014 Raoul Snyman # +# Portions copyright (c) 2008-2014 Tim Bentley, Gerald Britton, Jonathan # +# Corwin, Samuel Findlay, Michael Gorven, Scott Guerrieri, Matthias Hub, # +# Meinert Jordan, Armin Köhler, Erik Lundin, Edwin Lunando, Brian T. Meyer. # +# Joshua Miller, Stevan Pettit, Andreas Preikschat, Mattias Põldaru, # +# Christian Richter, Philip Ridout, Simon Scudder, Jeffrey Smith, # +# Maikel Stuivenberg, Martin Thompson, Jon Tibble, Dave Warnock, # +# Frode Woldsund, Martin Zibricky, Patrick Zimmermann # +# --------------------------------------------------------------------------- # +# This program is free software; you can redistribute it and/or modify it # +# under the terms of the GNU General Public License as published by the Free # +# Software Foundation; version 2 of the License. # +# # +# This program is distributed in the hope that it will be useful, but WITHOUT # +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # +# more details. # +# # +# You should have received a copy of the GNU General Public License along # +# with this program; if not, write to the Free Software Foundation, Inc., 59 # +# Temple Place, Suite 330, Boston, MA 02111-1307 USA # +############################################################################### +""" +This module contains tests for the Words of Worship song importer. +""" + +import os + +from tests.helpers.songfileimport import SongImportTestHelper +from openlp.plugins.songs.lib.importers.wordsofworship import WordsOfWorshipImport + +TEST_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..', 'resources', 'wordsofworshipsongs')) + + +class TestWordsOfWorshipFileImport(SongImportTestHelper): + + def __init__(self, *args, **kwargs): + self.importer_class_name = 'WordsOfWorshipImport' + self.importer_module_name = 'wordsofworship' + super(TestWordsOfWorshipFileImport, self).__init__(*args, **kwargs) + + def test_song_import(self): + """ + Test that loading a Words of Worship file works correctly + """ + self.file_import([os.path.join(TEST_PATH, 'Amazing Grace (6 Verses).wow-song')], + self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace (6 Verses).json'))) diff --git a/tests/resources/wordsofworshipsongs/Amazing Grace (6 Verses).json b/tests/resources/wordsofworshipsongs/Amazing Grace (6 Verses).json new file mode 100644 index 000000000..563872ae7 --- /dev/null +++ b/tests/resources/wordsofworshipsongs/Amazing Grace (6 Verses).json @@ -0,0 +1,33 @@ +{ + "authors": [ + "John Newton (1725-1807)" + ], + "title": "Amazing Grace (6 Verses)", + "verse_order_list": [], + "verses": [ + [ + "Amazing grace! how sweet the sound\nThat saved a wretch like me;\nI once was lost, but now am found,\nWas blind, but now I see.", + "V" + ], + [ + "'Twas grace that taught my heart to fear,\nAnd grace my fears relieved;\nHow precious did that grace appear,\nThe hour I first believed!", + "V" + ], + [ + "Through many dangers, toils and snares\nI have already come;\n'Tis grace that brought me safe thus far,\nAnd grace will lead me home.", + "V" + ], + [ + "The Lord has promised good to me,\nHis word my hope secures;\nHe will my shield and portion be\nAs long as life endures.", + "V" + ], + [ + "Yes, when this heart and flesh shall fail,\nAnd mortal life shall cease,\nI shall possess within the veil\nA life of joy and peace.", + "V" + ], + [ + "When we've been there ten thousand years,\nBright shining as the sun,\nWe've no less days to sing God's praise\nThan when we first begun.", + "V" + ] + ] +} diff --git a/tests/resources/wordsofworshipsongs/Amazing Grace (6 Verses).wow-song b/tests/resources/wordsofworshipsongs/Amazing Grace (6 Verses).wow-song new file mode 100644 index 0000000000000000000000000000000000000000..e45d22e2ffa3dcfafa966ae5984e40bc78698536 GIT binary patch literal 965 zcmZ9L&1)1v5XC$3Q&8L=JeSI-uWSMNpneM-q|+BY+>nB*JCXGJsKY70iOEq_IBGm zF_(JtlN4uXmlr{b>xYOOp4axbhu|{Vf)B_^N4aBesBuhL+E|+XnX?ULd&4DJ>Jm4F zqu{TZYMz+GDM>7tI-4m-ZpO4|8;a1eHB~AwHoJ2K7r|qHADvQWe%FI;{ZW*o(M&aZ z5M)Qn#P`xI{HsQZeh0WOPpU3j$*~RC8d_&Uv*;%%6fO_^-T%_;ZHE{Ix@#%+? Nx5ppfpC0ue<6q+@B>eyY literal 0 HcmV?d00001 From 88e4c2e0b9f872e4e2cf0462ca0697503b670c3a Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Mon, 3 Nov 2014 20:30:43 +0000 Subject: [PATCH 3/9] Try to fix 0 track length by waiting. Fixes bug 1387293 Fixes: https://launchpad.net/bugs/1387293 --- openlp/plugins/media/forms/mediaclipselectorform.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openlp/plugins/media/forms/mediaclipselectorform.py b/openlp/plugins/media/forms/mediaclipselectorform.py index 86727d336..416e2504e 100644 --- a/openlp/plugins/media/forms/mediaclipselectorform.py +++ b/openlp/plugins/media/forms/mediaclipselectorform.py @@ -446,6 +446,12 @@ class MediaClipSelectorForm(QtGui.QDialog, Ui_MediaClipSelector, RegistryPropert # Set media length info self.playback_length = self.vlc_media_player.get_length() log.debug('playback_length: %d ms' % self.playback_length) + # if length is 0, wait a bit, maybe vlc will change its mind... + loop_count = 0 + while self.playback_length == 0 and loop_count < 20: + sleep(0.1) + self.playback_length = self.vlc_media_player.get_length() + log.debug('in loop, playback_length: %d ms' % self.playback_length) self.position_slider.setMaximum(self.playback_length) # setup start and end time rounded_vlc_ms_length = int(round(self.playback_length / 100.0) * 100.0) From af6a5322e22bc81031736c6d9a6b659b72d5969c Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Tue, 4 Nov 2014 12:58:03 +0100 Subject: [PATCH 4/9] Make it possible to break out of waiting loop --- openlp/plugins/media/forms/mediaclipselectorform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openlp/plugins/media/forms/mediaclipselectorform.py b/openlp/plugins/media/forms/mediaclipselectorform.py index 416e2504e..2cdbbc6ef 100644 --- a/openlp/plugins/media/forms/mediaclipselectorform.py +++ b/openlp/plugins/media/forms/mediaclipselectorform.py @@ -451,6 +451,7 @@ class MediaClipSelectorForm(QtGui.QDialog, Ui_MediaClipSelector, RegistryPropert while self.playback_length == 0 and loop_count < 20: sleep(0.1) self.playback_length = self.vlc_media_player.get_length() + loop_count += 1 log.debug('in loop, playback_length: %d ms' % self.playback_length) self.position_slider.setMaximum(self.playback_length) # setup start and end time From 77c7da2d20ff569d8a7c90eabdb554daf0a609db Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Wed, 5 Nov 2014 09:42:33 +0100 Subject: [PATCH 5/9] Change duplicate check to pass int-string tuples to workers, to workaround windows issue, see bug #1388850 Fixes: https://launchpad.net/bugs/1388850 --- .../songs/forms/duplicatesongremovalform.py | 14 ++++---- openlp/plugins/songs/lib/songcompare.py | 16 +++++---- .../openlp_plugins/songs/test_lib.py | 34 +++++++++---------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/openlp/plugins/songs/forms/duplicatesongremovalform.py b/openlp/plugins/songs/forms/duplicatesongremovalform.py index c411c8c1c..4bcce1c44 100644 --- a/openlp/plugins/songs/forms/duplicatesongremovalform.py +++ b/openlp/plugins/songs/forms/duplicatesongremovalform.py @@ -48,14 +48,15 @@ log = logging.getLogger(__name__) def song_generator(songs): """ - This is a generator function to return tuples of two songs. When completed then all songs have once been returned - combined with any other songs. + This is a generator function to return tuples of tuple with two songs and their position in the song array. + When completed then all songs have once been returned combined with any other songs. :param songs: All songs in the database. """ for outer_song_counter in range(len(songs) - 1): for inner_song_counter in range(outer_song_counter + 1, len(songs)): - yield (songs[outer_song_counter], songs[inner_song_counter]) + yield ((outer_song_counter, songs[outer_song_counter].search_lyrics), + (inner_song_counter, songs[inner_song_counter].search_lyrics)) class DuplicateSongRemovalForm(OpenLPWizard, RegistryProperties): @@ -187,16 +188,17 @@ class DuplicateSongRemovalForm(OpenLPWizard, RegistryProperties): # Do not accept any further tasks. Also this closes the processes if all tasks are done. pool.close() # While the processes are still working, start to look at the results. - for song_tuple in result: + for pos_tuple in result: self.duplicate_search_progress_bar.setValue(self.duplicate_search_progress_bar.value() + 1) # The call to process_events() will keep the GUI responsive. self.application.process_events() if self.break_search: pool.terminate() return - if song_tuple is None: + if pos_tuple is None: continue - song1, song2 = song_tuple + song1 = songs[pos_tuple[0]] + song2 = songs[pos_tuple[1]] duplicate_added = self.add_duplicates_to_song_list(song1, song2) if duplicate_added: self.found_duplicates_edit.appendPlainText(song1.title + " = " + song2.title) diff --git a/openlp/plugins/songs/lib/songcompare.py b/openlp/plugins/songs/lib/songcompare.py index 8a6cc7130..ddd5e4552 100644 --- a/openlp/plugins/songs/lib/songcompare.py +++ b/openlp/plugins/songs/lib/songcompare.py @@ -59,12 +59,14 @@ def songs_probably_equal(song_tupel): :param song_tupel: A tuple of two songs to compare. """ song1, song2 = song_tupel - if len(song1.search_lyrics) < len(song2.search_lyrics): - small = song1.search_lyrics - large = song2.search_lyrics + pos1, lyrics1 = song1 + pos2, lyrics2 = song2 + if len(lyrics1) < len(lyrics2): + small = lyrics1 + large = lyrics2 else: - small = song2.search_lyrics - large = song1.search_lyrics + small = lyrics2 + large = lyrics1 differ = difflib.SequenceMatcher(a=large, b=small) diff_tuples = differ.get_opcodes() diff_no_typos = _remove_typos(diff_tuples) @@ -77,7 +79,7 @@ def songs_probably_equal(song_tupel): length_of_equal_blocks += _op_length(element) if length_of_equal_blocks >= MIN_BLOCK_SIZE: - return song1, song2 + return pos1, pos2 # Check 2: Similarity based on the relative length of the longest equal block. # Calculate the length of the largest equal block of the diff set. length_of_longest_equal_block = 0 @@ -85,7 +87,7 @@ def songs_probably_equal(song_tupel): if element[0] == "equal" and _op_length(element) > length_of_longest_equal_block: length_of_longest_equal_block = _op_length(element) if length_of_longest_equal_block > len(small) * 2 // 3: - return song1, song2 + return pos1, pos2 # Both checks failed. We assume the songs are not equal. return None diff --git a/tests/functional/openlp_plugins/songs/test_lib.py b/tests/functional/openlp_plugins/songs/test_lib.py index 140126f26..3b6f9bf87 100644 --- a/tests/functional/openlp_plugins/songs/test_lib.py +++ b/tests/functional/openlp_plugins/songs/test_lib.py @@ -58,8 +58,6 @@ class TestLib(TestCase): i love that old cross where the dearest and best for a world of lost sinners was slain so ill cherish the old rugged cross till my trophies at last i lay down i will cling to the old rugged cross and exchange it some day for a crown''' - self.song1 = MagicMock() - self.song2 = MagicMock() def clean_string_test(self): """ @@ -92,53 +90,53 @@ class TestLib(TestCase): Test the songs_probably_equal function with twice the same song. """ # GIVEN: Two equal songs. - self.song1.search_lyrics = self.full_lyrics - self.song2.search_lyrics = self.full_lyrics + song_tuple1 = (2, self.full_lyrics) + song_tuple2 = (4, self.full_lyrics) # WHEN: We compare those songs for equality. - result = songs_probably_equal((self.song1, self.song2)) + result = songs_probably_equal((song_tuple1, song_tuple2)) # THEN: The result should be a tuple.. - assert result == (self.song1, self.song2), 'The result should be the tuble of songs' + assert result == (2, 4), 'The result should be the tuble of song positions' def songs_probably_equal_short_song_test(self): """ Test the songs_probably_equal function with a song and a shorter version of the same song. """ # GIVEN: A song and a short version of the same song. - self.song1.search_lyrics = self.full_lyrics - self.song2.search_lyrics = self.short_lyrics + song_tuple1 = (1, self.full_lyrics) + song_tuple2 = (3, self.short_lyrics) # WHEN: We compare those songs for equality. - result = songs_probably_equal((self.song1, self.song2)) + result = songs_probably_equal((song_tuple1, song_tuple2)) # THEN: The result should be a tuple.. - assert result == (self.song1, self.song2), 'The result should be the tuble of songs' + assert result == (1, 3), 'The result should be the tuble of song positions' def songs_probably_equal_error_song_test(self): """ Test the songs_probably_equal function with a song and a very erroneous version of the same song. """ # GIVEN: A song and the same song with lots of errors. - self.song1.search_lyrics = self.full_lyrics - self.song2.search_lyrics = self.error_lyrics + song_tuple1 = (4, self.full_lyrics) + song_tuple2 = (7, self.error_lyrics) # WHEN: We compare those songs for equality. - result = songs_probably_equal((self.song1, self.song2)) + result = songs_probably_equal((song_tuple1, song_tuple2)) - # THEN: The result should be a tuple of songs.. - assert result == (self.song1, self.song2), 'The result should be the tuble of songs' + # THEN: The result should be a tuple of song positions. + assert result == (4, 7), 'The result should be the tuble of song positions' def songs_probably_equal_different_song_test(self): """ Test the songs_probably_equal function with two different songs. """ # GIVEN: Two different songs. - self.song1.search_lyrics = self.full_lyrics - self.song2.search_lyrics = self.different_lyrics + song_tuple1 = (5, self.full_lyrics) + song_tuple2 = (8, self.different_lyrics) # WHEN: We compare those songs for equality. - result = songs_probably_equal((self.song1, self.song2)) + result = songs_probably_equal((song_tuple1, song_tuple2)) # THEN: The result should be None. assert result is None, 'The result should be None' From 02a159bf654073a57078240b4c730e4aff06bd21 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Wed, 5 Nov 2014 14:04:43 +0100 Subject: [PATCH 6/9] Added one more test file for WoW import --- .../songs/test_wordsofworshipimport.py | 2 ++ .../When morning gilds the skies.json | 29 ++++++++++++++++++ .../When morning gilds the skies.wsg | Bin 0 -> 999 bytes 3 files changed, 31 insertions(+) create mode 100644 tests/resources/wordsofworshipsongs/When morning gilds the skies.json create mode 100644 tests/resources/wordsofworshipsongs/When morning gilds the skies.wsg diff --git a/tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py b/tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py index 952e16971..853868e91 100644 --- a/tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py +++ b/tests/functional/openlp_plugins/songs/test_wordsofworshipimport.py @@ -52,3 +52,5 @@ class TestWordsOfWorshipFileImport(SongImportTestHelper): """ self.file_import([os.path.join(TEST_PATH, 'Amazing Grace (6 Verses).wow-song')], self.load_external_result_data(os.path.join(TEST_PATH, 'Amazing Grace (6 Verses).json'))) + self.file_import([os.path.join(TEST_PATH, 'When morning gilds the skies.wsg')], + self.load_external_result_data(os.path.join(TEST_PATH, 'When morning gilds the skies.json'))) diff --git a/tests/resources/wordsofworshipsongs/When morning gilds the skies.json b/tests/resources/wordsofworshipsongs/When morning gilds the skies.json new file mode 100644 index 000000000..c7a4426dd --- /dev/null +++ b/tests/resources/wordsofworshipsongs/When morning gilds the skies.json @@ -0,0 +1,29 @@ +{ + "authors": [ + "Author Unknown. Tr. Edward Caswall" + ], + "title": "When morning gilds the skies", + "verse_order_list": [], + "verses": [ + [ + "When morning gilds the skies\nMy heart awaking cries:\n'May Jesus Christ be prais'd!'\nAlike at work and prayer to Jesus I repair:\n'May Jesus Christ be prais'd!'", + "V" + ], + [ + "Does sadness fill my mind?\nA solace here I find:\n'May Jesus Christ be praised!'\nWhen evil thoughts molest,\nWith this I shield my breast:\n'May Jesus Christ be prais'd!'", + "V" + ], + [ + "To God, the Word, on high\nThe hosts of angels cry:\n'May Jesus Christ be prais'd!'\nLet mortals, too, upraise\nTheir voice in hymns of praise:\n'May Jesus Christ be prais'd!'", + "V" + ], + [ + "Let earth's wide circle round\nIn joyful notes resound:\n'May Jesus Christ be prais'd!'\nLet air, and sea, and sky,\nFrom depth to height, reply:\n'May Jesus Christ be prais'd!'", + "V" + ], + [ + "Be this while life is mine\nMy canticle divine\n'May Jesus Christ be prais'd!'\nBe this the eternal song,\nThrough all the ages long:\n'May Jesus Christ be prais'd!'", + "V" + ] + ] +} diff --git a/tests/resources/wordsofworshipsongs/When morning gilds the skies.wsg b/tests/resources/wordsofworshipsongs/When morning gilds the skies.wsg new file mode 100644 index 0000000000000000000000000000000000000000..901a2dc4f30d8f8735d2356cd245a58f241f6a7b GIT binary patch literal 999 zcmaiz!EW0y42IJI+o7jnz_7+>R}MSmFryEU%aEq)kYYV8L(ZzjSe#`s5ak%Y?aezX z?U2)I9}Gd1ev;o$ZFmE>nP3OWs2}Kuf`X zeZK%!7T4Zr(9QWk4U+!v`L@{`jR9{kgKz)|ha)=Jp1XiDBHpBlX=+F?w4`U6w;Dj6>-%UnWtppKWS zND=l-N`W-X(1wHfQ+Q1HSoB3sku7&Kfu1ZvMX~Y-k-K2eRsnwVu&(;YWEK%Py|*$*|t(69G?K#h`{#z;`;t zl?jP*7nb(AH8Q``e`5I|V8R%T*9i-CS*)FMs2vy`zZ^BSLhu?IBaln|yy$e)BK!(X W;67Xdtm74Yv%QHHmPUG2o%{tDWjM$H literal 0 HcmV?d00001 From 992ac3bbb8e3d0830cc5c94c4eeaf06e07b553d5 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Wed, 5 Nov 2014 21:44:45 +0000 Subject: [PATCH 7/9] Second attempt to fix duplicate-song-detection on windows --- .../songs/forms/duplicatesongremovalform.py | 20 ++++++++++--------- openlp/plugins/songs/lib/songcompare.py | 11 +++++----- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/openlp/plugins/songs/forms/duplicatesongremovalform.py b/openlp/plugins/songs/forms/duplicatesongremovalform.py index 4bcce1c44..16867ea0b 100644 --- a/openlp/plugins/songs/forms/duplicatesongremovalform.py +++ b/openlp/plugins/songs/forms/duplicatesongremovalform.py @@ -33,6 +33,7 @@ The duplicate song removal logic for OpenLP. import logging import multiprocessing import os +import functools from PyQt4 import QtCore, QtGui @@ -46,17 +47,16 @@ from openlp.plugins.songs.lib.songcompare import songs_probably_equal log = logging.getLogger(__name__) -def song_generator(songs): +def tuple_generator(number_of_songs): """ - This is a generator function to return tuples of tuple with two songs and their position in the song array. - When completed then all songs have once been returned combined with any other songs. + This is a generator function to return tuples of two songs position. When completed then all songs position have + once been returned combined with any other songs position. - :param songs: All songs in the database. + :param number_of_songs: Number of songs in the DB. """ - for outer_song_counter in range(len(songs) - 1): - for inner_song_counter in range(outer_song_counter + 1, len(songs)): - yield ((outer_song_counter, songs[outer_song_counter].search_lyrics), - (inner_song_counter, songs[inner_song_counter].search_lyrics)) + for outer_song_counter in range(number_of_songs - 1): + for inner_song_counter in range(outer_song_counter + 1, number_of_songs): + yield (outer_song_counter, inner_song_counter) class DuplicateSongRemovalForm(OpenLPWizard, RegistryProperties): @@ -184,7 +184,9 @@ class DuplicateSongRemovalForm(OpenLPWizard, RegistryProperties): # Create a worker/process pool to check the songs. process_number = max(1, multiprocessing.cpu_count() - 1) pool = multiprocessing.Pool(process_number) - result = pool.imap_unordered(songs_probably_equal, song_generator(songs), 30) + # Create array with all lyrics + song_lyrics = [song.search_lyrics for song in songs] + result = pool.imap_unordered(functools.partial(songs_probably_equal, song_lyrics), tuple_generator(len(songs)), 30) # Do not accept any further tasks. Also this closes the processes if all tasks are done. pool.close() # While the processes are still working, start to look at the results. diff --git a/openlp/plugins/songs/lib/songcompare.py b/openlp/plugins/songs/lib/songcompare.py index ddd5e4552..9101245f5 100644 --- a/openlp/plugins/songs/lib/songcompare.py +++ b/openlp/plugins/songs/lib/songcompare.py @@ -52,15 +52,14 @@ MIN_BLOCK_SIZE = 70 MAX_TYPO_SIZE = 3 -def songs_probably_equal(song_tupel): +def songs_probably_equal(songs, pos_tupel): """ Calculate and return whether two songs are probably equal. :param song_tupel: A tuple of two songs to compare. """ - song1, song2 = song_tupel - pos1, lyrics1 = song1 - pos2, lyrics2 = song2 + lyrics1 = songs[pos_tupel[0]] + lyrics2 = songs[pos_tupel[1]] if len(lyrics1) < len(lyrics2): small = lyrics1 large = lyrics2 @@ -79,7 +78,7 @@ def songs_probably_equal(song_tupel): length_of_equal_blocks += _op_length(element) if length_of_equal_blocks >= MIN_BLOCK_SIZE: - return pos1, pos2 + return pos_tupel[0], pos_tupel[1] # Check 2: Similarity based on the relative length of the longest equal block. # Calculate the length of the largest equal block of the diff set. length_of_longest_equal_block = 0 @@ -87,7 +86,7 @@ def songs_probably_equal(song_tupel): if element[0] == "equal" and _op_length(element) > length_of_longest_equal_block: length_of_longest_equal_block = _op_length(element) if length_of_longest_equal_block > len(small) * 2 // 3: - return pos1, pos2 + return pos_tupel[0], pos_tupel[1] # Both checks failed. We assume the songs are not equal. return None From b6e0036383852404042a0f3595fbc525ce966d1f Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Thu, 6 Nov 2014 10:34:07 +0100 Subject: [PATCH 8/9] Re-re-fix duplicate song check on windows, and now it actually works in builds. --- openlp.py | 9 ++++++++- .../songs/forms/duplicatesongremovalform.py | 20 +++++++++---------- openlp/plugins/songs/lib/songcompare.py | 11 +++++----- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/openlp.py b/openlp.py index 5d507606d..bbd7f1922 100755 --- a/openlp.py +++ b/openlp.py @@ -28,7 +28,9 @@ ############################################################################### import sys +import multiprocessing +from openlp.core.common import is_win, is_macosx from openlp.core import main @@ -36,9 +38,14 @@ if __name__ == '__main__': """ Instantiate and run the application. """ + # Add support for using multiprocessing from frozen Windows executable (built using PyInstaller), + # see https://docs.python.org/3/library/multiprocessing.html#multiprocessing.freeze_support + if is_win(): + multiprocessing.freeze_support() # Mac OS X passes arguments like '-psn_XXXX' to the application. This argument is actually a process serial number. # However, this causes a conflict with other OpenLP arguments. Since we do not use this argument we can delete it # to avoid any potential conflicts. - if sys.platform.startswith('darwin'): + #if sys.platform.startswith('darwin'): + if is_macosx(): sys.argv = [x for x in sys.argv if not x.startswith('-psn')] main() diff --git a/openlp/plugins/songs/forms/duplicatesongremovalform.py b/openlp/plugins/songs/forms/duplicatesongremovalform.py index 16867ea0b..4bcce1c44 100644 --- a/openlp/plugins/songs/forms/duplicatesongremovalform.py +++ b/openlp/plugins/songs/forms/duplicatesongremovalform.py @@ -33,7 +33,6 @@ The duplicate song removal logic for OpenLP. import logging import multiprocessing import os -import functools from PyQt4 import QtCore, QtGui @@ -47,16 +46,17 @@ from openlp.plugins.songs.lib.songcompare import songs_probably_equal log = logging.getLogger(__name__) -def tuple_generator(number_of_songs): +def song_generator(songs): """ - This is a generator function to return tuples of two songs position. When completed then all songs position have - once been returned combined with any other songs position. + This is a generator function to return tuples of tuple with two songs and their position in the song array. + When completed then all songs have once been returned combined with any other songs. - :param number_of_songs: Number of songs in the DB. + :param songs: All songs in the database. """ - for outer_song_counter in range(number_of_songs - 1): - for inner_song_counter in range(outer_song_counter + 1, number_of_songs): - yield (outer_song_counter, inner_song_counter) + for outer_song_counter in range(len(songs) - 1): + for inner_song_counter in range(outer_song_counter + 1, len(songs)): + yield ((outer_song_counter, songs[outer_song_counter].search_lyrics), + (inner_song_counter, songs[inner_song_counter].search_lyrics)) class DuplicateSongRemovalForm(OpenLPWizard, RegistryProperties): @@ -184,9 +184,7 @@ class DuplicateSongRemovalForm(OpenLPWizard, RegistryProperties): # Create a worker/process pool to check the songs. process_number = max(1, multiprocessing.cpu_count() - 1) pool = multiprocessing.Pool(process_number) - # Create array with all lyrics - song_lyrics = [song.search_lyrics for song in songs] - result = pool.imap_unordered(functools.partial(songs_probably_equal, song_lyrics), tuple_generator(len(songs)), 30) + result = pool.imap_unordered(songs_probably_equal, song_generator(songs), 30) # Do not accept any further tasks. Also this closes the processes if all tasks are done. pool.close() # While the processes are still working, start to look at the results. diff --git a/openlp/plugins/songs/lib/songcompare.py b/openlp/plugins/songs/lib/songcompare.py index 9101245f5..ddd5e4552 100644 --- a/openlp/plugins/songs/lib/songcompare.py +++ b/openlp/plugins/songs/lib/songcompare.py @@ -52,14 +52,15 @@ MIN_BLOCK_SIZE = 70 MAX_TYPO_SIZE = 3 -def songs_probably_equal(songs, pos_tupel): +def songs_probably_equal(song_tupel): """ Calculate and return whether two songs are probably equal. :param song_tupel: A tuple of two songs to compare. """ - lyrics1 = songs[pos_tupel[0]] - lyrics2 = songs[pos_tupel[1]] + song1, song2 = song_tupel + pos1, lyrics1 = song1 + pos2, lyrics2 = song2 if len(lyrics1) < len(lyrics2): small = lyrics1 large = lyrics2 @@ -78,7 +79,7 @@ def songs_probably_equal(songs, pos_tupel): length_of_equal_blocks += _op_length(element) if length_of_equal_blocks >= MIN_BLOCK_SIZE: - return pos_tupel[0], pos_tupel[1] + return pos1, pos2 # Check 2: Similarity based on the relative length of the longest equal block. # Calculate the length of the largest equal block of the diff set. length_of_longest_equal_block = 0 @@ -86,7 +87,7 @@ def songs_probably_equal(songs, pos_tupel): if element[0] == "equal" and _op_length(element) > length_of_longest_equal_block: length_of_longest_equal_block = _op_length(element) if length_of_longest_equal_block > len(small) * 2 // 3: - return pos_tupel[0], pos_tupel[1] + return pos1, pos2 # Both checks failed. We assume the songs are not equal. return None From ff4ddefa779e23cb7aefc17bf0ccd5afd1575d09 Mon Sep 17 00:00:00 2001 From: Tomas Groth Date: Thu, 6 Nov 2014 10:42:54 +0100 Subject: [PATCH 9/9] pep8 fix --- openlp.py | 1 - openlp/core/ui/projector/__init__.py | 2 +- tests/functional/openlp_core_ui/test_settingsform.py | 2 +- tests/functional/openlp_plugins/images/test_imagetab.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/openlp.py b/openlp.py index bbd7f1922..e383cd86a 100755 --- a/openlp.py +++ b/openlp.py @@ -45,7 +45,6 @@ if __name__ == '__main__': # Mac OS X passes arguments like '-psn_XXXX' to the application. This argument is actually a process serial number. # However, this causes a conflict with other OpenLP arguments. Since we do not use this argument we can delete it # to avoid any potential conflicts. - #if sys.platform.startswith('darwin'): if is_macosx(): sys.argv = [x for x in sys.argv if not x.startswith('-psn')] main() diff --git a/openlp/core/ui/projector/__init__.py b/openlp/core/ui/projector/__init__.py index 9b51f110b..9a09145ac 100644 --- a/openlp/core/ui/projector/__init__.py +++ b/openlp/core/ui/projector/__init__.py @@ -28,4 +28,4 @@ ############################################################################### """ The Projector driver module. -""" \ No newline at end of file +""" diff --git a/tests/functional/openlp_core_ui/test_settingsform.py b/tests/functional/openlp_core_ui/test_settingsform.py index 17a8b165f..7ea2339a7 100644 --- a/tests/functional/openlp_core_ui/test_settingsform.py +++ b/tests/functional/openlp_core_ui/test_settingsform.py @@ -157,4 +157,4 @@ class TestSettingsForm(TestCase): # THEN: The general tab's cancel() method should have been called, but not the themes tab mocked_general_cancel.assert_called_with() - self.assertEqual(0, mocked_theme_cancel.call_count, 'The Themes tab\'s cancel() should not have been called') \ No newline at end of file + self.assertEqual(0, mocked_theme_cancel.call_count, 'The Themes tab\'s cancel() should not have been called') diff --git a/tests/functional/openlp_plugins/images/test_imagetab.py b/tests/functional/openlp_plugins/images/test_imagetab.py index fade284a4..a4614e816 100644 --- a/tests/functional/openlp_plugins/images/test_imagetab.py +++ b/tests/functional/openlp_plugins/images/test_imagetab.py @@ -95,4 +95,4 @@ class TestImageMediaItem(TestCase, TestMixin): self.form.save() # THEN: the post process should be requested self.assertEqual(1, self.form.settings_form.register_post_process.call_count, - 'Image Post processing should have been requested') \ No newline at end of file + 'Image Post processing should have been requested')