diff --git a/openlp/plugins/biblemanager/BibleDBImpl.py b/openlp/plugins/biblemanager/BibleDBImpl.py index bb163cdbf..4439f9c4d 100644 --- a/openlp/plugins/biblemanager/BibleDBImpl.py +++ b/openlp/plugins/biblemanager/BibleDBImpl.py @@ -30,7 +30,12 @@ sys.path.insert(0,(os.path.join(mypath, '..', '..', '..'))) from openlp.utils import ConfigHelper -logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s') +import logging +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M', + filename='plugins.log', + filemode='w') class BibleDBException(Exception): pass class BibleInvalidDatabaseError(Exception): @@ -107,14 +112,17 @@ mapper(Book, book_table) mapper(Verse, verse_table) class BibleDBImpl: + global log + log=logging.getLogger("BibleDBMgr") + log.info("BibleDB manager loaded") def __init__(self, biblename, btype = 'sqlite'): # Connect to database path = ConfigHelper.getBiblePath() - #print path - #print biblename + #log.debug( path + #log.debug( biblename self.biblefile = os.path.join(path, biblename+".bible") - #print self.biblefile - #print btype + #log.debug( self.biblefile + #log.debug( btype if btype == 'sqlite': self.db = create_engine("sqlite:///"+self.biblefile) elif btype == 'mysql': @@ -137,24 +145,24 @@ class BibleDBImpl: verse_table.create() self.loadMeta("dbversion", "0.1") - def createChapter(self, bk, chap, textlist): - print "createChapter ", bk, chap, textlist + def createChapter(self, bookname, chap, textlist): + log.debug( "createChapter %s,%s,%s", bookname, chap, textlist) metadata.bind.echo = False session = self.Session() s = text (""" select id FROM book where book.name == :b """) - data = self.db.execute(s, b=bk).fetchone() + data = self.db.execute(s, b=bookname).fetchone() id = data[0] # id is first record in list. - #print "id = " , id + #log.debug( "id = " , id for v , t in textlist.iteritems(): versemeta = Verse(book_id=id, chapter=int(chap), verse=int(v), text=(t)) session.add(versemeta) session.commit() - def createBook(self, bk): - print "createBook ", bk + def createBook(self, bookid, bookname, bookabbrev): + log.debug( "createBook %s,%s,%s", bookid, bookname, bookabbrev) metadata.bind.echo = False session = self.Session() - bookmeta = Book(int(5), bk, bk) + bookmeta = Book(int(5), bookname, bookabbrev) session.add(bookmeta) session.commit() @@ -169,6 +177,11 @@ class BibleDBImpl: s = text (""" select value FROM meta where key == :k """) return self.db.execute(s, k=key).fetchone() + def deleteMeta(self, key): + metadata.bind.echo = False + s = text (""" delete FROM meta where key == :k """) + self.db.execute(s, k=key) + def _loadTestaments(self): metadata.bind.echo = False session = self.Session() @@ -192,7 +205,7 @@ class BibleDBImpl: fverse=open(versesfile, 'r') for line in fbooks: - #print line + #log.debug( line) p = line.split(",") p[2] = self._cleanText(p[2]) p[3] = self._cleanText(p[3]) @@ -202,47 +215,47 @@ class BibleDBImpl: book_ptr = "" for line in fverse: - #print line + #log.debug( line) p = line.split(",", 3) # split into 3 units and leave the rest as a single field p[0] = self._cleanText(p[0]) p[3] = self._cleanText(p[3]) if book_ptr is not p[0]: query = session.query(Book).filter(Book.name==p[0]) - #print query - #print query.first() - #print query.first().id + #log.debug( query) + #log.debug( query.first()) + #log.debug( query.first().id) book_ptr = p[0] - #print text + #log.debug( text) versemeta = Verse(book_id=query.first().id, chapter=int(p[1]), verse=int(p[2]), text=(p[3])) session.add(versemeta) session.commit() def getBibleBook(self, bookname): - print "getBibleBook ", bookname + log.debug( "getBibleBook %s", bookname) metadata.bind.echo = False s = text (""" select name FROM book where book.name == :b """) return self.db.execute(s, b=bookname).fetchone() def getBibleChapter(self, bookname, chapter): - print "getBibleChapter ", bookname, chapter + log.debug( "getBibleChapter %s,%s", bookname, chapter ) metadata.bind.echo = False s = text (""" select book.name FROM verse,book where verse.book_id == book.id AND verse.chapter == :c and book.name == :b """) return self.db.execute(s, c=chapter, b=bookname).fetchone() def getBibleText(self, bookname, chapter, sverse, everse): - print "getBibleText ", bookname, chapter, sverse, everse + log.debug( "getBibleText %s,%s,%s,%s", bookname, chapter, sverse, everse) metadata.bind.echo = False s = text (""" select verse.verse, verse.text FROM verse,book where verse.book_id == book.id AND verse.chapter == :c AND (verse.verse between :v1 and :v2) and book.name == :b """) return self.db.execute(s, c=chapter, v1=sverse , v2=everse, b=bookname).fetchall() def dumpBible(self): - print ".........Dumping Bible Database" - print "...............................Books " + log.debug( ".........Dumping Bible Database") + log.debug( "...............................Books ") s = text (""" select * FROM book """) - print self.db.execute(s).fetchall() - print "...............................Verses " + log.debug( self.db.execute(s).fetchall()) + log.debug( "...............................Verses ") s = text (""" select * FROM verse """) - print self.db.execute(s).fetchall() + log.debug( self.db.execute(s).fetchall()) def _cleanText(self, text): text = text.replace('\n', '') diff --git a/openlp/plugins/biblemanager/BibleHTTPImpl.py b/openlp/plugins/biblemanager/BibleHTTPImpl.py index fa8ddc295..ece7cfa89 100644 --- a/openlp/plugins/biblemanager/BibleHTTPImpl.py +++ b/openlp/plugins/biblemanager/BibleHTTPImpl.py @@ -20,7 +20,17 @@ import os, os.path import sys import urllib2 +import logging +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M', + filename='plugins.log', + filemode='w') + class BibleHTTPImpl: + global log + log=logging.getLogger("BibleHTTPMgr") + log.info("BibleHTTP manager loaded") def __init__(self): """ Finds all the bibles defined for the system @@ -35,9 +45,9 @@ class BibleHTTPImpl: def setBibleSource(self,biblesource): self.biblesource = biblesource - def getBibleChapter(self, version, book, chapter): - urlstring = "http://bible.crosswalk.com/OnlineStudyBible/bible.cgi?word="+book+"+"+str(chapter)+"&version="+version - print urlstring + def getBibleChapter(self, version, bookid,bookname, chapter): + urlstring = "http://bible.crosswalk.com/OnlineStudyBible/bible.cgi?word="+bookname+"+"+str(chapter)+"&version="+version + log.debug( urlstring) xml_string = "" req = urllib2.Request(urlstring) req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)') @@ -46,8 +56,8 @@ class BibleHTTPImpl: xml_string = handle.read() except IOError, e: if hasattr(e, 'reason'): - print 'Reason : ' - print e.reason + log.error( 'Reason : ') + log.error( e.reason) i= xml_string.find("NavCurrentChapter") xml_string = xml_string[i:len(xml_string)] @@ -58,16 +68,16 @@ class BibleHTTPImpl: i= xml_string.find("") # Remove the heading for the book xml_string = xml_string[i + 3 :len(xml_string)] #remove the at the front versePos = xml_string.find("
") - #print versePos + #log.debug( versePos) bible = {} cleanbible = {} while versePos > 0: versePos = xml_string.find("", versePos) + 6 i = xml_string.find("", versePos) - #print versePos, i + #log.debug( versePos, i) verse= xml_string[versePos:i] # Got the Chapter #verse = int(temp) - #print "Chapter = " + str(temp) + #log.debug( "Chapter = " + str(temp)) versePos = i + 8 # move the starting position to negining of the text i = xml_string.find("", versePos) # fine the start of the next verse if i == -1: @@ -75,12 +85,12 @@ class BibleHTTPImpl: verseText = xml_string[versePos: i] versePos = 0 else: - #print i, versePos + #log.debug( i, versePos) verseText = xml_string[versePos: i] versePos = i bible[verse] = self._cleanVerse(verseText) - #print bible + #log.debug( bible) return bible def _cleanVerse(self, text): diff --git a/openlp/plugins/biblemanager/BibleManager.py b/openlp/plugins/biblemanager/BibleManager.py index a08a4a7c8..41891c6b6 100644 --- a/openlp/plugins/biblemanager/BibleManager.py +++ b/openlp/plugins/biblemanager/BibleManager.py @@ -26,7 +26,17 @@ from openlp.plugins.biblemanager.BibleDBImpl import BibleDBImpl from openlp.plugins.biblemanager.BibleHTTPImpl import BibleHTTPImpl from openlp.plugins.plugin import Plugin +import logging +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M', + filename='plugins.log', + filemode='w') + class BibleManager(Plugin): + global log + log=logging.getLogger("BibleMgr") + log.info("Bible manager loaded") def __init__(self): """ Finds all the bibles defined for the system @@ -35,6 +45,7 @@ class BibleManager(Plugin): Init confirms the bible exists and stores the database path. """ + log.debug( "Bible Initialising") self.bibleDBCache = {} self.bibleHTTPCache = {} self.booksOfBible = {} @@ -44,7 +55,7 @@ class BibleManager(Plugin): self.verses = {} self.verseData = {} self.biblePath = ConfigHelper.getBiblePath() - #print self.biblePath + #log.debug( self.biblePath ) files = os.listdir(self.biblePath) fbibles=open("../resources/bibles_en.txt", 'r') fbibledata=open("../resources/bible_books.txt", 'r') @@ -57,68 +68,69 @@ class BibleManager(Plugin): nhttp.setBibleSource(biblesource) self.bibleHTTPCache[b] = nhttp # - for line in fbibles: - p = line.split(",") - self.booksOfBible[p[0]] = p[1].replace('\n', '') - self.listOfBooks.insert(int(p[1].replace('\n', '')), p[0]) - for line in fbibledata: - p = line.split(",") - self.booksChapters[p[0]]=p[1] - self.CWids[p[0]]=p[2].replace('\n', '') - v = p[3].replace('\n', '') - self.verseData[p[0]] = v - #print "\n", self.booksOfBible - #print "\n", self.booksChapters - #print "\n", self.CWids - #print "\n", self.verseData + #Load in memory objects + for line in fbibles: + p = line.split(",") + self.booksOfBible[p[0]] = p[1].replace('\n', '') + self.listOfBooks.insert(int(p[1].replace('\n', '')), p[0]) + for line in fbibledata: + p = line.split(",") + self.booksChapters[p[0]]=p[1] + self.CWids[p[0]]=p[2].replace('\n', '') + v = p[3].replace('\n', '') + self.verseData[p[0]] = v + #log.debug( "\nbooks of the Bible", self.booksOfBible) + #log.debug( "\n", self.booksChapters) + #log.debug( "\n", self.CWids) + #log.debug( "\n", self.verseData) - #print self.bibleDBCache - #print self.bibleHTTPCache + #log.debug( self.bibleDBCache) + #log.debug( self.bibleHTTPCache ) + log.debug( "Bible Initialised") - def registerHTTPBible(self, name, biblesource, proxy, proxyport, proxyid, proxypass): + def registerHTTPBible(self, biblename, biblesource, proxy, proxyport, proxyid, proxypass): """ Return a list of bibles from a given URL. The selected Bible can then be registered and LazyLoaded into a database """ - if self._isNewBible(name): - nbible = BibleDBImpl(name) # Create new Bible + if self._isNewBible(biblename): + nbible = BibleDBImpl(biblename) # Create new Bible nbible.createTables() # Create Database - self.bibleDBCache[name] = nbible + self.bibleDBCache[biblename] = nbible nhttp = BibleHTTPImpl() nhttp.setBibleSource(biblesource) - self.bibleHTTPCache[name] = nhttp + self.bibleHTTPCache[biblename] = nhttp nbible.loadMeta("WEB", biblesource) # register a lazy loading interest - def registerBible(self, name, booksfile, versefile): + def registerFileBible(self, biblename, booksfile, versefile): """ Method to load a bible from a set of files into a database. If the database exists it is deleted and the database is reloaded from scratch. """ - if self._isNewBible(name): - nbible = BibleDBImpl(name) # Create new Bible + if self._isNewBible(biblename): + nbible = BibleDBImpl(biblename) # Create new Bible nbible.createTables() # Create Database nbible.loadData(booksfile, versefile) - self.bibleDBCache[name] = nbible + self.bibleDBCache[biblename] = nbible - def loadBible(self,bible): - print "loadBible ", bible + def loadBible(self,biblename): + log.debug( "loadBible %s", biblename) """ Downloads all the books of the bible and loads it into the database """ - for bk in self.listOfBooks: - cptrs = self.booksChapters[ self.booksOfBible[bk]] - print bk , self.booksChapters[ self.booksOfBible[bk]] + for bookname in self.listOfBooks: + cptrs = self.booksChapters[ self.booksOfBible[bookname]] + log.debug( "book and chapter %s %s", bookname , self.booksChapters[ self.booksOfBible[bookname]] ) for chptr in range(1 , int(cptrs)): # loop through all the chapters in book - c = self.bibleDBCache[bible].getBibleChapter(bk, chptr) # check to see if book/chapter exists - print c + c = self.bibleDBCache[biblename].getBibleChapter(bookname, chptr) # check to see if book/chapter exists + log.debug( "got chapter %s", c) if not c: - print "missing ", bk, chptr - self._loadBook(bible,bk) - self._loadChapter(bible, bk, chptr) - + log.debug( "missing %s,%s", bookname, chptr) + self._loadBook(biblename,bookname) + self._loadChapter(biblename, bookname, chptr) def getBibles(self): """ @@ -136,64 +148,75 @@ class BibleManager(Plugin): return self.listOfBooks def getBookChapterCount(self, book): - print "getBookChapterCount ", book + log.debug( "getBookChapterCount %s", book) """ Returns all the number of Chapters for a given book """ - i = self.booksOfBible[book] - return self.booksChapters[i] + bookid = self.booksOfBible[book] + return self.booksChapters[bookid] def getBookVerseCount(self, book, chapter): - print "getBookVerseCount ", book, chapter + log.debug( "getBookVerseCount %s,%s", book, chapter) """ Returns all the number of verses for a given book and chapter """ - i = self.booksOfBible[book] - v = self.verseData[i].split(":") - print v - return v[chapter-1] # sup 1 for zero indexing + bookid = self.booksOfBible[book] + v = self.verseData[bookid].split(":") + log.debug( v) + return v[chapter-1] # sub 1 for zero indexing - def getVerseText(self, bible, book, chapter, sverse, everse = 0 ): + def getVerseText(self, bible, bookname, chapter, sverse, everse = 0 ): """ Returns a list of verses for a given Book, Chapter and ranges of verses. If the end verse(everse) is less then the start verse(sverse) then only one verse is returned + Bible - Which bible + Book - full text description so needs to be converted. + Rest can be guessed at ! """ - #print self.bibleDBCache - #print self.bibleHTTPCache - print "getchapter ", bible, book, chapter, sverse, everse - c = self.bibleDBCache[bible].getBibleChapter(book, chapter) # check to see if book/chapter exists - print c + #log.debug( self.bibleDBCache) + #log.debug( self.bibleHTTPCache) + log.debug( "getchapter %s,%s,%s,%s,%s", bible, bookname, chapter, sverse, everse) + bookid = self.booksOfBible[bookname] # convert to id ie Genesis --> 1 Revelation --> 73 + # SORT OUT BOOKNAME BOOK ID. + # NAME COMES IN TO ID AND BACK TO NAME ? + c = self.bibleDBCache[bible].getBibleChapter(bookname, chapter) # check to see if book/chapter exists + bookabbrev = "" + log.debug( "Bible Chapter %s", c ) if not c: - self._loadBook(bible,book) - self._loadChapter(bible, book, chapter) + self._loadBook(bible,bookid, bookname, bookabbrev) + self._loadChapter(bible, bookid,bookname, chapter) if everse < sverse: everse = sverse - text = self.bibleDBCache[bible].getBibleText(book, chapter, sverse, everse) - #print text + text = self.bibleDBCache[bible].getBibleText(bookname, chapter, sverse, everse) + #log.debug( text) #self.bibleDBCache[bible].dumpBible() return text - def _loadBook(self, bible, book): - print "loadbook ", bible, book - cl = self.bibleDBCache[bible].getBibleBook(book) - #print cl + def _loadBook(self, bible, bookid, bookname, bookabbrev): + log.debug( "loadbook %s,%s,%s,%s", bible, bookid, bookname, bookabbrev) + cl = self.bibleDBCache[bible].getBibleBook(bookname) + log.debug( "get bible book %s" , cl) if not cl : - self.bibleDBCache[bible].createBook(book) + self.bibleDBCache[bible].createBook(bookid, bookname, bookabbrev) - def _loadChapter(self, bible, book, chapter): - print "loadChapter ", bible, book, chapter - chaptlist = self.bibleHTTPCache[bible].getBibleChapter(bible, book, chapter) - self.bibleDBCache[bible].createChapter(book, chapter, chaptlist) + def _loadChapter(self, bible, bookid,bookname, chapter): + log.debug( "loadChapter %s,%s,%s,%s", bible, bookid,bookname, chapter) + try : + print self.bibleHTTPCache[bible] + chaptlist = self.bibleHTTPCache[bible].getBibleChapter(bible, bookid,bookname, chapter) + self.bibleDBCache[bible].createChapter(bookname, chapter, chaptlist) + except: + log.error("Bible %s not found in Http cache", bible) def _isNewBible(self, name): """ Check cache to see if new bible """ for b , o in self.bibleDBCache.iteritems(): - print b + log.debug( b ) if b == name : return False return True diff --git a/openlp/plugins/biblemanager/test/test_bibleManager.py b/openlp/plugins/biblemanager/test/test_bibleManager.py index 68d87e4a1..d812cac44 100644 --- a/openlp/plugins/biblemanager/test/test_bibleManager.py +++ b/openlp/plugins/biblemanager/test/test_bibleManager.py @@ -27,70 +27,87 @@ sys.path.insert(0,(os.path.join(mypath, '..', '..','..','..'))) from openlp.plugins.biblemanager.BibleManager import BibleManager from openlp.utils import ConfigHelper +import logging +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M', + filename='plugins.log', + filemode='w') + +console=logging.StreamHandler() +# set a format which is simpler for console use +formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') +# tell the handler to use this format +console.setFormatter(formatter) +logging.getLogger('').addHandler(console) +log=logging.getLogger('') + +logging.info("\nLogging started") + class TestBibleManager: - + log=logging.getLogger("testBibleMgr") def setup_class(self): - print "\nRegister BM" + log.debug("\n.......Register BM") self.bm = BibleManager() def testRegisterBibleFiles(self): # Register a bible from files - print "\ntestRegisterBibleFiles" - self.bm.registerBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv') - self.bm.registerBible("NIV",'biblebooks_niv_short.csv','bibleverses_niv_short.csv') + log.debug("\n.......testRegisterBibleFiles") + self.bm.registerFileBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv') + self.bm.registerFileBible("NIV",'biblebooks_niv_short.csv','bibleverses_niv_short.csv') b = self.bm.getBibles() for b1 in b: - print b1 + log.debug( b1) assert(b1 in b) def testRegisterBibleHTTP(self): # Register a bible from files - print "\ntestRegisterBibleHTTP" + log.debug( "\n.......testRegisterBibleHTTP") self.bm.registerHTTPBible("asv","Crosswalk", "", "", "", "") #self.bm.registerBible("NIV", "ge", 1) b = self.bm.getBibles() for b1 in b: - print b1 + log.debug( b1) assert(b1 in b) def testGetBibles(self): - print "\ntestGetBibles" + log.debug( "\n.......testGetBibles") # make sure the shuffled sequence does not lose any elements b = self.bm.getBibles() for b1 in b: - print b1 + log.debug( b1) assert(b1 in b) - def testGetBooks(self): - print "\ntestGetBooks" + def testGetBibleBooks(self): + log.debug( "\n.......testGetBibleBooks") c = self.bm.getBibleBooks("NIV") for c1 in c: - print c1 + log.debug( c1) assert(c1 in c) - def testGetChapterCount(self): - print "\ntestGetChapterCount" + def testGetBookChapterCount(self): + log.debug( "\n.......testGetBookChapterCount") assert(self.bm.getBookChapterCount("Matthew") == '28') - def testGetVerseCount(self): - print "\ntestGetVerseCount\n" + def testGetBookVerseCount(self): + log.debug( "\n.......testGetBookVerseCount") assert(self.bm.getBookVerseCount("Genesis", 1) == '31') assert(self.bm.getBookVerseCount("Genesis", 2) == '25') assert(self.bm.getBookVerseCount("Matthew", 1) == '25') assert(self.bm.getBookVerseCount("Revelation", 1) == '20') def testGetVerseText(self): - print "\ntestGetVerseText" - c = self.bm.getVerseText("TheMessage",'Genesis',1,2, 1) - print c + log.debug( "\n.......testGetVerseText") + c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1) + log.debug( c ) c = self.bm.getVerseText('NIV','Genesis',1,1,2) - print c - c = self.bm.getVerseText('asv','re',1,1,2) - print c - c = self.bm.getVerseText('asv','re',1,5,9) - print c + log.debug( c ) + c = self.bm.getVerseText('asv','Revelation',1,1,2) + log.debug( c ) + c = self.bm.getVerseText('asv','Revelation',1,5,9) + log.debug( c ) def testLoadBible(self): - print "\ntestLoadBible" - self.bm.loadBible('asv') + log.debug( "\n.......testLoadBible") + #self.bm.loadBible('asv')