Fix bugs and add logging to test classes.

Sort out varable names to be meaningful.

bzr-revno: 82
This commit is contained in:
Tim Bentley 2008-11-06 18:49:36 +00:00
parent 5b4210c137
commit 3752f4279d
4 changed files with 191 additions and 128 deletions

View File

@ -30,7 +30,12 @@ sys.path.insert(0,(os.path.join(mypath, '..', '..', '..')))
from openlp.utils import ConfigHelper
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s')
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
class BibleDBException(Exception):
pass
class BibleInvalidDatabaseError(Exception):
@ -107,14 +112,17 @@ mapper(Book, book_table)
mapper(Verse, verse_table)
class BibleDBImpl:
global log
log=logging.getLogger("BibleDBMgr")
log.info("BibleDB manager loaded")
def __init__(self, biblename, btype = 'sqlite'):
# Connect to database
path = ConfigHelper.getBiblePath()
#print path
#print biblename
#log.debug( path
#log.debug( biblename
self.biblefile = os.path.join(path, biblename+".bible")
#print self.biblefile
#print btype
#log.debug( self.biblefile
#log.debug( btype
if btype == 'sqlite':
self.db = create_engine("sqlite:///"+self.biblefile)
elif btype == 'mysql':
@ -137,24 +145,24 @@ class BibleDBImpl:
verse_table.create()
self.loadMeta("dbversion", "0.1")
def createChapter(self, bk, chap, textlist):
print "createChapter ", bk, chap, textlist
def createChapter(self, bookname, chap, textlist):
log.debug( "createChapter %s,%s,%s", bookname, chap, textlist)
metadata.bind.echo = False
session = self.Session()
s = text (""" select id FROM book where book.name == :b """)
data = self.db.execute(s, b=bk).fetchone()
data = self.db.execute(s, b=bookname).fetchone()
id = data[0] # id is first record in list.
#print "id = " , id
#log.debug( "id = " , id
for v , t in textlist.iteritems():
versemeta = Verse(book_id=id, chapter=int(chap), verse=int(v), text=(t))
session.add(versemeta)
session.commit()
def createBook(self, bk):
print "createBook ", bk
def createBook(self, bookid, bookname, bookabbrev):
log.debug( "createBook %s,%s,%s", bookid, bookname, bookabbrev)
metadata.bind.echo = False
session = self.Session()
bookmeta = Book(int(5), bk, bk)
bookmeta = Book(int(5), bookname, bookabbrev)
session.add(bookmeta)
session.commit()
@ -169,6 +177,11 @@ class BibleDBImpl:
s = text (""" select value FROM meta where key == :k """)
return self.db.execute(s, k=key).fetchone()
def deleteMeta(self, key):
metadata.bind.echo = False
s = text (""" delete FROM meta where key == :k """)
self.db.execute(s, k=key)
def _loadTestaments(self):
metadata.bind.echo = False
session = self.Session()
@ -192,7 +205,7 @@ class BibleDBImpl:
fverse=open(versesfile, 'r')
for line in fbooks:
#print line
#log.debug( line)
p = line.split(",")
p[2] = self._cleanText(p[2])
p[3] = self._cleanText(p[3])
@ -202,47 +215,47 @@ class BibleDBImpl:
book_ptr = ""
for line in fverse:
#print line
#log.debug( line)
p = line.split(",", 3) # split into 3 units and leave the rest as a single field
p[0] = self._cleanText(p[0])
p[3] = self._cleanText(p[3])
if book_ptr is not p[0]:
query = session.query(Book).filter(Book.name==p[0])
#print query
#print query.first()
#print query.first().id
#log.debug( query)
#log.debug( query.first())
#log.debug( query.first().id)
book_ptr = p[0]
#print text
#log.debug( text)
versemeta = Verse(book_id=query.first().id, chapter=int(p[1]), verse=int(p[2]), text=(p[3]))
session.add(versemeta)
session.commit()
def getBibleBook(self, bookname):
print "getBibleBook ", bookname
log.debug( "getBibleBook %s", bookname)
metadata.bind.echo = False
s = text (""" select name FROM book where book.name == :b """)
return self.db.execute(s, b=bookname).fetchone()
def getBibleChapter(self, bookname, chapter):
print "getBibleChapter ", bookname, chapter
log.debug( "getBibleChapter %s,%s", bookname, chapter )
metadata.bind.echo = False
s = text (""" select book.name FROM verse,book where verse.book_id == book.id AND verse.chapter == :c and book.name == :b """)
return self.db.execute(s, c=chapter, b=bookname).fetchone()
def getBibleText(self, bookname, chapter, sverse, everse):
print "getBibleText ", bookname, chapter, sverse, everse
log.debug( "getBibleText %s,%s,%s,%s", bookname, chapter, sverse, everse)
metadata.bind.echo = False
s = text (""" select verse.verse, verse.text FROM verse,book where verse.book_id == book.id AND verse.chapter == :c AND (verse.verse between :v1 and :v2) and book.name == :b """)
return self.db.execute(s, c=chapter, v1=sverse , v2=everse, b=bookname).fetchall()
def dumpBible(self):
print ".........Dumping Bible Database"
print "...............................Books "
log.debug( ".........Dumping Bible Database")
log.debug( "...............................Books ")
s = text (""" select * FROM book """)
print self.db.execute(s).fetchall()
print "...............................Verses "
log.debug( self.db.execute(s).fetchall())
log.debug( "...............................Verses ")
s = text (""" select * FROM verse """)
print self.db.execute(s).fetchall()
log.debug( self.db.execute(s).fetchall())
def _cleanText(self, text):
text = text.replace('\n', '')

View File

@ -20,7 +20,17 @@ import os, os.path
import sys
import urllib2
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
class BibleHTTPImpl:
global log
log=logging.getLogger("BibleHTTPMgr")
log.info("BibleHTTP manager loaded")
def __init__(self):
"""
Finds all the bibles defined for the system
@ -35,9 +45,9 @@ class BibleHTTPImpl:
def setBibleSource(self,biblesource):
self.biblesource = biblesource
def getBibleChapter(self, version, book, chapter):
urlstring = "http://bible.crosswalk.com/OnlineStudyBible/bible.cgi?word="+book+"+"+str(chapter)+"&version="+version
print urlstring
def getBibleChapter(self, version, bookid,bookname, chapter):
urlstring = "http://bible.crosswalk.com/OnlineStudyBible/bible.cgi?word="+bookname+"+"+str(chapter)+"&version="+version
log.debug( urlstring)
xml_string = ""
req = urllib2.Request(urlstring)
req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')
@ -46,8 +56,8 @@ class BibleHTTPImpl:
xml_string = handle.read()
except IOError, e:
if hasattr(e, 'reason'):
print 'Reason : '
print e.reason
log.error( 'Reason : ')
log.error( e.reason)
i= xml_string.find("NavCurrentChapter")
xml_string = xml_string[i:len(xml_string)]
@ -58,16 +68,16 @@ class BibleHTTPImpl:
i= xml_string.find("<B>") # Remove the heading for the book
xml_string = xml_string[i + 3 :len(xml_string)] #remove the <B> at the front
versePos = xml_string.find("<BLOCKQUOTE>")
#print versePos
#log.debug( versePos)
bible = {}
cleanbible = {}
while versePos > 0:
versePos = xml_string.find("<B><I>", versePos) + 6
i = xml_string.find("</I></B>", versePos)
#print versePos, i
#log.debug( versePos, i)
verse= xml_string[versePos:i] # Got the Chapter
#verse = int(temp)
#print "Chapter = " + str(temp)
#log.debug( "Chapter = " + str(temp))
versePos = i + 8 # move the starting position to negining of the text
i = xml_string.find("<B><I>", versePos) # fine the start of the next verse
if i == -1:
@ -75,12 +85,12 @@ class BibleHTTPImpl:
verseText = xml_string[versePos: i]
versePos = 0
else:
#print i, versePos
#log.debug( i, versePos)
verseText = xml_string[versePos: i]
versePos = i
bible[verse] = self._cleanVerse(verseText)
#print bible
#log.debug( bible)
return bible
def _cleanVerse(self, text):

View File

@ -26,7 +26,17 @@ from openlp.plugins.biblemanager.BibleDBImpl import BibleDBImpl
from openlp.plugins.biblemanager.BibleHTTPImpl import BibleHTTPImpl
from openlp.plugins.plugin import Plugin
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
class BibleManager(Plugin):
global log
log=logging.getLogger("BibleMgr")
log.info("Bible manager loaded")
def __init__(self):
"""
Finds all the bibles defined for the system
@ -35,6 +45,7 @@ class BibleManager(Plugin):
Init confirms the bible exists and stores the database path.
"""
log.debug( "Bible Initialising")
self.bibleDBCache = {}
self.bibleHTTPCache = {}
self.booksOfBible = {}
@ -44,7 +55,7 @@ class BibleManager(Plugin):
self.verses = {}
self.verseData = {}
self.biblePath = ConfigHelper.getBiblePath()
#print self.biblePath
#log.debug( self.biblePath )
files = os.listdir(self.biblePath)
fbibles=open("../resources/bibles_en.txt", 'r')
fbibledata=open("../resources/bible_books.txt", 'r')
@ -57,6 +68,7 @@ class BibleManager(Plugin):
nhttp.setBibleSource(biblesource)
self.bibleHTTPCache[b] = nhttp
#
#Load in memory objects
for line in fbibles:
p = line.split(",")
self.booksOfBible[p[0]] = p[1].replace('\n', '')
@ -67,58 +79,58 @@ class BibleManager(Plugin):
self.CWids[p[0]]=p[2].replace('\n', '')
v = p[3].replace('\n', '')
self.verseData[p[0]] = v
#print "\n", self.booksOfBible
#print "\n", self.booksChapters
#print "\n", self.CWids
#print "\n", self.verseData
#log.debug( "\nbooks of the Bible", self.booksOfBible)
#log.debug( "\n", self.booksChapters)
#log.debug( "\n", self.CWids)
#log.debug( "\n", self.verseData)
#print self.bibleDBCache
#print self.bibleHTTPCache
#log.debug( self.bibleDBCache)
#log.debug( self.bibleHTTPCache )
log.debug( "Bible Initialised")
def registerHTTPBible(self, name, biblesource, proxy, proxyport, proxyid, proxypass):
def registerHTTPBible(self, biblename, biblesource, proxy, proxyport, proxyid, proxypass):
"""
Return a list of bibles from a given URL.
The selected Bible can then be registered and LazyLoaded into a database
"""
if self._isNewBible(name):
nbible = BibleDBImpl(name) # Create new Bible
if self._isNewBible(biblename):
nbible = BibleDBImpl(biblename) # Create new Bible
nbible.createTables() # Create Database
self.bibleDBCache[name] = nbible
self.bibleDBCache[biblename] = nbible
nhttp = BibleHTTPImpl()
nhttp.setBibleSource(biblesource)
self.bibleHTTPCache[name] = nhttp
self.bibleHTTPCache[biblename] = nhttp
nbible.loadMeta("WEB", biblesource) # register a lazy loading interest
def registerBible(self, name, booksfile, versefile):
def registerFileBible(self, biblename, booksfile, versefile):
"""
Method to load a bible from a set of files into a database.
If the database exists it is deleted and the database is reloaded
from scratch.
"""
if self._isNewBible(name):
nbible = BibleDBImpl(name) # Create new Bible
if self._isNewBible(biblename):
nbible = BibleDBImpl(biblename) # Create new Bible
nbible.createTables() # Create Database
nbible.loadData(booksfile, versefile)
self.bibleDBCache[name] = nbible
self.bibleDBCache[biblename] = nbible
def loadBible(self,bible):
print "loadBible ", bible
def loadBible(self,biblename):
log.debug( "loadBible %s", biblename)
"""
Downloads all the books of the bible
and loads it into the database
"""
for bk in self.listOfBooks:
cptrs = self.booksChapters[ self.booksOfBible[bk]]
print bk , self.booksChapters[ self.booksOfBible[bk]]
for bookname in self.listOfBooks:
cptrs = self.booksChapters[ self.booksOfBible[bookname]]
log.debug( "book and chapter %s %s", bookname , self.booksChapters[ self.booksOfBible[bookname]] )
for chptr in range(1 , int(cptrs)): # loop through all the chapters in book
c = self.bibleDBCache[bible].getBibleChapter(bk, chptr) # check to see if book/chapter exists
print c
c = self.bibleDBCache[biblename].getBibleChapter(bookname, chptr) # check to see if book/chapter exists
log.debug( "got chapter %s", c)
if not c:
print "missing ", bk, chptr
self._loadBook(bible,bk)
self._loadChapter(bible, bk, chptr)
log.debug( "missing %s,%s", bookname, chptr)
self._loadBook(biblename,bookname)
self._loadChapter(biblename, bookname, chptr)
def getBibles(self):
"""
@ -136,64 +148,75 @@ class BibleManager(Plugin):
return self.listOfBooks
def getBookChapterCount(self, book):
print "getBookChapterCount ", book
log.debug( "getBookChapterCount %s", book)
"""
Returns all the number of Chapters for a given
book
"""
i = self.booksOfBible[book]
return self.booksChapters[i]
bookid = self.booksOfBible[book]
return self.booksChapters[bookid]
def getBookVerseCount(self, book, chapter):
print "getBookVerseCount ", book, chapter
log.debug( "getBookVerseCount %s,%s", book, chapter)
"""
Returns all the number of verses for a given
book and chapter
"""
i = self.booksOfBible[book]
v = self.verseData[i].split(":")
print v
return v[chapter-1] # sup 1 for zero indexing
bookid = self.booksOfBible[book]
v = self.verseData[bookid].split(":")
log.debug( v)
return v[chapter-1] # sub 1 for zero indexing
def getVerseText(self, bible, book, chapter, sverse, everse = 0 ):
def getVerseText(self, bible, bookname, chapter, sverse, everse = 0 ):
"""
Returns a list of verses for a given Book, Chapter and ranges of verses.
If the end verse(everse) is less then the start verse(sverse)
then only one verse is returned
Bible - Which bible
Book - full text description so needs to be converted.
Rest can be guessed at !
"""
#print self.bibleDBCache
#print self.bibleHTTPCache
print "getchapter ", bible, book, chapter, sverse, everse
c = self.bibleDBCache[bible].getBibleChapter(book, chapter) # check to see if book/chapter exists
print c
#log.debug( self.bibleDBCache)
#log.debug( self.bibleHTTPCache)
log.debug( "getchapter %s,%s,%s,%s,%s", bible, bookname, chapter, sverse, everse)
bookid = self.booksOfBible[bookname] # convert to id ie Genesis --> 1 Revelation --> 73
# SORT OUT BOOKNAME BOOK ID.
# NAME COMES IN TO ID AND BACK TO NAME ?
c = self.bibleDBCache[bible].getBibleChapter(bookname, chapter) # check to see if book/chapter exists
bookabbrev = ""
log.debug( "Bible Chapter %s", c )
if not c:
self._loadBook(bible,book)
self._loadChapter(bible, book, chapter)
self._loadBook(bible,bookid, bookname, bookabbrev)
self._loadChapter(bible, bookid,bookname, chapter)
if everse < sverse:
everse = sverse
text = self.bibleDBCache[bible].getBibleText(book, chapter, sverse, everse)
#print text
text = self.bibleDBCache[bible].getBibleText(bookname, chapter, sverse, everse)
#log.debug( text)
#self.bibleDBCache[bible].dumpBible()
return text
def _loadBook(self, bible, book):
print "loadbook ", bible, book
cl = self.bibleDBCache[bible].getBibleBook(book)
#print cl
def _loadBook(self, bible, bookid, bookname, bookabbrev):
log.debug( "loadbook %s,%s,%s,%s", bible, bookid, bookname, bookabbrev)
cl = self.bibleDBCache[bible].getBibleBook(bookname)
log.debug( "get bible book %s" , cl)
if not cl :
self.bibleDBCache[bible].createBook(book)
self.bibleDBCache[bible].createBook(bookid, bookname, bookabbrev)
def _loadChapter(self, bible, book, chapter):
print "loadChapter ", bible, book, chapter
chaptlist = self.bibleHTTPCache[bible].getBibleChapter(bible, book, chapter)
self.bibleDBCache[bible].createChapter(book, chapter, chaptlist)
def _loadChapter(self, bible, bookid,bookname, chapter):
log.debug( "loadChapter %s,%s,%s,%s", bible, bookid,bookname, chapter)
try :
print self.bibleHTTPCache[bible]
chaptlist = self.bibleHTTPCache[bible].getBibleChapter(bible, bookid,bookname, chapter)
self.bibleDBCache[bible].createChapter(bookname, chapter, chaptlist)
except:
log.error("Bible %s not found in Http cache", bible)
def _isNewBible(self, name):
"""
Check cache to see if new bible
"""
for b , o in self.bibleDBCache.iteritems():
print b
log.debug( b )
if b == name :
return False
return True

View File

@ -27,70 +27,87 @@ sys.path.insert(0,(os.path.join(mypath, '..', '..','..','..')))
from openlp.plugins.biblemanager.BibleManager import BibleManager
from openlp.utils import ConfigHelper
class TestBibleManager:
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
console=logging.StreamHandler()
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
log=logging.getLogger('')
logging.info("\nLogging started")
class TestBibleManager:
log=logging.getLogger("testBibleMgr")
def setup_class(self):
print "\nRegister BM"
log.debug("\n.......Register BM")
self.bm = BibleManager()
def testRegisterBibleFiles(self):
# Register a bible from files
print "\ntestRegisterBibleFiles"
self.bm.registerBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv')
self.bm.registerBible("NIV",'biblebooks_niv_short.csv','bibleverses_niv_short.csv')
log.debug("\n.......testRegisterBibleFiles")
self.bm.registerFileBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv')
self.bm.registerFileBible("NIV",'biblebooks_niv_short.csv','bibleverses_niv_short.csv')
b = self.bm.getBibles()
for b1 in b:
print b1
log.debug( b1)
assert(b1 in b)
def testRegisterBibleHTTP(self):
# Register a bible from files
print "\ntestRegisterBibleHTTP"
log.debug( "\n.......testRegisterBibleHTTP")
self.bm.registerHTTPBible("asv","Crosswalk", "", "", "", "")
#self.bm.registerBible("NIV", "ge", 1)
b = self.bm.getBibles()
for b1 in b:
print b1
log.debug( b1)
assert(b1 in b)
def testGetBibles(self):
print "\ntestGetBibles"
log.debug( "\n.......testGetBibles")
# make sure the shuffled sequence does not lose any elements
b = self.bm.getBibles()
for b1 in b:
print b1
log.debug( b1)
assert(b1 in b)
def testGetBooks(self):
print "\ntestGetBooks"
def testGetBibleBooks(self):
log.debug( "\n.......testGetBibleBooks")
c = self.bm.getBibleBooks("NIV")
for c1 in c:
print c1
log.debug( c1)
assert(c1 in c)
def testGetChapterCount(self):
print "\ntestGetChapterCount"
def testGetBookChapterCount(self):
log.debug( "\n.......testGetBookChapterCount")
assert(self.bm.getBookChapterCount("Matthew") == '28')
def testGetVerseCount(self):
print "\ntestGetVerseCount\n"
def testGetBookVerseCount(self):
log.debug( "\n.......testGetBookVerseCount")
assert(self.bm.getBookVerseCount("Genesis", 1) == '31')
assert(self.bm.getBookVerseCount("Genesis", 2) == '25')
assert(self.bm.getBookVerseCount("Matthew", 1) == '25')
assert(self.bm.getBookVerseCount("Revelation", 1) == '20')
def testGetVerseText(self):
print "\ntestGetVerseText"
log.debug( "\n.......testGetVerseText")
c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1)
print c
log.debug( c )
c = self.bm.getVerseText('NIV','Genesis',1,1,2)
print c
c = self.bm.getVerseText('asv','re',1,1,2)
print c
c = self.bm.getVerseText('asv','re',1,5,9)
print c
log.debug( c )
c = self.bm.getVerseText('asv','Revelation',1,1,2)
log.debug( c )
c = self.bm.getVerseText('asv','Revelation',1,5,9)
log.debug( c )
def testLoadBible(self):
print "\ntestLoadBible"
self.bm.loadBible('asv')
log.debug( "\n.......testLoadBible")
#self.bm.loadBible('asv')