raoul
/
openlp
forked from openlp/openlp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor CSV reader 

bzr-revno: 97
This commit is contained in:
Tim Bentley 2008-11-17 21:22:14 +00:00
parent 947150c437
commit 5dcd808de2
7 changed files with 272 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
openlp/plugins/biblemanager/BibleCSVImpl.py Normal file
View File

@ -0,0 +1,78 @@
"""
OpenLP - Open Source Lyrics Projection
Copyright (c) 2008 Raoul Snyman
Portions copyright (c) 2008 Martin Thompson, Tim Bentley
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
"""
import os, os.path
import sys
import urllib2
mypath=os.path.split(os.path.abspath(__file__))[0]
sys.path.insert(0,(os.path.join(mypath, '..', '..', '..')))
from openlp.plugins.biblemanager.BibleDBImpl import BibleDBImpl
from openlp.plugins.biblemanager.BibleCommon import BibleCommon
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
class BibleCSVImpl(BibleCommon):
global log
log=logging.getLogger("BibleCSVImpl")
log.info("BibleCVSImpl loaded")
def __init__(self, bibledb):
"""
Loads a Bible from a pair of CVS files passed in
This class assumes the files contain all the information and
a clean bible is being loaded.
"""
self.bibledb = bibledb
def loadData(self, booksfile, versesfile):
self.bibledb.saveMeta("version", "Bible Version")
self.bibledb.saveMeta("Copyright", "(c) Some Bible company")
self.bibledb.saveMeta("Permission", "You Have Some")
#session = self.Session()
#Populate the Tables
fbooks=open(booksfile, 'r')
fverse=open(versesfile, 'r')
for line in fbooks:
#log.debug( line)
p = line.split(",")
p2 = p[2].replace('"', '')
p3 = p[3].replace('"', '')
self.bibledb.createBook(int(p[1]), p2, p3)
book_ptr = ""
id = 0
for line in fverse:
#log.debug( line)
p = line.split(",", 3) # split into 3 units and leave the rest as a single field
p0 = p[0].replace('"', '')
p3 = p[3].replace('"', '')
if book_ptr is not p0:
cl = self.bibledb.getBibleBook(p0)
id = self.bibledb.getBibleBookId(p0)
book_ptr = cl
log.debug( id )
self.bibledb.addVerse(id[0], p[1], p[2], p3)

5
openlp/plugins/biblemanager/BibleCommon.py
View File

@ -68,7 +68,7 @@ class BibleCommon:
j=text.find("</h", i)
text = text[ : (i - 1)]+text[(j+4)]
i = text.find("<h")
# Remove Support References from the Text
x = text.find("<sup>")
while x > -1:
@ -91,6 +91,7 @@ class BibleCommon:
text= text.replace(chr(189), '1/2')
text= text.replace("&quot;", '"')
text= text.replace("&apos;", "'")
i = text.find("<")
while i > -1 :
j = text.find(">", i)
@ -99,3 +100,5 @@ class BibleCommon:
text= text.replace('>', '')
return text.rstrip()

62
openlp/plugins/biblemanager/BibleDBImpl.py
View File

@ -145,8 +145,16 @@ class BibleDBImpl(BibleCommon):
testament_table.create()
book_table.create()
verse_table.create()
self.loadMeta("dbversion", "0.1")
self._loadTestaments
self.saveMeta("dbversion", "0.1")
self._loadTestaments()
def addVerse(self, bookid, chap, verse, text):
log.debug( "addVerse %s,%s,%s,%s", bookid, chap, verse, text)
metadata.bind.echo = False
session = self.Session()
versemeta = Verse(book_id=int(bookid), chapter=int(chap), verse=int(verse), text=(text))
session.add(versemeta)
session.commit()
def createChapter(self, bookname, chap, textlist):
log.debug( "createChapter %s,%s,%s", bookname, chap, textlist)
@ -168,9 +176,8 @@ class BibleDBImpl(BibleCommon):
bookmeta = Book(int(5), bookname, bookabbrev)
session.add(bookmeta)
session.commit()
self._loadTestaments()
def loadMeta(self, key, value):
def saveMeta(self, key, value):
metadata.bind.echo = False
session = self.Session()
bmeta= BibleMeta(key, value)
@ -197,52 +204,19 @@ class BibleDBImpl(BibleCommon):
testmeta = ONTestament(name="Apocrypha")
session.add(testmeta)
session.commit()
def loadData(self, booksfile, versesfile):
self.loadMeta("version", "Bible Version")
self.loadMeta("Copyright", "(c) Some Bible company")
self.loadMeta("Permission", "You Have Some")
self._loadTestaments()
session = self.Session()
#Populate the Tables
fbooks=open(booksfile, 'r')
fverse=open(versesfile, 'r')
for line in fbooks:
#log.debug( line)
p = line.split(",")
p[2] = self._cleanText(p[2])
p[3] = self._cleanText(p[3])
bookmeta = Book(int(p[1]), p[2], p[3])
session.add(bookmeta)
session.commit()
book_ptr = ""
for line in fverse:
#log.debug( line)
p = line.split(",", 3) # split into 3 units and leave the rest as a single field
p[0] = self._cleanText(p[0])
p[3] = self._cleanText(p[3])
if book_ptr is not p[0]:
query = session.query(Book).filter(Book.name==p[0])
#log.debug( query)
#log.debug( query.first())
#log.debug( query.first().id)
book_ptr = p[0]
#log.debug( text)
versemeta = Verse(book_id=query.first().id, chapter=int(p[1]), verse=int(p[2]), text=(p[3]))
session.add(versemeta)
session.commit()
def getBibleBook(self, bookname):
log.debug( "getBibleBook %s", bookname)
metadata.bind.echo = False
s = text (""" select name FROM book where book.name == :b """)
return self.db.execute(s, b=bookname).fetchone()
def getBibleBookId(self, bookname):
log.debug( "getBibleBook %s", bookname)
metadata.bind.echo = False
s = text (""" select id FROM book where book.name == :b """)
return self.db.execute(s, b=bookname).fetchone()
def getBibleChapter(self, bookname, chapter):
log.debug( "getBibleChapter %s,%s", bookname, chapter )
metadata.bind.echo = False

18
openlp/plugins/biblemanager/BibleManager.py
View File

@ -24,6 +24,7 @@ sys.path.insert(0,(os.path.join(mypath, '..', '..', '..')))
from openlp.utils import ConfigHelper
from openlp.plugins.biblemanager.BibleDBImpl import BibleDBImpl
from openlp.plugins.biblemanager.BibleHTTPImpl import BibleHTTPImpl
from openlp.plugins.biblemanager.BibleCSVImpl import BibleCSVImpl
from openlp.plugins.plugin import Plugin
import logging
@ -104,17 +105,17 @@ class BibleManager(Plugin):
nhttp = BibleHTTPImpl()
nhttp.setBibleSource(biblesource)
self.bibleHTTPCache[biblename] = nhttp
nbible.loadMeta("WEB", biblesource) # register a lazy loading interest
nbible.saveMeta("WEB", biblesource) # register a lazy loading interest
if proxyurl != None:
nbible.loadMeta("proxy", proxyurl) # store the proxy URL
nbible.saveMeta("proxy", proxyurl) # store the proxy URL
nhttp.setProxy(proxyurl)
if proxyid != None:
nbible.loadMeta("proxyid", proxyid) # store the proxy userid
nbible.saveMeta("proxyid", proxyid) # store the proxy userid
if proxypass != None:
nbible.loadMeta("proxypass", proxypass) # store the proxy password
nbible.saveMeta("proxypass", proxypass) # store the proxy password
def registerFileBible(self, biblename, booksfile, versefile):
def registerFileBible(self, biblename, booksfile, versefile, filetype):
"""
Method to load a bible from a set of files into a database.
If the database exists it is deleted and the database is reloaded
@ -123,8 +124,11 @@ class BibleManager(Plugin):
if self._isNewBible(biblename):
nbible = BibleDBImpl(biblename) # Create new Bible
nbible.createTables() # Create Database
nbible.loadData(booksfile, versefile)
self.bibleDBCache[biblename] = nbible
self.bibleDBCache[biblename] = nbible # cache the database for use later
bcsv = BibleCSVImpl(nbible) # create the loader and pass in the database
if filetype == "csv":
bcsv.loadData(booksfile, versefile)
def loadBible(self,biblename):
"""

20
openlp/plugins/biblemanager/test/test_bibleManager.py
View File

@ -50,7 +50,7 @@ class TestBibleManager:
log.debug("\n.......Register BM")
self.bm = BibleManager()
def testRegisterBibleFiles(self):
def testRegisterCSVBibleFiles(self):
# Register a bible from files
log.debug("\n.......testRegisterBibleFiles")
self.bm.registerFileBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv')
@ -60,7 +60,7 @@ class TestBibleManager:
log.debug( b1)
assert(b1 in b)
def testRegisterBibleHTTP(self):
def testRegisterHTTPBible(self):
# Register a bible from files
log.debug( "\n.......testRegisterBibleHTTP")
self.bm.registerHTTPBible("asv","Crosswalk", "", "", "")
@ -100,17 +100,15 @@ class TestBibleManager:
def testGetVerseText(self):
log.debug( "\n.......testGetVerseText")
c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1)
#c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1)
#log.debug( c )
#c = self.bm.getVerseText('NIV','Genesis',1,1,2)
#log.debug( c )
c = self.bm.getVerseText('asv','Genesis',10,1,20)
log.debug( c )
c = self.bm.getVerseText('NIV','Genesis',1,1,2)
log.debug( c )
c = self.bm.getVerseText('asv','Revelation',1,1,2)
log.debug( c )
c = self.bm.getVerseText('asv','Revelation',1,5,9)
log.debug( c )
c = self.bm.getVerseText('nasb','Revelation',10,5,9)
c = self.bm.getVerseText('nasb','Genesis',10,1,20)
log.debug( c )
c = self.bm.getVerseText('nkj','Revelation',10,5,9)
c = self.bm.getVerseText('nkj','Revelation',10,1,20)
log.debug( c )
def testLoadBible(self):

91
openlp/plugins/biblemanager/test/test_bibleManagerAPI.py Normal file
View File

@ -0,0 +1,91 @@
"""
OpenLP - Open Source Lyrics Projection
Copyright (c) 2008 Raoul Snyman
Portions copyright (c) 2008 Martin Thompson, Tim Bentley
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
"""
import random
import unittest
import os, os.path
import sys
mypath=os.path.split(os.path.abspath(__file__))[0]
sys.path.insert(0,(os.path.join(mypath, '..', '..','..','..')))
from openlp.plugins.biblemanager.BibleManager import BibleManager
from openlp.utils import ConfigHelper
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
console=logging.StreamHandler()
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
log=logging.getLogger('')
logging.info("\nLogging started")
class TestBibleManager:
log=logging.getLogger("testBibleMgr")
def setup_class(self):
log.debug("\n.......Register BM")
self.bm = BibleManager()
def testGetBibles(self):
log.debug( "\n.......testGetBibles")
# make sure the shuffled sequence does not lose any elements
b = self.bm.getBibles()
for b1 in b:
log.debug( b1)
assert(b1 in b)
def testGetBibleBooks(self):
log.debug( "\n.......testGetBibleBooks")
c = self.bm.getBibleBooks("NIV")
for c1 in c:
log.debug( c1)
assert(c1 in c)
def testGetBookChapterCount(self):
log.debug( "\n.......testGetBookChapterCount")
assert(self.bm.getBookChapterCount("Matthew") == '28')
def testGetBookVerseCount(self):
log.debug( "\n.......testGetBookVerseCount")
assert(self.bm.getBookVerseCount("Genesis", 1) == '31')
assert(self.bm.getBookVerseCount("Genesis", 2) == '25')
assert(self.bm.getBookVerseCount("Matthew", 1) == '25')
assert(self.bm.getBookVerseCount("Revelation", 1) == '20')
def testGetVerseText(self):
log.debug( "\n.......testGetVerseText")
#c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1)
#log.debug( c )
#c = self.bm.getVerseText('NIV','Genesis',1,1,2)
#log.debug( c )
c = self.bm.getVerseText('asv','Genesis',10,1,20)
log.debug( c )
c = self.bm.getVerseText('nasb','Genesis',10,1,20)
log.debug( c )
c = self.bm.getVerseText('nkj','Revelation',10,1,20)
log.debug( c )

61
openlp/plugins/biblemanager/test/test_bibleManagerCSV.py Normal file
View File

@ -0,0 +1,61 @@
"""
OpenLP - Open Source Lyrics Projection
Copyright (c) 2008 Raoul Snyman
Portions copyright (c) 2008 Martin Thompson, Tim Bentley
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
"""
import random
import unittest
import os, os.path
import sys
mypath=os.path.split(os.path.abspath(__file__))[0]
sys.path.insert(0,(os.path.join(mypath, '..', '..','..','..')))
from openlp.plugins.biblemanager.BibleManager import BibleManager
from openlp.utils import ConfigHelper
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
console=logging.StreamHandler()
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
log=logging.getLogger('')
logging.info("\nLogging started")
class TestBibleManager:
log=logging.getLogger("testBibleMgr")
def setup_class(self):
log.debug("\n.......Register BM")
self.bm = BibleManager()
def testRegisterCSVBibleFiles(self):
# Register a bible from files
log.debug("\n.......testRegisterBibleFiles")
self.bm.registerFileBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv', "csv")
self.bm.registerFileBible("NIV",'biblebooks_niv_short.csv','bibleverses_niv_short.csv', "csv")
b = self.bm.getBibles()
for b1 in b:
log.debug( b1)
assert(b1 in b)