raoul/openlp
1
0
Fork 0
forked from openlp/openlp
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor CSV reader

Browse Source 
bzr-revno: 97
This commit is contained in:
Tim Bentley 2008-11-17 21:22:14 +00:00
parent 947150c437
commit 5dcd808de2
7 changed files with 272 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
openlp/plugins/biblemanager/BibleCSVImpl.py Normal file
View File

@ -0,0 +1,78 @@
"""
OpenLP - Open Source Lyrics Projection
Copyright (c) 2008 Raoul Snyman
Portions copyright (c) 2008 Martin Thompson, Tim Bentley
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
"""
import os, os.path
import sys
import urllib2
mypath=os.path.split(os.path.abspath(__file__))[0]
sys.path.insert(0,(os.path.join(mypath, '..', '..', '..')))
from openlp.plugins.biblemanager.BibleDBImpl import BibleDBImpl
from openlp.plugins.biblemanager.BibleCommon import BibleCommon
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
class BibleCSVImpl(BibleCommon):
global log
log=logging.getLogger("BibleCSVImpl")
log.info("BibleCVSImpl loaded")
def __init__(self, bibledb):
"""
Loads a Bible from a pair of CVS files passed in
This class assumes the files contain all the information and
a clean bible is being loaded.
"""
self.bibledb = bibledb
def loadData(self, booksfile, versesfile):
self.bibledb.saveMeta("version", "Bible Version")
self.bibledb.saveMeta("Copyright", "(c) Some Bible company")
self.bibledb.saveMeta("Permission", "You Have Some")
#session = self.Session()
#Populate the Tables
fbooks=open(booksfile, 'r')
fverse=open(versesfile, 'r')
for line in fbooks:
#log.debug( line)
p = line.split(",")
p2 = p[2].replace('"', '')
p3 = p[3].replace('"', '')
self.bibledb.createBook(int(p[1]), p2, p3)
book_ptr = ""
id = 0
for line in fverse:
#log.debug( line)
p = line.split(",", 3) # split into 3 units and leave the rest as a single field
p0 = p[0].replace('"', '')
p3 = p[3].replace('"', '')
if book_ptr is not p0:
cl = self.bibledb.getBibleBook(p0)
id = self.bibledb.getBibleBookId(p0)
book_ptr = cl
log.debug( id )
self.bibledb.addVerse(id[0], p[1], p[2], p3)

3
openlp/plugins/biblemanager/BibleCommon.py
View File

@ -91,6 +91,7 @@ class BibleCommon:
text= text.replace(chr(189), '1/2') text= text.replace(chr(189), '1/2')
text= text.replace(""", '"') text= text.replace(""", '"')
text= text.replace("'", "'") text= text.replace("'", "'")
i = text.find("<") i = text.find("<")
while i > -1 : while i > -1 :
j = text.find(">", i) j = text.find(">", i)
@ -99,3 +100,5 @@ class BibleCommon:
text= text.replace('>', '') text= text.replace('>', '')
return text.rstrip() return text.rstrip()

60
openlp/plugins/biblemanager/BibleDBImpl.py
View File

@ -145,8 +145,16 @@ class BibleDBImpl(BibleCommon):
testament_table.create() testament_table.create()
book_table.create() book_table.create()
verse_table.create() verse_table.create()
self.loadMeta("dbversion", "0.1") self.saveMeta("dbversion", "0.1")
self._loadTestaments self._loadTestaments()
def addVerse(self, bookid, chap, verse, text):
log.debug( "addVerse %s,%s,%s,%s", bookid, chap, verse, text)
metadata.bind.echo = False
session = self.Session()
versemeta = Verse(book_id=int(bookid), chapter=int(chap), verse=int(verse), text=(text))
session.add(versemeta)
session.commit()
def createChapter(self, bookname, chap, textlist): def createChapter(self, bookname, chap, textlist):
log.debug( "createChapter %s,%s,%s", bookname, chap, textlist) log.debug( "createChapter %s,%s,%s", bookname, chap, textlist)
@ -168,9 +176,8 @@ class BibleDBImpl(BibleCommon):
bookmeta = Book(int(5), bookname, bookabbrev) bookmeta = Book(int(5), bookname, bookabbrev)
session.add(bookmeta) session.add(bookmeta)
session.commit() session.commit()
self._loadTestaments()
def loadMeta(self, key, value): def saveMeta(self, key, value):
metadata.bind.echo = False metadata.bind.echo = False
session = self.Session() session = self.Session()
bmeta= BibleMeta(key, value) bmeta= BibleMeta(key, value)
@ -198,51 +205,18 @@ class BibleDBImpl(BibleCommon):
session.add(testmeta) session.add(testmeta)
session.commit() session.commit()
def loadData(self, booksfile, versesfile):
self.loadMeta("version", "Bible Version")
self.loadMeta("Copyright", "(c) Some Bible company")
self.loadMeta("Permission", "You Have Some")
self._loadTestaments()
session = self.Session()
#Populate the Tables
fbooks=open(booksfile, 'r')
fverse=open(versesfile, 'r')
for line in fbooks:
#log.debug( line)
p = line.split(",")
p[2] = self._cleanText(p[2])
p[3] = self._cleanText(p[3])
bookmeta = Book(int(p[1]), p[2], p[3])
session.add(bookmeta)
session.commit()
book_ptr = ""
for line in fverse:
#log.debug( line)
p = line.split(",", 3) # split into 3 units and leave the rest as a single field
p[0] = self._cleanText(p[0])
p[3] = self._cleanText(p[3])
if book_ptr is not p[0]:
query = session.query(Book).filter(Book.name==p[0])
#log.debug( query)
#log.debug( query.first())
#log.debug( query.first().id)
book_ptr = p[0]
#log.debug( text)
versemeta = Verse(book_id=query.first().id, chapter=int(p[1]), verse=int(p[2]), text=(p[3]))
session.add(versemeta)
session.commit()
def getBibleBook(self, bookname): def getBibleBook(self, bookname):
log.debug( "getBibleBook %s", bookname) log.debug( "getBibleBook %s", bookname)
metadata.bind.echo = False metadata.bind.echo = False
s = text (""" select name FROM book where book.name == :b """) s = text (""" select name FROM book where book.name == :b """)
return self.db.execute(s, b=bookname).fetchone() return self.db.execute(s, b=bookname).fetchone()
def getBibleBookId(self, bookname):
log.debug( "getBibleBook %s", bookname)
metadata.bind.echo = False
s = text (""" select id FROM book where book.name == :b """)
return self.db.execute(s, b=bookname).fetchone()
def getBibleChapter(self, bookname, chapter): def getBibleChapter(self, bookname, chapter):
log.debug( "getBibleChapter %s,%s", bookname, chapter ) log.debug( "getBibleChapter %s,%s", bookname, chapter )
metadata.bind.echo = False metadata.bind.echo = False

18
openlp/plugins/biblemanager/BibleManager.py
View File

@ -24,6 +24,7 @@ sys.path.insert(0,(os.path.join(mypath, '..', '..', '..')))
from openlp.utils import ConfigHelper from openlp.utils import ConfigHelper
from openlp.plugins.biblemanager.BibleDBImpl import BibleDBImpl from openlp.plugins.biblemanager.BibleDBImpl import BibleDBImpl
from openlp.plugins.biblemanager.BibleHTTPImpl import BibleHTTPImpl from openlp.plugins.biblemanager.BibleHTTPImpl import BibleHTTPImpl
from openlp.plugins.biblemanager.BibleCSVImpl import BibleCSVImpl
from openlp.plugins.plugin import Plugin from openlp.plugins.plugin import Plugin
import logging import logging
@ -104,17 +105,17 @@ class BibleManager(Plugin):
nhttp = BibleHTTPImpl() nhttp = BibleHTTPImpl()
nhttp.setBibleSource(biblesource) nhttp.setBibleSource(biblesource)
self.bibleHTTPCache[biblename] = nhttp self.bibleHTTPCache[biblename] = nhttp
nbible.loadMeta("WEB", biblesource) # register a lazy loading interest nbible.saveMeta("WEB", biblesource) # register a lazy loading interest
if proxyurl != None: if proxyurl != None:
nbible.loadMeta("proxy", proxyurl) # store the proxy URL nbible.saveMeta("proxy", proxyurl) # store the proxy URL
nhttp.setProxy(proxyurl) nhttp.setProxy(proxyurl)
if proxyid != None: if proxyid != None:
nbible.loadMeta("proxyid", proxyid) # store the proxy userid nbible.saveMeta("proxyid", proxyid) # store the proxy userid
if proxypass != None: if proxypass != None:
nbible.loadMeta("proxypass", proxypass) # store the proxy password nbible.saveMeta("proxypass", proxypass) # store the proxy password
def registerFileBible(self, biblename, booksfile, versefile): def registerFileBible(self, biblename, booksfile, versefile, filetype):
""" """
Method to load a bible from a set of files into a database. Method to load a bible from a set of files into a database.
If the database exists it is deleted and the database is reloaded If the database exists it is deleted and the database is reloaded
@ -123,8 +124,11 @@ class BibleManager(Plugin):
if self._isNewBible(biblename): if self._isNewBible(biblename):
nbible = BibleDBImpl(biblename) # Create new Bible nbible = BibleDBImpl(biblename) # Create new Bible
nbible.createTables() # Create Database nbible.createTables() # Create Database
nbible.loadData(booksfile, versefile) self.bibleDBCache[biblename] = nbible # cache the database for use later
self.bibleDBCache[biblename] = nbible bcsv = BibleCSVImpl(nbible) # create the loader and pass in the database
if filetype == "csv":
bcsv.loadData(booksfile, versefile)
def loadBible(self,biblename): def loadBible(self,biblename):
""" """

20
openlp/plugins/biblemanager/test/test_bibleManager.py
View File

@ -50,7 +50,7 @@ class TestBibleManager:
log.debug("\n.......Register BM") log.debug("\n.......Register BM")
self.bm = BibleManager() self.bm = BibleManager()
def testRegisterBibleFiles(self): def testRegisterCSVBibleFiles(self):
# Register a bible from files # Register a bible from files
log.debug("\n.......testRegisterBibleFiles") log.debug("\n.......testRegisterBibleFiles")
self.bm.registerFileBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv') self.bm.registerFileBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv')
@ -60,7 +60,7 @@ class TestBibleManager:
log.debug( b1) log.debug( b1)
assert(b1 in b) assert(b1 in b)
def testRegisterBibleHTTP(self): def testRegisterHTTPBible(self):
# Register a bible from files # Register a bible from files
log.debug( "\n.......testRegisterBibleHTTP") log.debug( "\n.......testRegisterBibleHTTP")
self.bm.registerHTTPBible("asv","Crosswalk", "", "", "") self.bm.registerHTTPBible("asv","Crosswalk", "", "", "")
@ -100,17 +100,15 @@ class TestBibleManager:
def testGetVerseText(self): def testGetVerseText(self):
log.debug( "\n.......testGetVerseText") log.debug( "\n.......testGetVerseText")
c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1) #c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1)
#log.debug( c )
#c = self.bm.getVerseText('NIV','Genesis',1,1,2)
#log.debug( c )
c = self.bm.getVerseText('asv','Genesis',10,1,20)
log.debug( c ) log.debug( c )
c = self.bm.getVerseText('NIV','Genesis',1,1,2) c = self.bm.getVerseText('nasb','Genesis',10,1,20)
log.debug( c ) log.debug( c )
c = self.bm.getVerseText('asv','Revelation',1,1,2) c = self.bm.getVerseText('nkj','Revelation',10,1,20)
log.debug( c )
c = self.bm.getVerseText('asv','Revelation',1,5,9)
log.debug( c )
c = self.bm.getVerseText('nasb','Revelation',10,5,9)
log.debug( c )
c = self.bm.getVerseText('nkj','Revelation',10,5,9)
log.debug( c ) log.debug( c )
def testLoadBible(self): def testLoadBible(self):

91
openlp/plugins/biblemanager/test/test_bibleManagerAPI.py Normal file
View File

@ -0,0 +1,91 @@
"""
OpenLP - Open Source Lyrics Projection
Copyright (c) 2008 Raoul Snyman
Portions copyright (c) 2008 Martin Thompson, Tim Bentley
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
"""
import random
import unittest
import os, os.path
import sys
mypath=os.path.split(os.path.abspath(__file__))[0]
sys.path.insert(0,(os.path.join(mypath, '..', '..','..','..')))
from openlp.plugins.biblemanager.BibleManager import BibleManager
from openlp.utils import ConfigHelper
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
console=logging.StreamHandler()
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
log=logging.getLogger('')
logging.info("\nLogging started")
class TestBibleManager:
log=logging.getLogger("testBibleMgr")
def setup_class(self):
log.debug("\n.......Register BM")
self.bm = BibleManager()
def testGetBibles(self):
log.debug( "\n.......testGetBibles")
# make sure the shuffled sequence does not lose any elements
b = self.bm.getBibles()
for b1 in b:
log.debug( b1)
assert(b1 in b)
def testGetBibleBooks(self):
log.debug( "\n.......testGetBibleBooks")
c = self.bm.getBibleBooks("NIV")
for c1 in c:
log.debug( c1)
assert(c1 in c)
def testGetBookChapterCount(self):
log.debug( "\n.......testGetBookChapterCount")
assert(self.bm.getBookChapterCount("Matthew") == '28')
def testGetBookVerseCount(self):
log.debug( "\n.......testGetBookVerseCount")
assert(self.bm.getBookVerseCount("Genesis", 1) == '31')
assert(self.bm.getBookVerseCount("Genesis", 2) == '25')
assert(self.bm.getBookVerseCount("Matthew", 1) == '25')
assert(self.bm.getBookVerseCount("Revelation", 1) == '20')
def testGetVerseText(self):
log.debug( "\n.......testGetVerseText")
#c = self.bm.getVerseText("TheMessage",'Genesis',1,2,1)
#log.debug( c )
#c = self.bm.getVerseText('NIV','Genesis',1,1,2)
#log.debug( c )
c = self.bm.getVerseText('asv','Genesis',10,1,20)
log.debug( c )
c = self.bm.getVerseText('nasb','Genesis',10,1,20)
log.debug( c )
c = self.bm.getVerseText('nkj','Revelation',10,1,20)
log.debug( c )

61
openlp/plugins/biblemanager/test/test_bibleManagerCSV.py Normal file
View File

@ -0,0 +1,61 @@
"""
OpenLP - Open Source Lyrics Projection
Copyright (c) 2008 Raoul Snyman
Portions copyright (c) 2008 Martin Thompson, Tim Bentley
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
"""
import random
import unittest
import os, os.path
import sys
mypath=os.path.split(os.path.abspath(__file__))[0]
sys.path.insert(0,(os.path.join(mypath, '..', '..','..','..')))
from openlp.plugins.biblemanager.BibleManager import BibleManager
from openlp.utils import ConfigHelper
import logging
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M',
filename='plugins.log',
filemode='w')
console=logging.StreamHandler()
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
log=logging.getLogger('')
logging.info("\nLogging started")
class TestBibleManager:
log=logging.getLogger("testBibleMgr")
def setup_class(self):
log.debug("\n.......Register BM")
self.bm = BibleManager()
def testRegisterCSVBibleFiles(self):
# Register a bible from files
log.debug("\n.......testRegisterBibleFiles")
self.bm.registerFileBible("TheMessage",'biblebooks_msg_short.csv','bibleverses_msg_short.csv', "csv")
self.bm.registerFileBible("NIV",'biblebooks_niv_short.csv','bibleverses_niv_short.csv', "csv")
b = self.bm.getBibles()
for b1 in b:
log.debug( b1)
assert(b1 in b)