diff --git a/.gitattributes b/.gitattributes index 39b603c..00a78f5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,25 +1,17 @@ * text=auto eol=lf -*.bat text eol=crlf -*.config text eol=lf *.css text eol=lf *.html text eol=lf *.js text eol=lf -*.prefs text *.py text eol=lf *.rst text eol=lf *.sh text eol=lf *.txt text eol=lf -*.po text eol=lf -*.pot text eol=lf -*.styl text eol=lf -*.xml text *.gif binary *.ico binary *.jpg binary *.lnk binary -*.mo binary *.png binary *.exe binary *.so binary diff --git a/Context/Main.py b/Context/Main.py deleted file mode 100644 index 747f73d..0000000 --- a/Context/Main.py +++ /dev/null @@ -1,1112 +0,0 @@ -import os -import re -import datetime -import mimetypes - -from lib import wikipage, htmldiff, menubar -from lib.common import dedent -from lib.format_date import format_date_relative - - -from SitePage import * -from WebKit.HTTPExceptions import * - -tidy_options = dict( - break_before_br=True, - show_body_only=True, - char_encoding='utf8') -try: - from tidy import parseString as utidy -except ImportError: - try: - from mx.Tidy import tidy as mxtidy - except ImportError: - tidy = lambda text: text - else: - tidy_options['output_error'] = False - tidy = lambda text: mxtidy(text, **tidy_options)[2] -else: - tidy = lambda text: utidy(text, **tidy_options) - - -class Main(SitePage): - - def __init__(self): - SitePage.__init__(self) - self._inputTypeCleaners = {} - self._inputTypeCleaners['xinha'] = self.cleanXinha - - def setupEarly(self): - req = self.request() - name = req.extraURLPath().strip('/') - if not name: - self.forward('/frontpage') - name, ext = self.splitExtension(name) - if name in self.wiki.globalWiki.specialNames: - # We got the .html ending or somesuch, and we shouldn't - # have, so we'll redirect - self.sendRedirectAndEnd(self.servletLink(name)) - return - version = self.getVersion() - self.page = self.wiki.page(name, version=version) - if name != self.page.urlName: - link = self.page.link - if '?' in link: - link += '&' - else: - link += '?' - link += self.encodeArgs() - self.sendRedirectAndEnd(link) - self.titlePrefix = '' - if ext == '.thumb.jpg': - req.setField('_action_', 'thumbnail') - elif ext != '.html': - req.setField('_action_', 'source') - - def sleep(self, transaction): - SitePage.sleep(self, transaction) - self.page = None - - def respondToPut(self, transaction): - self.webdav() - - respondToLock = respondToUnlock = respondToPut - - def actions(self): - return ['edit', 'preview', 'save', 'cancel', 'history', - 'externalEdit', 'webdav', 'backlinks', 'diff', - 'changeMimeType', 'simple', 'source', 'comment', - 'thumbnail', 'attach'] - - def pageClass(self): - if (not self.page.exists() - and self.request().field('commenting', None)): - return 'comment' - return self.page.pageClass - - def authorUser(self): - return self.page.authorUser - - def writeGoogleAds(self): - if self.view() not in ('writeEdit', 'writePreview', 'writeChangeMimeType'): - SitePage.writeGoogleAds(self) - - ######################################## - ## Utility methods - ######################################## - - def splitExtension(self, name): - if name.endswith('.thumb.jpg'): - ext = '.thumb.jpg' - name = name[:-len('.thumb.jpg')] - elif '.' in name: - name, ext = os.path.splitext(name) - else: - ext = '.html' - return name, ext - - def getVersion(self): - version = self.request().field('version', None) or None - if isinstance(version, (list, tuple)): - version = version[0] - try: - version = int(version) - if version < 1: - raise ValueError - except: - version = None - return version - - def cleanInput(self, inputType, text): - if text.startswith('base64,'): - text = text[7:].decode('base64') - text = self._inputTypeCleaners.get(inputType, lambda x: x)(text) - return text - - def assertEdit(self): - if self.page.exists(): - self.assertPermission('edit') - else: - self.assertPermission(['create', 'edit']) - - ######################################## - ## Edit - ######################################## - - def edit(self): - self.assertEdit() - req = self.request() - self.titlePrefix = 'Edit: ' - mimeType = req.field('mimeType', self.page.mimeType) - if mimeType != self.page.mimeType and self.page.exists(): - assert 0, "Content conversion has not yet been implemented" - self._setPageClass = 'posting' - self.commenting = req.field('commenting', 0) - if self.commenting: - self._setPageClass = 'comment' - self.attaching = req.field('attaching', 0) - if self.attaching: - self._setPageClass = 'attachment' - assert not self.commenting or not self.attaching - if self.commenting: - self.orig = self.wiki.page(self.commenting) - if (not req.hasField('title') - and not self.orig.title.lower().startswith('re:')): - req.setField('title', 'Re: %s' % self.orig.title) - if self.attaching: - self.orig = self.wiki.page(self.attaching) - if not req.hasField('title'): - req.setField('title', 'Attachment to: %s' % self.orig.title) - if not req.hasField('mimeType'): - req.setField('mimeType', 'application/*') - self.setView('writeEdit') - self.writeHTML() - - _comment_name_re = re.compile(r'-*comment-?\d+$') - def comment(self): - if self._comment_name_re.search(self.page.urlName): - newName = self._comment_name_re.sub('', self.page.urlName) - else: - newName = self.page.urlName - p = self.wiki.page(newName + '-comment-000') - raise HTTPTemporaryRedirect( - p.link + - '?_action_=edit&commenting=%s' - % self.urlEncode(self.page.name)) - - def attach(self): - p = self.wiki.page(self.page.name + 'attach-000') - raise HTTPTemporaryRedirect( - p.link + - '?_action_=edit&attaching=%s' - % self.urlEncode(self.page.name)) - - def preview(self): - self.saveCookieAuthorInfoFromRequest() - self.setView('writePreview') - self.titlePrefix = 'Preview: ' - req = self.request() - self.writeHTML() - - def save(self): - self.assertEdit() - req = self.request() - self.saveCookieAuthorInfoFromRequest() - if self.page.name.endswith('000'): - # This means we have to create a new page name that is - # uniquely numbered. @@: This isn't actually safe, - # since instantiating the page won't make it "exist", - # and another thread could still use the same number - name = self.page.name[:-3] - n = 1 - while 1: - if not self.wiki.page('%s%i' % (name, n)).exists(): - break - n += 1 - oldPage = self.page - self.page = self.wiki.page('%s%i' % (name, n)) - self.page.urlName = oldPage.urlName[:-3] + str(n) - self.page.title = req.field('title') - if self.page.title.find('000') != -1: - self.page.title = self.page.title.replace('000', str(n)) - else: - self.page.title = req.field('title') - inputType = req.field('inputType') - text = req.field('text', '') - textUpload = req.field('textUpload', '') - mimeType = req.field('mimeType') - try: - filename = textUpload.filename - if mimeType.startswith('application/'): - mt, encoding = mimetypes.guess_type(filename) - if mt: - mimeType = mt - textUpload = textUpload.value - self.page.originalFilename = filename - except AttributeError: - pass - if textUpload: - text = textUpload - text = self.cleanInput(inputType, text) - self.page.mimeType = mimeType - self.page.title = req.field('title') - if req.field('hidden', None): - self.assertPermission('hide') - self.page.hidden = req.field('hidden', '') - if req.hasField('comments'): - self.page.comments = req.field('comments') - self.page.lastChangeLog = req.field('changeLog', '') - if self.user(): - self.page.lastChangeUser = self.user().username() - else: - self.page.lastChangeUser = 'anonymous' - if req.field('authorName', None): - self.page.authorName = req.field('authorName') - if req.field('authorURL', None): - self.page.authorURL = req.field('authorURL') - if req.field('authorEmail', None): - self.page.authorEmail = req.field('authorEmail') - if self.user(): - self.page.authorUser = self.user() - if text or not self.page.text: - self.page.text = text or '' - self.page.distributionOriginal = False - pageClass = req.field('pageClass', None) - if req.field('commenting', None): - parentPage = self.wiki.page(req.field('commenting')) - self.page.connections += [(parentPage, 'comment')] - if pageClass is None: - pageClass = 'comment' - if req.field('attaching', None): - parentPage = self.wiki.page(req.field('attaching')) - self.page.connections += [(parentPage, 'attachment')] - if pageClass is None: - pageClass = 'attachment' - if pageClass is None: - pageClass = 'posting' - self.page.pageClass = pageClass - self.page.save() - self.message('Changes saved') - self.sendRedirectAndEnd(self.link(unversioned=True)) - - def cancel(self): - self.message('Edit cancelled') - self.sendRedirectAndEnd(self.link(unversioned=True)) - - def history(self): - self.assertPermission(['history', 'view']) - self.setView('writeHistory') - self.titlePrefix = 'History of: ' - self.writeHTML() - - def backlinks(self): - self.assertPermission(['navigation', 'view']) - self.setView('writeBacklinks') - self.titlePrefix = 'Backlinks to: ' - self.writeHTML() - - def diff(self): - if self.request().field('delete', None): - self.history() - else: - self.assertPermission(['diff', 'history', 'view']) - firstVersion = self.request().field('firstVersion', None) or None - self.firstPage = self.wiki.page(self.page.name, version=firstVersion) - otherVersion = self.request().field('otherVersion', None) or None - self.otherPage = self.wiki.page(self.page.name, version=otherVersion) - self.setView('writeDiff') - self.titlePrefix = ('Diff %s to %s of:' % - (firstVersion or 'Current', otherVersion or 'Current')) - self.writeHTML() - - def externalEdit(self): - self.assertEdit() - self.setView(None) - res = self.response() - req = self.request() - if not req.hasCookie('_SID_'): - raise HTTPBadRequest( - "Your browser must support cookies to use the external " - "editor.") - res.setHeader('content-type', 'application/x-zope-edit') - self.write('url:http://%s%s\n' - % (req.environ()['HTTP_HOST'], - self.link(action='webdav'))) - self.write('meta_type:Wiki\n') - # no HTTP or cookie authentication required: - self.write('auth:\n') - self.write('cookie: _SID_=%s\n' - % (req.cookie('_SID_'))) - self.write('\n') - self.write('## title=%s\n' % self.page.title) - self.write('## class=%s\n' % (self.page.pageClass or 'posting')) - self.write('## log=\n') - self.write('\n') - self.write(self.page.text.replace('\r', '')) - - def webdav(self): - self.assertEdit() - # @@: Really, this should make sure that we send a real - # 401, not an HTML login form. But that's not easy at the - # moment :( HTTPForbidden instead? - self.setView(None) - req = self.request() - method = req.environ()['REQUEST_METHOD'].upper() - metavars = {'user': 'lastChangeUser', - 'log': 'lastChangeLog', - 'title': 'title', - 'class': 'pageClass'} - - if method in ('LOCK', 'UNLOCK'): - # We don't support these now - return - - if method == 'GET': - self.write(self.page.text) - return - - if method == 'PUT': - f = req.rawInput(rewind=1) - text = f.read() - lines = text.splitlines() - metadata = {} - while 1: - if not lines: - break - line = lines[0].strip() - if not line: - lines.pop(0) - continue - if not line.startswith('##'): - break - if '=' not in line: - break - lines.pop(0) - line = line[2:] - name, value = line.split('=', 1) - name = name.strip().lower() - value = value.strip() - metadata[name] = value - - for name, value in metadata.items(): - assert metavars.has_key(name), "Bad name %r (must be one of %s)" % (name, ', '.join(metavars.keys())) - setattr(self.page, metavars[name], value) - # @@: this won't work for binary pages - self.page.text = '\n'.join(lines) - self.page.save() - - def changeMimeType(self): - self.assertEdit() - self.setView('writeChangeMimeType') - self.writeHTML() - - def simple(self): - self.suppressFooter = True - self.writeHTML() - - def source(self): - mimeType = self.page.mimeType - if mimeType in ['text/x-restructured-text', 'text/x-python']: - mimeType = 'text/plain' - self.response().setHeader('content-type', mimeType) - self.write(self.page.text) - - def thumbnail(self): - thumbnail = self.page.thumbnail - if not thumbnail: - raise HTTPNotFound - self.response().setHeader('content-type', 'image/jpeg') - self.write(self.page.thumbnail) - - ######################################## - ## Views - ######################################## - - def title(self): - if self.page.version: - title = '%s version %s' % (self.page.title, self.page.version) - else: - title = self.page.title - return self.titlePrefix + title - - def writeHeader(self): - if not self.page.name in ('wikisandbox',): - self.write('\n') - SitePage.writeHeader(self) - - def writeRelatedLinks(self): - SitePage.writeRelatedLinks(self) - if self.checkPermission('edit'): - # add universal edit button - self.write('' % self.link(action='edit')) - - def link(self, action=None, unversioned=False, **kw): - if action: - kw['_action_'] = action - if unversioned: - link = self.page.wiki.linkTo(self.page.name) - else: - link = self.page.link - if not kw: - return link - if '?' in link: - link += '&' - else: - link += '?' - link += '&'.join(['%s=%s' % (n, self.urlEncode(v)) - for n, v in kw.items()]) - return link - - def writeContent(self): - if self.page.hidden: - self.assertPermission('viewhidden') - self.write(self.page.html) - creation = self.page.config.getbool('displaycreationdate', False) - modified = self.page.config.getbool('displaymodifieddate', False) - self.write('
\n') - data = [] - if creation and self.page.creationDate: - data.append( - 'Created %s' - % format_date_relative(self.page.creationDate)) - if (modified - and self.page.modifiedDate - and (self.page.creationDate != self.page.modifiedDate - or not creation)): - if creation and self.page.creationDate: - data.append( - 'Modified %s' - % format_date_relative(self.page.modifiedDate)) - else: - data.append(format_date_relative(self.page.modifiedDate)) - - for page, type in self.page.connections: - desc = {'comment': 'Comment on %s', - 'attachment': 'Attachment to %s', - }.get(type, '%s to %%s' % type) - data.append( - desc % ('%s' % (page.link, page.title))) - if self.page.config.getbool('showauthor', False): - if self.page.authorUser: - data.append('by %s' % self.page.authorUser.signature()) - elif self.page.authorName: - author = self.page.authorName - if self.page.authorURL: - author = '%s' % ( - self.page.authorURL, author) - data.append('by %s' % author) - if self.checkPermission(['create', 'edit'], pageClass='comment'): - data.append('Add comment' - % self.link('comment')) - self.write('
\n'.join(data).encode('utf-8')) - self.write('
\n') - if self.page.commentPages: - self.write('

Comments:

\n') - self.displayComments(self.page) - - def displayComments(self, page): - comments = page.commentPages - if not comments: - return - self.write('
\n') - comments.sort(lambda a, b: cmp(a.creationDate, b.creationDate)) - for comment in comments: - self.write(comment.html) - authorName = comment.authorName or comment.lastChangeUser - authorName = self.htmlEncode(authorName) - if comment.authorURL: - authorName = '%s' % ( - self.htmlEncode(comment.authorURL), authorName or 'URL') - if comment.authorUser: - authorName = comment.authorUser.signature() - data = [] - if authorName: - data = [authorName] - showCreate = None - if comment.config.getbool('displaycreationdate', False): - showCreate = comment.creationDate - data.append(format_date_relative(comment.creationDate)) - if (comment.config.getbool('displaymodifieddate', False) - and comment.modifiedDate != showCreate): - data.append( - 'Modified: %s' % - format_date_relative(comment.modifiedDate)) - if self.checkPermission(['create', 'edit'], - pageClass='comment'): - link = comment.link + '?_action_=comment' - data.append('Reply' % self.htmlEncode(link)) - perm_link = '#' % self.htmlEncode(comment.link) - if data: - data[0] = perm_link + ' ' + data[0] - else: - data = [perm_link] - if data: - self.write('
%s
' - % '
\n'.join(data)) - self.write('
\n') - self.displayComments(comment) - self.write('
\n') - - def menus(self): - menu = SitePage.menus(self) - menu.insert(2, 'Page') - if not self.page.readOnly: - menu.insert(3, 'Edit') - return menu - - def menuPage(self): - menu_title = self.page.title - if len(menu_title) > 15: - menu_title = menu_title[:14] + '…' - menu = ('%s' % menu_title, [ - ('View', self.link(unversioned=True)), - ('Source', self.page.sourceLink), - ('Backlinks', self.link(action='backlinks', unversioned=True)), - ('History', self.link(action='history', unversioned=True))]) - return menu - - def menuEdit(self): - menu = [] - if self.checkPermission('edit'): - menu = [ - ('Edit this page', self.link(action='edit')), - ('External editor ' - % self.wiki.linkTo('edit_icon.gif'), - self.link(action='externalEdit'))] - if (self.page.exists() - and self.checkPermission(['create', 'edit'], pageClass='comment')): - menu.append( - ('Add comment', self.link(action='comment'))) - if (self.page.pageClass != 'attachment' and self.page.exists() - and self.checkPermission(['create', 'edit'], pageClass='attachment')): - menu.append( - ('Attach file', self.link(action='attach'))) - if not menu: - return (menubar.Literal, '') - return ('Edit', menu) - - ## @@: CONTINUE - - def writeEdit(self): - req = self.request() - text = req.field('text', self.page.text) - title = req.field('title', self.page.title) - htmlTitle = self.htmlEncode(title) - log = req.field('changeLog', '') - htmlLog = self.htmlEncode(log) - mimeType = req.field('mimeType', self.page.mimeType) - editField = self.editFieldFor(self.page, mimeType) - quickFindLink = self.servletLink('quickfind', args={'callParent': 'alert'}) - if self.page.exists(): - changeLink = '' - else: - changeLink = 'Change...' % \ - self.pageLink(self.page.name + ".html", action='changeMimeType', - args={'commenting': req.field('commenting', None)}) - if self.canPreview(self.page, mimeType): - previewButton = '' - else: - previewButton = '' - if req.field('commenting', ''): - commenting = ('' - % self.htmlEncode(req.field('commenting'))) - else: - commenting = '' - - if req.field('attaching', ''): - attaching = ('' - % self.htmlEncode(req.field('attaching'))) - else: - attaching = '' - mimeHelpLink = self.helpLink('mimetypes', 'Help on MIME types') - relatedAdd = self.popupLink('quickfind?callParent=relatedAdd', 'Add...') - relatedHelpLink = self.helpLink('relatedterms', 'Help on related terms') - if self.checkPermission('hide'): - if (self.page.hidden - or req.field('hidden', '') - or not self.page.exists() - and self.wiki.config.getbool('starthidden', False)): - hiddenChecked = ' checked' - else: - hiddenChecked = '' - hidden = ('' % (hiddenChecked, - self.helpLink('hiddenpages', 'Help on hidden pages'))) - else: - hidden = '' - action = self.link() - - if not self.user(): - name, email, url = map(self.htmlEncode, self.cookieAuthorInfo()) - metaFields = dedent('''\ - - - - - - - -
-
- (will not be displayed) -
-
''' % (name, email, url)) - else: - metaFields = '' - if self.page.exists(): - describe = dedent('''\ - Describe your changes:
-
- ''' % locals()) - else: - describe = '' - self.write(dedent('''\ -
- - - %(commenting)s - %(attaching)s - MIME type: - %(mimeType)s - %(changeLink)s - %(mimeHelpLink)s -
- - %(editField)s -
- - %(describe)s - - - - %(hidden)s
- %(metaFields)s - ''' % locals())) - self.write(dedent('''\ - - %(previewButton)s - -
- ''' % locals())) - if req.field('commenting', None): - commentPage = self.wiki.page(req.field('commenting')) - self.write('

Commenting on:

\n') - self.write(commentPage.html) - - def cookieAuthorInfo(self): - req = self.request() - savedData = req.cookies().get('userinfo') - if savedData: - savedData = savedData.decode('base64') - name, email, url = savedData.split('|') - else: - name = email = url = '' - name = req.field('authorName', name) - email = req.field('authorEmail', email) - url = req.field('authorURL', url) - return name, email, url - - def saveCookieAuthorInfoFromRequest(self): - req = self.request() - name = req.field('authorName', '') - email = req.field('authorEmail', '') - url = req.field('authorURL', '') - self.saveCookieAuthorInfo(name, email, url) - - def saveCookieAuthorInfo(self, name, email, url): - name = name.replace('|', '') - email = email.replace('|', '') - url = url.replace('|', '') - if (not url.startswith('http://') - and not url.startswith('https://')): - url = 'http://' + url - value = '%s|%s|%s' % (name, email, url) - value = value.encode('base64') - self.response().setCookie('userinfo', value, expires='NEVER') - - def writeEditRelated(self): - p = self.page - if p.relatedDateLimit: - relatedDateLimit = p.relatedDateLimit.seconds / 60 / 60 / 24 - else: - relatedDateLimit = '' - self.write(dedent('''\ - Summarizing options: %(summarizingHelp)s
- - -   - -   - Use date: - -
- - Date limit: - - days -   - Item limit: -
- ''' % { - 'summarizingHelp': self.helpLink('wikisummarizing', - 'Help on the Wiki\'s summarizing'), - 'relatedSummariesCheck': - self.test(p.relatedSummaries, ' checked'), - 'relatedShowDatesCheck': - self.test(p.relatedShowDates, ' checked'), - 'creationDateSelected': - self.test(p.relatedSortField == 'creationDate', ' selected'), - 'modifiedDateSelected': - self.test(p.relatedSortField == 'modifiedDate', ' selected'), - 'relatedDateLimit': relatedDateLimit, - 'relatedEntryLimit': p.relatedEntryLimit or '', - })) - - def canPreview(self, page, mimeType): - return mimeType.startswith('text/') - - def editFieldFor(self, page, mimeType): - if mimeType == 'text/html': - return self.editFieldHTML(page) - elif mimeType == 'text/x-restructured-text': - return self.editFieldRest(page) - elif mimeType.startswith('text/'): - return self.editFieldText(page) - else: - return self.editFieldBinary(page, mimeType) - - def editFieldText(self, page): - req = self.request() - text = req.field('text', page.text) - return ('\n' - '' - % self.htmlEncode(text)) - - def editFieldRest(self, page): - req = self.request() - text = req.field('text', page.text) - # Code for selection from: - # http://www.alexking.org/blog/2003/06/02/inserting-at-the-cursor-using-javascript/ - text = self.htmlEncode(text) - markupHelpLink = str(self.helpLink('wikimarkup', - ' Help on markup', useImage=False)) - insertLink = self.popupLink('quickfind?callParent=restlink', - 'Insert wiki link') - return dedent('''\n - -
- %(insertLink)s | %(markupHelpLink)s (note: - no HTML tags allowed)
- - ''' % locals()) - - def editFieldHTML(self, page): - req = self.request() - text = req.field('text', page.text) - #if req.field('commenting', '') and not text: - # text = ('
-- %s %s\n' - # % (wikipage.canonicalName(self.user().name()), - # self.user().name(), - # datetime.datetime.now().strftime('%d %b \'%y'))) - return dedent('''\n - - - - - - ''' % self.htmlEncode(text)) - - def editFieldBinary(self, page, mimeType): - text = self.request().field('text', page.text) - if text != page.text: - saveText = text - else: - saveText = '' - src = '' - if saveText: - src += ('' - % (saveText.encode('base64'))) - return dedent('''\ - %s - Upload: -
- Comments:
- - ''' % (src, self.htmlEncode(page.comments))) - - def writePreview(self): - req = self.request() - text = req.field('text', '') - if req.hasField('text_upload'): - uploaded = req.field('text_upload') - try: - uploaded = uploaded.file.read() - except AttributeError: - pass - if uploaded: - text = uploaded - if req.hasField('old_upload') and not text: - filename = self.getSecureHidden('old_upload') - # @@: continue - mimeType = req.field('mimeType') - inputType = req.field('inputType') - text = self._inputTypeCleaners.get(inputType, lambda x:x)(text) - self.write(self.page.preview(text, mimeType)) - self.write('
\n') - self.writeEdit() - - def writeHistory(self): - if self.request().field('delete', None): - versions = self.request().field('deleteVersion', None) - if isinstance(versions, (str, unicode)): - versions = (versions,) - elif isinstance(versions, list): - versions = tuple(versions) - try: - versions = map(int, versions) - except: - versions = None - if versions: - for page in self.page.versions(): - if page.version in versions: - self.assertPermission('delete', page=page) - delete = self.request().field('deleteUsers', None) - if delete: - self.assertPermission('deleteuser') - users = self.page.delete(versions) - self.write('

Selected versions deleted.

') - myusername = self.user() and self.user().username() - if delete and users: - deleted = [] - for username in users: - if username != myusername: - try: - user = self.userManager().userForUsername(username) - self.userManager().deleteUser(user) - deleted.append(username) - except: - pass - if deleted: - msg = 'Users deleted: ' + ', '.join(deleted) - # @@ we should change ownership of all remaining pages of users - elif myusername in users: - msg = 'No hara-kiri allowed.' - else: - msg = 'Users were already deleted.' - self.write('

%s

' % msg) - versions = self.page.versions() - if not versions: - self.write('

There are no archived versions available.

') - return - delete = False - for page in versions: - if self.checkPermission('delete', page=page): - delete = True - break - self.write('
' - % (self.link(unversioned=True))) - self.write('\n') - header = ['Version', 'Created on', 'Log', 'User', 'Compare'] - if delete: - header.append('Delete') - header = ''.join(['%s' % h for h in header]) - self.write('%s\n' % header) - firstIndex = len(versions)-2 - otherIndex = firstIndex+1 - for index, page in enumerate(versions): - self.write('\n' % - ['odd', 'even'][index%2]) - self.write('\n' - % (page.link, page.version or 'current')) - self.write('\n' % self.format_date(page.modifiedDate, nonbreaking=True)) - self.write('\n' - % self.htmlEncode(page.lastChangeLog or '')) - self.write('\n' - % self.htmlEncode(page.lastChangeUser or '')) - self.write('\n' - % (page.version, self.test(index==firstIndex, ' checked'), - page.version, self.test(index==otherIndex, ' checked'))) - if delete: - if self.checkPermission('delete', page=page): - self.write('') - self.write('\n') - if delete and self.checkPermission('deleteuser'): - self.write('') - self.write('
' - '%s%s%s%s' - '' - '' - '' % page.version) - else: - self.write(' 
' - 'Check this if you want to delete the user accounts as well:' - '' - '
') - self.write('\n') - self.write('\n') - self.write('\n') - if delete: - self.write('\n') - self.write('
\n') - - def writeBacklinks(self): - self.write('\n') - for index, page in enumerate(self.page.backlinks): - self.write('\n' - % (['odd', 'even'][index%2], - page.link, - page.title)) - self.write('
%s' - '
\n') - - def writeDiff(self): - compType = 'thorough' - for name in self.request().fields().keys(): - if name.startswith('compare_'): - compType = name[len('compare_'):] - if compType == 'thorough': - Matcher = htmldiff.HTMLMatcher - source1 = self.firstPage.html - source2 = self.otherPage.html - elif compType == 'brief': - Matcher = htmldiff.NoTagHTMLMatcher - source1 = self.firstPage.html - source2 = self.otherPage.html - elif compType == 'source': - Matcher = htmldiff.TextMatcher - source1 = self.firstPage.text - source2 = self.otherPage.text - else: - assert 0, "Unknown comparison type: %r" % compType - matcher = Matcher(source1, source2) - diff = matcher.htmlDiff() - start = 'version %s' % (self.firstPage.version or 'Current') - end = 'version %s' % (self.otherPage.version or 'Current') - self.write('

Added to %s (present in %s)
' - 'Deleted from %s (present in %s)

\n' - % (start, end, end, start)) - self.write(diff) - - def writeChangeMimeType(self): - req = self.request() - self.write('

Current MIME type: %s\n%s

\n' - % (self.page.mimeType, - self.helpLink('mimetypes', 'Help on MIME types'))) - self.write('

Edit with a different MIME type:

\n') - allTypes = [t for t in self.wiki.availableMimeTypes - if not t.startswith('image/')] - allTypes.append('image/*') - for mimeType in allTypes: - self.write('%s\n' - % (self.pageLink(self.page.name, - action='edit', - args={'mimeType': mimeType, - 'commenting': req.field('commenting', None)}), - mimeType)) - - _linkRE = re.compile('href\s*=\s*"(.*?)"', re.I+re.S) - def cleanXinha(self, text): - return self._linkRE.sub(self._urlSubber, str(tidy(text))) - - def _urlSubber(self, match): - url = match.group(1) - baseLink = self.servletLink('', absolute=True) - if url.startswith(baseLink): - url = url[len(baseLink):] - return 'href="%s"' % url diff --git a/Context/SitePage.py b/Context/SitePage.py deleted file mode 100644 index 4cd336a..0000000 --- a/Context/SitePage.py +++ /dev/null @@ -1,429 +0,0 @@ -import os -import shutil -import time - -from lib import wiki, wikiconfig, user, menubar -from lib.common import pprint, dprint, dedent -from lib.formatdate import format_date -from lib.securehidden import SecureSigner - -from Component import CPage -from Component.notify import NotifyComponent - -from LoginKit import UserComponent -from LoginKit.rfc822usermanager import RFC822UserManager - -from TaskKit.Task import Task -from TaskKit.Scheduler import Scheduler - -from WebKit import AppServer -from WebKit.HTTPExceptions import * - -from WebUtils.Funcs import htmlEncode - -__all__ = ['SitePage', 'pprint', 'dprint'] - - -class SitePage(CPage): - - components = [NotifyComponent()] - _adsenseID = None - - def awake(self, transaction): - CPage.awake(self, transaction) - domain = self.request().environ().get('HTTP_HOST', 'SERVER_NAME') - if ':' in domain: - domain = domain.split(':', 1)[0] - self.wiki = self.TheGlobalWiki.site(domain=domain) - self.wiki.basehref = self.request().adapterName() + '/' - self.suppressFooter = False - self.response().setHeader( - 'content-type', 'text/html; charset=utf-8') - self.setupEarly() - self.config = self.wiki.config.merge_page_class(self.pageClass()) - if self._adsenseID is None: - SitePage._adsenseID = self.config.get('googleadsenseid', '') - self.assertPermission('view', self.pageClass()) - self.setup() - - def setupEarly(self): - """ - This is called before setup(), and before we resolve .pageClass() - (which gets used to create the config). Most pages should - override seutp() instead. - """ - pass - - def setup(self): - pass - - def checkPermission(self, *args, **kw): - try: - self.assertPermission(*args, **kw) - return True - except (HTTPAuthenticationRequired, HTTPForbidden): - return False - - _cachedPermissions = {} - - def assertPermission(self, action='view', pageClass=None, page=None): - if isinstance(action, (str, unicode)): - action = (action,) - elif isinstance(action, list): - action = tuple(action) - userID = self.user() and self.user().userID() - if not page: - cacheKey = (userID, action, pageClass) - try: - result = self._cachedPermissions[cacheKey] - if result == 'auth': - raise HTTPAuthenticationRequired - elif result: - raise HTTPForbidden(result) - else: - return - except KeyError: - pass - if pageClass is not None and pageClass != self.pageClass(): - config = self.wiki.config.merge_page_class(pageClass) - else: - config = self.config - config_section = None - for actual_action in action: - config_section = config.get(actual_action) - if config_section: - break - if not config_section: - msg = ("You must configure the roles to %s this page" - % actual_action) - if not page: - self._cachedPermissions[cacheKey] = msg - raise HTTPForbidden(msg) - role = config_section.get('requiredrole', 'none').strip().lower() - role = filter(None, role.split()) - if not role or role == ['none']: - if not page: - self._cachedPermissions[cacheKey] = None - return - if not self.user(): - if not page: - self._cachedPermissions[cacheKey] = 'auth' - raise HTTPAuthenticationRequired - if 'user' in role: - if not page: - self._cachedPermissions[cacheKey] = None - return - user_roles = self.user().roles() - if userID: - if page: - authorID = page.authorUser and page.authorUser.userID() - else: - authorID = self.authorUser() and self.authorUser().userID() - if userID == authorID: - user_roles.append('author') - for has_role in user_roles: - if has_role in role: - if not page: - self._cachedPermissions[cacheKey] = None - return - msg = ("You must have the role %s to %s this page" - % (' or '.join(role), actual_action)) - if not page: - self._cachedPermissions[cacheKey] = msg - raise HTTPForbidden(msg) - - ######################################## - ## Utility methods - ######################################## - - def loginRequired(self): - return False - - def pageClass(self): - return 'posting' - - def authorUser(self): - return None - - def writeSimplePageList(self, pages): - self.write('') - for index, page in enumerate(pages): - self.write('\n' - % (['even', 'odd'][index%2], - page.link, - page.title)) - self.write('
%s
\n') - - def test(self, op, t, f=''): - if op: - return t - else: - return f - - def encodeArgs(self, args=None): - q = self.urlEncode - if args is None: - args = self.request().fields() - if isinstance(args, dict): - args = args.items() - all = [] - for name, value in args: - if isinstance(value, str): - all.append((name, value)) - else: - for v in value: - all.append((name, v)) - return '&'.join([ - '%s=%s' % (q(name), q(value)) - for name, value in all]) - - ######################################## - ## Page layout - ######################################## - - def writeHeadParts(self): - CPage.writeHeadParts(self) - self.writeRelatedLinks() - self.writeJavascript() - - def writeRelatedLinks(self): - self.write('\n' - % self.wiki.linkTo('feeds/recent_changes.xml')) - self.write('\n' - % self.wiki.linkTo('feeds/new_pages.xml')) - - def writeStyleSheet(self): - self.write('\n' - % self.request().adapterName()) - - def writeJavascript(self): - self.write('\n' - % self.request().adapterName()) - - def writeBodyParts(self): - self.writeHeader() - self.writeMessages() - self.viewMethod()() - self.writeFooter() - - def writeHeader(self): - menu = [getattr(self, 'menu' + name)() for name in self.menus()] - bar, parts = menubar.menubarHTML(menu) - self.write(parts) - self.write('
') - self.write(bar) - self.write('
') - #self.write('
\n') - self.writeGoogleAds() - if self.htTitle(): - self.write('

%s

\n' % self.htTitle()) - - def writeGoogleAds(self): - if not self._adsenseID: - return - self.write(dedent('''\ -
-
''' % self._adsenseID)) - - def menus(self): - return ['User', 'Title', 'Goto', 'Help', 'Search'] - - def menuUser(self): - name = self.user() and self.user().username() - if name: - name = '%s' % name - else: - name = '' - return (menubar.Literal, name) - - def menuTitle(self): - return (menubar.Literal, '%s:' - % (self.wiki.config.getbool('blog', False) and 'Blog' or 'Wiki')) - - def menuGoto(self): - menu = [ - ('Home', self.wiki.basehref), - ('Recent Changes', self.wiki.basehref + 'recentchanges'), - ('Orphaned Pages', self.wiki.basehref + 'orphans'), - ('Wanted Pages', self.wiki.basehref + 'wanted'), - ] - if self.checkPermission(action='edit', - pageClass='posting'): - menu.append(('Create %s' % (self.wiki.config.getbool('blog', - 'False') and 'Post' or 'Page'), - "javascript:window.location='%s/' + " - "escape(window.prompt('Enter the name " - "for the new page').replace(/ /g, '-')) + " - "'?_action_=edit'" - % self.request().adapterName())) - if self.user() and 'admin' in self.user().roles(): - menu.append(('Administration', self.wiki.basehref + 'admin')) - menu.append((menubar.Separator, '')) - if self.user(): - menu.append(('Logout', '?_actionLogout=yes')) - else: - menu.append(('Login', self.wiki.basehref + 'login?returnTo=%s' - % self.request().environ()['REQUEST_URI'].split('?')[0])) - return ('Goto', menu) - - def menuHelp(self): - return ('Help', [ - ('About this wiki', self.wiki.basehref + 'thiswiki.html'), - ('Help with markup', self.wiki.basehref + 'wikimarkup.html'), - (menubar.Separator, ''), - ('Related terms', self.wiki.basehref + 'relatedterms.html'), - ]) - - def menuSearch(self): - return (menubar.Literal, - '    ') - - def sendRedirectAndEnd(self, url): - """ - This alternate version of sendRedirectAndEnd checks if we are - setting any cookies, and if so and the user is also using - Internet Explorer, then we do the redirect with Javascript - instead of with HTTP headers. - """ - if not self.response().cookies(): - # If there aren't any cookies, then we always do the - # full redirect - return CPage.sendRedirectAndEnd(self, url) - req = self.request() - agent = req.environ().get('HTTP_USER_AGENT') - if not agent or agent.find('MSIE') == -1: - return CPage.sendRedirectAndEnd(self, url) - # Otherwise we're dealing with MSIE, which has problems - # with setting cookies and then immediately redirecting. - self.write('Redirecting' - % self.htmlEncode(url)) - self.write('' - % url) - self.write('

Redirecting...

') - self.write('Redirecting to %s...' - % (self.htmlEncode(url), url)) - self.write('') - self.endResponse() - - def format_date(self, date, nonbreaking=False): - return format_date(date, nonbreaking=nonbreaking) - - def popupLink(self, link, text): - return ('%s' % (link, link, text)) - - def helpLink(self, dest, text, useImage=True): - if useImage: - text = ('help' % text) - link = self.wiki.page(dest).link - return self.popupLink(link, text) - - def pageLink(self, name, action=None, args=None): - args = args or {} - if action: - args['_action_'] = action - argsRendered = ['%s=%s' % (self.urlEncode(n), self.urlEncode(v)) - for (n, v) in args.items() - if v is not None] - url = self.request().adapterName() + '/' + name - if argsRendered: - if '?' in url: - url += '&' - else: - url += '?' - url += '&'.join(argsRendered) - return url - - def secureHidden(self, name, value, timeout=None): - return ('' % - (name, - self.htmlEncode(self._secureSigner.secureValue(value, timeout=timeout)))) - - def getSecureHidden(self, name): - return self._secureSigner.parseSecure(self.request().field(name)) - - def htmlEncode(s): - if isinstance(s, unicode): - s = s.encode('utf-8') - return htmlEncode(s) - htmlEncode = staticmethod(htmlEncode) - - def write(self, s): - try: - CPage.write(self, s) - except UnicodeDecodeError: - CPage.write(self, s.encode('utf-8')) - - -############################################################ -## Global setup -############################################################ - -thisDir = os.path.dirname(__file__) -parentDir = os.path.dirname(thisDir) -configFilename = AppServer.globalAppServer.serverSidePath('Configs/wiki.ini') -if not os.path.exists(configFilename): - configTemplateFilename = os.path.join(parentDir, 'wiki.ini') - print "Coping %s -> %s" % (configTemplateFilename, configFilename) - shutil.copyfile(configTemplateFilename, - configFilename) -config = wikiconfig.WikiConfig() -config.load(configFilename) -AppServer.globalAppServer._imp.watchFile(configFilename) -TheGlobalWiki = wiki.GlobalWiki(config) -for filename in os.listdir(thisDir): - if filename.endswith('.py'): - TheGlobalWiki.addSpecialName(filename[:-3]) -TheGlobalWiki.addSpecialName('rss') - -SitePage.TheGlobalWiki = TheGlobalWiki - -user_path = TheGlobalWiki.config.get('userpath') -if user_path is None: - user_path = os.path.join(TheGlobalWiki.root, 'users') -manager = RFC822UserManager(path=user_path, - userClass=user.WikiUser) -user.manager = manager -SitePage.components.append(UserComponent(manager, loginServlet='/login')) -signatureFilename = os.path.join(TheGlobalWiki.root, 'secret.txt') -SitePage._secureSigner = SecureSigner(signatureFilename) - - -class PublishTask(Task): - - def __init__(self, globalWiki): - self.globalWiki = globalWiki - - def run(self): - for wiki in self.globalWiki.cachedWikis.values(): - wiki.publish() - - -scheduler = Scheduler() -scheduler.start() -scheduler.addPeriodicAction( - time.time(), 10, PublishTask(TheGlobalWiki), 'PublishTask') diff --git a/Context/__init__.py b/Context/__init__.py deleted file mode 100644 index e9bd5fc..0000000 --- a/Context/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# It should not be necessary, but in the following way -# you can configure xinha to appear in the current directory, -# though it is physically located in a separate directory: -# urlJoins = ['xinha/'] - -def contextInitialize(appServer, path): - pass - diff --git a/Context/admin.py b/Context/admin.py deleted file mode 100644 index a0dd11a..0000000 --- a/Context/admin.py +++ /dev/null @@ -1,97 +0,0 @@ -from SitePage import * -from lib.import_atom import import_atom - -class admin(SitePage): - - def title(self): - return 'Administration' - - def loginRequired(self): - return True - - def pageClass(self): - return 'admin' - - def actions(self): - return ['rebuildIndex', 'rebuildHTML', 'rebuildStatic', - 'recreateThumbnails', 'import_atom'] - - def rebuildIndex(self): - self.wiki.rebuildIndex() - self.done() - - def rebuildHTML(self): - self.wiki.rebuildHTML() - self.done() - - def rebuildStatic(self): - self.wiki.rebuildStatic() - self.done() - - def recreateThumbnails(self): - self.wiki.recreateThumbnails() - self.done() - - def done(self): - self.sendRedirectAndEnd('admin') - - def import_atom(self): - output = [] - atom_data = self.request().field('atom_upload') - if not isinstance(atom_data, str): - atom_data = atom_data.value - redirects = import_atom( - self.wiki, atom_data, output.append) - redirects_page = self.wiki.page('redirects') - if not redirects_page.exists(): - redirects_page.hidden = True - redirects_page.pageType = 'admin' - redirects_page.mimeType = 'text/x-redirect-list' - redirect_map = {} - for line in redirects_page.text.splitlines(): - from_link, to_link = line.split(None, 1) - redirect_map.setdefault(from_link, []).append(to_link) - for from_link, to_link in redirects: - if to_link in redirect_map.get(from_link, []): - continue - redirect_map.setdefault(from_link, []).append(to_link) - redirect_map = redirect_map.items() - redirect_map.sort() - lines = [] - for from_link, to_links in redirect_map: - for to_link in to_links: - lines.append('%s %s\n' % (from_link, to_link)) - redirects_page.text = ''.join(lines) - redirects_page.lastChangeLog = 'Updated from atom import' - redirects_page.save() - for line in output: - self.message(line) - self.sendRedirectAndEnd('admin') - - def writeContent(self): - self.write('

Commands:

\n') - self.write('
\n') - for desc, name in [('Rebuild index', 'rebuildIndex'), - ('Rebuild/rerender all HTML', 'rebuildHTML'), - ('Rebuild static site', 'rebuildStatic'), - ('Recreate thumbnails', 'recreateThumbnails'), - ]: - self.write('

\n' - % (name, desc)) - self.write('
') - self.write(''' - -
- -

ATOM (or maybe RSS) feed to import:

-

-

-
- ''') diff --git a/Context/default.css b/Context/default.css deleted file mode 100644 index 620cf86..0000000 --- a/Context/default.css +++ /dev/null @@ -1,275 +0,0 @@ -/* -:Author: David Goodger -:Contact: goodger@python.org -:Date: $Date$ -:Revision: $Revision$ -:Copyright: This stylesheet has been placed in the public domain. - -Default cascading style sheet for the HTML output of Docutils. - -See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to -customize this style sheet. -*/ - -/* used to remove borders from tables and images */ -.borderless, table.borderless td, table.borderless th { - border: 0 } - -table.borderless td, table.borderless th { - /* Override padding for "table.docutils td" with "! important". - The right padding separates the table cells. */ - padding: 0 0.5em 0 0 ! important } - -.first { - /* Override more specific margin styles with "! important". */ - margin-top: 0 ! important } - -.last, .with-subtitle { - margin-bottom: 0 ! important } - -.hidden { - display: none } - -a.toc-backref { - text-decoration: none ; - color: black } - -blockquote.epigraph { - margin: 2em 5em ; } - -dl.docutils dd { - margin-bottom: 0.5em } - -/* Uncomment (and remove this text!) to get bold-faced definition list terms -dl.docutils dt { - font-weight: bold } -*/ - -div.abstract { - margin: 2em 5em } - -div.abstract p.topic-title { - font-weight: bold ; - text-align: center } - -div.admonition, div.attention, div.caution, div.danger, div.error, -div.hint, div.important, div.note, div.tip, div.warning { - margin: 2em ; - border: medium outset ; - padding: 1em } - -div.admonition p.admonition-title, div.hint p.admonition-title, -div.important p.admonition-title, div.note p.admonition-title, -div.tip p.admonition-title { - font-weight: bold ; - font-family: sans-serif } - -div.attention p.admonition-title, div.caution p.admonition-title, -div.danger p.admonition-title, div.error p.admonition-title, -div.warning p.admonition-title { - color: red ; - font-weight: bold ; - font-family: sans-serif } - -/* Uncomment (and remove this text!) to get reduced vertical space in - compound paragraphs. -div.compound .compound-first, div.compound .compound-middle { - margin-bottom: 0.5em } - -div.compound .compound-last, div.compound .compound-middle { - margin-top: 0.5em } -*/ - -div.dedication { - margin: 2em 5em ; - text-align: center ; - font-style: italic } - -div.dedication p.topic-title { - font-weight: bold ; - font-style: normal } - -div.figure { - margin-left: 2em ; - margin-right: 2em } - -div.footer, div.header { - clear: both; - font-size: smaller } - -div.line-block { - display: block ; - margin-top: 1em ; - margin-bottom: 1em } - -div.line-block div.line-block { - margin-top: 0 ; - margin-bottom: 0 ; - margin-left: 1.5em } - -div.sidebar { - margin-left: 1em ; - border: medium outset ; - padding: 1em ; - background-color: #ffffee ; - width: 40% ; - float: right ; - clear: right } - -div.sidebar p.rubric { - font-family: sans-serif ; - font-size: medium } - -div.system-messages { - margin: 5em } - -div.system-messages h1 { - color: red } - -div.system-message { - border: medium outset ; - padding: 1em } - -div.system-message p.system-message-title { - color: red ; - font-weight: bold } - -div.topic { - margin: 2em } - -h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, -h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { - margin-top: 0.4em } - -h1.title { - text-align: center } - -h2.subtitle { - text-align: center } - -hr.docutils { - width: 75% } - -img.align-left { - clear: left } - -img.align-right { - clear: right } - -ol.simple, ul.simple { - margin-bottom: 1em } - -ol.arabic { - list-style: decimal } - -ol.loweralpha { - list-style: lower-alpha } - -ol.upperalpha { - list-style: upper-alpha } - -ol.lowerroman { - list-style: lower-roman } - -ol.upperroman { - list-style: upper-roman } - -p.attribution { - text-align: right ; - margin-left: 50% } - -p.caption { - font-style: italic } - -p.credits { - font-style: italic ; - font-size: smaller } - -p.label { - white-space: nowrap } - -p.rubric { - font-weight: bold ; - font-size: larger ; - color: maroon ; - text-align: center } - -p.sidebar-title { - font-family: sans-serif ; - font-weight: bold ; - font-size: larger } - -p.sidebar-subtitle { - font-family: sans-serif ; - font-weight: bold } - -p.topic-title { - font-weight: bold } - -pre.address { - margin-bottom: 0 ; - margin-top: 0 ; - font-family: serif ; - font-size: 100% } - -pre.literal-block, pre.doctest-block { - margin-left: 2em ; - margin-right: 2em } - -span.classifier { - font-family: sans-serif ; - font-style: oblique } - -span.classifier-delimiter { - font-family: sans-serif ; - font-weight: bold } - -span.interpreted { - font-family: sans-serif } - -span.option { - white-space: nowrap } - -span.pre { - white-space: pre } - -span.problematic { - color: red } - -span.section-subtitle { - /* font-size relative to parent (h1..h6 element) */ - font-size: 80% } - -table.citation { - border-left: solid 1px gray; - margin-left: 1px } - -table.docinfo { - margin: 2em 4em } - -table.docutils { - margin-top: 0.5em ; - margin-bottom: 0.5em } - -table.footnote { - border-left: solid 1px black; - margin-left: 1px } - -table.docutils td, table.docutils th, -table.docinfo td, table.docinfo th { - padding-left: 0.5em ; - padding-right: 0.5em ; - vertical-align: top } - -table.docutils th.field-name, table.docinfo th.docinfo-name { - font-weight: bold ; - text-align: left ; - white-space: nowrap ; - padding-left: 0 } - -h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, -h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { - font-size: 100% } - -ul.auto-toc { - list-style-type: none } diff --git a/Context/edit_icon.gif b/Context/edit_icon.gif deleted file mode 100644 index 771dbad..0000000 Binary files a/Context/edit_icon.gif and /dev/null differ diff --git a/Context/exclamation.gif b/Context/exclamation.gif deleted file mode 100644 index 76d6386..0000000 Binary files a/Context/exclamation.gif and /dev/null differ diff --git a/Context/favicon.ico b/Context/favicon.ico deleted file mode 100644 index 74566ff..0000000 Binary files a/Context/favicon.ico and /dev/null differ diff --git a/Context/feeds.py b/Context/feeds.py deleted file mode 100644 index 6ebdc8a..0000000 --- a/Context/feeds.py +++ /dev/null @@ -1,40 +0,0 @@ -from SitePage import * - -class feeds(SitePage): - - def awake(self, trans): - SitePage.awake(self, trans) - req = self.request() - name = req.extraURLPath().strip('/') - self.feedName = name - - def writeHTML(self): - if self.feedName: - self.writeFeed(self.feedName) - else: - SitePage.writeHTML(self) - - def writeContent(self): - self.write('

There are two RSS feeds available:

') - self.write('

Recent Changes:' - ' a feed of all changes to the site.

\n' - '

New posts: ' - ' a feed of all new posts to the site.

' - % (self.wiki.linkTo('feeds/recent_changes.xml'), - self.wiki.linkTo('feeds/new_pages.xml'))) - - def writeFeed(self, name): - self.response().setHeader('Content-type', - 'application/rss+xml; charset=utf-8') - if name == 'recent_changes.xml': - f = open(self.wiki.syndicateRecentChangesFilename) - self.write(f.read()) - f.close() - elif name == 'new_pages.xml': - f = open(self.wiki.syndicateNewPagesFilename) - self.write(f.read()) - f.close() - else: - self.response().setHeader('Status', '404 Not Found') - self.write('Not Found') - diff --git a/Context/forgotten.py b/Context/forgotten.py deleted file mode 100644 index 9088436..0000000 --- a/Context/forgotten.py +++ /dev/null @@ -1,10 +0,0 @@ -from LoginKit.forgotten import ForgottenPasswordComponent -from SitePage import * - -class forgotten(SitePage): - - components = SitePage.components + [ForgottenPasswordComponent()] - - def defaultAction(self): - self.forgottenPasswordForm() - diff --git a/Context/frontpage.py b/Context/frontpage.py deleted file mode 100644 index 2cd4c6f..0000000 --- a/Context/frontpage.py +++ /dev/null @@ -1,66 +0,0 @@ -from SitePage import SitePage -import time - -class frontpage(SitePage): - - _cached_content = None - _cache_time = 0 - _cache_timeout = 60 - - def title(self): - return self.wiki.config.get('rss') and \ - self.wiki.config.get('rss').get('title') or 'The Wiki Frontpage' - - def writeRelatedLinks(self): - SitePage.writeRelatedLinks(self) - description = self.wiki.config.get('rss') and \ - self.wiki.config.get('rss').get('description') - if description: - self.write('\n' % description) - if self.user() and self.checkPermission('edit'): - # add universal edit button - page = self.wiki.page('index') - self.write('' % page.link) - - def writeContent(self): - if self.wiki.config.getbool('blog', False): - if not self._cached_contenttime or \ - time() - self._cache_time > self._cache_timeout: - frontpage._cached_content = self.getBlogContent() - frontpage._cache_time = time.time() - self.write(self._cached_content) - else: - self.write(self.getWikiContent()) - - def getBlogContent(self): - result = [] - write = result.append - recent = [p for p in self.wiki.recentCreated() - if p.pageClass == 'posting'][:10] - for page in recent: - write('

%s

\n' % - (page.link, page.title)) - write(page.html) - comments = page.commentPages - if not comments: - comment_text = 'No comments' - elif len(comments) == 1: - comment_text = '1 thread' - else: - comment_text = '%i threads' % len(comments) - write('
# ** %s ** ' - '%s
\n' - % (page.link, self.format_date(page.creationDate), - page.link, comment_text)) - write('
\n') - result.pop() # remove last
- return ''.join(result) - - def getWikiContent(self): - page = self.wiki.page('index') - result = page.html - if self.user() and self.checkPermission('edit'): - result += ('\n
' - 'Edit this page
\n') % page.link - return result diff --git a/Context/images/ed_wikilink.gif b/Context/images/ed_wikilink.gif deleted file mode 100644 index f379816..0000000 Binary files a/Context/images/ed_wikilink.gif and /dev/null differ diff --git a/Context/login.py b/Context/login.py deleted file mode 100644 index 42073b6..0000000 --- a/Context/login.py +++ /dev/null @@ -1,201 +0,0 @@ -from lib.common import dedent -from SitePage import * - - -class login(SitePage): - - def setup(self): - self._formErrors = {} - if (self.session() and self.session().hasValue('userID') - and self.request().field('_actionLoginUsername_', False)): - returnTo = self.request().field('returnTo', '') - self.sendRedirectAndEnd(returnTo - or self.request().adapterName() + '/') - - def pageClass(self): - return 'none' - - def newUsers(self): - return self.wiki.config.getbool('newusers', True) - - def actions(self): - if self.newUsers(): - return ['create'] - else: - return [] - - def title(self): - if self.view() == 'writeCreateForm': - return 'Create User Account' - elif self.view() == 'writeSuccess': - return 'User Created' - else: - return 'Login' - - def create(self): - self.setView('writeCreateForm') - req = self.request() - if not req.field('save', ''): - return - username = req.field('username', '') - name = req.field('name', '') - email = req.field('email', '') - website = req.field('website', '') - if not website.startswith('http'): - website = 'http://' + website - password = req.field('password') - errors = self._formErrors - if not username: - errors['username'] = 'Username is required' - elif self.userManager().userExists(username): - errors['username'] = 'That username is taken' - if not name: - errors['name'] = 'Your name is required' - if not self.check_email(email): - errors['email'] = 'Your email address is required' - if not password: - errors['password'] = 'Please enter a password' - if password != req.field('confirm_password', ''): - errors['password'] = 'Your password does not match your confirmation' - captcha = self.captcha() - if captcha: - id= req.field('captcha_id', None) - input = req.field('captcha_input', '') - if not captcha.check(input, id): - errors['captcha'] = 'This was not the expected input' - if errors: - self.writeHTML() - return - user = self.userManager().newUser() - user.setUsername(username) - user.setEmail(email) - user.setWebsite(website) - user.setPassword(password) - user.setName(name) - self.setUser(user) - self.message('User %s created' % username) - returnTo = req.field('returnTo', '') - if returnTo: - self.sendRedirectAndEnd(returnTo) - else: - self.setView('writeSuccess') - self.writeHTML() - - def writeContent(self): - self.write(self.simpleLoginForm()) - self.write('

Forgot your password?

\n' % - self.servletLink('forgotten')) - if self.newUsers(): - self.write('

Or create a new user account...

\n') - self.writeCreateForm() - - def writeSuccess(self): - self.write('User created!') - - def writeCreateForm(self, returnTo=''): - values = {} - values['action'] = self.servletLink('login') - for key in 'username name email website password confirm_password captcha returnTo'.split(): - values[key] = self.htmlEncode(self.request().field(key, '')) - if self._formErrors.has_key(key): - values['error_%s' % key] = '%s
\n' % self._formErrors[key] - else: - values['error_%s' % key] = '' - if returnTo: - values['returnTo'] = self.htmlEncode(returnTo) - - values['captcha'] = '' - captcha = self.captcha() - if captcha: - field = self.request().field - id= field('captcha_id', None) - input = field('captcha_input', '') - if captcha.check(input, id): - values['captcha'] = dedent('''\ - - - ''' % (input, id)) - else: - values['captcha'] = captcha.create() - values['captcha'] = dedent('''\ - -

To avoid abuse of this Wiki, we have to make a simple
- plausibility test that checks whether you are a serious user.

- %(error_captcha)s -

Please fill in:

%(captcha)s - - ''' % values) - - self.write(dedent('''\ -
-

No email confirmation required; just fill in the values - and go!

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %(captcha)s - - - - -
Username:%(error_username)s -
Full name:%(error_name)s -
Email address:%(error_email)s -
Website:%(error_website)s -
Password:%(error_password)s -
Confirm:%(error_confirm_password)s -
-
- ''' % values)) - - def check_email(self, email): - email = email.split('@', 2) - if len(email) != 2: - return False - if len(email[0]) < 1: - return False - if len(email[1]) < 5: - return False - if not '.' in email[1]: - return False - return True - - def captcha(self): - captcha = str(self.config.get('captcha')) - if captcha: - module = 'lib.captchas.' + captcha - return __import__(module, None, None, module).Captcha() - else: - return None diff --git a/Context/menubar.js b/Context/menubar.js deleted file mode 100644 index 3c6306b..0000000 --- a/Context/menubar.js +++ /dev/null @@ -1,457 +0,0 @@ -//***************************************************************************** -// Do not remove this notice. -// -// Copyright 2000-2004 by Mike Hall. -// See http://www.brainjar.com for terms of use. -//***************************************************************************** - -//---------------------------------------------------------------------------- -// Code to determine the browser and version. -//---------------------------------------------------------------------------- - -function Browser() { - - var ua, s, i; - - this.isIE = false; // Internet Explorer - this.isOP = false; // Opera - this.isNS = false; // Netscape - this.version = null; - - ua = navigator.userAgent; - - s = "Opera"; - if ((i = ua.indexOf(s)) >= 0) { - this.isOP = true; - this.version = parseFloat(ua.substr(i + s.length)); - return; - } - - s = "Netscape6/"; - if ((i = ua.indexOf(s)) >= 0) { - this.isNS = true; - this.version = parseFloat(ua.substr(i + s.length)); - return; - } - - // Treat any other "Gecko" browser as Netscape 6.1. - - s = "Gecko"; - if ((i = ua.indexOf(s)) >= 0) { - this.isNS = true; - this.version = 6.1; - return; - } - - s = "MSIE"; - if ((i = ua.indexOf(s))) { - this.isIE = true; - this.version = parseFloat(ua.substr(i + s.length)); - return; - } -} - -var browser = new Browser(); - -//---------------------------------------------------------------------------- -// Code for handling the menu bar and active button. -//---------------------------------------------------------------------------- - -var activeButton = null; - -// Capture mouse clicks on the page so any active button can be -// deactivated. - -if (browser.isIE) - document.onmousedown = pageMousedown; -else - document.addEventListener("mousedown", pageMousedown, true); - -function pageMousedown(event) { - - var el; - - // If there is no active button, exit. - - if (activeButton == null) - return; - - // Find the element that was clicked on. - - if (browser.isIE) - el = window.event.srcElement; - else - el = (event.target.tagName ? event.target : event.target.parentNode); - - // If the active button was clicked on, exit. - - if (el == activeButton) - return; - - // If the element is not part of a menu, reset and clear the active - // button. - - if (getContainerWith(el, "DIV", "menu") == null) { - resetButton(activeButton); - activeButton = null; - } -} - -function buttonClick(event, menuId) { - - var button; - - // Get the target button element. - - if (browser.isIE) - button = window.event.srcElement; - else - button = event.currentTarget; - - // Blur focus from the link to remove that annoying outline. - - button.blur(); - - // Associate the named menu to this button if not already done. - // Additionally, initialize menu display. - - if (button.menu == null) { - button.menu = document.getElementById(menuId); - if (button.menu.isInitialized == null) - menuInit(button.menu); - } - - // Reset the currently active button, if any. - - if (activeButton != null) - resetButton(activeButton); - - // Activate this button, unless it was the currently active one. - - if (button != activeButton) { - depressButton(button); - activeButton = button; - } - else - activeButton = null; - - return false; -} - -function buttonMouseover(event, menuId) { - - var button; - - // Find the target button element. - - if (browser.isIE) - button = window.event.srcElement; - else - button = event.currentTarget; - - // If any other button menu is active, make this one active instead. - - if (activeButton != null && activeButton != button) - buttonClick(event, menuId); -} - -function depressButton(button) { - - var x, y; - - // Update the button's style class to make it look like it's - // depressed. - - button.className += " menuButtonActive"; - - // Position the associated drop down menu under the button and - // show it. - - x = getPageOffsetLeft(button); - y = getPageOffsetTop(button) + button.offsetHeight; - - // For IE, adjust position. - - if (browser.isIE) { - x += button.offsetParent.clientLeft; - y += button.offsetParent.clientTop; - } - - button.menu.style.left = x + "px"; - button.menu.style.top = y + "px"; - button.menu.style.visibility = "visible"; -} - -function resetButton(button) { - - // Restore the button's style class. - - removeClassName(button, "menuButtonActive"); - - // Hide the button's menu, first closing any sub menus. - - if (button.menu != null) { - closeSubMenu(button.menu); - button.menu.style.visibility = "hidden"; - } -} - -//---------------------------------------------------------------------------- -// Code to handle the menus and sub menus. -//---------------------------------------------------------------------------- - -function menuMouseover(event) { - - var menu; - - // Find the target menu element. - - if (browser.isIE) - menu = getContainerWith(window.event.srcElement, "DIV", "menu"); - else - menu = event.currentTarget; - - // Close any active sub menu. - - if (menu.activeItem != null) - closeSubMenu(menu); -} - -function menuItemMouseover(event, menuId) { - - var item, menu, x, y; - - // Find the target item element and its parent menu element. - - if (browser.isIE) - item = getContainerWith(window.event.srcElement, "A", "menuItem"); - else - item = event.currentTarget; - menu = getContainerWith(item, "DIV", "menu"); - - // Close any active sub menu and mark this one as active. - - if (menu.activeItem != null) - closeSubMenu(menu); - menu.activeItem = item; - - // Highlight the item element. - - item.className += " menuItemHighlight"; - - // Initialize the sub menu, if not already done. - - if (item.subMenu == null) { - item.subMenu = document.getElementById(menuId); - if (item.subMenu.isInitialized == null) - menuInit(item.subMenu); - } - - // Get position for submenu based on the menu item. - - x = getPageOffsetLeft(item) + item.offsetWidth; - y = getPageOffsetTop(item); - - // Adjust position to fit in view. - - var maxX, maxY; - - if (browser.isIE) { - maxX = Math.max(document.documentElement.scrollLeft, document.body.scrollLeft) + - (document.documentElement.clientWidth != 0 ? document.documentElement.clientWidth : document.body.clientWidth); - maxY = Math.max(document.documentElement.scrollTop, document.body.scrollTop) + - (document.documentElement.clientHeight != 0 ? document.documentElement.clientHeight : document.body.clientHeight); - } - if (browser.isOP) { - maxX = document.documentElement.scrollLeft + window.innerWidth; - maxY = document.documentElement.scrollTop + window.innerHeight; - } - if (browser.isNS) { - maxX = window.scrollX + window.innerWidth; - maxY = window.scrollY + window.innerHeight; - } - maxX -= item.subMenu.offsetWidth; - maxY -= item.subMenu.offsetHeight; - - if (x > maxX) - x = Math.max(0, x - item.offsetWidth - item.subMenu.offsetWidth - + (menu.offsetWidth - item.offsetWidth)); - y = Math.max(0, Math.min(y, maxY)); - - // Position and show it. - - item.subMenu.style.left = x + "px"; - item.subMenu.style.top = y + "px"; - item.subMenu.style.visibility = "visible"; - - // Stop the event from bubbling. - - if (browser.isIE) - window.event.cancelBubble = true; - else - event.stopPropagation(); -} - -function closeSubMenu(menu) { - - if (menu == null || menu.activeItem == null) - return; - - // Recursively close any sub menus. - - if (menu.activeItem.subMenu != null) { - closeSubMenu(menu.activeItem.subMenu); - menu.activeItem.subMenu.style.visibility = "hidden"; - menu.activeItem.subMenu = null; - } - removeClassName(menu.activeItem, "menuItemHighlight"); - menu.activeItem = null; -} - -//---------------------------------------------------------------------------- -// Code to initialize menus. -//---------------------------------------------------------------------------- - -function menuInit(menu) { - - var itemList, spanList; - var textEl, arrowEl; - var itemWidth; - var w, dw; - var i, j; - - // For IE, replace arrow characters. - - if (browser.isIE) { - menu.style.lineHeight = "2.5ex"; - spanList = menu.getElementsByTagName("SPAN"); - for (i = 0; i < spanList.length; i++) - if (hasClassName(spanList[i], "menuItemArrow")) { - spanList[i].style.fontFamily = "Webdings"; - spanList[i].firstChild.nodeValue = "4"; - } - } - - // Find the width of a menu item. - - itemList = menu.getElementsByTagName("A"); - if (itemList.length > 0) - itemWidth = itemList[0].offsetWidth; - else - return; - - // For items with arrows, add padding to item text to make the - // arrows flush right. - - for (i = 0; i < itemList.length; i++) { - spanList = itemList[i].getElementsByTagName("SPAN"); - textEl = null; - arrowEl = null; - for (j = 0; j < spanList.length; j++) { - if (hasClassName(spanList[j], "menuItemText")) - textEl = spanList[j]; - if (hasClassName(spanList[j], "menuItemArrow")) { - arrowEl = spanList[j]; - } - } - if (textEl != null && arrowEl != null) { - textEl.style.paddingRight = (itemWidth - - (textEl.offsetWidth + arrowEl.offsetWidth)) + "px"; - // For Opera, remove the negative right margin to fix a display bug. - if (browser.isOP) - arrowEl.style.marginRight = "0px"; - } - } - - // Fix IE hover problem by setting an explicit width on first item of - // the menu. - - if (browser.isIE) { - w = itemList[0].offsetWidth; - itemList[0].style.width = w + "px"; - dw = itemList[0].offsetWidth - w; - w -= dw; - itemList[0].style.width = w + "px"; - } - - // Mark menu as initialized. - - menu.isInitialized = true; -} - -//---------------------------------------------------------------------------- -// General utility functions. -//---------------------------------------------------------------------------- - -function getContainerWith(node, tagName, className) { - - // Starting with the given node, find the nearest containing element - // with the specified tag name and style class. - - while (node != null) { - if (node.tagName != null && node.tagName == tagName && - hasClassName(node, className)) - return node; - node = node.parentNode; - } - - return node; -} - -function hasClassName(el, name) { - - var i, list; - - // Return true if the given element currently has the given class - // name. - - list = el.className.split(" "); - for (i = 0; i < list.length; i++) - if (list[i] == name) - return true; - - return false; -} - -function removeClassName(el, name) { - - var i, curList, newList; - - if (el.className == null) - return; - - // Remove the given class name from the element's className property. - - newList = new Array(); - curList = el.className.split(" "); - for (i = 0; i < curList.length; i++) - if (curList[i] != name) - newList.push(curList[i]); - el.className = newList.join(" "); -} - -function getPageOffsetLeft(el) { - - var x; - - // Return the x coordinate of an element relative to the page. - - x = el.offsetLeft; - if (el.offsetParent != null) - x += getPageOffsetLeft(el.offsetParent); - - return x; -} - -function getPageOffsetTop(el) { - - var y; - - // Return the x coordinate of an element relative to the page. - - y = el.offsetTop; - if (el.offsetParent != null) - y += getPageOffsetTop(el.offsetParent); - - return y; -} diff --git a/Context/miniXmlButton.gif b/Context/miniXmlButton.gif deleted file mode 100644 index cfa86cc..0000000 Binary files a/Context/miniXmlButton.gif and /dev/null differ diff --git a/Context/moin-diff.gif b/Context/moin-diff.gif deleted file mode 100644 index e89ba8a..0000000 Binary files a/Context/moin-diff.gif and /dev/null differ diff --git a/Context/orphans.py b/Context/orphans.py deleted file mode 100644 index 00ad4fa..0000000 --- a/Context/orphans.py +++ /dev/null @@ -1,13 +0,0 @@ -from SitePage import * - -class orphans(SitePage): - - def title(self): - return 'Orphaned pages' - - def writeContent(self): - pages = self.wiki.orphanPages() - self.write('

These %i pages are not linked to from ' - 'any other pages.

\n' - % len(pages)) - self.writeSimplePageList(pages) diff --git a/Context/question_icon.gif b/Context/question_icon.gif deleted file mode 100644 index 6cfc061..0000000 Binary files a/Context/question_icon.gif and /dev/null differ diff --git a/Context/quickfind.py b/Context/quickfind.py deleted file mode 100644 index 3fa877a..0000000 --- a/Context/quickfind.py +++ /dev/null @@ -1,146 +0,0 @@ -from SitePage import * - -repopulate = ''' -var lastValue = ""; - -function repopulate() { - var search = document.getElementById("search").value.toLowerCase().split(/\s+/); - var resultNames = new Array(); - var resultTitlesCapped = new Array(); - var resultTitles = new Array(); - var resultMimeTypes = new Array(); - var item, i, title, pos; - for (i = 0; i < allNames.length; i++) { - item = allNames[i]; - if (searchMatches(search, item)) { - resultNames[resultNames.length] = item; - title = capitalizeSearch(search, allTitles[i]) - resultTitlesCapped[resultTitlesCapped.length] = title; - resultTitles[resultTitles.length] = allTitles[i]; - resultMimeTypes[resultMimeTypes.length] = allMimeTypes[i]; - } - } - var select = document.getElementById("pages"); - for (i = select.length; i >= resultNames.length; i--) { - select.options[i] = null; - } - for (i = 0; i < resultNames.length; i++) { - select.options[i] = new Option(resultTitlesCapped[i], - resultNames[i] + "**" + resultMimeTypes[i] - + "**" + resultTitles[i]); - } - if (! resultNames.length) { - select.options[0] = new Option("No results", ""); - } -} - -function searchMatches(search, term) { - term = term.toLowerCase(); - var i; - for (i = 0; i < search.length; i++) { - if (term.indexOf(search[i]) == -1) { - return false; - } - } - return true; -} - -function capitalizeSearch(search, term) { - var i; - for (i = 0; i < search.length; i++) { - pos = term.toLowerCase().indexOf(search[i]); - if (pos != -1) { - term = term.substring(0, pos) - + term.substring(pos, pos+search[i].length).toUpperCase() - + term.substring(pos+search[i].length, term.length); - } - } - return term; -} - -function research() { - var search = document.getElementById("search").value; - if (search != lastValue) { - lastValue = search; - repopulate(); - } -} - -function getSelected() { - var select = document.getElementById("pages"); - var items = select.options[select.selectedIndex].value.split("**"); - return items; -} - -function callParent(func_name) { - var items = getSelected(); - //window.alert("Calling " + func_name + " of " + window.opener - //+ " which is " + window.opener[func_name]); - window.opener[func_name](items[0], items[1], items[2]); -} - -function callParentBare(func_name) { - var search = document.getElementById("search").value; - var name = search.toLowerCase(); - name = name.replace(/[^a-z]/g, ""); - window.opener[func_name](name, "text/html", search); -} -''' - -class quickfind(SitePage): - - def awake(self, trans): - SitePage.awake(self, trans) - self.suppressFooter = True - self.callParent = self.request().field('callParent', '') - - def title(self): - return 'Quick Page Find' - - def writeArray(self, name, array): - self.write('var %s = new Array(\n' % name) - self.write(',\n '.join( - [repr(str(v)) for v in array])) - self.write(');\n') - - def writeContent(self): - onSelect = None - if self.callParent: - onSelect = 'callParent(%s); window.close();' \ - % repr(self.callParent) - if onSelect: - selectAttr = ' onChange="%s"' % onSelect - else: - selectAttr = '' - - onCreate = "callParentBare(%s); window.close();" \ - % repr(self.callParent) - - self.write(''' -

All pages on system:

- Search:
- - - -
-
- - ''' % (onCreate, selectAttr)) - - self.write('') diff --git a/Context/recentchanges.py b/Context/recentchanges.py deleted file mode 100644 index b35d647..0000000 --- a/Context/recentchanges.py +++ /dev/null @@ -1,70 +0,0 @@ -from SitePage import * - - -class recentchanges(SitePage): - - def writeContent(self): - recentType = self.request().field('type', 'changes') - if recentType == 'changes': - pages = self.wiki.recentPages() - elif recentType == 'created': - pages = self.wiki.recentCreated() - else: - assert 0, "Unknown type: %r" % recentType - - limit = int(self.request().field('limit', 0)) - if limit: - pages = pages[:limit] - - if recentType == 'changes': - self.write('

RSS feed of recent changes: \n' - % (self.wiki.linkTo('feeds/recent_changes.xml'), - self.wiki.linkTo('miniXmlButton.gif'))) - elif recentType == 'created': - self.write('

RSS feed of new posts: \n' - % (self.wiki.linkTo('feeds/new_pages.xml'), - self.wiki.linkTo('miniXmlButton.gif'))) - - - self.write(''' - - - - - - - ''') - index = 0 - anyError = False - for page in pages: - index += 1 - if index % 2: - rowClass = 'even' - else: - rowClass = 'odd' - if page.hasParseErrors: - anyError = True - bang = ('\n' - % (self.wiki.linkTo('exclamation.gif'))) - else: - bang = '' - self.write('\n' - % (rowClass, page.link, page.title, bang)) - self.write('\n' - % self.format_date(page.modifiedDate, nonbreaking=True)) - self.write('\n' - % self.htmlEncode(page.lastChangeLog or '')) - self.write('\n' - % self.htmlEncode(page.lastChangeUser or '')) - self.write('\n') - self.write('
PageLast modifiedLog messageUser
%s%s%s%s%s
') - if anyError: - self.write(''' -

[errors] - indicates that errors occured while trying to parse the - document.

- ''' % self.wiki.linkTo('exclamation.gif')) - - def title(self): - return 'Recent Changes' diff --git a/Context/search.py b/Context/search.py deleted file mode 100644 index f35ce5f..0000000 --- a/Context/search.py +++ /dev/null @@ -1,83 +0,0 @@ -from SitePage import * -import re - -class search(SitePage): - - def awake(self, trans): - SitePage.awake(self, trans) - req = self.request() - self.titleSearch = req.field('searchTitle', '').strip() - self.bodySearch = req.field('searchBody', '').strip() - self.gotoSearch = req.field('searchGoto', '').strip() - self.genericSearch = req.field('search', '') - if isinstance(self.genericSearch, list): - self.genericSearch = self.genericSearch[0] - self.genericSearch = self.genericSearch.strip() - if self.genericSearch: - if self.genericSearch.lower().startswith('title:'): - junk, self.titleSearch = self.genericSearch.split(':', 1) - self.titleSearch = self.titleSearch.strip() - elif self.genericSearch.lower().startswith('goto:'): - junk, self.gotoSearch = self.genericSearch.split(':', 1) - self.gotoSearch = self.gotoSearch.strip() - else: - self.bodySearch = self.genericSearch - self.results = None - self.explanation = 'Search' - if self.bodySearch: - self.explanation = 'Search for "%s"' % self.htmlEncode(self.bodySearch) - self.results = self.wiki.search(self.bodySearch) - elif self.titleSearch: - self.explanation = 'Title search for "%s"' % self.htmlEncode(self.titleSearch) - self.results = self.wiki.searchTitles(self.titleSearch) - elif self.gotoSearch: - self.explanation = 'Goto "%s"' % self.htmlEncode(self.gotoSearch) - self.results = self.wiki.searchNames(self.gotoSearch) - if self.results: - if len(self.results) > 1: - self.message('Also matched: %s' - % ', '.join(['%s' - % (page.link, page.title) - for page in self.results[1:]])) - self.sendRedirectAndEnd(self.results[0].link) - - def writeContent(self): - self.writeForm() - if self.results is not None: - if not self.results: - self.write('

No pages found

\n') - else: - self.writeResults(self.results) - - def title(self): - return self.explanation - - def htTitle(self): - return '' - - def writeForm(self): - self.write(''' - ''' - % (self.htmlEncode(self.genericSearch or self.titleSearch or self.bodySearch))) - - def writeResults(self, results): - results.sort(lambda a, b: cmp(a.name, b.name)) - self.write('
\n') - query = self.bodySearch or self.titleSearch - regex = re.compile('(%s)' % re.escape(query), re.I) - for page in results: - self.write('
%s
\n' - % (page.link, self.htmlEncode(page.title))) - segment = self.htmlEncode(page.searchSegment(query, length=200)) - segment = regex.sub(r'\1', segment) - self.write('
%s
' - % segment) - self.write('
\n') - diff --git a/Context/wanted.py b/Context/wanted.py deleted file mode 100644 index 8947d36..0000000 --- a/Context/wanted.py +++ /dev/null @@ -1,32 +0,0 @@ -from SitePage import * - -class wanted(SitePage): - - def title(self): - return 'Wanted pages' - - def writeContent(self): - pages = self.wiki.wantedPages() - # sort by most wanted, then name: - pages.sort(lambda a, b: cmp((-len(a[1]), a[0].name), - (-len(b[1]), b[0].name))) - self.write('

These %i pages are being linked to, but ' - 'have not been created yet.

\n' - % len(pages)) - self.write('\n') - self.write('\n') - self.write('\n') - self.write('\n') - self.write('\n') - index = 0 - for page, wants in pages: - index += 1 - for subindex, want in enumerate(wants): - self.write('\n' - % (want.link, self.htmlEncode(want.title))) - self.write('
Wanted pageWanted by
' - % ['odd', 'even'][index%2]) - if not subindex: - self.write('%s' - % (page.link, page.title)) - self.write('%s
') diff --git a/Context/wiki.css b/Context/wiki.css deleted file mode 100644 index fe4ae55..0000000 --- a/Context/wiki.css +++ /dev/null @@ -1,474 +0,0 @@ -/* - Webware for Python Wiki (http://wiki.webwareforpython.org) - - Common style sheet for the Webware Wiki pages -*/ - -/* First import default style for pages created with Docutils: */ - -@import url(default.css); - -/* Customization for the Wiki goes here: */ - -body { - background-color: #FFFFFF; - font-family: sans-serif; - font-size: 11pt; - line-height: 14pt; - padding: 4pt; -} -i, em { - font-family: serif; - font-size: 115%; -} -a.target { - color: blue; -} -a.reference { - text-decoration: none; - border-bottom: 1px dotted #666666; -} -a.reference:hover { - background-color: #eeeeee; -} -span.nowiki { - background-color: #ffffcc; -} -a.wiki { - text-decoration: none; - border-bottom: 1px dotted #0000cc; -} -a.wiki:hover { - background-color: #eeeeff; -} -a.version { - display: block; -} -a.version:hover { - text-decoration: none; - background-color: #ddddff; -} -cite { - font-style: normal; - font-family: monospace; - font-weight: bold; -} -table { - margin-top: 6pt; - margin-bottom: 6pt; - border-color: #909097; - empty-cells: show; - border-spacing: 0pt; - border-collapse: collapse; -} -td, th { - font-family: sans-serif; - font-size: 11pt; - padding: 2pt 4pt; - border-color: green; - vertical-align: top -} -tr.even { - background-color: #dddddd; -} -tr.header { - background-color: #000066; - color: #ffffff; -} -hr { - width: 75% } -p { - margin-top: 6pt; - margin-bottom: 6pt; - text-align: justify; -} -li { - margin-bottom: 5pt; - list-style-type: circle; -} -dt { - margin-bottom: 2pt; -} -dd { - margin-bottom: 5pt; -} -h1 { - font-size: 17pt; -} -h2 { - font-size: 15pt; -} -h3 { - font-size: 14pt; -} -h4 { - font-size: 13pt; -} -h5 { - font-size: 12pt; -} -.contents { - font-family: sans-serif; -} -.contents ul { - list-style: none; - padding-left: 0em; - margin-left: 2em; -} -.contents ul li { - font-size: medium; - margin-bottom: 3pt; -} -.contents ul ul { - list-style-type: none; - margin-top: 2pt; - padding-left: 0em; - margin-left: 1.5em; -} -.contents ul ul li { - font-size: small; - margin-bottom: 1pt; -} -.contents .topic-title { - font-size: 16pt; -} -code, .literal, .literal-block, .pre, .py { - font-family: monospace; - font-size: 10pt; - color: #052; -} -tt { - color: #000066 -} -tt.literal, span.pre { - background-color: #FFFFFF; -} -pre.py, pre.literal-block { - margin: 0; - padding: 2pt 1pt 1pt 2pt; - background-color: #F0F0F8; -} -.typed { - font-weight: bold; -} -.error { - color: red; -} -.warning { - color: brown; -} -div.admonition, div.attention, div.caution, div.danger, div.error, -div.hint, div.important, div.note, div.tip, div.warning { - background-color: #cccccc; - padding: 2pt; -} - -/* Highlighting: */ - -.function {color: #000077; font-weight: bold} -.keyword {color: #004444;} -.comment {color: #770000; font-style: italic} -.normal {color: #000000;} -.string {color: #006600;} -.symbol {color: #000000;} - -.htmltag {color: #000077;} -.htmlsymbol {color: #000000;} -.htmlnormal {color: #000000;} -.htmlcomment {color: #770000; font-style: italic} -.htmlstring {color: #006600;} -.htmlattr {color: #000000;} - -.stmlfunction {color: #000077; font-weight: bold} -.stmlattr {color: #000000;} -.stmlstring {color: #006600;} -.stmlexpr {color: #004444;} - -/* Page tabs: */ - -#page_nav ul { - display: block; - list-style: none outside; - padding: 0; - margin: 0 0 0 10px; - font-size: medium; -} -#page_nav li { - padding: 0; - margin: 0 4px 0 0; - border: 1px solid #cecbc6; - background-color: #000066; - color: #ffffff; - font-weight: bold; - /* the next 4 lines are magic */ - display: block; - float: left; - position: relative; - bottom: 1.32em; -} -#page_nav a:link, -#page_nav a:visited { - background-color: #000066; - color: #ffffff; - text-decoration: none; - padding-left: 1em; - padding-right: 1em; -} -#page_nav a:hover { - background-color: #cecbc6; - color: #000066; -} -a.viewing#viewing { - /* For some reason I can't get it to - work without both the class and ID */ - background-color: #ffffff; - color: #000066; - border: 1px solid #000066; -} -div#bottom_nav { - border-top: 1px solid black; - background-color: #000066; - color: #ffffff; - padding: 0px 5px 1px 5px; -} -span.navgroup { - border: 1px solid white; - font-weight: bold; -} -a#recentchanges_xml { - padding-left: 0.25em; - padding-right: 0.5em; -} -a#recentchanges { - padding-right: 0.25em; -} -.navgroup a { - color: #ffffff; - text-decoration: none; - padding-left: 1em; - padding-right: 1em; -} -select.navgroup { - color: #ffffff; - background-color: #000066; - border: thin solid #ffffff; - font-weight: bold; -} -.navgroup a:hover { - background-color: #cecbc6; - color: #000066; -} - -/* Magic stuff to stop display:float */ - -#page_navclear { - display: none; -} -html>body #page_navclear { - display: block; - clear: both; -} - -/* Search results */ - -dd.summary { - font-size: small; -} -form#searchbar { - border: 1px solid black; - background-color: #ddddff; - padding: 0.5em; -} -#searchbar tt { - color: #000099; - font-weight: bold; -} - -/* Diff results */ - -.insert { background-color: #aaffaa } -.delete { background-color: #ff8888 } -.tagInsert { background-color: #004400; color: #ffffff } -.tagDelete { background-color: #770000; color: #ffffff } -.tagInsert tt { color: #99ff99 } - -/* summary/keyword pages */ - -.modifiedDate { font-size: small } - -/* NotifyComponent */ - -.notifyMessage { - border: thin black solid; - width: 70%; - background-color: #006600; - color: #ffffff; -} -.notifyMessage a { - color: #bbffbb; -} -.notifyMessage a:visited { - color: #ddddff; -} - -/* Login/forms */ - -table.loginForm { - background-color: #d0d0da; - color: #000066; - border: 2px solid; - border-color: #f0f0fc #909097 #909097 #f0f0fc; -} -.loginForm input { - border: 1px solid; - border-color: #909097 #f0f0fc #f0f0fc #909097; -} -.formError { - background-color: #993333; - color: #ffffff; - font-size: 10pt; - padding: 1pt 4pt; -} -a.button { - background-color: #d0d0da; - color: #000000; - font-size: 10pt; - text-decoration: none; - padding: 1pt 4pt; - border: 2px groove #d0d0da; -} -a.button:hover { - background-color: #f0f0fc; - border: 2px groove #f0f0fc; -} -a.menu { - display: block; - padding: 2px; - text-decoration: none; - font-weight: bold; -} -a.menu:hover { - background-color: #cccccc; -} - -/* Related entries */ - -.related { - /* border-top: 2px solid black; */ -} -.relatedEntry { - border-bottom: 1px dotted #999999; -} -.relatedDate { - text-align: right; - font-size: small; -} - -/* Menubar */ - -div.menuBar, -div.menuBar a.menuButton, -div.menu, -div.menu a.menuItem { - font-family: "MS Sans Serif", Arial, sans-serif; - font-size: 8pt; - font-style: normal; - font-weight: normal; - color: #000000; -} -div.menuBar { - background-color: #d0d0da; - border: 2px solid; - border-color: #f0f0fc #909097 #909097 #f0f0fc; - padding: 4px 2px; - text-align: left; -} -div.menuBar span.menuUser { - background-color: #f0f0fc; - color: #990000; - margin: 0px 2px; - padding: 1px 6px; - text-align: right; - float: right; -} -div.menuBar span.menuTitle { - color: #000099; - margin: 1px; - padding: 2px 6px; -} -div.menuBar a.menuButton { - background-color: transparent; - border: 1px solid #d0d0da; - color: #000000; - cursor: default; - left: 0px; - margin: 1px; - padding: 2px 6px; - position: relative; - text-decoration: none; - top: 0px; - z-index: 100; -} -div.menuBar a.menuButton:hover { - background-color: transparent; - border-color: #f0f0fc #909097 #909097 #f0f0fc; - color: #000000; -} -div.menuBar a.menuButtonActive, -div.menuBar a.menuButtonActive:hover { - background-color: #a0a0c8; - border-color: #909097 #f0f0fc #f0f0fc #909097; - color: #ffffff; - left: 1px; - top: 1px; -} -div.menu { - background-color: #d0d0da; - border: 2px solid; - border-color: #f0f0fc #909097 #909097 #f0f0fc; - left: 0px; - padding: 0px 1px; - position: absolute; - top: 0px; - visibility: hidden; - z-index: 101; -} -div.menu a.menuItem { - color: #000000; - cursor: default; - display: block; - padding: 3px 1em; - text-decoration: none; - white-space: nowrap; -} -div.menu a.menuItem:hover, div.menu a.menuItemHighlight { - background-color: #000090; - color: #ffffff; -} -div.menu a.menuItem span.menuItemText {} -div.menu a.menuItem span.menuItemArrow { - margin-right: -.75em; -} -div.menu div.menuItemSep { - border-top: 1px solid #909097; - border-bottom: 1px solid #f0f0fc; - margin: 4px 2px; -} -input.menuSearch { - font-family: "MS Sans Serif", Arial, sans-serif; - font-size: 7pt; - padding: 0px; - border: 1px solid; - border-color: #909097 #f0f0fc #f0f0fc #909097; -} -div.menu span.disabled { - color: #666666; -} -div.source-code { - background-color: #000000; - border: inset #999999 3px; - overflow: auto; -} -div.source-code a.source-link { - float: right; -} \ No newline at end of file diff --git a/INSTALL.txt b/INSTALL.txt deleted file mode 100644 index 5f3af7c..0000000 --- a/INSTALL.txt +++ /dev/null @@ -1,107 +0,0 @@ -Ian's Webware Wiki version 0.2 -============================== - -How to install this thing: --------------------------- - -The Webware Wiki requires the following software: - -* Python >= 2.3 (http://www.python.org) - -* uTidylib >= 0.2 (http://utidylib.berlios.de) or - mxTidy >= 3.0 (http://www.egenix.com/products/python/mxExperimental/mxTidy/) - -* Docutils >= 0.4 (http://docutils.sourceforge.net) - -* If you want to run the test modules, you need to install - py.test (http://codespeak.net/py/dist/test.html). - -* Webware version 1.0 and the Component 0.2 and LoginKit 0.1 plug-ins, - (http://www.webwareforpython.org/downloads/Webware/Webware-1.0.tar.gz, - http://www.webwareforpython.org/downloads/Component/Component-0.2.tar.gz, - http://www.webwareforpython.org/downloads/LoginKit/LoginKit-0.1.tar.gz) - - In order to install Webware, unpack it to a directory like /usr/local/Webware - and move the Component and LoginKit directories into the Webware directory. - Then run the install.py script inside the Webware directory. - -* Now you need to install the actual Wiki software. The download location is - - http://www.webwareforpython.org/downloads/Wiki/WebwareWiki-0.2.tar.gz - - You can unpack it as a subdirectory of the Webware directory or anywhere - else, since this is not a Webware plugin, but just an additional library. - - You can also check out the latest version from the Subversion repository:: - - svn co svn://svn.w4py.org/Wiki/trunk Wiki - -* For WYSIWYG editing, the Wiki software needs to be supplemented with Xinha - (http://xinha.webfactional.com), we were using Xinha 0.95 for this version. - Unpack the software into the subdirectory ``Wiki/Context/xinha`` (note that - the xinha subdirectory does not yet exist). - -* The next step is to set up a new Webware Wiki working directory:: - - $WEBWARE_DIR/bin/MakeAppWorkDir.py \ - -c Wiki -d $WIKI_DIR/Context -l $WIKI_DIR $WIKI_WORKDIR - - Here, WEBWARE_DIR is the directory where Webware is installed - (e.g. /usr/local/Webware), WIKI_DIR is the directory where the Wiki - library is installed (e.g. /usr/local/Webware/Wiki), and WIKI_WORKDIR - is where the Webware Wiki working directory shall be created - (e.g. /home/wiki/WorkDir). - -* Copy the standard ``wiki.ini`` configuration file to the working directory:: - - cp $WIKI_DIR/wiki.ini $WIKI_WORKDIR/Configs - - Customize ``$WIKI_WORKDIR/Configs/wiki.ini`` to fit your needs. The - ``basepath`` setting in the global section is where all your Wikis will - be stored (e.g. /var/lib/wiki). This should be writable by the AppServer. - Also, for any domains there should be a section, like ``[vhost(localhost)]`` - -- if you have multiple domains that should serve the same wiki, use:: - - [vhost(localhost)] - canonical = canonical.domain.name - - You must also customize the configuration files used by Webware. - Particularly, in ``$WIKI_WORKDIR/Configs/Application.config``, set:: - - ExtraPathInfo = True - - Also, set ``ErrorEmailServer`` and ``ErrorEmailHeaders``. - Remove all Webware contexts that you don't want to use. - You should also customize the app server using the configuration file - ``$WIKI_WORKDIR/Configs/AppServer.config``. - - In ``$WIKI_WORKDIR/Configs/AppServer.config``, add the path ``$WIKI_DIR`` - to the PlugInDirs list and make other appropriate changes. - -Then you should be all set! - -How to run this thing: ----------------------- - -Start the Webware application as usual, e.g. by running - - $WIKI_WORKDIR/AppServer - -or by running the ``webkit`` start script under Unix. Under Windows, -you need to run ``AppServer.bat`` or use ``AppServerService.py``. -See also http://www.webwareforpython.org/Webware/WebKit/Docs/UsersGuide.html. - -With the default settings in ``$WIKI_WORKDIR/Configs/AppServer.config`` -you should now be able to access the Wiki at http://localhost:8080. - -If you want to use Apache or another web server see also -http://www.w4py.org/Webware/WebKit/Docs/InstallGuide.html -and http://wiki.w4py.org/webserverintegration.html. - -After creating a user account on the login page, you should grant yourself -as the first user the "admin" role, by editing the users/User-1.txt file -in the directory where the wiki is stored, so you can administer the wiki. - - - -- Ian Bicking, 26 Apr 2004 - -- Updated by Christoph Zwerschke, 3 Jan 2009 diff --git a/README.md b/README.md deleted file mode 100644 index a852afc..0000000 --- a/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Ye Olde Webware for Python Wiki - -This GitHub repository serves as an archive for both the [code](https://github.com/WebwareForPython/w4py-olde-wiki/tree/master) and the [content](https://webwareforpython.github.io/w4py-olde-wiki/) of the old Webware for Python Wiki. - -Note that [Webware for Python](https://webwareforpython.github.io/w4py) is still maintained, but this Wiki has been frozen in 2010 and will not be updated any more. diff --git a/__init__.py b/__init__.py deleted file mode 100644 index a1b37a8..0000000 --- a/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Ian's Webware Wiki - -__version__ = '0.2' \ No newline at end of file diff --git a/ajaxinwebware.html b/ajaxinwebware.html new file mode 100644 index 0000000..e7a98d8 --- /dev/null +++ b/ajaxinwebware.html @@ -0,0 +1,84 @@ + + + + + +AJAX in Webware + + + +
+

AJAX in Webware

+ +

John Dickinson has created three files which, together, provide a fairly simple way to implement some Ajax functionality into Webware servlets. AjaxPage provides a super class to your servlet (but below your site page). All JavaScript necessary is contained in ajax.js and ajax_polling.js. ajax.js provides basic functionality useful for most situations, and ajax_polling.js provides support for long-running requests and out-of-band server to client commands.

+
+

AjaxPage

+

The AjaxPage class creates a function ajax_allowed() which returns a list of function names. These function names refer to Python functions that are able to be called by an Ajax-enabled web page. This is very similar in functionality to webware's actions. Child classes should override ajax_allowed().

+

AjaxPage also provides the class PyJavascript. This class simply tanslates a Python expression into a string. For example:

+
foo = PyJavascript('bar')
+foo.hello('hi').another.element.func() # evaluates as the string "bar.hello('hi').another.element.func()"
+

PyJavascript is based in large part on code from Nevow 0.4.1 (nevow.org).

+

The AjaxPage class uses PyJavascript to set up several useful references to JavaScript functions.

+
+
+

ajax.js and ajax_polling.js

+

ajax.js provides all the client-side heavy lifting required to get Ajax functionality into a web page. Several pieces of it are based on ideas from Apple developer code (developer.apple.com) and Nevow 0.4.1 (nevow.org).

+

ajax_polling.js provides functionality that allows the server to process long-running requests from the client and allows the server to send commands to the client without the user on the client side triggering an event.

+
+
+

The Anatomy of an AJAX-enabled page

+
    +
  1. Client triggers an action. This request is sent to ajax_controller().

  2. +
  3. +
    ajax_controller() calls the appropriate functions on the server.
    +
      +
    1. Once the server-side function has returned, the results are sent back down the pipe to the client...

    2. +
    3. ...unless it took longer that TIMEOUT seconds to complete (the timeout is defaulted to 100). If the function take a while to finish, ajax_controller() puts the results into the results bucket that ajax_response() is polling.

    4. +
    +
    +
    +
  4. +
  5. ajax_response(), which is being polled every few seconds by the client, sees that there is something for the client in the results bucket, grabs it, and sends it to the cient.

  6. +
+

The polling behavior of an ajax-enabled page may be controlled by overriding ajax_clientPollingTimeout. Setting its return value to None prevents the polling from happening at all. Otherwise, this function needs to return the number of seconds to wait until the client polls the server again.

+
+
+

Example

+
class my_servlet(AjaxPage):
+  def ajax_allowed(self):
+    return ['check_if_even']
+
+  def check_if_even(self,val):
+    is_even = (int(val) % 2) == 0
+    if is_even:
+      ret = self.setTag('response_id','The value you entered is even!')
+    else:
+      ret = self.alert('Oh no! The value you entered is not even!')
+    return '%s; %s' % (self.setReadonly('even_num',False),ret) # don't forget to unlock the field
+
+  def writeContent(self):
+    wl = self.writeln
+    wl('<form action="some_page.py" method="post">')
+
+    # will first set the input field to read only, then call check_if_even with the field's value passed in
+    ajax_cmd = self.generic_ajax(self.setReadonly('even_num',True),'check_if_even',self.this.value)
+
+    wl('<input name="even_num" id="even_num" type="text" onchange="%s"/>' % ajax_cmd)
+    wl('<span id="response_id"></span>') # see check_if_even() for tag name
+    wl('</form>')
+

Downloads:

+ +

Robert Forkel has compiled an extended example named AjaxSuggest. This example shows how to implement functionality a la Google suggest. Note: This example will be already included in the Examples context of Webware for Python starting with version 0.9.1.

+
+
+ + + diff --git a/ajaxjavascript.js b/ajaxjavascript.js new file mode 100644 index 0000000..16f74df --- /dev/null +++ b/ajaxjavascript.js @@ -0,0 +1,122 @@ +//some code based on ideas from Apple developer code (developer.apple.com) and nevow 0.4.1 (www.nevow.org) + +// refactoring of existing ajax functionality to allow server-initiated data to be processed client side +// and to prevent server timeouts + +function createRequest() +{ + var req; + if (window.XMLHttpRequest) { + req = new XMLHttpRequest(); + } else if (window.ActiveXObject) { + req = new ActiveXObject("Microsoft.XMLHTTP"); + } + return req; +} + +function openConnection(req,url) +{ + if (req) { + req.onreadystatechange = function() { + if (req.readyState==4) { + if (req.status == 200) { + try { + eval(req.responseText); + } catch (e) { + ; //ignore errors + } + } + } + }; + req.open("GET", url, true); + req.send(null); + } +} + +//generic ajax call +function generic_ajax(pre_action,func) { + if (pre_action) { + eval(pre_action); + } + var additionalArguments = '' + for (i = 2; i') + if self.ajax_clientPollingInterval() is not None: + self.writeln('') diff --git a/ajaxservlet.html b/ajaxservlet.html new file mode 100644 index 0000000..8195597 --- /dev/null +++ b/ajaxservlet.html @@ -0,0 +1,138 @@ + + + + + +Ajaxservlet + + + +
+

Ajaxservlet

+ +

OpenRico is a Javascript for Rich Application library including ajax, drag&drop and many visual effects. +Here explains ajax servlet with OpenRico.

+

The Ajax Servlet library AjaxServlet.py:

+
from WebKit.RPCServlet import RPCServlet
+
+class AjaxServlet(RPCServlet):
+    def awake(self, transaction):
+        RPCServlet.awake(self, transaction)
+        self._response = transaction.response()
+        self._request = transaction.request()
+
+        self._session = transaction.session()
+        self._resp = []
+    def sleep(self, transaction):
+        RPCServlet.sleep(self, transaction)
+        self._response = None
+        self._request = None
+        self._session = None
+        self._resp = None
+
+    def respondToGet(self, trans):
+        try:
+            # ajax servlet don't need cache
+            resp = self.response()
+            resp.setHeader('Cache-Control', 'no-cache')
+            resp.setHeader('Pragma', 'no-cache');
+
+            self.writeln('<ajax-response>')
+            self._respond()
+            self.writeln('</response></ajax-response>')
+
+            self.sendOK('text/xml', ''.join(self._resp), trans)
+        finally:
+            pass # do error handling or database release
+
+    def request(self): return self._request
+    def response(self): return self._response
+    def transaction(self): return self._transaction
+
+    def _respond(self):
+        """ Here goes the response
+        self.write('<span>Here is ajax response!!</span>')
+
+        NOTE: You must call responseToElement() or responseToObject() before write something
+        """
+        raise NotImplementedError
+
+
+    def responseToElement(self, elementId):
+        self.write('<response type="element" id="%s">' % elementId)
+
+    def responseToObject(self, objectId):
+        self.write('<response type="object" id="%s">' % objectId)
+
+    def write(self, *args):
+        for arg in args:
+            self._resp.append(self.encode(str(arg)))
+
+    def writeln(self, *args):
+        self.write(*args)
+        self.write("\n")
+
+    def decode(self, str, encoding='euc-kr'):
+        """ Decode JavaScript quoted string to native string """
+        ret = []
+
+        while s:
+            if s[:2].lower() == '%u':   # 유니코드 인코딩
+                char = s[:6]
+                unicode_chr = int( char[2:], 16)
+                try: ret.append( unichr(unicode_chr).encode(encoding) )
+                except UnicodeEncodeError: ret.append( '&#%s;' % unicode_chr )
+                s = s[6:]
+            else:
+                ret.append( s[0] )
+                s = s[1:]
+
+        return ''.join(ret)
+
+
+    def encode(self, str, encoding='euc-kr'):
+        """ Encode string to HTML unicode representation """
+        p = string.printable
+        ret = []
+        for c in unicode(str, encoding):
+            if c in p: ret.append(c)
+            else: ret.append( '&#%d;' % ord(c) )
+        return ''.join(ret)
+

search zipcode servlet example::

+
from AjaxServlet import *
+from Zipcode import *
+
+class ajax_search_zipcode(AjaxServlet):
+    def _respond(self):
+        self.responseToElement('search_result')
+        zipcode_list = Zipcode.search(self.decode(self.request().field('dong')))
+
+        self.write("""<table cellspacing="1" cellpadding="1" width="100%" bgcolor="black" bordercolordark="#DDDDDD" bordercolorlight="#666666">
+   <tr align="center" bgcolor="#DDDDDD" height="22">
+       <td bgcolor="#DDDDDD">우편번호</td>
+       <td bgcolor="#DDDDDD">주 소</td>
+       <td bgcolor="#DDDDDD">선택</td>
+   </tr>\n""")
+
+        for z in zipcode_list:
+            self.write( """<tr bgcolor="#DDDDDD">
+    <td bgcolor="#FFFFFF" style="white-space: nowrap;">%s</td>
+    <td bgcolor="#FFFFFF" style="white-space: nowrap;">%s %s %s %s</td>
+    <td bgcolor="#FFFFFF"><input type="button" value="선택" onClick="postaddr('%s', '%s');"/></td>
+</tr>\n""" % ( z[0], z[1], z[2], z[3], z[4], z[0], z[1] + z[2] + z[3] ) )
+        self.write('</table>')
+

for real working example:

+
    +
  1. goto http://www.bodd.co.kr/member_register (only korean site)

  2. +
  3. click image button.

  4. +
  5. input "신림9" and click first button

  6. +
+

-- whitekid

+
+ + + diff --git a/ajaxsuggest.py b/ajaxsuggest.py new file mode 100644 index 0000000..e47668c --- /dev/null +++ b/ajaxsuggest.py @@ -0,0 +1,198 @@ +""" +most code taken and adapted from + http://wiki.w4py.org/ajax_in_webware.html +and + http://www.dynamicajax.com/fr/AJAX_Suggest_Tutorial-271_290_312.html + +author: robert forkel +""" +# inherit from AjaxPage, which in turn inherits from ExamplePage +from ajaxpage import AjaxPage + +MAX = 10 # only pass a limited number of suggestions + +class AjaxSuggest(AjaxPage): + + def writeHeadParts(self): + AjaxPage.writeHeadParts(self) + + # write css for the suggestion box + self.write(""" + +""") + # write javascript + self.write(""" + +""") + + def writeContent(self): + self.write("""\ +
+

+ Start typing in some lowercase letters, and get words starting with these characters suggested:
+ +

+

+
+""") + + def ajax_allowed(self): + """ + register the suggest method for use with ajax + """ + return ['suggest'] + + + def suggest(self, prefix): + """ + we return a javascript function call as string + + the javascript function we want called is 'handleSuggestions' and we pass + an array of strings starting with prefix. + """ + s = filter(lambda w: w.startswith(prefix), SUGGESTIONS) + if not s: + s = ['none'] + # + # note: to pass more general python objects to the client side, use json, + # e.g. using json-py's (https://sourceforge.net/projects/json-py/) JsonWriter. + # + return "handleSuggestions([%s]);" % ",".join(map(lambda w: "'%s'" % w, s[:MAX])) + + +from random import choice +from string import ascii_lowercase +SUGGESTIONS = [] +for i in range(500): + word = [] + for j in range(5): + word.append(choice(ascii_lowercase)) + SUGGESTIONS.append(''.join(word)) diff --git a/appideas.html b/appideas.html new file mode 100644 index 0000000..d5c7038 --- /dev/null +++ b/appideas.html @@ -0,0 +1,299 @@ + + + + + +App Ideas + + + +
+

App Ideas

+ +

Here is a list of ideas for open source apps to build with, or for +Webware. If you add an idea or comment, please sign your name and +leave a date stamp. Many of the project ideas listed below will be +interrelated, so use cross-references where appropriate.

+

Extensions to the Webware and WebKit core (as opposed to other Kits or +stand-alone applications) belong on the `Wish List`_ page.

+
+

RSS kit

+

I'm not sure what this would look like, but some object to facilitate +the collection of RSS feeds and embedding them into a web page. I +guess there would be a few parts coded into a single class, or a +single class with helper classes:

+
    +
  • Configuration info: which site to connect to, how often to refresh.

  • +
  • Read and parse the RSS file at the specified interval.

  • +
  • Send some kind of "observe me now" signal to the object that's +serving the final output, so it can refresh itself. Perhaps +Threading.Event or a port of Java's Observable/Observer interface +can help with this.

  • +
  • Offer the RSS data in a Python data structure to any routine that +asks for it. Perhaps a method in the RSS-retrieval object can offer +this, or it can be in a well-known attribute somewhere.

  • +
  • Have some boilerplate output objects that convert the RSS data into +HTML hyperlinks, for simple sites that want an "out of the box" +solution. These classes should be scalable for adding cosmetic +enhancements via stylesheets or subclasses.

  • +
+

-- Mike Orr - 14 Apr 2002

+
+
+

Miscellaneous small objects

+

The following other web application servers may have objects worth +porting to Webware:

+

Horde Horde may have some objects worth porting. "The Horde +Application Framework is a general-purpose web application framework +in PHP, providing classes for dealing with preferences, compression, +browser detection, connection tracking, MIME handling, and more."

+

SkunkWeb

+

-- Mike Orr - 04 Apr 2002

+
+
+

Web mail reader

+

Webmail software that could be ported to Webware or ideas stolen from.

+

Horde IMP -- IMAP/POP e-mail access.

+

(This isn't something I need anytime soon, but would be worthwhile +someday.

+

-- Mike Orr - 04 Apr 2002

+

BoboMail is a decent POP webmail client that runs off CGI and bobo +(an old version of Zope). It's fairly decent code, and even if a lot +of stuff gets rewritten it would serve a good starting point for +writing such a client. -- Ian Bicking - 08 Apr 2002

+
+
+

Python Card Interface

+

(See Python Card Interface)

+

PythonCard -- looks very interesting, and maybe it could provide an +alternative to a browser interface, in those cases where the browser +provides a less-than-optimal interface (e.g., data entry). Using +XmlRpc, maybe there's a way to make this sort of thing easy.

+

-- Ian Bicking - 23 Jan 2002

+
+
+

Webware Objects

+

(See Webware Objects)

+

Just finished reading MiddleKit docs. What are the chances of adding +a feature to MiddleKit that will read an XSD file, instead of CSV. +I'm more than willing to take the point on this ... since I'm bringing +it up, but I would like some help in some of the nitty-gritty details +of MiddleKit. It's new to me. Otherwise I'm just re-writing a whole +bunch of stuff that's already been done.

+

Thanks -- Ray Leyva - 20 Jan 2002

+

I know ambitious name, but it's an ambitious plan. Generate simple +reusable objects that know how to render, replicate, and store +themselves. I'm initially interested in creating objects that will +automagically tie themselves into a database once imported into a +framework, and immediately become available for data entry using +internally defined display / validation schemes

+

This came from my initial, and now semi-abandoned attempts to create a +single interface WebwareReportKit.

+

I have so far created a validation, rendering, and reporting scheme. +I'm starting work on the storage components right now. Don't want to +redo the MiddleKit stuff, but until I know where I'm going with the +general object layout I don't want to commit to any single OR mapping.

+

Things to note, the items I've completed work with XML / XSL / XSD, +and AFAIK the current best crop of those tools live in the java world. +I don't have the time to be able to both build this infrastructure, +and help out on the PyXML stuff. Wish I did. I use XmlRpc to tie +them together, of course this could be SOAP / CORBA or whatever remote +invocation procedure you like to use.

+

WebKit is the Application server that I'm basing the framework off of, +but I'm designing these objects with modularity in mind. IE : I don't +want to rule out using Java based App servers ... if push comes to +shove, and some client somewhere REALLY doesn't want to use Python / +Webware ;D

+

Is this of interest to anyone. I'm trying to figure out the best way +to make this available ... I really want to give back to the +community.

+

All this because I figured out that for me to be able to print the +stuff from WebwareReportKit I would need to have a semi-intelligent +data-entry component first ... duh! ;D

+

PS : I'll get to that re-org sometime soon! :D +-- Ray Leyva - 19 Jan 2002

+

I have been working on (and then reworking over and over again...) +some XML tools, imaginatively known as XMLTools, which attempt to +provide mappings from database schemas to XML "instances" through +descriptions of those instances, which are themselves highly +simplified documents with the same purpose as XML Schema documents +(which is what I presume you mean by XSD).

+

The aim is to support XML instance validation, templating and storage +in various forms (such as in relational databases). An +object-relational mapping could be a consequence of this work, but +that doesn't interest me that much right now.

+

Eventually, I want to bring XMLForms back with its core replaced by +XMLTools, so that the main purpose of XMLForms is the integration with +Webware (and possibly other Web frameworks).

+

-- Paul Boddie - 02 Apr 2002

+
+
+

WebwareReportKit

+

This would be more of a general compendium of standard reports that +can be generated. For example:

+
    +
  • Invoices

  • +
  • Bill Of Materials ( BOM )

  • +
  • Legal documents

  • +
  • Real Estate Documents

  • +
  • Generic process mandated documents.

  • +
  • Heck .. tax forms!

  • +
+

This could be used in conjunction with WebWare, and a database to +generate data driven reports. It'd be a very useful component because +a great many applications require reporting functions.

+

PS: Are we putting newest ideas at the top, and oldest at the bottom? +If so I'd like to ask for general permission to re-order the older +inputs to reflect that.

+

-- Ray Leyva - 27 Nov 2001 <br />

+

That certainly does sound useful! Have you looked at Biggles for +producing charts? ... fine with me about re-ordering -- Tavis Rudd +- 27 Nov 2001

+

Sorry, moved away from python, and code based reporting tools +... moved towards XSL:FO solutions. -- Ray Leyva - 19 Jan 2002

+
+
+

WebwareXManagement

+

(See WebwareXManagement)

+

the link point to a page about some meta discussion about the several +SomethingManagementSystems. The "X" stands for "something" and not a +well known display manager :-) -- Stephan Diehl - 22 Nov 2001

+
+
+

WebwareUserManager

+

(See Webware User Manager)

+

A good user management system is essential for most of the ideas +listed below! Click on the link above for more notes and +discussion. -- Tavis Rudd - 20 Nov 2001

+
+
+

Webware Document Management System

+

(See Webware Document Management System)

+

This might be the same than Webware Content Management System +and/or a Webware Knowledge Management System. More can be seen on +the topic page -- Stephan Diehl - 21 Nov 2001

+
+
+

Webware Wiki

+

(See Webware Wiki)

+

a wiki built with Webware. Wikis are wicked! -- Tavis Rudd - 30 +Oct 2001

+
+
+

Webware CVS Manager

+

(See Webware CVS Manager)

+

A WebKit/Webware implementation of the infamous viewcvs.cgi +script. WebKit's persistence and potential for caching could make a +huge improvement in responsiveness and allow sites like SourceForge to +consume less hardware for this feature.

+

-- Chuck Esterbrook - 31 Oct 2001

+
+
+

Webware DB Admin

+

(See Webware DB Admin)

+

A database management frontend. Like phpMyAdmin, but database +agnostic. phpMyAdmin is one of the most popular applications on +SourceForge. Someone said that "the application sells the platform", +well a high quality tool like this would be a huge selling point for +Webware. -- Tavis Rudd - 01 Nov 2001

+
+
+

Webware Content Management System

+

(See Webware Content Management System)

+

A ContentManagementSystem built with Webware (Similar to Zope and +Macromedia's SiteSpring). Comprising:

+
    +
  • WebwareUserManager

  • +
  • WebAdminInterface

  • +
  • WebwareCVSManager

  • +
  • WebwareTaskManager

  • +
  • SafeDelegation

  • +
  • WebwareSearchEngine

  • +
  • WebwareFolders (like ZopeFolders)

  • +
+

NOTE: this AppIdea is strongly interrelated with the +WebwareSourceforge AppIdea and WebwareXManagement.

+

-- Tavis Rudd - 31 Oct 2001

+
+
+

Webware Cofax

+

(See Webware Cofax)

+

Cofax is the Java software that powers Knight-Ridder newspaper web +sites, like http://inq.philly.com/.

+
+
+

Webware Sourceforge

+

(See Webware Sourceforge)

+

a sourceforge/tigris like application built with Webware. -- +Tavis Rudd - 30 Oct 2001

+
+
+

Webware List Archive

+

(See Webware List Archive)

+

a geocrawler-like email archive viewer (that actually works!) -- +Tavis Rudd - 31 Oct 2001

+
+
+

Webware Cart

+

(See Webware Cart)

+

an ecommerce shopping cart -- Tavis Rudd - 31 Oct 2001

+
+
+

Webware Forum

+

(See Webware Forum)

+

a discussion forum component -- Tavis Rudd - 01 Nov 2001

+
+
+

Python Web Stats

+

(See Python Web Stats)

+

a modular web log analyzer with a webware front-end, that can also be +used programmatically or with other frontends. The existing Open +Source log analyzers suck! -- Tavis Rudd - 31 Oct 2001

+

I have a totally (overly?) interface-agnostic web log analyzer that I +never polished with a nice frontend. If someone wants to try +something with this, contact me -- Ian Bicking - 02 Nov 2001

+
+
+

Webware Slash

+

(See Webware Slash)

+

a slashdot news site that sould let users upgrade from a common news +app such as PHPNuke. The existing PHP sites get slow under load and +the code is getting pretty complex and ugly. A python update will get +attract frustrated PHP webmasters -- Aaron Held - 2 Nov 2001

+
+
+

Webware Vaults

+

(See Webware Vaults)

+

a Parnassus-like site, not just using WebKit's base features, but also +using some of the other kits and extensions -- Paul Boddie - 06 Nov +2001

+
+
+

Webware Groupware

+

(See Webware Groupware)

+

Looking to create a Webware based Groupware application. It should +include email, and calendaring functions. There's a million of them +out there, but not for Webware ;D

+

With this as a starting point I would like to eventually get to +something like a WyattERP (it's hosted on SourceForge), except ASP +and Python/WebWare based.

+ +

-- Ray Leyva - 17 Nov 2001 <br />

+
+
+

Non-Web Interfaces

+

Facilitating non-browser interfaces -- Ian Bicking

+
+
+

Implications?

+

Why is there the avoidance of discussion on commercial implications of Wiki and related material? An example would be for how we might implement it our (ASP) ecommerce software http://www.veracart.com +please inform me if such a discussion is taking place already: y@mytechsupport.com

+
+
+ + + diff --git a/applicationdesign.html b/applicationdesign.html new file mode 100644 index 0000000..a17f755 --- /dev/null +++ b/applicationdesign.html @@ -0,0 +1,36 @@ + + + + + +Application Design + + + +
+

Application Design

+ +

This pages links to recommendations and notes about how you should set +up your application:

+ +

See also: Webware Recipes

+
+ + + diff --git a/appserver.html b/appserver.html new file mode 100644 index 0000000..bfd0d8d --- /dev/null +++ b/appserver.html @@ -0,0 +1,77 @@ + + + + + +AppServer + + + +
+

AppServer

+ +

Webware's core is its "Application Server", which is known as "WebKit".

+

See http://webopaedia.com/TERM/a/application_server.html for more +notes on application servers.

+
+

NOTE:

+

Several people have suggested that the name "WebKit" should be +scrapped when Webware is transitioned to a DistUtils based package +architecture. ChuckEsterbrook likes the name and has rejected these +suggestions so far, but the discussion isn't over yet. My argument is +that it is better to position and describe Webware as 'A +Python-Powered Application Server', that comes with a suite of useful +components, than saying that "Webware for Python is a suite of +software components for developing object-oriented, web-based +applications." The former is clearer and more direct. The latter is +ambiguous and needs clarification. The name "WebKit" and the latter +description imply that the AppServer is just another component of +Webware. This is misleading: the AppServer is the heart of Webware.

+

Using the name WebKit also makes the module/package structure and path +management issues more complex than they should be.

+

-- Tavis Rudd - 29 Oct 2001

+

I wouldn't mind seeing other kits which handle other forms of +communication. What about MailKit, for example? One could imagine the +nature of requests and responses to be quite different in that kind of +environment.

+

-- PaulBoddie - 30 Oct 2001

+

Such 'kits' could still be created if the name 'WebKit' was removed. +A sensible package layout would be something like this:

+
    +
  • Webware [package]

    +
      +
    • MultiPortServer [module]

    • +
    • AppServer [module]

    • +
    • Application [module]

    • +
    • Transaction [module]

    • +
    • Servlet [module]

    • +
    • Request [module]

    • +
    • Response [module]

    • +
    • Session [module]

    • +
    • SessionStore [module]

    • +
    • Cookie [module]

    • +
    • WebUtils [sub-package]

    • +
    • MiscUtils [sub-package]

    • +
    +
  • +
  • MiddleKit [package]

  • +
  • UserKit [package]

  • +
  • FunFormKit [package]

  • +
  • Cheetah [package]

  • +
  • XMLForms [package]

  • +
  • other 'kits' and related packages ...

  • +
+

-- Tavis Rudd - 30 Oct 2001

+

I brought this up again on the list a few weeks ago and Chuck said his +final answer to my request to get rid of the name 'WebKit' was no. +I'll agree to disagree.

+

-- Tavis Rudd - 23 Nov 2001

+
+
+ + + diff --git a/automatichtmlvalidation.html b/automatichtmlvalidation.html new file mode 100644 index 0000000..08b09bc --- /dev/null +++ b/automatichtmlvalidation.html @@ -0,0 +1,65 @@ + + + + + +Automatic HTML Validation + + + +
+

Automatic HTML Validation

+ +
+

Problem

+
    +
  • Doing "View source" from the browser doesn't work if your page was +produced in response to a POST, since your browser may not re-POST +the form variables (mozilla doesn't, at least).

  • +
  • Debugging by hand is tedious, and if posible should be avoided by +using HTML validators.

  • +
  • You can't use online web validators (which validate a given URL) in +the context of a web application, because your web application often +requires a session id, which the validator obviously can't send to +your server

  • +
+
+
+

Approach

+

This approach uses Web Design Group's HTML validator, which can be +installed on your local machine (available at +http://www.htmlhelp.com/tools/validator/ ).

+

You hook it up to your servlets so that the HTML output your servlet +produces is automatically validated, and the validation output and +full HTML source (with highlighted errors) is appended to the normal +output and sent to the browser.

+
+
+

Details

+
    +
  • Install the WDG offline-validator. In particular, you need the +"validate" script which can be called from the command line. Debian +users: apt-get install wdg-html-validator

  • +
  • Apply the patch at +http://www.opensky.ca/~jdhildeb/webware_validator_streambuffer.patch +to webware. This makes it possible to get the accumulated HTML in +the response object from within your servlets, before the response +has been committed.

  • +
  • Add the validateHTML method (available at +http://www.opensky.ca/~jdhildeb/webwarevalidator.py ) to your +SitePage, and call it at the end of writeBodyParts (you'll have to +override Page.writeBodyParts() if you haven't already done +so). validateHTML won't get all the HTML produced by your servlet if +you call response.flush() explicitly, so don't do that. :)

  • +
+

Enjoy!

+

-- JasonHildebrand - 20 Sep 2002

+
+
+ + + diff --git a/badmarshaldata.html b/badmarshaldata.html new file mode 100644 index 0000000..718fdbf --- /dev/null +++ b/badmarshaldata.html @@ -0,0 +1,36 @@ + + + + + +Bad Marshal Data + + + +
+

Bad Marshal Data

+ +

From the 0.6 Install Guide:

+
+

The most common installation problem is a Python exception +appearing in your browser that says "bad marshal data". This is +always caused by pointing the web browser to the app server:

+

http://localhost:8086/WebKit.cgi/

+

But the app server hides behind a web server so the correct URL +would be:

+

http://localhost/WebKit.cgi/

+

That requires that web server and app server are set up to talk to +each other.

+
+

See InstallGuide.html in Webware/WebKit/Docs for full details, or +you can read the one for the last production release at:

+

http://webware.sourceforge.net/Webware/WebKit/Docs/InstallGuide.html

+

-- ChuckEsterbrook - 30 Nov 2001

+
+ + + diff --git a/batteriesincluded.html b/batteriesincluded.html new file mode 100644 index 0000000..458af7b --- /dev/null +++ b/batteriesincluded.html @@ -0,0 +1,25 @@ + + + + + +Batteries Included + + + +
+

Batteries Included

+ +

A packaged version of Webware that comes with the latest versions of +other tools that are commonly used with Webware.

+

Examples of these tools are Cheetah, FunFormKit, XmlForms, and +Python database-adaptor modules such as psycopg.

+

-- TavisRudd - 31 Oct 2001

+
+ + + diff --git a/batteriesincludedproj.html b/batteriesincludedproj.html new file mode 100644 index 0000000..79c9274 --- /dev/null +++ b/batteriesincludedproj.html @@ -0,0 +1,99 @@ + + + + + +Batteries Included Proj + + + +
+

Batteries Included Proj

+ +
+

Project Summary

+
+

Synopsis

+

Create a BatteriesIncluded distribution of Webware that comes with +other related tools, such as Cheetah, FunFormKit, XmlForms, the +database adaptors, etc.

+
+
+

Project Moderator

+

TavisRudd

+
+
+

Project Members

+
    +
  • TavisRudd

  • +
+
+
+

Current Status

+

Packaging/installation system using distutils has been created as part +of WebwareExpRefactoring.

+
+
+

Problem

+

There are several tools that are designed for, or complement, Webware +that are not currently distributed with it and must be downloaded and +installed separately. As a result Webware installations take longer +than they should.

+
+
+

Solution

+

Uses Python's DistUtils to create a meta-distribution that gathers all +the most useful Webware tools in one place and installs them in one +operation.

+
+
+

Risks

+
    +
  • incompatible license problems

  • +
  • overkill

  • +
+
+
+

Delivery Target

+

Webware 0.7

+
+
+
+

Notes

+

Contents

+
    +
  • Webware and all the core components

  • +
  • Cheetah

  • +
  • FunFormKit

  • +
  • Database modules like psycopg

  • +
  • a plotting module like Biggles (http://biggles.sf.net) for +quickly adding charts to websites

  • +
  • example sites

  • +
+

Implementation

+
    +
  • use a master DistUtils setup.py script to control the whole +installation process so that all the components can installed in one +fell swoop. "python setup.py install" will install Webware, while +"python setup.py install-batteries" will install the third-party +packages.

  • +
  • the end-user will control what will and won't be installed out of +all the included stuff using the setup.cfg

  • +
  • make Webware's setup.py compile and install mod_webkit or the iis +version (when complete). This could be controlled via the setup.cfg +file.

  • +
+
+

General Discussion

+

Add your thoughts and comments here, along with your sig -- +TavisRudd - 31 Oct 2001

+
+
+
+ + + diff --git a/bbs.html b/bbs.html new file mode 100644 index 0000000..1084212 --- /dev/null +++ b/bbs.html @@ -0,0 +1,21 @@ + + + + + +Bbs + + + +
+

Bbs

+ +

empty

+
+ + + diff --git a/browsingalternateportsinmozilla.html b/browsingalternateportsinmozilla.html new file mode 100644 index 0000000..a6ed3ef --- /dev/null +++ b/browsingalternateportsinmozilla.html @@ -0,0 +1,43 @@ + + + + + +Browsing Alternate Ports in Mozilla + + + +
+

Browsing Alternate Ports in Mozilla

+ +

(This not exactly a Webware issue, but you may experience this in the +course of using Webware.)

+

On a LAN machine, I set my HTTP server to run on port 79. I could +browse it with http://foo:79/ using the lynx, konqueror and opera +browsers, but mozilla and galeon gave me this mysterious message:

+
+

Access to the port number given has been disabled for security +reason.

+
+

I posted this to netscape.public.mozilla.general and found out that +this behavior is in response to the "cross-protocol scripting" +vulnerability described at http://www.kb.cert.org/vuls/id/476267.

+

I also learned that you can override the banned ports by putting a +preference in the all.js script of your mozilla site installation:

+
pref("network.security.ports.banned.override", "79");
+

My all.js was found at /usr/lib/mozilla/defaults/pref/all.js. I +located it with rpm -q mozilla -l | grep all.js.

+

Credit: Thanks to the folks at netscape.public.mozilla.general for +their help. For details, see the thread starting on 2001-12-26, +"Access to the port number given has been disabled for security +reasons".

+

-- ChuckEsterbrook - 27 Dec 2001

+

Which ports are banned? All below 1024 except 80?

+
+ + + diff --git a/buildingalist.html b/buildingalist.html new file mode 100644 index 0000000..e235560 --- /dev/null +++ b/buildingalist.html @@ -0,0 +1,148 @@ + + + + + +Building A List + + + +
+

Building A List

+ +

Some might argue that building an opt-in mailing list is not e-commerce, +because it is not directly involved in the "transactional" commerce. +However, anyone who has been marketing online for more than a month +knows that the gold is in the list -- you will continue to struggle +selling online until you build a list, or become Yahoo!

+

So how do you start, and what does it have to do with Webware?

+
+
First, you need a marketing plan, specifically:
+
    +
  1. who are you selling to

  2. +
  3. what do they want

  4. +
  5. what makes you different from your competitors

  6. +
+
+
+

Second, you will need an followup autoresponder service You can program the web part of +this is python/webware, but unless you want to become an expert in handling spam +complaints and staying off blacklists, you might want to leave the email broadcasting +to someone else. YMMV.

+
+
Third, you need an "ethical bribe". This is something that is
+
    +
  1. relevant and specific to your offer

  2. +
  3. valuable to the prospect

  4. +
  5. easy for you to provide (low cost in money and time)

  6. +
+
+
+

The plan is to offer the bribe as a motivation to get +the suspect (e.g. fist time visitor) to opt-in. Once +the suspect opts-in, he becomes a prospect, and you can +begin to follow up with him until he knows, likes, and +trusts you enough to buy when the time is right.

+

What does this have to do with Webware?

+

The ability to generate dynamic pages allows you to create +a website that captures the opt-in aggressively without +being annoying.

+
    +
  1. You can control when exit popups appear.

  2. +
  3. You can test multiple offers and easily +track where the visitor came from.

  4. +
  5. You can offer a repeat visitor something +he hasn't seen before.

  6. +
+
+

Using Popups Without Being Annoying

+

If someone visits your website, and leaves without opting in +it would be nice to know why. An exit popup survey is the +perfect tool.

+

Before you go hollering, let me say that I work primarily in +B2B markets, and my target market is not too terribly annoyed +by popups. In a B2C market, you'll need to test it.

+

What about popup blockers? Well, there is not a lot we can +do about them. But even if 90% of your popups get blocked, +that's 10% that got through, and 10% of your abandonment +traffic got a chance to tell you why they left. That's better +than nothing.

+

If your visitor is coming from a Google AdWords Ad, you need +to supress the popup. That's pretty easy with Webware.

+
+
+
if "google" not in referrer:
+

# include popup code

+
+
+
+

Ok, now the part about not being annoying.

+

Suppose someone visits your site, then leaves and gets the popup. +He answers the question, and clicks ok.

+

Later, he comes back, pokes about, then leaves again. Are you +going to show him the same popup? No.

+

What if he just closed the popup the first time he left? Are you +going to annoy him again by showing it again? Probably not.

+

How do you do this? Set a cookie (about 90 days) the first time +the popup code gets loaded. When he comes back, you know from +the cookie that he has already seen the popup, so you suppress it +just like you did for Google.

+

If you have flash audio clips that play automatically, you'll want +to use the same technique to say that they don't play automatically +when the page is revisited.

+
+
+

Testing Different Response Modifiers

+

In order to make your marketing messages most effective, you need +to continually measure, change, and measure again.

+

The most significant Response Modifier is "traffic source." It's +easy to set up different pages for different traffic sources if +you have control of the link -- you can just link to a different +page in the site. (But in terms of concentrating page rank, you +may not want to do this all of the time.) If you don't have control +of the link, you'll just need to trust the http-referrer header.

+

In any case, you'll want to track each traffic source, measure +how many "hits" vs. "actions" you get (that's your conversion rate) +and possibly create different offers for each.

+

Another important Response Modifier is the headline. How do you +test a headline?

+

When someone first visits your page, you randomly select one of +two (or more) headlines, and include that in the page. You also +set a cookie (about 30 days) so he sees the same headline when +he comes back or refreshes the page. When the visitor finally +does take action (opt-in or purchase), you record that as an +action for that particular headline. The first version to get +40 actions is the winner and becomes the new control.

+

Use the same pattern to test as many other response modifiers +as you want. (BTW: you should never stop testing.)

+
+
+

Offering Fresh Content

+

Caveat: This part is just a theory. I have not tested it, and I +don't know whether it works or not. After I do test it and get +significant results, I'll come back and update this paragraph.

+

Suppose you have a dozen different sites that are related. Each +site has a different ethical bribe, a different offer, and a +different autoresponder sequence. Because the sites are related, +someone who opts-in at one site is likely to be interested in +the other sites and offers.

+

Suppose someone visits site A, opts-in, and downloads the ebook. +The next time he visits site A, it may make sense to cross-promote +site B. You can do that because you know he has already opted-in +on A, so including the same offer as before is just wasting +screen real estate. And if he goes to site B after opting-in +on A and B, you can make offer C. Similarly if he keeps ignoring +offer B, stop showing it to him.

+

If you would like more information about testing and tracking, +or about building pages that compell the opt-in, you can join +my shy-yes list.

+

-- TerrelShumway 10 July 2004

+
+
+ + + diff --git a/canwebwarehandleit.html b/canwebwarehandleit.html new file mode 100644 index 0000000..63b5e93 --- /dev/null +++ b/canwebwarehandleit.html @@ -0,0 +1,73 @@ + + + + + +Can Webware Handle It? + + + +
+

Can Webware Handle It?

+ +

People often ask 'Is Webware ready for prime-time use?' or 'Can +Webware really handle it?' The answer is yes, but this is the wrong +question.

+

A better question would be 'can Webware help me build an application +that can reliably handle heavy loads?' Webware is not an application, +methodology, or recipe for success. It is a development framework and +a collection of useful tools. If you use the framework and tools +intelligently, and design your application well, then yes it will help +you build an application that can reliably handle heavy loads. If you +use them poorly, or botch your application design, don't cross your +fingers.

+

An even better question is 'does Webware's architecture encourage and +facilitate effective application designs?' I believe it does, for the +following reasons:

+ + +

Drop by the Webware Propaganda for some more info.

+

Keep in mind that Webware is only one part of your toolkit. +Non-trivial applications will rely on many other resources. These +might include server and network hardware; operating system(s); +persistence mechanisms, such as filesystems and databases; web +servers(s); web caches, such as squid; external credit card +processors; shipping processors; and, of course, web browsers. The +performance and reliability of your application will depend on how you +utilize these resources in conjunction with Webware.

+

Here's some notes on Using Webware Effectively.

+

-- TavisRudd - 30 Mar 2002

+

Have a look at some recent Webware benchmarks.

+

-- ChrisZwerschke - 18 Apr 2010

+
+ + + diff --git a/chadwalstrom.html b/chadwalstrom.html new file mode 100644 index 0000000..cbb5cc5 --- /dev/null +++ b/chadwalstrom.html @@ -0,0 +1,31 @@ + + + + + +Chad Walstrom + + + +
+

Chad Walstrom

+ +

Chad Walstrom is a long-time Linux and Python enthusiast. He is also a Debian +Developer, maintaining packages such as pnm2ppa, cheetah, clamsmtp, and +GNU GNATS (gnats).

+
+
Homepage
+

http://wookimus.net/~chewie

+
+
Email
+

chewie+w4py@wookimus.net

+
+
+
+ + + diff --git a/cheetah.html b/cheetah.html new file mode 100644 index 0000000..734c747 --- /dev/null +++ b/cheetah.html @@ -0,0 +1,62 @@ + + + + + +Cheetah + + + +
+

Cheetah

+ +

http://cheetahtemplate.org

+

Cheetah is a Python-powered template engine and code generator. It +can be used as a standalone utility or it can be combined with other +tools. Cheetah has many potential uses, but web developers looking +for a viable alternative to ASP, JSP, PHP and PSP are expected to be +its principle user group.

+

Cheetah:

+ +

Cheetah integrates tightly with Webware for Python +(http://webwareforpython.org): a Python-powered application server and +persistent servlet framework. Webware provides automatic session, +cookie, and user management and can be used with almost any +operating-system, web server, or database. Through Python, it works +with XML, SOAP, XML-RPC, CORBA, COM, DCOM, LDAP, IMAP, POP3, FTP, SSL, +etc.. Python supports structured exception handling, threading, object +serialization, unicode, string internationalization, advanced +cryptography, and more. It can also be extended with code and +libraries written in C, C++, Java and other languages.

+

Like Python, Cheetah and Webware are Open Source Software and are +supported by active user communities. Together, they are a powerful +and elegant framework for building dynamic web sites.

+

Like its namesake, Cheetah is fast, flexible and powerful.

+

-- TavisRudd - 29 Oct 2001

+
+ + + diff --git a/chriszwerschke.html b/chriszwerschke.html new file mode 100644 index 0000000..d6a9bfd --- /dev/null +++ b/chriszwerschke.html @@ -0,0 +1,22 @@ + + + + + +Christoph Zwerschke + + + +
+

Christoph Zwerschke

+ +

Is maintaining the project since 2007.

+

Please use our mailing lists for any questions, suggestions or contributions.

+
+ + + diff --git a/chuckesterbrook.html b/chuckesterbrook.html new file mode 100644 index 0000000..ca1e969 --- /dev/null +++ b/chuckesterbrook.html @@ -0,0 +1,29 @@ + + + + + +Chuck Esterbrook + + + +
+

Chuck Esterbrook

+ +

I founded the Webware project in the spring of 2000. For a long time, I managed the +project, wrote code, applied patches, made design choices, wrote +documentation, cut releases and answered questions on the discussion +list. Of course, other team members were doing similar things.

+

In 2007, I handed project management and leadership to the capable hands of Christoph Zwerschke so that I could focus more on my next project, The Cobra Programming Language.

+

I'm still the primary maintainer for MiddleKit. If you need help in that area, I'll probably respond to your webware-discuss message. If I don't, track me down.

+

My home page is at http://ChuckEsterbrook.com.

+

The Webware home page is at http://webware.sourceforge.net.

+

My latest project is The Cobra Programming language, found at http://cobra-language.com/

+
+ + + diff --git a/compatibilityinformation.html b/compatibilityinformation.html new file mode 100644 index 0000000..c3c31c6 --- /dev/null +++ b/compatibilityinformation.html @@ -0,0 +1,66 @@ + + + + + +Compatibility Information + + + +
+

Compatibility Information

+ +

This is the place to document all known compabilities and +incompatibilities between Webware, Python, and third-party +modules. The information here supplements official documentation, and +is meant to provide information on a more frequently updated basis +than the documentation revision cycle may allow.

+
+

Python Compatibilities

+ +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Python 1.5

Python 2.0

Python 2.1

Python 2.2

Webware 0.7

no

ok

ok

ok

Cheetah 0.9.12

ok

ok

ok

ok

FunFormKit 0.3.2

ok

ok

ok

ok

+

-- EdmundLian - 11 April 2002

+
+
+ + + diff --git a/component.html b/component.html new file mode 100644 index 0000000..5c73a3b --- /dev/null +++ b/component.html @@ -0,0 +1,160 @@ + + + + + +Component + + + +
+

Component

+ + +
+

Downloads and Websites

+ +

The repository is a Subversion repository. To check out a copy:

+
$ svn co http://svn.webwareforpython.org/Component/trunk Component
+
    +
  • FreeBSD ports

    +

    cd /usr/ports/www/py-webware-component && make install clean

    +
  • +
+
+
+

License and Prerequesites

+

Component is licensed under an MIT-style license. This gives you +permission to most anything you want with Component.

+

Component is a toolkit for Webware For Python. Webware is the only +prerequisite.

+
+
+

What Are They?

+

Many enhancements in Webware require subclassing WebKit.Page. But +each of these enhancements is incompatible with the others, because +you have to choose an specific inheritance hierarchy to use them. You +can't subclass SecurePage and SidebarPage and MVCPage all +at once.

+

One solution is Mixins, using multiple inheritance to add various +functions. This technique is available in Webware through +MiscUtils.MixIn. However, there are many places where Mixins are +mutually incompatible -- for instance, code often has to intercept the +awake() method, but delegating to each Mixin requires that they +all know about each other.

+

Components are essentially a way of creating Mixins that all know +about each other.

+
+

What's a Component Good For?

+

Components are useful if you have code that should know about the +request and response, and possible intercept it. If the code doesn't +need to know about the request at all, you should just create a normal +Python library, and call into it explicitly. Components can also be a +convenient way to add new convenience methods to your servlet.

+
+
+
+

Using CPage

+

In order to use components, all your servlets must inherit from +Component.CPage. This subclass of WebKit.Page adds some new +convenient methods, but mostly just adds component support.

+

New methods:

+
+
writeHeader():
+

Called automatically in writeBodyParts. If you are not using +a template, this can be used to write the HTML that goes at the +top of every page (the text written goes just after <body>).

+
+
writeFooter():
+

writeHeader compliment.

+
+
setView(view_name):
+

CPage adds the idea of views. The default view is +writeContent -- that's the innermost method that is called. +This can be used with actions so that the action method performs +some work, then changes the view to display a different body. +This way actions occur before content display, but can effect +later display without losing the normal Python templating +mechanism (writeHead, etc). Templating components (like +ZPTKit) may use this concept of view to indicate a template to +use.

+
+
view():
+

Returns the view, as a string.

+
+
sendResponseAndEnd(status, headers={}, body=None):
+

For error responses. This can be used like:

+
def awake(self, trans):
+    CPage.awake(self, trans)
+    try:
+        self.id = int(
+            self.request().extraURLPath().split('/')[1])
+    except (ValueError, IndexError):
+        self.sendResponseAndEnd(500)
+

A simple error body is created if you don't pass one in.

+
+
+

preAction and postAction are modified to do nothing (in +WebKit.Page they output a little HTML).

+
+
+

Using Components

+

CPage looks for any components you have in the .components +class variable (which should be a list). One very simple component is +included, NotifyComponent. As an example you can use this +component like:

+
from Component import CPage
+from Component.notify import NotifyComponent
+
+class SitePage(CPage):
+
+    components = [NotifyComponent()]
+
+    def writeHeader(self):
+        self.writeMessages()
+

If the component was configured, it would probably take arguments to +its constructor (this simple component has no configuration). A +component can add new methods to the servlet. In this case it adds a +.message(text) method, which adds your message to the session, and +a .writeMessages() method which writes the messages and clears them +from the session. (Also a .messageText() method if you don't want +to write the messages immediately.)

+

It's up to the component to add methods or attributes to the servlet. +It can also respond to "events" -- one example (and one of the only +current events) is the awake event, where it could intercept the +request. An example of this is in LoginKit, where the component +checks the servlet for permission settings, and may abort the +transaction with a login screen (or a Forbidden response) during +awake().

+
+
+

Writing Components

+

Read the source -- there are many docstrings in Component.py.

+
+
+ + + diff --git a/computeoncethencache.html b/computeoncethencache.html new file mode 100644 index 0000000..30f4747 --- /dev/null +++ b/computeoncethencache.html @@ -0,0 +1,33 @@ + + + + + +Compute Once Then Cache + + + +
+

Compute Once Then Cache

+ +

About this trick:

+
+
+
def compute():
+

# ...

+
+
+

data = compute()

+
+

I think I succeeded in applying this. +But, +Where/How can I check that the socalled 'data' variable is imported/computed only once ?

+

P-Y Delens

+
+ + + diff --git a/configschema.html b/configschema.html new file mode 100644 index 0000000..92b3703 --- /dev/null +++ b/configschema.html @@ -0,0 +1,162 @@ + + + + + +Config Schema + + + +
+

Config Schema

+ +
On Thu, 2002-04-25 at 07:47, Geoffrey Talvola wrote:
+> Already partially done.  Look at WebKit/Docs/Application.configlist and
+> WebKit/Docs/AppServer.configlist.  These files are used to auto-generate the
+> documentation sections of the user's guide using
+> WebKit/DocSupport/config.py.
+

OnceAndOnlyOnce suggests that the foo.configlist should come from +the same place as defaultConfig.

+
+

WEP: Configuration Schema

+
+

rough sketch of implementation

+
class ConfigurationOption:
+    def __init__(self,name,default,help):
+        ...
+
+class ConfigurationSchema:
+    def __init__(self,namespace,version,options):
+        ...
+
+    def default(self,key):
+        return self._optionMap[key].default
+
+    def defaults(self):
+        config = {}
+        for opt in self._options:
+            config[opt.name] = opt.default
+    ...
+
+class Configureable:
+    #configSchema() supercedes defaultConfig()
+    #config() uses the schema
+    #setting() can (optionally) warn if a key is requested that
+    #is not in the schema
+
+    def loadConfig(self,file): # factor out from userConfig
+        """
+        can (optionally) warn if there are keys present in the
+        file that are not in the schema (e.g. a setting gets
+        refactored or removed, user upgrades and cannot figure
+        out why his setting is now ignored.)
+        """
+
+
+

Usage

+
# Application.py
+
+class Application(...):
+
+    _configSchema = ConfigSchema( "Application", "0.7",
+        [
+         ConfigOption("PrintConfigAtStartup", 1, "does what it says"),
+         ConfigOption("AdminPassword","webware", "blah blah blah"),
+         ...])
+
+    def configSchema(self):
+        return self._configSchema
+
+
+

Benefits

+
    +
  • documentation and code are less likely to get out of sync

  • +
  • developers are more likely to provide documentation consistently

  • +
  • when schema changes, things are less likely to break without giving a clue as to what the problem is

  • +
  • config files can be upgraded automatically (if versioning is implemented)

  • +
  • an introspective tool can be written for configuring webware components/applications

  • +
  • if setting names or default values change, they change in one place.

  • +
+
+
+
+

Issues

+

future proofing:

+
+

Namespaces

+

In Application, there are a dozen settings that deal with error +handling, half of a dozen that deal with sessions, and a few that are +already dictionaries with only an informal internal structure.

+

It would be nice to be able to group these into their own schema with +a separate namespace.

+

"AdminPassword" is in Application.config. If the current Admin +context were to get replaced (or if the "ErrorEmail" stuff were to get +refactored), it would be nice to be able to break these out into +another config file.

+
+
+

Versioning

+

it would be nice to be able to automatically upgrade .config files, +either in an =upgrade.py= script, or as the file is read into +memmory. This could accommodate name changes, namespace changes,

+

an option could be deprecated by adding a deprecated attribute to the +ConfigOption.

+

an option could be marked as experimental in the same way.

+
+
+

Suggestion

+

a reserved "ConfigSchema" key/namespace to be included in all +configuration files:

+
# foo.config
+{
+    "ConfigSchema": {
+        "Namespace": "Foo",
+        "Version": "0.0.1",
+        "Owner": "FooKit",
+    }
+
+    ...
+
+}
+

this could also extend to "inline" namespaces such as "ErrorControl" +or "SessionControl": if a dictionary has a "ConfigSchema" key, it can +be versioned and handled by the config tool and treated as a +"namespace".

+

If a config file does not have a "ConfigSchema" key, it is treated as +a legacy config with a mild suggestion that the user update the file +to match current conventions.

+
+
+

How to handle namespaces?

+

e.g. current code does: +application.setting("Debug")["Sessions"]. New code might say +application.setting("ErrorControl")["EmailHost"] ... but this +would get out of control quickly. Maybe Configurable could be made +to handle "Debug.Sessions" or "ErrorControl.EmailHost"

+

Maybe Configurable should become a delegate instead of a +superclass, then a refactored ErrorManager, could look up +setting("EmailHost") which might delegate to +application.setting("EmailHost",namespace="ErrorControl") or +even:

+
import config
+host = config.setting("EmailHost",namespace=("Application","ErrorControl"))
+...
+

OK. Lets calm down and take it one step at a time.

+

---++ Blue Sky

+

if ConfigOption could be related somehow to optik.Option, +settings could be overridden on the startup command +line. (OnceAndOnlyOnce again)

+

A simple type system could eliminate the need to look at the name for +"Filename" or "Dir" in ConfigurableForServerSidePath. Use +option.type == TYPE_SERVER_PATH as the predicate.

+

-- TerrelShumway - 25 Apr 2002

+
+
+
+ + + diff --git a/copingwithtabs.html b/copingwithtabs.html new file mode 100644 index 0000000..e34cb21 --- /dev/null +++ b/copingwithtabs.html @@ -0,0 +1,61 @@ + + + + + +Coping with Tabs + + + +
+

Coping with Tabs

+ +

Since tabs (at least at the moment) are a reality in Webware, perhaps +we can collect some practical tips for coping with them. It probably +makes sense to collect these by editor.

+

-- JasonHildebrand - 27 Sep 2002

+
+

?? why can't we use spaces? I use spaces for my Python coding using +Vim, and I use WebKit. I don't seem to have any +trouble.. -JamesBecker

+
+

You can use spaces in your own Webware-based application -- no problem. +But if you need to modify Webware itself, you'll run into trouble if you +mix tabs and spaces.

+

-- JasonHildebrand - 15 Apr 2004

+
+

Vim

+

To teach Vim to switch to "noexpandtabs" mode when editing a file in +any subdirectory of "Webware", add these lines to your ~/.vimrc

+
autocmd BufEnter */Webware/* set noexpandtab
+autocmd BufLeave */Webware/* set expandtab
+
+
+

sed

+

Converting python files from tabs to spaces is a snap.

+
> cd ~/dev/Webware
+> for file in `find . -name "*.py"`
+do
+    sed -e "s/^V<tab>/    /g" $file > $file.new     # hit ctrl-v and the tab key after the first slash
+    mv $file.new $file
+done
+

--VictorNg April 15, 2004

+
+
+

(un)expand

+

The GNU Coreutils contains the utilties expand and unexpand convert tabs to spaces and vice-versa. A tab is considered to align at specific columns, by default every eight columns or four columns with option -t4, so the specific number of spaces that correspond to a tab depends on where those spaces or tab occur. Unless you specify the -a option, unexpand will only entab the initial whitespace (which goes along with the Webware styleguides).

+

--ChrisZwerschke July 26, 2005

+
+
+

DrPython

+

The Editor/Mini-IDE "DrPython" supports converting from tabs to spaces and vice versa via the menu entries "Edit - Whitespace - Set indentation to tabs..." and "Edit - Whitespace - Set indentation to spaces..."

+

--ChrisZwerschke July 26, 2005

+
+
+ + + diff --git a/creditcards.html b/creditcards.html new file mode 100644 index 0000000..0ed54a4 --- /dev/null +++ b/creditcards.html @@ -0,0 +1,120 @@ + + + + + +Credit Cards + + + +
+

Credit Cards

+ +

This page is about accepting, storing and processing credit cards.

+
+

A brief, high level overview of accepting payments on-line, with a focus on credit cards is at http://selfpromotion.com/credit.t

+

-- ChuckEsterbrook - 18 Mar 2002

+
+

This information comes courtesy of Jeff Johnson:

+
+

How to Store Credit Card Numbers

+

For starters, you'll likely store them in a SQL database. You'll want +to encrypt them for safekeeping, but might also choose to store the +last 4 digits as plain text so that customer service reps can tell a +customer which card they used (without seeing the whole number).

+

You can encrypt them using either M2Crypto or the Python rotor +library.

+
+

The rotor module implements an encryption algorithm used in the second +world war. Beware that data encrypted with this algorithm are easily +cracked!

+

--AlbertBrandl

+

Here's an interesting link about credit card handling, including the +pitfalls of not retaining card information: +http://www.arsdigita.com/books/panda/ecommerce#credit_cards

+

-- PaulBoddie - 06 Mar 2002

+
+

Here's what Jeff Johnson said:

+

I store them encrypted using the rotor library, I might use M2Crypto if I +were to do it again but so far rotor is fine. I also store the last 4 +digits as plain text so CSRs can tell the customer which card they used +without seeing the whole number. Then, I store the hash so we can search +the database and compare numbers without passing the actual number. Here's +my code, altered a little for security reasons:

+
def decryptCreditCard(self,number):
+ r = rotor.newrotor(???,???)
+ number = binascii.a2b_hex(number)
+ number = r.decrypt(number)
+ number = re.sub(r"\D","",number)
+ return number
+
+def encryptCreditCard(self,number):
+ number = re.sub(r"\D","",number)
+ r = rotor.newrotor(???,???)
+ number = r.encrypt(number)
+ number = binascii.b2a_hex(number)
+ return number
+
+def creditCardShaHex(self,number):
+ number = re.sub(r"\D","",number)
+ return binascii.b2a_hex(sha.new(number).digest())
+
+def creditCardFromNumber(self,number):
+ number = re.sub(r"\D","",number)
+ numberShaHex = self.creditCardShaHex(number)
+ sql = " select creditCardID,number,expiration,numberShaHex,numberPart from CreditCard where numberShaHex = '%s' " % numberShaHex
+ rs = self.recordset("DEG",sql)
+ if rs:
+       return rs[0]
+ else:
+       return None
+
+def creditCardFromID(self,creditCardID):
+ sql = " select creditCardID,number,expiration,numberShaHex,numberPart from CreditCard where creditCardID = %s " % creditCardID
+ rs = self.recordset("DEG",sql)
+ if rs:
+       return rs[0]
+ else:
+       return None
+
+def insertCreditCard(self,number,expiration):
+ # Get rid of non-digits (\D).
+ number = re.sub(r"\D","",number)
+ expiration = re.sub(r"\D","",expiration)
+
+ # Encrypt the number before storing it.
+ numberRotor = self.encryptCreditCard(number,expiration)
+ # To find a credit card in the database, get the sha hex value, then query the database.
+ numberShaHex = self.creditCardShaHex(number)
+ # If a customer wants to know which card they used, read the last 4 digits back to them.
+ numberPart = number[-4:]
+
+ sql = """ select nextval('CreditCardSeq') as creditCardID """
+ rs = self.recordset("DEG",sql)
+ creditCardID = rs[0].creditCardID
+
+ sql = """
+       insert into CreditCard
+       (creditCardID,number,expiration,numberShaHex,numberPart)
+       values
+       (%(creditCardID)d,'%(numberRotor)s','%(expiration)s','%(numberShaHex)s','%(numberPart)s')
+       """ % locals()
+ rs = self.recordset("DEG",sql)
+ return creditCardID
+
+
+

SSL

+

You'll need SSL for encryption. You can run Apache 1.3.x + mod_ssl and +get SSL certs from a variety of +companies. http://google.com/search?q=buy+ssl+certificates

+

You can use stunnel to talk SSL to the credit card company.

+

-- ChuckEsterbrook - 04 Mar 2002

+
+
+ + + diff --git a/databaseintegration.html b/databaseintegration.html new file mode 100644 index 0000000..ee2dbf7 --- /dev/null +++ b/databaseintegration.html @@ -0,0 +1,101 @@ + + + + + +Database Integration + + + +
+

Database Integration

+ +

Eventually, you are going to need persistent objects, or just some way +to store and retrieve data. The approach you use depends on many +factors, such as:

+ +
+

Possible Solutions

+

The following is a list of possible solutions:

+
    +
  • Use an object-oriented database or object store like ZODB

  • +
  • Use an RDBMS and the Webware MiddleKit object-relational mapper

  • +
  • Use an RDBMS and some other object-relational mapper. Possibilities include: +* PyDO +* Database Objects (dbObj) +* SQLObject +* Modeling

  • +
  • Forget about object-relational mapping--use the Python DB API and an RDBMS directly

  • +
  • Use DBUtils on top of the Python DB API

  • +
  • Use Python's pickle and shelve modules

  • +
  • Roll your own system with a flat file

  • +
+
+
+

Pros and Cons of Each Solution

+ +
+
+

Comments about various RDBMS and OODBMS Systems

+
    +
  • Informix (proprietary)

  • +
  • InterBase (open source)

  • +
  • MySQL (open source)

  • +
  • Oracle (proprietary)

  • +
  • PostgreSQL (open source)

  • +
  • SAP DB (now open source, based on ADABAS code)

  • +
  • SQL Server (proprietary)

  • +
  • Sybase (proprietary)

  • +
+
+
+

More about RDBMS and OODBMS Systems

+
    +
  • Database Debunkings is a great place to look for intelligent +discussion about database issues by people like C.J. Date and Fabian +Pascal.

  • +
  • searchDatabase.com seems to be another great site to start +exploring the literature on databases.

  • +
  • When will they ever learn is a critique (by Fabian Pascal) of an +exchange on whether MySQL and Innobase are DBMSs, or even +relational. C.J. Date makes an appearance, and you get to hear the +horses speak directly.

  • +
  • Linux RDBMS Library Compilation of free readings on Linux +relational databases.

  • +
  • What exactly is a relational database? C.J. Date explains in A +Closer Look at Relational Database. This site requires free +registration.

  • +
  • Fabian Pascal explains why SQL sucks and why all SQL DBMS violate +important relational features, some more than others in Little +Relationship to Relational.

  • +
  • Think OODBMS are the bee's-knees? Better read this first.

  • +
+

-- EdmundLian - 30 Dec 2001

+
+
+ + + diff --git a/dbutils.html b/dbutils.html new file mode 100644 index 0000000..2644395 --- /dev/null +++ b/dbutils.html @@ -0,0 +1,29 @@ + + + + + +DBUtils + + + +
+

DBUtils

+ +

DBUtils is a suite of tools providing solid, persistent and pooled connections +to a database that can be used in all kinds of multi-threaded environments +like Webware for Python or other web application servers. The suite supports +DB-API 2 compliant database interfaces and the classic PyGreSQL interface.

+

The current version is available for download here and at

+ +

The documentation is also contained in the download package.

+
+ + + diff --git a/developerguidelines.html b/developerguidelines.html new file mode 100644 index 0000000..7944f14 --- /dev/null +++ b/developerguidelines.html @@ -0,0 +1,69 @@ + + + + + +Developer Guidelines + + + +
+

Developer Guidelines

+ +

Here are some guidelines for Webware Developers:

+
+

Design

+

If your change is non-backward-compatible or involves design +considerations, please discuss it on the webware-devel list before +checking it in (and preferably before implementing it!). Your ideas +are probably good, but someone else on the list may come up with +better ones!

+

Following Python's development model, Webware also has a BDFL. Chuck +Esterbrook is Webware's BDFL. As such, he has final say over Webware +design issues and if he makes a pronouncement, please obey it.

+

Jay and Geoff can also make pronouncements if Chuck isn't available or +if they are better qualified to decide on whatever issue is at hand. +But pronouncements from Jay or Geoff are overruled by a pronouncement +from the BDFL.

+
+
+

Coding

+

Follow the Webware Style Guidelines. +http://www.webwareforpython.org/Docs/StyleGuidelines.html

+
+
+

Testing

+

Please test before checking in changes. If the component you are +modifying has a test suite, please add a test case to the test suite +if appropriate.

+
+
+

Documentation

+

Please update the Release Notes. If a new feature has been added, +please update the User's Guide. If a new setting has been added, +please update the Configuration Guide.

+
+
+

Becoming a Webware Developer

+

If you agree to abide by the rules listed above and you want SVN write +access, send an email to Chuck Esterbrook and Geoff Talvola. You +should let them know what it is you plan to do with SVN -- fix bugs, +improve the docs, add new features, write a regression test suite, or +whatever. Also, if you can show evidence of one or more of these:

+
    +
  • Previous history of writing quality Webware patches

  • +
  • Previous history of writing quality 3rd-party extensions to Webware

  • +
  • Previous history of writing quality open-source Python code

  • +
  • Or any other evidence that you'll be a good addition to the Webware team

  • +
+

Then Chuck or Geoff will probably give you write access.

+

-- GeoffTalvola - 06 Jun 2002

+
+
+ + + diff --git a/directorystructure.html b/directorystructure.html new file mode 100644 index 0000000..b988653 --- /dev/null +++ b/directorystructure.html @@ -0,0 +1,55 @@ + + + + + +Directory Structure + + + +
+

Directory Structure

+ +

There are two aspects to DirectoryStructure: client-side (URI) and +server-side (file-system, database, etc.).

+

This article by Tim Berners-Lee +http://www.w3.org/Provider/Style/URI.html is a great start if you're +trying to decide on a URI directory structure for your site (thanks to +IanBicking and KendallClark for posting about it)!

+

-- TavisRudd - 16 Nov 2001

+
At 10:39 AM 10/27/2001 +0200, Georg Balmer wrote:
+>I have 2 basic questions
+>
+>--[1]--------------------------------------------
+>
+>How do you layout the directory structure ?
+[snip]
+>or case B:
+>
+>someDirectory/Webware/WebKit/
+>                   ...
+>
+>someOtherDirectory/appOne/pages/
+>someOtherDirectory/appOne/lib/
+

Case B. Mingling your projects with Webware is a bad idea. It makes it +harder to upgrade Webware and confuses people into thinking they have +to modify Webware to make Webware apps.

+

One notable exception is when you bake your own plug-ins. Those can be +convenient to drop into Webware/ because then they get picked up +automatically. However, there are 2 settings related to this in +WebKit/Configs/AppServer.config which allows you to put your +plug-ins outside of Webware.

+

Just for clarification: If you are building a Webware site, you are +probably NOT creating a plug-in.

+

---

+

I recommend using MakeAppWorkDir.py to create a working directory +for your application separate from your Webware directory.

+

-- GeoffTalvola - 29 Oct 2001

+
+ + + diff --git a/distutils.html b/distutils.html new file mode 100644 index 0000000..e7ebc10 --- /dev/null +++ b/distutils.html @@ -0,0 +1,40 @@ + + + + + +distutils + + + +
+

distutils

+ +

The term "distutils" refers to the "Python Distribution Utilities". +Since their introduction in 2000 they have become the standard way to +package, distribute and install Python modules and applications.

+

See http://python.sourceforge.net/devel-docs/dist/dist.html for more +information.

+

The current version of Webware does not use distutils. The transition +to a distutils architecture will hopefully be made by Webware 0.8.

+
+
There are numerous advantages to switching to distutils:
+
    +
  • users don't need to maintain a local copy of all the Webware modules.

  • +
  • the core modules can be used easily by other frameworks

  • +
  • it can be packaged for Debian and other OS distros.

  • +
  • installs on Windows now make use of the GUI installer.

  • +
  • distutils offers many advanced packaging facilities!

  • +
  • distutils is the Pythonic way to do installs.

  • +
+
+
+

-- TavisRudd - 29 Oct 2001

+
+ + + diff --git a/documentmode.html b/documentmode.html new file mode 100644 index 0000000..319aee9 --- /dev/null +++ b/documentmode.html @@ -0,0 +1,27 @@ + + + + + +Document-mode + + + +
+

Document-mode

+ +

A style of Wiki writing that presents a synopsis, or synthesis, of a +discussion. The synopsis is owned by all of the participants of that +discussion, although it might be moderated. See +http://www.usemod.com/cgi-bin/mb.pl?DocumentMode for a fuller +explanation. It is complemented by ThreadMode, which is better for +freeform discussions.

+

-- TavisRudd - 23 Nov 2001

+
+ + + diff --git a/downloads/Component-0.1.tar.gz b/downloads/Component-0.1.tar.gz new file mode 100644 index 0000000..88b37d8 Binary files /dev/null and b/downloads/Component-0.1.tar.gz differ diff --git a/downloads/Component-0.2.tar.gz b/downloads/Component-0.2.tar.gz new file mode 100644 index 0000000..5466ba7 Binary files /dev/null and b/downloads/Component-0.2.tar.gz differ diff --git a/downloads/DBUtils-1.0.tar.gz b/downloads/DBUtils-1.0.tar.gz new file mode 100644 index 0000000..010dfb7 Binary files /dev/null and b/downloads/DBUtils-1.0.tar.gz differ diff --git a/downloads/DBUtils-1.1.tar.gz b/downloads/DBUtils-1.1.tar.gz new file mode 100644 index 0000000..c31bd9d Binary files /dev/null and b/downloads/DBUtils-1.1.tar.gz differ diff --git a/downloads/LoginKit-0.1.tar.gz b/downloads/LoginKit-0.1.tar.gz new file mode 100644 index 0000000..2b23eb7 Binary files /dev/null and b/downloads/LoginKit-0.1.tar.gz differ diff --git a/downloads/LoginKit-0.2.tar.gz b/downloads/LoginKit-0.2.tar.gz new file mode 100644 index 0000000..83cd85d Binary files /dev/null and b/downloads/LoginKit-0.2.tar.gz differ diff --git a/downloads/Webware-1.0.3.tar.gz b/downloads/Webware-1.0.3.tar.gz new file mode 100644 index 0000000..4a32db9 Binary files /dev/null and b/downloads/Webware-1.0.3.tar.gz differ diff --git a/downloads/Webware-1.1.1.tar.gz b/downloads/Webware-1.1.1.tar.gz new file mode 100644 index 0000000..cf77b21 Binary files /dev/null and b/downloads/Webware-1.1.1.tar.gz differ diff --git a/downloads/WebwareWiki-0.1.tar.gz b/downloads/WebwareWiki-0.1.tar.gz new file mode 100644 index 0000000..b53070b Binary files /dev/null and b/downloads/WebwareWiki-0.1.tar.gz differ diff --git a/downloads/WebwareWiki-0.2.tar.gz b/downloads/WebwareWiki-0.2.tar.gz new file mode 100644 index 0000000..31bdaf4 Binary files /dev/null and b/downloads/WebwareWiki-0.2.tar.gz differ diff --git a/downloads/ZPTKit-0.4.1.tar.gz b/downloads/ZPTKit-0.4.1.tar.gz new file mode 100644 index 0000000..7d132f7 Binary files /dev/null and b/downloads/ZPTKit-0.4.1.tar.gz differ diff --git a/downloads/index.html b/downloads/index.html new file mode 100644 index 0000000..340a156 --- /dev/null +++ b/downloads/index.html @@ -0,0 +1,44 @@ + + + + +Downloads + + + +
+

Downloads

+ + + + + + + diff --git a/dynamicpricing.html b/dynamicpricing.html new file mode 100644 index 0000000..fea732b --- /dev/null +++ b/dynamicpricing.html @@ -0,0 +1,163 @@ + + + + + +Dynamic Pricing + + + +
+

Dynamic Pricing

+ +
+

Introduction

+

Price points are extremely important in business as their selection +can make a huge difference in gross revenue.

+

To illustrate the importance of price point, consider a product which +might be priced between $5 and $20. The number of customers that +purchase at these different prices will change for different reasons:

+ ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

$/month

% that buy

score

comment

$5

5%

25

$10

7%

70

higher price sometimes yields higher +perceived value by customer

$15

6%

90

higher price eventually drives +customers away

$20

3%

60

higher price eventually causes many +customers to balk

+

So $15 is head and shoulders above the rest: more that 29% higher +grossing than the second most profitable price point.

+

Selecting a price point is done by almost all major businesses that +have goods and services to sell. Often a combination of surveys, +consultants, spreadsheets and software are employed by businesses +large enough to afford all that. Price testing is also conducted.

+

It's notable that Amazon's initial random price testing caused an +uproar among customers. Some reports state that Amazon refined their +approach by giving customers the lowest price available at the end of +the test and others state that Amazon ceased testing +indefinitely. See: +http://www.google.com/search?hl=en&q=amazon+dynamic+pricing

+
+
+

Proposed Approach

+

A dynamic web site offers a unique environment to determing price +points dynamically, e.g., to adjust prices automatically as they need +adjusting.

+

One approach could be to set a base price and after a certain number +of potential sales, adjust it. If the last batch scored less than its +predecessor, the price drops. Otherwise, the price increases.

+

A period is defined in terms of the number of visitors that actually +get to the pricing information for the opportunity to purchase. We'll +call those visitors the potential customers. The purchase rate is +the # of customers divided by the # of potential customers. The +purchase rate times the price gives the score for that period.

+

Here are some fragmented thoughts expressed in Python:

+
bp = 4.95   # base price
+inc = 1.00  # increment amount
+dec = 1.00  # decrement amount
+period = 100        # number of visitors before analyzing
+
+periods = []        # list of periods
+
+class Period:
+    """
+    attributes:
+        date  - the date the period started
+        score - the important number. how well this period did
+        rate  - rate of customers who saw pricing information and purchased
+        parameters - dictionary of parameters at the time such as bp, inc, dec, etc.
+    """
+    pass
+
+if len(periods)>1:
+    prev = period[-1]
+    beforePrev = period[-2]
+    if prev.score>beforePrev.score:
+        newPrice = price + inc
+    elif prev.score<beforePrev.score:
+        newPrice = price - dec
+

By giving all potential customers the same price for a given period of +time, customers are treated fairly (contrasted with the original +Amazon approach). The period price adjustments are no different than +manual adjustments made by other companies for their products. What's +special is that the adjustments are automatic and precise.

+

What's also special is that the price adjustments are made whenever +needed. The underlying factors could be all kinds of things:

+
    +
  • change in product quality

  • +
  • change in economy

  • +
  • appearance or disappearance of a competitor

  • +
  • presentation of product

  • +
  • and much, much more

  • +
+

But the computation doesn't need to know the factors and will +therefore, never miss any, because the score is the bottom line.

+

Additional ideas to refine this approach are:

+
    +
  • Have a minimum number of days for the current price point, at least 1. This would avoid customers experiencing changing prices in a single day, which could make them feel uncomfortable.

  • +
  • Likewise, have a maximum number of days in order to put a limit to the timewise duration of a pricing period.

  • +
  • Refrain from price adjustments if the score difference was less than some minimum percentage such as 1%.

  • +
  • Adjust prices dynamically according to the difference in scores. Here's a thought:

    +
    factor = 0.5
    +scoreRelDiff = (period[-1].score - period[-2].score) / period[-2].score
    +price += price*scoreRelDiff*factor
    +
  • +
  • Another approach could use multiple moving averages. For example, if the average score of the past 5 days is higher than that of the past 10 days, increase the price. This "moving average" approach is used by some stock market investors.

  • +
  • Some sites would need to be cautious of the cost of goods sold (COGS) in order to avoid selling product at a loss.

  • +
+
+
+

Summary

+

The advantages of Dynamic Pricing are:

+
    +
  • maximizing profits

  • +
  • adjusting price whenever needed

  • +
  • automating price adjustment

  • +
  • treating customers in a given period equally

  • +
+

There's a huge number of approaches that could be taken and lots of +different topics to be discussed. Feel free to add to this Wiki page.

+

-- ChuckEsterbrook - 17 Mar 2002

+

---

+

This is brilliant -- thanks Chuck! JamesBecker

+
+
+ + + diff --git a/edmundlian.html b/edmundlian.html new file mode 100644 index 0000000..075e872 --- /dev/null +++ b/edmundlian.html @@ -0,0 +1,28 @@ + + + + + +Edmund Lian + + + +
+

Edmund Lian

+ +

I began experimenting with Webware after declaring myself a refugee of +the complexity and inflexibility in +"a-well-known-Python-application-server-that-will-remain-anonymous". You +know which one...

+

I am an independent consultant (specializing in complex +marketing/strategy/technology interaction issues) who also dabbles in +software development.

+

-- EdmundLian - 31 Oct 2001

+
+ + + diff --git a/environmentvariables.html b/environmentvariables.html new file mode 100644 index 0000000..885aaf6 --- /dev/null +++ b/environmentvariables.html @@ -0,0 +1,62 @@ + + + + + +Environment Variables + + + +
+

Environment Variables

+ +

Many Python scripts start with the following line:

+
#!/usr/bin/env python
+

On UNIX-like systems, this instructs the operating system to find the python executable according to the environment. To determine which version of Python gets run, just make sure that your PATH environment variable is set up correctly.

+

If Python 2.0 is installed as /usr/local/bin/python and Python 1.5.2 is installed as /usr/bin/python, and you want to run Python 2.0, then you should make sure that /usr/local/bin appears on your "path" before /usr/bin. How and when this is done depends a lot on your system and how you want to work.

+
+

Setting the Default Environment

+

Let's say you want to always use Python 2.0 as a particular user. This +means that you need to edit your shell's resource file so that when it +is started, it puts the directory in which Python 2.0 is found before +the directory in which Python 1.5.2 is found.

+

Check your system documentation for details of which file you should +edit and how you should edit it. Here's an example, though - for +bash (the Bourne Again Shell), edit the .bashrc file in your +home directory and add this at the end of the file:

+
export PATH=/usr/local/bin:${PATH}
+

This assumes that you did install Python 2.0 in /usr/local when +considering MultiplePythonVersions.

+
+
+

Setting the Environment for AppServer

+

If you just want to run AppServer with a changed environment so that +only that program uses Python 2.0, and you are running bash, you +can try this:

+
PATH=/usr/local/bin:${PATH} ./AppServer
+

Why not put this in a script:

+
#!/bin/sh
+PATH=/usr/local/bin:${PATH} ./AppServer
+
+
+

What You Should Not Do

+

Don't be tempted to go round changing the first line of all Python +scripts to something like this:

+
#!/usr/bin/env python20
+

If you do have Python 2.0 installed as something other than python +then consider making a symbolic link from another directory to the +executable, like this:

+
cd /usr/local/bin
+ln -s /usr/bin/python20 python
+

Now add that directory (in this case, /usr/local/bin) to the start +of your "path", like we did above.

+

-- PaulBoddie - 31 Oct 2001

+
+
+ + + diff --git a/ericradman.html b/ericradman.html new file mode 100644 index 0000000..fd9f8e6 --- /dev/null +++ b/ericradman.html @@ -0,0 +1,24 @@ + + + + + +Eric Radman + + + +
+

Eric Radman

+ +

I work as the System Administrator for TEI Internet Serices +(http://www.teisprint.net). I'm a big fan of open source software, and keep a +journal of my experiences at http://eradman.com.

+

theman@eradman.com

+
+ + + diff --git a/extrapathinfo.html b/extrapathinfo.html new file mode 100644 index 0000000..522a34e --- /dev/null +++ b/extrapathinfo.html @@ -0,0 +1,34 @@ + + + + + +ExtraPathInfo + + + +
+

ExtraPathInfo

+ +

If Webware's ExtraPathInfo setting is true and you type a URL with a +nonexistent directory component (e.g., +http://site.com/dir/subdir/NONEXISTENT), WebKit returns the +parent directory's index.py instead of a 404 Not Found error, because +it thinks the invalid directory is actually extra path info. But the +servlet, if it's not designed to use the extra path info, will just +display its normal thing, which is wrong because it's the wrong +servlet. Worse, any relative links on the page will loop back to that +servlet, because they'll have the invalid pathname component(s) in the +middle of the URL.

+

The consensus among Webware developers is, this is not a bug, it's a +feature. It's just something to be aware of if you use +ExtraPathInfo.

+

-- MikeOrr - 21 Feb 2002

+
+ + + diff --git a/featuresafterv09.html b/featuresafterv09.html new file mode 100644 index 0000000..26f57d0 --- /dev/null +++ b/featuresafterv09.html @@ -0,0 +1,52 @@ + + + + + +Features after v. 0.9 + + + +
+

Features after v. 0.9

+ +
+
From a thread on the mailing lists:
+

I'd like to do some cleaning up of Webware after our v0.9 build. The automated tests are good preparation to that cleanup. When there are a good set of test cases in place, I'd like to make Webware easier to install. I'm also improving my Cheetah ServletFactory which I can add as a kit. And I have a very simple script that functions like Ant or make--it's a place to put short scripts when developing such as "generate MiddleKit sql", "run sql on my database", "clean up temporary files", "run python interactively with the same python path as the appserver", "upload my web application to the production server." "run webware, but using the CVS version instead of v0.81"

+

What improvements would other people like? I will compile a list and post it on the wiki. I'll also sort it in priority order as judged by the enthusiastic responses on this list.

+
+
+
+

Proposals

+
+
    +
  • 29 Dec 04 - Ian Bicking - While we're talking about improvements, I might re-note the existance of WSGIKit.

  • +
  • 04 Jan 05 - Tom Schwaller - Are Aspects (simple solution in CherryPy) something people would like to see integrated in Webware? Or are continuations? Maybe using generators and annotations could make Webware much more sophisticated?

  • +
  • +
    19 Feb 05 - Lloyd
    +
      +
    • recently i was looking over the (abandoned?) "webware experimental refactoring" project, and it had some interesting improvements that, to me, seemed like a natural progression for webware:

    • +
    • contexts become "applications" - get their own sessions and state;

    • +
    • multiple applications/contexts can run under a single server instance;

    • +
    • formalized method for executing (context/)application-specific code once on startup, with access to various webware components. (what's the recommended way to do this now? stick it in a site page at module level?)

    • +
    • out-of-the-box connection pooling

    • +
    • get webware into debian's repository? :-)

    • +
    +
    +
    +
  • +
+
+
+
+

Issues

+

WSGIKit - Should it be developed in parallel, or should the old Webware be dropped? There are several votes for keeping them in parallel.

+
+
+ + + diff --git a/fileextensions.html b/fileextensions.html new file mode 100644 index 0000000..29005bf --- /dev/null +++ b/fileextensions.html @@ -0,0 +1,50 @@ + + + + + +File Extensions + + + +
+

File Extensions

+ +
+

Problem

+

I created a pure HTML Cheetah test.tmpl file, and compiled it to +test.py. When I tried to visit the url "~MyContext/test", I got a 404 +error. But if I went to "~MyContext/test.py", the page is returned +correctly. Why is this happening?

+
+
+

Solution

+

IanBicking wrote:

+

Maybe because there's an ambiguity of what file you are calling -- +test.tmpl or test.py. When you give the extension it is unambiguous.

+

I thought Webware would ignore the unregistered extension, though.

+

ChuckEsterbrook wrote:

+

Actually, it's the other way around. You have to specify what +extensions to ignore. An old TO DO item is to switch WebKit to a +setting called ExtensionsToServe.

+

But in the mean time, go to WebKit/Configs/Application.config and add +.tmpl to the ExtensionsToIgnore setting.

+

I (Edmund) add:

+

It's not enough to do this. You must also add things like .tmpl~ +(backup files created by emacs, in my case) to the +ExtensionsToIgnore setting, or you will still get 404 errors.

+

-- EdmundLian - 02 Dec 2001

+

I noticed also that Cheetah leaves _bak files around, too, so I needed +to add .py_bak to the list of extensions to ignore. This may be a +problem if you regenerate an HTML file with Cheetah, too (meaning, +.html_bak would need to be ignored, too).

+

-- DavidHancock - 07 Jul 2002

+
+
+ + + diff --git a/filestreamingandcontentdisposition.html b/filestreamingandcontentdisposition.html new file mode 100644 index 0000000..4180359 --- /dev/null +++ b/filestreamingandcontentdisposition.html @@ -0,0 +1,96 @@ + + + + + +File streaming and Content-Disposition Header + + + +
+

File streaming and Content-Disposition Header

+ +

This is a followup from the Webware-discuss list (see Has anyone +tried to stream a reportlab)

+

The Content Disposition Header is defined in RFC 2183. Its purpose +(among others) is to give a hint about the name of the following +content. This is quite useful, when one tries to pipe a file to the +browser. Without this header, the browser will show the scriptname, +instead of the filename, when trying to use the Save as function.

+

The following code should run right away as a servlet. It shows the +content of some directory and lets you download its files. It is +tested with Webware (10 day old CVS), OneShot.cgi Adapter and the +following browsers. The following list gives you an idea, what +browsers will show the right filename (or not):

+
    +
  • Opera on Linux (works)

  • +
  • Konqueror (doesn't work)

  • +
  • Mozilla 0.96 on Linux (doesn't work, though it should according to +documentation) IE5 should work.

  • +
+

---

+
from WebKit.Page import Page
+import os, mimetypes
+
+FILEDIR = "/path/to/directory"
+
+class Main(Page):
+   def title(self):
+        return 'Webware Download Test Page'
+   def writeBody(self):
+        response = self.response()
+        request = self.request()
+        adapterName = request.adapterName()
+        if request.hasField('file'):
+            filename = '%s/%s' % (FILEDIR,request.field('file'))
+            (mtype,enctype) = mimetypes.guess_type(filename)
+            fd = open(filename)
+
+            response.setHeader('Content-Type',mtype)
+            response.setHeader('Content-Disposition','attachment; filename="%s"' % request.field('file'))
+            response.flush()
+
+            chunksize = response.streamOut().bufferSize()
+            outchunk = fd.read(chunksize)
+            while outchunk:
+                self.write(outchunk)
+                outchunk = fd.read(chunksize)
+
+        else:
+            files = os.listdir(FILEDIR)
+            self.writeln('<h1>Webware Download Test</h1>')
+            self.writeln('<ul>')
+            for file in files:
+                self.writeln('<li><a href="%s/Test/Main.py?file=%s">%s</a>' % (adapterName,file,file))
+                self.writeln('</ul>')
+

---

+

I'm not entirely sure if this code is really streaming the file or if +the Webware outStream is buffering. I think that "response.flush()" +will disable buffering, but I'm not sure.

+

Please comment and feel free to put more elegant versions in +place. Maybe we should start a UsefullRfcs page.

+

-- StephanDiehl - 14 Nov 2001

+

As far as the buffering goes, by calling response.flush(), you tell +webware to go ahead and send what you have written to the stream so +far. By default, the outStream waits until you are done writting to +send anything, calling flush allows you to send data as it becomes +available.

+

-- JayLove - 27 Dec 2001

+

It seems to me that the above code is not correct because the streamed +content consists not only of the contents of the intended file but also +the html headers, which are not desired. To solve this I override +writeHTML like this:

+
def writeHTML(self):
+    if self.request().hasField('file'):
+        self.writeBody()
+    else:
+        Page.writeHTML(self)
+

-- RubenMendes - 4 Oct 2004

+
+ + + diff --git a/formauthentication.html b/formauthentication.html new file mode 100644 index 0000000..d084130 --- /dev/null +++ b/formauthentication.html @@ -0,0 +1,22 @@ + + + + + +Form Authentication + + + +
+

Form Authentication

+ +

Please see Model Two plus One for an example of FormAuthentication.

+

-- LukeHolden - 07 Jan 2003

+
+ + + diff --git a/formexample1.py b/formexample1.py new file mode 100644 index 0000000..85b9919 --- /dev/null +++ b/formexample1.py @@ -0,0 +1,70 @@ +"""Demo for using FormEncode with Webware.""" + +# Tested with Webware 1.1 and FormEncode 1.2.2 + +from formencode import compound, Invalid, htmlfill, Schema, validators +from WebKit.Examples.ExamplePage import ExamplePage + + +form_template = '''

Tell me about yourself

+ +
+ +


+ +

+ +


+ +

+ +

Your favorite color:
+ + Red   + Blue   + Black   + Green

+ +

+
''' + + +class FormSchema(Schema): + name = validators.String(not_empty=True) + age = validators.Int(min=13, max=99) + color = compound.All(validators.Set(), + validators.OneOf(['red', 'blue', 'black', 'green'])) + + +class FormExample1(ExamplePage): + """Demo for using FormEncode with Webware.""" + + def getDefaults(self): + return dict(age='enter your age', color=['blue']) + + def writeStyleSheet(self): + ExamplePage.writeStyleSheet(self) + self.writeln('''''') + + def awake(self, trans): + ExamplePage.awake(self, trans) + if self.request().hasField('name'): + fields = self.request().fields() + try: + fields = FormSchema.to_python(fields, self) + except Invalid, e: + errors = dict((k, v.encode('utf-8')) + for k, v in e.unpack_errors().iteritems()) + else: + errors = None + else: + fields = self.getDefaults() + errors = None + self.rendered_form = htmlfill.render(form_template, + defaults=fields, errors=errors) + + def writeContent(self): + self.write(self.rendered_form) diff --git a/formexample2.py b/formexample2.py new file mode 100644 index 0000000..2875dcb --- /dev/null +++ b/formexample2.py @@ -0,0 +1,69 @@ +"""Demo for using FormEncode with Webware.""" + +# Tested with Webware 1.1 and FormEncode 1.2.3 + +from formencode import compound, Invalid, htmlfill, Schema, validators +from formencode.htmlfill_schemabuilder import parse_schema +from WebKit.Examples.ExamplePage import ExamplePage + + +form_template = '''

Tell me about yourself

+ +
+ +


+ +

+ +


+ +

+ +

Your favorite color:
+ + Red   + Blue   + Black   + Green

+ +

+
''' + + +class FormExample2(ExamplePage): + """Demo for using FormEncode with Webware.""" + + schema = parse_schema(form_template) + + def getDefaults(self): + return dict(age='enter your age', color=['blue']) + + def writeStyleSheet(self): + ExamplePage.writeStyleSheet(self) + self.writeln('''''') + + def awake(self, trans): + ExamplePage.awake(self, trans) + if self.request().hasField('name'): + fields = self.request().fields() + try: + fields = self.schema.to_python(fields, self) + except Invalid, e: + errors = dict((k, v.encode('utf-8')) + for k, v in e.unpack_errors().iteritems()) + else: + errors = None + else: + fields = self.getDefaults() + errors = None + self.rendered_form = htmlfill.render(form_template, + defaults=fields, errors=errors) + + def writeContent(self): + self.write(self.rendered_form) diff --git a/formgenfirststeps.html b/formgenfirststeps.html new file mode 100644 index 0000000..3a97da0 --- /dev/null +++ b/formgenfirststeps.html @@ -0,0 +1,75 @@ + + + + + +Formgen First Steps + + + +
+

Formgen First Steps

+ +
+

Work in progress

+

Resources:

+ +
+
+

FormGn - Gernerieren HTML-Forms

+
+

Feldtypen

+

siehe: webhelpers.html.tags

+
+

Note

+

Test with cut and paste sphinx source

+
+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Feldtyp

Syntax

Text

text(name, value=None, id=None, **attrs)

Textarea

text(name, content='', id=None, **attrs)

Hidden

hidden(name, value=None, id=None, **attrs)

Password

password(name, value=None, id=None, **attrs)

Checkbox

checkbox(name, value='1', checked=False, label=None, id=None, **attrs)

Radio

Select

Submit

+
+
+
+ + + diff --git a/formvalidation.html b/formvalidation.html new file mode 100644 index 0000000..2e0ec51 --- /dev/null +++ b/formvalidation.html @@ -0,0 +1,37 @@ + + + + + +Form Validation + + + +
+

Form Validation

+ +

Form validation serves to ensure the validity of the data submitted to +the servlet. Not checking your data is one of the primary security +holes in web applications. The most egregious holes are when you use +code like this:

+
req = self.request()
+os.system("echo hello there | mail -s 'subscribe request' %s" % req.field('email'))
+

You might have a select box with several email address, which will +work just fine. But if the user manipulates the page so as to submit +something like: user@public.com;rm -rf /, you'll see the problem.

+

Besides security, form validation can deal with the +submit-signal-error-resubmit loop that complex form should allow. +Simply signalling an error and asking the user to go back is a +less-than-optimal situation.

+

Some systems that deal with this for Webware are FunFormKit, +FormEncode and XMLForms. Zope has Formulator, and Python CGIs have +pyweblib.

+

-- IanBicking - 02 Nov 2001

+
+ + + diff --git a/funformkit.html b/funformkit.html new file mode 100644 index 0000000..041ef03 --- /dev/null +++ b/funformkit.html @@ -0,0 +1,27 @@ + + + + + +FunFormKit + + + +
+

FunFormKit

+ +

FunFormKit is a form validation system, as well as form generation. It +is currently maintained by IanBicking, and exists at

+
+

http://www.colorstudy.com/software/funformkit/

+
+

FunFormKit is defunct, and is replaced by FormEncode.

+

-- IanBicking

+
+ + + diff --git a/futurework.html b/futurework.html new file mode 100644 index 0000000..62c2632 --- /dev/null +++ b/futurework.html @@ -0,0 +1,187 @@ + + + + + +Future Work + + + +
+

Future Work

+ +

Future Work

+

Webware for Python

+
+
Version
+

1.1b1

+
+
Released
+

04/18/10

+
+
+
+

Warning

+

You can find more information about known bugs and future work +in the Wiki and in the SourceForge tracker systems, +both accessible from the Webware Home Page. +The SourceForge task manager is currently not used since these systems should +be sufficient and we do not want to scatter issues in too many systems.

+
+
+

Future Work/Limitations

+

Sprinkled throughout the code are comments tagged with @@ which are +hopefully accompanied by a date and someone's initials. These comments +represent things to be done. The double at-sign (@@) convention was +chosen because it doesn't appear to be used for anything else.

+

In addition to the inline comments, some significant items have been +recorded below. These are future ideas, with no commitments or timelines +as to when/if they'll be realized. The Python WebKit is open source, +so feel free to jump in!

+
+

Known Bugs

+

All major known bugs that existed previously have been fixed.

+
+
+

To Do

+
+

Major Items

+
    +
  • Some of our old style guidelines are in conflict with newer Python +conventions (PEP8, properties etc.). We have already changed the +Webware convention of using tabs, but most naming conventions cannot +be changed so easily without breaking the API.

  • +
  • Think about a migration path towards Pyhon 3, check code with 2to3 tool.

  • +
  • Use distutils/setuptools instead of our own installer, use Python eggs +instead of our own plug-in concept.

  • +
  • The installer should try building and/or installing mod_webkit and wkcgi +automatically.

  • +
  • Use Sphinx instead of our own tools in DocSupport, use reStructuredText +as the format for all docs and maybe also for the docstrings.

  • +
  • Separate more clearly and systematically which messages are printed to +stdout and which to stderr.

  • +
  • Make use of the logging module (so far it is only used with unit testing) +instead of conditional print statements with "debug" and "verbose" flags.

  • +
  • Role-based security and user-authentication. Goal is to eliminate, +as much as possible, developer-written security logic. +This should be provided by the WebKit and be configurable.

  • +
  • Better support and documentation for distribution, load balancing and +fault tolerance.

  • +
  • More sophisticated admin tools including password protection, +clearing logs, displaying a maximum of information at a time, etc. +Consider using module 'resource'.

  • +
  • Investigate case insensitive URLs, especially for the Windows platform.

  • +
  • Support unicode strings and different encodings. HTTPContent should get an +encoding attribute with 'utf-8' as the default (set in Application.config). +This could be output in Page.writeMetaData() and used as the default for +an optional encoding parameter in HTTPContent.write(), so you can write +unicode strings to output. So far, Webware works best if you use ordinary +strings with a consistent encoding throughout the whole application.

  • +
  • Integrate support for i18n and l10n.

  • +
  • Plug-ins:

    +
      +
    • In ExamplePage, automatically support examples of any plug-in

    • +
    • Better docs

    • +
    • Properties.config. 'Load', 0, 1 or the name of the required op sys

    • +
    +
  • +
+
+
+

General

+
    +
  • Hunt down: @@ tags (which signify "To Be Done"s), FUTURE items +in class doc strings, NotImplementedErrors, -- tags

  • +
  • Code clean up. Use bin/checksrc and pylint.

  • +
  • Right now, if the Application itself (as opposed to Servlets) throws +an exception, it doesn't get captured nicely. However, it is displayed +in the app server's console.

  • +
  • The exception handler is pretty nice and has features like logging, +e-mail, gathering debugging info, etc. +However, on occasions it can throw exceptions too. +There should be a simpler, secondary exception handler for when this happens.

  • +
  • Review the timestamp caching logic and its relation to .pyc files if any.

  • +
  • Add "Last-modified:" to generic files that are served via WebKit.

  • +
  • If a Python file has only one class that inherits from Servlet, +then use that as the Servlet class +(rather than requiring the name be the same as the file).

  • +
+
+
+

Testing

+
    +
  • Provide testing web page where people can report their testing results +including version numbers, etc.

  • +
  • Provide higher level automation of testing. For example, a testing script +should be able to launch various app servers multiple times.

  • +
  • Provide highly automated benchmarking so we can track changes in performance.

  • +
  • Add more unit tests, expand the regression test suite.

  • +
+
+
+

Docs

+
    +
  • Use Sphinx instead of our own tools in DocSupport, use reStructuredText +as the format for all docs and maybe also for the docstrings.

  • +
  • Add a Getting Started Guide and a screencast.

  • +
  • Beef up the User's Guide and Tutorial.

  • +
  • User's Guide: Create a caching section to discuss the virtues of doing so +(the Color example became 12 X faster on the server side).

  • +
+
+
+

Food for thought, considerations, reviews

+
    +
  • Consider including FormKit, FunFormKit or FormEncode: +A plug-in to aid the construction and validation of forms.

  • +
  • Consider adding a simple helper lib for generating HTML +(such as SimpleHTMLGen) to the WebUtils package.

  • +
  • Better support for WSGI. +Currently we only have the WSGIAdapter interfacing to the app server.

  • +
  • Consider this statement from the FastCGI docs: +"Redirects are handled similar to CGI. Location headers with values +that begin with "/" are treated as internal-redirects; otherwise, +they are treated as external redirects (302)."

  • +
  • FastCGI app server: +The idea is that if the app server itself supports FastCGI, +then it can be used directly with FastCGI enabled web servers +sans the infamous "adapter". +Dan Green has brought this up in Webware-discuss.

  • +
  • Consider if we need to support <form action="x.py?a=1" method="post"> +where you will have both a query string and posted data.

  • +
  • Application modifies sys.path so that servlets can say +from SuperServlet import SuperServlet where SuperServlet is located in +the same directory as the Servlet. We'd prefer a more sophisticated technique +which does not modify sys.path and does not affect other servlets. (Or maybe +this would go away with a new one-process-per-application architecture.)

  • +
  • The ThreadedAppServer is not optimal for multiprocessor systems and modern +multi-core CPUs, due to the Python GIL. On Python 2.x, the GIL implementation +is particularly bad. Check whether we need to build something new such as +a MultiprocessAppServer, or use existing app server or tools such as Twisted.

  • +
+
+ +
+
+
+ + + diff --git a/geofftalvola.html b/geofftalvola.html new file mode 100644 index 0000000..49cf758 --- /dev/null +++ b/geofftalvola.html @@ -0,0 +1,24 @@ + + + + + +Geoff Talvola + + + +
+

Geoff Talvola

+ +

I live in Cambridge, MA and work for Parlance Corporation, a +speech recognition company in Medford, MA. I am one of the developers +of Webware. And of course, I am a heavy user of Webware in my work.

+

-- 29 Oct 2001

+
+ + + diff --git a/httpauthentication.html b/httpauthentication.html new file mode 100644 index 0000000..fea33b8 --- /dev/null +++ b/httpauthentication.html @@ -0,0 +1,106 @@ + + + + + +HTTP Authentication + + + +
+

HTTP Authentication

+ +

HTTP authentication is the standard way to authenticate access to +resources. It has a number of flaws, and it not the most common way +to do authentication on the web -- using standard forms is more +common. However, in some circumstances it is the only possible way -- +in particular, using non-browser tools to access pages (like a DAV +client, an XML-RPC client, etc.)

+

HTTP authentication occurs when the server returns a 401 status (as in +self.response().setStatus(401, 'Authorization Required') ) The +server also must set another header, WWW-Authenticate: Basic realm="<i>realm name</i>" The realm name is the only message the +user will see explaining what they are logging into (though they +should know what page they are trying to access).

+

Clients will sent a header like Authorization: Basic xxx (where xxx is +the authorization information)

+

Clients generally do not send authentication information until they +are asked to do so (by receiving a 401 error). In typical usage, this +back and forth will occur for every page access. (Though I don't see +any reason you can't use the session and cookies like you would with +normal, form-based authentication).

+

There is another style of authentication (this style is termed +"Basic"), called "Digest" -- it sends a hash of the password, in a +challenge-response style. This is more secure, as you can't sniff +passwords. However, for reasons I cannot fathom, many major browsers +<i>still</i> do not support this kind of authentication. Instead, you +may use SSL to secure the password.

+
+

Setting up Apache

+

Apache does not normally send the Authorization header onto clients. +To get around this you must use mod_rewrite (this is also typical with +Zope, but I haven't gotten their technique to work for me)

+

Put this rewrite rule in:

+
<Location /WK>
+RewriteEngine On
+RewriteCond %{HTTP:Authorization} ^(.*)
+RewriteRule /WK(.*) /WK2$1?AUTH=%1 [QSA,L,PT]
+</Location>
+

/WK2 is the real location of the WebKit hander -- either a +location set up for mod_webkit, or another adapter. The +RewriteCond line will capture the Authorization: header, which can +be referenced by %1. The RewriteRule line will add a variable +AUTH to the query string.

+

(I'm not entirely sure about the security of this -- the +authentication information doesn't show up in the log, though)

+
+
+

In your servlet

+

In the servlet, you must both send the right information, and then +receive that authorization. To require authentication:

+
res = self.response()
+res.setStatus(401, 'Authorization Required')
+res.setHeader('WWW-Authenticate', 'Basic realm="%s"' % realm)
+

To process it:

+
import base64
+
+def authorized(self):
+    req = self.request()
+    httpAuth = req.field('AUTH', None)
+    if not httpAuth: return 0
+    authType, auth = httpAuth.split(' ', 1)
+    assert authType.lower() == 'basic', 'Only basic HTTP authentication is supported'
+    name, password = base64.decodestring(auth.strip()).split(':', 1)
+    return self.authorizeUser(name, password)
+

(If you are using HTTPAdapter, you will receive the authentication +information in self.request().environ()['HTTP_AUTHORIZATION'] )

+

-- IanBicking - 10 May 2002

+
+
+

Alternative Approach

+

You can compile Apache 1.3 with SECURITY_HOLE_PASS_AUTHORIZATION +which will cause apache to pass the HTTP_AUTHORIZATION environment +variable to your script. If you don't want to recompile apache, you +can use a much simpler mod_rewrite setup to add an additional +environment variable:

+
<Location /WK>
+    WKServer localhost 8086
+    SetHandler webkit-handler
+    RewriteEngine On
+    RewriteRule /WK(.*) - [E=X-HTTP_AUTHORIZATION:%{HTTP:Authorization},PT]
+</Location>
+

With this you scan check +self.request().environ()['HTTP_AUTHORIZATION'] and +self.request().environ()['X-HTTP_AUTHORIZATION'] to find +authorization Information.

+

There is also a patch to mod_webkit implementing passing of +Authorisation Info to the Application Server available at +http://blogs.23.nu/c0re/stories/1410/

+
+
+ + + diff --git a/ianbicking.html b/ianbicking.html new file mode 100644 index 0000000..8757694 --- /dev/null +++ b/ianbicking.html @@ -0,0 +1,33 @@ + + + + + +Ian Bicking + + + +
+

Ian Bicking

+ +

Ian Bicking, me...

+
+
homepage:
+

http://ianbicking.org

+
+
blog:
+

http://blog.ianbicking.org

+
+
email:
+

ianb@colorstudy.com

+
+
+

I wrote this Wiki software

+
+ + + diff --git a/index.html b/index.html new file mode 100644 index 0000000..363a3d5 --- /dev/null +++ b/index.html @@ -0,0 +1,184 @@ + + + + + +Ye Olde Webware for Python Wiki + + + +
+

Ye Olde Webware for Python Wiki

+ +
+

Introduction

+

Welcome to the archived Wiki of the Webware for Python project.

+

This Wiki has been created by Ian Bicking in October 2001, +using the Webware framework. Its purpose was to facilitate discussion +about Webware: its design, its use and its future. This Wiki was intended +to supplement, not replace, the Webware homepage, webware lists +and docs . It collected interesting ideas and recipes by Webware users +and developers until May 2010. It now has been closed and archived as a +read-only reference and historical document. Please note that many links +are now broken, and some of the content may not be applicable to the latest +versions of Webware for Python any more.

+
+
+

Starting Points

+ ++++ + + + + + + + + +

Python Propaganda

Who Is Using Webware?

Webware Propaganda

Can Webware handle it?

+

See also the Register for a list of all pages of this Wiki.

+
+
+

Tips for Using Webware

+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

ApplicationDesign

How to design your webware application.

WebServerIntegration

How to integrate Webware with various webservers.

WebwareRecipes

Detailed explanations on how to accomplish common tasks.

RealtimeCharts

Tools to generate charts on demand.

DatabaseIntegration

Detailed explanations and links to tools +for working with databases.

TroubleshootingFAQ

Things that have tripped people up.

WebwareSecurity

How to secure your Webware applications.

WebwareHosting

What to look for in a host.

ModRewriteRecipes

Apache's mod_rewrite: the solution to all questions +regarding URLs, eliminating Webware-specific parts of the URL, +and dealing with virtual hosts.

Web-Commerce

E-Commerce, Credit cards, dynamic pricing, etc.

MiscTips

A catch-all for items that don't fit anywhere else.

+
+
+

Webware Documentation

+ ++++ + + + + + + + + + + + +

WebwareNews

News items from the webware project, in chronological order

TransactionAnatomy

Describes the internal process for handling a transaction.

CompatibilityInformation

Documents all known compatibility issues between Webware, +Python, and third-party modules.

+
+
+

Off-Site Documentation

+
+
Building a Pop-Up Calendar:
+

Off-Site Tutorial to a simple calendar control.

+
+
Introduction to Webware:
+

Offsite: from a talk at LinuxTag 2002 (50% Webware, 50% MiddleKit)

+
+
+
+ +
+

Discussion on Improving Webware

+
+
FutureWork:
+

This is a copy of the current Webware/WebKit/Docs/Future.txt document.

+
+
FeaturesAfterV09:
+

Jan 2005 - Summary of discussion on mailing lists about what we should +work on in the near future.

+
+
WishList:
+

An informal wishlist of things people would like to see improved in +Webware. Wishes that become planned features are moved from this +list to the ToDo list.

+
+
ToDo:
+

A more formal list of improvements that are planned for specific +releases (e.g. 0.9).

+
+
WishesGranted:
+

a list of wishes and todo items that have been completed.

+
+
SubProjects:
+

An index of the development sub-projects that are focussed on +discussing and implementing specific items on the ToDo list.

+
+
WebwareExpRefactoring:
+

An 'unofficial' refactoring of the Webware 0.6 codebase that addresses +many of the items on the WishList and ToDo list.

+
+
RelatedReading:
+

Third-party documents on high-level design issues

+
+
SubmittingPatches:
+

Guidelines for submitting patches

+
+
DeveloperGuidelines:
+

Guidelines for Webware Developers (Read this if you want CVS access)

+
+
SubversionRepositoryAccess:
+

As of March 2005, the Webware source is stored in Subversion. Here is how to access it.

+
+
ThirdPartyPages:
+

links to other pages that discuss Webware

+
+
+
+
+ + + diff --git a/interceptconnection.html b/interceptconnection.html new file mode 100644 index 0000000..a7259ed --- /dev/null +++ b/interceptconnection.html @@ -0,0 +1,28 @@ + + + + + +Intercept Connection + + + +
+

Intercept Connection

+ +

TcpWatch is a great program to view what goes on during a transaction. +You can find it at http://www.zope.org/Members/hathawsh/tcpwatch

+

You run it as:

+
$ python tcpwatch.py 8080 myserver.com 80</code>
+

And when you connect to localhost:8080, you'll get pages from +myserver.com:80 -- except that everything that goes back and forth +will be tracked through a nice Tk interface.

+

-- IanBicking - 15 Nov 2001

+
+ + + diff --git a/internetgroupware.html b/internetgroupware.html new file mode 100644 index 0000000..9d187ac --- /dev/null +++ b/internetgroupware.html @@ -0,0 +1,52 @@ + + + + + +Internet Groupware + + + +
+

Internet Groupware

+ +

SourceForge.net provides a valuable service to the Open Source +community, but its implementation and user interface could be improved +in many ways. (TavisRudd in WebwareSourceforge)

+

The biggest improvement would be to get the browser out of the way. I +cringe every time I need to search a mailing list (or worse, a +Sourceforge "forum") for a specific keyword or conversation through +some strange web-interface. IMAP, which was designed for exactly that +purpose, does the job thousands of times faster with far less drain on +both human and computer/network resources. (NNTP is also ok, but +requires the client software to do the searching, thus causing +more load on the server by having to transfer all of the articles, +rather than just the interesting ones.)

+

I think that a key benefit of the webware approach over the Zope +approach is that you can design your application without being +constrained by a browser. Roundup's design is a good example of +this. It uses the most approriate protocol/tool for each task, +instead of trying to shoehorn everything into "stateless" HTTP +transactions.

+

Therefore: a "webware application" such as this proposed collaboration +suite, should be first an application with a thin adapter to +create a browser-based UI generated using webware components.

+

Please read these before you start:

+ +

And beware of this (another "feature" of the current sourceforge development model):

+ +

(Can you tell that I strongly dislike web-only interfaces? 8-D)

+

-- TerrelShumway - 16 Apr 2002

+
+ + + diff --git a/javascriptdependencyresolution.html b/javascriptdependencyresolution.html new file mode 100644 index 0000000..1bc996a --- /dev/null +++ b/javascriptdependencyresolution.html @@ -0,0 +1,31 @@ + + + + + +Javascriptdependencyresolution + + + +
+

Javascriptdependencyresolution

+ +

As we all know, JavaScript sucks in many ways, one of them being its lack of a module import system. So, if you have a substantial amount of JavaScript code, there are basically three choices:

+
    +
  • stuff everything into one huge file,

  • +
  • break the code down into multiple files that load recursively by issuing multiple HTTP requests,

  • +
  • manually add all JS URLs, in carefully maintained order, to each and every page that needs them.

  • +
+

The first choice is unpleasant for the developer, whereas the second is unpleasant for the user (who gets to watch the sometimes slow loading). Approach 3 is unpleasant to both parties.

+

An alternative is to use dependency resolution between JavaScript files on the server side. The dependencies can be declared in JavaScript comments:

+

// requireScript script1, script2, script0815

+

The resolution is done in a servlet, whose URL is prefixed to the topmost JS file. In this way, one big JS file is assembled that can be sent in a single transaction. The file will include all required files recursively and in proper order, such that required files are always included before their dependent ones. The assembled JS file can be cached. The servlet file JSResolver.py +contains additional explanations.

+
+ + + diff --git a/justalink.html b/justalink.html new file mode 100644 index 0000000..78d1ddf --- /dev/null +++ b/justalink.html @@ -0,0 +1,367 @@ + + + + + + + + + +
+ + + +
+

Section Title

+
+

Section Title

+
+

Section Title

+
+

Section Title

+
+
Section Title
+
+
Section Title
+
+Section Title +
+Section Title +
+Section Title +
+Section Title +
+Section Title +
+ +++ + + + + +

paragraph

+ +++ + + + + +

paragraph

+
+
+
term 1
+

Definition 1.

+
+
term 2
+

Definition 2, paragraph 1.

+

Definition 2, paragraph 2.

+
+
term 3classifier
+

Definition 3.

+
+
term 4classifier oneclassifier two
+

Definition 4.

+
+
+
+ ++++++ + + + + + + + + + + + + + + + + + + + + + + + +

Header row, column 1 +(header rows optional)

Header 2

Header 3

Header 4

body row 1, column 1

column 2

column 3

column 4

body row 2

Cells may span columns.

body row 3

Cells may +span rows.

    +
  • Table cells

  • +
  • contain

  • +
  • body elements.

  • +
+

body row 4

+

Some care must be taken with grid tables to avoid undesired interactions with cell text in rare cases. For example, the following table contains a cell in row 2 spanning from column 2 to column 4:

+ ++++++ + + + + + + + + + + + + + + + +

row 1, col 1

column 2

column 3

column 4

row 2

row 3

+

If a vertical bar is used in the text of that cell, it could have unintended effects if accidentally aligned with column boundaries:

+ ++++++ + + + + + + + + + + + + + + + +

row 1, col 1

column 2

column 3

column 4

row 2

Use the command ls | more.

row 3

+

Several solutions are possible. All that is needed is to break the continuity of the cell outline rectangle. One possibility is to shift the text by adding an extra space before:

+ ++++++ + + + + + + + + + + + + + + + +

row 1, col 1

column 2

column 3

column 4

row 2

Use the command ls | more.

row 3

+

Another possibility is to add an extra line to row 2:

+ ++++++ + + + + + + + + + + + + + + + +

row 1, col 1

column 2

column 3

column 4

row 2

Use the command ls | more.

row 3

+

Simple Tables

+

Simple tables provide a compact and easy to type but limited row-oriented table representation for simple data sets. Cell contents are typically single paragraphs, although arbitrary body elements may be represented in most cells. Simple tables allow multi-line rows (in all but the first column) and column spans, but not row spans. See Grid Tables above for a complete table representation.

+

Simple tables are described with horizontal borders made up of "=" and "-" characters. The equals sign ("=") is used for top and bottom table borders, and to separate optional header rows from the table body. The hyphen ("-") is used to indicate column spans in a single row by underlining the joined columns, and may optionally be used to explicitly and/or visually separate rows.

+

A simple table begins with a top border of equals signs with one or more spaces at each column boundary (two or more spaces recommended). Regardless of spans, the top border must fully describe all table columns. There must be at least two columns in the table (to differentiate it from section headers). The last of the optional header rows is underlined with '=', again with spaces at column boundaries. There may not be a blank line below the header row separator; it would be interpreted as the bottom border of the table. The bottom boundary of the table consists of '=' underlines, also with spaces at column boundaries. For example, here is a truth table, a three-column table with one header row and four body rows:

+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + +

A

B

A and B

False

False

False

True

False

False

False

True

False

True

True

True

+

Underlines of '-' may be used to indicate column spans by "filling in" column margins to join adjacent columns. Column span underlines must be complete (they must cover all columns) and align with established column boundaries. Text lines containing column span underlines may not contain any other text. A column span underline applies only to one row immediately above it. For example, here is a table with a column span in the header:

+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Inputs

Output

A

B

A or B

False

False

False

True

False

True

False

True

True

True

True

True

+

Each line of text must contain spaces at column boundaries, except where cells have been joined by column spans. Each line of text starts a new row, except when there is a blank cell in the first column. In that case, that line of text is parsed as a continuation line. For this reason, cells in the first column of new rows (not continuation lines) must contain some text; blank cells would lead to a misinterpretation. An empty comment ("..") is sufficient and will be omitted from the processed output (see Comments below). Also, this mechanism limits cells in the first column to only one line of text. Use grid tables if this limitation is unacceptable.

+

Underlines of '-' may also be used to visually separate rows, even if there are no column spans. This is especially useful in long tables, where rows are many lines long.

+

Blank lines are permitted within simple tables. Their interpretation depends on the context. Blank lines between rows are ignored. Blank lines within multi-line rows may separate paragraphs or other body elements within cells.

+

The rightmost column is unbounded; text may continue past the edge of the table (as indicated by the table borders). However, it is recommended that borders be made long enough to contain the entire text.

+

The following example illustrates continuation lines (row 2 consists of two lines of text, and four lines for row 3), a blank line separating paragraphs (row 3, column 2), and text extending past the right edge of the table:

+ ++++ + + + + + + + + + + + + + + + + +

col 1

col 2

1

Second column of row 1.

2

Second column of row 2. +Second line of paragraph.

3

    +
  • Second column of row 3.

  • +
  • Second item in bullet +list (row 3, column 2).

  • +
+
+

Explicit Markup Blocks

+
+
+
+
+
+
+
+
+
+
+
+
+ + + diff --git a/justanyone.html b/justanyone.html new file mode 100644 index 0000000..8f7b382 --- /dev/null +++ b/justanyone.html @@ -0,0 +1,24 @@ + + + + + +Justanyone + + + +
+

Justanyone

+ +

In real life:

+

Kevin J. Rice

+

I work for Textura, LLC (http://texturallc.com) in Lake Bluff, Illinois, USA.

+

My home page is : http://justanyone.com

+
+ + + diff --git a/kylevanderbeek.html b/kylevanderbeek.html new file mode 100644 index 0000000..e69d85c --- /dev/null +++ b/kylevanderbeek.html @@ -0,0 +1,22 @@ + + + + + +Kyle VanderBeek + + + +
+

Kyle VanderBeek

+ +

I work for IronPort Systems and am evaluating Webware for use in some of our web projects.

+

I believe in spaces not tabs.

+
+ + + diff --git a/lib/PySourceColor.py b/lib/PySourceColor.py deleted file mode 100644 index 6fcbaf8..0000000 --- a/lib/PySourceColor.py +++ /dev/null @@ -1,1205 +0,0 @@ -""" -############################################################################# -# PySourceColor.py -############################################################################# -# A python source to colorized html converter. -# Hacked by M.E.Farmer Jr. 2004 -# Python license -############################################################################# -# Now supports two types of output markup: -# - HTML markup does not create w3c valid html, but it works on every -# browser i've tried so far.(I.E.,Mozilla/Firefox,Opera,wxHTML). -# - CSS markup is w3c valid html 4.01 strict, -# but will not render correctly everywhere! -# Too bad CSS is not supported on all browsers. -############################################################################# -# Features: -# -Can seperate and colorize: -# 12 types of strings -# 2 comment types -# numbers -# operators -# class / name -# def / name -# decorator / name -# keywords -# arguments class/def/deco -# text -# -Eight colorschemes built-in: -# null -# mono -# dark (default) -# dark2 -# lite -# idle -# viewcvs -# pythonwin -# -Two types of markup: -# html (default) -# css/html -# -Any combination of four text styles: -# none (default) -# bold -# italic -# underline -############################################################################# -# Example usage: -############################################################################# -# # import -# import PySourceColor as psc -# psc.convert('c:/Python22/PySourceColor.py', colors=psc.idle, show=1) -#---------------------------------------------------------------------------- -# # from module import * -# from PySourceColor import * -# convert('c:/Python22/Lib', colors=lite, markup="css") -#---------------------------------------------------------------------------- -# # How to use a custom colorscheme. -# from PySourceColor import * -# new = { -# ERRORTOKEN: ('bui','#FF8080',''), -# DECORATOR_NAME: ('','#AACBBC',''), -# DECORATOR: ('','#333333',''), -# NAME: ('','#1133AA','#DDFF22'), -# NUMBER: ('','#236676','#FF5555'), -# OPERATOR: ('b','#435345','#BBBB11'), -# COMMENT: ('','#545366','#AABBFF'), -# DOUBLECOMMENT: ('','#553455','#FF00FF'), -# CLASS_NAME: ('','#000000','#FFFFFF'), -# DEF_NAME: ('u','#897845','#000022'), -# KEYWORD: ('','#345345','#FFFF22'), -# SINGLEQUOTE: ('','#223344','#AADDCC'), -# SINGLEQUOTE_R: ('','#344522',''), -# SINGLEQUOTE_U: ('','#234234',''), -# DOUBLEQUOTE: ('','#334421',''), -# DOUBLEQUOTE_R: ('','#345345',''), -# DOUBLEQUOTE_U: ('','#678673',''), -# TRIPLESINGLEQUOTE: ('','#FFFFFF','#000000'), -# TRIPLESINGLEQUOTE_R: ('bu','#443256','#DDFFDA'), -# TRIPLESINGLEQUOTE_U: ('','#423454','#DDFFDA'), -# TRIPLEDOUBLEQUOTE: ('ib','#000000','#FFFFFF'), -# TRIPLEDOUBLEQUOTE_R: ('ub','#000000','#FFFFFF'), -# TRIPLEDOUBLEQUOTE_U: ('', '#CCAABB','#FFFAFF'), -# PAGEBACKGROUND: '#FFFAAA', -# } -# if __name__ == '__main__': -# import sys -# convert(sys.argv[1], './css.html',colors=new, markup='css', show=1) -# convert(sys.argv[1], './html.html',colors=new, markup='html', show=1) -############################################################################# -""" -__all__ = ['ERRORTOKEN','DECORATOR_NAME', 'DECORATOR', 'ARGS', - 'NAME', 'NUMBER', 'OPERATOR', 'COMMENT', - 'DOUBLECOMMENT', 'CLASS_NAME', 'DEF_NAME', 'KEYWORD', - 'SINGLEQUOTE','SINGLEQUOTE_R','SINGLEQUOTE_U','DOUBLEQUOTE', - 'DOUBLEQUOTE_R', 'DOUBLEQUOTE_U', 'TRIPLESINGLEQUOTE', - 'TRIPLESINGLEQUOTE_R', 'TRIPLESINGLEQUOTE_U', 'TRIPLEDOUBLEQUOTE', - 'TRIPLEDOUBLEQUOTE_R', 'TRIPLEDOUBLEQUOTE_U', 'PAGEBACKGROUND', - 'null', 'mono', 'lite', 'dark','dark2', 'pythonwin','idle', - 'viewcvs', 'Usage', 'cli', 'str2stdout', 'path2stdout', 'Parser', - 'str2file', 'str2html', 'path2file', 'path2html', 'convert', - 'walkdir', 'defaultColors', 'showpage'] -__title__ = 'PySourceColor' -__version__ = "1.9.9" -__date__ = '27 November 2004' -__author__ = "M.E.Farmer Jr." -__credits__ = '''This was originally based on a python recipe -submitted by Juergen Hermann to ASPN. -M.E.Farmer 2004 -Python license -''' -import os -import cgi -import sys -import time -import glob -import getopt -import keyword -import token -import tokenize -import cStringIO -import traceback -import webbrowser - -# Do not edit -NAME = token.NAME -NUMBER = token.NUMBER -COMMENT = tokenize.COMMENT -OPERATOR = token.OP -ERRORTOKEN = token.ERRORTOKEN -ARGS = token.NT_OFFSET + 1 -DOUBLECOMMENT = token.NT_OFFSET + 2 -CLASS_NAME = token.NT_OFFSET + 3 -DEF_NAME = token.NT_OFFSET + 4 -KEYWORD = token.NT_OFFSET + 5 -SINGLEQUOTE = token.NT_OFFSET + 6 -SINGLEQUOTE_R = token.NT_OFFSET + 7 -SINGLEQUOTE_U = token.NT_OFFSET + 8 -DOUBLEQUOTE = token.NT_OFFSET + 9 -DOUBLEQUOTE_R = token.NT_OFFSET + 10 -DOUBLEQUOTE_U = token.NT_OFFSET + 11 -TRIPLESINGLEQUOTE = token.NT_OFFSET + 12 -TRIPLESINGLEQUOTE_R = token.NT_OFFSET + 13 -TRIPLESINGLEQUOTE_U = token.NT_OFFSET + 14 -TRIPLEDOUBLEQUOTE = token.NT_OFFSET + 15 -TRIPLEDOUBLEQUOTE_R = token.NT_OFFSET + 16 -TRIPLEDOUBLEQUOTE_U = token.NT_OFFSET + 17 -PAGEBACKGROUND = token.NT_OFFSET + 18 -DECORATOR = token.NT_OFFSET + 19 -DECORATOR_NAME = token.NT_OFFSET + 20 - -# Do not edit (markup classname lookup) -MARKUPDICT = { - ERRORTOKEN: 'err', - DECORATOR_NAME: 'decn', - DECORATOR: 'dec', - ARGS: 'args', - NAME: 'name', - NUMBER: 'num', - OPERATOR: 'op', - COMMENT: 'comm', - DOUBLECOMMENT: 'dcom', - CLASS_NAME: 'clsn', - DEF_NAME: 'defn', - KEYWORD: 'key', - SINGLEQUOTE: 'sq', - SINGLEQUOTE_R: 'sqr', - SINGLEQUOTE_U: 'squ', - DOUBLEQUOTE: 'dq', - DOUBLEQUOTE_R: 'dqr', - DOUBLEQUOTE_U: 'dqu', - TRIPLESINGLEQUOTE: 'tsq', - TRIPLESINGLEQUOTE_R: 'tsqr', - TRIPLESINGLEQUOTE_U: 'tsqu', - TRIPLEDOUBLEQUOTE: 'tdq', - TRIPLEDOUBLEQUOTE_R: 'tdqr', - TRIPLEDOUBLEQUOTE_U: 'tdqu', - } - -###################################################################### -# Edit colors and styles to taste -# Create your own scheme, just copy one below , rename and edit. -# fore and back color are rgb hex and must be specified. #RRGGBB -# Styles are optional: b=bold, i=italic, u=underline -# Cutom styles must at least define NAME, ERRORTOKEN, PAGEBACKGROUND, -# all missing elements will default to NAME. -###################################################################### -# Copy null and use it as a starter colorscheme. -null = {# tokentype: ('tags', 'textforecolor', 'textbackcolor') - ERRORTOKEN: ('','#FF8080',''),# Error token - DECORATOR_NAME: ('','#000000',''),# Decorator name - DECORATOR: ('','#000000',''),# @ - ARGS: ('','#000000',''), - NAME: ('','#000000',''),# All other text - NUMBER: ('','#000000',''),# 0->10 - OPERATOR: ('','#000000',''),# ()<>=!.:;^>%, etc. - COMMENT: ('','#000000',''),# Single comment - DOUBLECOMMENT: ('','#000000',''),## Double comment - CLASS_NAME: ('','#000000',''),# Class name - DEF_NAME: ('','#000000',''),# Def name - KEYWORD: ('','#000000',''),# Python keywords - SINGLEQUOTE: ('','#000000',''),# 'SINGLEQUOTE' - SINGLEQUOTE_R: ('','#000000',''),# r'SINGLEQUOTE' - SINGLEQUOTE_U: ('','#000000',''),# u'SINGLEQUOTE' - DOUBLEQUOTE: ('','#000000',''),# "DOUBLEQUOTE" - DOUBLEQUOTE_R: ('','#000000',''),# r"DOUBLEQUOTE" - DOUBLEQUOTE_U: ('','#000000',''),# u"DOUBLEQUOTE" - TRIPLESINGLEQUOTE: ('','#000000',''),# '''TRIPLESINGLEQUOTE''' - TRIPLESINGLEQUOTE_R: ('','#000000',''),# r'''TRIPLESINGLEQUOTE''' - TRIPLESINGLEQUOTE_U: ('','#000000',''),# u'''TRIPLESINGLEQUOTE''' - TRIPLEDOUBLEQUOTE: ('','#000000',''),# """TRIPLEDOUBLEQUOTE""" - TRIPLEDOUBLEQUOTE_R: ('','#000000',''),# r"""TRIPLEDOUBLEQUOTE""" - TRIPLEDOUBLEQUOTE_U: ('','#000000',''),# u"""TRIPLEDOUBLEQUOTE""" - PAGEBACKGROUND: '#FFFFFF'# set the page background - } - -mono = { - ERRORTOKEN: ('','#FF8080',''), - DECORATOR_NAME: ('bu','#000000',''), - DECORATOR: ('b','#000000',''), - ARGS: ('b','#000000',''), - NAME: ('','#000000',''), - NUMBER: ('b','#000000',''), - OPERATOR: ('b','#000000',''), - COMMENT: ('i','#000000',''), - DOUBLECOMMENT: ('b','#000000',''), - CLASS_NAME: ('bu','#000000',''), - DEF_NAME: ('b','#000000',''), - KEYWORD: ('b','#000000',''), - SINGLEQUOTE: ('','#000000',''), - SINGLEQUOTE_R: ('','#000000',''), - SINGLEQUOTE_U: ('','#000000',''), - DOUBLEQUOTE: ('','#000000',''), - DOUBLEQUOTE_R: ('','#000000',''), - DOUBLEQUOTE_U: ('','#000000',''), - TRIPLESINGLEQUOTE: ('','#000000',''), - TRIPLESINGLEQUOTE_R: ('','#000000',''), - TRIPLESINGLEQUOTE_U: ('','#000000',''), - TRIPLEDOUBLEQUOTE: ('i','#000000',''), - TRIPLEDOUBLEQUOTE_R: ('i','#000000',''), - TRIPLEDOUBLEQUOTE_U: ('i','#000000',''), - PAGEBACKGROUND: '#FFFFFF' - } - -dark = { - ERRORTOKEN: ('','#FF8080',''), - DECORATOR_NAME: ('b','#FFBBAA',''), - DECORATOR: ('b','#CC5511',''), - ARGS: ('b','#CCCCDD',''), - NAME: ('','#FFFFFF',''), - NUMBER: ('','#FF0000',''), - OPERATOR: ('b','#FAF785',''), - COMMENT: ('','#45FCA0',''), - DOUBLECOMMENT: ('i','#A7C7A9',''), - CLASS_NAME: ('b','#B599FD',''), - DEF_NAME: ('b','#EBAE5C',''), - KEYWORD: ('b','#8680FF',''), - SINGLEQUOTE: ('','#F8BAFE',''), - SINGLEQUOTE_R: ('','#F8BAFE',''), - SINGLEQUOTE_U: ('','#F8BAFE',''), - DOUBLEQUOTE: ('','#FF80C0',''), - DOUBLEQUOTE_R: ('','#FF80C0',''), - DOUBLEQUOTE_U: ('','#FF80C0',''), - TRIPLESINGLEQUOTE: ('','#FF9595',''), - TRIPLESINGLEQUOTE_R: ('','#FF9595',''), - TRIPLESINGLEQUOTE_U: ('','#FF9595',''), - TRIPLEDOUBLEQUOTE: ('','#B3FFFF',''), - TRIPLEDOUBLEQUOTE_R: ('','#B3FFFF',''), - TRIPLEDOUBLEQUOTE_U: ('','#B3FFFF',''), - PAGEBACKGROUND: '#000000' - } - -dark2 = { - ERRORTOKEN: ('','#FF8080',''), - DECORATOR_NAME: ('b','#FFBBAA',''), - DECORATOR: ('b','#CC5511',''), - ARGS: ('b','#FFFFFF',''), - NAME: ('','#c0c0c0',''), - NUMBER: ('b','#00FF00',''), - OPERATOR: ('b','#FF090F',''), - COMMENT: ('i','#F0F709','#844200'), - DOUBLECOMMENT: ('i','#F0F709','#844200'), - CLASS_NAME: ('b','#7E58C7',''), - DEF_NAME: ('b','#FF8040',''), - KEYWORD: ('b','#4726E1',''), - SINGLEQUOTE: ('','#8080C0',''), - SINGLEQUOTE_R: ('','#8080C0',''), - SINGLEQUOTE_U: ('','#8080C0',''), - DOUBLEQUOTE: ('','#ADB9F1',''), - DOUBLEQUOTE_R: ('','#ADB9F1',''), - DOUBLEQUOTE_U: ('','#ADB9F1',''), - TRIPLESINGLEQUOTE: ('','#00C1C1',''), - TRIPLESINGLEQUOTE_R: ('','#00C1C1',''), - TRIPLESINGLEQUOTE_U: ('','#00C1C1',''), - TRIPLEDOUBLEQUOTE: ('','#33e3e3',''), - TRIPLEDOUBLEQUOTE_R: ('','#33e3e3',''), - TRIPLEDOUBLEQUOTE_U: ('','#33e3e3',''), - PAGEBACKGROUND: '#000000' - } - -lite = { - ERRORTOKEN: ('','#FF8080',''), - DECORATOR_NAME: ('b','#BB4422',''), - DECORATOR: ('b','#3333af',''), - ARGS: ('','#000000',''), - NAME: ('','#000000',''), - NUMBER: ('','#FF2200',''), - OPERATOR: ('b','#303000',''), - COMMENT: ('','#007F00',''), - DOUBLECOMMENT: ('','#606060',''), - CLASS_NAME: ('','#0000FF',''), - DEF_NAME: ('b','#9C7A00',''), - KEYWORD: ('b','#0000AF',''), - SINGLEQUOTE: ('','#600080',''), - SINGLEQUOTE_R: ('','#600080',''), - SINGLEQUOTE_U: ('','#600080',''), - DOUBLEQUOTE: ('','#A0008A',''), - DOUBLEQUOTE_R: ('','#A0008A',''), - DOUBLEQUOTE_U: ('','#A0008A',''), - TRIPLESINGLEQUOTE: ('','#337799',''), - TRIPLESINGLEQUOTE_R: ('','#337799',''), - TRIPLESINGLEQUOTE_U: ('','#337799',''), - TRIPLEDOUBLEQUOTE: ('','#1188AA',''), - TRIPLEDOUBLEQUOTE_R: ('','#1188AA',''), - TRIPLEDOUBLEQUOTE_U: ('','#1188AA',''), - PAGEBACKGROUND: '#FFFFFF' - } - -idle = { - ERRORTOKEN: ('','#FF8080',''), - DECORATOR_NAME: ('','#900090',''), - DECORATOR: ('','#000000',''), - NAME: ('','#000000',''), - NUMBER: ('','#000000',''), - OPERATOR: ('','#000000',''), - COMMENT: ('','#DD0000',''), - DOUBLECOMMENT: ('','#DD0000',''), - CLASS_NAME: ('','#0000FF',''), - DEF_NAME: ('','#0000FF',''), - KEYWORD: ('','#FF7700',''), - SINGLEQUOTE: ('','#00AA00',''), - SINGLEQUOTE_R: ('','#00AA00',''), - SINGLEQUOTE_U: ('','#00AA00',''), - DOUBLEQUOTE: ('','#00AA00',''), - DOUBLEQUOTE_R: ('','#00AA00',''), - DOUBLEQUOTE_U: ('','#00AA00',''), - TRIPLESINGLEQUOTE: ('','#00AA00',''), - TRIPLESINGLEQUOTE_R: ('','#00AA00',''), - TRIPLESINGLEQUOTE_U: ('','#00AA00',''), - TRIPLEDOUBLEQUOTE: ('','#00AA00',''), - TRIPLEDOUBLEQUOTE_R: ('','#00AA00',''), - TRIPLEDOUBLEQUOTE_U: ('','#00AA00',''), - PAGEBACKGROUND: '#FFFFFF' - } - -pythonwin = { - ERRORTOKEN: ('','#FF8080',''), - DECORATOR_NAME: ('b','#303030',''), - DECORATOR: ('b','#DD0080',''), - ARGS: ('','#000000',''), - NAME: ('','#303030',''), - NUMBER: ('','#008080',''), - OPERATOR: ('','#000000',''), - COMMENT: ('','#007F00',''), - DOUBLECOMMENT: ('','#7F7F7F',''), - CLASS_NAME: ('b','#0000FF',''), - DEF_NAME: ('b','#007F7F',''), - KEYWORD: ('b','#000080',''), - SINGLEQUOTE: ('','#808000',''), - SINGLEQUOTE_R: ('','#808000',''), - SINGLEQUOTE_U: ('','#808000',''), - DOUBLEQUOTE: ('','#808000',''), - DOUBLEQUOTE_R: ('','#808000',''), - DOUBLEQUOTE_U: ('','#808000',''), - TRIPLESINGLEQUOTE: ('','#808000',''), - TRIPLESINGLEQUOTE_R: ('','#808000',''), - TRIPLESINGLEQUOTE_U: ('','#808000',''), - TRIPLEDOUBLEQUOTE: ('','#808000',''), - TRIPLEDOUBLEQUOTE_R: ('','#808000',''), - TRIPLEDOUBLEQUOTE_U: ('','#808000',''), - PAGEBACKGROUND: '#FFFFFF' - } - -viewcvs = { - ERRORTOKEN: ('b','#FF8080',''), - DECORATOR_NAME: ('','#000000',''), - DECORATOR: ('','#000000',''), - ARGS: ('','#000000',''), - NAME: ('','#000000',''), - NUMBER: ('','#000000',''), - OPERATOR: ('','#000000',''), - COMMENT: ('i','#b22222',''), - DOUBLECOMMENT: ('i','#b22222',''), - CLASS_NAME: ('','#000000',''), - DEF_NAME: ('b','#0000ff',''), - KEYWORD: ('b','#a020f0',''), - SINGLEQUOTE: ('b','#bc8f8f',''), - SINGLEQUOTE_R: ('b','#bc8f8f',''), - SINGLEQUOTE_U: ('b','#bc8f8f',''), - DOUBLEQUOTE: ('b','#bc8f8f',''), - DOUBLEQUOTE_R: ('b','#bc8f8f',''), - DOUBLEQUOTE_U: ('b','#bc8f8f',''), - TRIPLESINGLEQUOTE: ('b','#bc8f8f',''), - TRIPLESINGLEQUOTE_R: ('b','#bc8f8f',''), - TRIPLESINGLEQUOTE_U: ('b','#bc8f8f',''), - TRIPLEDOUBLEQUOTE: ('b','#bc8f8f',''), - TRIPLEDOUBLEQUOTE_R: ('b','#bc8f8f',''), - TRIPLEDOUBLEQUOTE_U: ('b','#bc8f8f',''), - PAGEBACKGROUND: '#FFFFFF' - } - -defaultColors = dark - -def Usage(): - """ ------------------------------------------------------------------------------ - PySourceColor.py ver: %s ------------------------------------------------------------------------------ - Module summary: - This module is designed to colorize python source code. - Standalone: - This module will work from the command line with options. - This module will work with redirected stdio. - Imported: - This module can be imported and used directly in your code. ------------------------------------------------------------------------------ - Command line options: - -h, --help - Optional-> Display this help message. - -t, --test - Optional-> Will ignore all others flags but --profile - test all schemes and markup combinations - -p, --profile - Optional-> Works only with --test or -t - runs profile.py and makes the test work in quiet mode. - -i, --in, --input - Use any of these for the current dir (.,cwd) - Input can be file or dir. - Input from stdin use one of the following (-,stdin) - If stdin is used as input stdout is output unless specified. - -o, --out, --output - Optional-> output dir for the colorized source. - default: output dir is the input dir. - To output html to stdout use one of the following (-,stdout) - Stdout can be used without stdin if you give a file as input. - -c, --color - Optional-> null, mono, dark, dark2, lite, idle, pythonwin, viewcvs - default: dark - -s, --show - Optional-> Show webpage after creation. - default: no show - -m, --markup - Optional-> html, css - default: HTML ------------------------------------------------------------------------------ - Option usage: - # Test and show pages - python PySourceColor.py -t - # Test and only show profile results - python PySAourceColor.py -t -p - # Colorize all .py,.pyw files in cwdir you can also use: (.,cwd) - python PySourceColor.py -i . - # Using long options w/ = - python PySourceColor.py --in=c:/myDir/my.py --color=lite --show - # Using short options w/out = - python PySourceColor.py -i c:/myDir/ -c idle -m css - # Using any mix - python PySourceColor.py --in . -o=c:/myDir --show ------------------------------------------------------------------------------ - Stdio usage: - # Stdio using no options - python PySourceColor.py < c:/MyFile.py >> c:/tmp/MyFile.html - # Using stdin alone automatically uses stdout for output: (stdin,-) - python PySourceColor.py -i- < c:/MyFile.py >> c:/tmp/myfile.html - # Stdout can also be written to directly from a file instead of stdin - python PySourceColor.py -i c:/MyFile.py -m css -o- >> c:/tmp/myfile.html - # Stdin can be used as input , but output can still be specified - python PySourceColor.py -i- -o c:/pydoc.py.html -s < c:/Python22/my.py -_____________________________________________________________________________ -""" - print Usage.__doc__% (__version__) - sys.exit(1) - -###################################################### Command line interface - -def cli(): - """Handle command line args and redirections""" - try: - # try to get command line args - opts, args = getopt.getopt(sys.argv[1:], - "hsqtpi:o:c:m:",["help", "show", "quiet", "profile", "test", - "input=", "output=", "color=", "markup="]) - except getopt.GetoptError: - # on error print help information and exit: - Usage() - - # init some names - input = None - output = None - colorscheme = None - markup = 'html' - show = 0 - quiet = 0 - test = 0 - profile = 0 - # if we have args then process them - for o, a in opts: - if o in ["-h", "--help"]: - Usage() - sys.exit() - if o in ["-o", "--output", "--out"]: - output = a - if o in ["-i", "--input", "--in"]: - input = a - if input in [".", "cwd"]: - input = os.getcwd() - if o in ["-s", "--show"]: - show = 1 - if o in ["-q", "--quiet"]: - quiet = 1 - if o in ["-t", "--test"]: - test = 1 - if o in ["-m", "--markup"]: - markup = str(a) - if o in ["-p", "--profile"]: - profile = 1 - if o in ["-c", "--color"]: - try: - colorscheme = globals().get(a.lower()) - except: - traceback.print_exc() - Usage() - if test: - if profile: - import profile - profile.run('_test(show=%s, quiet=%s)'%(show,quiet)) - else: - # Parse this script in every possible colorscheme and markup - _test(show,quiet) - elif input in [None, "-", "stdin"] or output in ["-", "stdout"]: - # determine if we are going to use stdio - if input not in [None, "-", "stdin"]: - if os.path.isfile(input) : - path2stdout(input) - else: - raise PathError, 'File does not exists!' - else: - try: - if sys.stdin.isatty(): - raise InputError, 'Please check input!' - else: - if output in [None,"-","stdout"]: - str2stdout(sys.stdin.read()) - else: - str2file(sys.stdin.read(), output, show) - except: - traceback.print_exc() - Usage() - else: - if os.path.exists(input): - # if there was at least an input given we can proceed - convert(input, output, colorscheme, show, markup, quiet) - else: - raise PathError, 'File does not exists!' - Usage() - -######################################################### Simple markup tests - -def _test(show=0, quiet=0): - """Test the parser and most of the functions. - - There are 15 _test total(seven colorschemes in two diffrent markups, - and a str2file test. Most functions are tested by this. - """ - fi = sys.argv[0] - if not fi.endswith('.exe'):# Do not test if frozen as an archive - path2file(fi, '/tmp/null.html', null, show=show, quiet=quiet) - path2file(fi, '/tmp/null_css.html', null, show=show, - markup='css', quiet=quiet) - path2file(fi, '/tmp/mono.html', mono, show=show, quiet=quiet) - path2file(fi, '/tmp/mono_css.html', mono, show=show, - markup='css', quiet=quiet) - path2file(fi, '/tmp/lite.html', lite, show=show, quiet=quiet) - path2file(fi, '/tmp/lite_css.html', lite, show=show, - markup='css', quiet=quiet) - path2file(fi, '/tmp/dark.html', dark, show=show, quiet=quiet) - path2file(fi, '/tmp/dark_css.html', dark, show=show, - markup='css', quiet=quiet) - path2file(fi, '/tmp/dark2.html', dark2, show=show, quiet=quiet) - path2file(fi, '/tmp/dark2_css.html', dark2, show=show, - markup='css', quiet=quiet) - path2file(fi, '/tmp/idle.html', idle, show=show, quiet=quiet) - path2file(fi, '/tmp/idle_css.html', idle, show=show, - markup='css', quiet=quiet) - path2file(fi, '/tmp/viewcvs.html', viewcvs, show=show, quiet=quiet) - path2file(fi, '/tmp/viewcvs_css.html', viewcvs, show=show, - markup='css', quiet=quiet) - path2file(fi, '/tmp/pythonwin.html', pythonwin, show=show, - quiet=quiet) - path2file(fi, '/tmp/pythonwin_css.html', pythonwin, show=show, - markup='css', quiet=quiet) - teststr=r'''"""This is a test of decorators and other things""" -@whatever(arg,arg2) -def my(arg,arg2): - """This is a docstring""" - print 'fgfdgfgfdgfdgfdgfdg' -# Comment before decorators -@staticmethod## Double comments are kewl -def hgfghf(): - """this is a docstring""" - """Second doc --> - -=================== -Just for giggles -=================== -## comments in a second docstring ->>> This = 123 -""" - ##double comments------------> - w = {'sdfdfdf':'1','dfdfdf':'2'} - p = r'dfgfg\nfgfdgfgfdfgfg\dfgfgfdgfdgfd\dfgfd\g' - j = p.split(" ") - Ur"""asdasdasdsadsadsadsadsadsadsadsa""" - U"""asdasdsadsadsadsadsad""" - r'sdfsdfdsfdsfdsfdfdf' - rU"sdfdsfdsfdsfdsfdfd" - u'gdfgfdgfgfdgfgfgfdg' - import os - os.path.split(r'/tmp/null') - e = u'ugdg'+r"""ytryt=""" - g = """dfgfdgfdgdfgdfgdfgfdg"""+"ghgfhgfhghghh" - # who said that - d = 'fgfdgfdgfdgfdgfdgfgfgfgfdgfdgfgfdgfdgfdgfddffdgfdgfdg\ -dfgdfgfdgfdgdfgdfgfgfgfdgfgdgdfgfg' - print d''' - htmlPath = os.path.abspath('/tmp/strtest.html') - str2file(teststr, htmlPath, colors=dark, show=show) - _printinfo(" wrote %s" % htmlPath, quiet) - else: - Usage() - -####################################################### User level funtctions - -def str2stdout(sourcestring, colors=None, form=None): - """Converts a code(string) to colorized HTML. Writes to stdout. - - form='code',or'snip' (for "
yourcode
" only) - colors=null,mono,lite,dark,idle,or pythonwin - """ - Parser(sourcestring, colors).format(form) - -def path2stdout(sourcepath, colors=None, form=None): - """Converts code(file) to colorized HTML. Writes to stdout. - - form='code',or'snip' (for "
yourcode
" only) - colors=null,mono,lite,dark,idle,or pythonwin - """ - sourcestring = open(sourcepath).read() - Parser(sourcestring, colors, title=sourcepath).format(form) - -def str2html(sourcestring, colors=None, form=None): - """Converts a code(string) to colorized HTML. Returns an HTML string. - - form='code',or'snip' (for "
yourcode
" only) - colors=null,mono,lite,dark,idle,or pythonwin - """ - stringIO = cStringIO.StringIO() - Parser(sourcestring, colors, out=stringIO).format(form) - stringIO.seek(0) - return stringIO.read() - -def str2file(sourcestring, outfile, colors=None, show=0): - """Converts a string to a file. - - makes no attempt at correcting bad pathnames - """ - html = str2html(sourcestring, colors) - f = open(outfile,'wt') - f.writelines(html) - f.close() - if show: - showpage(outfile) - -def path2html(sourcepath, colors=None, form=None): - """Converts code(file) to colorized HTML. Returns an HTML string. - - form='code',or'snip' (for "
yourcode
" only) - colors=null,mono,lite,dark,idle,or pythonwin - """ - stringIO = cStringIO.StringIO() - sourcestring = open(sourcepath).read() - Parser(sourcestring, colors, title=sourcepath, out=stringIO).format(form) - stringIO.seek(0) - return stringIO.read() - -def convert(source, outdir=None, colors=None, - show=0, markup='html', quiet=0): - """Takes a file or dir as input and places the html in the outdir. - - If outdir is none it defaults to the input dir - """ - c=0 - # If it is a filename then path2file - if not os.path.isdir(source): - if os.path.isfile(source): - c+=1 - path2file(source, outdir, colors, show, markup, quiet) - else: - raise PathError, 'File does not exists!' - # If we pass in a dir we need to walkdir for files. - # Then we need to colorize them with path2file - else: - fileList = walkdir(source) - if fileList != None: - # make sure outdir is a dir - if outdir != None: - if os.path.splitext(outdir)[1] != '': - outdir = os.path.split(outdir)[0] - for item in fileList: - c+=1 - path2file(item, outdir, colors, show, markup, quiet) - _printinfo('Completed colorizing %s files.'%str(c), quiet) - else: - _printinfo("No files to convert in dir.", quiet) - -def path2file(sourcePath, out=None, colors=None, show=0, - markup='html', quiet=0): - """ Converts python source to html file""" - # If no outdir is given we use the sourcePath - if out == None:#this is a guess - htmlPath = sourcePath + '.html' - else: - # If we do give an out_dir, and it does - # not exist , it will be created. - if os.path.splitext(out)[1] == '': - if not os.path.isdir(out): - os.makedirs(out) - sourceName = os.path.basename(sourcePath) - htmlPath = os.path.join(out,sourceName)+'.html' - # If we do give an out_name, and its dir does - # not exist , it will be created. - else: - outdir = os.path.split(out)[0] - if not os.path.isdir(outdir): - os.makedirs(outdir) - htmlPath = out - htmlPath = os.path.abspath(htmlPath) - # Open the text and do the parsing. - source = open(sourcePath).read() - Parser(source, colors, sourcePath, open(htmlPath, 'wt'), - markup).format() - _printinfo(" wrote %s" % htmlPath, quiet) - if show: - # load HTML page into the default web browser. - showpage(htmlPath) - return htmlPath - -def walkdir(dir): - """Return a list of .py and .pyw files from a given directory. - - This function can be written as a generator Python 2.3, or a genexp - in Python 2.4. But 2.2 and 2.1 would be left out.... - """ - # Get a list of files that match *.py* - GLOB_PATTERN = os.path.join(dir, "*.[p][y]*") - pathlist = glob.glob(GLOB_PATTERN) - # Now filter out all but py and pyw - filterlist = [x for x in pathlist - if x.endswith('.py') - or x.endswith('.pyw')] - if filterlist != []: - # if we have a list send it - return filterlist - else: - return None - -def showpage(path): - """Helper function to open webpages""" - try: - webbrowser.open_new(os.path.abspath(path)) - except: - traceback.print_exc() - -def _printinfo(message, quiet): - """Helper to print messages""" - if not quiet: - print message - -########################################################### Custom Exceptions - -class PySourceColorError(Exception): - pass# base for custom error - -class PathError(PySourceColorError): - pass# custom error - -class InputError(PySourceColorError): - pass# custom error - -########################################################## Python code parser - -class Parser: - - """MoinMoin python parser heavily chopped :)""" - - def __init__(self, raw, colors=None, title='', out=sys.stdout, - markup='html', header=0, footer=0): - """Store the source text & set some flags""" - if colors == None: - colors = defaultColors - self.raw = raw.expandtabs().strip() - self.title = os.path.basename(title) - self.out = out - self.argFlag = 0 - self.classFlag = 0 - self.defFlag = 0 - self.decoratorFlag = 0 - self.markup = markup.upper() - self.colors = colors - self.header = header - self.footer = footer - - def format(self, form=None): - """Parse and send the colorized source""" - if form in ('snip','code'): - self.addEnds = 0 - else: - self.addEnds = 1 - # Store line offsets in self.lines - self.lines = [0, 0] - pos = 0 - - # Gather lines - while 1: - pos = self.raw.find('\n', pos) + 1 - if not pos: break - self.lines.append(pos) - self.lines.append(len(self.raw)) - - # Wrap text in a filelike object - self.pos = 0 - text = cStringIO.StringIO(self.raw) - - # Html start - if self.addEnds: - self._doPageStart() - else: - self._doSnippetStart() - - # Parse the source and write out the results. - ## Tokenize calls the __call__ - ## function for each token till done. - try: - tokenize.tokenize(text.readline, self) - except tokenize.TokenError, ex: - msg = ex[0] - line = ex[1][0] - self.out.write("

ERROR: %s

%s\n"% - (msg, self.raw[self.lines[line]:])) - traceback.print_exc() - - # Html end - if self.addEnds: - self._doPageEnd() - else: - self._doSnippetEnd() - - def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line): - """Token handler. Order is important do not rearrange.""" - style = '' - # Calculate new positions - oldpos = self.pos - newpos = self.lines[srow] + scol - self.pos = newpos + len(toktext) - - # Handle newlines - if toktype in (token.NEWLINE, tokenize.NL): - self.out.write('\n') - return - - # Send the original whitespace, if needed - if newpos > oldpos: - self.out.write(self.raw[oldpos:newpos]) - - # Skip indenting tokens - if toktype in (token.INDENT, token.DEDENT): - self.pos = newpos - return - - # Map token type to a color group - if token.LPAR <= toktype and toktype <= token.OP: - # Trap decorators py2.4> - if toktext == '@': - toktype = DECORATOR - # Set a flag if this was the decorator start so - # the decorator name and arguments can be identified - self.decoratorFlag = self.argFlag = 1 - else: - # Find the start for arguments - if toktext == '(' and self.argFlag: - self.argFlag = 2 - # Find the end for arguments - elif toktext == ':': - self.argFlag = 0 - toktype = token.OP - - elif toktype == token.NAME and keyword.iskeyword(toktext): - toktype = KEYWORD - # Set a flag if this was the class / def start so - # the class / def name and arguments can be identified - if toktext =='class': - self.classFlag = self.argFlag = 1 - elif toktext == 'def': - self.defFlag = self.argFlag = 1 - - # Maps the color if the last token set flags - # class, def, decorator name - elif self.classFlag or self.defFlag or self.decoratorFlag: - if self.classFlag: - toktype = CLASS_NAME - self.classFlag = 0 - elif self.defFlag: - toktype = DEF_NAME - self.defFlag = 0 - elif self.decoratorFlag: - toktype = DECORATOR_NAME - self.decoratorFlag = 0 - - # Seperate the diffrent comment types - # Order of evaluation is important do not change. - elif toktype == token.STRING: - text = toktext.lower() - # TRIPLE DOUBLE QUOTE's - if (text[:3].lower() == '"""'): - toktype = TRIPLEDOUBLEQUOTE - elif (text[:4] == 'r"""' or - text[:5] == 'ru"""'): - toktype = TRIPLEDOUBLEQUOTE_R - elif (text[:4] == 'u"""' or - text[:5] == 'ur"""'): - toktype = TRIPLEDOUBLEQUOTE_U - # DOUBLE QUOTE's - elif (text[:1] == '"'): - toktype = DOUBLEQUOTE - elif (text[:2] == 'r"' or - text[:3] == 'ru"'): - toktype = DOUBLEQUOTE_R - elif (text[:2] == 'u"' or - text[:3] == 'ur"'): - toktype = DOUBLEQUOTE_U - # TRIPLE SINGLE QUOTE's - elif (text[:3] == "'''"): - toktype = TRIPLESINGLEQUOTE - elif (text[:4] == "r'''" or - text[:5] == "ru'''"): - toktype = TRIPLESINGLEQUOTE_R - elif (text[:4] == "u'''" or - text[:5] == "ur'''"): - toktype = TRIPLESINGLEQUOTE_U - # SINGLE QUOTE's - elif (text[:1] == "'"): - toktype = SINGLEQUOTE - elif (text[:2] == "r'" or - text[:3] == "ru'" ): - toktype = SINGLEQUOTE_R - elif (text[:2] == "u'" or - text[:3] == "ur'" ): - toktype = SINGLEQUOTE_U - - # Seperate the diffrent comment types - elif toktype == tokenize.COMMENT: - if toktext[:2] == "##": - toktype = DOUBLECOMMENT - - # Seperate errors from decorators - elif toktype == token.ERRORTOKEN: - # trap decorators...\n') - - def _doSnippetEnd(self): - # Start of html snippet - self.out.write('\n') - - ######################################################## markup selectors - - def _doPageStart(self): - getattr(self, '_do%sStart'%(self.markup))() - - def _doPageHeader(self): - getattr(self, '_do%sHeader'%(self.markup))() - - def _doPageFooter(self): - getattr(self, '_do%sFooter'%(self.markup))() - - def _doPageEnd(self): - getattr(self, '_do%sEnd'%(self.markup))() - - #################################################### color/style retrival - - def _getTags(self, key): - # style tags - return self.colors.get(key, self.colors[NAME])[0] - - def _getForeColor(self, key): - # get text foreground color, if not set to black - color = self.colors.get(key, self.colors[NAME])[1] - if color[:1] != '#': - color = '#000000' - return color - - def _getBackColor(self, key): - # get text background color - return self.colors.get(key, self.colors[NAME])[2] - - def _getPageColor(self): - # get page background color - return self.colors.get(PAGEBACKGROUND, '#FFFFFF') - - def _getStyle(self, key): - # get the token style from the color dictionary. - rawcolor = self.colors.get(key, self.colors[NAME]) - return rawcolor - - def _getMarkupClass(self, key): - return MARKUPDICT.get(key, MARKUPDICT[NAME]) - - def _setDocumentCreatedBy(self): - return '\n'%( - __title__,__version__,time.ctime()) - - ################################################### HTML markup functions - - def _doHTMLStart(self): - # Start of html page - self.out.write('\n') - self.out.write('%s\n'%(self.title)) - self.out.write(self._setDocumentCreatedBy()) - self.out.write('\n') - # Get background - color = self._getPageColor() - self.out.write('\n'%color) - # Write a little info at the top. - if self.header: - self._doPageHeader() - self.out.write('
')
-
-    def _sendHTMLText(self, toktype, toktext):
-        # If it is an error set a red box around the bad tokens
-        # older browsers will ignore it
-        if toktype == ERRORTOKEN:
-            style = ' style="border: solid 1.5pt #FF0000;"'
-        else:
-            style = ''
-        tags, color = self._getStyle(toktype)[:2]
-        tagstart=[]
-        tagend=[]
-        # check for styles and set them if needed.
-        if 'b' in tags:#Bold
-            tagstart.append('')
-            tagend.append('')
-        if 'i' in tags:#Italics
-            tagstart.append('')
-            tagend.append('')
-        if 'u' in tags:#Underline
-            tagstart.append('')
-            tagend.append('')
-        # HTML tags should be paired like so : Doh!
-        tagend.reverse()
-        starttag="".join(tagstart)
-        endtag="".join(tagend)
-        sendtext = cgi.escape(toktext)
-        # send text
-        ## Output optimization
-        # skip font tag if black text, but styles will still be sent. (b,u,i)
-        if color !='#000000':
-            startfont = ''%(color, style)
-            endfont = ''
-        else:
-            startfont, endfont = ('','')
-        self.out.write(''.join([startfont,starttag,sendtext,endtag,endfont]))
-        return
-
-    def _doHTMLHeader(self):
-        # Optional
-        color = self._getForeColor(token.NAME)
-        self.out.write(' 

#%s %s

\n'% - (color, self.title, time.ctime())) - - def _doHTMLFooter(self): - # Optional - color = self._getForeColor(token.NAME) - self.out.write('

#%s %s

\n'% - (color, self.title,time.ctime())) - - def _doHTMLEnd(self): - # End of html page - self.out.write('
\n') - # Write a little info at the bottom - if self.footer: - self._doPageFooter() - self.out.write('\n') - - #################################################### CSS markup functions - - def _getCSSStyle(self, key): - # Get the tags and colors from the dictionary - tags, forecolor, backcolor = self._getStyle(key) - style=[] - if tags: - if 'b' in tags:# Bold - style.append('font-weight:bold;') - if 'i' in tags:# Italic - style.append('font-style:italic;') - if 'u' in tags:# Underline - style.append('text-decoration:underline;') - style.append('color:%s;'% forecolor) - if backcolor: - style.append('background-color:%s;'%backcolor) - return (self._getMarkupClass(key),''.join(style)) - - def _doCSSStart(self): - # Start of css/html page - self.out.write('') - self.out.write('%s\n'%(self.title)) - self.out.write(self._setDocumentCreatedBy()) - self.out.write('\n') - self.out.write('\n\n\n') - # Write a little info at the top. - if self.header: - self._doPageHeader() - self.out.write('
')
-
-    def _sendCSSText(self, toktype, toktext):
-        ##Output optimization
-        # send text plain if black and no tags.
-        tags, color = self._getStyle(toktype)[:2]
-        if color == '#000000' and not tags:
-            startspan, endspan = ('','')
-        else:
-            startspan = ''%(self._getMarkupClass(toktype))
-            endspan = ''
-        sendtext = cgi.escape(toktext)
-        self.out.write(''.join([startspan,sendtext,endspan]))
-        return
-
-    def _doCSSHeader(self):
-        # Optional
-        self.out.write('

#%s %s

\n'% - (self.title, time.ctime())) - - def _doCSSFooter(self): - # Optional - self.out.write('

#%s %s

\n'% - (self.title, time.ctime())) - - def _doCSSEnd(self): - # End of css/html page - self.out.write('
\n') - # Write a little info at the bottom - if self.footer: - self._doPageFooter() - self.out.write('\n') - -############################################################################# - -if __name__ == '__main__': - cli() - -############################################################################# -# 2004 M.E.Farmer Jr. -# Python license diff --git a/lib/__init__.py b/lib/__init__.py deleted file mode 100644 index 792d600..0000000 --- a/lib/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/lib/captchas/__init__.py b/lib/captchas/__init__.py deleted file mode 100644 index 792d600..0000000 --- a/lib/captchas/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/lib/captchas/music.py b/lib/captchas/music.py deleted file mode 100644 index d6d775b..0000000 --- a/lib/captchas/music.py +++ /dev/null @@ -1,106 +0,0 @@ -"""music - a Python captcha for musicians""" - -__version__ = "1.0" -__license__ = "Python" -__copyright__ = "Copyright 2005 by Christoph Zwerschke" - -from random import randint - -class Captcha: - """A captcha for Python programmers. - - Methods: - create(width): create the HTML form fields - width: the length of the input field - check(input, id): check the input from the user - input, id: use the result values from the form - """ - - size = (6,4) - colors = ['black', 'white'] - names = 'do-re-mi-fa-so-la-ti-do-re'.split('-') - - def create(self, width=6, id=None): - if id: - notes = map(int, id) - else: - notes = [] - for i in range(width): - notes.append(randint(0, 8)) - id = ''.join(map(str, notes)) - s = ['' - '' - '
'] - s.append('Please enter the melody' - ' in solfege notation:') - s.append('
') - s.append('(e.g., "doremiresola")') - s.append('
' - % ((width*2+1)*self.size[0], 12*self.size[1])) - s.append(self.__notes(notes)) - s.append('
 ' - ' =  ') - s.append('' % ((width*3,)*2)) - s.append('' % id) - s.append('
') - return '\n'.join(s) - - def check(self, input, id): - input = input.lower() - input = filter(lambda c: c.isalpha(), input) - solution = self.__solfege(id) - return input == solution - - def __solfege(self, id): - if not isinstance(id, str): - return None - solfege = [] - for n in id: - try: - n = int(n) - except: - n = -1 - if not 0 <= n <= 8: - return None - solfege.append(self.names[n]) - return ''.join(solfege) - - def __notes(self, notes): - s = ['
'] - for row in range(10): - col = 0 - for note in notes: - s.append(self.__note(col, row)) - col += 1 - s.append(self.__note(col, row, row+note-8 in (0,1))) - col += 1 - s.append(self.__note(col, row)) - # Workaround for MSIE-Bug: - s.append('' - % ((len(notes)*2+1)*self.size[0], - 10*self.size[1], self.colors[1])) - s.append('
') - return '\n'.join(s) - - def __note(self, col, row, filled=False): - s = ['position:absolute', - 'left:%dpx' % (self.size[0]*col), - 'top:%dpx' % (self.size[1]*row), - 'width:%dpx' % self.size[0], - 'height:%dpx' % self.size[1], - 'border-width:1px 0px', - #'border-color:blue', - 'border-top-color:%s' % self.colors[ - not(row % 2 or filled)], - 'border-bottom-color:%s' % self.colors[ - not(row % 2 or filled) or row==9], - 'border-style:solid', - 'background-color:%s' % self.colors[not filled]] - s = ';'.join(s) - s = '' % s - return s diff --git a/lib/captchas/python.py b/lib/captchas/python.py deleted file mode 100644 index 4e1c52d..0000000 --- a/lib/captchas/python.py +++ /dev/null @@ -1,65 +0,0 @@ -"""python - a Python captcha for Python programmers""" - -__version__ = "1.0" -__license__ = "Python" -__copyright__ = "Copyright 2005 by Christoph Zwerschke" - -from random import randint, choice - -class Captcha: - """A captcha for Python programmers. - - Methods: - create(width): create the HTML form fields - width: the length of the input field - check(input, id): check the input from the user - input, id: use the result values from the form - """ - - alphabet = 'pythonic' - - def create(self, width=6, id=None): - if not id: - code = randint(1, 7) - word = [] - for i in range(width): - word.append(choice(self.alphabet)) - word = ''.join(word) - id = word + str(code) - expr = self.__expr(id) - s = ['

Have a look' - ' at the following expression:

'] - s.append('
%s
' % expr) - s.append('

Now please enter the result: ') - s.append('

' % ((width,)*2)) - s.append('' % id) - return '\n'.join(s) - - def check(self, input, id): - expr = self.__expr(id) - input = input.strip().strip('\'"') - try: - solution = eval(expr) - except: - solution = None - return input == solution - - def __expr(self, id): - if not isinstance(id, str): - return None - code = id[-1:] - try: - code = int(code) - except: - code = 0 - if not 1 <= code <= 8: - return None - word = id[:-1] - for c in word: - if c not in self.alphabet: - return None - return ("''.join(map(lambda c:" - "'%s'[\n'%s'.index(c)^%d],'%s'))" - % (self.alphabet, self.alphabet, code, word)) diff --git a/lib/common.py b/lib/common.py deleted file mode 100644 index 3edea33..0000000 --- a/lib/common.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -These are routines common to the entire application. -We keep them separate to avoid circular dependencies. -""" - -import cgi -import re -import inspect -import pprint as pprint_module -pprint = pprint_module.pprint -import os - -__all__ = ['canonicalName', 'htmlEncode', 'guessURLName', 'dprint', - 'pprint', 'guessTitle', 'dedent'] - -_canonicalNameRE = re.compile(r'[^a-z0-9]') -def canonicalName(name): - """Turns a wiki name into its canonical form. - - This may only have letters, numbers and hyphens in it. - """ - return str(_canonicalNameRE.sub('', name.lower())) - -_urlNameRE = re.compile(r'[^a-z0-9 ]') -def guessURLName(name): - name = str(_urlNameRE.sub('', name.lower())) - return name.replace(' ', '-') - -def guessTitle(name): - """Turns a canonical name into a title.""" - return ' '.join([w.capitalize() for w in name.split('-')]) - -def htmlEncode(val, cgiEscape=cgi.escape): - return cgiEscape(val, 1) - -def dprint(*args, **kw): - caller_frame = inspect.stack()[1] - caller_name = caller_frame[3] - caller_line = caller_frame[2] - caller_filename = caller_frame[1] - caller_module = os.path.splitext(os.path.basename(caller_filename))[0] - del caller_frame - print "%s from %s.%s:%s %s" % ( - '-'*10, caller_module, caller_name, caller_line, '-'*10) - for arg in args: - if isinstance(arg, (str, unicode)): - print arg, - else: - print pprint_module.pformat(arg) - items = kw.items() - items.sort() - for name, value in items: - print name, pprint_module.pformat(value) - -try: - from textwrap import dedent -except ImportError: # Fallback for Python < 2.3 - def dedent(t): - lines = text.expandtabs().split('\n') - margin = None - for line in lines: - content = line.lstrip() - if not content: - continue - indent = len(line) - len(content) - if margin is None: - margin = indent - else: - margin = min(margin, indent) - if margin is not None and margin > 0: - for i in range(len(lines)): - lines[i] = lines[i][margin:] - return '\n'.join(lines) diff --git a/lib/config/__init__.py b/lib/config/__init__.py deleted file mode 100644 index 792d600..0000000 --- a/lib/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/lib/config/configwrapper.py b/lib/config/configwrapper.py deleted file mode 100644 index 53e4414..0000000 --- a/lib/config/configwrapper.py +++ /dev/null @@ -1,263 +0,0 @@ -import os -import iniparser - -class _ConfigParser(iniparser.INIParser): - - """ - An internal subclass of the abstract ini parser, which saves - everything into one dictionary. - """ - - def __init__(self, allow_empty_sections=True): - self.allow_empty_sections = allow_empty_sections - iniparser.INIParser.__init__(self) - - def reset(self): - self.data = {} - self.key_order = {} - self.section_order = [] - iniparser.INIParser.reset(self) - - def assignment(self, name, content): - section = self.data.setdefault(self.section, {}) - name = clean_name(name) - section.setdefault(name, []).append(content) - self.key_order.setdefault(self.section, []).append(name) - - def new_section(self, section): - section = section.strip().lower() - if not section or section == 'general': - section = None - self.section = section - if not section in self.section_order: - self.section_order.append(section) - -class NoDefault: - pass - -def clean_name(name): - name = name.strip().lower() - for char in ' \t_-': - name = name.replace(char, '') - return name - -class Config(object): - - """ - A configuration object. Acts a little like a dictionary, but not - really. - - Configuration can be nested with ``[section]`` markers, or can be - flat (everything globally available). If in sections then - ``section.key`` is used, otherwise just ``key``. - """ - - # If False, then no global sections are allowed - allow_empty_sections = True - - def __init__(self, filename, file=None, - allow_empty_sections=NoDefault): - if allow_empty_sections is not NoDefault: - self.allow_empty_sections = allow_empty_sections - self.filename = filename - p = _ConfigParser(allow_empty_sections=self.allow_empty_sections) - if file is None: - p.load(filename) - else: - p.loadfile(file) - self.raw_data = p.data - self._section_order = p.section_order - self._key_order = p.key_order - self.name = os.path.splitext(os.path.basename(filename))[0] - - def keys(self, section=NoDefault): - if section is NoDefault: - all = [] - for section_name in self._section_order: - for key_name in self._key_order[section_name]: - if section_name: - key_name = '%s|%s' % (section_name, key_name) - all.append(key_name) - return all - else: - try: - return self._key_order[section] - except KeyError: - raise KeyError( - "Section [%s] not found (from sections %r)" - %(section, self._key_order.keys())) - - def sections(self): - return self.raw_data.keys() - - def getraw(self, key, section=None): - """ - Mostly for internal use, returns a list of all matching keys. - """ - if '|' in key: - assert section is None, ( - "Cannot have | in a key (%r) and an explicit section (%r)" - % (key, section)) - section, key = key.split('|', 1) - section = self.clean_section(section) - try: - return self.raw_data[section][key] - except KeyError: - return () - - def clean_section(self, section_name): - if not section_name or section_name == 'general': - return None - return section_name.lower() - - def get(self, key, default=None, section=None): - """ - Get a single value, returning `default` if none found. - """ - value = self.getraw(key, section=section) - if not value: - return default - else: - return value[0] - - def getlist(self, key, default=(), section=None): - """ - Get a list of all matching keys. Example:: - - foo = bar - foo = baz - - Then:: - - >>> config.getlist('foo') - ['bar', 'baz'] - """ - value = self.getraw(key, section=section) - if not value: - return default - else: - return value - - true_values = ['t', 'true', '1', 'y', 'yes', 'on'] - false_values = ['f', 'false', '0', 'n', 'no', 'off'] - - def getbool(self, key, default=False, section=None): - """ - Get a single boolean value. Boolean values are things - like ``true``, ``false``, etc. - """ - value = self.getraw(key, section=section) - if not value: - return default - value = value[0] - if isinstance(value, (str, unicode)): - value = value.lower() - if value in self.true_values: - return True - elif value in self.false_values: - return False - else: - raise ValueError( - "Yes/No value expected for %s (got %r)" - % (key, vaule)) - else: - return value - - def getint(self, key, default=None, section=None): - """ - Get an integer value. - """ - v = self.get(key, default=default, section=section) - if v is None: - return v - else: - return int(v) - - def getinlinelist(self, key, default=(), section=None): - """ - Get a list, where the list is defined like: - - foo = bar, baz - - Gives:: - - >>> config.getinlinelist('foo') - ['bar', 'baz'] - """ - result = [] - for item in self.getlist(key, default, section): - item = item.strip() - if not item: - continue - if ',' in item: - result.extend([i.strip() for i in item.split(',')]) - else: - result.append(item) - return result - - def getstartswith(self, startswith, section): - """ - Returns a list of [(key, value)] for all keys in the section that - start with startswith. - """ - result = [] - for key in self.keys(section=section): - if not key.startswith(startswith): - continue - for value in self.getraw(key, section=section): - result.append((key, value)) - return result - - def __repr__(self): - return '<%s filename=%s>' % ( - self.__class__.__name__, self.filename) - - def __str__(self): - data = [] - data.append('%s from %s:' % ( - self.__class__.__name__, self.filename)) - for name, value_list in self.raw_data.get(None, {}).items(): - for value in value_list: - data.append('%s: %s' % (name, value)) - for section_name, value_dict in self.raw_data.items(): - if section_name is None: - continue - data.append('[%s]' % section_name) - for name, value_list in value_dict.items(): - for value in value_list: - data.append('%s: %s' % (name, value)) - return '\n'.join(data) - -def load_options(parser, options, config, if_exists=True): - """ - Given a parser object and the parsed options and a configuration - object or filename, this will load the configuration into the - parsed options. - """ - types = {} - dests = {} - aliases = {} - for option in parser.option_list: - if not option.dest: - # Some builtin options, like --help - continue - name = clean_name(option.dest) - dests[name] = option.dest - if option.action in ('store_true', 'store_false'): - types[name] = 'getbool' - else: - types[name] = 'get' - for lst in option._long_opts, option._short_opts: - for opt in lst: - aliases[clean_name(opt)] = name - if isinstance(config, str): - if if_exists and not os.path.exists(config): - return - config = Config(config) - for config_key in config.keys(): - name = aliases.get(config_key, config_key) - if name not in dests: - raise ValueError( - "Bad configuration file: option %r unexpected" % config_key) - value = getattr(config, types[name])(config_key) - setattr(options, dests[name], value) diff --git a/lib/config/iniparser.py b/lib/config/iniparser.py deleted file mode 100644 index eb8b7a4..0000000 --- a/lib/config/iniparser.py +++ /dev/null @@ -1,241 +0,0 @@ -""" -A parser for .ini-syntax files. INIParser should be subclassed to -create a data structure from the file. See INIParser for more -""" - -class ParseError(Exception): - - def __init__(self, message, filename=None, lineno=None, column=None, line=None): - # Note: right now column is not used, but in some contexts it - # could be used to print out a line like: - # error, must be an integer: - # a = one - # ^ - self.message = message - self.filename = filename - self.lineno = lineno - self.column = column - self.line = line - - def __str__(self): - msg = self.message - if self.filename and self.lineno: - msg += ' in %s:%s' % (self.filename, self.lineno) - elif self.filename: - msg += ' in %s' % self.filename - elif self.lineno: - msg += ' at line %s' % self.lineno - return msg - -class INIParser: - - """ - A parser for .ini-syntax files. - - Implements all features I know of in .ini files: - * sections with a [ in the first column - * assignment via a=b or a: b - * rfc822-style continuation lines (i.e., start the next line - with indentation to make it a continuation). - * ; or # for comments - - This class should be subclassed. Subclasses may only need to - implement the .assignment() method. You may want to use the - .section attribute, which holds the current section (or None if no - section has been defined yet). - - Use .parse_error(message) when you encounter a problem; this will - create an exception that will note the filename and line in which - the problem occurs. - """ - - def __init__(self): - self.reset() - - def reset(self): - pass - - def load(self, filename, encoding='ascii'): - fileobj = open(filename, 'rb') - self.loadfile(fileobj, filename=filename, encoding=encoding) - fileobj.close() - - def loadfile(self, fileobj, filename=None, encoding='ascii'): - self.start_lineno = 0 - if filename is None: - filename = getattr(fileobj, 'name', None) - self.filename = filename - # lineno is what we are parsing, start_lineno is the last - # assignment we processed (for multi-line assignments) - self.start_lineno = 0 - self.lineno = 0 - self.encoding = encoding - def strip_newline(l): - if l.endswith('\n'): - return l[:-1] - else: - return l - self.stream = [strip_newline(l) for l in fileobj.readlines()] - self.process_file() - del self.filename - del self.encoding - del self.lineno - - def loadstring(self, string, filename=None): - self.stream = string.splitlines() - self.filename = filename - self.start_lineno = 0 - self.lineno = 0 - self.encoding = None - self.process_file() - del self.stream - del self.filename - del self.lineno - del self.encoding - - def process_file(self): - self.section = None - last_name = None - accumulated_content = None - for line in self.stream: - self.lineno += 1 - # @@: should catch encoding error: - if self.encoding: - line = line.decode(self.encoding) - end_assignment = False - if not line.strip(): # empty line - if last_name: - self.process_assignment( - last_name, - accumulated_content) - last_name = accumulated_content = None - self.start_lineno = self.lineno - continue - elif line[0] in (' ', '\t'): # continuation line - if not last_name: - self.error_continuation_without_assignment(line) - else: - accumulated_content.append(line) - continue - elif self.get_section(line) is not None: # section line - if last_name: - self.process_assignment( - last_name, - accumulated_content) - last_name = accumulated_content = None - self.start_lineno = self.lineno - self.new_section(self.get_section(line)) - elif self.get_comment(line) is not None: # comment line - if last_name: - self.process_assignment( - last_name, - accumulated_content) - last_name = accumulated_content = None - self.start_lineno = self.lineno - self.add_comment(self.get_comment(line)) - else: # normal assignment - if last_name: - self.process_assignment( - last_name, - accumulated_content) - last_name, accumulated_content = self.split_name_value(line) - self.start_lineno = self.lineno - if last_name: - self.process_assignment( - last_name, - accumulated_content) - - def split_name_value(self, line): - colon_pos = line.find(':') - equal_pos = line.find('=') - if colon_pos == -1 and equal_pos == -1: - self.error_missing_equal(line) - return None - if (colon_pos == -1 - or (equal_pos != -1 and equal_pos < colon_pos)): - pos = equal_pos - else: - pos = colon_pos - return line[:pos], [line[pos+1:]] - - def get_comment(self, line): - """ - Returns None if not a comment - """ - line = line.lstrip() - if line.startswith(';') or line.startswith('#'): - return line[1:] - return None - - def get_section(self, line): - """ - Returns None if not a section - """ - line = line.strip() - if not line.startswith('['): - return None - if not line.endswith(']'): - self.error_section_without_end_bracket(line) - return None - return line[1:-1] - - def process_assignment(self, name, accumulated_content): - content = '\n'.join([l.lstrip() for l in accumulated_content]) - self.assignment(name.strip(), content) - - def assignment(self, name, content): - raise NotImplementedError - - def new_section(self, section): - if not section: - self.error_no_section_name() - self.section = section - - def add_comment(self, comment): - pass - - def error_continuation_without_assignment(self, line): - self.parse_error('Invalid indentation', line) - - def error_section_without_end_bracket(self, line): - self.parse_error('Invalid section (must end with ])', line) - - def error_missing_equal(self, line): - self.parse_error( - 'Lines should look like "name=value" or "name: value"', - line) - - def error_no_section_name(self): - self.parse_error( - 'Empty section name ([])') - - def parse_error(self, msg, line=None): - raise ParseError( - msg, - filename=self.filename, - lineno=self.lineno, - line=line) - -class BasicParser(INIParser): - - """ - A simple subclass of INIParser; creates a nested data structure - like ``{'section_name': {'variable': ['values']}}`` - - Usage:: - - >>> p = BasicParser() - >>> p.load('config.ini') - >>> data = p.data - """ - - def reset(self): - self.data = {} - INIParser.reset(self) - - def assignment(self, name, content): - if not self.section: - self.parse_error( - 'Assignments can only occur inside sections; no section has been defined yet') - section = self.data.setdefault(self.section, {}) - section.setdefault(name, []).append(content) diff --git a/lib/config/inischema.py b/lib/config/inischema.py deleted file mode 100644 index 683eb16..0000000 --- a/lib/config/inischema.py +++ /dev/null @@ -1,431 +0,0 @@ -""" -.ini file schemas - -You can define a schema for your .ini configuration files, which -defines the types and defaults for the values. You can also define -a catchall attribute. - -TODO ----- - -Currently, this does not deal with sections at all, and sections are -not allowed. That wil take some more thought, as the schemas will -probably be per-section (though one section may inherit from another, -or defaults may be inherited, etc, which should be allowed for). - -Documentation isn't kept track of, nor is it generated. It should -possible to indicate with ``opt(help=...)``, and as an option -to ``INISchema.ini_repr()`` you should be able to put in documentation -in comments. - -Comments aren't kept track of. - -Key order isn't kept track of. - -Minimal .ini files should be possible to generate -- only generating -keys when the default doesn't match the actual value. - -There should be a way to check that the entire config is loaded, and -there are no missing values (options which weren't set and have no -default). - -Usage ------ - -:: - - class VHostSchema(INISchema): - - server_name = opt() - port = optint(default=80) - # optlist means this can show up multiple times: - server_alias = optlist(default=[]) - document_root = opt() - - vhost = VHostSchema() - vhost.load('config.ini') - connect(vhost.server_name, vhost.port) - # etc. - -Any schema can contain an ``optdefault()`` object, which will -pick up any keys that aren't specified otherwise (if not present, -extra keys are an error). Then you'll get a dictionary of lists, -for all the extra config values. (Use -``optdefault(allow_multiple=False)`` if you want a dictionary of -strings). - -If you expect multiple values, use ``optlist(subtype=optsomething())`` -for a key. The values will be collected in a list; if you don't -indicate ``subtype`` then ``opt()`` is used. Other types are -fairly easy to make through subclassing. - -You can generate config files by using ``schema.ini_repr()`` which -will return a string version of the ini file. - -Implementation --------------- - -This makes heavy use of descriptors. If you are not familiar with -descriptors, see http://users.rcn.com/python/download/Descriptor.htm - -It also makes light use of metaclasses. -""" - - -import iniparser - -class INIMeta(type): - - def __new__(meta, class_name, bases, d): - cls = type.__new__(meta, class_name, bases, d) - cls.__classinit__.im_func(cls, d) - return cls - -class ParseValueError(Exception): - pass - -class NoDefault: - pass - -class INISchema(object): - - __metaclass__ = INIMeta - - _default_option = None - _default_values = None - - _config_names = {} - _config_names_lower = {} - - case_insensitive = False - - def __classinit__(cls, d): - # We don't initialize INISchema itself: - if cls.__bases__ == (object,): - return - cls._config_names = cls._config_names.copy() - cls._config_names_lower = cls._config_names_lower.copy() - for name, value in d.items(): - if isinstance(value, opt): - cls.add_option(name, value) - # @@: We should look for None-ified options, and remove - # them from cls._config_names - - def add_option(cls, attr_name, option): - """ - Classmethod: add the option using the given attribute name. - This can be called after the class has been created, to - dynamically build up the options. - """ - if isinstance(option, optdefault): - # We use a list so that the descriptor behavior doesn't - # apply here: - cls._default_option = [option] - option.attr_name = attr_name - return - if option.names is None: - option.names = [attr_name] - option.attr_name = attr_name - for option_name in option.names: - cls._config_names[option_name] = option - cls._config_names_lower[option_name.lower()] = option - option.set_schema(cls) - setattr(cls, attr_name, option) - - add_option = classmethod(add_option) - - def __init__(self): - self._ini_attrs = {} - - def set_config_value(self, name, value): - if self.case_insensitive: - name = name.lower() - config_names = self._config_names_lower - else: - config_names = self._config_names - if config_names.has_key(name): - setattr(self, config_names[name].attr_name, value) - elif not self._default_option: - raise ParseValueError( - "The setting %r was not expected (from %s)" - % (name, ', '.join(config_names.keys()) or 'none')) - else: - self._default_option[0].set_config_value( - self, name, value) - - def _parser(self): - return SchemaINIParser(self) - - def load(self, filename, **kw): - """ - Loads the filename. Use the encoding keyword argument to - specify the file's encoding. - """ - self._parser().load(filename, **kw) - - def loadstring(self, string, **kw): - """ - Loads the string, which is the content of the ini files. - Use the filename keyword argument to indicate the filename - source (or another way to identify the source of the string - in error messages). - """ - self._parser().loadstring(string, **kw) - - def as_dict(self, fold_defaults=False): - """ - Returns the loaded configuration as a dictionary. - """ - # @@: default values won't show up here - v = self._ini_attrs.copy() - if fold_defaults: - if self._default_values: - v.update(self._default_values) - elif self._default_option: - v[self._default_option[0].attr_name] = self._default_values or {} - return v - - def ini_repr(self): - """ - Returns the loaded values as a string, suitable as a - configuration file. - """ - config_names = [] - used_options = {} - for option in self._config_names.values(): - if used_options.has_key(option): - continue - used_options[option] = None - config_names.append((option.names[0], option)) - config_names.sort() - if self._default_option: - config_names.append((None, self._default_option[0])) - result = [] - for name, option in config_names: - result.append(option.ini_assignment( - self, getattr(self, option.attr_name))) - return ''.join(result) - -class opt(object): - - default = NoDefault - - def __init__(self, names=None, **kw): - self.names = names - self.attr_name = None - self.schema = None - for name, value in kw.items(): - if not hasattr(self, name): - raise TypeError( - "The keyword argument %s is unknown" - % name) - setattr(self, name, value) - - def set_schema(self, schema): - self.schema = schema - - def __get__(self, obj, type=None): - if obj is None: - return self - try: - return obj._ini_attrs[self.attr_name] - except KeyError: - if self.default is not NoDefault: - return self.default - raise AttributeError( - "The attribute %s has not been set on %r" - % (self.attr_name, obj)) - - def __set__(self, obj, value): - new_value = self.convert(obj, value) - self.validate(obj, new_value) - self.set_value(obj, new_value) - - def convert(self, obj, value): - return value - - def validate(self, obj, value): - pass - - def set_value(self, obj, value): - obj._ini_attrs[self.attr_name] = value - - def __delete__(self, obj): - try: - del obj._ini_attrs[self.attr_name] - except KeyError: - raise AttributeError( - "%r does not have an attribute %s" - % (obj, self.attr_name)) - - def ini_assignment(self, obj, value, name=None): - if name is None: - name = self.names[0] - return '%s=%s\n' % (name, - self.ini_fold(self.ini_repr(obj, value))) - - def ini_fold(self, value): - lines = value.splitlines() - if len(lines) <= 1: - return value - lines = [lines[0]] + [' ' + l for l in lines[1:]] - return '\n'.join(lines) - - def ini_repr(self, obj, value): - return str(value) - -optstring = opt - -class optint(opt): - - max = None - min = None - - bad_type_message = "You must give an integer number, not %(value)r" - too_large_message = "The value %(value)s is too large" - too_small_message = "The value %(value)s is too small" - coerce = int - - def convert(self, obj, value): - try: - new_value = self.coerce(value) - except ValueError: - raise ParseValueError( - self.bad_type_message % {'value': value}) - if self.max is not None and new_value > self.max: - raise ParseValueError( - self.too_large_message % {'value': value}) - if self.min is not None and new_value < self.min: - raise ParseValueError( - self.too_small_message % {'value': value}) - return new_value - -class optfloat(optint): - - coerce = float - bad_type_message = "You must give a float number, not %(value)r" - -class optbool(opt): - - true_values = ('yes', 'true', '1', 'on') - false_values = ('no', 'false', '0', 'off') - - def convert(self, obj, value): - if value.lower() in self.true_values: - return True - elif value.lower() in self.false_values: - return False - else: - raise ParseValueError( - "Should be a boolean value (true/false, on/off, yes/no), not %r" - % value) - - def ini_repr(self, obj, value): - if value: - return 'true' - else: - return 'false' - -class optlist(opt): - - subtype = opt - - def __init__(self, *args, **kw): - opt.__init__(self, *args, **kw) - if isinstance(self.subtype, type): - self.subtype = self.subtype() - - def convert(self, obj, value): - return self.subtype.convert(obj, value) - - def validate(self, obj, value): - self.subtype.validate(obj, value) - - def set_value(self, obj, value): - obj._ini_attrs.setdefault(self.attr_name, []).append(value) - - def ini_assignment(self, obj, value, name=None): - if name is None: - name = self.names[0] - assert not isinstance(value, (str, unicode)), ( - "optlist attributes should receive lists or sequences, not " - "strings (%r)" % value) - all = [self.subtype.ini_assignment(obj, sub, name=name) - for sub in value] - return ''.join(all) - -class optdefault(opt): - - allow_multiple = True - - def set_config_value(self, obj, name, value): - if obj._default_values is None: - obj._default_values = {} - if self.allow_multiple: - obj._default_values.setdefault(name, []).append(value) - else: - if obj._default_values.has_key(name): - raise ParseValueError( - "You have already set the configuration key %r" - % name) - obj._default_values[name] = value - - def __get__(self, obj, type=None): - if obj is None: - return self - return obj._default_values or {} - - def __set__(self, obj, value): - raise AttributeError( - "The attribute %s cannot be set" % self.attr_name) - - def ini_assignment(self, obj, value, name=None): - assert name is None, "default values can't accept a name" - all = [] - all_values = self.__get__(obj).items() - all_values.sort() - for name, value in all_values: - if isinstance(value, (str, unicode)): - value = [value] - for subvalue in value: - all.append('%s=%s\n' % ( - name, self.ini_fold(self.ini_repr(obj, subvalue)))) - return ''.join(all) - -class optconverter(opt): - - misc_error_message = "%(error_type)s: %(message)s" - converter_func = None - - def convert(self, obj, value): - try: - return self.converter_func(value) - except Exception, e: - raise ParseValueError( - self.misc_error_message % { - 'message': str(e), - 'error_type': e.__class__.__name__}) - - _converters = {} - def get_converter(cls, name, converter_func): - if cls._converters.has_key(id(converter_func)): - return cls._converters[(name, id(func))] - else: - converter = cls(name, converter_func=converter_func) - cls._converters[(name, id(converter_func))] = converter - return converter - get_converter = classmethod(get_converter) - -class SchemaINIParser(iniparser.INIParser): - - def __init__(self, schema): - self.schema = schema - - def new_section(self, section): - self.parse_error("Schemas do not yet support sections") - - def assignment(self, name, content): - try: - self.schema.set_config_value(name, content) - except ParseValueError, e: - self.parse_error(e.args[0]) diff --git a/lib/config/lazyiniparser.py b/lib/config/lazyiniparser.py deleted file mode 100644 index 628a900..0000000 --- a/lib/config/lazyiniparser.py +++ /dev/null @@ -1,168 +0,0 @@ -""" -A parser that keeps lots of information around, so the file can be -reconstructed almost exactly like it originally was entered. Also, if -there are errors with values, they can be tracked back to a file and -line number. -""" - -from iniparser import INIParser, ParseError - -class ConversionError(Exception): - pass - -def canonical_name(name): - return name.lower().replace(' ', '').replace('_', '') - -class LazyINIParser(INIParser): - - def __init__(self, allow_empty_sections=False): - self.allow_empty_sections = allow_empty_sections - INIParser.__init__(self) - - def reset(self): - self.configuration = Configuration() - self.last_comment = [] - - def add_comment(self, line): - if line.startswith(' '): - line = line[1:] - self.last_comment.append(line) - - def assignment(self, name, content): - item = Item(section=self.section, - name=name, - content=content, - comment='\n'.join(self.last_comment), - filename=self.filename, - lineno=self.start_lineno) - self.last_comment = [] - if not self.section: - self.new_section('') - self.section.add_item(item) - - def new_section(self, section): - if not section and not self.allow_empty_sections: - self.error_no_section_name() - self.section = Section( - name=section.strip(), - comment='\n'.join(self.last_comment)) - self.configuration.add_section(self.section) - self.last_comment = [] - -class Configuration(object): - - def __init__(self): - self.sections = [] - self.sections_by_name = {} - - def add_section(self, section): - self.sections.append(section) - # @@: I shouldn't be doing this here, I'll do it in lazyloader - #names = self.split_names(section.name) - #d = self.sections_by_name - #for name in names[:-1]: - # d = d.setdefault(name, {}) - #d.setdefault(names[-1], []).append(section) - - def split_names(self, name): - names = [] - while 1: - dot_pos = name.find('.') - paren_pos = name.find('(') - if dot_pos == -1 and paren_pos == -1: - next = canonical_name(name) - if next: - names.append(next) - return names - if (dot_pos == -1 or (dot_pos > paren_pos - and paren_pos != -1)): - next = canonical_name(name[:paren_pos]) - if next: - names.append(next) - name = name[paren_pos+1:] - next_pos = name.find(')') - assert next_pos != -1, ( - "Bad section name, ) expected: %r" % name) - names.append(name[:next_pos]) - name = name[next_pos+1:] - else: - assert dot_pos != -1 - assert paren_pos == -1 or dot_pos < paren_pos - next = canonical_name(name[:dot_pos]) - assert next, ( - "Empty name") - names.append(next) - name = name[dot_pos+1:] - - def source(self): - return '\n\n'.join([s.source() for s in self.sections]) - -class Section(object): - - def __init__(self, name, comment): - self.name = name - self.comment = comment - self.items = [] - self.canonical = {} - - def add_item(self, item): - self.items.append(item) - self.canonical.setdefault( - canonical_name(item.name), []).append(item) - - def __repr__(self): - return '<%s name=%r>' % (self.__class__.__name__, self.name) - - def source(self): - s = '' - if self.comment: - s += '\n'.join(['# ' + l - for l in self.comment.splitlines()]) + '\n' - s += '[%s]\n' % self.name - s += ''.join([i.source() for i in self.items]) - return s - - -class Item(object): - - def __init__(self, section, name, content, comment, - filename, lineno): - self.section = section - self.name = name - self.content = content - self.comment = comment - self.filename = filename - self.lineno = lineno - - def value(self, name, converter=None, catch_all_exceptions=False): - if catch_all_exceptions: - ExcClass = Exception - else: - ExcClass = ConversionError - if converter is not None: - try: - return converter(self.content) - except ExcClass, e: - msg = str(e) - raise ParseError( - msg, - filename=self.filename, - lineno=self.lineno, - column=None) - else: - return self.content - - def __str__(self): - return self.content - - def __repr__(self): - return '<%s name=%r; value=%r>' % ( - self.__class__.__name__, self.name, self.content) - - def source(self): - s = '' - if self.comment: - s += '\n'.join(['# ' + l - for l in self.comment.splitlines()]) + '\n' - s += '%s = %s\n' % (self.name, self.content) - return s diff --git a/lib/config/lazyloader.py b/lib/config/lazyloader.py deleted file mode 100644 index dd1a58a..0000000 --- a/lib/config/lazyloader.py +++ /dev/null @@ -1,228 +0,0 @@ -""" -A config file loader that can load and nest multiple config files, the -config files can have structure, and the values can be tracked back to -their original file and line number. - -To start, you'd do something like: - - >>> config = LazyLoader() - -You could use ``.load(filename)`` to load a config file; for the -examples it is convenient to instead use loadstring, and give a fake -filename: - - >>> config_data = \"\"\" - ... [server] - ... port = 8000 - ... host = localhost - ... document_root = /var/www - ... \"\"\" - >>> config.loadstring(config_data, filename='config_data.conf') - >>> config['server']['port'] - '8000' - >>> config['server'].convert('port', int) - 8000 - >>> config['server'].convert('host', int) - Traceback (most recent call last): - ... - ValueError: Error in config_data.conf (section [server]), line 4 ('localhost'): - ValueError: invalid literal for int(): localhost - -Note that names are normalized, removing case, underscores, and -spaces. So to get to the document root: - - >>> config['server']['documentroot'] - '/var/www' - -You can also merge in values; for instance, consider a virtual host -that overrides global values: - - >>> vhost_data = \"\"\" - ... [vhost(my.host.com)] - ... document_root = /path/to/root - ... \"\"\" - >>> vhost_config = LazyLoader() - >>> vhost_config.loadstring(vhost_data, filename='vhost_data.conf') - >>> vhost_config['vhost'].keys() - ['my.host.com'] - -Note that key and section names can be nested with .'s, and ()'s quote -the values (so the key is ['my.host.com'] instead of -['my']['host']['com']). Then we may want to merge this in, based on a -condition (e.g., the hostname matches my.host.com): - - >>> config['server'].merge(vhost_config['vhost']['my.host.com']) - >>> config['server']['documentroot'] - '/path/to/root' - -""" - -from lazyiniparser import LazyINIParser, Item -from nested import NestedDict -import inischema -import re -from UserDict import UserDict, DictMixin - -class LazyLoader(NestedDict): - - def __init__(self, configs=None, mutable=False, nest=True, - section_name=None, master=None): - self.section_name = section_name - self.master = master - NestedDict.__init__(self, configs=configs, mutable=mutable, - nest=nest) - - def load(self, filename): - parser = LazyINIParser(allow_empty_sections=True) - parser.load(filename) - config = self._convert_configuration(parser.configuration) - self.add_config(config) - - def loadstring(self, s, filename=None): - parser = LazyINIParser(allow_empty_sections=True) - parser.loadstring(s, filename=filename) - config = self._convert_configuration(parser.configuration) - self.add_config(config) - - def merge(self, lazyloader): - if self.master: - self.master._propagate_merge([self.section_name], lazyloader) - else: - self._propagate_merge([], lazyloader) - - def _propagate_merge(self, slave_keys, lazyloader): - if self.master: - slave_keys.insert(0, self.section_name) - self.master._propagate_merge(slave_keys, lazyloader) - else: - configs = filter(None, lazyloader.configs) - new_configs = [] - for config in configs: - if slave_keys: - new_config = {} - set_config = new_config - for key in slave_keys[:-1]: - set_config[key] = {} - set_config = set_config[key] - set_config[slave_keys[-1]] = config - else: - new_config = config - self.add_config(new_config) - - def __getitem__(self, key): - try: - return NestedDict.__getitem__(self, key) - except KeyError, e: - if self.section_name is None: - section = 'global section' - else: - section = 'section [%s]' % self.section_name - raise KeyError( - "Key %r not found in %s" % (key, section)) - - def _convert_configuration(self, conf): - data = {} - for section in conf.sections: - section_keys = self._parse_keys(section.name) - if section_keys == ['global']: - section_keys = [] - for item in section.items: - item_keys = self._parse_keys(item.name) - all_keys = section_keys + item_keys - pos = data - for key in all_keys[:-1]: - pos = pos.setdefault(key, {}) - pos.setdefault(all_keys[-1], []).append(item) - return data - - def _parse_keys(self, name): - keys = [] - orig_name = name - name = name.strip() - while 1: - next_period = name.find('.') - next_paren = name.find('(') - if next_paren == -1 and next_period == -1: - next = self._canonical_name(name) - if next: - keys.append(next) - return keys - elif (next_paren == -1 - or next_period != -1 and next_period < next_paren): - next = self._canonical_name(name[:next_period]) - if next: - keys.append(next) - name = name[next_period+1:] - else: - assert next_paren != -1 - assert next_period == -1 or next_paren < next_period - next = self._canonical_name(name[:next_paren]) - if next: - keys.append(next) - name = name[next_paren+1:] - next_close = name.find(')') - if next_close == -1: - raise inischema.ParseValueError( - "Key name contains a ( with no closing ): %r" - % orig_name) - next = name[:next_close] - keys.append(next) - name = name[next_close+1:] - - _canonical_re = re.compile('[_ \t-]') - def _canonical_name(self, name): - return self._canonical_re.sub('', name.lower()) - - def _convert_single(self, key, value_list): - if isinstance(value_list[0], (dict, DictMixin, UserDict)): - return self.__class__( - value_list, mutable=self.mutable, nest=True, - section_name=key, master=self) - elif isinstance(value_list[0][-1], Item): - return value_list[0][-1].content - else: - return value_list[0][-1] - - def getlist(self, key, add_inherited=True): - results = self.raw_get(key, add_inherited=add_inherited) - converted = [] - for result_set in results: - if not isinstance(result_set, (list, tuple)): - result_set = [result_set] - for item in result_set: - if isinstance(item, (dict, UserDict, DictMixin)): - value = self.__class__( - [item], mutable=self.mutable, nest=self.nest) - elif isinstance(item, Item): - value = item.content - else: - value = item - converted.append(value) - return converted - - def _make_converter(self, key, converter): - if not isinstance(converter, inischema.opt): - converter = inischema.optconverter.get_converter( - key, converter_func=converter) - return converter - - def convert(self, key, converter): - converter = self._make_converter(key, converter) - try: - return converter.convert(None, self[key]) - except inischema.ParseValueError, e: - item = self.raw_get(key)[0][0] - message = ('Error in %s (section [%s]), line %i (%r):\n%s' - % (item.filename, item.section.name, item.lineno, - self[key], e)) - raise ValueError(message) - - def convertlist(self, key, converter, add_inherited=True): - converter = self._make_converter(key, converter) - value_list = self.getlist(key, add_inherited=add_inherited) - new_list = [] - try: - for value in value_list: - new_list.append(converter.convert(value)) - except inischema.ParseValueError, e: - pass diff --git a/lib/config/nested.py b/lib/config/nested.py deleted file mode 100644 index 24b2f65..0000000 --- a/lib/config/nested.py +++ /dev/null @@ -1,168 +0,0 @@ -""" -Layers multiple dictionaries. - -Nested dictionaries can be traversed, with each dictionary shadowing -the previous dictionaries. So for example: - - >>> d = NestedDict([{'foo': 'bar'}]) - >>> d['foo'] - 'bar' - >>> d2 = d.clone(add_dict={'foo': 'bar2'}) - >>> d2['foo'] - 'bar2' - >>> d2.getlist('foo') - ['bar2', 'bar'] - -This works for deeply nested dictionaries, not just at the top level; -each nested dictionary gets wrapped in a NestedDict as well. -""" - -from UserDict import DictMixin, UserDict - -class NestedDict(DictMixin): - - def __init__(self, configs=None, mutable=True, nest=True): - if configs is None: - configs = [{}] - assert isinstance(configs, (list, tuple)), ( - "The configs must be a list or tuple, not: %r" - % configs) - self.configs = [conf.copy() for conf in configs] - self.mutable = mutable - self.nest = nest - - def __getitem__(self, key): - results = self.raw_get(key) - if not results: - raise KeyError, "Key not found: %r" % key - return self._convert_single(key, results) - - def add_config(self, config, position=0): - assert not isinstance(config, (str, unicode)), ( - "Bad configuration (not a mapping): %r" % config) - if position is None: - self.configs.append(config) - else: - self.configs.insert(position, config) - - def set_configs(self, new_configs): - self.configs[:] = [] - for config in new_configs: - self.add_config(config, None) - - def raw_get(self, key, add_inherited=True): - results = [] - if add_inherited: - all_configs = self.configs - else: - all_configs = [self.configs[-1]] - for config in all_configs: - if key in config: - if isinstance(config, (str, unicode)): - print [key, config, all_configs, self.configs] - results.append(config[key]) - return results - - def getlist(self, key, add_inherited=True): - results = self.raw_get(key, add_inherited=add_inherited) - converted = [] - for result_set in results: - if not isinstance(result_set, (list, tuple)): - result_set = [result_set] - converted.extend(self._convert_many(key, result_set)) - return converted - - def _convert_single(self, key, value_list): - if (not self.nest - or not isinstance(value_list[0], - (dict, DictMixin, UserDict))): - # @@: doesn't handle all dict alternatives - return value_list[0] - elif len(value_list) == 1 and self.mutable: - return value_list[0] - else: - return self.__class__( - value_list, mutable=self.mutable, nest=True) - - def _convert_many(self, key, value_list): - return [self._convert_single(key, [v]) for v in value_list] - - def __setitem__(self, key, value): - if self.mutable: - self.configs[0][key] = value - else: - raise KeyError, ( - "Dictionary is read-only") - - def __delitem__(self, key): - if not self.mutable: - raise KeyError, ( - "Dictionary is read-only") - if self.configs[0].has_key(key): - del self.configs[0][key] - elif self.has_key(key): - raise KeyError, ( - "You cannot delete the key %r, as it belongs to the " - "a master configuration %r" - % (key, self.master)) - else: - raise KeyError, ( - "Key does not exist: %r" % key) - - def keys(self): - return [key for key in self] - - def __contains__(self, key): - for config in self.configs: - if config.has_key(key): - return True - return False - - def has_key(self, key): - return key in self - - def __iter__(self): - used = {} - for config in self.configs: - for key in config: - if key in used: - continue - used[key] = None - yield key - - _clone_sentry = [] - - def clone(self, add_dict=None, mutable=_clone_sentry, - nest=_clone_sentry): - if mutable is self._clone_sentry: - mutable = self.mutable - if nest is self._clone_sentry: - nest = self.nest - new = self.configs[:] - if add_dict is not None: - new.insert(0, add_dict) - return self.__class__(new, mutable=mutable, nest=nest) - - def copy(self): - return dict(self.iteritems()) - - def __eq__(self, other): - if other is None: - return False - if (not hasattr(other, 'keys') - or not hasattr(other, '__getitem__')): - return False - for key in other: - if not key in self: - return False - for name, value in self.iteritems(): - if other[name] != value: - return False - return True - - def __cmp__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return '<%s %r>' % (self.__class__.__name__, - dict(self.iteritems())) diff --git a/lib/config/test_iniparser.py b/lib/config/test_iniparser.py deleted file mode 100644 index e10031f..0000000 --- a/lib/config/test_iniparser.py +++ /dev/null @@ -1,78 +0,0 @@ -import iniparser -import re - - -def parse(s, filename='test.ini'): - p = iniparser.BasicParser() - p.loadstring(s, filename=filename) - return p - - -def raises(sample, match): - if isinstance(match, str): - match = re.compile(match) - try: - parse(sample) - except iniparser.ParseError, e: - if match and not match.search(str(e)): - raise - else: - assert 0, "Parsing %r should have raised an error" % sample - - -def test_no_section(): - raises( - """# A file... -a=10 -""", r'^Assignments can only.*') - -def test_no_equal(): - raises( - """[a file] -this is a test -""", r'^Lines should look like.*') - -def test_bad_section(): - raises( - """[file -""", r'^Invalid section.*') - -def test_empty_section(): - raises( - """[] -""", r'^Empty section name.*') - -def test_equal_colon(): - sec = parse("""[test] -a=1:2 -b: 1=3""").data['test'] - assert sec['a'] == ['1:2'] - assert sec['b'] == ['1=3'] - -def test_multi_section(): - data = parse("""[test] -#a=1 -a=2 -a=3 -a=4 -[test2] -a=1 -""").data - assert data['test']['a'] == ['2', '3', '4'] - assert data['test2']['a'] == ['1'] - -def test_continuation(): - data = parse("""[test] -a = a - pretty - bird -\tb=3 -b=another - line -[test2] -[test3] -c=[blah] - [blah]""").data - assert data['test']['a'] == ['a\npretty\nbird\nb=3'] - assert data['test']['b'] == ['another\nline'] - assert data['test3']['c'] == ['[blah]\n[blah]'] diff --git a/lib/config/test_inischema.py b/lib/config/test_inischema.py deleted file mode 100644 index 04e1ac1..0000000 --- a/lib/config/test_inischema.py +++ /dev/null @@ -1,111 +0,0 @@ -from inischema import * - - -class Schema1(INISchema): - - number = optint() - maybe = optbool(names=['maybe', 'or not']) - float = optfloat() - string = opt() - string2 = opt() - rep = optlist(subtype=optint()) - whatever = opt(default=5) - - -class Schema2(INISchema): - - a = optint() - default = optdefault() - - -class Schema3(Schema2): - pass - -Schema3.add_option('default', optdefault(allow_multiple=False)) -Schema3.add_option('b', optlist(subtype=optint())) - - -def parse_schema(schema, string, filename='test.ini'): - if not isinstance(schema, INISchema): - schema = schema() - schema.loadstring(string) - return schema - - -load_ini = """ -number = 1 -float = 1 -or not = true -string = something -string2=something -rep=1 -rep=5 -rep=10 -""" - - -def test_load(): - s = parse_schema(Schema1, load_ini) - assert s.as_dict() == { - 'number': 1, - 'maybe': True, - 'float': 1.0, - 'string': 'something', - 'string2': 'something', - 'rep': [1, 5, 10]} - assert s.number == 1 - assert s.rep == [1, 5, 10] - assert s.whatever == 5 - assert s.string == 'something' - -def test_unload(): - s = parse_schema(Schema1, load_ini) - assert s.ini_repr() == """\ -float=1.0 -maybe=true -number=1 -rep=1 -rep=5 -rep=10 -string=something -string2=something -whatever=5 -""" - -default_ini = """ -a = 1 -b = 2 -c = 3 -""" - -def test_default(): - s = parse_schema(Schema2, default_ini) - assert s.as_dict() == { - 'a': 1, - 'default': {'b': ['2'], - 'c': ['3']}} - assert s.as_dict(fold_defaults=True) == { - 'a': 1, - 'b': ['2'], - 'c': ['3']} - -def test_default2(): - s = parse_schema(Schema3, default_ini) - assert s.as_dict() == { - 'a': 1, - 'b': [2], - 'default': {'c': '3'}} - assert s.as_dict(fold_defaults=True) == { - 'a': 1, 'b': [2], 'c': '3'} - -def test_gen(): - s = parse_schema(Schema2, default_ini) - assert s.ini_repr() == "a=1\nb=2\nc=3\n" - s.a = 2 - assert s.ini_repr() == "a=2\nb=2\nc=3\n" - -def test_gen2(): - s = parse_schema(Schema3, default_ini) - assert s.ini_repr() == "a=1\nb=2\nc=3\n" - s.a = 2 - assert s.ini_repr() == "a=2\nb=2\nc=3\n" diff --git a/lib/config/test_lazyiniparser.py b/lib/config/test_lazyiniparser.py deleted file mode 100644 index ff653ef..0000000 --- a/lib/config/test_lazyiniparser.py +++ /dev/null @@ -1,27 +0,0 @@ -import lazyiniparser - - -def parse(s, filename='test.ini'): - p = lazyiniparser.LazyINIParser() - p.loadstring(s, filename=filename) - return p.configuration - - -data = """\ -# a test -[section 1] -a = 2 -a2 = 3 -# one more... -a = 4 -""" - -def test_simple(): - c = parse(data) - assert len(c.sections) == 1 - assert c.sections[0].name == 'section 1' - assert c.sections[0].comment == 'a test' - items = c.sections[0].items - assert [i.name for i in items] == ['a', 'a2', 'a'] - assert [i.lineno for i in items] == [3, 4, 6] - assert c.source() == data diff --git a/lib/config/test_lazyloader.py b/lib/config/test_lazyloader.py deleted file mode 100644 index 9d178fd..0000000 --- a/lib/config/test_lazyloader.py +++ /dev/null @@ -1,111 +0,0 @@ -import sys -import lazyloader -from test_fixture import DoctestCollector, sorted, assert_error - -data1 = """\ -[ages] - -jeanette = 72 -dave = 54 - -[children] - -jeanette = dave -dave = ian -dave = monica -""" - -def test_create(): - config = lazyloader.LazyLoader() - config.loadstring(data1, filename="data1.conf") - assert sorted(config.keys()) == ['ages', 'children'] - assert config['ages']['jeanette'] == '72' - assert config['ages']['dave'] == '54' - assert config['children']['dave'] == 'monica' - assert config['children']['jeanette'] == 'dave' - assert config['children'].getlist('dave') == ['ian', 'monica'] - assert config['ages'].convert('jeanette', int) == 72 - assert_error( - config['children'].convert, 'jeanette', int, - error=ValueError, - text_re=r'data1.conf.*line 8.*invalid literal') - assert_error( - config['children'].convert, 'joe', str, - error=KeyError, - text_re=r'joe.*section.*children') - assert_error( - config.__getitem__, 'child', - error=KeyError) - -data2 = """\ -[children] -# Another child... -jeanette = mary -""" - -def test_fold(): - config = lazyloader.LazyLoader() - config.loadstring(data1, filename="data1.conf") - config.loadstring(data2, filename="data2.conf") - assert config['children']['jeanette'] == 'mary' - assert (sorted(config['children'].getlist('jeanette')) - == ['dave', 'mary']) - assert config['children']['dave'] == 'monica' - assert sorted(config.keys()) == ['ages', 'children'] - assert sorted(config['children'].keys()) == ['dave', 'jeanette'] - -data3 = """\ -children.mary = vada -[global] -children.mary = armen -""" - -data4 = """\ -george = chelsea -""" - -data5 = """\ -[something(here!)] -test.this.out = foo -""" - -def test_global(): - config = lazyloader.LazyLoader() - config.loadstring(data1, filename="data1.conf") - config.loadstring(data2, filename="data2.conf") - config.loadstring(data3, filename="data3.conf") - assert config['children']['mary'] == 'armen' - assert config['children'].getlist('mary') == ['vada', 'armen'] - merged = lazyloader.LazyLoader() - merged.loadstring(data4, filename="data4.conf") - config['children'].merge(merged) - assert config['children']['george'] == 'chelsea' - config.merge(merged) - assert config['george'] == 'chelsea' - merge2 = lazyloader.LazyLoader() - merge2.loadstring(data5, filename="data5.conf") - assert merge2['something']['here!']['test']['this']['out'] == 'foo' - config.merge(merge2) - assert config['something']['here!']['test']['this']['out'] == 'foo' - -collect_doctest = DoctestCollector(lazyloader) - -def parse_keys(n): - p = lazyloader.LazyLoader() - return p._parse_keys(n) - -def test_parse_section(): - data = [ - ('a', 'a'), - ('a.b', 'a', 'b'), - ('this . that', 'this', 'that'), - ('this...that', 'this', 'that'), - ('foo_bar(foo_bar)', 'foobar', 'foo_bar'), - ('A.(B).C', 'a', 'B', 'c'), - ('A(B)C', 'a', 'B', 'c'), - ('a ( b ) . c . d . ( e )', 'a', ' b ', 'c', 'd', ' e '), - ] - for trial in data: - input = trial[0] - output = list(trial[1:]) - assert parse_keys(input) == output diff --git a/lib/config/test_nested.py b/lib/config/test_nested.py deleted file mode 100644 index 83521de..0000000 --- a/lib/config/test_nested.py +++ /dev/null @@ -1,56 +0,0 @@ -import nested -from test_fixture import DoctestCollector, sorted - -NestedDict = nested.NestedDict - - -def test_empty(): - d = NestedDict() - d['a'] = 1 - assert d['a'] == 1 - d['a'] = 2 - assert d['a'] == 2 - src = {'c': 2} - d['b'] = src - assert d['b'] == src - assert d['b'] is src - assert d['b']['c'] == 2 - d2 = d.clone() - assert d == d2 - d2['a'] = 3 - assert d.items() != d2.items() - assert list(d.iteritems()) != list(d2.iteritems()) - print list(d.iteritems()), list(d2.iteritems()) - print d.configs, d2.configs - assert d != d2 - assert repr(d) != repr(d2) - assert d2['a'] == 3 - d3 = d.copy() - assert d == d3 - assert d != None - assert d != [] - assert d != object() - -def test_nested(): - src = { - 'a': 1, - 'b': { - 'c': 2, - 'd': 3, - }} - shadow = { - 'b': { - 'c': 5, - 'e': 6, - }} - d = NestedDict([shadow, src]) - assert d['a'] == 1 - assert d['b']['c'] == 5 - assert d['b']['d'] == 3 - assert sorted(d.keys()) == ['a', 'b'] - assert sorted(d['b'].keys()) == ['c', 'd', 'e'] - assert isinstance(d['b'], NestedDict) - concrete = {'a': 1, 'b': {'c': 5, 'd': 3, 'e': 6}} - assert d == concrete - -collect_doctest = DoctestCollector(nested) diff --git a/lib/convert_html.py b/lib/convert_html.py deleted file mode 100644 index 778bc74..0000000 --- a/lib/convert_html.py +++ /dev/null @@ -1,54 +0,0 @@ -from common import htmlEncode -#import py2html, PyFontify -import PySourceColor -import converter_registry - -def convert_html(page, text, mime_type): - return text - -converter_registry.register(convert_html, 'text/html') - -def convert_text(page, text, mime_type): - return ''' -
source
-
%s
''' % (page.sourceLinkForMimeType(mime_type), - htmlEncode(text)) - -converter_registry.register(convert_text, 'text/*') - -def convert_application(page, text, mime_type): - return ''' - View file (%s) - ''' % (page.sourceLinkForMimeType(mime_type), mime_type) - -converter_registry.register(convert_application, 'application/*') - -def convert_image(page, text, mime_type): - attrs = '' - if page.width: - attrs += ' width="%i"' % page.width - if page.height: - attrs += ' height="%i"' % page.height - return '' % ( - page.sourceLinkForMimeType(mime_type), - attrs) - -converter_registry.register(convert_image, 'image/*') - -def convert_generic(page, text, mime_type): - return text - -converter_registry.register(convert_generic, '*') - -def convert_python(page, text, mime_type): - - #pp = py2html.PrettyPrint(format='rawhtml', mode='color', - # tagfct=PyFontify.fontify) - #html = pp.filter(text) - html = PySourceColor.str2html(text, PySourceColor.dark) - return ''' -
- source - %s
''' % (page.sourceLinkForMimeType(mime_type), html) - -converter_registry.register(convert_python, 'text/x-python') diff --git a/lib/convert_rest.py b/lib/convert_rest.py deleted file mode 100644 index ee32618..0000000 --- a/lib/convert_rest.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Code for rendering restructured text -""" -import traceback -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -from docutils import core -from docutils.utils import SystemMessage -from docutils import nodes -from docutils.readers import standalone -from docutils.transforms import Transform - -import converter_registry -from common import canonicalName, guessURLName, htmlEncode - - -def convert_rest(page, text, mime_type=None): - if not text.strip(): - # ReST doesn't like empty documents - return '' - try: - html = _clean_html(core.publish_string( - source=text, - reader=Reader(), - parser_name='restructuredtext', - writer_name='html', - settings_overrides={'traceback': 1})) - return html - except SystemMessage, error: - return _format_error(error, None) - except Exception, error: - print "Error rendering page" - out = StringIO() - traceback.print_exc(file=out) - return _format_error(error, out.getvalue()) - -converter_registry.register(convert_rest, 'text/x-restructured-text') - -def _clean_html(html): - return html[html.find('')+6:html.find('')] - -def _format_error(error, tb): - if isinstance(error, SystemMessage): - # We expect a format like col:line: (LEVEL/INT) Message\ntext - msg = error.args[0] - col, line, rest = msg.split(':', 2) - level, rest = rest.split(')', 1) - level = level.strip()[1:] - message, text = rest.split('\n', 1) - return '''
-

- SystemMessage: %s (%s:%s)

-

%s

-
%s
-
''' % (htmlEncode(level), - col, line, - htmlEncode(message), - htmlEncode(text)) - else: - return '''
-

-

%s

-
%s
-
''' % (htmlEncode(str(error)), - htmlEncode(tb)) - - -class WikiLinkResolver(nodes.SparseNodeVisitor): - - def visit_reference(self, node): - if node.resolved or not node.hasattr('refname'): - return - refname = node['refname'] - node.resolved = 1 - node['class'] = 'wiki' - refuri = guessURLName(refname) - node['refuri'] = refuri - del node['refname'] - -class WikiLink(Transform): - - default_priority = 800 - - def apply(self): - visitor = WikiLinkResolver(self.document) - self.document.walk(visitor) - -class Reader(standalone.Reader): - - supported = standalone.Reader.supported + ('wiki',) - - def get_transforms(self): - transforms = standalone.Reader.get_transforms(self) - transforms.append(WikiLink) - return transforms diff --git a/lib/converter_registry.py b/lib/converter_registry.py deleted file mode 100644 index ba17afa..0000000 --- a/lib/converter_registry.py +++ /dev/null @@ -1,16 +0,0 @@ -_registry = {} - -def register(func, mime_type): - assert mime_type not in _registry, ( - "Registry duplicate for mime type %s" - % mime_type) - _registry[mime_type] = func - -def convert(page, text, mime_type): - possible = [mime_type, mime_type.split('/')[0] + '/*', '*'] - for mtype in possible: - if mtype in _registry: - return _registry[mtype](page, text, mime_type) - raise ValueError, ( - "No converter registered for type %s" - % mime_type) diff --git a/lib/default_static.tmpl b/lib/default_static.tmpl deleted file mode 100644 index 41d1e85..0000000 --- a/lib/default_static.tmpl +++ /dev/null @@ -1,13 +0,0 @@ - - - $page.title - - - -

$page.title

- $page.staticHTML - -
Last updated: ${page.modifiedDate.strftime('%c')} -
- - diff --git a/lib/distpages/relatedterms.meta b/lib/distpages/relatedterms.meta deleted file mode 100644 index a9553f0..0000000 --- a/lib/distpages/relatedterms.meta +++ /dev/null @@ -1 +0,0 @@ -title: Related Terms \ No newline at end of file diff --git a/lib/distpages/relatedterms.txt b/lib/distpages/relatedterms.txt deleted file mode 100644 index c8e0a2c..0000000 --- a/lib/distpages/relatedterms.txt +++ /dev/null @@ -1,13 +0,0 @@ -Related terms are like a kind of backlink/inlining into other pages. - -When you list a page as being a related term, when you view that page you will -see *this* page inlined into that page (along with any other pages that also -are considered related). You can also relate this to a page/term that does not -yet exist in the wiki. This is a way of creating a kind of table of contents -for select topics. You can use normal page terms, like "Related Terms", and -it is converted to ``relatedterms`` for you. - -There is more to be done with this, mostly for customizing the target term -- -so that things like a limit on the number of pages brought in, the sort order, -indexing, RSS generation, and customization of how the page is inlined -(just titles, summaries, entire pages). It's just an idea for now. diff --git a/lib/distpages/thiswiki.meta b/lib/distpages/thiswiki.meta deleted file mode 100644 index e6c40e2..0000000 --- a/lib/distpages/thiswiki.meta +++ /dev/null @@ -1 +0,0 @@ -title: This Wiki \ No newline at end of file diff --git a/lib/distpages/thiswiki.txt b/lib/distpages/thiswiki.txt deleted file mode 100644 index 2b6afae..0000000 --- a/lib/distpages/thiswiki.txt +++ /dev/null @@ -1,89 +0,0 @@ -This is a `Webware for Python`_ Wiki_ written by `Ian Bicking`_. - -.. _Webware for Python: http://www.webwareforpython.org -.. _Wiki: http://wiki.org/wiki.cgi?WhatIsWiki -.. _Ian Bicking: http://ianbicking.org - -Markup ------- - -This wiki uses `ReStructuredText`_ for its markup. Restructured -text is intended as a general-purpose markup for creating documents, -with specific emphasis on documenting programming. As such it is the -perfect fit for this site. ReStructuredText is also used by several -other projects, so if you learn how to use it for this wiki you can -transfer that knowledge other places. - -.. _ReStructuredText: http://docutils.sourceforge.net/rst.html - -To learn how to use ReStructuredText, you may wish to look at the -`Quick Reference`_ - -.. _Quick Reference: http://docutils.sourceforge.net/docs/rst/quickref.html - -Note that WikiNames are *not* used in this software! Instead a ``_`` -must be appended to a name to make it into a link, and use:: - - `Long wiki name`_ - -for multi-word names. All names are made lower-case and spaces are -replaced with hyphens (i.e., the link is to the page ``long-wiki-name.html``). - -To play around with the markup to this wiki feel free to use the `Wiki Sandbox`_. - -External Editor ---------------- - -If you don't want to use ``textarea`` editing (which is no fun), this -site uses `External Editor`_ to allow text to be edited with a -standard text editor. The Zope Product aspect is not required (the -equivalent is already built into this wiki), but the "helper -application" is used to hook into the editor. This is -the ``zopeedit.py`` application. To invoke the editor once you have -set up the client, click the pen image (|penimage|). This is highly -recommended -- client installation is easy, available for multiple -platforms, and the editing experience is far superior. - -.. _External Editor: http://www.zope.org/Members/Caseman/ExternalEditor -.. |penimage| image:: edit_icon.gif - -If you are using a browser in the Mozilla family (Mozilla, Firefox,...) you can -also use the cool `Mozex`_ extension, which lets you configure your favourite -text editor (Emacs, Vim, Notepad,...) for textareas. - -.. _Mozex: http://mozex.mozdev.org - -WYSIWYG Editor --------------- - -For editing HTML pages directly in WYSIWYG fashion, this wiki uses Xinha. -This should work out of the box with all modern web browsers supporting -Javascript. You will even get a special button for Wiki internal links. - -Other Notes ------------ - -Updates to this site are published to a `New Pages`_ and a `Recent Changes`_ -RSS feed, which you can subscribe to in order to receive updates. - -.. _New Pages: /feeds/new_pages.xml -.. _Recent Changes: /feeds/recent_changes.xml - -Acquiring the Wiki ------------------- - -This wiki was written specifically for the `Webware for Python`_ web site, -though in most ways it is a general-purpose wiki. -Of course, it runs on Webware and Python. - -You can download the code for this Wiki at - - http://www.webwareforpython.org/downloads/Wiki/ - -or check out the latest version from the Subversion repository with - - $ svn co http://svn.w4py.org/Wiki/trunk Wiki - -Then follow the installation instructions in ``Wiki/README.txt``. - -See also: `Wiki Features`_, `Wiki ToDo`_, and `Wiki Bugs`_. diff --git a/lib/distpages/wikibugs.meta b/lib/distpages/wikibugs.meta deleted file mode 100644 index f510bcb..0000000 --- a/lib/distpages/wikibugs.meta +++ /dev/null @@ -1 +0,0 @@ -title: Wiki Bugs \ No newline at end of file diff --git a/lib/distpages/wikibugs.txt b/lib/distpages/wikibugs.txt deleted file mode 100644 index 0885eb4..0000000 --- a/lib/distpages/wikibugs.txt +++ /dev/null @@ -1,11 +0,0 @@ -Please list any bugs you find here. -Feature requests should go on `Wiki ToDo`_. - -Indirect Wiki Names -~~~~~~~~~~~~~~~~~~~ - -This doesn't work, but it should:: - - Maybe you should look at this_ - - .. _this: `another wiki page`_ diff --git a/lib/distpages/wikifeatures.meta b/lib/distpages/wikifeatures.meta deleted file mode 100644 index f871be6..0000000 --- a/lib/distpages/wikifeatures.meta +++ /dev/null @@ -1 +0,0 @@ -title: Wiki Features \ No newline at end of file diff --git a/lib/distpages/wikifeatures.txt b/lib/distpages/wikifeatures.txt deleted file mode 100644 index 08d8d59..0000000 --- a/lib/distpages/wikifeatures.txt +++ /dev/null @@ -1,119 +0,0 @@ -Some of the features of this wiki: - -Restructured Text: - `Restructured Text`_ is fully supported, and is extended in a way - that is natural both for a wiki and for reST. Any links (like - ``link_``) that cannot be resolved internally in a document are - considered to be wiki links to other pages on the system. - - Because Restructured Text is a full-featured markup language, it - does not cut corners based on a limited initial domain. It is not - wiki-specific, and does not produce output that is in any way - wiki-specific or of inferior layout. It does not use WikiNames. - -.. _Restructured Text: http://docutils.sourceforge.net/rst.html - -WYSIWYG Editor: - WYSIWYG editing using Xinha_. Includes a custom button to - insert intra-wiki links. - -.. _Xinha: http://docutils.sourceforge.net/rst.html - -External Editor: - The wiki supports using an `external editor`_ to edit content, to - avoid through-the-web editing (browser textareas are poor for - editing). - -.. _external editor: http://www.plope.com/software/ExternalEditor - -Universal Edit Button: - The wiki also supports the `Universal Edit Button`_ to edit content. - -.. _Universal Edit Button: http://universaleditbutton.org - -Searching: - Text and title searching. Incremental page searches - when finding pages to link to. - -Non-text pages: - "Pages" can be of any type, including binary and image files - (e.g., PDF). - -RSS Feed: - XML RSS feeds for `new pages`_ and `recent changes`_ - available for updates of the site. - -.. _new pages: /feeds/new_pages.xml -.. _recent changes: /feeds/recent_changes.xml - -Versioning: - All edits are versioned, and a history is kept. You can compare - versions. - -Administration: - Administrators can easily rollback changes, remove old versions - or remove user accounts. This helps a lot in cleaning from spam. - -Captchas: - You can plug in your own captcha for creating user accounts. - Currently there are captchas suited for Python programmers and - musicians. Robots and stupid spammers will not be able to - create user accounts, but for serious users it is easy. - -Static publishing: - The wiki can be published to static files, with URLs that will be - analogous to the in-wiki URLs. You can provide your own Cheetah - template to control the page content. Pages are regenerated as - their wiki equivalents are edited. Links to pages that do not - exist are removed from the static published content; creating - those pages will regenerate any pages that contained the - previously dangling link. - - These pages can be statically published both locally, and to an - SFTP or FTP account. - -Virtual domains: - Multiple domains can be served off a single installation; both - aliases (e.g., http://wiki.webwareforpython.org and - http://wiki.w4py.org), or entirely separate sites - (http://wiki.cheetahtemplate.org). - -Configuration: - Configuration is done through a simple ``.ini`` file. - Configuration can be global or domain-specific. One possibility - would be to configure two domains to point to the same content, - but for one domain to be marked read only. As this wiki becomes - more configurable, this could allow for it to grow into more - CMS-like features. - -Simple persistence: - All data is stored in simple text files; pages in ``.txt`` files, - metadata in rfc822_-style files, RSS in its own RSS file - (as the canonical source of data), configuration in an ``.ini`` - file. There is no binary data and no pickles (except for some - indexing which is done in bdb files, which are not canonical sources - of data -- you can delete and regenerate those files at any - time). This makes upgrading easy -- you only need to update the - code, not the data. - -.. _rfc822: http://www.ietf.org/rfc/rfc0822.txt - -No requirements: - Doesn't require any special tools, like RCS. It is self-contained - (beyond the Webware requirement, and modules in the standard - Python library). Though it has grown some other optional - libraries. - -Strong object model: - The logic for the *wiki* is separated from the logic for the - *interface*. The wiki is written in independent modules which are - not bound to Webware or any particular interface. - -`Related Terms`_: - Another way of relating wiki pages to each other. More on its - page. This is the beginning of blog functionality. - -Comments: - Comments are a certain kind of related page -- both individually - addressable and editable, but are also shown inline in the - commented-upon page. diff --git a/lib/distpages/wikimarkup.meta b/lib/distpages/wikimarkup.meta deleted file mode 100644 index 8e94d79..0000000 --- a/lib/distpages/wikimarkup.meta +++ /dev/null @@ -1 +0,0 @@ -title: Wiki Markup \ No newline at end of file diff --git a/lib/distpages/wikimarkup.txt b/lib/distpages/wikimarkup.txt deleted file mode 100644 index 383a9ba..0000000 --- a/lib/distpages/wikimarkup.txt +++ /dev/null @@ -1,24 +0,0 @@ -In this Wiki, you can use the reStructuredText_ markup language. - -.. _restructuredText: http://docutils.sourceforge.net/rst.html - -For a quick introduction, please read: - -* `ReStructuredText Primer`_ -* `Quick reStructuredText`_ - -.. _`ReStructuredText Primer`: http://docutils.sourceforge.net/docs/user/rst/quickstart.html -.. _`Quick reStructuredText`: http://docutils.sourceforge.net/docs/user/rst/quickref.html - -reStructuredText parser is a component of Docutils_. - -For reference, please use the documentation available on the Docutils website: - -* `Introduction to reStructuredText`_ -* `reStructuredText Markup Specification`_ -* `Complete reStructuredText documentation`_ - -.. _Docutils: http://docutils.sourceforge.net -.. _`Introduction to reStructuredText`: _http://docutils.sourceforge.net/docs/ref/rst/introduction.html -.. _`reStructuredText Markup Specification`: http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html -.. _`Complete reStructuredText documentation`: http://docutils.sourceforge.net/rst.html diff --git a/lib/distpages/wikisandbox.meta b/lib/distpages/wikisandbox.meta deleted file mode 100644 index 3e46edd..0000000 --- a/lib/distpages/wikisandbox.meta +++ /dev/null @@ -1 +0,0 @@ -title: Wiki Sandbox \ No newline at end of file diff --git a/lib/distpages/wikisandbox.txt b/lib/distpages/wikisandbox.txt deleted file mode 100644 index 3e48abf..0000000 --- a/lib/distpages/wikisandbox.txt +++ /dev/null @@ -1,3 +0,0 @@ -This is the Wiki Sandbox. - -Feel free to play with the wiki here. diff --git a/lib/distpages/wikitodo.meta b/lib/distpages/wikitodo.meta deleted file mode 100644 index e893323..0000000 --- a/lib/distpages/wikitodo.meta +++ /dev/null @@ -1 +0,0 @@ -title: Wiki ToDo \ No newline at end of file diff --git a/lib/distpages/wikitodo.txt b/lib/distpages/wikitodo.txt deleted file mode 100644 index 4182595..0000000 --- a/lib/distpages/wikitodo.txt +++ /dev/null @@ -1,182 +0,0 @@ -Ideas for future Improvement ----------------------------- - -Some things I have in mind, though it's very unsure when and if ever: - -* Templating (configurable on a per-domain basis). Probably with Cheetah_. - Choosable in some fashion as well, particularly a "printable" template. - -* More Indexing (backlinks are indexed) - -* More configuration options to control output and suppress links. - -* Mail notification on changes (though we have RSS now) - -* Some sort of categorization (maybe with more metadata; keyword - metadata, or a more controlled form of categorization) - -* InterWiki-links - -* Locking of pages for editing - -* `Related terms`_ starts the idea, but it needs a lot more flushing - out; more on that page. - -* Look at http://www.mnot.net/python/RSS.py , see if it should - replace ``rsspersist.py`` - -* Maybe something with Atom, like described in - http://www.xml.com/pub/a/2004/04/14/atomwiki.html - -* Make implicit user logins -- with cookie or by IP address -- by user - preference. - -* Allow for deleting and moving pages. I'm thinking this would be - implemented with a new page type, application/x-redirect, and with - deletes. A rename or move would be a copy, delete, replace with - redirect. - -* Give untitled links titles by fetching the target and looking for a - ````. Maybe part of restructured text parsing, and an - addition to the WYSIWYG editor, or maybe a post-processing filter. - -* Referrer tracking. Accept trackbacks (as comments?). - -* Many RSS (or whatever format) files. One for each page (just append - ``.rss``). One for each user? One for site as well. - -* Note the many features on this page: - http://wikifeatures.wiki.taoriver.net/moin.cgi/IdeasToPlace and - also: - http://wikifeatures.wiki.taoriver.net/moin.cgi/FeatureSummaries - specifically... - -* `Fading Links`_: links change color depending on how often they are - followed. Rates not just popularity of a page, but popularity of a - connection. - -* `Automatic Translation`_. Nice because it's easy. - -* `Email Address Protection`_. Or, more generally, pluggable output - filters. In this case, a little Javascript "encryption". - -* `Fine Grained Addressing`_. This is like `Purple Number`_. - This could be added fairly easily to ``text/html`` content, but - restructured text will be harder. - -* `Event Casting`_, i.e., some sort of remote event notification. - Perhaps also trackback pings to all newly created links. - -* Obscure links behind redirects, like described here__. But only - "unapproved" links -- i.e., new links. (Note, we are already - telling search engines not to index the sandbox pages, which are - most prone to these abuses.) IP tracking and banning might also go - with this. Revert+ban (block IP, delete user account) would be - convenient, but it isn't that bad yet. - -* Boilerplate pages. There's a number of pages (growing) related to - instructions about the Wiki. When the Wiki starts up, it should - automatically install any new pages (pages that are listed in the - boilerplate, but aren't on the site). - -* Put templates, embedded content (e.g., navigation text), CSS into - Wiki. - -* Locking or edit conflict detection. - -* Links to pages that don't exist would pass the content of the anchor, - so that we have a decent idea of what the title should look like. - -* Attachments. Archives would be spiffy too -- download the archive, - or view the archive contents (maybe just a listing). Serve as a - simple code repository. (Single Python scripts are already - supported well enough.) - -* Permissions. Restrict editing, creation, commenting based on roles, - assign roles. - -* Display entire page in RSS when a page is created. If no log - message on an edit, include some sort of abbreviated diff. - -* `Machine Readable Pages`_. I.e., an XMLRPC_ interface. Should be - easy, but I'm not sure how many interesting clients there are. - -* `Near Links`_. This might mean tracking the available pages in - several wikis, and noting similarities in page names (either - matches, or near matches, like `Like Pages`_). - -.. __: http://simon.incutio.com/archive/2004/05/11/approved - -* Some notes/links on wikis and blogs: - http://www.ourpla.net/cgi-bin/pikie.cgi?WikiWeblogs - -.. _Fading Links: http://wikifeatures.wiki.taoriver.net/moin.cgi/FadingLinks -.. _Automatic Translation: http://wikifeatures.wiki.taoriver.net/moin.cgi/AutomaticTranslation -.. _Email Address Protection: http://wikifeatures.wiki.taoriver.net/moin.cgi/EmailAddressProtection -.. _Fine Grained Addressing: http://wikifeatures.wiki.taoriver.net/moin.cgi/FineGrainedAddressing -.. _Purple number: http://www.eekim.com/software/purple/purple.html -.. _Event Casting: http://wikifeatures.wiki.taoriver.net/moin.cgi/EventCasting -.. _Machine Readable Pages: http://wikifeatures.wiki.taoriver.net/moin.cgi/MachineReadablePages -.. _Near Links: http://wikifeatures.wiki.taoriver.net/moin.cgi/NearLink -.. _Like Pages: http://wiki.wordpress.org/?pagename=LikePages - -Anything related to the markup should be put into docutils -- I think -wiki links are nearly all we need in the way of extensions; everything -else can be done in docutils directly. (Though perhaps there should -be a general directive, which simply passes parameters to the Wiki to -be rendered at display time) - -Requested Features ------------------- - -Put whatever you want here. - -* Possibility to generate static pages for fast and scalable read-only - with any server. *request granted* - -* Breadcrumb links (showing your recent path through the wiki) - - IB: What's the value over the back button? - - CG: Well, if you click back then click a link, it erases the forward history, - so it's handy for that. I don't think I described it right -- it's more of - a static-length page history with most recently visited pages at the top - (and when you visit a page in that history, it pushes it back up to the top). - It lets you do non-linear history traversal. :) It's not really THAT useful, - but MoinMoin has it, and I find that it comes in handy when you want to go - back and forth between two wikipages that aren't linked directly to each other. - Plus, it's prominently visible, so you can quickly jump back 5 pages with ease! - -* Petition the docutils guys to make their formatting warning messages include - a piece (or the whole) line that generated the error. ;) - -* Javascript function that clears the "click here to search..." field when you - select it. (IB: it's supposed to do that now; works for me on Mozilla at least) - -* ReST Quick-Reference link on editing page (popup window) - -More ideas ----------- - -Things I don't really have in mind, but maybe would be a good idea: - -* Macros, like in MoinMoin_ -- would replace custom pages - (like `recent changes`_) - -* User preferences - -* Transclusion (including the contents of one page in another; - maybe it would be important with binary pages) - -* Internationalization (I have so little in-code text now, it would be - both easy and uninteresting) - -* Blog-like top-comment mode - -* Workflow. Cool, but hard. Note `Staged Commits`_. - -.. _Staged Commits: http://wikifeatures.wiki.taoriver.net/moin.cgi/StagedCommits - -.. _recent changes: recentchanges -.. _MoinMoin: http://moinmoin.wikiwikiweb.de/HelpOnMacros -.. _Cheetah: http://www.cheetahtemplate.org/ diff --git a/lib/feedparser.py b/lib/feedparser.py deleted file mode 100644 index bb802df..0000000 --- a/lib/feedparser.py +++ /dev/null @@ -1,2858 +0,0 @@ -#!/usr/bin/env python -"""Universal feed parser - -Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds - -Visit http://feedparser.org/ for the latest version -Visit http://feedparser.org/docs/ for the latest documentation - -Required: Python 2.1 or later -Recommended: Python 2.3 or later -Recommended: CJKCodecs and iconv_codec <http://cjkpython.i18n.org/> -""" - -__version__ = "4.1"# + "$Revision: 1.92 $"[11:15] + "-cvs" -__license__ = """Copyright (c) 2002-2006, Mark Pilgrim, All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE.""" -__author__ = "Mark Pilgrim <http://diveintomark.org/>" -__contributors__ = ["Jason Diamond <http://injektilo.org/>", - "John Beimler <http://john.beimler.org/>", - "Fazal Majid <http://www.majid.info/mylos/weblog/>", - "Aaron Swartz <http://aaronsw.com/>", - "Kevin Marks <http://epeus.blogspot.com/>"] -_debug = 0 - -# HTTP "User-Agent" header to send to servers when downloading feeds. -# If you are embedding feedparser in a larger application, you should -# change this to your application name and URL. -USER_AGENT = "UniversalFeedParser/%s +http://feedparser.org/" % __version__ - -# HTTP "Accept" header to send to servers when downloading feeds. If you don't -# want to send an Accept header, set this to None. -ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1" - -# List of preferred XML parsers, by SAX driver name. These will be tried first, -# but if they're not installed, Python will keep searching through its own list -# of pre-installed parsers until it finds one that supports everything we need. -PREFERRED_XML_PARSERS = ["drv_libxml2"] - -# If you want feedparser to automatically run HTML markup through HTML Tidy, set -# this to 1. Requires mxTidy <http://www.egenix.com/files/python/mxTidy.html> -# or utidylib <http://utidylib.berlios.de/>. -TIDY_MARKUP = 0 - -# List of Python interfaces for HTML Tidy, in order of preference. Only useful -# if TIDY_MARKUP = 1 -PREFERRED_TIDY_INTERFACES = ["uTidy", "mxTidy"] - -# ---------- required modules (should come with any Python distribution) ---------- -import sgmllib, re, sys, copy, urlparse, time, rfc822, types, cgi, urllib, urllib2 -try: - from cStringIO import StringIO as _StringIO -except: - from StringIO import StringIO as _StringIO - -# ---------- optional modules (feedparser will work without these, but with reduced functionality) ---------- - -# gzip is included with most Python distributions, but may not be available if you compiled your own -try: - import gzip -except: - gzip = None -try: - import zlib -except: - zlib = None - -# If a real XML parser is available, feedparser will attempt to use it. feedparser has -# been tested with the built-in SAX parser, PyXML, and libxml2. On platforms where the -# Python distribution does not come with an XML parser (such as Mac OS X 10.2 and some -# versions of FreeBSD), feedparser will quietly fall back on regex-based parsing. -try: - import xml.sax - xml.sax.make_parser(PREFERRED_XML_PARSERS) # test for valid parsers - from xml.sax.saxutils import escape as _xmlescape - _XML_AVAILABLE = 1 -except: - _XML_AVAILABLE = 0 - def _xmlescape(data): - data = data.replace('&', '&') - data = data.replace('>', '>') - data = data.replace('<', '<') - return data - -# base64 support for Atom feeds that contain embedded binary data -try: - import base64, binascii -except: - base64 = binascii = None - -# cjkcodecs and iconv_codec provide support for more character encodings. -# Both are available from http://cjkpython.i18n.org/ -try: - import cjkcodecs.aliases -except: - pass -try: - import iconv_codec -except: - pass - -# chardet library auto-detects character encodings -# Download from http://chardet.feedparser.org/ -try: - import chardet - if _debug: - import chardet.constants - chardet.constants._debug = 1 -except: - chardet = None - -# ---------- don't touch these ---------- -class ThingsNobodyCaresAboutButMe(Exception): pass -class CharacterEncodingOverride(ThingsNobodyCaresAboutButMe): pass -class CharacterEncodingUnknown(ThingsNobodyCaresAboutButMe): pass -class NonXMLContentType(ThingsNobodyCaresAboutButMe): pass -class UndeclaredNamespace(Exception): pass - -sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*') -sgmllib.special = re.compile('<!') -sgmllib.charref = re.compile('&#(x?[0-9A-Fa-f]+)[^0-9A-Fa-f]') - -SUPPORTED_VERSIONS = {'': 'unknown', - 'rss090': 'RSS 0.90', - 'rss091n': 'RSS 0.91 (Netscape)', - 'rss091u': 'RSS 0.91 (Userland)', - 'rss092': 'RSS 0.92', - 'rss093': 'RSS 0.93', - 'rss094': 'RSS 0.94', - 'rss20': 'RSS 2.0', - 'rss10': 'RSS 1.0', - 'rss': 'RSS (unknown version)', - 'atom01': 'Atom 0.1', - 'atom02': 'Atom 0.2', - 'atom03': 'Atom 0.3', - 'atom10': 'Atom 1.0', - 'atom': 'Atom (unknown version)', - 'cdf': 'CDF', - 'hotrss': 'Hot RSS' - } - -try: - UserDict = dict -except NameError: - # Python 2.1 does not have dict - from UserDict import UserDict - def dict(aList): - rc = {} - for k, v in aList: - rc[k] = v - return rc - -class FeedParserDict(UserDict): - keymap = {'channel': 'feed', - 'items': 'entries', - 'guid': 'id', - 'date': 'updated', - 'date_parsed': 'updated_parsed', - 'description': ['subtitle', 'summary'], - 'url': ['href'], - 'modified': 'updated', - 'modified_parsed': 'updated_parsed', - 'issued': 'published', - 'issued_parsed': 'published_parsed', - 'copyright': 'rights', - 'copyright_detail': 'rights_detail', - 'tagline': 'subtitle', - 'tagline_detail': 'subtitle_detail'} - def __getitem__(self, key): - if key == 'category': - return UserDict.__getitem__(self, 'tags')[0]['term'] - if key == 'categories': - return [(tag['scheme'], tag['term']) for tag in UserDict.__getitem__(self, 'tags')] - realkey = self.keymap.get(key, key) - if type(realkey) == types.ListType: - for k in realkey: - if UserDict.has_key(self, k): - return UserDict.__getitem__(self, k) - if UserDict.has_key(self, key): - return UserDict.__getitem__(self, key) - return UserDict.__getitem__(self, realkey) - - def __setitem__(self, key, value): - for k in self.keymap.keys(): - if key == k: - key = self.keymap[k] - if type(key) == types.ListType: - key = key[0] - return UserDict.__setitem__(self, key, value) - - def get(self, key, default=None): - if self.has_key(key): - return self[key] - else: - return default - - def setdefault(self, key, value): - if not self.has_key(key): - self[key] = value - return self[key] - - def has_key(self, key): - try: - return hasattr(self, key) or UserDict.has_key(self, key) - except AttributeError: - return False - - def __getattr__(self, key): - try: - return self.__dict__[key] - except KeyError: - pass - try: - assert not key.startswith('_') - return self.__getitem__(key) - except: - raise AttributeError, "object has no attribute '%s'" % key - - def __setattr__(self, key, value): - if key.startswith('_') or key == 'data': - self.__dict__[key] = value - else: - return self.__setitem__(key, value) - - def __contains__(self, key): - return self.has_key(key) - -def zopeCompatibilityHack(): - global FeedParserDict - del FeedParserDict - def FeedParserDict(aDict=None): - rc = {} - if aDict: - rc.update(aDict) - return rc - -_ebcdic_to_ascii_map = None -def _ebcdic_to_ascii(s): - global _ebcdic_to_ascii_map - if not _ebcdic_to_ascii_map: - emap = ( - 0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15, - 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31, - 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7, - 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26, - 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33, - 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94, - 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63, - 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34, - 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,201, - 202,106,107,108,109,110,111,112,113,114,203,204,205,206,207,208, - 209,126,115,116,117,118,119,120,121,122,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231, - 123,65,66,67,68,69,70,71,72,73,232,233,234,235,236,237, - 125,74,75,76,77,78,79,80,81,82,238,239,240,241,242,243, - 92,159,83,84,85,86,87,88,89,90,244,245,246,247,248,249, - 48,49,50,51,52,53,54,55,56,57,250,251,252,253,254,255 - ) - import string - _ebcdic_to_ascii_map = string.maketrans( \ - ''.join(map(chr, range(256))), ''.join(map(chr, emap))) - return s.translate(_ebcdic_to_ascii_map) - -_urifixer = re.compile('^([A-Za-z][A-Za-z0-9+-.]*://)(/*)(.*?)') -def _urljoin(base, uri): - uri = _urifixer.sub(r'\1\3', uri) - return urlparse.urljoin(base, uri) - -class _FeedParserMixin: - namespaces = {'': '', - 'http://backend.userland.com/rss': '', - 'http://blogs.law.harvard.edu/tech/rss': '', - 'http://purl.org/rss/1.0/': '', - 'http://my.netscape.com/rdf/simple/0.9/': '', - 'http://example.com/newformat#': '', - 'http://example.com/necho': '', - 'http://purl.org/echo/': '', - 'uri/of/echo/namespace#': '', - 'http://purl.org/pie/': '', - 'http://purl.org/atom/ns#': '', - 'http://www.w3.org/2005/Atom': '', - 'http://purl.org/rss/1.0/modules/rss091#': '', - - 'http://webns.net/mvcb/': 'admin', - 'http://purl.org/rss/1.0/modules/aggregation/': 'ag', - 'http://purl.org/rss/1.0/modules/annotate/': 'annotate', - 'http://media.tangent.org/rss/1.0/': 'audio', - 'http://backend.userland.com/blogChannelModule': 'blogChannel', - 'http://web.resource.org/cc/': 'cc', - 'http://backend.userland.com/creativeCommonsRssModule': 'creativeCommons', - 'http://purl.org/rss/1.0/modules/company': 'co', - 'http://purl.org/rss/1.0/modules/content/': 'content', - 'http://my.theinfo.org/changed/1.0/rss/': 'cp', - 'http://purl.org/dc/elements/1.1/': 'dc', - 'http://purl.org/dc/terms/': 'dcterms', - 'http://purl.org/rss/1.0/modules/email/': 'email', - 'http://purl.org/rss/1.0/modules/event/': 'ev', - 'http://rssnamespace.org/feedburner/ext/1.0': 'feedburner', - 'http://freshmeat.net/rss/fm/': 'fm', - 'http://xmlns.com/foaf/0.1/': 'foaf', - 'http://www.w3.org/2003/01/geo/wgs84_pos#': 'geo', - 'http://postneo.com/icbm/': 'icbm', - 'http://purl.org/rss/1.0/modules/image/': 'image', - 'http://www.itunes.com/DTDs/PodCast-1.0.dtd': 'itunes', - 'http://example.com/DTDs/PodCast-1.0.dtd': 'itunes', - 'http://purl.org/rss/1.0/modules/link/': 'l', - 'http://search.yahoo.com/mrss': 'media', - 'http://madskills.com/public/xml/rss/module/pingback/': 'pingback', - 'http://prismstandard.org/namespaces/1.2/basic/': 'prism', - 'http://www.w3.org/1999/02/22-rdf-syntax-ns#': 'rdf', - 'http://www.w3.org/2000/01/rdf-schema#': 'rdfs', - 'http://purl.org/rss/1.0/modules/reference/': 'ref', - 'http://purl.org/rss/1.0/modules/richequiv/': 'reqv', - 'http://purl.org/rss/1.0/modules/search/': 'search', - 'http://purl.org/rss/1.0/modules/slash/': 'slash', - 'http://schemas.xmlsoap.org/soap/envelope/': 'soap', - 'http://purl.org/rss/1.0/modules/servicestatus/': 'ss', - 'http://hacks.benhammersley.com/rss/streaming/': 'str', - 'http://purl.org/rss/1.0/modules/subscription/': 'sub', - 'http://purl.org/rss/1.0/modules/syndication/': 'sy', - 'http://purl.org/rss/1.0/modules/taxonomy/': 'taxo', - 'http://purl.org/rss/1.0/modules/threading/': 'thr', - 'http://purl.org/rss/1.0/modules/textinput/': 'ti', - 'http://madskills.com/public/xml/rss/module/trackback/':'trackback', - 'http://wellformedweb.org/commentAPI/': 'wfw', - 'http://purl.org/rss/1.0/modules/wiki/': 'wiki', - 'http://www.w3.org/1999/xhtml': 'xhtml', - 'http://www.w3.org/XML/1998/namespace': 'xml', - 'http://schemas.pocketsoap.com/rss/myDescModule/': 'szf' -} - _matchnamespaces = {} - - can_be_relative_uri = ['link', 'id', 'wfw_comment', 'wfw_commentrss', 'docs', 'url', 'href', 'comments', 'license', 'icon', 'logo'] - can_contain_relative_uris = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description'] - can_contain_dangerous_markup = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description'] - html_types = ['text/html', 'application/xhtml+xml'] - - def __init__(self, baseuri=None, baselang=None, encoding='utf-8'): - if _debug: sys.stderr.write('initializing FeedParser\n') - if not self._matchnamespaces: - for k, v in self.namespaces.items(): - self._matchnamespaces[k.lower()] = v - self.feeddata = FeedParserDict() # feed-level data - self.encoding = encoding # character encoding - self.entries = [] # list of entry-level data - self.version = '' # feed type/version, see SUPPORTED_VERSIONS - self.namespacesInUse = {} # dictionary of namespaces defined by the feed - - # the following are used internally to track state; - # this is really out of control and should be refactored - self.infeed = 0 - self.inentry = 0 - self.incontent = 0 - self.intextinput = 0 - self.inimage = 0 - self.inauthor = 0 - self.incontributor = 0 - self.inpublisher = 0 - self.insource = 0 - self.sourcedata = FeedParserDict() - self.contentparams = FeedParserDict() - self._summaryKey = None - self.namespacemap = {} - self.elementstack = [] - self.basestack = [] - self.langstack = [] - self.baseuri = baseuri or '' - self.lang = baselang or None - if baselang: - self.feeddata['language'] = baselang - - def unknown_starttag(self, tag, attrs): - if _debug: sys.stderr.write('start %s with %s\n' % (tag, attrs)) - # normalize attrs - attrs = [(k.lower(), v) for k, v in attrs] - attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs] - - # track xml:base and xml:lang - attrsD = dict(attrs) - baseuri = attrsD.get('xml:base', attrsD.get('base')) or self.baseuri - self.baseuri = _urljoin(self.baseuri, baseuri) - lang = attrsD.get('xml:lang', attrsD.get('lang')) - if lang == '': - # xml:lang could be explicitly set to '', we need to capture that - lang = None - elif lang is None: - # if no xml:lang is specified, use parent lang - lang = self.lang - if lang: - if tag in ('feed', 'rss', 'rdf:RDF'): - self.feeddata['language'] = lang - self.lang = lang - self.basestack.append(self.baseuri) - self.langstack.append(lang) - - # track namespaces - for prefix, uri in attrs: - if prefix.startswith('xmlns:'): - self.trackNamespace(prefix[6:], uri) - elif prefix == 'xmlns': - self.trackNamespace(None, uri) - - # track inline content - if self.incontent and self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'): - # element declared itself as escaped markup, but it isn't really - self.contentparams['type'] = 'application/xhtml+xml' - if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml': - # Note: probably shouldn't simply recreate localname here, but - # our namespace handling isn't actually 100% correct in cases where - # the feed redefines the default namespace (which is actually - # the usual case for inline content, thanks Sam), so here we - # cheat and just reconstruct the element based on localname - # because that compensates for the bugs in our namespace handling. - # This will horribly munge inline content with non-empty qnames, - # but nobody actually does that, so I'm not fixing it. - tag = tag.split(':')[-1] - return self.handle_data('<%s%s>' % (tag, ''.join([' %s="%s"' % t for t in attrs])), escape=0) - - # match namespaces - if tag.find(':') <> -1: - prefix, suffix = tag.split(':', 1) - else: - prefix, suffix = '', tag - prefix = self.namespacemap.get(prefix, prefix) - if prefix: - prefix = prefix + '_' - - # special hack for better tracking of empty textinput/image elements in illformed feeds - if (not prefix) and tag not in ('title', 'link', 'description', 'name'): - self.intextinput = 0 - if (not prefix) and tag not in ('title', 'link', 'description', 'url', 'href', 'width', 'height'): - self.inimage = 0 - - # call special handler (if defined) or default handler - methodname = '_start_' + prefix + suffix - try: - method = getattr(self, methodname) - return method(attrsD) - except AttributeError: - return self.push(prefix + suffix, 1) - - def unknown_endtag(self, tag): - if _debug: sys.stderr.write('end %s\n' % tag) - # match namespaces - if tag.find(':') <> -1: - prefix, suffix = tag.split(':', 1) - else: - prefix, suffix = '', tag - prefix = self.namespacemap.get(prefix, prefix) - if prefix: - prefix = prefix + '_' - - # call special handler (if defined) or default handler - methodname = '_end_' + prefix + suffix - try: - method = getattr(self, methodname) - method() - except AttributeError: - self.pop(prefix + suffix) - - # track inline content - if self.incontent and self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'): - # element declared itself as escaped markup, but it isn't really - self.contentparams['type'] = 'application/xhtml+xml' - if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml': - tag = tag.split(':')[-1] - self.handle_data('</%s>' % tag, escape=0) - - # track xml:base and xml:lang going out of scope - if self.basestack: - self.basestack.pop() - if self.basestack and self.basestack[-1]: - self.baseuri = self.basestack[-1] - if self.langstack: - self.langstack.pop() - if self.langstack: # and (self.langstack[-1] is not None): - self.lang = self.langstack[-1] - - def handle_charref(self, ref): - # called for each character reference, e.g. for ' ', ref will be '160' - if not self.elementstack: return - ref = ref.lower() - if ref in ('34', '38', '39', '60', '62', 'x22', 'x26', 'x27', 'x3c', 'x3e'): - text = '&#%s;' % ref - else: - if ref[0] == 'x': - c = int(ref[1:], 16) - else: - c = int(ref) - text = unichr(c).encode('utf-8') - self.elementstack[-1][2].append(text) - - def handle_entityref(self, ref): - # called for each entity reference, e.g. for '©', ref will be 'copy' - if not self.elementstack: return - if _debug: sys.stderr.write('entering handle_entityref with %s\n' % ref) - if ref in ('lt', 'gt', 'quot', 'amp', 'apos'): - text = '&%s;' % ref - else: - # entity resolution graciously donated by Aaron Swartz - def name2cp(k): - import htmlentitydefs - if hasattr(htmlentitydefs, 'name2codepoint'): # requires Python 2.3 - return htmlentitydefs.name2codepoint[k] - k = htmlentitydefs.entitydefs[k] - if k.startswith('&#') and k.endswith(';'): - return int(k[2:-1]) # not in latin-1 - return ord(k) - try: name2cp(ref) - except KeyError: text = '&%s;' % ref - else: text = unichr(name2cp(ref)).encode('utf-8') - self.elementstack[-1][2].append(text) - - def handle_data(self, text, escape=1): - # called for each block of plain text, i.e. outside of any tag and - # not containing any character or entity references - if not self.elementstack: return - if escape and self.contentparams.get('type') == 'application/xhtml+xml': - text = _xmlescape(text) - self.elementstack[-1][2].append(text) - - def handle_comment(self, text): - # called for each comment, e.g. <!-- insert message here --> - pass - - def handle_pi(self, text): - # called for each processing instruction, e.g. <?instruction> - pass - - def handle_decl(self, text): - pass - - def parse_declaration(self, i): - # override internal declaration handler to handle CDATA blocks - if _debug: sys.stderr.write('entering parse_declaration\n') - if self.rawdata[i:i+9] == '<![CDATA[': - k = self.rawdata.find(']]>', i) - if k == -1: k = len(self.rawdata) - self.handle_data(_xmlescape(self.rawdata[i+9:k]), 0) - return k+3 - else: - k = self.rawdata.find('>', i) - return k+1 - - def mapContentType(self, contentType): - contentType = contentType.lower() - if contentType == 'text': - contentType = 'text/plain' - elif contentType == 'html': - contentType = 'text/html' - elif contentType == 'xhtml': - contentType = 'application/xhtml+xml' - return contentType - - def trackNamespace(self, prefix, uri): - loweruri = uri.lower() - if (prefix, loweruri) == (None, 'http://my.netscape.com/rdf/simple/0.9/') and not self.version: - self.version = 'rss090' - if loweruri == 'http://purl.org/rss/1.0/' and not self.version: - self.version = 'rss10' - if loweruri == 'http://www.w3.org/2005/atom' and not self.version: - self.version = 'atom10' - if loweruri.find('backend.userland.com/rss') <> -1: - # match any backend.userland.com namespace - uri = 'http://backend.userland.com/rss' - loweruri = uri - if self._matchnamespaces.has_key(loweruri): - self.namespacemap[prefix] = self._matchnamespaces[loweruri] - self.namespacesInUse[self._matchnamespaces[loweruri]] = uri - else: - self.namespacesInUse[prefix or ''] = uri - - def resolveURI(self, uri): - return _urljoin(self.baseuri or '', uri) - - def decodeEntities(self, element, data): - return data - - def push(self, element, expectingText): - self.elementstack.append([element, expectingText, []]) - - def pop(self, element, stripWhitespace=1): - if not self.elementstack: return - if self.elementstack[-1][0] != element: return - - element, expectingText, pieces = self.elementstack.pop() - output = ''.join(pieces) - if stripWhitespace: - output = output.strip() - if not expectingText: return output - - # decode base64 content - if base64 and self.contentparams.get('base64', 0): - try: - output = base64.decodestring(output) - except binascii.Error: - pass - except binascii.Incomplete: - pass - - # resolve relative URIs - if (element in self.can_be_relative_uri) and output: - output = self.resolveURI(output) - - # decode entities within embedded markup - if not self.contentparams.get('base64', 0): - output = self.decodeEntities(element, output) - - # remove temporary cruft from contentparams - try: - del self.contentparams['mode'] - except KeyError: - pass - try: - del self.contentparams['base64'] - except KeyError: - pass - - # resolve relative URIs within embedded markup - if self.mapContentType(self.contentparams.get('type', 'text/html')) in self.html_types: - if element in self.can_contain_relative_uris: - output = _resolveRelativeURIs(output, self.baseuri, self.encoding) - - # sanitize embedded markup - if self.mapContentType(self.contentparams.get('type', 'text/html')) in self.html_types: - if element in self.can_contain_dangerous_markup: - output = _sanitizeHTML(output, self.encoding) - - if self.encoding and type(output) != type(u''): - try: - output = unicode(output, self.encoding) - except: - pass - - # categories/tags/keywords/whatever are handled in _end_category - if element == 'category': - return output - - # store output in appropriate place(s) - if self.inentry and not self.insource: - if element == 'content': - self.entries[-1].setdefault(element, []) - contentparams = copy.deepcopy(self.contentparams) - contentparams['value'] = output - self.entries[-1][element].append(contentparams) - elif element == 'link': - self.entries[-1][element] = output - if output: - self.entries[-1]['links'][-1]['href'] = output - else: - if element == 'description': - element = 'summary' - self.entries[-1][element] = output - if self.incontent: - contentparams = copy.deepcopy(self.contentparams) - contentparams['value'] = output - self.entries[-1][element + '_detail'] = contentparams - elif (self.infeed or self.insource) and (not self.intextinput) and (not self.inimage): - context = self._getContext() - if element == 'description': - element = 'subtitle' - context[element] = output - if element == 'link': - context['links'][-1]['href'] = output - elif self.incontent: - contentparams = copy.deepcopy(self.contentparams) - contentparams['value'] = output - context[element + '_detail'] = contentparams - return output - - def pushContent(self, tag, attrsD, defaultContentType, expectingText): - self.incontent += 1 - self.contentparams = FeedParserDict({ - 'type': self.mapContentType(attrsD.get('type', defaultContentType)), - 'language': self.lang, - 'base': self.baseuri}) - self.contentparams['base64'] = self._isBase64(attrsD, self.contentparams) - self.push(tag, expectingText) - - def popContent(self, tag): - value = self.pop(tag) - self.incontent -= 1 - self.contentparams.clear() - return value - - def _mapToStandardPrefix(self, name): - colonpos = name.find(':') - if colonpos <> -1: - prefix = name[:colonpos] - suffix = name[colonpos+1:] - prefix = self.namespacemap.get(prefix, prefix) - name = prefix + ':' + suffix - return name - - def _getAttribute(self, attrsD, name): - return attrsD.get(self._mapToStandardPrefix(name)) - - def _isBase64(self, attrsD, contentparams): - if attrsD.get('mode', '') == 'base64': - return 1 - if self.contentparams['type'].startswith('text/'): - return 0 - if self.contentparams['type'].endswith('+xml'): - return 0 - if self.contentparams['type'].endswith('/xml'): - return 0 - return 1 - - def _itsAnHrefDamnIt(self, attrsD): - href = attrsD.get('url', attrsD.get('uri', attrsD.get('href', None))) - if href: - try: - del attrsD['url'] - except KeyError: - pass - try: - del attrsD['uri'] - except KeyError: - pass - attrsD['href'] = href - return attrsD - - def _save(self, key, value): - context = self._getContext() - context.setdefault(key, value) - - def _start_rss(self, attrsD): - versionmap = {'0.91': 'rss091u', - '0.92': 'rss092', - '0.93': 'rss093', - '0.94': 'rss094'} - if not self.version: - attr_version = attrsD.get('version', '') - version = versionmap.get(attr_version) - if version: - self.version = version - elif attr_version.startswith('2.'): - self.version = 'rss20' - else: - self.version = 'rss' - - def _start_dlhottitles(self, attrsD): - self.version = 'hotrss' - - def _start_channel(self, attrsD): - self.infeed = 1 - self._cdf_common(attrsD) - _start_feedinfo = _start_channel - - def _cdf_common(self, attrsD): - if attrsD.has_key('lastmod'): - self._start_modified({}) - self.elementstack[-1][-1] = attrsD['lastmod'] - self._end_modified() - if attrsD.has_key('href'): - self._start_link({}) - self.elementstack[-1][-1] = attrsD['href'] - self._end_link() - - def _start_feed(self, attrsD): - self.infeed = 1 - versionmap = {'0.1': 'atom01', - '0.2': 'atom02', - '0.3': 'atom03'} - if not self.version: - attr_version = attrsD.get('version') - version = versionmap.get(attr_version) - if version: - self.version = version - else: - self.version = 'atom' - - def _end_channel(self): - self.infeed = 0 - _end_feed = _end_channel - - def _start_image(self, attrsD): - self.inimage = 1 - self.push('image', 0) - context = self._getContext() - context.setdefault('image', FeedParserDict()) - - def _end_image(self): - self.pop('image') - self.inimage = 0 - - def _start_textinput(self, attrsD): - self.intextinput = 1 - self.push('textinput', 0) - context = self._getContext() - context.setdefault('textinput', FeedParserDict()) - _start_textInput = _start_textinput - - def _end_textinput(self): - self.pop('textinput') - self.intextinput = 0 - _end_textInput = _end_textinput - - def _start_author(self, attrsD): - self.inauthor = 1 - self.push('author', 1) - _start_managingeditor = _start_author - _start_dc_author = _start_author - _start_dc_creator = _start_author - _start_itunes_author = _start_author - - def _end_author(self): - self.pop('author') - self.inauthor = 0 - self._sync_author_detail() - _end_managingeditor = _end_author - _end_dc_author = _end_author - _end_dc_creator = _end_author - _end_itunes_author = _end_author - - def _start_itunes_owner(self, attrsD): - self.inpublisher = 1 - self.push('publisher', 0) - - def _end_itunes_owner(self): - self.pop('publisher') - self.inpublisher = 0 - self._sync_author_detail('publisher') - - def _start_contributor(self, attrsD): - self.incontributor = 1 - context = self._getContext() - context.setdefault('contributors', []) - context['contributors'].append(FeedParserDict()) - self.push('contributor', 0) - - def _end_contributor(self): - self.pop('contributor') - self.incontributor = 0 - - def _start_dc_contributor(self, attrsD): - self.incontributor = 1 - context = self._getContext() - context.setdefault('contributors', []) - context['contributors'].append(FeedParserDict()) - self.push('name', 0) - - def _end_dc_contributor(self): - self._end_name() - self.incontributor = 0 - - def _start_name(self, attrsD): - self.push('name', 0) - _start_itunes_name = _start_name - - def _end_name(self): - value = self.pop('name') - if self.inpublisher: - self._save_author('name', value, 'publisher') - elif self.inauthor: - self._save_author('name', value) - elif self.incontributor: - self._save_contributor('name', value) - elif self.intextinput: - context = self._getContext() - context['textinput']['name'] = value - _end_itunes_name = _end_name - - def _start_width(self, attrsD): - self.push('width', 0) - - def _end_width(self): - value = self.pop('width') - try: - value = int(value) - except: - value = 0 - if self.inimage: - context = self._getContext() - context['image']['width'] = value - - def _start_height(self, attrsD): - self.push('height', 0) - - def _end_height(self): - value = self.pop('height') - try: - value = int(value) - except: - value = 0 - if self.inimage: - context = self._getContext() - context['image']['height'] = value - - def _start_url(self, attrsD): - self.push('href', 1) - _start_homepage = _start_url - _start_uri = _start_url - - def _end_url(self): - value = self.pop('href') - if self.inauthor: - self._save_author('href', value) - elif self.incontributor: - self._save_contributor('href', value) - elif self.inimage: - context = self._getContext() - context['image']['href'] = value - elif self.intextinput: - context = self._getContext() - context['textinput']['link'] = value - _end_homepage = _end_url - _end_uri = _end_url - - def _start_email(self, attrsD): - self.push('email', 0) - _start_itunes_email = _start_email - - def _end_email(self): - value = self.pop('email') - if self.inpublisher: - self._save_author('email', value, 'publisher') - elif self.inauthor: - self._save_author('email', value) - elif self.incontributor: - self._save_contributor('email', value) - _end_itunes_email = _end_email - - def _getContext(self): - if self.insource: - context = self.sourcedata - elif self.inentry: - context = self.entries[-1] - else: - context = self.feeddata - return context - - def _save_author(self, key, value, prefix='author'): - context = self._getContext() - context.setdefault(prefix + '_detail', FeedParserDict()) - context[prefix + '_detail'][key] = value - self._sync_author_detail() - - def _save_contributor(self, key, value): - context = self._getContext() - context.setdefault('contributors', [FeedParserDict()]) - context['contributors'][-1][key] = value - - def _sync_author_detail(self, key='author'): - context = self._getContext() - detail = context.get('%s_detail' % key) - if detail: - name = detail.get('name') - email = detail.get('email') - if name and email: - context[key] = '%s (%s)' % (name, email) - elif name: - context[key] = name - elif email: - context[key] = email - else: - author = context.get(key) - if not author: return - emailmatch = re.search(r'''(([a-zA-Z0-9\_\-\.\+]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([a-zA-Z0-9\-]+\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\]?))''', author) - if not emailmatch: return - email = emailmatch.group(0) - # probably a better way to do the following, but it passes all the tests - author = author.replace(email, '') - author = author.replace('()', '') - author = author.strip() - if author and (author[0] == '('): - author = author[1:] - if author and (author[-1] == ')'): - author = author[:-1] - author = author.strip() - context.setdefault('%s_detail' % key, FeedParserDict()) - context['%s_detail' % key]['name'] = author - context['%s_detail' % key]['email'] = email - - def _start_subtitle(self, attrsD): - self.pushContent('subtitle', attrsD, 'text/plain', 1) - _start_tagline = _start_subtitle - _start_itunes_subtitle = _start_subtitle - - def _end_subtitle(self): - self.popContent('subtitle') - _end_tagline = _end_subtitle - _end_itunes_subtitle = _end_subtitle - - def _start_rights(self, attrsD): - self.pushContent('rights', attrsD, 'text/plain', 1) - _start_dc_rights = _start_rights - _start_copyright = _start_rights - - def _end_rights(self): - self.popContent('rights') - _end_dc_rights = _end_rights - _end_copyright = _end_rights - - def _start_item(self, attrsD): - self.entries.append(FeedParserDict()) - self.push('item', 0) - self.inentry = 1 - self.guidislink = 0 - id = self._getAttribute(attrsD, 'rdf:about') - if id: - context = self._getContext() - context['id'] = id - self._cdf_common(attrsD) - _start_entry = _start_item - _start_product = _start_item - - def _end_item(self): - self.pop('item') - self.inentry = 0 - _end_entry = _end_item - - def _start_dc_language(self, attrsD): - self.push('language', 1) - _start_language = _start_dc_language - - def _end_dc_language(self): - self.lang = self.pop('language') - _end_language = _end_dc_language - - def _start_dc_publisher(self, attrsD): - self.push('publisher', 1) - _start_webmaster = _start_dc_publisher - - def _end_dc_publisher(self): - self.pop('publisher') - self._sync_author_detail('publisher') - _end_webmaster = _end_dc_publisher - - def _start_published(self, attrsD): - self.push('published', 1) - _start_dcterms_issued = _start_published - _start_issued = _start_published - - def _end_published(self): - value = self.pop('published') - self._save('published_parsed', _parse_date(value)) - _end_dcterms_issued = _end_published - _end_issued = _end_published - - def _start_updated(self, attrsD): - self.push('updated', 1) - _start_modified = _start_updated - _start_dcterms_modified = _start_updated - _start_pubdate = _start_updated - _start_dc_date = _start_updated - - def _end_updated(self): - value = self.pop('updated') - parsed_value = _parse_date(value) - self._save('updated_parsed', parsed_value) - _end_modified = _end_updated - _end_dcterms_modified = _end_updated - _end_pubdate = _end_updated - _end_dc_date = _end_updated - - def _start_created(self, attrsD): - self.push('created', 1) - _start_dcterms_created = _start_created - - def _end_created(self): - value = self.pop('created') - self._save('created_parsed', _parse_date(value)) - _end_dcterms_created = _end_created - - def _start_expirationdate(self, attrsD): - self.push('expired', 1) - - def _end_expirationdate(self): - self._save('expired_parsed', _parse_date(self.pop('expired'))) - - def _start_cc_license(self, attrsD): - self.push('license', 1) - value = self._getAttribute(attrsD, 'rdf:resource') - if value: - self.elementstack[-1][2].append(value) - self.pop('license') - - def _start_creativecommons_license(self, attrsD): - self.push('license', 1) - - def _end_creativecommons_license(self): - self.pop('license') - - def _addTag(self, term, scheme, label): - context = self._getContext() - tags = context.setdefault('tags', []) - if (not term) and (not scheme) and (not label): return - value = FeedParserDict({'term': term, 'scheme': scheme, 'label': label}) - if value not in tags: - tags.append(FeedParserDict({'term': term, 'scheme': scheme, 'label': label})) - - def _start_category(self, attrsD): - if _debug: sys.stderr.write('entering _start_category with %s\n' % repr(attrsD)) - term = attrsD.get('term') - scheme = attrsD.get('scheme', attrsD.get('domain')) - label = attrsD.get('label') - self._addTag(term, scheme, label) - self.push('category', 1) - _start_dc_subject = _start_category - _start_keywords = _start_category - - def _end_itunes_keywords(self): - for term in self.pop('itunes_keywords').split(): - self._addTag(term, 'http://www.itunes.com/', None) - - def _start_itunes_category(self, attrsD): - self._addTag(attrsD.get('text'), 'http://www.itunes.com/', None) - self.push('category', 1) - - def _end_category(self): - value = self.pop('category') - if not value: return - context = self._getContext() - tags = context['tags'] - if value and len(tags) and not tags[-1]['term']: - tags[-1]['term'] = value - else: - self._addTag(value, None, None) - _end_dc_subject = _end_category - _end_keywords = _end_category - _end_itunes_category = _end_category - - def _start_cloud(self, attrsD): - self._getContext()['cloud'] = FeedParserDict(attrsD) - - def _start_link(self, attrsD): - attrsD.setdefault('rel', 'alternate') - attrsD.setdefault('type', 'text/html') - attrsD = self._itsAnHrefDamnIt(attrsD) - if attrsD.has_key('href'): - attrsD['href'] = self.resolveURI(attrsD['href']) - expectingText = self.infeed or self.inentry or self.insource - context = self._getContext() - context.setdefault('links', []) - context['links'].append(FeedParserDict(attrsD)) - if attrsD['rel'] == 'enclosure': - self._start_enclosure(attrsD) - if attrsD.has_key('href'): - expectingText = 0 - if (attrsD.get('rel') == 'alternate') and (self.mapContentType(attrsD.get('type')) in self.html_types): - context['link'] = attrsD['href'] - else: - self.push('link', expectingText) - _start_producturl = _start_link - - def _end_link(self): - value = self.pop('link') - context = self._getContext() - if self.intextinput: - context['textinput']['link'] = value - if self.inimage: - context['image']['link'] = value - _end_producturl = _end_link - - def _start_guid(self, attrsD): - self.guidislink = (attrsD.get('ispermalink', 'true') == 'true') - self.push('id', 1) - - def _end_guid(self): - value = self.pop('id') - self._save('guidislink', self.guidislink and not self._getContext().has_key('link')) - if self.guidislink: - # guid acts as link, but only if 'ispermalink' is not present or is 'true', - # and only if the item doesn't already have a link element - self._save('link', value) - - def _start_title(self, attrsD): - self.pushContent('title', attrsD, 'text/plain', self.infeed or self.inentry or self.insource) - _start_dc_title = _start_title - _start_media_title = _start_title - - def _end_title(self): - value = self.popContent('title') - context = self._getContext() - if self.intextinput: - context['textinput']['title'] = value - elif self.inimage: - context['image']['title'] = value - _end_dc_title = _end_title - _end_media_title = _end_title - - def _start_description(self, attrsD): - context = self._getContext() - if context.has_key('summary'): - self._summaryKey = 'content' - self._start_content(attrsD) - else: - self.pushContent('description', attrsD, 'text/html', self.infeed or self.inentry or self.insource) - - def _start_abstract(self, attrsD): - self.pushContent('description', attrsD, 'text/plain', self.infeed or self.inentry or self.insource) - - def _end_description(self): - if self._summaryKey == 'content': - self._end_content() - else: - value = self.popContent('description') - context = self._getContext() - if self.intextinput: - context['textinput']['description'] = value - elif self.inimage: - context['image']['description'] = value - self._summaryKey = None - _end_abstract = _end_description - - def _start_info(self, attrsD): - self.pushContent('info', attrsD, 'text/plain', 1) - _start_feedburner_browserfriendly = _start_info - - def _end_info(self): - self.popContent('info') - _end_feedburner_browserfriendly = _end_info - - def _start_generator(self, attrsD): - if attrsD: - attrsD = self._itsAnHrefDamnIt(attrsD) - if attrsD.has_key('href'): - attrsD['href'] = self.resolveURI(attrsD['href']) - self._getContext()['generator_detail'] = FeedParserDict(attrsD) - self.push('generator', 1) - - def _end_generator(self): - value = self.pop('generator') - context = self._getContext() - if context.has_key('generator_detail'): - context['generator_detail']['name'] = value - - def _start_admin_generatoragent(self, attrsD): - self.push('generator', 1) - value = self._getAttribute(attrsD, 'rdf:resource') - if value: - self.elementstack[-1][2].append(value) - self.pop('generator') - self._getContext()['generator_detail'] = FeedParserDict({'href': value}) - - def _start_admin_errorreportsto(self, attrsD): - self.push('errorreportsto', 1) - value = self._getAttribute(attrsD, 'rdf:resource') - if value: - self.elementstack[-1][2].append(value) - self.pop('errorreportsto') - - def _start_summary(self, attrsD): - context = self._getContext() - if context.has_key('summary'): - self._summaryKey = 'content' - self._start_content(attrsD) - else: - self._summaryKey = 'summary' - self.pushContent(self._summaryKey, attrsD, 'text/plain', 1) - _start_itunes_summary = _start_summary - - def _end_summary(self): - if self._summaryKey == 'content': - self._end_content() - else: - self.popContent(self._summaryKey or 'summary') - self._summaryKey = None - _end_itunes_summary = _end_summary - - def _start_enclosure(self, attrsD): - attrsD = self._itsAnHrefDamnIt(attrsD) - self._getContext().setdefault('enclosures', []).append(FeedParserDict(attrsD)) - href = attrsD.get('href') - if href: - context = self._getContext() - if not context.get('id'): - context['id'] = href - - def _start_source(self, attrsD): - self.insource = 1 - - def _end_source(self): - self.insource = 0 - self._getContext()['source'] = copy.deepcopy(self.sourcedata) - self.sourcedata.clear() - - def _start_content(self, attrsD): - self.pushContent('content', attrsD, 'text/plain', 1) - src = attrsD.get('src') - if src: - self.contentparams['src'] = src - self.push('content', 1) - - def _start_prodlink(self, attrsD): - self.pushContent('content', attrsD, 'text/html', 1) - - def _start_body(self, attrsD): - self.pushContent('content', attrsD, 'application/xhtml+xml', 1) - _start_xhtml_body = _start_body - - def _start_content_encoded(self, attrsD): - self.pushContent('content', attrsD, 'text/html', 1) - _start_fullitem = _start_content_encoded - - def _end_content(self): - copyToDescription = self.mapContentType(self.contentparams.get('type')) in (['text/plain'] + self.html_types) - value = self.popContent('content') - if copyToDescription: - self._save('description', value) - _end_body = _end_content - _end_xhtml_body = _end_content - _end_content_encoded = _end_content - _end_fullitem = _end_content - _end_prodlink = _end_content - - def _start_itunes_image(self, attrsD): - self.push('itunes_image', 0) - self._getContext()['image'] = FeedParserDict({'href': attrsD.get('href')}) - _start_itunes_link = _start_itunes_image - - def _end_itunes_block(self): - value = self.pop('itunes_block', 0) - self._getContext()['itunes_block'] = (value == 'yes') and 1 or 0 - - def _end_itunes_explicit(self): - value = self.pop('itunes_explicit', 0) - self._getContext()['itunes_explicit'] = (value == 'yes') and 1 or 0 - -if _XML_AVAILABLE: - class _StrictFeedParser(_FeedParserMixin, xml.sax.handler.ContentHandler): - def __init__(self, baseuri, baselang, encoding): - if _debug: sys.stderr.write('trying StrictFeedParser\n') - xml.sax.handler.ContentHandler.__init__(self) - _FeedParserMixin.__init__(self, baseuri, baselang, encoding) - self.bozo = 0 - self.exc = None - - def startPrefixMapping(self, prefix, uri): - self.trackNamespace(prefix, uri) - - def startElementNS(self, name, qname, attrs): - namespace, localname = name - lowernamespace = str(namespace or '').lower() - if lowernamespace.find('backend.userland.com/rss') <> -1: - # match any backend.userland.com namespace - namespace = 'http://backend.userland.com/rss' - lowernamespace = namespace - if qname and qname.find(':') > 0: - givenprefix = qname.split(':')[0] - else: - givenprefix = None - prefix = self._matchnamespaces.get(lowernamespace, givenprefix) - if givenprefix and (prefix == None or (prefix == '' and lowernamespace == '')) and not self.namespacesInUse.has_key(givenprefix): - raise UndeclaredNamespace, "'%s' is not associated with a namespace" % givenprefix - if prefix: - localname = prefix + ':' + localname - localname = str(localname).lower() - if _debug: sys.stderr.write('startElementNS: qname = %s, namespace = %s, givenprefix = %s, prefix = %s, attrs = %s, localname = %s\n' % (qname, namespace, givenprefix, prefix, attrs.items(), localname)) - - # qname implementation is horribly broken in Python 2.1 (it - # doesn't report any), and slightly broken in Python 2.2 (it - # doesn't report the xml: namespace). So we match up namespaces - # with a known list first, and then possibly override them with - # the qnames the SAX parser gives us (if indeed it gives us any - # at all). Thanks to MatejC for helping me test this and - # tirelessly telling me that it didn't work yet. - attrsD = {} - for (namespace, attrlocalname), attrvalue in attrs._attrs.items(): - lowernamespace = (namespace or '').lower() - prefix = self._matchnamespaces.get(lowernamespace, '') - if prefix: - attrlocalname = prefix + ':' + attrlocalname - attrsD[str(attrlocalname).lower()] = attrvalue - for qname in attrs.getQNames(): - attrsD[str(qname).lower()] = attrs.getValueByQName(qname) - self.unknown_starttag(localname, attrsD.items()) - - def characters(self, text): - self.handle_data(text) - - def endElementNS(self, name, qname): - namespace, localname = name - lowernamespace = str(namespace or '').lower() - if qname and qname.find(':') > 0: - givenprefix = qname.split(':')[0] - else: - givenprefix = '' - prefix = self._matchnamespaces.get(lowernamespace, givenprefix) - if prefix: - localname = prefix + ':' + localname - localname = str(localname).lower() - self.unknown_endtag(localname) - - def error(self, exc): - self.bozo = 1 - self.exc = exc - - def fatalError(self, exc): - self.error(exc) - raise exc - -class _BaseHTMLProcessor(sgmllib.SGMLParser): - elements_no_end_tag = ['area', 'base', 'basefont', 'br', 'col', 'frame', 'hr', - 'img', 'input', 'isindex', 'link', 'meta', 'param'] - - def __init__(self, encoding): - self.encoding = encoding - if _debug: sys.stderr.write('entering BaseHTMLProcessor, encoding=%s\n' % self.encoding) - sgmllib.SGMLParser.__init__(self) - - def reset(self): - self.pieces = [] - sgmllib.SGMLParser.reset(self) - - def _shorttag_replace(self, match): - tag = match.group(1) - if tag in self.elements_no_end_tag: - return '<' + tag + ' />' - else: - return '<' + tag + '></' + tag + '>' - - def feed(self, data): - data = re.compile(r'<!((?!DOCTYPE|--|\[))', re.IGNORECASE).sub(r'<!\1', data) - #data = re.sub(r'<(\S+?)\s*?/>', self._shorttag_replace, data) # bug [ 1399464 ] Bad regexp for _shorttag_replace - data = re.sub(r'<([^<\s]+?)\s*/>', self._shorttag_replace, data) - data = data.replace(''', "'") - data = data.replace('"', '"') - if self.encoding and type(data) == type(u''): - data = data.encode(self.encoding) - sgmllib.SGMLParser.feed(self, data) - - def normalize_attrs(self, attrs): - # utility method to be called by descendants - attrs = [(k.lower(), v) for k, v in attrs] - attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs] - return attrs - - def unknown_starttag(self, tag, attrs): - # called for each start tag - # attrs is a list of (attr, value) tuples - # e.g. for <pre class='screen'>, tag='pre', attrs=[('class', 'screen')] - if _debug: sys.stderr.write('_BaseHTMLProcessor, unknown_starttag, tag=%s\n' % tag) - uattrs = [] - # thanks to Kevin Marks for this breathtaking hack to deal with (valid) high-bit attribute values in UTF-8 feeds - for key, value in attrs: - if type(value) != type(u''): - value = unicode(value, self.encoding) - uattrs.append((unicode(key, self.encoding), value)) - strattrs = u''.join([u' %s="%s"' % (key, value) for key, value in uattrs]).encode(self.encoding) - if tag in self.elements_no_end_tag: - self.pieces.append('<%(tag)s%(strattrs)s />' % locals()) - else: - self.pieces.append('<%(tag)s%(strattrs)s>' % locals()) - - def unknown_endtag(self, tag): - # called for each end tag, e.g. for </pre>, tag will be 'pre' - # Reconstruct the original end tag. - if tag not in self.elements_no_end_tag: - self.pieces.append("</%(tag)s>" % locals()) - - def handle_charref(self, ref): - # called for each character reference, e.g. for ' ', ref will be '160' - # Reconstruct the original character reference. - self.pieces.append('&#%(ref)s;' % locals()) - - def handle_entityref(self, ref): - # called for each entity reference, e.g. for '©', ref will be 'copy' - # Reconstruct the original entity reference. - self.pieces.append('&%(ref)s;' % locals()) - - def handle_data(self, text): - # called for each block of plain text, i.e. outside of any tag and - # not containing any character or entity references - # Store the original text verbatim. - if _debug: sys.stderr.write('_BaseHTMLProcessor, handle_text, text=%s\n' % text) - self.pieces.append(text) - - def handle_comment(self, text): - # called for each HTML comment, e.g. <!-- insert Javascript code here --> - # Reconstruct the original comment. - self.pieces.append('<!--%(text)s-->' % locals()) - - def handle_pi(self, text): - # called for each processing instruction, e.g. <?instruction> - # Reconstruct original processing instruction. - self.pieces.append('<?%(text)s>' % locals()) - - def handle_decl(self, text): - # called for the DOCTYPE, if present, e.g. - # <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" - # "http://www.w3.org/TR/html4/loose.dtd"> - # Reconstruct original DOCTYPE - self.pieces.append('<!%(text)s>' % locals()) - - _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match - def _scan_name(self, i, declstartpos): - rawdata = self.rawdata - n = len(rawdata) - if i == n: - return None, -1 - m = self._new_declname_match(rawdata, i) - if m: - s = m.group() - name = s.strip() - if (i + len(s)) == n: - return None, -1 # end of buffer - return name.lower(), m.end() - else: - self.handle_data(rawdata) -# self.updatepos(declstartpos, i) - return None, -1 - - def output(self): - '''Return processed HTML as a single string''' - return ''.join([str(p) for p in self.pieces]) - -class _LooseFeedParser(_FeedParserMixin, _BaseHTMLProcessor): - def __init__(self, baseuri, baselang, encoding): - sgmllib.SGMLParser.__init__(self) - _FeedParserMixin.__init__(self, baseuri, baselang, encoding) - - def decodeEntities(self, element, data): - data = data.replace('<', '<') - data = data.replace('<', '<') - data = data.replace('>', '>') - data = data.replace('>', '>') - data = data.replace('&', '&') - data = data.replace('&', '&') - data = data.replace('"', '"') - data = data.replace('"', '"') - data = data.replace(''', ''') - data = data.replace(''', ''') - if self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'): - data = data.replace('<', '<') - data = data.replace('>', '>') - data = data.replace('&', '&') - data = data.replace('"', '"') - data = data.replace(''', "'") - return data - -class _RelativeURIResolver(_BaseHTMLProcessor): - relative_uris = [('a', 'href'), - ('applet', 'codebase'), - ('area', 'href'), - ('blockquote', 'cite'), - ('body', 'background'), - ('del', 'cite'), - ('form', 'action'), - ('frame', 'longdesc'), - ('frame', 'src'), - ('iframe', 'longdesc'), - ('iframe', 'src'), - ('head', 'profile'), - ('img', 'longdesc'), - ('img', 'src'), - ('img', 'usemap'), - ('input', 'src'), - ('input', 'usemap'), - ('ins', 'cite'), - ('link', 'href'), - ('object', 'classid'), - ('object', 'codebase'), - ('object', 'data'), - ('object', 'usemap'), - ('q', 'cite'), - ('script', 'src')] - - def __init__(self, baseuri, encoding): - _BaseHTMLProcessor.__init__(self, encoding) - self.baseuri = baseuri - - def resolveURI(self, uri): - return _urljoin(self.baseuri, uri) - - def unknown_starttag(self, tag, attrs): - attrs = self.normalize_attrs(attrs) - attrs = [(key, ((tag, key) in self.relative_uris) and self.resolveURI(value) or value) for key, value in attrs] - _BaseHTMLProcessor.unknown_starttag(self, tag, attrs) - -def _resolveRelativeURIs(htmlSource, baseURI, encoding): - if _debug: sys.stderr.write('entering _resolveRelativeURIs\n') - p = _RelativeURIResolver(baseURI, encoding) - p.feed(htmlSource) - return p.output() - -class _HTMLSanitizer(_BaseHTMLProcessor): - acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b', 'big', - 'blockquote', 'br', 'button', 'caption', 'center', 'cite', 'code', 'col', - 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'fieldset', - 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', - 'ins', 'kbd', 'label', 'legend', 'li', 'map', 'menu', 'ol', 'optgroup', - 'option', 'p', 'pre', 'q', 's', 'samp', 'select', 'small', 'span', 'strike', - 'strong', 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'tfoot', 'th', - 'thead', 'tr', 'tt', 'u', 'ul', 'var'] - - acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'axis', 'border', 'cellpadding', 'cellspacing', - 'char', 'charoff', 'charset', 'checked', 'cite', 'class', 'clear', 'cols', - 'colspan', 'color', 'compact', 'coords', 'datetime', 'dir', 'disabled', - 'enctype', 'for', 'frame', 'headers', 'height', 'href', 'hreflang', 'hspace', - 'id', 'ismap', 'label', 'lang', 'longdesc', 'maxlength', 'media', 'method', - 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'prompt', 'readonly', - 'rel', 'rev', 'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size', - 'span', 'src', 'start', 'summary', 'tabindex', 'target', 'title', 'type', - 'usemap', 'valign', 'value', 'vspace', 'width'] - - unacceptable_elements_with_end_tag = ['script', 'applet'] - - def reset(self): - _BaseHTMLProcessor.reset(self) - self.unacceptablestack = 0 - - def unknown_starttag(self, tag, attrs): - if not tag in self.acceptable_elements: - if tag in self.unacceptable_elements_with_end_tag: - self.unacceptablestack += 1 - return - attrs = self.normalize_attrs(attrs) - attrs = [(key, value) for key, value in attrs if key in self.acceptable_attributes] - _BaseHTMLProcessor.unknown_starttag(self, tag, attrs) - - def unknown_endtag(self, tag): - if not tag in self.acceptable_elements: - if tag in self.unacceptable_elements_with_end_tag: - self.unacceptablestack -= 1 - return - _BaseHTMLProcessor.unknown_endtag(self, tag) - - def handle_pi(self, text): - pass - - def handle_decl(self, text): - pass - - def handle_data(self, text): - if not self.unacceptablestack: - _BaseHTMLProcessor.handle_data(self, text) - -def _sanitizeHTML(htmlSource, encoding): - p = _HTMLSanitizer(encoding) - p.feed(htmlSource) - data = p.output() - if TIDY_MARKUP: - # loop through list of preferred Tidy interfaces looking for one that's installed, - # then set up a common _tidy function to wrap the interface-specific API. - _tidy = None - for tidy_interface in PREFERRED_TIDY_INTERFACES: - try: - if tidy_interface == "uTidy": - from tidy import parseString as _utidy - def _tidy(data, **kwargs): - return str(_utidy(data, **kwargs)) - break - elif tidy_interface == "mxTidy": - from mx.Tidy import Tidy as _mxtidy - def _tidy(data, **kwargs): - nerrors, nwarnings, data, errordata = _mxtidy.tidy(data, **kwargs) - return data - break - except: - pass - if _tidy: - utf8 = type(data) == type(u'') - if utf8: - data = data.encode('utf-8') - data = _tidy(data, output_xhtml=1, numeric_entities=1, wrap=0, char_encoding="utf8") - if utf8: - data = unicode(data, 'utf-8') - if data.count('<body'): - data = data.split('<body', 1)[1] - if data.count('>'): - data = data.split('>', 1)[1] - if data.count('</body'): - data = data.split('</body', 1)[0] - data = data.strip().replace('\r\n', '\n') - return data - -class _FeedURLHandler(urllib2.HTTPDigestAuthHandler, urllib2.HTTPRedirectHandler, urllib2.HTTPDefaultErrorHandler): - def http_error_default(self, req, fp, code, msg, headers): - if ((code / 100) == 3) and (code != 304): - return self.http_error_302(req, fp, code, msg, headers) - infourl = urllib.addinfourl(fp, headers, req.get_full_url()) - infourl.status = code - return infourl - - def http_error_302(self, req, fp, code, msg, headers): - if headers.dict.has_key('location'): - infourl = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers) - else: - infourl = urllib.addinfourl(fp, headers, req.get_full_url()) - if not hasattr(infourl, 'status'): - infourl.status = code - return infourl - - def http_error_301(self, req, fp, code, msg, headers): - if headers.dict.has_key('location'): - infourl = urllib2.HTTPRedirectHandler.http_error_301(self, req, fp, code, msg, headers) - else: - infourl = urllib.addinfourl(fp, headers, req.get_full_url()) - if not hasattr(infourl, 'status'): - infourl.status = code - return infourl - - http_error_300 = http_error_302 - http_error_303 = http_error_302 - http_error_307 = http_error_302 - - def http_error_401(self, req, fp, code, msg, headers): - # Check if - # - server requires digest auth, AND - # - we tried (unsuccessfully) with basic auth, AND - # - we're using Python 2.3.3 or later (digest auth is irreparably broken in earlier versions) - # If all conditions hold, parse authentication information - # out of the Authorization header we sent the first time - # (for the username and password) and the WWW-Authenticate - # header the server sent back (for the realm) and retry - # the request with the appropriate digest auth headers instead. - # This evil genius hack has been brought to you by Aaron Swartz. - host = urlparse.urlparse(req.get_full_url())[1] - try: - assert sys.version.split()[0] >= '2.3.3' - assert base64 != None - user, passw = base64.decodestring(req.headers['Authorization'].split(' ')[1]).split(':') - realm = re.findall('realm="([^"]*)"', headers['WWW-Authenticate'])[0] - self.add_password(realm, host, user, passw) - retry = self.http_error_auth_reqed('www-authenticate', host, req, headers) - self.reset_retry_count() - return retry - except: - return self.http_error_default(req, fp, code, msg, headers) - -def _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, handlers): - """URL, filename, or string --> stream - - This function lets you define parsers that take any input source - (URL, pathname to local or network file, or actual data as a string) - and deal with it in a uniform manner. Returned object is guaranteed - to have all the basic stdio read methods (read, readline, readlines). - Just .close() the object when you're done with it. - - If the etag argument is supplied, it will be used as the value of an - If-None-Match request header. - - If the modified argument is supplied, it must be a tuple of 9 integers - as returned by gmtime() in the standard Python time module. This MUST - be in GMT (Greenwich Mean Time). The formatted date/time will be used - as the value of an If-Modified-Since request header. - - If the agent argument is supplied, it will be used as the value of a - User-Agent request header. - - If the referrer argument is supplied, it will be used as the value of a - Referer[sic] request header. - - If handlers is supplied, it is a list of handlers used to build a - urllib2 opener. - """ - - if hasattr(url_file_stream_or_string, 'read'): - return url_file_stream_or_string - - if url_file_stream_or_string == '-': - return sys.stdin - - if urlparse.urlparse(url_file_stream_or_string)[0] in ('http', 'https', 'ftp'): - if not agent: - agent = USER_AGENT - # test for inline user:password for basic auth - auth = None - if base64: - urltype, rest = urllib.splittype(url_file_stream_or_string) - realhost, rest = urllib.splithost(rest) - if realhost: - user_passwd, realhost = urllib.splituser(realhost) - if user_passwd: - url_file_stream_or_string = '%s://%s%s' % (urltype, realhost, rest) - auth = base64.encodestring(user_passwd).strip() - # try to open with urllib2 (to use optional headers) - request = urllib2.Request(url_file_stream_or_string) - request.add_header('User-Agent', agent) - if etag: - request.add_header('If-None-Match', etag) - if modified: - # format into an RFC 1123-compliant timestamp. We can't use - # time.strftime() since the %a and %b directives can be affected - # by the current locale, but RFC 2616 states that dates must be - # in English. - short_weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] - months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] - request.add_header('If-Modified-Since', '%s, %02d %s %04d %02d:%02d:%02d GMT' % (short_weekdays[modified[6]], modified[2], months[modified[1] - 1], modified[0], modified[3], modified[4], modified[5])) - if referrer: - request.add_header('Referer', referrer) - if gzip and zlib: - request.add_header('Accept-encoding', 'gzip, deflate') - elif gzip: - request.add_header('Accept-encoding', 'gzip') - elif zlib: - request.add_header('Accept-encoding', 'deflate') - else: - request.add_header('Accept-encoding', '') - if auth: - request.add_header('Authorization', 'Basic %s' % auth) - if ACCEPT_HEADER: - request.add_header('Accept', ACCEPT_HEADER) - request.add_header('A-IM', 'feed') # RFC 3229 support - opener = apply(urllib2.build_opener, tuple([_FeedURLHandler()] + handlers)) - opener.addheaders = [] # RMK - must clear so we only send our custom User-Agent - try: - return opener.open(request) - finally: - opener.close() # JohnD - - # try to open with native open function (if url_file_stream_or_string is a filename) - try: - return open(url_file_stream_or_string) - except: - pass - - # treat url_file_stream_or_string as string - return _StringIO(str(url_file_stream_or_string)) - -_date_handlers = [] -def registerDateHandler(func): - '''Register a date handler function (takes string, returns 9-tuple date in GMT)''' - _date_handlers.insert(0, func) - -# ISO-8601 date parsing routines written by Fazal Majid. -# The ISO 8601 standard is very convoluted and irregular - a full ISO 8601 -# parser is beyond the scope of feedparser and would be a worthwhile addition -# to the Python library. -# A single regular expression cannot parse ISO 8601 date formats into groups -# as the standard is highly irregular (for instance is 030104 2003-01-04 or -# 0301-04-01), so we use templates instead. -# Please note the order in templates is significant because we need a -# greedy match. -_iso8601_tmpl = ['YYYY-?MM-?DD', 'YYYY-MM', 'YYYY-?OOO', - 'YY-?MM-?DD', 'YY-?OOO', 'YYYY', - '-YY-?MM', '-OOO', '-YY', - '--MM-?DD', '--MM', - '---DD', - 'CC', ''] -_iso8601_re = [ - tmpl.replace( - 'YYYY', r'(?P<year>\d{4})').replace( - 'YY', r'(?P<year>\d\d)').replace( - 'MM', r'(?P<month>[01]\d)').replace( - 'DD', r'(?P<day>[0123]\d)').replace( - 'OOO', r'(?P<ordinal>[0123]\d\d)').replace( - 'CC', r'(?P<century>\d\d$)') - + r'(T?(?P<hour>\d{2}):(?P<minute>\d{2})' - + r'(:(?P<second>\d{2}))?' - + r'(?P<tz>[+-](?P<tzhour>\d{2})(:(?P<tzmin>\d{2}))?|Z)?)?' - for tmpl in _iso8601_tmpl] -del tmpl -_iso8601_matches = [re.compile(regex).match for regex in _iso8601_re] -del regex -def _parse_date_iso8601(dateString): - '''Parse a variety of ISO-8601-compatible formats like 20040105''' - m = None - for _iso8601_match in _iso8601_matches: - m = _iso8601_match(dateString) - if m: break - if not m: return - if m.span() == (0, 0): return - params = m.groupdict() - ordinal = params.get('ordinal', 0) - if ordinal: - ordinal = int(ordinal) - else: - ordinal = 0 - year = params.get('year', '--') - if not year or year == '--': - year = time.gmtime()[0] - elif len(year) == 2: - # ISO 8601 assumes current century, i.e. 93 -> 2093, NOT 1993 - year = 100 * int(time.gmtime()[0] / 100) + int(year) - else: - year = int(year) - month = params.get('month', '-') - if not month or month == '-': - # ordinals are NOT normalized by mktime, we simulate them - # by setting month=1, day=ordinal - if ordinal: - month = 1 - else: - month = time.gmtime()[1] - month = int(month) - day = params.get('day', 0) - if not day: - # see above - if ordinal: - day = ordinal - elif params.get('century', 0) or \ - params.get('year', 0) or params.get('month', 0): - day = 1 - else: - day = time.gmtime()[2] - else: - day = int(day) - # special case of the century - is the first year of the 21st century - # 2000 or 2001 ? The debate goes on... - if 'century' in params.keys(): - year = (int(params['century']) - 1) * 100 + 1 - # in ISO 8601 most fields are optional - for field in ['hour', 'minute', 'second', 'tzhour', 'tzmin']: - if not params.get(field, None): - params[field] = 0 - hour = int(params.get('hour', 0)) - minute = int(params.get('minute', 0)) - second = int(params.get('second', 0)) - # weekday is normalized by mktime(), we can ignore it - weekday = 0 - # daylight savings is complex, but not needed for feedparser's purposes - # as time zones, if specified, include mention of whether it is active - # (e.g. PST vs. PDT, CET). Using -1 is implementation-dependent and - # and most implementations have DST bugs - daylight_savings_flag = 0 - tm = [year, month, day, hour, minute, second, weekday, - ordinal, daylight_savings_flag] - # ISO 8601 time zone adjustments - tz = params.get('tz') - if tz and tz != 'Z': - if tz[0] == '-': - tm[3] += int(params.get('tzhour', 0)) - tm[4] += int(params.get('tzmin', 0)) - elif tz[0] == '+': - tm[3] -= int(params.get('tzhour', 0)) - tm[4] -= int(params.get('tzmin', 0)) - else: - return None - # Python's time.mktime() is a wrapper around the ANSI C mktime(3c) - # which is guaranteed to normalize d/m/y/h/m/s. - # Many implementations have bugs, but we'll pretend they don't. - return time.localtime(time.mktime(tm)) -registerDateHandler(_parse_date_iso8601) - -# 8-bit date handling routines written by ytrewq1. -_korean_year = u'\ub144' # b3e2 in euc-kr -_korean_month = u'\uc6d4' # bff9 in euc-kr -_korean_day = u'\uc77c' # c0cf in euc-kr -_korean_am = u'\uc624\uc804' # bfc0 c0fc in euc-kr -_korean_pm = u'\uc624\ud6c4' # bfc0 c8c4 in euc-kr - -_korean_onblog_date_re = \ - re.compile('(\d{4})%s\s+(\d{2})%s\s+(\d{2})%s\s+(\d{2}):(\d{2}):(\d{2})' % \ - (_korean_year, _korean_month, _korean_day)) -_korean_nate_date_re = \ - re.compile(u'(\d{4})-(\d{2})-(\d{2})\s+(%s|%s)\s+(\d{,2}):(\d{,2}):(\d{,2})' % \ - (_korean_am, _korean_pm)) -def _parse_date_onblog(dateString): - '''Parse a string according to the OnBlog 8-bit date format''' - m = _korean_onblog_date_re.match(dateString) - if not m: return - w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s:%(second)s%(zonediff)s' % \ - {'year': m.group(1), 'month': m.group(2), 'day': m.group(3),\ - 'hour': m.group(4), 'minute': m.group(5), 'second': m.group(6),\ - 'zonediff': '+09:00'} - if _debug: sys.stderr.write('OnBlog date parsed as: %s\n' % w3dtfdate) - return _parse_date_w3dtf(w3dtfdate) -registerDateHandler(_parse_date_onblog) - -def _parse_date_nate(dateString): - '''Parse a string according to the Nate 8-bit date format''' - m = _korean_nate_date_re.match(dateString) - if not m: return - hour = int(m.group(5)) - ampm = m.group(4) - if (ampm == _korean_pm): - hour += 12 - hour = str(hour) - if len(hour) == 1: - hour = '0' + hour - w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s:%(second)s%(zonediff)s' % \ - {'year': m.group(1), 'month': m.group(2), 'day': m.group(3),\ - 'hour': hour, 'minute': m.group(6), 'second': m.group(7),\ - 'zonediff': '+09:00'} - if _debug: sys.stderr.write('Nate date parsed as: %s\n' % w3dtfdate) - return _parse_date_w3dtf(w3dtfdate) -registerDateHandler(_parse_date_nate) - -_mssql_date_re = \ - re.compile('(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})(\.\d+)?') -def _parse_date_mssql(dateString): - '''Parse a string according to the MS SQL date format''' - m = _mssql_date_re.match(dateString) - if not m: return - w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s:%(second)s%(zonediff)s' % \ - {'year': m.group(1), 'month': m.group(2), 'day': m.group(3),\ - 'hour': m.group(4), 'minute': m.group(5), 'second': m.group(6),\ - 'zonediff': '+09:00'} - if _debug: sys.stderr.write('MS SQL date parsed as: %s\n' % w3dtfdate) - return _parse_date_w3dtf(w3dtfdate) -registerDateHandler(_parse_date_mssql) - -# Unicode strings for Greek date strings -_greek_months = \ - { \ - u'\u0399\u03b1\u03bd': u'Jan', # c9e1ed in iso-8859-7 - u'\u03a6\u03b5\u03b2': u'Feb', # d6e5e2 in iso-8859-7 - u'\u039c\u03ac\u03ce': u'Mar', # ccdcfe in iso-8859-7 - u'\u039c\u03b1\u03ce': u'Mar', # cce1fe in iso-8859-7 - u'\u0391\u03c0\u03c1': u'Apr', # c1f0f1 in iso-8859-7 - u'\u039c\u03ac\u03b9': u'May', # ccdce9 in iso-8859-7 - u'\u039c\u03b1\u03ca': u'May', # cce1fa in iso-8859-7 - u'\u039c\u03b1\u03b9': u'May', # cce1e9 in iso-8859-7 - u'\u0399\u03bf\u03cd\u03bd': u'Jun', # c9effded in iso-8859-7 - u'\u0399\u03bf\u03bd': u'Jun', # c9efed in iso-8859-7 - u'\u0399\u03bf\u03cd\u03bb': u'Jul', # c9effdeb in iso-8859-7 - u'\u0399\u03bf\u03bb': u'Jul', # c9f9eb in iso-8859-7 - u'\u0391\u03cd\u03b3': u'Aug', # c1fde3 in iso-8859-7 - u'\u0391\u03c5\u03b3': u'Aug', # c1f5e3 in iso-8859-7 - u'\u03a3\u03b5\u03c0': u'Sep', # d3e5f0 in iso-8859-7 - u'\u039f\u03ba\u03c4': u'Oct', # cfeaf4 in iso-8859-7 - u'\u039d\u03bf\u03ad': u'Nov', # cdefdd in iso-8859-7 - u'\u039d\u03bf\u03b5': u'Nov', # cdefe5 in iso-8859-7 - u'\u0394\u03b5\u03ba': u'Dec', # c4e5ea in iso-8859-7 - } - -_greek_wdays = \ - { \ - u'\u039a\u03c5\u03c1': u'Sun', # caf5f1 in iso-8859-7 - u'\u0394\u03b5\u03c5': u'Mon', # c4e5f5 in iso-8859-7 - u'\u03a4\u03c1\u03b9': u'Tue', # d4f1e9 in iso-8859-7 - u'\u03a4\u03b5\u03c4': u'Wed', # d4e5f4 in iso-8859-7 - u'\u03a0\u03b5\u03bc': u'Thu', # d0e5ec in iso-8859-7 - u'\u03a0\u03b1\u03c1': u'Fri', # d0e1f1 in iso-8859-7 - u'\u03a3\u03b1\u03b2': u'Sat', # d3e1e2 in iso-8859-7 - } - -_greek_date_format_re = \ - re.compile(u'([^,]+),\s+(\d{2})\s+([^\s]+)\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+([^\s]+)') - -def _parse_date_greek(dateString): - '''Parse a string according to a Greek 8-bit date format.''' - m = _greek_date_format_re.match(dateString) - if not m: return - try: - wday = _greek_wdays[m.group(1)] - month = _greek_months[m.group(3)] - except: - return - rfc822date = '%(wday)s, %(day)s %(month)s %(year)s %(hour)s:%(minute)s:%(second)s %(zonediff)s' % \ - {'wday': wday, 'day': m.group(2), 'month': month, 'year': m.group(4),\ - 'hour': m.group(5), 'minute': m.group(6), 'second': m.group(7),\ - 'zonediff': m.group(8)} - if _debug: sys.stderr.write('Greek date parsed as: %s\n' % rfc822date) - return _parse_date_rfc822(rfc822date) -registerDateHandler(_parse_date_greek) - -# Unicode strings for Hungarian date strings -_hungarian_months = \ - { \ - u'janu\u00e1r': u'01', # e1 in iso-8859-2 - u'febru\u00e1ri': u'02', # e1 in iso-8859-2 - u'm\u00e1rcius': u'03', # e1 in iso-8859-2 - u'\u00e1prilis': u'04', # e1 in iso-8859-2 - u'm\u00e1ujus': u'05', # e1 in iso-8859-2 - u'j\u00fanius': u'06', # fa in iso-8859-2 - u'j\u00falius': u'07', # fa in iso-8859-2 - u'augusztus': u'08', - u'szeptember': u'09', - u'okt\u00f3ber': u'10', # f3 in iso-8859-2 - u'november': u'11', - u'december': u'12', - } - -_hungarian_date_format_re = \ - re.compile(u'(\d{4})-([^-]+)-(\d{,2})T(\d{,2}):(\d{2})((\+|-)(\d{,2}:\d{2}))') - -def _parse_date_hungarian(dateString): - '''Parse a string according to a Hungarian 8-bit date format.''' - m = _hungarian_date_format_re.match(dateString) - if not m: return - try: - month = _hungarian_months[m.group(2)] - day = m.group(3) - if len(day) == 1: - day = '0' + day - hour = m.group(4) - if len(hour) == 1: - hour = '0' + hour - except: - return - w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s%(zonediff)s' % \ - {'year': m.group(1), 'month': month, 'day': day,\ - 'hour': hour, 'minute': m.group(5),\ - 'zonediff': m.group(6)} - if _debug: sys.stderr.write('Hungarian date parsed as: %s\n' % w3dtfdate) - return _parse_date_w3dtf(w3dtfdate) -registerDateHandler(_parse_date_hungarian) - -# W3DTF-style date parsing adapted from PyXML xml.utils.iso8601, written by -# Drake and licensed under the Python license. Removed all range checking -# for month, day, hour, minute, and second, since mktime will normalize -# these later -def _parse_date_w3dtf(dateString): - def __extract_date(m): - year = int(m.group('year')) - if year < 100: - year = 100 * int(time.gmtime()[0] / 100) + int(year) - if year < 1000: - return 0, 0, 0 - julian = m.group('julian') - if julian: - julian = int(julian) - month = julian / 30 + 1 - day = julian % 30 + 1 - jday = None - while jday != julian: - t = time.mktime((year, month, day, 0, 0, 0, 0, 0, 0)) - jday = time.gmtime(t)[-2] - diff = abs(jday - julian) - if jday > julian: - if diff < day: - day = day - diff - else: - month = month - 1 - day = 31 - elif jday < julian: - if day + diff < 28: - day = day + diff - else: - month = month + 1 - return year, month, day - month = m.group('month') - day = 1 - if month is None: - month = 1 - else: - month = int(month) - day = m.group('day') - if day: - day = int(day) - else: - day = 1 - return year, month, day - - def __extract_time(m): - if not m: - return 0, 0, 0 - hours = m.group('hours') - if not hours: - return 0, 0, 0 - hours = int(hours) - minutes = int(m.group('minutes')) - seconds = m.group('seconds') - if seconds: - seconds = int(seconds) - else: - seconds = 0 - return hours, minutes, seconds - - def __extract_tzd(m): - '''Return the Time Zone Designator as an offset in seconds from UTC.''' - if not m: - return 0 - tzd = m.group('tzd') - if not tzd: - return 0 - if tzd == 'Z': - return 0 - hours = int(m.group('tzdhours')) - minutes = m.group('tzdminutes') - if minutes: - minutes = int(minutes) - else: - minutes = 0 - offset = (hours*60 + minutes) * 60 - if tzd[0] == '+': - return -offset - return offset - - __date_re = ('(?P<year>\d\d\d\d)' - '(?:(?P<dsep>-|)' - '(?:(?P<julian>\d\d\d)' - '|(?P<month>\d\d)(?:(?P=dsep)(?P<day>\d\d))?))?') - __tzd_re = '(?P<tzd>[-+](?P<tzdhours>\d\d)(?::?(?P<tzdminutes>\d\d))|Z)' - __tzd_rx = re.compile(__tzd_re) - __time_re = ('(?P<hours>\d\d)(?P<tsep>:|)(?P<minutes>\d\d)' - '(?:(?P=tsep)(?P<seconds>\d\d(?:[.,]\d+)?))?' - + __tzd_re) - __datetime_re = '%s(?:T%s)?' % (__date_re, __time_re) - __datetime_rx = re.compile(__datetime_re) - m = __datetime_rx.match(dateString) - if (m is None) or (m.group() != dateString): return - gmt = __extract_date(m) + __extract_time(m) + (0, 0, 0) - if gmt[0] == 0: return - return time.gmtime(time.mktime(gmt) + __extract_tzd(m) - time.timezone) -registerDateHandler(_parse_date_w3dtf) - -def _parse_date_rfc822(dateString): - '''Parse an RFC822, RFC1123, RFC2822, or asctime-style date''' - data = dateString.split() - if data[0][-1] in (',', '.') or data[0].lower() in rfc822._daynames: - del data[0] - if len(data) == 4: - s = data[3] - i = s.find('+') - if i > 0: - data[3:] = [s[:i], s[i+1:]] - else: - data.append('') - dateString = " ".join(data) - if len(data) < 5: - dateString += ' 00:00:00 GMT' - tm = rfc822.parsedate_tz(dateString) - if tm: - return time.gmtime(rfc822.mktime_tz(tm)) -# rfc822.py defines several time zones, but we define some extra ones. -# 'ET' is equivalent to 'EST', etc. -_additional_timezones = {'AT': -400, 'ET': -500, 'CT': -600, 'MT': -700, 'PT': -800} -rfc822._timezones.update(_additional_timezones) -registerDateHandler(_parse_date_rfc822) - -def _parse_date(dateString): - '''Parses a variety of date formats into a 9-tuple in GMT''' - for handler in _date_handlers: - try: - date9tuple = handler(dateString) - if not date9tuple: continue - if len(date9tuple) != 9: - if _debug: sys.stderr.write('date handler function must return 9-tuple\n') - raise ValueError - map(int, date9tuple) - return date9tuple - except Exception, e: - if _debug: sys.stderr.write('%s raised %s\n' % (handler.__name__, repr(e))) - pass - return None - -def _getCharacterEncoding(http_headers, xml_data): - '''Get the character encoding of the XML document - - http_headers is a dictionary - xml_data is a raw string (not Unicode) - - This is so much trickier than it sounds, it's not even funny. - According to RFC 3023 ('XML Media Types'), if the HTTP Content-Type - is application/xml, application/*+xml, - application/xml-external-parsed-entity, or application/xml-dtd, - the encoding given in the charset parameter of the HTTP Content-Type - takes precedence over the encoding given in the XML prefix within the - document, and defaults to 'utf-8' if neither are specified. But, if - the HTTP Content-Type is text/xml, text/*+xml, or - text/xml-external-parsed-entity, the encoding given in the XML prefix - within the document is ALWAYS IGNORED and only the encoding given in - the charset parameter of the HTTP Content-Type header should be - respected, and it defaults to 'us-ascii' if not specified. - - Furthermore, discussion on the atom-syntax mailing list with the - author of RFC 3023 leads me to the conclusion that any document - served with a Content-Type of text/* and no charset parameter - must be treated as us-ascii. (We now do this.) And also that it - must always be flagged as non-well-formed. (We now do this too.) - - If Content-Type is unspecified (input was local file or non-HTTP source) - or unrecognized (server just got it totally wrong), then go by the - encoding given in the XML prefix of the document and default to - 'iso-8859-1' as per the HTTP specification (RFC 2616). - - Then, assuming we didn't find a character encoding in the HTTP headers - (and the HTTP Content-type allowed us to look in the body), we need - to sniff the first few bytes of the XML data and try to determine - whether the encoding is ASCII-compatible. Section F of the XML - specification shows the way here: - http://www.w3.org/TR/REC-xml/#sec-guessing-no-ext-info - - If the sniffed encoding is not ASCII-compatible, we need to make it - ASCII compatible so that we can sniff further into the XML declaration - to find the encoding attribute, which will tell us the true encoding. - - Of course, none of this guarantees that we will be able to parse the - feed in the declared character encoding (assuming it was declared - correctly, which many are not). CJKCodecs and iconv_codec help a lot; - you should definitely install them if you can. - http://cjkpython.i18n.org/ - ''' - - def _parseHTTPContentType(content_type): - '''takes HTTP Content-Type header and returns (content type, charset) - - If no charset is specified, returns (content type, '') - If no content type is specified, returns ('', '') - Both return parameters are guaranteed to be lowercase strings - ''' - content_type = content_type or '' - content_type, params = cgi.parse_header(content_type) - return content_type, params.get('charset', '').replace("'", '') - - sniffed_xml_encoding = '' - xml_encoding = '' - true_encoding = '' - http_content_type, http_encoding = _parseHTTPContentType(http_headers.get('content-type')) - # Must sniff for non-ASCII-compatible character encodings before - # searching for XML declaration. This heuristic is defined in - # section F of the XML specification: - # http://www.w3.org/TR/REC-xml/#sec-guessing-no-ext-info - try: - if xml_data[:4] == '\x4c\x6f\xa7\x94': - # EBCDIC - xml_data = _ebcdic_to_ascii(xml_data) - elif xml_data[:4] == '\x00\x3c\x00\x3f': - # UTF-16BE - sniffed_xml_encoding = 'utf-16be' - xml_data = unicode(xml_data, 'utf-16be').encode('utf-8') - elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') and (xml_data[2:4] != '\x00\x00'): - # UTF-16BE with BOM - sniffed_xml_encoding = 'utf-16be' - xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8') - elif xml_data[:4] == '\x3c\x00\x3f\x00': - # UTF-16LE - sniffed_xml_encoding = 'utf-16le' - xml_data = unicode(xml_data, 'utf-16le').encode('utf-8') - elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and (xml_data[2:4] != '\x00\x00'): - # UTF-16LE with BOM - sniffed_xml_encoding = 'utf-16le' - xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8') - elif xml_data[:4] == '\x00\x00\x00\x3c': - # UTF-32BE - sniffed_xml_encoding = 'utf-32be' - xml_data = unicode(xml_data, 'utf-32be').encode('utf-8') - elif xml_data[:4] == '\x3c\x00\x00\x00': - # UTF-32LE - sniffed_xml_encoding = 'utf-32le' - xml_data = unicode(xml_data, 'utf-32le').encode('utf-8') - elif xml_data[:4] == '\x00\x00\xfe\xff': - # UTF-32BE with BOM - sniffed_xml_encoding = 'utf-32be' - xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8') - elif xml_data[:4] == '\xff\xfe\x00\x00': - # UTF-32LE with BOM - sniffed_xml_encoding = 'utf-32le' - xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8') - elif xml_data[:3] == '\xef\xbb\xbf': - # UTF-8 with BOM - sniffed_xml_encoding = 'utf-8' - xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8') - else: - # ASCII-compatible - pass - xml_encoding_match = re.compile('^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data) - except: - xml_encoding_match = None - if xml_encoding_match: - xml_encoding = xml_encoding_match.groups()[0].lower() - if sniffed_xml_encoding and (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode', 'iso-10646-ucs-4', 'ucs-4', 'csucs4', 'utf-16', 'utf-32', 'utf_16', 'utf_32', 'utf16', 'u16')): - xml_encoding = sniffed_xml_encoding - acceptable_content_type = 0 - application_content_types = ('application/xml', 'application/xml-dtd', 'application/xml-external-parsed-entity') - text_content_types = ('text/xml', 'text/xml-external-parsed-entity') - if (http_content_type in application_content_types) or \ - (http_content_type.startswith('application/') and http_content_type.endswith('+xml')): - acceptable_content_type = 1 - true_encoding = http_encoding or xml_encoding or 'utf-8' - elif (http_content_type in text_content_types) or \ - (http_content_type.startswith('text/')) and http_content_type.endswith('+xml'): - acceptable_content_type = 1 - true_encoding = http_encoding or 'us-ascii' - elif http_content_type.startswith('text/'): - true_encoding = http_encoding or 'us-ascii' - elif http_headers and (not http_headers.has_key('content-type')): - true_encoding = xml_encoding or 'iso-8859-1' - else: - true_encoding = xml_encoding or 'utf-8' - return true_encoding, http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type - -def _toUTF8(data, encoding): - '''Changes an XML data stream on the fly to specify a new encoding - - data is a raw sequence of bytes (not Unicode) that is presumed to be in %encoding already - encoding is a string recognized by encodings.aliases - ''' - if _debug: sys.stderr.write('entering _toUTF8, trying encoding %s\n' % encoding) - # strip Byte Order Mark (if present) - if (len(data) >= 4) and (data[:2] == '\xfe\xff') and (data[2:4] != '\x00\x00'): - if _debug: - sys.stderr.write('stripping BOM\n') - if encoding != 'utf-16be': - sys.stderr.write('trying utf-16be instead\n') - encoding = 'utf-16be' - data = data[2:] - elif (len(data) >= 4) and (data[:2] == '\xff\xfe') and (data[2:4] != '\x00\x00'): - if _debug: - sys.stderr.write('stripping BOM\n') - if encoding != 'utf-16le': - sys.stderr.write('trying utf-16le instead\n') - encoding = 'utf-16le' - data = data[2:] - elif data[:3] == '\xef\xbb\xbf': - if _debug: - sys.stderr.write('stripping BOM\n') - if encoding != 'utf-8': - sys.stderr.write('trying utf-8 instead\n') - encoding = 'utf-8' - data = data[3:] - elif data[:4] == '\x00\x00\xfe\xff': - if _debug: - sys.stderr.write('stripping BOM\n') - if encoding != 'utf-32be': - sys.stderr.write('trying utf-32be instead\n') - encoding = 'utf-32be' - data = data[4:] - elif data[:4] == '\xff\xfe\x00\x00': - if _debug: - sys.stderr.write('stripping BOM\n') - if encoding != 'utf-32le': - sys.stderr.write('trying utf-32le instead\n') - encoding = 'utf-32le' - data = data[4:] - newdata = unicode(data, encoding) - if _debug: sys.stderr.write('successfully converted %s data to unicode\n' % encoding) - declmatch = re.compile('^<\?xml[^>]*?>') - newdecl = '''<?xml version='1.0' encoding='utf-8'?>''' - if declmatch.search(newdata): - newdata = declmatch.sub(newdecl, newdata) - else: - newdata = newdecl + u'\n' + newdata - return newdata.encode('utf-8') - -def _stripDoctype(data): - '''Strips DOCTYPE from XML document, returns (rss_version, stripped_data) - - rss_version may be 'rss091n' or None - stripped_data is the same XML document, minus the DOCTYPE - ''' - entity_pattern = re.compile(r'<!ENTITY([^>]*?)>', re.MULTILINE) - data = entity_pattern.sub('', data) - doctype_pattern = re.compile(r'<!DOCTYPE([^>]*?)>', re.MULTILINE) - doctype_results = doctype_pattern.findall(data) - doctype = doctype_results and doctype_results[0] or '' - if doctype.lower().count('netscape'): - version = 'rss091n' - else: - version = None - data = doctype_pattern.sub('', data) - return version, data - -def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, referrer=None, handlers=[]): - '''Parse a feed from a URL, file, stream, or string''' - result = FeedParserDict() - result['feed'] = FeedParserDict() - result['entries'] = [] - if _XML_AVAILABLE: - result['bozo'] = 0 - if type(handlers) == types.InstanceType: - handlers = [handlers] - try: - f = _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, handlers) - data = f.read() - except Exception, e: - result['bozo'] = 1 - result['bozo_exception'] = e - data = '' - f = None - - # if feed is gzip-compressed, decompress it - if f and data and hasattr(f, 'headers'): - if gzip and f.headers.get('content-encoding', '') == 'gzip': - try: - data = gzip.GzipFile(fileobj=_StringIO(data)).read() - except Exception, e: - # Some feeds claim to be gzipped but they're not, so - # we get garbage. Ideally, we should re-request the - # feed without the 'Accept-encoding: gzip' header, - # but we don't. - result['bozo'] = 1 - result['bozo_exception'] = e - data = '' - elif zlib and f.headers.get('content-encoding', '') == 'deflate': - try: - data = zlib.decompress(data, -zlib.MAX_WBITS) - except Exception, e: - result['bozo'] = 1 - result['bozo_exception'] = e - data = '' - - # save HTTP headers - if hasattr(f, 'info'): - info = f.info() - result['etag'] = info.getheader('ETag') - last_modified = info.getheader('Last-Modified') - if last_modified: - result['modified'] = _parse_date(last_modified) - if hasattr(f, 'url'): - result['href'] = f.url - result['status'] = 200 - if hasattr(f, 'status'): - result['status'] = f.status - if hasattr(f, 'headers'): - result['headers'] = f.headers.dict - if hasattr(f, 'close'): - f.close() - - # there are four encodings to keep track of: - # - http_encoding is the encoding declared in the Content-Type HTTP header - # - xml_encoding is the encoding declared in the <?xml declaration - # - sniffed_encoding is the encoding sniffed from the first 4 bytes of the XML data - # - result['encoding'] is the actual encoding, as per RFC 3023 and a variety of other conflicting specifications - http_headers = result.get('headers', {}) - result['encoding'], http_encoding, xml_encoding, sniffed_xml_encoding, acceptable_content_type = \ - _getCharacterEncoding(http_headers, data) - if http_headers and (not acceptable_content_type): - if http_headers.has_key('content-type'): - bozo_message = '%s is not an XML media type' % http_headers['content-type'] - else: - bozo_message = 'no Content-type specified' - result['bozo'] = 1 - result['bozo_exception'] = NonXMLContentType(bozo_message) - - result['version'], data = _stripDoctype(data) - - baseuri = http_headers.get('content-location', result.get('href')) - baselang = http_headers.get('content-language', None) - - # if server sent 304, we're done - if result.get('status', 0) == 304: - result['version'] = '' - result['debug_message'] = 'The feed has not changed since you last checked, ' + \ - 'so the server sent no data. This is a feature, not a bug!' - return result - - # if there was a problem downloading, we're done - if not data: - return result - - # determine character encoding - use_strict_parser = 0 - known_encoding = 0 - tried_encodings = [] - # try: HTTP encoding, declared XML encoding, encoding sniffed from BOM - for proposed_encoding in (result['encoding'], xml_encoding, sniffed_xml_encoding): - if not proposed_encoding: continue - if proposed_encoding in tried_encodings: continue - tried_encodings.append(proposed_encoding) - try: - data = _toUTF8(data, proposed_encoding) - known_encoding = use_strict_parser = 1 - break - except: - pass - # if no luck and we have auto-detection library, try that - if (not known_encoding) and chardet: - try: - proposed_encoding = chardet.detect(data)['encoding'] - if proposed_encoding and (proposed_encoding not in tried_encodings): - tried_encodings.append(proposed_encoding) - data = _toUTF8(data, proposed_encoding) - known_encoding = use_strict_parser = 1 - except: - pass - # if still no luck and we haven't tried utf-8 yet, try that - if (not known_encoding) and ('utf-8' not in tried_encodings): - try: - proposed_encoding = 'utf-8' - tried_encodings.append(proposed_encoding) - data = _toUTF8(data, proposed_encoding) - known_encoding = use_strict_parser = 1 - except: - pass - # if still no luck and we haven't tried windows-1252 yet, try that - if (not known_encoding) and ('windows-1252' not in tried_encodings): - try: - proposed_encoding = 'windows-1252' - tried_encodings.append(proposed_encoding) - data = _toUTF8(data, proposed_encoding) - known_encoding = use_strict_parser = 1 - except: - pass - # if still no luck, give up - if not known_encoding: - result['bozo'] = 1 - result['bozo_exception'] = CharacterEncodingUnknown( \ - 'document encoding unknown, I tried ' + \ - '%s, %s, utf-8, and windows-1252 but nothing worked' % \ - (result['encoding'], xml_encoding)) - result['encoding'] = '' - elif proposed_encoding != result['encoding']: - result['bozo'] = 1 - result['bozo_exception'] = CharacterEncodingOverride( \ - 'documented declared as %s, but parsed as %s' % \ - (result['encoding'], proposed_encoding)) - result['encoding'] = proposed_encoding - - if not _XML_AVAILABLE: - use_strict_parser = 0 - if use_strict_parser: - # initialize the SAX parser - feedparser = _StrictFeedParser(baseuri, baselang, 'utf-8') - saxparser = xml.sax.make_parser(PREFERRED_XML_PARSERS) - saxparser.setFeature(xml.sax.handler.feature_namespaces, 1) - saxparser.setContentHandler(feedparser) - saxparser.setErrorHandler(feedparser) - source = xml.sax.xmlreader.InputSource() - source.setByteStream(_StringIO(data)) - if hasattr(saxparser, '_ns_stack'): - # work around bug in built-in SAX parser (doesn't recognize xml: namespace) - # PyXML doesn't have this problem, and it doesn't have _ns_stack either - saxparser._ns_stack.append({'http://www.w3.org/XML/1998/namespace':'xml'}) - try: - saxparser.parse(source) - except Exception, e: - if _debug: - import traceback - traceback.print_stack() - traceback.print_exc() - sys.stderr.write('xml parsing failed\n') - result['bozo'] = 1 - result['bozo_exception'] = feedparser.exc or e - use_strict_parser = 0 - if not use_strict_parser: - feedparser = _LooseFeedParser(baseuri, baselang, known_encoding and 'utf-8' or '') - feedparser.feed(data) - result['feed'] = feedparser.feeddata - result['entries'] = feedparser.entries - result['version'] = result['version'] or feedparser.version - result['namespaces'] = feedparser.namespacesInUse - return result - -if __name__ == '__main__': - if not sys.argv[1:]: - print __doc__ - sys.exit(0) - else: - urls = sys.argv[1:] - zopeCompatibilityHack() - from pprint import pprint - for url in urls: - print url - print - result = parse(url) - pprint(result) - print - -#REVISION HISTORY -#1.0 - 9/27/2002 - MAP - fixed namespace processing on prefixed RSS 2.0 elements, -# added Simon Fell's test suite -#1.1 - 9/29/2002 - MAP - fixed infinite loop on incomplete CDATA sections -#2.0 - 10/19/2002 -# JD - use inchannel to watch out for image and textinput elements which can -# also contain title, link, and description elements -# JD - check for isPermaLink='false' attribute on guid elements -# JD - replaced openAnything with open_resource supporting ETag and -# If-Modified-Since request headers -# JD - parse now accepts etag, modified, agent, and referrer optional -# arguments -# JD - modified parse to return a dictionary instead of a tuple so that any -# etag or modified information can be returned and cached by the caller -#2.0.1 - 10/21/2002 - MAP - changed parse() so that if we don't get anything -# because of etag/modified, return the old etag/modified to the caller to -# indicate why nothing is being returned -#2.0.2 - 10/21/2002 - JB - added the inchannel to the if statement, otherwise its -# useless. Fixes the problem JD was addressing by adding it. -#2.1 - 11/14/2002 - MAP - added gzip support -#2.2 - 1/27/2003 - MAP - added attribute support, admin:generatorAgent. -# start_admingeneratoragent is an example of how to handle elements with -# only attributes, no content. -#2.3 - 6/11/2003 - MAP - added USER_AGENT for default (if caller doesn't specify); -# also, make sure we send the User-Agent even if urllib2 isn't available. -# Match any variation of backend.userland.com/rss namespace. -#2.3.1 - 6/12/2003 - MAP - if item has both link and guid, return both as-is. -#2.4 - 7/9/2003 - MAP - added preliminary Pie/Atom/Echo support based on Sam Ruby's -# snapshot of July 1 <http://www.intertwingly.net/blog/1506.html>; changed -# project name -#2.5 - 7/25/2003 - MAP - changed to Python license (all contributors agree); -# removed unnecessary urllib code -- urllib2 should always be available anyway; -# return actual url, status, and full HTTP headers (as result['url'], -# result['status'], and result['headers']) if parsing a remote feed over HTTP -- -# this should pass all the HTTP tests at <http://diveintomark.org/tests/client/http/>; -# added the latest namespace-of-the-week for RSS 2.0 -#2.5.1 - 7/26/2003 - RMK - clear opener.addheaders so we only send our custom -# User-Agent (otherwise urllib2 sends two, which confuses some servers) -#2.5.2 - 7/28/2003 - MAP - entity-decode inline xml properly; added support for -# inline <xhtml:body> and <xhtml:div> as used in some RSS 2.0 feeds -#2.5.3 - 8/6/2003 - TvdV - patch to track whether we're inside an image or -# textInput, and also to return the character encoding (if specified) -#2.6 - 1/1/2004 - MAP - dc:author support (MarekK); fixed bug tracking -# nested divs within content (JohnD); fixed missing sys import (JohanS); -# fixed regular expression to capture XML character encoding (Andrei); -# added support for Atom 0.3-style links; fixed bug with textInput tracking; -# added support for cloud (MartijnP); added support for multiple -# category/dc:subject (MartijnP); normalize content model: 'description' gets -# description (which can come from description, summary, or full content if no -# description), 'content' gets dict of base/language/type/value (which can come -# from content:encoded, xhtml:body, content, or fullitem); -# fixed bug matching arbitrary Userland namespaces; added xml:base and xml:lang -# tracking; fixed bug tracking unknown tags; fixed bug tracking content when -# <content> element is not in default namespace (like Pocketsoap feed); -# resolve relative URLs in link, guid, docs, url, comments, wfw:comment, -# wfw:commentRSS; resolve relative URLs within embedded HTML markup in -# description, xhtml:body, content, content:encoded, title, subtitle, -# summary, info, tagline, and copyright; added support for pingback and -# trackback namespaces -#2.7 - 1/5/2004 - MAP - really added support for trackback and pingback -# namespaces, as opposed to 2.6 when I said I did but didn't really; -# sanitize HTML markup within some elements; added mxTidy support (if -# installed) to tidy HTML markup within some elements; fixed indentation -# bug in _parse_date (FazalM); use socket.setdefaulttimeout if available -# (FazalM); universal date parsing and normalization (FazalM): 'created', modified', -# 'issued' are parsed into 9-tuple date format and stored in 'created_parsed', -# 'modified_parsed', and 'issued_parsed'; 'date' is duplicated in 'modified' -# and vice-versa; 'date_parsed' is duplicated in 'modified_parsed' and vice-versa -#2.7.1 - 1/9/2004 - MAP - fixed bug handling " and '. fixed memory -# leak not closing url opener (JohnD); added dc:publisher support (MarekK); -# added admin:errorReportsTo support (MarekK); Python 2.1 dict support (MarekK) -#2.7.4 - 1/14/2004 - MAP - added workaround for improperly formed <br/> tags in -# encoded HTML (skadz); fixed unicode handling in normalize_attrs (ChrisL); -# fixed relative URI processing for guid (skadz); added ICBM support; added -# base64 support -#2.7.5 - 1/15/2004 - MAP - added workaround for malformed DOCTYPE (seen on many -# blogspot.com sites); added _debug variable -#2.7.6 - 1/16/2004 - MAP - fixed bug with StringIO importing -#3.0b3 - 1/23/2004 - MAP - parse entire feed with real XML parser (if available); -# added several new supported namespaces; fixed bug tracking naked markup in -# description; added support for enclosure; added support for source; re-added -# support for cloud which got dropped somehow; added support for expirationDate -#3.0b4 - 1/26/2004 - MAP - fixed xml:lang inheritance; fixed multiple bugs tracking -# xml:base URI, one for documents that don't define one explicitly and one for -# documents that define an outer and an inner xml:base that goes out of scope -# before the end of the document -#3.0b5 - 1/26/2004 - MAP - fixed bug parsing multiple links at feed level -#3.0b6 - 1/27/2004 - MAP - added feed type and version detection, result['version'] -# will be one of SUPPORTED_VERSIONS.keys() or empty string if unrecognized; -# added support for creativeCommons:license and cc:license; added support for -# full Atom content model in title, tagline, info, copyright, summary; fixed bug -# with gzip encoding (not always telling server we support it when we do) -#3.0b7 - 1/28/2004 - MAP - support Atom-style author element in author_detail -# (dictionary of 'name', 'url', 'email'); map author to author_detail if author -# contains name + email address -#3.0b8 - 1/28/2004 - MAP - added support for contributor -#3.0b9 - 1/29/2004 - MAP - fixed check for presence of dict function; added -# support for summary -#3.0b10 - 1/31/2004 - MAP - incorporated ISO-8601 date parsing routines from -# xml.util.iso8601 -#3.0b11 - 2/2/2004 - MAP - added 'rights' to list of elements that can contain -# dangerous markup; fiddled with decodeEntities (not right); liberalized -# date parsing even further -#3.0b12 - 2/6/2004 - MAP - fiddled with decodeEntities (still not right); -# added support to Atom 0.2 subtitle; added support for Atom content model -# in copyright; better sanitizing of dangerous HTML elements with end tags -# (script, frameset) -#3.0b13 - 2/8/2004 - MAP - better handling of empty HTML tags (br, hr, img, -# etc.) in embedded markup, in either HTML or XHTML form (<br>, <br/>, <br />) -#3.0b14 - 2/8/2004 - MAP - fixed CDATA handling in non-wellformed feeds under -# Python 2.1 -#3.0b15 - 2/11/2004 - MAP - fixed bug resolving relative links in wfw:commentRSS; -# fixed bug capturing author and contributor URL; fixed bug resolving relative -# links in author and contributor URL; fixed bug resolvin relative links in -# generator URL; added support for recognizing RSS 1.0; passed Simon Fell's -# namespace tests, and included them permanently in the test suite with his -# permission; fixed namespace handling under Python 2.1 -#3.0b16 - 2/12/2004 - MAP - fixed support for RSS 0.90 (broken in b15) -#3.0b17 - 2/13/2004 - MAP - determine character encoding as per RFC 3023 -#3.0b18 - 2/17/2004 - MAP - always map description to summary_detail (Andrei); -# use libxml2 (if available) -#3.0b19 - 3/15/2004 - MAP - fixed bug exploding author information when author -# name was in parentheses; removed ultra-problematic mxTidy support; patch to -# workaround crash in PyXML/expat when encountering invalid entities -# (MarkMoraes); support for textinput/textInput -#3.0b20 - 4/7/2004 - MAP - added CDF support -#3.0b21 - 4/14/2004 - MAP - added Hot RSS support -#3.0b22 - 4/19/2004 - MAP - changed 'channel' to 'feed', 'item' to 'entries' in -# results dict; changed results dict to allow getting values with results.key -# as well as results[key]; work around embedded illformed HTML with half -# a DOCTYPE; work around malformed Content-Type header; if character encoding -# is wrong, try several common ones before falling back to regexes (if this -# works, bozo_exception is set to CharacterEncodingOverride); fixed character -# encoding issues in BaseHTMLProcessor by tracking encoding and converting -# from Unicode to raw strings before feeding data to sgmllib.SGMLParser; -# convert each value in results to Unicode (if possible), even if using -# regex-based parsing -#3.0b23 - 4/21/2004 - MAP - fixed UnicodeDecodeError for feeds that contain -# high-bit characters in attributes in embedded HTML in description (thanks -# Thijs van de Vossen); moved guid, date, and date_parsed to mapped keys in -# FeedParserDict; tweaked FeedParserDict.has_key to return True if asking -# about a mapped key -#3.0fc1 - 4/23/2004 - MAP - made results.entries[0].links[0] and -# results.entries[0].enclosures[0] into FeedParserDict; fixed typo that could -# cause the same encoding to be tried twice (even if it failed the first time); -# fixed DOCTYPE stripping when DOCTYPE contained entity declarations; -# better textinput and image tracking in illformed RSS 1.0 feeds -#3.0fc2 - 5/10/2004 - MAP - added and passed Sam's amp tests; added and passed -# my blink tag tests -#3.0fc3 - 6/18/2004 - MAP - fixed bug in _changeEncodingDeclaration that -# failed to parse utf-16 encoded feeds; made source into a FeedParserDict; -# duplicate admin:generatorAgent/@rdf:resource in generator_detail.url; -# added support for image; refactored parse() fallback logic to try other -# encodings if SAX parsing fails (previously it would only try other encodings -# if re-encoding failed); remove unichr madness in normalize_attrs now that -# we're properly tracking encoding in and out of BaseHTMLProcessor; set -# feed.language from root-level xml:lang; set entry.id from rdf:about; -# send Accept header -#3.0 - 6/21/2004 - MAP - don't try iso-8859-1 (can't distinguish between -# iso-8859-1 and windows-1252 anyway, and most incorrectly marked feeds are -# windows-1252); fixed regression that could cause the same encoding to be -# tried twice (even if it failed the first time) -#3.0.1 - 6/22/2004 - MAP - default to us-ascii for all text/* content types; -# recover from malformed content-type header parameter with no equals sign -# ('text/xml; charset:iso-8859-1') -#3.1 - 6/28/2004 - MAP - added and passed tests for converting HTML entities -# to Unicode equivalents in illformed feeds (aaronsw); added and -# passed tests for converting character entities to Unicode equivalents -# in illformed feeds (aaronsw); test for valid parsers when setting -# XML_AVAILABLE; make version and encoding available when server returns -# a 304; add handlers parameter to pass arbitrary urllib2 handlers (like -# digest auth or proxy support); add code to parse username/password -# out of url and send as basic authentication; expose downloading-related -# exceptions in bozo_exception (aaronsw); added __contains__ method to -# FeedParserDict (aaronsw); added publisher_detail (aaronsw) -#3.2 - 7/3/2004 - MAP - use cjkcodecs and iconv_codec if available; always -# convert feed to UTF-8 before passing to XML parser; completely revamped -# logic for determining character encoding and attempting XML parsing -# (much faster); increased default timeout to 20 seconds; test for presence -# of Location header on redirects; added tests for many alternate character -# encodings; support various EBCDIC encodings; support UTF-16BE and -# UTF16-LE with or without a BOM; support UTF-8 with a BOM; support -# UTF-32BE and UTF-32LE with or without a BOM; fixed crashing bug if no -# XML parsers are available; added support for 'Content-encoding: deflate'; -# send blank 'Accept-encoding: ' header if neither gzip nor zlib modules -# are available -#3.3 - 7/15/2004 - MAP - optimize EBCDIC to ASCII conversion; fix obscure -# problem tracking xml:base and xml:lang if element declares it, child -# doesn't, first grandchild redeclares it, and second grandchild doesn't; -# refactored date parsing; defined public registerDateHandler so callers -# can add support for additional date formats at runtime; added support -# for OnBlog, Nate, MSSQL, Greek, and Hungarian dates (ytrewq1); added -# zopeCompatibilityHack() which turns FeedParserDict into a regular -# dictionary, required for Zope compatibility, and also makes command- -# line debugging easier because pprint module formats real dictionaries -# better than dictionary-like objects; added NonXMLContentType exception, -# which is stored in bozo_exception when a feed is served with a non-XML -# media type such as 'text/plain'; respect Content-Language as default -# language if not xml:lang is present; cloud dict is now FeedParserDict; -# generator dict is now FeedParserDict; better tracking of xml:lang, -# including support for xml:lang='' to unset the current language; -# recognize RSS 1.0 feeds even when RSS 1.0 namespace is not the default -# namespace; don't overwrite final status on redirects (scenarios: -# redirecting to a URL that returns 304, redirecting to a URL that -# redirects to another URL with a different type of redirect); add -# support for HTTP 303 redirects -#4.0 - MAP - support for relative URIs in xml:base attribute; fixed -# encoding issue with mxTidy (phopkins); preliminary support for RFC 3229; -# support for Atom 1.0; support for iTunes extensions; new 'tags' for -# categories/keywords/etc. as array of dict -# {'term': term, 'scheme': scheme, 'label': label} to match Atom 1.0 -# terminology; parse RFC 822-style dates with no time; lots of other -# bug fixes -#4.1 - MAP - removed socket timeout; added support for chardet library diff --git a/lib/filequeue.py b/lib/filequeue.py deleted file mode 100644 index 1b94adf..0000000 --- a/lib/filequeue.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Implements a simple queue, appending values to a file, and reading -them off the front of the file. Multi-thread but not multi-process -safe. -""" - -import threading -import os - -__all__ = ['FileQueue'] - -class FileQueue(object): - - def __init__(self, filename): - self.filename = filename - self._lock = threading.Lock() - - def append(self, value): - """ - Appends a value to the queue. - """ - if not isinstance(value, str): - raise ValueError( - "Only strings may be added to a FileQueue, not %r" - % value) - self._lock.acquire() - try: - if '\n' in value: - raise ValueError( - "Values cannot have newlines: %r" % value) - f = open(self.filename, 'a') - f.write(value + '\n') - f.close() - finally: - self._lock.release() - - def extend(self, values): - self._lock.acquire() - try: - f = open(self.filename, 'a') - for value in values: - assert '\n' not in value, ( - "Values cannot have newlines: %r" % value) - f.write(value + '\n') - f.close() - finally: - self._lock.release() - - def set(self, value): - """ - Like append, but only adds the value to the queue if it - is not already in the queue. - """ - self._lock.acquire() - try: - try: - f = open(self.filename, 'r') - except IOError: - lines = [] - else: - lines = f.readlines() - f.close() - if (value + '\n') in lines: - return - f = open(self.filename, 'a') - f.write(value + '\n') - f.close() - finally: - self._lock.release() - - def pop(self): - """ - Pops the top value off the queue. Returns None if nothing - is left on the queue. - """ - value = self.popmany(1) - if value: - return value[0] - else: - return None - - def popall(self): - self._lock.acquire() - try: - try: - f = open(self.filename, 'r') - except IOError, e: - if e.errno == 2: # no such file - return [] - all = f.readlines() - f.close() - f = open(self.filename, 'w') - f.close() - return [v[:-1] for v in all] - finally: - self._lock.release() - - def popmany(self, n): - """ - Pops up to n entries at a time. - """ - self._lock.acquire() - try: - try: - size = os.stat(self.filename).st_size - except OSError: - return [] - if not size: - return [] - f = open(self.filename, 'r') - all = f.readlines() - f.close() - if not all: - return [] - f = open(self.filename, 'w') - f.write(''.join(all[n:])) - f.close() - return [v[:-1] for v in all[:n]] - finally: - self._lock.release() - diff --git a/lib/format_date.py b/lib/format_date.py deleted file mode 100644 index 9c261f2..0000000 --- a/lib/format_date.py +++ /dev/null @@ -1,77 +0,0 @@ -import time -try: - from mx import DateTime -except ImportError: - DateTime = None -try: - from datetime import datetime -except ImportError: - datetime = None - -def _days(d): - if isinstance(d, datetime): - return time.mktime(d.timetuple()) / 60 / 60 / 24 - elif isinstance(d, DateTime.DateTimeType): - return d.day_of_year - elif isinstance(d, time.struct_time): - return time.mktime(d) / 60 / 60 / 24 - else: - return d.dayOfYear() - -def format_date_relative(date): - """ - Formats a date relative to the current time. The result - will be dates like 'Yesterday', 'Wednesday 10:00am', - '12 May 2003', etc. - - Specifically: - - * If in the last 24 hours, just give the time. - * If yesterday, give 'yesterday TIME' - * If in the last 7 days, give 'DAY_OF_WEEK TIME' - * If in the same calendar year, give 'DAY_OF_MONTH MONTH' - * Otherwise gives 'DAY_OF_MONTH MONTH YEAR' - - It uses english names when appropriate (e.g., Apr or Thu). - """ - if date is None: - return '' - now = DateTime.now() - year = date.year - if callable(year): - year = year() - month = date.month - if callable(month): - month = month() - day = date.day - if callable(day): - day = day() - day_of_year = _days(date) - if now.year == year: - if now.month == month: - if _days(now) - 7 < day_of_year: - if now.day == day: - return format_time(date) - elif now.day - 1 == day: - return 'Yesterday %s' % format_time(date) - else: - return date.strftime('%a ') + format_time(date) - elif _days(now) - 21 < day_of_year: - return date.strftime('%a %d %b') - else: - return date.strftime('%d %b') - else: - return date.strftime('%d %b') - else: - return date.strftime('%d %b \'%y') - -def format_time(date): - """ - Formats the time, like 4:20pm; unlike strftime, it lower-cases - the am/pm, and doesn't create hours with leading zeros. - """ - text = date.strftime('%I:%M%p') - text = text[:-2] + text[-2:].lower() - if text.startswith('0'): - text = text[1:] - return text diff --git a/lib/formatdate.py b/lib/formatdate.py deleted file mode 100644 index 78bb9f5..0000000 --- a/lib/formatdate.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Does time-sensitive formatting of dates. -""" -import datetime - -def format_date(date, nonbreaking=False): - """ - Format a date relative to the current time (i.e., greater - precision if the date is recent). Returns a string. Accepts - a datetime object or an integer timestamp. - - If nonbreaking=True, then spaces will be replace with   - """ - if isinstance(date, (int, long)): - date = datetime.datetime.fromtimestamp(date) - now = datetime.datetime.now() - dist = now - date - if now.year != date.year: - result = date.strftime("%d %b '%y") - elif dist.days < 1: - result = format_hour(date) - elif dist.days < 2: - result = 'Yest. %s' % format_hour(date) - elif dist.days < 7: - result = date.strftime('%a ') + format_hour(date) - else: - result = date.strftime('%d %b') - if nonbreaking: - result = result.replace(' ', ' ') - return result - -def format_hour(date): - hour = date.hour - ampm = 'am' - if hour >= 12: - hour -= 12 - ampm = 'pm' - if hour == 0: - hour = 12 - return '%i:%02i%s' % (hour, date.minute, ampm) diff --git a/lib/ftpclient.py b/lib/ftpclient.py deleted file mode 100644 index e127d45..0000000 --- a/lib/ftpclient.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -A simple FTP client; See also sshclient -""" - -import ftplib -import os - -__all__ = ['upload_files'] - -def upload_files(hostname, fileList, username=None, - password=''): - if username is None: - username = os.environ['USER'] - ftp = ftplib.FTP(hostname, username, password) - for source, dest in fileList: - f = open(source) - dir = os.path.dirname(dest) - while 1: - try: - ftp.storbinary('STOR %s' % dest, f) - except ftplib.error_perm: - pass - else: - break - ftp_makedirs(ftp, dir) - f.close() - ftp.close() - -def ftp_makedirs(ftp, dir): - """ - Makes the directories for dir, creating any directories as - necessary. - """ - tryDir = dir - while 1: - try: - ftp.mkd(tryDir) - except ftplib.error_perm: - tryDir = os.path.dirname(tryDir) - else: - if tryDir == dir: - return - tryDir = dir - -if __name__ == '__main__': - import sys - upload_files(sys.argv[1], [p.split(':', 1) for p in sys.argv[3:]], - password=sys.argv[2]) diff --git a/lib/html_abstracts.py b/lib/html_abstracts.py deleted file mode 100644 index e4e4382..0000000 --- a/lib/html_abstracts.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Parses HTML to find an 'abstract', i.e., a summary of the page. -""" - -import re - -tagRE = re.compile(r'<(/?[a-zA-Z0-9\-_]+)([^>]*)>') -classRE = re.compile(r'class="(.*?)"', re.I) - -def find_abstract(html): - """ - Search for something with the class 'abstract', or if nothing has - such an attribute, a the first <p> or <div> tag we can find, or - if there's not even that, the entire document. - """ - fullHTML = html - while 1: - match = tagRE.search(html) - if not match: - return guess_abstract(fullHTML) - html = html[match.end():] - tag = match.group(1) - attrs = match.group(2) - classMatch = classRE.search(attrs) - if not classMatch: - continue - classes = classMatch.group(1).lower().split() - if 'abstract' in classes: - return find_match_tag(html, tag.lower()) - -def find_match_tag(html, tagStart): - """ - Give all the html that comes after, say, a <p> tag, find all - the html up to the </p> tag, allowing for nested tags. - """ - fullHTML = html - tagNest = 0 - while 1: - match = tagRE.search(html) - if not match: - return fullHTML - tag = match.group(1).lower() - if tag == tagStart: - tagNest += 1 - elif tag == '/' + tagStart: - if not tagNest: - return fullHTML[:-(len(html)-match.start())] - tagNest -= 1 - html = html[match.end():] - -def guess_abstract(html): - """ - Finds the first <p> tag, and we consider it to be the - abstract. - """ - fullHTML = html - while 1: - match = tagRE.search(html) - if not match: - return fullHTML - html = html[match.end():] - tag = match.group(1).lower() - if tag == 'p': - return find_match_tag(html, tag) diff --git a/lib/htmldiff.py b/lib/htmldiff.py deleted file mode 100755 index 4b51692..0000000 --- a/lib/htmldiff.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python -""" -htmldiff.py -(C) Ian Bicking <ianb@colorstudy.com> - -Finds the differences between two HTML files. *Not* line-by-line -comparison (more word-by-word). - -Command-line usage: - ./htmldiff.py test1.html test2.html - -Better results if you use mxTidy first. The output is HTML. -""" - -from difflib import SequenceMatcher -import re -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO -import cgi - -def htmlEncode(s, esc=cgi.escape): - return esc(s, 1) - -commentRE = re.compile('<!--.*?-->', re.S) -tagRE = re.compile('<.*?>', re.S) -headRE = re.compile('<\s*head\s*>', re.S | re.I) - -class HTMLMatcher(SequenceMatcher): - - def __init__(self, source1, source2): - SequenceMatcher.__init__(self, None, source1, source2) - - def set_seq1(self, a): - SequenceMatcher.set_seq1(self, self.splitHTML(a)) - - def set_seq2(self, b): - SequenceMatcher.set_seq2(self, self.splitHTML(b)) - - def splitTags(self, t): - result = [] - pos = 0 - while 1: - match = tagRE.search(t, pos=pos) - if not match: - result.append(t[pos:]) - break - result.append(t[pos:match.start()]) - result.append(match.group(0)) - pos = match.end() - return result - - def splitWords(self, t): - return t.strip().split() - - def splitHTML(self, t): - t = commentRE.sub('', t) - r = self.splitTags(t) - result = [] - for item in r: - if item.startswith('<'): - result.append(item) - else: - result.extend(self.splitWords(item)) - return result - - def htmlDiff(self, addStylesheet=False): - opcodes = self.get_opcodes() - a = self.a - b = self.b - out = StringIO() - #print [o[0] for o in opcodes] - for tag, i1, i2, j1, j2 in opcodes: - if tag == 'equal': - for item in a[i1:i2]: - out.write(item) - out.write(' ') - if tag == 'delete' or tag == 'replace': - self.textDelete(a[i1:i2], out) - if tag == 'insert' or tag == 'replace': - self.textInsert(b[j1:j2], out) - html = out.getvalue() - out.close() - if addStylesheet: - html = self.addStylesheet(html, self.stylesheet()) - return html - - def textDelete(self, lst, out): - inSpan = False - for item in lst: - if item.startswith('<'): - if inSpan: - out.write(self.endDeleteText()) - inSpan = False - out.write(self.formatDeleteTag(item)) - else: - if not inSpan: - out.write(self.startDeleteText()) - inSpan = True - out.write(item) - out.write(' ') - if inSpan: - out.write(self.endDeleteText()) - - def textInsert(self, lst, out): - inSpan = False - for item in lst: - if item.startswith('<'): - if inSpan: - out.write(self.endInsertText()) - inSpan = False - out.write(self.formatInsertTag(item)) - out.write(item) - out.write(' ') - else: - if not inSpan: - out.write(self.startInsertText()) - inSpan = True - out.write(item) - out.write(' ') - if inSpan: - out.write(self.endInsertText()) - - def stylesheet(self): - return ''' -.insert { background-color: #aaffaa } -.delete { background-color: #ff8888 } -.tagInsert { background-color: #007700; color: #ffffff } -.tagDelete { background-color: #770000; color: #ffffff } -''' - - def addStylesheet(self, html, ss): - match = headRE.search(html) - if match: - pos = match.end() - else: - pos = 0 - return ('%s<style type="text/css"><!--\n%s\n--></style>%s' - % (html[:pos], ss, html[pos:])) - - def startInsertText(self): - return '<span class="insert">' - def endInsertText(self): - return '</span> ' - def startDeleteText(self): - return '<span class="delete">' - def endDeleteText(self): - return '</span> ' - def formatInsertTag(self, tag): - return ('<span class="tagInsert">insert: <tt>%s</tt></span> ' - % htmlEncode(tag)) - def formatDeleteTag(self, tag): - return ('<span class="tagDelete">delete: <tt>%s</tt></span> ' - % htmlEncode(tag)) - -class NoTagHTMLMatcher(HTMLMatcher): - def formatInsertTag(self, tag): - return '' - def formatDeleteTag(self, tag): - return '' - -def htmldiff(source1, source2, addStylesheet=False): - """ - Return the difference between two pieces of HTML - - >>> htmldiff('test1', 'test2') - '<span class="delete">test1 </span> <span class="insert">test2 </span> ' - >>> htmldiff('test1', 'test1') - 'test1 ' - >>> htmldiff('<b>test1</b>', '<i>test1</i>') - '<span class="tagDelete">delete: <tt><b></tt></span> <span class="tagInsert">insert: <tt><i></tt></span> <i> test1 <span class="tagDelete">delete: <tt></b></tt></span> <span class="tagInsert">insert: <tt></i></tt></span> </i> ' - """ - h = HTMLMatcher(source1, source2) - return h.htmlDiff(addStylesheet) - -def diffFiles(f1, f2): - source1 = open(f1).read() - source2 = open(f2).read() - return htmldiff(source1, source2, True) - -class SimpleHTMLMatcher(HTMLMatcher): - """ - Like HTMLMatcher, but returns a simpler diff - """ - def startInsertText(self): - return '+[' - def endInsertText(self): - return ']' - def startDeleteText(self): - return '-[' - def endDeleteText(self): - return ']' - def formatInsertTag(self, tag): - return '+[%s]' % tag - def formatDeleteTag(self, tag): - return '-[%s]' % tag - -def simplehtmldiff(source1, source2): - """ - Simpler form of htmldiff; mostly for testing, like: - - >>> simplehtmldiff('test1', 'test2') - '-[test1 ]+[test2 ]' - >>> simplehtmldiff('<b>Hello world!</b>', '<i>Hello you!</i>') - '-[<b>]+[<i>]<i> Hello -[world! ]-[</b>]+[you! ]+[</i>]</i> ' - """ - h = SimpleHTMLMatcher(source1, source2) - return h.htmlDiff() - -class TextMatcher(HTMLMatcher): - - - def set_seq1(self, a): - SequenceMatcher.set_seq1(self, a.split('\n')) - - def set_seq2(self, b): - SequenceMatcher.set_seq2(self, b.split('\n')) - - def htmlDiff(self, addStylesheet=False): - opcodes = self.get_opcodes() - a = self.a - b = self.b - out = StringIO() - for tag, i1, i2, j1, j2 in opcodes: - if tag == 'equal': - self.writeLines(a[i1:i2], out) - if tag == 'delete' or tag == 'replace': - out.write(self.startDeleteText()) - self.writeLines(a[i1:i2], out) - out.write(self.endDeleteText()) - if tag == 'insert' or tag == 'replace': - out.write(self.startInsertText()) - self.writeLines(b[j1:j2], out) - out.write(self.endInsertText()) - html = out.getvalue() - out.close() - if addStylesheet: - html = self.addStylesheet(html, self.stylesheet()) - return html - - def writeLines(self, lines, out): - for line in lines: - line = htmlEncode(line) - line = line.replace(' ', '  ') - line = line.replace('\t', '        ') - if line.startswith(' '): - line = ' ' + line[1:] - out.write('<tt>%s</tt><br>\n' % line) - -if __name__ == '__main__': - import sys - if not sys.argv[1:]: - print "Usage: %s file1 file2" % sys.argv[0] - print "or to test: %s test" % sys.argv[0] - elif sys.argv[1] == 'test' and not sys.argv[2:]: - import doctest - doctest.testmod() - else: - print diffFiles(sys.argv[1], sys.argv[2]) - diff --git a/lib/import_atom.py b/lib/import_atom.py deleted file mode 100644 index c11056a..0000000 --- a/lib/import_atom.py +++ /dev/null @@ -1,104 +0,0 @@ -import feedparser -from common import htmlEncode, canonicalName, guessURLName -from datetime import datetime, timedelta -import time - -def import_atom(wiki, atom_file, logger): - feed = feedparser.parse(atom_file) - logger('Importing feed titled <a href="%s">%s</a>' - % (feed.feed.get('link', '(unknown)'), htmlEncode(feed.feed.title))) - redirects = [] - for entry in feed.entries: - import_entry(wiki, entry, logger, redirects) - return redirects - -def import_entry(wiki, entry, logger, redirects): - title = entry.title - name = guessURLName(title) - base = '' - while wiki.exists(name + str(base)): - if not base: - base = 1 - else: - base += 1 - name = name + str(base) - page = wiki.page(name) - page.title = title - if entry.get('modified_parsed'): - modified_date = datetime(*entry.modified_parsed[:7]) - page.createdDate = modified_date - id = entry.get('id') - if id: - try: - page.atomID = id - except ValueError, e: - logger("Could not set ATOM ID to %r for page %s; error: %s" - % (id, name, e)) - ## @@: currently summary is not persistent - # summary = entry.summary - contents = entry.content - page.mimeType = str(contents[0]['type']) - page.text = '\n'.join([c['value'] for c in contents]) - page.pageClass = 'posting' - page.creationDate = time.mktime(convert_date(entry.issued).timetuple()) - author = entry.get('author_detail') - if author and author.get('name'): - author_name = author['name'] - if author.get('name'): - page.authorName = author.name - if author.get('email'): - page.authorEmail = author.email - if author.get('url') and author.url.lower().strip() != 'http://': - page.authorURL = author.url - if author.get('email'): - author_name = '%s <%s>' % (author_name, author['email']) - page.lastChangeUser = author_name - parents = [link for link in entry.links if link.rel == 'parent'] - if parents: - page.pageClass = 'comment' - for parent in parents: - parentPage = wiki.pageByAtomID(parent.href) - if not parentPage: - logger('Parent page not found: %r' % parent.href) - print "Not found: %r" % parent.href - continue - page.connections += [(parentPage, 'comment')] - logger('%s a comment on %s' - % (title, parentPage.name)) - print "%s -> %s" % (title, parentPage.name) - page.save() - logger("Saved page %s as %s" - % (title, name)) - print "Imported %s" % name - if entry.has_key('link'): - redirects.append((entry.link, page.link)) - -def convert_date(isodate): - date, time_part = isodate.split('T', 1) - year, month, day = date.split('-', 2) - if '-' in time_part: - pos = time_part.find('-') - elif '+' in time_part: - pos = time_part.find('+') - else: - pos = -1 - if pos != -1: - time_part, tz = time_part[:pos], time_part[pos:] - else: - tz = '' - hour, minute, second = time_part.split(':', 2) - result = datetime( - int(year), int(month), int(day), - int(hour), int(minute), int(second)) - if tz: - plus_minus = tz[0] - if plus_minus == '-': - mult = -1 - else: - mult = 1 - hour_offset, minute_offset = tz[1:].split(':', 1) - delta = timedelta(hours=mult*int(minute_offset), - minutes=mult*int(hour_offset)) - result += delta - return result - diff --git a/lib/menubar.py b/lib/menubar.py deleted file mode 100644 index be531be..0000000 --- a/lib/menubar.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Functions to generate HTML for the menubar. -""" - -__all__ = ['Literal', 'Separator', 'menubarHTML'] - -class Literal: - pass - -class Separator: - pass - -class Namespace: - - def __init__(self, prefix): - self.prefix = prefix - self.id = 1 - - def get(self): - this, self.id = self.id, self.id+1 - return '%s_%i' % (self.prefix, this) - -def menubarHTML(lst, namespace="menu"): - """ - Converts a list into the HTML for use with the menubar.js - code (see also http://www.brainjar.com/dhtml/menubar/) - - Each link is a tuple of (title, href). Or, href can itself be a - list of tuples. So a nested list looks like:: - - [('File', [('Open', '...'), ...])] - - If you want to include a non-menu element (particularly in the - top-level bar) use a title of Literal (which is a symbol in this - module). - - This returns a tuple of (menubar, subelements). The menubar - should be put wherever you want it to show up on the page, while - the subelements should go directly after <body> or just before - </body>. - - Pages that use this should also include ``menubar.js`` and the - necessary CSS. - """ - - menubar = [] - subToRender = {} - subelements = [] - ns = Namespace(namespace) - - menubar.append('<div class="menuBar">\n') - - for title, args in lst: - if title is Literal: - menubar.append(args) - elif isinstance(args, list): - name = ns.get() - menubar.append( - '<a class="menuButton" href="" ' - 'onclick="return buttonClick(event, \'%s\')" ' - 'onmouseover="buttonMouseover(event, \'%s\')">%s</a>\n' - % (name, name, title)) - subToRender[(title, name)] = args - else: - menubar.append( - '<a class="menuButton" href="%s" ' - 'onmouseover="buttonMouseover(event, \'\')">%s</a>\n' - % (args, title)) - - menubar.append('</div>') - - while subToRender: - (parentTitle, name), args = subToRender.popitem() - subelements.append('<!-- %s menus: -->\n' % parentTitle) - subelements.append( - '<div id="%s" class="menu" onmouseover="menuMouseover(event)">\n' - % name) - for title, subargs in args: - if title is Literal: - subelements.append(subargs) - elif title is Separator: - subelements.append('<div class="menuItemSep"></div>\n') - elif isinstance(subargs, list): - name = ns.get() - subelements.append( - '<a class="menuItem" href="" onclick="return false;" ' - 'onmouseover="menuItemMouseover(event, \'%s\');">' - '<span class="menuItemText">%s</span>' - '<span class="menuItemArrow">⓮</span></a>\n' - % (name, title)) - subToRender[(title, name)] = subargs - else: - subelements.append( - '<a class="menuItem" href="%s">%s</a>\n' - % (subargs, title)) - subelements.append('</div>\n') - - return ''.join(menubar), ''.join(subelements) - -__test__ = { - 'simple': r""" - >>> def t(v): - ... a, b = menubarHTML(v) - ... print a.strip(), '\n', b.strip() - >>> t([('File', 'open.html')]) - <div class="menubar"> - <a class="menuButton" href="open.html" onmouseover="buttonMouseover(event, '')"> - </div> - >>> t([('File', [('Open', 'open.html'), ('Close', 'close.html')])]) - <div class="menubar"> - <a class="menuButton" href="" onclick="return buttonClick(event, 'menu_1')" onmouseover="buttonMouseover(event, 'menu_1')"> - </div> <!-- File menus: --> - <div id="menu_1" class="menu" onmouseover="menuMouseover(event)"> - <a class="menuItem" href="open.html">Open</a> - <a class="menuItem" href="close.html">Close</a> - </div> - """, - } - -if __name__ == '__main__': - import doctest - doctest.testmod() - diff --git a/lib/pooledtemplate.py b/lib/pooledtemplate.py deleted file mode 100644 index ac63d47..0000000 --- a/lib/pooledtemplate.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Pooled Cheetah templates -""" - -from Cheetah.Template import Template as CheetahTemplate - -class Template(object): - - _keywords = ['cacheSize', 'searchList'] - cacheSize = 10 - searchList = [] - - def __init__(self, *args, **kw): - for name in self._keywords: - if kw.has_key(name): - setattr(self, name, kw[name]) - del kw[name] - self.args = args - self.kw = kw - self.pool = [] - - def eval(self, **namespace): - tmpl = None - try: - tmpl, tmplNamespace = self.getTemplate() - tmplNamespace.clear() - tmplNamespace.update(namespace) - result = str(tmpl) - finally: - if tmpl: - self.returnTemplate(tmpl, tmplNamespace) - return result - - def getTemplate(self): - try: - return self.pool.pop() - except IndexError: - return self.newTemplate() - - def newTemplate(self): - namespace = {} - kw = self.kw.copy() - kw['searchList'] = self.searchList + [namespace] - tmpl = CheetahTemplate(*self.args, **kw) - return tmpl, namespace - - def returnTemplate(self, tmpl, namespace): - if len(self.pool) >= self.cacheSize: - # Throw it away - return - self.pool.append((tmpl, namespace)) diff --git a/lib/propertymeta.py b/lib/propertymeta.py deleted file mode 100644 index a811f05..0000000 --- a/lib/propertymeta.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -propertymeta.py -Ian Bicking <ianb@colorstudy.com> -""" - -class MakeProperties(type): - """ - A metaclass that runs `makeProperties` on your classes. Use - it like:: - - class MyClass(object): - - __metaclass__ = MakeProperties - - def attr__get(self): - return blah - - ``attr__get`` will be magically turned into a property. - """ - - def __new__(mcs, className, bases, d): - makeProperties(d) - return type.__new__(mcs, className, bases, d) - -def makeProperties(obj): - """ - This function takes a dictionary of methods and finds - methods named like: - * attr__get - * attr__set - * attr__del - * attr__doc - Except for attr__doc, these should be methods. It - then creates properties from these methods, like - property(attr__get, attr__set, attr__del, attr__doc). - Missing methods are okay. - - It also takes methods like: - * attr__class - Which is turned into a class method. - - You can pass either an object or a dictionary in, where - the dictionary is obj.__dict__ (the dictionary interface - allows you to run this on yet-to-be-created objects, - like in a metaclass `__new__` method). - """ - - if isinstance(obj, dict): - def setFunc(var, value): - obj[var] = value - d = obj - else: - def setFunc(var, value): - setattr(obj, var, value) - d = obj.__dict__ - - props = {} - for var, value in d.items(): - if var.endswith('__set'): - props.setdefault(var[:-5], {})['set'] = value - elif var.endswith('__get'): - props.setdefault(var[:-5], {})['get'] = value - elif var.endswith('__del'): - props.setdefault(var[:-5], {})['del'] = value - elif var.endswith('__doc'): - props.setdefault(var[:-5], {})['doc'] = value - elif var.endswith('__class'): - setFunc(var[:-7], classmethod(value)) - - for var, setters in props.items(): - if len(setters) == 1 and setters.has_key('doc'): - continue - if d.has_key(var): - continue - setFunc(var, - property(setters.get('get'), setters.get('set'), - setters.get('del'), setters.get('doc'))) - - -def unmakeProperties(obj): - """ - Accompanies makeProperties -- when you dynamically modify a - class, adding or removing getters and setters, the property - will persist (unless you delete the property itself). This - goes through and gets rid of properties where any of its - methods (get, set, delete) have been removed. - """ - - if isinstance(obj, dict): - def delFunc(obj, var): - del obj[var] - d = obj - else: - delFunc = delattr - d = obj.__dict__ - - for var, value in d.items(): - if isinstance(value, property): - for prop in [value.fget, value.fset, value.fdel]: - if prop and not d.has_key(prop.__name__): - delFunc(obj, var) - break diff --git a/lib/rfc822persist.py b/lib/rfc822persist.py deleted file mode 100644 index b2ebb1a..0000000 --- a/lib/rfc822persist.py +++ /dev/null @@ -1,275 +0,0 @@ -r""" -Provides a dictionary-like interface, where keys and values are -stored in an rfc822 formatted file. Data is automatically saved -when keys are set. - -Examples:: - - >>> r = RFC822StringDict('''key: value''') - >>> r['key'] - 'value' - >>> r.get('nothing', 'default') - 'default' - >>> r['new'] = '1' - >>> r.string - 'key: value' - >>> r.save() - >>> r.string - 'key: value\nnew: 1\n' - >>> r['NEW'] - '1' - >>> r['test'] = 'a\nb\n c' - >>> r['test'] - 'a\nb\n c' - >>> r.save() - >>> r2 = RFC822StringDict(r.string) - >>> r2['test'] - 'a\nb\n c' - -Note that keys are case insensitive. All values are stored as -strings, with whitespace stripped. If you need to store, say, an -integer, be sure to convert the value you get back. If you pass -lazy=False to the constuctor, then you need not call .save() to -write to disk. -""" - -import rfc822 -from UserDict import DictMixin -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO -import os - -class RFC822Dict(DictMixin): - - def __init__(self, filename, lazy=True): - self.filename = filename - self._data = None - self.lazy = lazy - self.dirty = False - - def __getitem__(self, key): - if self._data is None: - self._readData() - value = self._data[key] - if '\n' in value: - lines = value.splitlines() - for i in range(1, len(lines)): - if lines[i].startswith(' .'): - lines[i] = lines[i][5:] - elif lines[i].startswith(' .'): - lines[i] = lines[i][2:] - else: - lines[i] = lines[i].strip() - value = '\n'.join(lines) - return value - - def __setitem__(self, key, value): - value = str(value).strip() - value = value.replace('\r', '') - if '\n' in value: - lines = value.splitlines() - value = '\n'.join([lines[0]] + [' .' + l for l in lines[1:]]) - if self._data is None: - self._readData() - self._data[key] = value - self.dirty = True - if not self.lazy: - self._saveData() - - def __delitem__(self, key): - if self._data is None: - self._readData() - del self._data[key] - self.dirty = True - if not self.lazy: - self._saveData() - - def getbool(self, key, default=False): - value = self.get(key) - if value is None: - return default - if value.lower() in ['on', 'yes', '1', 'true']: - return True - else: - return False - - def setbool(self, key, value): - if value: - self[key] = 'true' - else: - self[key] = 'false' - - def keys(self): - if self._data is None: - self._readData() - return self._data.keys() - - def reset(self): - self._data = None - - def save(self): - self.dirty = False - self._saveData() - - def _readData(self): - if not os.path.exists(self.filename): - self._data = rfc822.Message(StringIO()) - else: - f = open(self.filename) - self._data = rfc822.Message(f) - f.close() - - def _saveData(self): - f = open(self.filename, 'w') - f.write(self._headerStr(self._data)) - f.close() - - def saveKeyNow(self, key): - f = open(self.filename, 'a') - for header in self._data.getallmatchingheaders(key): - f.write(header) - if not header.endswith('\n'): - f.write('\n') - f.close() - - def _headerStr(self, message): - headers = message.headers - result = StringIO() - for header in headers: - result.write(header) - if not header.endswith('\n'): - result.write('\n') - return result.getvalue() - -class RFC822StringDict(RFC822Dict): - - """ - An RFC822Dict that stores to a string, instead of a file. - """ - - def __init__(self, string, lazy=True): - self.string = string - self._data = None - self.lazy = lazy - - def _readData(self): - f = StringIO(self.string) - self._data = rfc822.Message(f) - f.close() - - def _saveData(self): - self.string = self._headerStr(self._data) - - def saveKeyNow(self, key): - pass - -class metaprop(object): - - """ - A descriptor for use with objects that have a 'metadata' - attribute that contains a dictionary like RFC822Dict. - - name: - The name of the key in the dictionary - default: - The value if no key is found; this will *not* be converted - converter: - A function that is called with the value from the dictionary. - unconverter: - A function that converts a value to be saved in the dictionary - - """ - - def __init__(self, name, default=None, - converter=None, unconverter=None): - self.name = name - if converter is not None or not hasattr(self, 'converter'): - self.converter = converter or utf_unstr - if unconverter is not None or not hasattr(self, 'unconverter'): - self.unconverter = unconverter or utf_str - if default is not None or not hasattr(self, 'default'): - self.default = default - - def default_func(self, obj): - return self.default - - def __get__(self, obj, type=None): - if obj is None: - return self - if obj.metadata.has_key(self.name): - value = obj.metadata[self.name] - if self.converter: - return self.converter(value) - else: - return self.default_func(obj) - - def __set__(self, obj, value): - if self.unconverter: - value = self.unconverter(value) - if value is None: - if obj.metadata.has_key(self.name): - del obj.metadata[self.name] - else: - obj.metadata[self.name] = value - - def __delete__(self, obj): - del obj.metadata[self.name] - -def utf_str(s): - if isinstance(s, unicode): - return s.encode('utf-8') - return s - -def utf_unstr(s): - return s.decode('utf-8') - -class metabool(metaprop): - - """ - A metaprop that coerces booleans nicely. - """ - - def __init__(self, name, default=False, delete_if_default=False): - metaprop.__init__(self, name, default=default) - self.delete_if_default = delete_if_default - - def converter(self, value): - if not value: - return False - if value.lower().strip() == 'true': - return True - return False - - def unconverter(self, value): - if self.delete_if_default and bool(value) == bool(self.default): - return None - if value: - return 'True' - else: - return 'False' - -__test__ = { - 'tests': - """ - >>> r = RFC822StringDict('''key: value''') - >>> r['new'] = ' ' - >>> r['new'] - '' - >>> r.string = 'something: whatever' - >>> r['new'] - '' - >>> r.reset() - >>> r['something'] - 'whatever' - """ - } - - - - - -if __name__ == '__main__': - import doctest - doctest.testmod() diff --git a/lib/rssobject.py b/lib/rssobject.py deleted file mode 100644 index 0c3f789..0000000 --- a/lib/rssobject.py +++ /dev/null @@ -1,366 +0,0 @@ -""" -This presents an object that can read from an RSS 2.0 file, be -modified in place, then be written out to an RSS 2.0 file. -Essentially it is an object representation of the RSS file. -""" - -__version__ = '0.1' - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO -import xml.sax -import xml.sax.handler -import xml.sax.saxutils -import time -from datetime import datetime -import os - -############################################################ -## Utilities -############################################################ - -class NoDefault: - pass - -class metasetter(object): - - def __init__(self, name, converter=None, unconverter=None, - default=NoDefault): - self.name = name - self.converter = converter - self.unconverter = unconverter - self.default = default - - def __get__(self, obj, type=None): - if self.default is NoDefault: - value = obj.metadata[self.name] - else: - value = obj.metadata.get(self.name, self.default) - if self.unconverter: - return self.unconverter(value) - else: - return value - - def __set__(self, obj, value): - if self.converter: - value = self.converter(value) - obj.metadata[self.name] = value - if self.name not in obj.metadataOrder: - obj.metadataOrder.append(self.name) - - def __delete__(self, obj): - del obj.metadata[self.name] - -def formatDate(value=None): - format = '%a, %d %b %Y %H:%M:%S GMT' - if value is None: - value = time.gmtime() - if isinstance(value, (int, float, time.struct_time)): - return time.strftime(format, value) - elif isinstance(value, (str, unicode)): - return value - else: - # @@: should also deal with time tuples - return value.strftime(format) - -def parseDate(datestr): - return datetime( - *time.strptime(datestr, '%a, %d %b %Y %H:%M:%S GMT')[:7]) - -def xmlEncode(val): - assert isinstance(val, (str, unicode)), "Bad type: %r" % val - return xml.sax.saxutils.escape(val, {'"': '"'}) - -############################################################ -## Public classes -############################################################ - -class RSS(object): - - defaultGenerator = 'rssobject.py v%s' % __version__ - - def __init__(self, filename=None, text=None, metadata=None, - items=None, sortOrder=None): - assert not text or not filename, ( - "You cannot provide both text and a filename/file object") - self.items = [] - self.metadata = {} - self.metadataOrder = [] - if filename and not isinstance(filename, (str, unicode)): - # assume it's a file-like object - self.parseFile(filename) - self.filename = None - elif filename: - if os.path.exists(filename): - f = open(filename) - self.parseFile(f) - f.close() - self.filename = filename - elif text: - self.parseFile(StringIO(text)) - self.filename = None - else: - self.metadata = {} - self.items = [] - self.sortOrder = sortOrder - if metadata: - self.metadata.update(metadata) - if items: - for item in items: - self.addItem(item, len(items)) - - def parseFile(self, f): - xml.sax.parse(f, SAXHandler(self)) - - def writeText(self, filename=None): - if filename is None: - assert self.filename, ( - "If you didn't pass a filename to the RSS constructor, " - "you must pass one to writeText") - filename = self.filename - if not isinstance(filename, (str, unicode)): - # assume it's a file-like object - f = filename - else: - f = open(filename, 'w') - f.write(str(self)) - f.close() - - def fillMetadata(self): - metadata = self.metadata.copy() - if not metadata.get('generator'): - metadata['generator'] = self.defaultGenerator - if not metadata.get('lastBuildDate'): - metadata['lastBuildDate'] = formatDate() - if not metadata.get('pubDate'): - metadata['pubDate'] = formatDate() - return metadata - - def __str__(self): - result = StringIO() - result.write('<?xml version="1.0"?>\n<rss version="2.0">\n' - '<channel>\n') - seenItems = False - metadata = self.fillMetadata() - items = [] - for name in self.metadataOrder: - if name == 'item': - if not seenItems: - items.append(('item', None)) - seenItems = True - continue - if name in metadata: - items.append((name, metadata[name])) - del metadata[name] - items.extend(metadata.items()) - if not seenItems: - items.append(('item', None)) - for name, value in items: - if name == 'item': - for i in self.items: - result.write(str(i)) - continue - if isinstance(value, unicode): - value = value.encode('utf-8') - result.write('<%s>%s</%s>\n' - % (name, xmlEncode(value), name)) - result.write('</channel>\n</rss>\n') - return result.getvalue() - - def addItem(self, item, maxItems): - self.items.append(item) - self.items = self.sortItems(self.items) - while len(self.items) > maxItems: - self.items.pop(0) - - def sortItems(self, items): - if not self.sortOrder: - return items - sorter = self.sortOrder - reversed = False - if isinstance(sorter, str): - if sorter.startswith('-'): - reversed = True - sorter = sorter[1:] - sorter = lambda a, b, sorter=sorter: ( - cmp(getattr(a, sorter), getattr(b, sorter))) - items = items[:] - items.sort(sorter) - if reversed: - items.reverse() - return items - - title = metasetter('title') - link = metasetter('link') - description = metasetter('description') - language = metasetter('language') - copyright = metasetter('copyright') - managingEditor = metasetter('managingEditor') - webMaster = metasetter('webMaster') - pubDate = metasetter('pubdate', formatDate) - lastBuildDate = metasetter('lastBuildDate', formatDate) - category = metasetter('category') - generator = metasetter('generator') - docs = metasetter('docs') - cloud = metasetter('cloud') - ttl = metasetter('ttl', str, int) - image = metasetter('image') - rating = metasetter('rating') - textInput = metasetter('textInput') - skipHours = metasetter('skipHours') - skipdays = metasetter('skipDays') - -class RSSItem(object): - - def __init__(self, metadataOrder=None, **kw): - self.metadataOrder = metadataOrder or [] - self.metadata = kw - - def __str__(self): - metadata = self.fillMetadata() - result = StringIO() - result.write('<item>\n') - items = [] - for name in self.metadataOrder: - if name in metadata: - items.append((name, metadata[name])) - del metadata[name] - items.extend(metadata.items()) - for name, value in items: - if isinstance(value, unicode): - value = value.encode('utf-8') - result.write('<%s>%s</%s>\n' % - (str(name), xmlEncode(value), str(name))) - result.write('</item>\n') - return result.getvalue() - - def fillMetadata(self): - metadata = self.metadata.copy() - if metadata.get('link') and not metadata.get('guid'): - metadata['guid'] = metadata['link'] - elif metadata.get('guid') and not metadata.get('link'): - metadata['link'] = metadata['guid'] - if not metadata.get('pubDate'): - metadata['pubDate'] = formatDate() - return metadata - - title = metasetter('title') - link = metasetter('link') - description = metasetter('description') - author = metasetter('author') - category = metasetter('category') - comments = metasetter('comments') - enclosure = metasetter('enclosure') - guid = metasetter('guid') - pubDate = metasetter('pubDate') - source = metasetter('source') - -############################################################ -## Parser -############################################################ - -class SAXHandler(xml.sax.handler.ContentHandler): - - metadataFields = ['title', 'link', 'description', 'language', - 'copyright', 'managingEditor', 'webMaster', - 'pubDate', 'lastBuildDate', 'category', - 'generator', 'docs', 'cloud', 'ttl', - 'image', 'rating', 'textInput', 'skipHours' - 'skipDays', - ] - - itemMetadataFields = ['title', 'link', 'description', 'author', - 'category', 'comments', 'enclosure', 'guid', - 'pubDate', 'source', - ] - - def __init__(self, rss): - self.rss = rss - - def startDocument(self): - self.rss.items = [] - self.rss.metadata = {} - self.rss.metadataOrder = [] - self.collectItem = None - self.collectMetadata = None - self.chars = None - - def startElement(self, name, attrs): - if name in ('rss', 'channel'): - return - if name == 'item': - assert self.chars is None, \ - "Extra characters lying around: %r" % self.chars - self.collectItem = {} - return - if self.collectItem is None: - lookingFor = self.metadataFields - else: - lookingFor = self.itemMetadataFields - assert name in lookingFor, \ - "The tag name %r is unknown (from %r)" % (name, lookingFor) - assert not attrs, ("We can't deal with attributes at this " - "time (%s: %s)" % (name, attrs.items())) - self.collectMetadata = name - assert self.chars is None, \ - "Extra characters lying around: %r" % self.chars - self.chars = [] - - def endElement(self, name): - if name in ('rss', 'channel'): - return - if name == 'item': - assert self.collectItem, "</item> outside of <item>" - assert self.chars is None, \ - "Extra characters lying around: %r" % self.chars - self.createItem() - self.collectItem = None - return - if self.collectItem is None: - metadata = self.rss.metadata - self.rss.metadataOrder.append(name) - else: - metadata = self.collectItem - metadata.setdefault('metadataOrder', []).append(name) - assert self.collectMetadata is not None, \ - "</%s> outside of <%s>" % (name, name) - metadata[self.collectMetadata] = ''.join(self.chars) - self.chars = None - self.collectMetadata = None - - def characters(self, char): - if self.chars is None and not char.strip(): - return - assert self.chars is not None, "Unexpected chars: %r" % char - self.chars.append(char) - - def createItem(self): - self.collectItem = dict([(str(v[0]), v[1]) - for v in - self.collectItem.items()]) - item = RSSItem(**self.collectItem) - self.rss.items.append(item) - -rssAttributes = [ - 'title', - 'link', - 'description', - 'language', - 'copyright', - 'managingEditor', - 'webMaster', - 'pubDate', - 'lastBuildDate', - 'category', - 'generator', - 'docs', - 'cloud', - 'ttl', - 'image', - 'rating', - 'textInput', - 'skipHours', - 'skipdays', - ] diff --git a/lib/securehidden.py b/lib/securehidden.py deleted file mode 100644 index 47c1745..0000000 --- a/lib/securehidden.py +++ /dev/null @@ -1,87 +0,0 @@ -import sha -import urllib -import os -import threading -import random -import time - -class SignatureError(Exception): - pass - -class SecureSigner(object): - - """ - This will sign fields, and pack the value and signature into a - single hidden field. You may also make the field expirable. - """ - - def __init__(self, secretFilename): - self.secretFilename = secretFilename - self._secret = None - self.secretLock = threading.Lock() - - def secret(self): - if self._secret is not None: - return self._secret - self.secretLock.acquire() - try: - if self._secret is None: - self._generateSecret() - finally: - self.secretLock.release() - return self._secret - - def _generateSecret(self): - if not os.path.exists(self.secretFilename): - self._secret = '' - for i in range(4): - self._secret += hex(random.randrange(0xffff))[2:] - f = open(self.secretFilename, 'w') - f.write(self._secret) - f.write('\n') - f.close() - else: - f = open(self.secretFilename) - self._secret = f.read().strip() - f.close() - - def secureValue(self, value, timeout=None): - """ - The value, signed and potentially with a timeout. The timeout - should be in seconds, e.g., 3600 for an hour. - """ - pieces = [] - if timeout: - expire = str(int(time.time()) + timeout) - else: - expire = '0' - digest = sha.new() - digest.update(value) - digest.update(expire) - digest.update(self.secret()) - pieces.append(expire) - pieces.append(digest.hexdigest()) - pieces.append(value) - return ' '.join(pieces) - - def parseSecure(self, input): - """ - Take the value as produced by .secureValue(), unpack it, confirm - the signature and expiration, and return the original value. If - something has happened -- the signature doesn't match, or it has - expired -- a SignatureError will be raised. - """ - expire, signature, value = input.split(' ', 2) - digest = sha.new() - digest.update(value) - digest.update(expire) - digest.update(self.secret()) - if not digest.hexdigest() == signature: - raise SignatureError, "Bad signature: %r" % signature - expire = int(expire) - if expire and time.time() > expire: - raise SignatureError( - "Signature expired on %s (now is %s)" - % (expire, time.time())) - return value - diff --git a/lib/sshclient.py b/lib/sshclient.py deleted file mode 100644 index 466d71a..0000000 --- a/lib/sshclient.py +++ /dev/null @@ -1,49 +0,0 @@ -import os - -__all__ = ['upload_files'] - -def upload_files(hostname, fileList, port=None, username=None, - password=None): - assert not password, "SSH does not support passwords at this time" - if ':' in hostname: - hostname, port = hostname.split(':', 1) - port = int(port) - if username == os.environ['USER']: - username = None - allDirs = {} - for source, dest in fileList: - allDirs[os.path.dirname(dest).replace('"', '').replace("'", '')] = 1 - allDirs = allDirs.keys() - if allDirs: - cmd = 'ssh' - cmd += ' ' + hostname - if port and port != 22: - cmd += ' -p %i' % port - if username: - cmd += ' -l %s' % username - cmd += ' ' - cmd += '"mkdir -p %s"' % ' '.join(["'%s'" % d for d in allDirs]) - stdin, stdouterr = os.popen4(cmd) - output = stdouterr.read() - if output: - print 'SSH output (%r): %s' % (cmd, output) - - cmd = 'sftp' - cmd += ' -b -' - if username: - cmd += ' %s@%s' % (username, hostname) - else: - cmd += ' ' + hostname - stdin, stdout, stderr = os.popen3(cmd) - for source, dest in fileList: - stdin.write('put %s %s\n' % (source, dest)) - stdin.close() - output = stderr.read() - output += stdout.read() - if output: - print 'SFTP output (%r): %s' % (cmd, output) - -if __name__ == '__main__': - import sys - upload_files(sys.argv[1], [p.split(':', 1) for p in sys.argv[2:]]) - diff --git a/lib/test.conf b/lib/test.conf deleted file mode 100644 index a18cad1..0000000 --- a/lib/test.conf +++ /dev/null @@ -1,17 +0,0 @@ -# Test configuration file - -basepath = ./test_data - -testval = A -testval2 = None - -[vhost(test.domain)] - -testval = B -testval2 = A -static href = http://static.domain/ - -[vhost(test.virtual)] - -canonical = test.domain -testval = C diff --git a/lib/test_convert_html.py b/lib/test_convert_html.py deleted file mode 100644 index 91ad6a8..0000000 --- a/lib/test_convert_html.py +++ /dev/null @@ -1,27 +0,0 @@ -from common import htmlEncode -import convert_html -import test_fixture - -dummy_page = test_fixture.Dummy( - method_sourceLinkForMimeType='source_link', - width=10, height=20) - - -def test_text(): - for v in ['a', '1 < 2', 'me&you']: - converted = convert_html.convert_text(dummy_page, - v, 'text/plain') - assert htmlEncode(v) in converted - assert 'source_link' in converted - -def test_application(): - converted = convert_html.convert_application(dummy_page, None, 'text/whatever') - assert 'source_link' in converted - -def test_convert_image(): - converted = convert_html.convert_image(dummy_page, None, 'image/whatever') - assert 'width="10"' in converted - assert 'height="20"' in converted - assert 'source_link' in converted - -# @@: should test convert_python diff --git a/lib/test_convert_rest.py b/lib/test_convert_rest.py deleted file mode 100644 index ec7ebbd..0000000 --- a/lib/test_convert_rest.py +++ /dev/null @@ -1,20 +0,0 @@ -from convert_rest import convert_rest - -tests = [(""" -Test ----- - -This is a test_ -""", - ''' -<div class="document" id="test"> -<h1 class="title">Test</h1> - -<p>This is a <a class="reference external" href="test">test</a></p> -</div> -''')] - -def test_conversions(): - for rest, html in tests: - assert convert_rest(None, rest) == html - diff --git a/lib/test_filequeue.py b/lib/test_filequeue.py deleted file mode 100644 index f0456c8..0000000 --- a/lib/test_filequeue.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import py -import filequeue - - -def runner(func): - if os.path.exists('tmp.queue'): - os.unlink('tmp.queue') - q = filequeue.FileQueue('tmp.queue') - func(q) - os.unlink('tmp.queue') - - -def test_queue(q): - assert q.pop() is None - assert q.popall() == [] - assert q.popmany(10) == [] - q.append('1') - assert q.pop() == '1' - assert q.pop() is None - q.append('2') - assert q.popall() == ['2'] - q.extend(['1', '2', '3']) - assert q.popmany(2) == ['1', '2'] - q.set('5') - q.set('5') - q.set('3') - assert q.popall() == ['3', '5'] - py.test.raises(ValueError, q.append, 1) - py.test.raises(ValueError, q.append, 'test\n') - -test_queue = runner(test_queue) diff --git a/lib/test_fixture.py b/lib/test_fixture.py deleted file mode 100644 index 3b5d38e..0000000 --- a/lib/test_fixture.py +++ /dev/null @@ -1,119 +0,0 @@ -import sys -if '.' not in sys.path: - sys.path.append('.') -import types -from cStringIO import StringIO - -import doctest - -from py.test.collect import Collector, Item - - -class Dummy(object): - - def __init__(self, **kw): - for name, value in kw.items(): - if name.startswith('method_'): - name = name[len('method_'):] - value = DummyMethod(value) - setattr(self, name, value) - - -class DummyMethod(object): - - def __init__(self, return_value): - self.return_value = return_value - - def __call__(self, *args, **kw): - return self.return_value - - -class DoctestCollector(Collector): - - def __init__(self, extpy_or_module): - if isinstance(extpy_or_module, types.ModuleType): - self.module = extpy_or_module - self.extpy = None - else: - self.extpy = extpy_or_module - self.module = self.extpy.getpymodule() - - def __call__(self, extpy): - # we throw it away, because this has been set up to explicitly - # check another module; maybe this isn't clean - if self.extpy is None: - self.extpy = extpy - return self - - def __iter__(self): - finder = doctest.DocTestFinder() - tests = finder.find(self.module) - for t in tests: - yield DoctestItem(self.extpy, t) - - -class DoctestItem(Item): - - def __init__(self, extpy, doctestitem, *args): - self.extpy = extpy - self.doctestitem = doctestitem - self.name = extpy.basename - self.args = args - - def execute(self, driver): - runner = doctest.DocTestRunner() - teardown = None - #driver.setup_path(self.extpy) - #target, teardown = driver.setup_method(self.extpy) - try: - (failed, tried), run_output = capture_stdout(runner.run, self.doctestitem) - if failed: - raise self.Failed(msg=run_output, tbindex=-2) - finally: - if teardown: - teardown(target) - - -def capture_stdout(func, *args, **kw): - newstdout = StringIO() - oldstdout = sys.stdout - sys.stdout = newstdout - try: - result = func(*args, **kw) - finally: - sys.stdout = oldstdout - return result, newstdout.getvalue() - - -def assert_error(func, *args, **kw): - kw.setdefault('error', Exception) - kw.setdefault('text_re', None) - error = kw.pop('error') - text_re = kw.pop('text_re') - if text_re and isinstance(text_re, str): - import re - real_text_re = re.compile(text_re, re.S) - else: - real_text_re = text_re - try: - value = func(*args, **kw) - except error, e: - if real_text_re and not real_text_re.search(str(e)): - assert False, ( - "Exception did not match pattern; exception:\n %r;\n" - "pattern:\n %r" - % (str(e), text_re)) - except Exception, e: - assert False, ( - "Exception type %s should have been raised; got %s instead (%s)" - % (error, e.__class__, e)) - else: - assert False, ( - "Exception was expected, instead successfully returned %r" - % (value)) - - -def sorted(l): - l = list(l) - l.sort() - return l diff --git a/lib/test_html_abstracts.py b/lib/test_html_abstracts.py deleted file mode 100644 index 8863968..0000000 --- a/lib/test_html_abstracts.py +++ /dev/null @@ -1,35 +0,0 @@ -from html_abstracts import * - -tests = [ - (""" - This is a test - blah blah - <p>and more</p> - - <div class="abstract">this is the abstract</div> - and more - """, "this is the abstract"), - (""" - Another test. - - And some more. - - blah blah <p class="abstract">here -there</p>stop -""", "here\nthere"), - (""" - <p> - The first para. - </p> - - <p> - The second para - </p> - """, "\n The first para.\n "), - ("""\nThe whole doc...\n\n""", - """\nThe whole doc...\n\n"""), - ] - -def test_match(): - for input, output in tests: - assert find_abstract(input).strip() == output.strip() diff --git a/lib/test_htmldiff.py b/lib/test_htmldiff.py deleted file mode 100644 index 5250fdb..0000000 --- a/lib/test_htmldiff.py +++ /dev/null @@ -1,341 +0,0 @@ -""" -datatest.util.htmldiff - -Compares two HTML strings and provides intelligent feedback on how -they differ. - -htmlcompare() and htmlcompareError() are the most useful functions. -They compare source HTML to a 'pattern' HTML (which can contain -wildcards). The two strings are compared for semantic equivalence, -not exact equality -- whitespace in text is normalized, tag case is -normalized, and attribute order is ignored. - -These wildcards are allowed: - -``*`` *alone* in text will match any text or tags. (Note, you cannot -use ``*text`` to match ``some text``) It will consume the source until -a matching tag is found (even if by consuming more it could have -caused a match). - -The ``<any>`` tag will match any tag (start or end), or any single -piece of text. - -A tag with the any attribute (like ``<a any>``) will match any -attributes (though the tag itself must match). If you include the any -attribute and other attributes, those other attributes must match (but -extra attributes will be ignored). - -An attribute like ``any-href`` will allow the source to include any -value for ``href``, or to exclude it altogether. ``href=\"*\"`` will -match any value for the ``href`` attribute, but will require that the -attribute exists in the source. - -Usage: - - >>> from datatest.util.htmldiff import htmlcompareError - >>> from datatest.util.doctestprinter import printer - >>> printer(htmlcompareError('<a href=\"something\">', '<a href=\"*\">')) - None - >>> printer(htmlcompareError('<a href=\"something\">a tag</a>', '<a href=\"something\">different tag</a>')) - Error: 'a tag' does not match 'different tag' - ============================================================ - Source(1:20): - . - <a href=\"something\">((|))a tag</a> - ------------------------------------------------------------ - Pattern(1:20): - . - <a href=\"something\">((|))different tag</a> - -""" - -from HTMLParser import HTMLParser, HTMLParseError -import cgi, re -from cStringIO import StringIO -import os - -def simplify(html): - """ - Tokenizes an HTML string, using HTMLSimplifier. - """ - s = HTMLSimplifier() - try: - s.feed(html) - except HTMLParseError, e: - raise HTMLParseError, "%s in %s" % (e, html) - s.close() - return s.data() - -class HTMLSimplifier(HTMLParser): - - """ - HTML tokenizer; tokenizes HTML into a stream of text and tags. - Does not parse the structure of the text, just the stream of - symbols. - - The output is a list of three-tuples, where the first item is the - tag name (or 'comment' for comments, or 'text' for text), the - second item is the data -- a dictionary of attributes, None for - end tags, and the text for comments and text. The third item is - the position in the text. - - Text is normalized -- whitespace is collapsed into a single space, - and leading and trailing whitespace is removed. - """ - - def reset(self): - self.tags = [] - HTMLParser.reset(self) - - def handle_starttag(self, tag, attrs): - self.tags.append((tag, dict(attrs), self.getpos())) - - def handle_endtag(self, tag): - self.tags.append((tag, None, self.getpos())) - - def handle_comment(self, data): - # @@: For some reason, comments are being treated like text. - self.tags.append(('comment', data, self.getpos())) - - def handle_data(self, data): - if self.tags and self.tags[-1][0] == 'text': - self.tags[-1][1] += data - else: - self.tags.append(['text', data, self.getpos()]) - - def data(self): - result = [] - for tag in self.tags: - if tag[0] == 'text': - text = self.normalizeText(tag[1]) - if text: - result.append(('text', text, tag[2])) - else: - result.append(tag) - return result - - def normalizeText(self, html): - html = re.sub(r'[ \n\t\r]+', ' ', html) - return html.strip() - -def htmlcompare(source, pattern, wildcard='*'): - """ - Compare source and pattern HTML. - - Both source and pattern are expected to be strings. Both will be - simplified (using `simplify`) and run through `compareTags`. If - the two match None is returned (because matches are boring), - otherwise (description, sourcePos, patternPos) is returned, where - description describes, in English, the problem, and sourcePos and - patternPos point to the last place where the two strings matched. - """ - source = simplify(source) - pattern = simplify(pattern) - v = ([], []) - try: - while source and pattern: - v[0].append(source) - v[0].append(pattern) - source, pattern = compareTags(source, pattern, - wildcard=wildcard) - except MismatchError, e: - return (e.mismatch, e.sourcePos, e.patternPos) - return None - -def htmlcompareError(source, pattern, wildcard='*'): - """ - Compares the source and pattern, returning None if they match, or a - helpful (string) error message if they do not. - - The error message describes the difference, and displays the - source and pattern text with '((|))' where they start to differ. - """ - val = htmlcompare(source, pattern, wildcard=wildcard) - if not val: - return None - msg, spos, dpos = val - out = StringIO() - out.write("Error: %s\n" % msg) - out.write('='*60+'\n') - if spos: - out.write("Source(%i:%i):\n" % spos) - before, after = cutString(source, spos) - out.write(before) - out.write(errorMarker()) - out.write(after.rstrip()) - out.write('\n') - else: - out.write('Source:\n') - out.write(source + "\n") - out.write('-' * 60 + "\n") - if dpos: - out.write('Pattern(%i:%i):\n' % dpos) - before, after = cutString(pattern, dpos) - out.write(before) - out.write(errorMarker()) - out.write(after.rstrip()) - out.write('\n') - else: - out.write('Pattern:\n') - out.write(pattern + "\n") - return out.getvalue() - - -def compareTags(source, pattern, wildcard='*'): - """ - Compares two lists of tags, as produced by - simplify/HTMLSimplifier. - - Consumes the first tag, and returns the - remaining tags. It may consume more than one tag if wildcards are - used. Raises MisMatchError if the source and pattern don't match. - - pattern can have wildcards. A portion of text that consists of only - '*' will match any text (but globbing is not supported, i.e., - 'item*' won't match 'items', or even 'item'). A wildcard will - match multiple pieces of text or tags until it finds a match for - the next token. So '*<b>stuff</b>' will match 'some <i>great</i> - <b>stuff</b>', but *won't* match 'some <b>bad</b> <b>stuff</b>' - (because the first <b> will be matched, but 'bad' != 'stuff'). - - You'll usually use htmlcompare() - """ - if not source and not pattern: - return source, pattern - if not source: - raise MismatchError("short source", None, pattern[0][2]) - if not pattern: - raise MismatchError("source too long", source[0][2], None) - if pattern[0][0] == 'text' and pattern[0][1] == wildcard: - pattern = pattern[1:] - while 1: - if not pattern: - return [], [] - if not source: - return compareTags(source, pattern, wildcard=wildcard) - if tagMatch(source[0], pattern[0], wildcard=wildcard): - source = source[1:] - pattern = pattern[1:] - break - source = source[1:] - return source, pattern - else: - if tagMatch(source[0], pattern[0], wildcard=wildcard): - return (source[1:], pattern[1:]) - else: - raise MismatchError("%s does not match %s" % - (_shorten(formatData(source[0])), - _shorten(formatData(pattern[0]))), - source[0][2], pattern[0][2]) - -def _shorten(v, length=40): - v = repr(v) - if len(v) > length: - v = v[:15] + "..." + v[-15:] - return v - -def tagMatch(source, pattern, wildcard='*'): - """ - Matches two 'tags', which can be an HTML opening tag, end tag, - comment, or text. - - pattern can have special signifiers to relax the matching. <any> - will match any single tag or piece of text. If you include an any - attribute, like <a any=\"\">, then any extra attributes in the - source will be ignored for the comparison. If you use * in an - attribute, like <a href=\"*\">, then any text in that attribute - will be matched (but the attribute must exist). If <a - any-href=\"\"> is found, then href may exist or not (but if it - exists, then it must match). - """ - if pattern[0] != 'any' and source[0] != pattern[0]: - # Tags don't match - return 0 - if pattern[0] in ['comment', 'text']: - if pattern[1].strip() == wildcard: - return 1 - return pattern[1] == source[1] - if pattern[1] is None: - if source[1] is None: - # Both are end tags - return 1 - else: - return 0 - elif source[1] is None: - return 0 - patternd = pattern[1].copy() - sourced = source[1].copy() - if patternd.has_key('any'): - del patternd['any'] - any = 1 - else: - any = 0 - for key, value in patternd.items(): - if key.startswith('any-'): - if sourced.has_key(key[4:]): - del sourced[key[4:]] - del patternd[key] - elif value == wildcard: - del patternd[key] - if not sourced.has_key(key): - return 0 - del sourced[key] - elif any: - if sourced.has_key(key) and sourced[key] != value: - return 0 - else: - if not sourced.has_key(key) or sourced[key] != value: - return 0 - else: - del sourced[key] - if not any and sourced: - return 0 - return 1 - -class MismatchError(AssertionError): - """ - The error produced when HTML doesn't match. - """ - def __init__(self, mismatch, sourcePos, patternPos, *args): - self.sourcePos = sourcePos - self.patternPos = patternPos - self.mismatch = mismatch - Exception.__init__(self, *args) - -def formatData(data): - """ - Turns a token (as produced by simplify or HTMLSimplifier) back - into HTML. - """ - if data[0] == 'text': - return cgi.escape(data[1]) - elif data[1] is None: - return '</%s>' % data[0] - elif data[0] == 'comment': - return '<!--%s-->' % data[1] - else: - attrs = data[1].items() - attrs.sort() - return '<%s%s>' % \ - (data[0], ''.join([' %s="%s"' % (a, v and cgi.escape(v, 1) or '') - for a, v in attrs])) - -def cutString(s, pos): - """ - Takes a source HTML string, and given pos (as returned by - simplify/HTMLSimplifier, which is a (line, column) tuple) returns - (before, after) split around the position. - """ - line, offset = pos - line -= 1 - lines = s.split('\n') - before = '\n'.join(lines[:line]) + '\n' + lines[line][:offset] - after = lines[line][offset:] + '\n' + '\n'.join(lines[line+1:]) - return before, after - -def errorMarker(): - basic = '((|))' - if os.environ.get('TERM', '') in ('xterm', 'rxvt', 'vt100'): - return '\x1b[41;37m%s\x1b[0m' % basic - else: - return basic diff --git a/lib/test_rssobject.py b/lib/test_rssobject.py deleted file mode 100644 index b741dac..0000000 --- a/lib/test_rssobject.py +++ /dev/null @@ -1,199 +0,0 @@ -import rssobject -import test_htmldiff - - -def assertXMLEqual(source, pattern): - result = test_htmldiff.htmlcompareError(source, pattern) - if result: - assert 0, result - -params_testCreates = [] - -def do_testCreates(name, kw, itemKW, output): - items = [rssobject.RSSItem(**kw) for kw in itemKW] - rss = rssobject.RSS(metadata=kw, items=items) - assertXMLEqual(str(rss), output) - -params_testRoundTrip = [] - -def test_creates(): - for params in params_testCreates: - do_testCreates(*params) - -def do_testRoundTrip(name, input): - rss = rssobject.RSS(text=input) - assertXMLEqual(str(rss), input) - -def test_roundtrip(): - for params in params_testRoundTrip: - do_testRoundTrip(*params) - -def addCreate(name, output, **kw): - itemKW = kw['items'] - del kw['items'] - params_testCreates.append((name, kw, itemKW, output)) - -def addRoundTrip(name, input): - params_testRoundTrip.append((name, input)) - -addCreate( - 'simple', - '''<?xml version="1.0"> - <rss version="2.0"> - <channel> - <lastBuildDate>now</lastBuildDate> - <generator>*</generator> - <pubDate>now</pubDate> - </channel> - </rss>''', - lastBuildDate='now', - pubDate='now', - items=[]) - -addCreate( - 'complex', - '''<?xml version="1.0"> - <rss version="2.0"> - <channel> - <lastBuildDate>*</lastBuildDate> - <pubDate>Tuesday</pubDate> - <generator>rssobject.py v0.1</generator> - <author>ianb@colorstudy.com</author> - <title>second - http://testblog.com/item2.html - - http://testblog.com/item1.html - http://testblog.com/item1.html - My first post! - * - First post! - - - http://testblog.com/item2.html - http://testblog.com/item2.html - ianb@colorstudy.com - Tuesday - second - - - - ''', - title='test', - link='http://testblog.com', - description='A cool test blog', - ttl=20, - author='ianb@colorstudy.com', - items=[{'title': 'First post!', - 'link': 'http://testblog.com/item1.html', - 'description': 'My first post!'}, - {'title': 'second', - 'guid': 'http://testblog.com/item2.html', - 'pubDate': 'Tuesday', - 'author': 'ianb@colorstudy.com'}], - ) - -addRoundTrip( - 'rssExample', - """ - - - Scripting News - http://www.scripting.com/ - A weblog about scripting and stuff like that. - en-us - - Copyright 1997-2002 Dave Winer - Mon, 30 Sep 2002 11:00:00 GMT - http://backend.userland.com/rss - - Radio UserLand v8.0.5 - dave@userland.com - dave@userland.com - 40 - Fri 09 Apr 2004 12:00:00 GMT - - - "rssflowersalignright"With any luck we should have one or two more days of namespaces stuff here on Scripting News. It feels like it's winding down. Later in the week I'm going to a <a href="http://harvardbusinessonline.hbsp.harvard.edu/b02/en/conferences/conf_detail.jhtml?id=s775stg&pid=144XCF">conference</a> put on by the Harvard Business School. So that should change the topic a bit. The following week I'm off to Colorado for the <a href="http://www.digitalidworld.com/conference/2002/index.php">Digital ID World</a> conference. We had to go through namespaces, and it turns out that weblogs are a great way to work around mail lists that are clogged with <a href="http://www.userland.com/whatIsStopEnergy">stop energy</a>. I think we solved the problem, have reached a consensus, and will be ready to move forward shortly. - - Mon, 30 Sep 2002 01:56:02 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#When:6:56:02PM - http://scriptingnews.userland.com/backissues/2002/09/29#When:6:56:02PM - - - Joshua Allen: <a href="http://www.netcrucible.com/blog/2002/09/29.html#a243">Who loves namespaces?</a> - - Sun, 29 Sep 2002 19:59:01 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#When:12:59:01PM - http://scriptingnews.userland.com/backissues/2002/09/29#When:12:59:01PM - - - <a href="http://www.docuverse.com/blog/donpark/2002/09/29.html#a68">Don Park</a>: "It is too easy for engineer to anticipate too much and XML Namespace is a frequent host of over-anticipation." - - Mon, 30 Sep 2002 01:52:02 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#When:6:52:02PM - http://scriptingnews.userland.com/backissues/2002/09/29#When:6:52:02PM - - - <a href="http://scriptingnews.userland.com/stories/storyReader$1768">Three Sunday Morning Options</a>. "I just got off the phone with Tim Bray, who graciously returned my call on a Sunday morning while he was making breakfast for his kids." We talked about three options for namespaces in RSS 2.0, and I think I now have the tradeoffs well outlined, and ready for other developers to review. If there is now a consensus, I think we can easily move forward. - - Sun, 29 Sep 2002 17:05:20 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#When:10:05:20AM - http://scriptingnews.userland.com/backissues/2002/09/29#When:10:05:20AM - - - <a href="http://blog.mediacooperative.com/mt-comments.cgi?entry_id=1435">Mark Pilgrim</a> weighs in behind option 1 on a Ben Hammersley thread. On the RSS2-Support list, Phil Ringnalda lists a set of <a href="http://groups.yahoo.com/group/RSS2-Support/message/54">proposals</a>, the first is equivalent to option 1. - - Sun, 29 Sep 2002 19:09:28 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#When:12:09:28PM - http://scriptingnews.userland.com/backissues/2002/09/29#When:12:09:28PM - - - <a href="http://effbot.org/zone/effnews-4.htm">Fredrik Lundh breaks</a> through, following Simon Fell's lead, now his Python aggregator works with Scripting News <a href="http://www.scripting.com/rss.xml">in</a> RSS 2.0. BTW, the spec is imperfect in regards to namespaces. We anticipated a 2.0.1 and 2.0.2 in the Roadmap for exactly this purpose. Thanks for your help, as usual, Fredrik. - - Sun, 29 Sep 2002 15:01:02 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#When:8:01:02AM - http://scriptingnews.userland.com/backissues/2002/09/29#When:8:01:02AM - - - Law and Order - http://scriptingnews.userland.com/backissues/2002/09/29#lawAndOrder - - - <p><a href="http://www.nbc.com/Law_&_Order/index.html"><img src="http://radio.weblogs.com/0001015/images/2002/09/29/lenny.gif" width="45" height="53" border="0" align="right" hspace="15" vspace="5" alt="A picture named lenny.gif"></a>A great line in a recent Law and Order. Lenny Briscoe, played by Jerry Orbach, is interrogating a suspect. The suspect tells a story and reaches a point where no one believes him, not even the suspect himself. Lenny says: "Now there's five minutes of my life that's lost forever." </p> - - - Sun, 29 Sep 2002 23:48:33 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#lawAndOrder - - - Rule 1 - http://scriptingnews.userland.com/backissues/2002/09/29#rule1 - - - <p>In the discussions over namespaces in RSS 2.0, one thing I hear a lot of, that is just plain wrong, is that when you move up by a major version number, breakage is expected and is okay. In the world I come from it is, emphatically, <i>not okay.</i> We spend huge resources to make sure that files, scripts and apps built in version N work in version N+1 without modification. Even the smallest change in the core engine can break apps. It's just not acceptable. When we make changes we have to be sure there's no breakage. I don't know where these other people come from, or if they make software that anyone uses, but the users I know don't stand for that. As we expose the tradeoffs it becomes clear that <i>that's the issue here.</i> We are not in Year Zero. There are users. Breaking them is not an option. A conclusion to lift the confusion: Version 0.91 and 0.92 files are valid 2.0 files. This is where we started, what seems like years ago.</p> - - <p>BTW, you can ask anyone who's worked for me in a technical job to explain rules 1 and 1b. (I'll clue you in. Rule 1 is "No Breakage" and Rule 1b is "Don't Break Dave.")</p> - - Sun, 29 Sep 2002 17:24:20 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#rule1 - - - - Really early morning no-coffee notes - http://scriptingnews.userland.com/backissues/2002/09/29#reallyEarlyMorningNocoffeeNotes - - <p>One of the lessons I've learned in 47.4 years: When someone accuses you of a <a href="http://www.dictionary.com/search?q=deceit">deceit</a>, there's a very good chance the accuser practices that form of deceit, and a reasonable chance that he or she is doing it as they point the finger. </p> - - <p><a href="http://www.docuverse.com/blog/donpark/2002/09/28.html#a66">Don Park</a>: "He poured a barrel full of pig urine all over the Korean Congress because he was pissed off about all the dirty politics going on."</p> - <p><a href="http://davenet.userland.com/1995/01/04/demoingsoftwareforfunprofi">1/4/95</a>: "By the way, the person with the big problem is probably a competitor."</p> - - <p>I've had a fair amount of experience in the last few years with what you might call standards work. XML-RPC, SOAP, RSS, OPML. Each has been different from the others. In all this work, the most positive experience was XML-RPC, and not just because of the technical excellence of the people involved. In the end, what matters more to me is <a href="http://www.dictionary.com/search?q=collegiality">collegiality</a>. Working together, person to person, for the sheer pleasure of it, is even more satisfying than a good technical result. Now, getting both is the best, and while XML-RPC is not perfect, it's pretty good. I also believe that if you have collegiality, technical excellence follows as a natural outcome.</p> - <p>One more bit of philosophy. At my checkup earlier this week, one of the things my cardiologist asked was if I was experiencing any kind of intellectual dysfunction. In other words, did I lose any of my sharpness as a result of the surgery in June. I told him yes I had and thanked him for asking. In an amazing bit of synchronicity, the next day John Robb <a href="http://jrobb.userland.com/2002/09/25.html#a2598">located</a> an article in New Scientist that said that scientists had found a way to prevent this from happening. I hadn't talked with John about my experience or the question the doctor asked. Yesterday I was telling the story to my friend Dave Jacobs. He said it's not a problem because I always had excess capacity in that area. Exactly right Big Dave and thanks for the vote of confidence.</p> - - - Sun, 29 Sep 2002 11:13:10 GMT - http://scriptingnews.userland.com/backissues/2002/09/29#reallyEarlyMorningNocoffeeNotes - - - -""") diff --git a/lib/test_wiki.py b/lib/test_wiki.py deleted file mode 100644 index b8a82a2..0000000 --- a/lib/test_wiki.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import shutil - -import wiki - -from wikiconfig import WikiConfig - - -def make_conf(): - conf = WikiConfig() - conf.load(os.path.join(os.path.dirname(__file__), "test.conf")) - return conf - - -def setup_module(module): - shutil.rmtree(os.path.join(os.path.dirname(__file__), 'test_data'), True) - os.mkdir(os.path.join(os.path.dirname(__file__), 'test_data')) - - -def test_global(): - conf = make_conf() - g = wiki.GlobalWiki(conf) - s = g.site('test.domain') - assert s.globalWiki is g - assert s.canonical is None - assert g.config['testval'] == 'A' - assert g.config['testval2'] == 'None' - assert s.config['testval'] == 'B' - assert s.config['testval2'] == 'A' - assert s.basepath.endswith(os.sep+'test.domain') - assert s.basepath.startswith(g.root) - s2 = g.site('test.virtual') - assert s2.globalWiki is g - assert s2.canonical is s - assert s2.config['testval'] == 'C' - assert s2.config['testval2'] == 'A' - - -def test_wiki(): - conf = make_conf() - w = wiki.GlobalWiki(conf).site('test.domain') - assert w.filenameForName('test').endswith('test.txt') - assert not w.exists('test') - assert w.search('testword') == [] - assert w.searchTitles('test') == [] - assert w.searchNames('test') == [] - dist = set(os.path.splitext(p)[0] for p in os.listdir('distpages')) - assert dist.issuperset(p.name for p in w.recentPages()) - assert dist.issuperset(p.name for p in w.orphanPages()) - assert dist.issuperset(p.name for p in w.allPages()) - assert w.linkTo('test') == 'http://test.domain/test.html' - assert w.linkTo('test.link') == 'http://test.domain/test.link' - assert w.linkTo('test', source=True) == 'http://test.domain/test.txt' - assert w.staticLinkTo('test') == 'http://static.domain/test.html' - assert w.staticLinkTo('test.link') == 'http://static.domain/test.link' - assert w.canonicalDomain == w.domain - w.rebuildHTML() - w.rebuildStatic() - w.recreateThumbnails() - w.rebuildIndex() - w.checkDistributionFiles() - rss = str(w.syndicateRecentChanges) - assert rss.startswith('') != -1 - for mime_type, ext in [('text/plain', '.txt'), - ('image/jpeg', '.jpg'), - ('text/x-restructured-text', '.txt'), - ('text/html', '.html'), - ]: - assert w.extensionForMimeType(mime_type) == ext diff --git a/lib/test_wikiindex.py b/lib/test_wikiindex.py deleted file mode 100644 index 7b7d1ac..0000000 --- a/lib/test_wikiindex.py +++ /dev/null @@ -1,79 +0,0 @@ -import wikiindex -import os -import py - -def set_match(s1, s2): - """ - Treats lists like dictionaries when testing equality - """ - if not len(s1) == len(s2): - return False - for item in s1: - if item not in s2: - return False - return True - -def recur_delete(dir): - if os.path.isdir(dir): - for fn in os.listdir(dir): - recur_delete(os.path.join(dir, fn)) - os.rmdir(dir) - else: - os.unlink(dir) - -def runner(func): - global base_dir - base_dir = os.path.join(os.path.dirname(__file__), 'tmp') - if os.path.exists(base_dir): - recur_delete(base_dir) - if not os.path.exists(base_dir): - os.makedirs(base_dir) - index = wikiindex.WikiIndex(base_dir) - func(index) - wikiindex._closeHandler() - -def test_links(index): - index.setLinks('test', ['test2', 'test3']) - assert set_match(index.forwardLinks('test'), ['test2', 'test3']) - assert set_match(index.backlinks('test'), []) - assert set_match(index.backlinks('test2'), ['test']) - index.setLinks('test2', ['test', 'blah']) - assert set_match(index.forwardLinks('test2'), ['test', 'blah']) - assert set_match(index.backlinks('test2'), ['test']) - assert set_match(index.backlinks('test'), ['test2']) - assert set_match(index.forwardLinks('test'), ['test2', 'test3']) - assert set_match(index.forwardLinks('test'), ['test2', 'test3']) - index.setLinks('test', ['test3']) - assert set_match(index.forwardLinks('test'), ['test3']) - assert set_match(index.backlinks('test2'), []) - py.test.raises(Exception, index.setLinks, 'test', 'blah') - -test_links = runner(test_links) - -def test_connections(index): - index.setConnections('test', [('test2', 'comment'), ('test3', 'attach')]) - assert set_match(index.connections('test'), - [('test2', 'comment'), ('test3', 'attach')]) - assert set_match(index.backConnections('test2'), - [('test', 'comment')]) - assert set_match(index.backConnections('test3'), - [('test', 'attach')]) - assert set_match(index.connections('test2'), []) - assert set_match(index.backConnections('test'), []) - index.setConnections('test2', [('test', 'attach')]) - assert set_match(index.connections('test2'), [('test', 'attach')]) - assert set_match(index.connections('test'), - [('test2', 'comment'), ('test3', 'attach')]) - assert set_match(index.backConnections('test3'), - [('test', 'attach')]) - index.setConnections('test', [('test3', 'attach')]) - assert set_match(index.backConnections('test3'), - [('test', 'attach')]) - assert set_match(index.backConnections('test2'), []) - assert set_match(index.connections('test'), [('test3', 'attach')]) - -def teardown_module(mod): - wikiindex._closeHandler() - recur_delete(base_dir) - -test_connections = runner(test_connections) diff --git a/lib/test_wikipage.py b/lib/test_wikipage.py deleted file mode 100644 index dc6285a..0000000 --- a/lib/test_wikipage.py +++ /dev/null @@ -1,122 +0,0 @@ -import test_wiki -import os -import wiki -import rfc822persist -from test_fixture import DoctestCollector -import re - -setup_module = test_wiki.setup_module - -def test_page(): - conf = test_wiki.make_conf() - w = wiki.GlobalWiki(conf).site('test.domain') - page = w.page('test1') - assert not page.exists() - assert not page.readOnly - assert page.title == 'Test1' - assert page.text == '' - page.title = 'Test 1' - assert page.title == 'Test 1' - # Haven't saved yet: - assert not page.exists() - assert page.modifiedDate is None - page.text = 'A test\n' - assert page.text == 'A test\n' - assert not page.exists() - page.save() - assert page.modifiedDate is not None - assert page.text == 'A test\n' - assert 'A test' in page.html - assert page.exists() - assert page.creationDate == page.modifiedDate - assert not page.hasParseErrors - assert page.mimeType == 'text/x-restructured-text' - assert page.width is None - assert page.height is None - assert page.comments == '' - assert not page.originalFilename - assert not page.hidden - assert not page.distributionOriginal - assert page.basePath == os.path.join(w.basepath, 'test1') - assert page.archiveBasePath == os.path.join(w.basepath, 'archive', 'test1') - # None = current version - assert page.version is None - assert page.link == 'http://test.domain/test1.html' - assert page.sourceLink == 'http://test.domain/test1.txt' - assert page.sourceLinkForMimeType('image/jpeg') == 'http://test.domain/test1.jpg' - assert page.thumbnailLinkForMimeType('image/jpeg') == 'http://test.domain/test1.thumb.jpg' - assert 'A test' in page.staticHTML - assert 'Another test' in page.preview('Another test', 'text/x-restructured-text') - assert 'Test 1' in page.summary - assert page.link in page.summary - assert page.wikiLinks() == [] - assert page.backlinks == [] - versions = page.versions() - assert len(versions) == 1 - assert versions[0].version == 1 - assert versions[0].name == page.name - assert not page.searchMatches('whatever') - assert page.searchMatches('test') - assert page.searchTitleMatches('1') - assert page.searchNameMatches('1') - assert 'A test' in page.searchSegment('test') - -def test_relations(): - conf = test_wiki.make_conf() - w = wiki.GlobalWiki(conf).site('test.domain') - page = w.page('test3') - page.text = 'whatever' - page.save() - page.text = 'link to test4_ there' - page.save() - assert len(page.versions()) == 2 - assert page.versions()[0].text != page.text - assert page.versions()[-1].text == page.text - links = page.wikiLinks() - assert len(links) == 1 - assert links[0] == 'test4' - page2 = w.page('test4') - back = page2.backlinks - assert len(back) == 1 - assert back[0].name == 'test3' - page2.connections = [(page, 'comment')] - page2.save() - p2conn = page2.connections - assert len(p2conn) == 1 - assert p2conn[0][1] == 'comment' - assert p2conn[0][0].name == 'test3' - pconn = page.backConnections - assert len(pconn) == 1 - assert pconn[0][1] == 'comment' - assert pconn[0][0].name == 'test4' - page.text = 'Link to test3' - page.mimeType = 'text/html' - page.save() - assert page2.backlinks == [] - assert page.wikiLinks() == ['test5'] - page5 = w.page('test5') - back = page5.backlinks - assert len(back) == 1 - assert back[0].name == 'test3' - print page.html - assert re.search(r']*href="[^"]*test5.html">', page.html) - assert 'class="nowiki"' in page.html - assert 'class="wiki"' not in page.html - page5.text = 'test' - page5.save() - print page.html - assert re.search(r']*href="[^"]*test5.html">', page.html) - assert 'class="nowiki"' not in page.html - assert 'class="wiki"' in page.html - -def test_error(): - conf = test_wiki.make_conf() - w = wiki.GlobalWiki(conf).site('test.domain') - page = w.page('badtest') - page.text = 'A test__!_\n\nthis is a test' - page.save() - assert page.hasParseErrors - assert 'system-messages' in page.html - assert 'Anonymous hyperlink mismatch' in page.html - -collect_doctest = DoctestCollector(rfc822persist) diff --git a/lib/user.py b/lib/user.py deleted file mode 100644 index d729fa0..0000000 --- a/lib/user.py +++ /dev/null @@ -1,60 +0,0 @@ -from LoginKit.simpleuser import SimpleUser - - -class WikiUser(SimpleUser): - - def __init__(self, manager): - SimpleUser.__init__(self, manager) - self._name = None - self._email = None - self._website = None - self._roles = [] - self._url = None - - def email(self): - return self._email - - def setEmail(self, value): - self._email = value - self.changed() - - def website(self): - return self._website - - def setWebsite(self, value): - self._website = value - self.changed() - - def roles(self): - return self._roles - - def setRoles(self, roles): - self._roles = roles - self.changed() - - def url(self): - return self._url or None - - def setURL(self, value): - self._url = url - self.changed() - - def signature(self): - if self.url(): - return '%s' % (self.url(), self.name()) - else: - return self.name() - - def simpleFields(self): - fields = SimpleUser.simpleFields(self) - fields['roles'] = ', '.join(self.roles()) - fields['url'] = self.url() - return fields - - def setSimpleFields(self, fields): - SimpleUser.setSimpleFields(self, fields) - self._roles = [ - r.strip() for r in fields.get('roles', '').split(',') - if r.strip()] - self._url = fields.get('url', '') - diff --git a/lib/wiki.py b/lib/wiki.py deleted file mode 100644 index f1e22fc..0000000 --- a/lib/wiki.py +++ /dev/null @@ -1,593 +0,0 @@ -import os -import wikipage -import rssobject -import propertymeta -import threading -import wikiindex -try: - import pooledtemplate -except ImportError: - pooledtemplate = None -import mimetypes -import filequeue -import sshclient -import ftpclient -import warnings -from common import canonicalName -import rfc822persist -try: - from WebKit.HTTPExceptions import HTTPNotFound -except ImportError: - HTTPNotFound = KeyError - -warnings.filterwarnings('ignore', - message='tempnam is a potential security risk to your program') - - -class GlobalWiki(object): - - def __init__(self, config): - """Initialize Wiki. - - root is the path to root storage location for Wiki sites. - This class will make any necessary subdirectories or other - structure. - - """ - self.config = config - self.root = os.path.join(os.path.dirname(__file__), config['basepath']) - assert os.path.exists(self.root), "Nonexistant root: %r" % self.root - self.specialNames = {} - self.cachedWikis = {} - self.cacheLock = threading.Lock() - self.allSites = {} - - def site(self, domain): - """Get site Wiki. - - Retrieves the Wiki that corresponds to this domain. - Currently by looking for a similarly named subdirectory - (creating directory if necessary). - - """ - assert '/' not in domain - # We go through all this trouble to make sure Wikis are - # unique per domain, and that they have a link to their - # canonical form if necessary, which is also unique; all - # done in a threadsafe manner. - try: - return self.cachedWikis[domain] - except KeyError: - self.cacheLock.acquire() - try: - try: - return self.cachedWikis[domain] - except KeyError: - return self._makeSite(domain) - finally: - self.cacheLock.release() - - def _makeSite(self, domain): - config = self.config.clone() - try: - local_config = self.config['vhost'][domain] - except KeyError: - raise HTTPNotFound, "The domain %s is unknown" % domain - canonical_domain = local_config.get('canonical') - if canonical_domain is not None and canonical_domain != domain: - canonical_config = self.config['vhost'][canonical_domain] - config.merge(canonical_config) - try: - canonical = self.cachedWikis[canonical_domain] - except KeyError: - canonical = self._makeSite(canonical_domain) - self.cachedWikis[canonical_domain] = canonical - else: - canonical = None - config.merge(local_config) - result = Wiki(globalWiki=self, config=config, - canonical=canonical, domain=domain) - self.cachedWikis[domain] = result - return result - - def addSpecialName(self, specialName): - """Add special Name. - - A 'special name' is a name which isn't in the Wiki, but - still exists as a page. Like 'recentchanges', which is - not a Wiki page. - - """ - self.specialNames[specialName] = None - - -class Wiki(object): - - __metaclass__ = propertymeta.MakeProperties - - def __init__(self, globalWiki, domain, config, canonical): - self.config = config - self.domain = domain - self.globalWiki = globalWiki - self.canonical = canonical - self.template = None - - if self.config.has_key('localbasepath'): - self.basepath = self.config['localbasepath'] - else: - if canonical: - domain = canonical.domain - else: - domain = self.domain - self.basepath = os.path.join(os.path.dirname(__file__), - self.config['basepath'], domain) - if not os.path.exists(self.basepath): - os.mkdir(self.basepath) - - # @@: this doesn't work well... - self.basehref = self.config.get('basehref', - 'http://%s' % self.domain) - if not self.basehref.endswith('/'): - self.basehref += '/' - - if self.config.getbool('staticpublish', False): - self.publishQueue = filequeue.FileQueue(os.path.join( - self.basepath, 'publish.queue')) - - if not canonical: - if (config.getbool('rebuildindex', False) or - not wikiindex.WikiIndex.exists(self.basepath)): - needRebuild = True - else: - needRebuild = False - self.index = wikiindex.WikiIndex(self.basepath) - if config.getbool('rebuildhtml', False): - self.rebuildHTML() - if needRebuild: - self.rebuildIndex() - if config.getbool('rebuildstatic', False): - self.rebuildStatic() - self.checkDistributionFiles() - else: - self.index = canonical.index - self._knownAtomIDs = {} - - def __repr__(self): - return '<%s>' % self.shortRepr() - - def shortRepr(self): - if not self.canonical: - return 'Wiki:%s' % self.domain - else: - return 'Wiki:%s aliases %s' % (self.domain, - self.canonical.domain) - - ############################################################ - ## Pages - ############################################################ - - def page(self, name, version=None): - """Get a page. - - Returns a page by the given name, with the given version - (None == current version). - - """ - urlName = name - name = canonicalName(name) - return wikipage.WikiPage(self, self.basepath, - name, urlName=urlName, version=version) - - def pageByAtomID(self, atomID): - if atomID in self._knownAtomIDs: - return self.page(self._knownAtomIDs[atomID]) - for page in self.allPages(): - self._knownAtomIDs[page.atomID] = page.name - if atomID in self._knownAtomIDs: - return self.page(self._knownAtomIDs[atomID]) - return None - - def filenameForName(self, filename): - return os.path.join(self.basepath, filename + '.txt') - - def exists(self, name): - """True if wiki page by name exists.""" - if self.globalWiki.specialNames.has_key(name): - return True - else: - return os.path.exists(self.filenameForName(name)) - - def search(self, text): - """Search titles and bodies of pages for ``text``. - - Returns a list of pages. - - """ - return [page for page in self.allPages() - if page.searchMatches(text)] - - def searchTitles(self, text): - """Search page titles for ``text``. - - Returns a list of pages - - """ - return [page for page in self.allPages() - if page.searchTitleMatches(text)] - - def searchNames(self, text): - return [page for page in self.allPages() - if page.searchNameMatches(text)] - - def recentPages(self): - """All pages, sorted by date modified, most recent first.""" - pages = self.allPages() - pages.sort(lambda a, b: cmp(b.modifiedDate, a.modifiedDate)) - return pages - - def recentCreated(self): - """All pages, sorted by date created, most recent first.""" - pages = self.allPages() - pages.sort(lambda a, b: cmp(b.creationDate, a.creationDate)) - return pages - - def orphanPages(self): - """All pages which are not linked to by another page.""" - orphans = [] - for page in self.allPages(): - if not page.backlinks: - orphans.append(page) - orphans.sort(lambda a, b: cmp(a.name, b.name)) - return orphans - - def wantedPages(self): - wantedPages = {} - existant = [] - for page in self.allPages(): - existant.append(page.name) - for link in self.index.forwardLinks(page.name): - wantedPages.setdefault(link, []).append(page) - for name in existant: - if wantedPages.has_key(name): - del wantedPages[name] - return [(self.page(name), wants) - for (name, wants) in wantedPages.items()] - - def allPages(self): - """All pages with content in the system.""" - return [self.page(filename[:-4]) - for filename in os.listdir(self.basepath) - if filename.endswith('.txt')] - - ############################################################ - ## Links - ############################################################ - - def linkTo(self, pageName, source=False): - """Return the href to refer to pageName.""" - if isinstance(pageName, wikipage.WikiPage): - page = pageName - pageName = pageName.name - else: - page = self.page(pageName) - pageName = pageName.lower() - if '.' in pageName: - # Already has an extension - return self.basehref + pageName - else: - url = self.basehref + pageName - if not source: - return url + str(self.config.get('wikiextension', '.html')) - else: - return url + self.extensionForMimeType(page.mimeType) - - def staticLinkTo(self, pageName): - if isinstance(pageName, wikipage.WikiPage): - pageName = pageName.name - if '.' in pageName: - # already has an extension - return self.config['statichref'] + pageName - else: - return (self.config['statichref'] - + pageName - + self.config.get('staticextension', '.html')) - - ############################################################ - ## Content - ############################################################ - - def notifyChange(self, page, action, **args): - """Notify page change. - - Called by the page everytime it is changed, so global - indexing and updating can occur. - - """ - if action != 'connected' and page.name not in ('wikisandbox',): - changed = self.syndicateRecentChanges - desc = page.lastChangeLog or '' - if page.lastChangeUser: - desc += ' -- %s' % page.lastChangeUser - item = rssobject.RSSItem( - title=page.title, - description=desc, - link='http://%s%s' % (self.canonicalDomain, page.link), - guid='http://%s%s?version=%s' % ( - self.canonicalDomain, - page.link, - page.versions()[-1].version), - ) - changed.addItem(item, 10) - changed.writeText() - self.index.setLinks(page.name, page.wikiLinks()) - - if action == 'create' and page.pageClass in ('posting',): - new = self.syndicateNewPages - link = 'http://%s%s' % (self.canonicalDomain, page.link) - item = rssobject.RSSItem( - title=page.title, - description=page.html, - link=link, - guid=link, - pubDate=rssobject.formatDate(page.creationDate)) - new.addItem(item, 10) - new.writeText() - - self.schedulePublish(page) - if action in ('create', 'delete'): - for linkingPage in self.backlinks(page): - self.schedulePublish(linkingPage) - if page._connectionsDirty: - page._connectionDirty = False - self.index.setConnections( - page.name, - [(p.name, type) for p, type in page.connections]) - - def canonicalDomain__get(self): - if self.canonical: - return self.canonical.domain - else: - return self.domain - - def schedulePublish(self, page): - if not self.config.getbool('staticpublish', False): - return - assert pooledtemplate, ( - "You must install Cheetah (cheetahtemplate.org) to use " - "static publishing") - self.publishQueue.set(page.name) - - def publish(self): - if not self.config.getbool('staticpublish', False): - return - pages = [self.page(n) for n in self.publishQueue.popmany(20)] - if not pages: - return - try: - meth = self.config.staticMethod - print 'Publishing via %s: %s' % ( - meth, - ', '.join([p.name for p in pages])) - if meth == 'file': - self.publishFile(pages) - elif meth in ('ssh', 'scp', 'sftp'): - self.publishRemote(pages, sshclient) - elif meth == 'ftp': - self.publishRemote(pages, ftpclient) - else: - assert 0, "Unknown static publishing method: %r" % meth - except: - self.publishQueue.extend([p.name for p in pages]) - raise - - def publishFile(self, pages): - for page in pages: - if not os.path.exists(self.config.staticPath): - os.mkdir(self.config.staticPath) - base = os.path.join(self.config.staticPath, page.name ) - f = open(base + '.html', 'wb') - f.write(self.publishedText(page)) - f.close() - if not page.mimeType == 'text/html': - f = open(base + self.extensionForMimeType(page.mimeType), 'wb') - f.write(page.text) - f.close() - - def publishRemote(self, pages, remoteModule): - fileList = self._publishTempFiles(pages) - try: - remoteModule.uploadFiles( - hostname=self.config.staticHostname, - fileList=fileList, - username=self.config.staticUsername, - password=self.config.staticPassword) - finally: - self._unpublishTempFiles(fileList) - - def _publishTempFiles(self, pages): - fileList = [] - for page in pages: - source = os.tempnam() - dest = os.path.join(self.config.staticPath, page.name ) - f = open(source, 'wb') - f.write(self.publishedText(page)) - f.close() - fileList.append((source, dest + '.html')) - if not page.mimeType == 'text/html': - source = os.tempnam() - f = open(source, 'wb') - f.write(page.text) - f.close() - fileList.append( - (source, - dest + self.extensionForMimeType(page.mimeType))) - return fileList - - def _unpublishTempFiles(self, fileList): - for source, dest in fileList: - os.unlink(source) - - def publishedText(self, page): - if self.template is None: - if not os.path.exists(self.config.staticTemplate): - filename = ( - os.path.join(os.path.dirname(__file__), - 'default_static.tmpl')) - else: - filename = self.config.staticTemplate - self.template = pooledtemplate.Template(file=filename) - return self.template.eval(page=page) - - def rebuildHTML(self): - print "Rerending HTML" - for page in self.allPages(): - page.rerender() - - def rebuildStatic(self): - print "Rebuilding static pages" - for page in self.allPages(): - self.schedulePublish(page) - - def recreateThumbnails(self): - print "Recreating thumbnails" - for page in self.allPages(): - page.recreateThumbnail() - - def syndicateRecentChanges__get(self): - """Returns the rssobject.RSS object.""" - rss = rssobject.RSS(self.syndicateRecentChangesFilename) - conf = self.config.get('rss', {}) - for attribute in rssobject.rssAttributes: - value = conf.get(attribute) - if value: - setattr(rss, attribute, value) - rss.link = 'http://%s%s' % (self.canonicalDomain, self.basehref) - return rss - - def syndicateRecentChangesFilename__get(self): - return os.path.join(self.basepath, 'recent_changes.xml') - - def syndicateNewPages__get(self): - def sorter(item1, item2): - return cmp( - rssobject.parseDate(item1.pubDate), - rssobject.parseDate(item2.pubDate)) - rss = rssobject.RSS(self.syndicateNewPagesFilename, sortOrder=sorter) - conf = self.config.get('rss', {}) - for attribute in rssobject.rssAttributes: - value = conf.get(attribute) - if value: - setattr(rss, attribute, value) - rss.link = 'http://%s%s' % (self.canonicalDomain, self.basehref) - return rss - - def syndicateNewPagesFilename__get(self): - return os.path.join(self.basepath, 'new_pages.xml') - - def extensionForMimeType(self, mimeType): - return _reverseMimeMap[mimeType] - - availableMimeTypes = [ - 'text/x-restructured-text', - 'text/html', - 'text/plain', - 'text/x-python', - 'image/gif', - 'image/png', - 'image/jpeg', - 'image/*', - 'application/*', - ] - - ############################################################ - ## Relations and indexing - ############################################################ - - def backlinks(self, page): - return [self.page(name) - for name in self.index.backlinks(page.name)] - - def connectedPages(self, page): - return [(self.page(name), type) - for name, type - in self.index.connections(page.name)] - - def connectPage(self, sourcePage, destPage, type): - connections = self.index.connections(destPage.name) - sourceName = sourcePage.name - for name, connType in connections: - if name == sourceName and connType == type: - return - connections.append((sourceName, type)) - self.index.setConnections(destPage.name, connections) - - def unconnectPage(self, sourcePage, destPage, type): - connections = self.index.connections(destPage.name) - try: - connections.remove((sourcePage.name, type)) - except ValueError: - pass - else: - self.index.setConnections(destPage.name, connections) - - def backConnections(self, page): - result = [] - for name, type in self.index.backConnections(page.name): - result.append((self.page(name), type)) - return result - - ############################################################ - ## Administrative and maintenance functions - ############################################################ - - def rebuildIndex(self): - print 'Rebuilding index...' - self.index.clear() - for page in self.allPages(): - self.index.setLinks(page.name, page.wikiLinks()) - self.index.setConnections( - page.name, - [(p.name, type) for p, type in page.connections]) - print 'The index has been rebuilt.' - - def checkDistributionFiles(self): - sourcePath = os.path.join(os.path.dirname(__file__), 'distpages') - names = [] - files = {} - for filename in os.listdir(sourcePath): - base, ext = os.path.splitext(filename) - files.setdefault(base, []).append(filename) - if ext == '.meta': - names.append(base) - for name in names: - page = self.page(name) - if not page.exists() or page.distributionOriginal: - newText = open(os.path.join(sourcePath, name+'.txt')).read() - d = rfc822persist.RFC822Dict( - os.path.join(sourcePath, name+'.meta')) - if page.text == newText: - for name, value in page.metadata.items(): - if d.has_key(name) and d[name] != value: - break - else: - continue - print "Updating %s from distribution" % name - page.updateRawMetadata(d) - page.text = newText - page.more = True - page.lastChangeLog = 'Updated from distribution' - page.save() - -_reverseMimeMap = {} -for _extDict in [mimetypes.types_map, mimetypes.common_types]: - for _key, _value in _extDict.items(): - _reverseMimeMap[_value] = _key - -# We can't add this to mimetypes, because then the .txt extension -# would be overwritten there. -_reverseMimeMap['text/x-restructured-text'] = '.txt' -# Theoretically it should be .jpeg, but .jpg is conventional: -_reverseMimeMap['image/jpeg'] = '.jpg' -# No .htm -_reverseMimeMap['text/html'] = '.html' -_reverseMimeMap['text/x-redirect-list'] = '.redirect' diff --git a/lib/wikiconfig.py b/lib/wikiconfig.py deleted file mode 100644 index 54dc71a..0000000 --- a/lib/wikiconfig.py +++ /dev/null @@ -1,51 +0,0 @@ -from config.lazyloader import LazyLoader - -__all__ = ['WikiConfig'] - -class NoDefault: - pass - -class WikiConfig(LazyLoader): - - def __init__(self, *args, **kw): - self._merged_page_classes = {} - LazyLoader.__init__(self, *args, **kw) - - def getbool(self, key, default=NoDefault): - try: - return self.convert(key, converter=self.convertbool) - except KeyError: - if default is NoDefault: - raise - return default - - def convertbool(self, value): - value = value.strip().lower() - if value in ('1', 'true', 'yes', 'on'): - return True - elif value in ('0', 'false', 'no', 'off'): - return False - else: - raise ValueError( - "Boolean expected (true/false)") - - def merge_page_class(self, page_class): - try: - return self._merged_page_classes[page_class] or self - except KeyError: - pass - values = self.getlist('pageclass') - to_add = [] - for value in values: - if value.has_key(page_class): - to_add.append(value[page_class]) - if to_add: - new = self.clone() - for item in to_add: - new.merge(item) - self._merged_page_classes[page_class] = new - return new - else: - self._merged_page_classes[page_class] = None - return self - diff --git a/lib/wikiindex.py b/lib/wikiindex.py deleted file mode 100644 index 6e3282d..0000000 --- a/lib/wikiindex.py +++ /dev/null @@ -1,163 +0,0 @@ -import shelve -import os -import atexit - -_shelvesToClose = [] - -def _closeHandler(): - while _shelvesToClose: - _shelvesToClose.pop().close() - -atexit.register(_closeHandler) - -class WikiIndex(object): - - _dbName = 'index.db' - - def __init__(self, dir): - self.dir = dir - self.filename = os.path.join(self.dir, self._dbName) - self.store = shelve.open(self.filename) - _shelvesToClose.append(self.store) - - def exists(cls, dir): - return os.path.exists(os.path.join(dir, cls._dbName)) - exists = classmethod(exists) - - def clear(self): - """Clear the index.""" - self.store.clear() - - def backlinks(self, pageName): - """Returns a list of pages that link to pageName.""" - return self._getLinks('backlink', pageName) - - def forwardLinks(self, pageName): - """Returns a list of pages to which pageName links.""" - return self._getLinks('forward', pageName) - - def _setBacklinks(self, pageName, links): - self._setLinks('backlink', pageName, links) - - def _setForwardLinks(self, pageName, links): - self._setLinks('forward', pageName, links) - - def setLinks(self, pageName, links): - """Indexes all the backlinks for a page. - - Also indexes forward links, so that we can tell - when a backlink needs to be removed. - - """ - oldLinks = self.forwardLinks(pageName) - oldLinks.sort() - newLinks = links[:] - newLinks.sort() - toRemove = [] - toAdd = [] - while links and oldLinks: - if links[0] > oldLinks[0]: - toRemove.append(oldLinks.pop(0)) - elif links[0] < oldLinks[0]: - toAdd.append(links.pop(0)) - else: - oldLinks.pop(0) - toRemove.extend(oldLinks) - toAdd.extend(newLinks) - for link in toRemove: - prev = self.backlinks(link) - try: - prev.remove(pageName) - except ValueError: - # @@: because it's not transactional and concurrency - # control is poor, we might not have consistent data - # here, so we might not find the link - pass - self._setBacklinks(link, prev) - for link in toAdd: - prev = self.backlinks(link) - prev.append(pageName) - prev.sort() - self._setBacklinks(link, prev) - self._setForwardLinks(pageName, links) - - def _getLinks(self, type, pageName): - result = self.store.get(str('%s.%s' % (type, pageName)), '') - result = filter(None, result.split(',')) - return result - - def _setLinks(self, type, pageName, links): - links = ','.join(unique(links)) - self.store['%s.%s' % (type, pageName)] = links - - def connections(self, pageName): - """Returns a list of all the connections. - - The list has the form [(connection_type_string, pageName), ...]. - This is pages connected *to* this page. - """ - result = self.store.get('connections.%s' % pageName, '') - values = result.split(',') - connections = [] - for value in values: - if not value: - continue - connectionType, pageName = value.split(':', 1) - connections.append((pageName, connectionType)) - return connections - - def setConnections(self, pageName, connections): - oldConnections = self.connections(pageName) - values = ','.join([ - '%s:%s' % (t, n) for n, t in connections]) - self.store['connections.%s' % pageName] = values - for connPage, type in connections: - back = self.backConnections(connPage) - if (pageName, type) in back: - # @@: Really this shouldn't happen, as it signals - # a data integrity problem - continue - back.append((pageName, type)) - self._setBackConnections(connPage, back) - for connPage, type in oldConnections: - if (connPage, type) in connections: - continue - back = self.backConnections(connPage) - if (pageName, type) not in back: - # @@: again shouldn't happen - continue - back.remove((pageName, type)) - self._setBackConnections(connPage, back) - - def backConnections(self, pageName): - result = self.store.get('back_connections.%s' % pageName, '') - values = result.split(',') - backConn = [] - for value in values: - if not value: - continue - connectionType, pageName = value.split(':', 1) - backConn.append((pageName, connectionType)) - return backConn - - def _setBackConnections(self, pageName, backConn): - values = ','.join([ - '%s:%s' % (t, n) for n, t in backConn]) - self.store['back_connections.%s' % pageName] = values - - def _packDict(self, d): - return ','.join(['%s:%s' % (k, v) - for (k, v) in d.items()]) - - def _unpackDict(self, s): - v = [t.split(':', 1) for t in s.split(',') if t] - try: - return dict(v) - except ValueError: - raise ValueError, "Bad dict: %r (became %r)" % (s, v) - -def unique(vals): - result = {} - for v in vals: - result[v] = None - return result.keys() diff --git a/lib/wikipage.py b/lib/wikipage.py deleted file mode 100644 index dde39fd..0000000 --- a/lib/wikipage.py +++ /dev/null @@ -1,756 +0,0 @@ -""" -The Wiki module primarily exports the `WikiPage` class: -""" - -import os, re, time -import shutil -from datetime import datetime, timedelta -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO -import propertymeta -from rfc822persist import RFC822Dict, metaprop, metabool -try: - import Image -except ImportError: - Image = None -import converter_registry -from html_abstracts import find_abstract -from common import * -try: - import user -except ImportError: - user = None - -# Just make sure these are loaded: -import convert_rest -del convert_rest -import convert_html -del convert_html - -__all__ = ['WikiPage'] - -class WikiPage(object): - """ - WikiPage is a class to represent one page in a WikiWikiWeb [#]_. - The page may or may not yet exist -- that is, it may not yet - have content. - - .. [#] http://c2.com/cgi-bin/wiki - - It has the following properties and methods: - - `html`: - A read-only property giving the HTML for the page. - If the page does not yet have content the text - ``"This page has not yet been created"`` is returned. - `text`: - The text for the page. To save new text, simply - assign to this property. - .. ignore: html - .. ignore: text - .. ignore: setText - - """ - - __metaclass__ = propertymeta.MakeProperties - - def __init__(self, wiki, dir, pageName, - urlName=None, version=None): - """ - Each page has a name, which is a unique identifier, for example - ``"FrontPage"``, which identifies the page in the URL and - for linking. - """ - self.dir = dir - self.wiki = wiki - self.name = pageName - self.version = version - self.metadata = RFC822Dict(self.basePath + '.meta') - self._text = None - self._summary = None - self._thumbnail = None - self._connectionsDirty = False - self._config = None - if not self.exists() and urlName is not None: - self.urlName = urlName - self.title = guessTitle(urlName) - - def __repr__(self): - text = '