#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
from titlecase import titlecase # NOQA
import locale
class TypogrifyError(Exception):
""" A base error class so we can catch or scilence typogrify's errors in templates """
pass
def process_ignores(text, ignore_tags=None):
""" Creates a list of tuples based on tags to be ignored.
Tags can be added as a list in the `ignore_tags`.
Returns in the following format:
[
('Text here', processed processed processedprocessed
processed
', False), ('processed
processed
processed
', False), ('processed', True), ('
processed', False), ('', True)] >>> process_ignores('
processed
processed
processed',['p']) [('
processed
', False), ('processed
processed', False)] """ position = 0 sections = [] if ignore_tags is None: ignore_tags = [] ignore_tags = ignore_tags + ['pre', 'code'] # default tags ignore_regex = r'<(%s)(?:\s.*?)?>.*?(\1)>' % '|'.join(ignore_tags) ignore_finder = re.compile(ignore_regex, re.IGNORECASE | re.DOTALL) for section in ignore_finder.finditer(text): start, end = section.span() if position != start: # if the current position isn't the match we # need to process everything in between sections.append((text[position:start], True)) # now we mark the matched section as ignored sections.append((text[start:end], False)) position = end # match the rest of the text if necessary # (this could in fact be the entire string) if position < len(text): sections.append((text[position:len(text)], True)) return sections def amp(text): """Wraps apersands in HTML with ```` so they can be styled with CSS. Apersands are also normalized to ``&``. Requires ampersands to have whitespace or an `` `` on both sides. >>> amp('One & two') 'One & two' >>> amp('One & two') 'One & two' >>> amp('One & two') 'One & two' >>> amp('One & two') 'One & two' It won't mess up & that are already wrapped, in entities or URLs >>> amp('One & two') 'One & two' >>> amp('“this” & that') '“this” & that' It should ignore standalone amps that are in attributes >>> amp('xyz') 'xyz' """ # tag_pattern from http://haacked.com/archive/2004/10/25/usingregularexpressionstomatchhtml.aspx # it kinda sucks but it fixes the standalone amps in attributes bug tag_pattern = '?\w+((\s+\w+(\s*=\s*(?:".*?"|\'.*?\'|[^\'">\s]+))?)+\s*|\s*)/?>' amp_finder = re.compile(r"(\s| )(&|&|&\#38;)(\s| )") intra_tag_finder = re.compile(r'(?P
CAPSmore CAPS") '
CAPSmore CAPS' >>> caps("A message from 2KU2 with digits") 'A message from 2KU2 with digits' >>> caps("Dotted caps followed by spaces should never include them in the wrap D.O.T. like so.") 'Dotted caps followed by spaces should never include them in the wrap D.O.T. like so.' All caps with with apostrophes in them shouldn't break. Only handles dump apostrophes though. >>> caps("JIMMY'S") 'JIMMY\\'S' >>> caps("D.O.T.HE34TRFID") 'D.O.T.HE34TRFID' """ try: import smartypants except ImportError: raise TypogrifyError("Error in {% caps %} filter: The Python SmartyPants library isn't installed.") tokens = smartypants._tokenize(text) result = [] in_skipped_tag = False cap_finder = re.compile(r"""( (\b[A-Z\d]* # Group 2: Any amount of caps and digits [A-Z]\d*[A-Z] # A cap string much at least include two caps (but they can have digits between them) [A-Z\d']*\b) # Any amount of caps and digits or dumb apostsrophes | (\b[A-Z]+\.\s? # OR: Group 3: Some caps, followed by a '.' and an optional space (?:[A-Z]+\.\s?)+) # Followed by the same thing at least once more (?:\s|\b|$)) """, re.VERBOSE) def _cap_wrapper(matchobj): """This is necessary to keep dotted cap strings to pick up extra spaces""" if matchobj.group(2): return """%s""" % matchobj.group(2) else: if matchobj.group(3)[-1] == " ": caps = matchobj.group(3)[:-1] tail = ' ' else: caps = matchobj.group(3) tail = '' return """%s%s""" % (caps, tail) # Add additional tags whose content should be # ignored here. Note -
and tag are
# ignored by default and therefore are not here
tags_to_skip_regex = re.compile("<(/)?(?:kbd|script)[^>]*>", re.IGNORECASE)
for token in tokens:
if token[0] == "tag":
# Don't mess with tags.
result.append(token[1])
close_match = tags_to_skip_regex.match(token[1])
if close_match and close_match.group(1) == None:
in_skipped_tag = True
else:
in_skipped_tag = False
else:
if in_skipped_tag:
result.append(token[1])
else:
result.append(cap_finder.sub(_cap_wrapper, token[1]))
output = "".join(result)
return output
def initial_quotes(text):
"""Wraps initial quotes in ``class="dquo"`` for double quotes or
``class="quo"`` for single quotes. Works in these block tags ``(h1-h6, p, li, dt, dd)``
and also accounts for potential opening inline elements ``a, em, strong, span, b, i``
>>> initial_quotes('"With primes"')
'"With primes"'
>>> initial_quotes("'With single primes'")
'\\'With single primes\\''
>>> initial_quotes('"With primes and a link"')
'"With primes and a link"'
>>> initial_quotes('“With smartypanted quotes”')
'“With smartypanted quotes”'
"""
quote_finder = re.compile(r"""((<(p|h[1-6]|li|dt|dd)[^>]*>|^) # start with an opening p, h1-6, li, dd, dt or the start of the string
\s* # optional white space!
(<(a|em|span|strong|i|b)[^>]*>\s*)*) # optional opening inline tags, with more optional white space for each.
(("|“|&\#8220;)|('|‘|&\#8216;)) # Find me a quote! (only need to find the left quotes and the primes)
# double quotes are in group 7, singles in group 8
""", re.VERBOSE)
def _quote_wrapper(matchobj):
if matchobj.group(7):
classname = "dquo"
quote = matchobj.group(7)
else:
classname = "quo"
quote = matchobj.group(8)
return """%s%s""" % (matchobj.group(1), classname, quote)
output = quote_finder.sub(_quote_wrapper, text)
return output
def smartypants(text):
"""Applies smarty pants to curl quotes.
>>> smartypants('The "Green" man')
'The “Green” man'
"""
try:
import smartypants
except ImportError:
raise TypogrifyError("Error in {% smartypants %} filter: The Python smartypants library isn't installed.")
else:
output = smartypants.smartypants(text)
return output
def french_insecable(text):
"""Replace the space between each double sign punctuation by a thin
non-breaking space.
This conform with the french typographic rules.
>>> french_insecable('Foo !')
u'Foo !'
>>> french_insecable('Foo ?')
u'Foo ?'
>>> french_insecable('Foo : bar')
u'Foo : bar'
>>> french_insecable('Foo ; bar')
u'Foo ; bar'
>>> french_insecable(u'\xab bar \xbb')
u'\\xab bar \\xbb'
>>> french_insecable('123 456')
u'123 456'
>>> french_insecable('123 %')
u'123 %'
Space inside attributes should be preserved :
>>> french_insecable('')
''
"""
tag_pattern = '?\w+((\s+\w+(\s*=\s*(?:".*?"|\'.*?\'|[^\'">\s]+))?)+\s*|\s*)/?>'
intra_tag_finder = re.compile(r'(?P(%s)?)(?P([^<]*))(?P(%s)?)' % (tag_pattern, tag_pattern))
nnbsp = u' '
space_finder = re.compile(r"""(?:
(\w\s[:;!\?\xbb])| # Group 1, space before punctuation
([\xab]\s\w)|
([0-9]\s[0-9])|
([0-9]\s\%)
)""", re.VERBOSE)
def _insecable_wrapper(groups):
"""This is necessary to keep dotted cap strings to pick up extra spaces"""
def substitute(matchobj):
return matchobj.group(0).replace(" ", nnbsp)
prefix = groups.group('prefix') or ''
text = space_finder.sub(substitute, groups.group('text'))
suffix = groups.group('suffix') or ''
return prefix + text + suffix
output = intra_tag_finder.sub(_insecable_wrapper, text)
return output
def localize(text):
""" Return the text processed with the appropriate system locale
"""
table = {"fr_FR" : lambda x : french_insecable(x)}
lang = locale.getdefaultlocale()[0]
processor = table.get(lang, lambda x : x)
return processor(text)
def widont(text):
"""Replaces the space between the last two words in a string with `` ``
Works in these block tags ``(h1-h6, p, li, dd, dt)`` and also accounts for
potential closing inline elements ``a, em, strong, span, b, i``
>>> widont('A very simple test')
'A very simple test'
Single word items shouldn't be changed
>>> widont('Test')
'Test'
>>> widont(' Test')
' Test'
>>> widont('- Test
')
'- Test
'
>>> widont('- Test
')
'- Test
'
>>> widont('In a couple of paragraphs
paragraph two
')
'In a couple of paragraphs
paragraph two
'
>>> widont('In a link inside a heading
')
'In a link inside a heading
'
>>> widont('In a link followed by other text
')
'In a link followed by other text
'
Empty HTMLs shouldn't error
>>> widont('
')
'
'
>>> widont('Divs get no love!')
'Divs get no love!'
>>> widont('
Neither do PREs
')
'Neither do PREs
'
>>> widont('But divs with paragraphs do!
')
'But divs with paragraphs do!
'
"""
widont_finder = re.compile(r"""((?:?(?:a|em|span|strong|i|b)[^>]*>)|[^<>\s]) # must be proceeded by an approved inline opening or closing tag or a nontag/nonspace
\s+ # the space to replace
([^<>\s]+ # must be flollowed by non-tag non-space characters
\s* # optional white space!
((a|em|span|strong|i|b)>\s*)* # optional closing inline tags with optional white space after each
(((p|h[1-6]|li|dt|dd)>)|$)) # end with a closing p, h1-6, li or the end of the string
""", re.VERBOSE)
output = widont_finder.sub(r'\1 \2', text)
return output
def applyfilters(text):
"""Applies the following filters: smartypants, caps, amp, initial_quotes
>>> typogrify('"Jayhawks" & KU fans act extremely obnoxiously
')
'“Jayhawks” & KU fans act extremely obnoxiously
'
"""
text = amp(text)
text = smartypants(text)
text = caps(text)
text = initial_quotes(text)
text = localize(text)
return text
def typogrify(text, ignore_tags=None):
"""The super typography filter
Applies filters to text that are not in tags contained in the
ignore_tags list.
"""
section_list = process_ignores(text, ignore_tags)
rendered_text = ""
for text_item, should_process in section_list:
if should_process:
rendered_text += applyfilters(text_item)
else:
rendered_text += text_item
# apply widont at the end, as its already smart about tags. Hopefully.
return widont(rendered_text)
def _test():
import doctest
doctest.testmod(verbose=True)
if __name__ == "__main__":
_test()