3 asciidoc - converts an AsciiDoc text file to HTML or DocBook
5 Copyright (C) 2002-2010 Stuart Rackham. Free use of this software is granted
6 under the terms of the GNU General Public License (GPL).
9 import sys, os, re, time, traceback, tempfile, subprocess, codecs, locale, unicodedata
11 ### Used by asciidocapi.py ###
12 VERSION = '8.6.5' # See CHANGLOG file for version history.
14 MIN_PYTHON_VERSION = 2.4 # Require this version of Python or better.
16 #---------------------------------------------------------------------------
18 #---------------------------------------------------------------------------
19 DEFAULT_BACKEND = 'html'
20 DEFAULT_DOCTYPE = 'article'
21 # Allowed substitution options for List, Paragraph and DelimitedBlock
22 # definition subs entry.
23 SUBS_OPTIONS = ('specialcharacters','quotes','specialwords',
24 'replacements', 'attributes','macros','callouts','normal','verbatim',
25 'none','replacements2')
26 # Default value for unspecified subs and presubs configuration file entries.
27 SUBS_NORMAL = ('specialcharacters','quotes','attributes',
28 'specialwords','replacements','macros','replacements2')
29 SUBS_VERBATIM = ('specialcharacters','callouts')
31 NAME_RE = r'(?u)[^\W\d][-\w]*' # Valid section or attribute name.
32 OR, AND = ',', '+' # Attribute list separators.
35 #---------------------------------------------------------------------------
36 # Utility functions and classes.
37 #---------------------------------------------------------------------------
39 class EAsciiDoc(Exception): pass
41 class OrderedDict(dict):
43 Dictionary ordered by insertion order.
44 Python Cookbook: Ordered Dictionary, Submitter: David Benjamin.
45 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/107747
47 def __init__(self, d=None, **kwargs):
49 if d is None: d = kwargs
50 dict.__init__(self, d)
51 def __delitem__(self, key):
52 dict.__delitem__(self, key)
53 self._keys.remove(key)
54 def __setitem__(self, key, item):
55 dict.__setitem__(self, key, item)
56 if key not in self._keys: self._keys.append(key)
62 d._keys = self._keys[:]
65 return zip(self._keys, self.values())
72 raise KeyError('dictionary is empty')
76 def setdefault(self, key, failobj = None):
77 dict.setdefault(self, key, failobj)
78 if key not in self._keys: self._keys.append(key)
79 def update(self, d=None, **kwargs):
84 if key not in self._keys: self._keys.append(key)
86 return map(self.get, self._keys)
90 Like a dictionary except values can be accessed as attributes i.e. obj.foo
91 can be used in addition to obj['foo'].
92 If an item is not present None is returned.
94 def __getattr__(self, key):
96 except KeyError: return None
97 def __setattr__(self, key, value):
99 def __delattr__(self, key):
101 except KeyError, k: raise AttributeError, k
103 return '<AttrDict ' + dict.__repr__(self) + '>'
104 def __getstate__(self):
106 def __setstate__(self,value):
107 for k,v in value.items(): self[k]=v
109 class InsensitiveDict(dict):
111 Like a dictionary except key access is case insensitive.
112 Keys are stored in lower case.
114 def __getitem__(self, key):
115 return dict.__getitem__(self, key.lower())
116 def __setitem__(self, key, value):
117 dict.__setitem__(self, key.lower(), value)
118 def has_key(self, key):
119 return dict.has_key(self,key.lower())
120 def get(self, key, default=None):
121 return dict.get(self, key.lower(), default)
122 def update(self, dict):
123 for k,v in dict.items():
125 def setdefault(self, key, default = None):
126 return dict.setdefault(self, key.lower(), default)
131 Used in conjunction with the 'trace' attribute to generate diagnostic
132 output. There is a single global instance of this class named trace.
134 SUBS_NAMES = ('specialcharacters','quotes','specialwords',
135 'replacements', 'attributes','macros','callouts',
138 self.name_re = '' # Regexp pattern to match trace names.
141 def __call__(self, name, before, after=None):
143 Print trace message if tracing is on and the trace 'name' matches the
144 document 'trace' attribute (treated as a regexp).
145 'before' is the source text before substitution; 'after' text is the
146 source text after substitutuion.
147 The 'before' and 'after' messages are only printed if they differ.
149 name_re = document.attributes.get('trace')
150 if name_re == 'subs': # Alias for all the inline substitutions.
151 name_re = '|'.join(self.SUBS_NAMES)
152 self.name_re = name_re
153 if self.name_re is not None:
154 msg = message.format(name, 'TRACE: ', self.linenos, offset=self.offset)
155 if before != after and re.match(self.name_re,name):
157 before = '\n'.join(before)
159 msg += '\n%s\n' % before
162 after = '\n'.join(after)
163 msg += '\n<<<\n%s\n>>>\n%s\n' % (before,after)
170 PROG = os.path.basename(os.path.splitext(__file__)[0])
173 # Set to True or False to globally override line numbers method
174 # argument. Has no effect when set to None.
178 def stdout(self,msg):
181 def stderr(self,msg=''):
182 self.messages.append(msg)
183 if __name__ == '__main__':
184 sys.stderr.write('%s: %s%s' % (self.PROG, msg, os.linesep))
186 def verbose(self, msg,linenos=True):
188 msg = self.format(msg,linenos=linenos)
191 def warning(self, msg,linenos=True,offset=0):
192 msg = self.format(msg,'WARNING: ',linenos,offset=offset)
193 document.has_warnings = True
196 def deprecated(self, msg, linenos=True):
197 msg = self.format(msg, 'DEPRECATED: ', linenos)
200 def format(self, msg, prefix='', linenos=True, cursor=None, offset=0):
201 """Return formatted message string."""
202 if self.linenos is not False and ((linenos or self.linenos) and reader.cursor):
204 cursor = reader.cursor
205 prefix += '%s: line %d: ' % (os.path.basename(cursor[0]),cursor[1]+offset)
208 def error(self, msg, cursor=None, halt=False):
211 If halt=True raise EAsciiDoc exception.
212 If halt=False don't exit application, continue in the hope of reporting
213 all fatal errors finishing with a non-zero exit code.
216 raise EAsciiDoc, self.format(msg,linenos=False,cursor=cursor)
218 msg = self.format(msg,'ERROR: ',cursor=cursor)
220 document.has_errors = True
222 def unsafe(self, msg):
223 self.error('unsafe: '+msg)
228 Return user's home directory or None if it is not defined.
230 result = os.path.expanduser('~')
237 Return True if we are not executing the system wide version
238 i.e. the configuration is in the executable's directory.
240 return os.path.isfile(os.path.join(APP_DIR, 'asciidoc.conf'))
242 def file_in(fname, directory):
243 """Return True if file fname resides inside directory."""
244 assert os.path.isfile(fname)
245 # Empty directory (not to be confused with None) is the current directory.
247 directory = os.getcwd()
249 assert os.path.isdir(directory)
250 directory = os.path.realpath(directory)
251 fname = os.path.realpath(fname)
252 return os.path.commonprefix((directory, fname)) == directory
257 def is_safe_file(fname, directory=None):
258 # A safe file must reside in directory directory (defaults to the source
260 if directory is None:
261 if document.infile == '<stdin>':
263 directory = os.path.dirname(document.infile)
264 elif directory == '':
268 or file_in(fname, directory)
269 or file_in(fname, APP_DIR)
270 or file_in(fname, CONF_DIR)
273 def safe_filename(fname, parentdir):
275 Return file name which must reside in the parent file directory.
276 Return None if file is not found or not safe.
278 if not os.path.isabs(fname):
279 # Include files are relative to parent document
281 fname = os.path.normpath(os.path.join(parentdir,fname))
282 if not os.path.isfile(fname):
283 message.warning('include file not found: %s' % fname)
285 if not is_safe_file(fname, parentdir):
286 message.unsafe('include file: %s' % fname)
291 """Assign all attributes from 'src' object to 'dst' object."""
292 for a,v in src.__dict__.items():
296 """Trim white space and, if necessary, quote characters from s."""
298 # Strip quotation mark characters from quoted strings.
299 if len(s) >= 3 and s[0] == '"' and s[-1] == '"':
304 """Return True if s is a valid regular expression else return False."""
310 """Join list of regular expressions re1,re2,... to single regular
311 expression (re1)|(re2)|..."""
315 # Delete named groups to avoid ambiguity.
317 result.append(re.sub(r'\?P<\S+?>','',s))
318 result = ')|('.join(result)
319 result = '('+result+')'
322 def validate(value,rule,errmsg):
323 """Validate value against rule expression. Throw EAsciiDoc exception with
324 errmsg if validation fails."""
326 if not eval(rule.replace('$',str(value))):
327 raise EAsciiDoc,errmsg
329 raise EAsciiDoc,errmsg
334 Return list with empty items from start of list removed.
336 for i in range(len(s)):
344 Return list with empty items from end of list removed.
346 for i in range(len(s)-1,-1,-1):
354 Return list with empty items from start and end of list removed.
362 Return True if object is list or tuple type.
364 return isinstance(obj,list) or isinstance(obj,tuple)
366 def dovetail(lines1, lines2):
368 Append list or tuple of strings 'lines2' to list 'lines1'. Join the last
369 non-blank item in 'lines1' with the first non-blank item in 'lines2' into a
372 assert is_array(lines1)
373 assert is_array(lines2)
374 lines1 = strip_list(lines1)
375 lines2 = strip_list(lines2)
376 if not lines1 or not lines2:
377 return list(lines1) + list(lines2)
378 result = list(lines1[:-1])
379 result.append(lines1[-1] + lines2[0])
380 result += list(lines2[1:])
383 def dovetail_tags(stag,content,etag):
384 """Merge the end tag with the first content line and the last
385 content line with the end tag. This ensures verbatim elements don't
386 include extraneous opening and closing line breaks."""
387 return dovetail(dovetail(stag,content), etag)
389 def parse_attributes(attrs,dict):
390 """Update a dictionary with name/value attributes from the attrs string.
391 The attrs string is a comma separated list of values and keyword name=value
392 pairs. Values must preceed keywords and are named '1','2'... The entire
393 attributes list is named '0'. If keywords are specified string values must
400 dict: {'2': 'world', '0': 'hello,world', '1': 'hello'}
402 attrs: '"hello", planet="earth"'
403 dict: {'planet': 'earth', '0': '"hello",planet="earth"', '1': 'hello'}
405 def f(*args,**keywords):
406 # Name and add aguments '1','2'... to keywords.
407 for i in range(len(args)):
408 if not str(i+1) in keywords:
409 keywords[str(i+1)] = args[i]
415 # Replace line separators with spaces so line spanning works.
416 s = re.sub(r'\s', ' ', attrs)
419 # Attributes must evaluate to strings, numbers or None.
421 if not (isinstance(v,str) or isinstance(v,int) or isinstance(v,float) or v is None):
424 s = s.replace('"','\\"')
426 s = map(lambda x: '"' + x.strip() + '"', s)
431 return # If there's a syntax error leave with {0}=attrs.
432 for k in d.keys(): # Drop any empty positional arguments.
433 if d[k] == '': del d[k]
437 def parse_named_attributes(s,attrs):
438 """Update a attrs dictionary with name="value" attributes from the s string.
439 Returns False if invalid syntax.
441 attrs: 'star="sun",planet="earth"'
442 dict: {'planet':'earth', 'star':'sun'}
444 def f(**keywords): return keywords
454 """Parse comma separated string of Python literals. Return a tuple of of
457 result = eval('tuple(['+s+'])')
459 raise EAsciiDoc,'malformed list: '+s
462 def parse_options(options,allowed,errmsg):
463 """Parse comma separated string of unquoted option names and return as a
464 tuple of valid options. 'allowed' is a list of allowed option values.
465 If allowed=() then all legitimate names are allowed.
466 'errmsg' is an error message prefix if an illegal option error is thrown."""
469 for s in re.split(r'\s*,\s*',options):
470 if (allowed and s not in allowed) or not is_name(s):
471 raise EAsciiDoc,'%s: %s' % (errmsg,s)
476 """Drop non-symbol characters and convert to lowercase."""
477 return re.sub(r'(?u)[^\w\-_]', '', s).lower()
480 """Return True if s is valid attribute, macro or tag name
481 (starts with alpha containing alphanumeric and dashes only)."""
482 return re.match(r'^'+NAME_RE+r'$',s) is not None
484 def subs_quotes(text):
485 """Quoted text is marked up and the resulting text is
487 keys = config.quotes.keys()
490 if i != -1 and q != '|' and q != '||':
491 lq = q[:i] # Left quote.
492 rq = q[i+1:] # Right quote.
495 tag = config.quotes[q]
497 # Unconstrained quotes prefix the tag name with a hash.
500 # Unconstrained quotes can appear anywhere.
501 reo = re.compile(r'(?msu)(^|.)(\[(?P<attrlist>[^[\]]+?)\])?' \
502 + r'(?:' + re.escape(lq) + r')' \
503 + r'(?P<content>.+?)(?:'+re.escape(rq)+r')')
505 # The text within constrained quotes must be bounded by white space.
506 # Non-word (\W) characters are allowed at boundaries to accomodate
507 # enveloping quotes and punctuation e.g. a='x', ('x'), 'x', ['x'].
508 reo = re.compile(r'(?msu)(^|[^\w;:}])(\[(?P<attrlist>[^[\]]+?)\])?' \
509 + r'(?:' + re.escape(lq) + r')' \
510 + r'(?P<content>\S|\S.*?\S)(?:'+re.escape(rq)+r')(?=\W|$)')
513 mo = reo.search(text,pos)
515 if text[mo.start()] == '\\':
516 # Delete leading backslash.
517 text = text[:mo.start()] + text[mo.start()+1:]
518 # Skip past start of match.
522 parse_attributes(mo.group('attrlist'), attrlist)
523 stag,etag = config.tag(tag, attrlist)
524 s = mo.group(1) + stag + mo.group('content') + etag
525 text = text[:mo.start()] + s + text[mo.end():]
526 pos = mo.start() + len(s)
529 def subs_tag(tag,dict={}):
530 """Perform attribute substitution and split tag string returning start, end
531 tag tuple (c.f. Config.tag())."""
534 s = subs_attrs(tag,dict)
536 message.warning('tag \'%s\' dropped: contains undefined attribute' % tag)
538 result = s.split('|')
541 elif len(result) == 2:
544 raise EAsciiDoc,'malformed tag: %s' % tag
546 def parse_entry(entry, dict=None, unquote=False, unique_values=False,
547 allow_name_only=False, escape_delimiter=True):
548 """Parse name=value entry to dictionary 'dict'. Return tuple (name,value)
549 or None if illegal entry.
550 If name= then value is set to ''.
551 If name and allow_name_only=True then value is set to ''.
552 If name! and allow_name_only=True then value is set to None.
553 Leading and trailing white space is striped from 'name' and 'value'.
554 'name' can contain any printable characters.
555 If the '=' delimiter character is allowed in the 'name' then
556 it must be escaped with a backslash and escape_delimiter must be True.
557 If 'unquote' is True leading and trailing double-quotes are stripped from
559 If unique_values' is True then dictionary entries with the same value are
560 removed before the parsed entry is added."""
562 mo = re.search(r'(?:[^\\](=))',entry)
564 mo = re.search(r'(=)',entry)
565 if mo: # name=value entry.
567 name = entry[:mo.start(1)]
569 name = name.replace(r'\=','=') # Unescape \= in name.
570 value = entry[mo.end(1):]
571 elif allow_name_only and entry: # name or name! entry.
581 name = strip_quotes(name)
582 if value is not None:
583 value = strip_quotes(value)
586 if value is not None:
587 value = value.strip()
592 for k,v in dict.items():
593 if v == value: del dict[k]
597 def parse_entries(entries, dict, unquote=False, unique_values=False,
598 allow_name_only=False,escape_delimiter=True):
599 """Parse name=value entries from from lines of text in 'entries' into
600 dictionary 'dict'. Blank lines are skipped."""
601 entries = config.expand_templates(entries)
602 for entry in entries:
603 if entry and not parse_entry(entry, dict, unquote, unique_values,
604 allow_name_only, escape_delimiter):
605 raise EAsciiDoc,'malformed section entry: %s' % entry
607 def dump_section(name,dict,f=sys.stdout):
608 """Write parameters in 'dict' as in configuration file section format with
610 f.write('[%s]%s' % (name,writer.newline))
611 for k,v in dict.items():
613 k = k.replace('=',r'\=') # Escape = in name.
614 # Quote if necessary.
615 if len(k) != len(k.strip()):
617 if v and len(v) != len(v.strip()):
620 # Don't dump undefined attributes.
625 s = '\\' + s # Escape so not treated as comment lines.
626 f.write('%s%s' % (s,writer.newline))
627 f.write(writer.newline)
629 def update_attrs(attrs,dict):
630 """Update 'attrs' dictionary with parsed attributes in dictionary 'dict'."""
631 for k,v in dict.items():
633 raise EAsciiDoc,'illegal attribute name: %s' % k
636 def is_attr_defined(attrs,dic):
638 Check if the sequence of attributes is defined in dictionary 'dic'.
639 Valid 'attrs' sequence syntax:
640 <attr> Return True if single attrbiute is defined.
641 <attr1>,<attr2>,... Return True if one or more attributes are defined.
642 <attr1>+<attr2>+... Return True if all the attributes are defined.
645 for a in attrs.split(OR):
646 if dic.get(a.strip()) is not None:
650 for a in attrs.split(AND):
651 if dic.get(a.strip()) is None:
655 return dic.get(attrs.strip()) is not None
657 def filter_lines(filter_cmd, lines, attrs={}):
659 Run 'lines' through the 'filter_cmd' shell command and return the result.
660 The 'attrs' dictionary contains additional filter attributes.
662 def findfilter(name,dir,filter):
663 """Find filter file 'fname' with style name 'name' in directory
664 'dir'. Return found file path or None if not found."""
666 result = os.path.join(dir,'filters',name,filter)
667 if os.path.isfile(result):
669 result = os.path.join(dir,'filters',filter)
670 if os.path.isfile(result):
674 # Return input lines if there's not filter.
675 if not filter_cmd or not filter_cmd.strip():
677 # Perform attributes substitution on the filter command.
678 s = subs_attrs(filter_cmd, attrs)
680 message.error('undefined filter attribute in command: %s' % filter_cmd)
682 filter_cmd = s.strip()
683 # Parse for quoted and unquoted command and command tail.
685 mo = re.match(r'^"(?P<cmd>[^"]+)"(?P<tail>.*)$', filter_cmd)
688 mo = re.match(r"^'(?P<cmd>[^']+)'(?P<tail>.*)$", filter_cmd)
690 # Unquoted catch all.
691 mo = re.match(r'^(?P<cmd>\S+)(?P<tail>.*)$', filter_cmd)
692 cmd = mo.group('cmd').strip()
694 if not os.path.dirname(cmd):
695 # Filter command has no directory path so search filter directories.
696 filtername = attrs.get('style')
697 d = document.attributes.get('docdir')
699 found = findfilter(filtername, d, cmd)
702 found = findfilter(filtername, USER_DIR, cmd)
705 found = findfilter(filtername, APP_DIR, cmd)
707 found = findfilter(filtername, CONF_DIR, cmd)
709 if os.path.isfile(cmd):
712 message.warning('filter not found: %s' % cmd)
714 filter_cmd = '"' + found + '"' + mo.group('tail')
715 if sys.platform == 'win32':
716 # Windows doesn't like running scripts directly so explicitly
717 # specify interpreter.
719 if cmd.endswith('.py'):
720 filter_cmd = 'python ' + filter_cmd
721 elif cmd.endswith('.rb'):
722 filter_cmd = 'ruby ' + filter_cmd
723 message.verbose('filtering: ' + filter_cmd)
725 p = subprocess.Popen(filter_cmd, shell=True,
726 stdin=subprocess.PIPE, stdout=subprocess.PIPE)
727 output = p.communicate(os.linesep.join(lines))[0]
729 raise EAsciiDoc,'filter error: %s: %s' % (filter_cmd, sys.exc_info()[1])
731 result = [s.rstrip() for s in output.split(os.linesep)]
734 filter_status = p.wait()
736 message.warning('filter non-zero exit code: %s: returned %d' %
737 (filter_cmd, filter_status))
738 if lines and not result:
739 message.warning('no output from filter: %s' % filter_cmd)
742 def system(name, args, is_macro=False, attrs=None):
744 Evaluate a system attribute ({name:args}) or system block macro
746 If is_macro is True then we are processing a system block macro otherwise
747 it's a system attribute.
748 The attrs dictionary is updated by the counter and set system attributes.
749 NOTE: The include1 attribute is used internally by the include1::[] macro
750 and is not for public use.
753 syntax = '%s::[%s]' % (name,args)
756 syntax = '{%s:%s}' % (name,args)
757 separator = writer.newline
758 if name not in ('eval','eval3','sys','sys2','sys3','include','include1','counter','counter2','set','set2','template'):
760 msg = 'illegal system macro name: %s' % name
762 msg = 'illegal system attribute name: %s' % name
768 message.warning('skipped %s: undefined attribute in: %s' % (name,args))
771 if name != 'include1':
772 message.verbose('evaluating: %s' % syntax)
773 if safe() and name not in ('include','include1'):
774 message.unsafe(syntax)
777 if name in ('eval','eval3'):
782 elif result is False:
784 elif result is not None:
787 message.warning('%s: evaluation error' % syntax)
788 elif name in ('sys','sys2','sys3'):
790 fd,tmp = tempfile.mkstemp()
794 cmd = cmd + (' > %s' % tmp)
798 message.warning('%s: non-zero exit status' % syntax)
800 if os.path.isfile(tmp):
801 lines = [s.rstrip() for s in open(tmp)]
805 raise EAsciiDoc,'%s: temp file read error' % syntax
806 result = separator.join(lines)
808 if os.path.isfile(tmp):
810 elif name in ('counter','counter2'):
811 mo = re.match(r'^(?P<attr>[^:]*?)(:(?P<seed>.*))?$', args)
812 attr = mo.group('attr')
813 seed = mo.group('seed')
814 if seed and (not re.match(r'^\d+$', seed) and len(seed) > 1):
815 message.warning('%s: illegal counter seed: %s' % (syntax,seed))
817 if not is_name(attr):
818 message.warning('%s: illegal attribute name' % syntax)
820 value = document.attributes.get(attr)
822 if not re.match(r'^\d+$', value) and len(value) > 1:
823 message.warning('%s: illegal counter value: %s'
826 if re.match(r'^\d+$', value):
829 expr = 'chr(ord("%s")+1)' % value
831 result = str(eval(expr))
833 message.warning('%s: evaluation error: %s' % (syntax, expr))
839 document.attributes[attr] = result
840 if attrs is not None:
842 if name == 'counter2':
844 elif name in ('set','set2'):
845 mo = re.match(r'^(?P<attr>[^:]*?)(:(?P<value>.*))?$', args)
846 attr = mo.group('attr')
847 value = mo.group('value')
850 if attr.endswith('!'):
853 if not is_name(attr):
854 message.warning('%s: illegal attribute name' % syntax)
856 if attrs is not None:
858 if name != 'set2': # set2 only updates local attributes.
859 document.attributes[attr] = value
864 elif name == 'include':
865 if not os.path.exists(args):
866 message.warning('%s: file does not exist' % syntax)
867 elif not is_safe_file(args):
868 message.unsafe(syntax)
870 result = [s.rstrip() for s in open(args)]
872 result = subs_attrs(result)
873 result = separator.join(result)
874 result = result.expandtabs(reader.tabsize)
877 elif name == 'include1':
878 result = separator.join(config.include1[args])
879 elif name == 'template':
880 if not args in config.sections:
881 message.warning('%s: template does not exist' % syntax)
884 for line in config.sections[args]:
885 line = subs_attrs(line)
888 result = '\n'.join(result)
891 if result and name in ('eval3','sys3'):
892 macros.passthroughs.append(result)
893 result = '\x07' + str(len(macros.passthroughs)-1) + '\x07'
896 def subs_attrs(lines, dictionary=None):
897 """Substitute 'lines' of text with attributes from the global
898 document.attributes dictionary and from 'dictionary' ('dictionary'
899 entries take precedence). Return a tuple of the substituted lines. 'lines'
900 containing undefined attributes are deleted. If 'lines' is a string then
903 - Attribute references are substituted in the following order: simple,
905 - Attribute references inside 'dictionary' entry values are substituted.
908 def end_brace(text,start):
909 """Return index following end brace that matches brace at start in
911 assert text[start] == '{'
914 for c in text[start:]:
915 # Skip braces that are followed by a backslash.
916 if result == len(text)-1 or text[result+1] != '\\':
917 if c == '{': n = n + 1
918 elif c == '}': n = n - 1
923 if type(lines) == str:
927 string_result = False
928 if dictionary is None:
929 attrs = document.attributes
931 # Remove numbered document attributes so they don't clash with
932 # attribute list positional attributes.
934 for k,v in document.attributes.items():
935 if not re.match(r'^\d+$', k):
937 # Substitute attribute references inside dictionary values.
938 for k,v in dictionary.items():
942 v = subs_attrs(str(v))
947 attrs.update(dictionary)
948 # Substitute all attributes in all lines.
951 # Make it easier for regular expressions.
952 line = line.replace('\\{','{\\')
953 line = line.replace('\\}','}\\')
954 # Expand simple attributes ({name}).
955 # Nested attributes not allowed.
956 reo = re.compile(r'(?su)\{(?P<name>[^\\\W][-\w]*?)\}(?!\\)')
959 mo = reo.search(line,pos)
961 s = attrs.get(mo.group('name'))
966 line = line[:mo.start()] + s + line[mo.end():]
967 pos = mo.start() + len(s)
968 # Expand conditional attributes.
969 # Single name -- higher precedence.
970 reo1 = re.compile(r'(?su)\{(?P<name>[^\\\W][-\w]*?)' \
971 r'(?P<op>\=|\?|!|#|%|@|\$)' \
972 r'(?P<value>.*?)\}(?!\\)')
973 # Multiple names (n1,n2,... or n1+n2+...) -- lower precedence.
974 reo2 = re.compile(r'(?su)\{(?P<name>[^\\\W][-\w'+OR+AND+r']*?)' \
975 r'(?P<op>\=|\?|!|#|%|@|\$)' \
976 r'(?P<value>.*?)\}(?!\\)')
977 for reo in [reo1,reo2]:
980 mo = reo.search(line,pos)
983 name = mo.group('name')
989 names = [s.strip() for s in name.split(sep) if s.strip() ]
991 if not re.match(r'^[^\\\W][-\w]*$',n):
992 message.error('illegal attribute syntax: %s' % attr)
994 # Process OR name expression: n1,n2,...
996 if attrs.get(n) is not None:
1002 # Process AND name expression: n1+n2+...
1004 if attrs.get(n) is None:
1010 lval = attrs.get(name)
1012 # mo.end() not good enough because '{x={y}}' matches '{x={y}'.
1013 end = end_brace(line,mo.start())
1014 rval = line[mo.start('value'):end-1]
1015 UNDEFINED = '{zzzzz}'
1017 if op == '=': s = rval
1018 elif op == '?': s = ''
1019 elif op == '!': s = rval
1020 elif op == '#': s = UNDEFINED # So the line is dropped.
1021 elif op == '%': s = rval
1022 elif op in ('@','$'):
1023 s = UNDEFINED # So the line is dropped.
1025 assert False, 'illegal attribute: %s' % attr
1027 if op == '=': s = lval
1028 elif op == '?': s = rval
1029 elif op == '!': s = ''
1030 elif op == '#': s = rval
1031 elif op == '%': s = UNDEFINED # So the line is dropped.
1032 elif op in ('@','$'):
1033 v = re.split(r'(?<!\\):',rval)
1034 if len(v) not in (2,3):
1035 message.error('illegal attribute syntax: %s' % attr)
1037 elif not is_re('^'+v[0]+'$'):
1038 message.error('illegal attribute regexp: %s' % attr)
1041 v = [s.replace('\\:',':') for s in v]
1042 re_mo = re.match('^'+v[0]+'$',lval)
1045 s = v[1] # {<name>@<re>:<v1>[:<v2>]}
1047 if len(v) == 3: # {<name>@<re>:<v1>:<v2>}
1049 else: # {<name>@<re>:<v1>}
1053 if len(v) == 2: # {<name>$<re>:<v1>}
1055 elif v[1] == '': # {<name>$<re>::<v2>}
1056 s = UNDEFINED # So the line is dropped.
1057 else: # {<name>$<re>:<v1>:<v2>}
1060 if len(v) == 2: # {<name>$<re>:<v1>}
1061 s = UNDEFINED # So the line is dropped.
1062 else: # {<name>$<re>:<v1>:<v2>}
1065 assert False, 'illegal attribute: %s' % attr
1067 line = line[:mo.start()] + s + line[end:]
1068 pos = mo.start() + len(s)
1069 # Drop line if it contains unsubstituted {name} references.
1070 skipped = re.search(r'(?su)\{[^\\\W][-\w]*?\}(?!\\)', line)
1072 trace('dropped line', line)
1074 # Expand system attributes (eval has precedence).
1076 re.compile(r'(?su)\{(?P<action>eval):(?P<expr>.*?)\}(?!\\)'),
1077 re.compile(r'(?su)\{(?P<action>[^\\\W][-\w]*?):(?P<expr>.*?)\}(?!\\)'),
1083 mo = reo.search(line,pos)
1085 expr = mo.group('expr')
1086 action = mo.group('action')
1087 expr = expr.replace('{\\','{')
1088 expr = expr.replace('}\\','}')
1089 s = system(action, expr, attrs=dictionary)
1090 if dictionary is not None and action in ('counter','counter2','set','set2'):
1091 # These actions create and update attributes.
1092 attrs.update(dictionary)
1094 # Drop line if the action returns None.
1097 line = line[:mo.start()] + s + line[mo.end():]
1098 pos = mo.start() + len(s)
1102 # Remove backslash from escaped entries.
1103 line = line.replace('{\\','{')
1104 line = line.replace('}\\','}')
1108 return '\n'.join(result)
1112 return tuple(result)
1114 def char_encoding():
1115 encoding = document.attributes.get('encoding')
1118 codecs.lookup(encoding)
1119 except LookupError,e:
1120 raise EAsciiDoc,str(e)
1124 return len(char_decode(s))
1126 east_asian_widths = {'W': 2, # Wide
1127 'F': 2, # Full-width (wide)
1129 'H': 1, # Half-width (narrow)
1130 'N': 1, # Neutral (not East Asian, treated as narrow)
1131 'A': 1} # Ambiguous (s/b wide in East Asian context,
1132 # narrow otherwise, but that doesn't work)
1133 """Mapping of result codes from `unicodedata.east_asian_width()` to character
1136 def column_width(s):
1137 text = char_decode(s)
1138 if isinstance(text, unicode):
1141 width += east_asian_widths[unicodedata.east_asian_width(c)]
1149 return s.decode(char_encoding())
1152 "'%s' codec can't decode \"%s\"" % (char_encoding(), s)
1158 return s.encode(char_encoding())
1163 """Convert seconds since the Epoch to formatted local time string."""
1164 t = time.localtime(t)
1165 s = time.strftime('%H:%M:%S',t)
1166 if time.daylight and t.tm_isdst == 1:
1167 result = s + ' ' + time.tzname[1]
1169 result = s + ' ' + time.tzname[0]
1170 # Attempt to convert the localtime to the output encoding.
1172 result = char_encode(result.decode(locale.getdefaultlocale()[1]))
1178 """Convert seconds since the Epoch to formatted local date string."""
1179 t = time.localtime(t)
1180 return time.strftime('%Y-%m-%d',t)
1184 """Lexical analysis routines. Static methods and attributes only."""
1188 raise AssertionError,'no class instances allowed'
1191 """Returns class of next element on the input (None if EOF). The
1192 reader is assumed to be at the first line following a previous element,
1193 end of file or line one. Exits with the reader pointing to the first
1194 line of the next element or EOF (leading blank lines are skipped)."""
1195 reader.skip_blank_lines()
1196 if reader.eof(): return None
1197 # Optimization: If we've already checked for an element at this
1198 # position return the element.
1199 if Lex.prev_element and Lex.prev_cursor == reader.cursor:
1200 return Lex.prev_element
1201 if AttributeEntry.isnext():
1202 result = AttributeEntry
1203 elif AttributeList.isnext():
1204 result = AttributeList
1205 elif BlockTitle.isnext() and not tables_OLD.isnext():
1207 elif Title.isnext():
1208 if AttributeList.style() == 'float':
1209 result = FloatingTitle
1212 elif macros.isnext():
1213 result = macros.current
1214 elif lists.isnext():
1215 result = lists.current
1216 elif blocks.isnext():
1217 result = blocks.current
1218 elif tables_OLD.isnext():
1219 result = tables_OLD.current
1220 elif tables.isnext():
1221 result = tables.current
1223 if not paragraphs.isnext():
1224 raise EAsciiDoc,'paragraph expected'
1225 result = paragraphs.current
1226 # Optimization: Cache answer.
1227 Lex.prev_cursor = reader.cursor
1228 Lex.prev_element = result
1232 def canonical_subs(options):
1233 """Translate composite subs values."""
1234 if len(options) == 1:
1235 if options[0] == 'none':
1237 elif options[0] == 'normal':
1238 options = config.subsnormal
1239 elif options[0] == 'verbatim':
1240 options = config.subsverbatim
1244 def subs_1(s,options):
1245 """Perform substitution specified in 'options' (in 'options' order)."""
1248 if document.attributes.get('plaintext') is not None:
1249 options = ('specialcharacters',)
1251 options = Lex.canonical_subs(options)
1253 if o == 'specialcharacters':
1254 result = config.subs_specialchars(result)
1255 elif o == 'attributes':
1256 result = subs_attrs(result)
1258 result = subs_quotes(result)
1259 elif o == 'specialwords':
1260 result = config.subs_specialwords(result)
1261 elif o in ('replacements','replacements2'):
1262 result = config.subs_replacements(result,o)
1264 result = macros.subs(result)
1265 elif o == 'callouts':
1266 result = macros.subs(result,callouts=True)
1268 raise EAsciiDoc,'illegal substitution option: %s' % o
1275 def subs(lines,options):
1276 """Perform inline processing specified by 'options' (in 'options'
1277 order) on sequence of 'lines'."""
1278 if not lines or not options:
1280 options = Lex.canonical_subs(options)
1281 # Join lines so quoting can span multiple lines.
1282 para = '\n'.join(lines)
1283 if 'macros' in options:
1284 para = macros.extract_passthroughs(para)
1286 if o == 'attributes':
1287 # If we don't substitute attributes line-by-line then a single
1288 # undefined attribute will drop the entire paragraph.
1289 lines = subs_attrs(para.split('\n'))
1290 para = '\n'.join(lines)
1292 para = Lex.subs_1(para,(o,))
1293 if 'macros' in options:
1294 para = macros.restore_passthroughs(para)
1295 return para.splitlines()
1298 def set_margin(lines, margin=0):
1299 """Utility routine that sets the left margin to 'margin' space in a
1300 block of non-blank lines."""
1301 # Calculate width of block margin.
1303 width = len(lines[0])
1305 i = re.search(r'\S',s).start()
1306 if i < width: width = i
1307 # Strip margin width from all lines.
1308 for i in range(len(lines)):
1309 lines[i] = ' '*margin + lines[i][width:]
1312 #---------------------------------------------------------------------------
1313 # Document element classes parse AsciiDoc reader input and write DocBook writer
1315 #---------------------------------------------------------------------------
1316 class Document(object):
1319 def getdoctype(self):
1320 return self.attributes.get('doctype')
1321 def setdoctype(self,doctype):
1322 self.attributes['doctype'] = doctype
1323 doctype = property(getdoctype,setdoctype)
1326 def getbackend(self):
1327 return self.attributes.get('backend')
1328 def setbackend(self,backend):
1330 backend = self.attributes.get('backend-alias-' + backend, backend)
1331 self.attributes['backend'] = backend
1332 backend = property(getbackend,setbackend)
1335 self.infile = None # Source file name.
1336 self.outfile = None # Output file name.
1337 self.attributes = InsensitiveDict()
1338 self.level = 0 # 0 => front matter. 1,2,3 => sect1,2,3.
1339 self.has_errors = False # Set true if processing errors were flagged.
1340 self.has_warnings = False # Set true if warnings were flagged.
1341 self.safe = False # Default safe mode.
1342 def update_attributes(self,attrs=None):
1344 Set implicit attributes and attributes in 'attrs'.
1347 self.attributes['localtime'] = time_str(t)
1348 self.attributes['localdate'] = date_str(t)
1349 self.attributes['asciidoc-version'] = VERSION
1350 self.attributes['asciidoc-file'] = APP_FILE
1351 self.attributes['asciidoc-dir'] = APP_DIR
1352 self.attributes['asciidoc-confdir'] = CONF_DIR
1353 self.attributes['user-dir'] = USER_DIR
1355 self.attributes['verbose'] = ''
1356 # Update with configuration file attributes.
1358 self.attributes.update(attrs)
1359 # Update with command-line attributes.
1360 self.attributes.update(config.cmd_attrs)
1361 # Extract miscellaneous configuration section entries from attributes.
1363 config.load_miscellaneous(attrs)
1364 config.load_miscellaneous(config.cmd_attrs)
1365 self.attributes['newline'] = config.newline
1366 # File name related attributes can't be overridden.
1367 if self.infile is not None:
1368 if self.infile and os.path.exists(self.infile):
1369 t = os.path.getmtime(self.infile)
1370 elif self.infile == '<stdin>':
1375 self.attributes['doctime'] = time_str(t)
1376 self.attributes['docdate'] = date_str(t)
1377 if self.infile != '<stdin>':
1378 self.attributes['infile'] = self.infile
1379 self.attributes['indir'] = os.path.dirname(self.infile)
1380 self.attributes['docfile'] = self.infile
1381 self.attributes['docdir'] = os.path.dirname(self.infile)
1382 self.attributes['docname'] = os.path.splitext(
1383 os.path.basename(self.infile))[0]
1385 if self.outfile != '<stdout>':
1386 self.attributes['outfile'] = self.outfile
1387 self.attributes['outdir'] = os.path.dirname(self.outfile)
1388 if self.infile == '<stdin>':
1389 self.attributes['docname'] = os.path.splitext(
1390 os.path.basename(self.outfile))[0]
1391 ext = os.path.splitext(self.outfile)[1][1:]
1392 elif config.outfilesuffix:
1393 ext = config.outfilesuffix[1:]
1397 self.attributes['filetype'] = ext
1398 self.attributes['filetype-'+ext] = ''
1399 def load_lang(self):
1401 Load language configuration file.
1403 lang = self.attributes.get('lang')
1405 filename = 'lang-en.conf' # Default language file.
1407 filename = 'lang-' + lang + '.conf'
1408 if config.load_from_dirs(filename):
1409 self.attributes['lang'] = lang # Reinstate new lang attribute.
1412 # The default language file must exist.
1413 message.error('missing conf file: %s' % filename, halt=True)
1415 message.warning('missing language conf file: %s' % filename)
1416 def set_deprecated_attribute(self,old,new):
1418 Ensures the 'old' name of an attribute that was renamed to 'new' is
1421 if self.attributes.get(new) is None:
1422 if self.attributes.get(old) is not None:
1423 self.attributes[new] = self.attributes[old]
1425 self.attributes[old] = self.attributes[new]
1426 def consume_attributes_and_comments(self,comments_only=False,noblanks=False):
1428 Returns True if one or more attributes or comments were consumed.
1429 If 'noblanks' is True then consumation halts if a blank line is
1436 if noblanks and not reader.read_next(): return result
1437 if blocks.isnext() and 'skip' in blocks.current.options:
1440 blocks.current.translate()
1441 if noblanks and not reader.read_next(): return result
1442 if macros.isnext() and macros.current.name == 'comment':
1445 macros.current.translate()
1446 if not comments_only:
1447 if AttributeEntry.isnext():
1450 AttributeEntry.translate()
1451 if AttributeList.isnext():
1454 AttributeList.translate()
1456 def parse_header(self,doctype,backend):
1458 Parses header, sets corresponding document attributes and finalizes
1459 document doctype and backend properties.
1460 Returns False if the document does not have a header.
1461 'doctype' and 'backend' are the doctype and backend option values
1462 passed on the command-line, None if no command-line option was not
1465 assert self.level == 0
1466 # Skip comments and attribute entries that preceed the header.
1467 self.consume_attributes_and_comments()
1468 if doctype is not None:
1469 # Command-line overrides header.
1470 self.doctype = doctype
1471 elif self.doctype is None:
1472 # Was not set on command-line or in document header.
1473 self.doctype = DEFAULT_DOCTYPE
1474 # Process document header.
1475 has_header = (Title.isnext() and Title.level == 0
1476 and AttributeList.style() != 'float')
1477 if self.doctype == 'manpage' and not has_header:
1478 message.error('manpage document title is mandatory',halt=True)
1481 # Command-line entries override header derived entries.
1482 self.attributes.update(config.cmd_attrs)
1483 # DEPRECATED: revision renamed to revnumber.
1484 self.set_deprecated_attribute('revision','revnumber')
1485 # DEPRECATED: date renamed to revdate.
1486 self.set_deprecated_attribute('date','revdate')
1487 if doctype is not None:
1488 # Command-line overrides header.
1489 self.doctype = doctype
1490 if backend is not None:
1491 # Command-line overrides header.
1492 self.backend = backend
1493 elif self.backend is None:
1494 # Was not set on command-line or in document header.
1495 self.backend = DEFAULT_BACKEND
1497 # Has been set in document header.
1498 self.backend = self.backend # Translate alias in header.
1499 assert self.doctype in ('article','manpage','book'), 'illegal document type'
1501 def translate(self,has_header):
1502 if self.doctype == 'manpage':
1503 # Translate mandatory NAME section.
1504 if Lex.next() is not Title:
1505 message.error('name section expected')
1508 if Title.level != 1:
1509 message.error('name section title must be at level 1')
1510 if not isinstance(Lex.next(),Paragraph):
1511 message.error('malformed name section body')
1512 lines = reader.read_until(r'^$')
1514 mo = re.match(r'^(?P<manname>.*?)\s+-\s+(?P<manpurpose>.*)$',s)
1516 message.error('malformed name section body')
1517 self.attributes['manname'] = mo.group('manname').strip()
1518 self.attributes['manpurpose'] = mo.group('manpurpose').strip()
1519 names = [s.strip() for s in self.attributes['manname'].split(',')]
1521 message.warning('to many manpage names')
1522 for i,name in enumerate(names):
1523 self.attributes['manname%d' % (i+1)] = name
1525 # Do postponed substitutions (backend confs have been loaded).
1526 self.attributes['doctitle'] = Title.dosubs(self.attributes['doctitle'])
1527 if config.header_footer:
1528 hdr = config.subs_section('header',{})
1529 writer.write(hdr,trace='header')
1530 if 'title' in self.attributes:
1531 del self.attributes['title']
1532 self.consume_attributes_and_comments()
1533 if self.doctype in ('article','book'):
1534 # Translate 'preamble' (untitled elements between header
1535 # and first section title).
1536 if Lex.next() is not Title:
1537 stag,etag = config.section2tags('preamble')
1538 writer.write(stag,trace='preamble open')
1539 Section.translate_body()
1540 writer.write(etag,trace='preamble close')
1541 elif self.doctype == 'manpage' and 'name' in config.sections:
1542 writer.write(config.subs_section('name',{}), trace='name')
1544 self.process_author_names()
1545 if config.header_footer:
1546 hdr = config.subs_section('header',{})
1547 writer.write(hdr,trace='header')
1548 if Lex.next() is not Title:
1549 Section.translate_body()
1550 # Process remaining sections.
1551 while not reader.eof():
1552 if Lex.next() is not Title:
1553 raise EAsciiDoc,'section title expected'
1555 Section.setlevel(0) # Write remaining unwritten section close tags.
1556 # Substitute document parameters and write document footer.
1557 if config.header_footer:
1558 ftr = config.subs_section('footer',{})
1559 writer.write(ftr,trace='footer')
1560 def parse_author(self,s):
1561 """ Return False if the author is malformed."""
1562 attrs = self.attributes # Alias for readability.
1564 mo = re.match(r'^(?P<name1>[^<>\s]+)'
1565 '(\s+(?P<name2>[^<>\s]+))?'
1566 '(\s+(?P<name3>[^<>\s]+))?'
1567 '(\s+<(?P<email>\S+)>)?$',s)
1569 # Names that don't match the formal specification.
1571 attrs['firstname'] = s
1573 firstname = mo.group('name1')
1574 if mo.group('name3'):
1575 middlename = mo.group('name2')
1576 lastname = mo.group('name3')
1579 lastname = mo.group('name2')
1580 firstname = firstname.replace('_',' ')
1582 middlename = middlename.replace('_',' ')
1584 lastname = lastname.replace('_',' ')
1585 email = mo.group('email')
1587 attrs['firstname'] = firstname
1589 attrs['middlename'] = middlename
1591 attrs['lastname'] = lastname
1593 attrs['email'] = email
1595 def process_author_names(self):
1596 """ Calculate any missing author related attributes."""
1597 attrs = self.attributes # Alias for readability.
1598 firstname = attrs.get('firstname','')
1599 middlename = attrs.get('middlename','')
1600 lastname = attrs.get('lastname','')
1601 author = attrs.get('author')
1602 initials = attrs.get('authorinitials')
1603 if author and not (firstname or middlename or lastname):
1604 self.parse_author(author)
1605 attrs['author'] = author.replace('_',' ')
1606 self.process_author_names()
1609 author = '%s %s %s' % (firstname, middlename, lastname)
1610 author = author.strip()
1611 author = re.sub(r'\s+',' ', author)
1613 initials = (char_decode(firstname)[:1] +
1614 char_decode(middlename)[:1] + char_decode(lastname)[:1])
1615 initials = char_encode(initials).upper()
1616 names = [firstname,middlename,lastname,author,initials]
1617 for i,v in enumerate(names):
1618 v = config.subs_specialchars(v)
1621 firstname,middlename,lastname,author,initials = names
1623 attrs['firstname'] = firstname
1625 attrs['middlename'] = middlename
1627 attrs['lastname'] = lastname
1629 attrs['author'] = author
1631 attrs['authorinitials'] = initials
1633 attrs['authored'] = ''
1637 """Static methods and attributes only."""
1638 REV_LINE_RE = r'^(\D*(?P<revnumber>.*?),)?(?P<revdate>.*?)(:\s*(?P<revremark>.*))?$'
1639 RCS_ID_RE = r'^\$Id: \S+ (?P<revnumber>\S+) (?P<revdate>\S+) \S+ (?P<author>\S+) (\S+ )?\$$'
1641 raise AssertionError,'no class instances allowed'
1644 assert Lex.next() is Title and Title.level == 0
1645 attrs = document.attributes # Alias for readability.
1646 # Postpone title subs until backend conf files have been loaded.
1647 Title.translate(skipsubs=True)
1648 attrs['doctitle'] = Title.attributes['title']
1649 document.consume_attributes_and_comments(noblanks=True)
1650 s = reader.read_next()
1653 # Process first header line after the title that is not a comment
1654 # or an attribute entry.
1656 mo = re.match(Header.RCS_ID_RE,s)
1658 document.parse_author(s)
1659 document.consume_attributes_and_comments(noblanks=True)
1660 if reader.read_next():
1661 # Process second header line after the title that is not a
1662 # comment or an attribute entry.
1666 mo = re.match(Header.RCS_ID_RE,s)
1668 mo = re.match(Header.REV_LINE_RE,s)
1669 document.consume_attributes_and_comments(noblanks=True)
1670 s = attrs.get('revnumber')
1672 mo = re.match(Header.RCS_ID_RE,s)
1674 revnumber = mo.group('revnumber')
1676 attrs['revnumber'] = revnumber.strip()
1677 author = mo.groupdict().get('author')
1678 if author and 'firstname' not in attrs:
1679 document.parse_author(author)
1680 revremark = mo.groupdict().get('revremark')
1681 if revremark is not None:
1682 revremark = [revremark]
1683 # Revision remarks can continue on following lines.
1684 while reader.read_next():
1685 if document.consume_attributes_and_comments(noblanks=True):
1687 revremark.append(reader.read())
1688 revremark = Lex.subs(revremark,['normal'])
1689 revremark = '\n'.join(revremark).strip()
1690 attrs['revremark'] = revremark
1691 revdate = mo.group('revdate')
1693 attrs['revdate'] = revdate.strip()
1694 elif revnumber or revremark:
1695 # Set revision date to ensure valid DocBook revision.
1696 attrs['revdate'] = attrs['docdate']
1697 document.process_author_names()
1698 if document.doctype == 'manpage':
1699 # manpage title formatted like mantitle(manvolnum).
1700 mo = re.match(r'^(?P<mantitle>.*)\((?P<manvolnum>.*)\)$',
1703 message.error('malformed manpage title')
1705 mantitle = mo.group('mantitle').strip()
1706 mantitle = subs_attrs(mantitle)
1707 if mantitle is None:
1708 message.error('undefined attribute in manpage title')
1709 # mantitle is lowered only if in ALL CAPS
1710 if mantitle == mantitle.upper():
1711 mantitle = mantitle.lower()
1712 attrs['mantitle'] = mantitle;
1713 attrs['manvolnum'] = mo.group('manvolnum').strip()
1715 class AttributeEntry:
1716 """Static methods and attributes only."""
1722 attributes = {} # Accumulates all the parsed attribute entries.
1724 raise AssertionError,'no class instances allowed'
1727 result = False # Assume not next.
1728 if not AttributeEntry.pattern:
1729 pat = document.attributes.get('attributeentry-pattern')
1731 message.error("[attributes] missing 'attributeentry-pattern' entry")
1732 AttributeEntry.pattern = pat
1733 line = reader.read_next()
1735 # Attribute entry formatted like :<name>[.<name2>]:[ <value>]
1736 mo = re.match(AttributeEntry.pattern,line)
1738 AttributeEntry.name = mo.group('attrname')
1739 AttributeEntry.name2 = mo.group('attrname2')
1740 AttributeEntry.value = mo.group('attrvalue') or ''
1741 AttributeEntry.value = AttributeEntry.value.strip()
1746 assert Lex.next() is AttributeEntry
1747 attr = AttributeEntry # Alias for brevity.
1748 reader.read() # Discard attribute entry from reader.
1749 while attr.value.endswith(' +'):
1750 if not reader.read_next(): break
1751 attr.value = attr.value[:-1] + reader.read().strip()
1752 if attr.name2 is not None:
1753 # Configuration file attribute.
1754 if attr.name2 != '':
1755 # Section entry attribute.
1757 # Some sections can have name! syntax.
1758 if attr.name in ('attributes','miscellaneous') and attr.name2[-1] == '!':
1759 section[attr.name] = [attr.name2]
1761 section[attr.name] = ['%s=%s' % (attr.name2,attr.value)]
1762 config.load_sections(section)
1763 config.load_miscellaneous(config.conf_attrs)
1765 # Markup template section attribute.
1766 if attr.name in config.sections:
1767 config.sections[attr.name] = [attr.value]
1769 message.warning('missing configuration section: %s' % attr.name)
1772 if attr.name[-1] == '!':
1773 # Names like name! undefine the attribute.
1774 attr.name = attr.name[:-1]
1776 # Strip white space and illegal name chars.
1777 attr.name = re.sub(r'(?u)[^\w\-_]', '', attr.name).lower()
1778 # Don't override most command-line attributes.
1779 if attr.name in config.cmd_attrs \
1780 and attr.name not in ('trace','numbered'):
1782 # Update document attributes with attribute value.
1783 if attr.value is not None:
1784 mo = re.match(r'^pass:(?P<attrs>.*)\[(?P<value>.*)\]$', attr.value)
1786 # Inline passthrough syntax.
1787 attr.subs = mo.group('attrs')
1788 attr.value = mo.group('value') # Passthrough.
1790 # Default substitution.
1791 # DEPRECATED: attributeentry-subs
1792 attr.subs = document.attributes.get('attributeentry-subs',
1793 'specialcharacters,attributes')
1794 attr.subs = parse_options(attr.subs, SUBS_OPTIONS,
1795 'illegal substitution option')
1796 attr.value = Lex.subs((attr.value,), attr.subs)
1797 attr.value = writer.newline.join(attr.value)
1798 document.attributes[attr.name] = attr.value
1799 elif attr.name in document.attributes:
1800 del document.attributes[attr.name]
1801 attr.attributes[attr.name] = attr.value
1803 class AttributeList:
1804 """Static methods and attributes only."""
1809 raise AssertionError,'no class instances allowed'
1812 if not 'attributelist-pattern' in document.attributes:
1813 message.error("[attributes] missing 'attributelist-pattern' entry")
1814 AttributeList.pattern = document.attributes['attributelist-pattern']
1817 result = False # Assume not next.
1818 line = reader.read_next()
1820 mo = re.match(AttributeList.pattern, line)
1822 AttributeList.match = mo
1827 assert Lex.next() is AttributeList
1828 reader.read() # Discard attribute list from reader.
1830 d = AttributeList.match.groupdict()
1831 for k,v in d.items():
1836 parse_attributes(v, attrs)
1838 AttributeList.attrs[k] = v
1839 AttributeList.subs(attrs)
1840 AttributeList.attrs.update(attrs)
1843 '''Substitute single quoted attribute values normally.'''
1844 reo = re.compile(r"^'.*'$")
1845 for k,v in attrs.items():
1846 if reo.match(str(v)):
1847 attrs[k] = Lex.subs_1(v[1:-1],SUBS_NORMAL)
1850 return AttributeList.attrs.get('style') or AttributeList.attrs.get('1')
1853 """Add attribute list to the dictionary 'd' and reset the
1855 if AttributeList.attrs:
1856 d.update(AttributeList.attrs)
1857 AttributeList.attrs = {}
1858 # Generate option attributes.
1860 options = parse_options(d['options'], (), 'illegal option name')
1861 for option in options:
1862 d[option+'-option'] = ''
1865 """Static methods and attributes only."""
1869 raise AssertionError,'no class instances allowed'
1872 result = False # Assume not next.
1873 line = reader.read_next()
1875 mo = re.match(BlockTitle.pattern,line)
1877 BlockTitle.title = mo.group('title')
1882 assert Lex.next() is BlockTitle
1883 reader.read() # Discard title from reader.
1884 # Perform title substitutions.
1886 Title.subs = config.subsnormal
1887 s = Lex.subs((BlockTitle.title,), Title.subs)
1888 s = writer.newline.join(s)
1890 message.warning('blank block title')
1891 BlockTitle.title = s
1894 """If there is a title add it to dictionary 'd' then reset title."""
1895 if BlockTitle.title:
1896 d['title'] = BlockTitle.title
1897 BlockTitle.title = None
1900 """Processes Header and Section titles. Static methods and attributes
1903 underlines = ('==','--','~~','^^','++') # Levels 0,1,2,3,4.
1909 section_numbers = [0]*len(underlines)
1911 linecount = None # Number of lines in title (1 or 2).
1913 raise AssertionError,'no class instances allowed'
1915 def translate(skipsubs=False):
1916 """Parse the Title.attributes and Title.level from the reader. The
1917 real work has already been done by parse()."""
1918 assert Lex.next() in (Title,FloatingTitle)
1919 # Discard title from reader.
1920 for i in range(Title.linecount):
1924 Title.attributes['title'] = Title.dosubs(Title.attributes['title'])
1928 Perform title substitutions.
1931 Title.subs = config.subsnormal
1932 title = Lex.subs((title,), Title.subs)
1933 title = writer.newline.join(title)
1935 message.warning('blank section title')
1939 lines = reader.read_ahead(2)
1940 return Title.parse(lines)
1943 """Parse title at start of lines tuple."""
1944 if len(lines) == 0: return False
1945 if len(lines[0]) == 0: return False # Title can't be blank.
1946 # Check for single-line titles.
1948 for level in range(len(Title.underlines)):
1949 k = 'sect%s' % level
1950 if k in Title.dump_dict:
1951 mo = re.match(Title.dump_dict[k], lines[0])
1953 Title.attributes = mo.groupdict()
1959 # Check for double-line titles.
1960 if not Title.pattern: return False # Single-line titles only.
1961 if len(lines) < 2: return False
1962 title,ul = lines[:2]
1963 title_len = column_width(title)
1964 ul_len = char_len(ul)
1965 if ul_len < 2: return False
1966 # Fast elimination check.
1967 if ul[:2] not in Title.underlines: return False
1968 # Length of underline must be within +-3 of title.
1969 if not ((ul_len-3 < title_len < ul_len+3)
1970 # Next test for backward compatibility.
1971 or (ul_len-3 < char_len(title) < ul_len+3)):
1973 # Check for valid repetition of underline character pairs.
1974 s = ul[:2]*((ul_len+1)/2)
1975 if ul != s[:ul_len]: return False
1976 # Don't be fooled by back-to-back delimited blocks, require at
1977 # least one alphanumeric character in title.
1978 if not re.search(r'(?u)\w',title): return False
1979 mo = re.match(Title.pattern, title)
1981 Title.attributes = mo.groupdict()
1982 Title.level = list(Title.underlines).index(ul[:2])
1985 # Check for expected pattern match groups.
1987 if not 'title' in Title.attributes:
1988 message.warning('[titles] entry has no <title> group')
1989 Title.attributes['title'] = lines[0]
1990 for k,v in Title.attributes.items():
1991 if v is None: del Title.attributes[k]
1993 Title.level += int(document.attributes.get('leveloffset','0'))
1996 Title.attributes['level'] = str(Title.level)
2000 """Load and validate [titles] section entries dictionary."""
2001 if 'underlines' in entries:
2002 errmsg = 'malformed [titles] underlines entry'
2004 underlines = parse_list(entries['underlines'])
2006 raise EAsciiDoc,errmsg
2007 if len(underlines) != len(Title.underlines):
2008 raise EAsciiDoc,errmsg
2009 for s in underlines:
2011 raise EAsciiDoc,errmsg
2012 Title.underlines = tuple(underlines)
2013 Title.dump_dict['underlines'] = entries['underlines']
2014 if 'subs' in entries:
2015 Title.subs = parse_options(entries['subs'], SUBS_OPTIONS,
2016 'illegal [titles] subs entry')
2017 Title.dump_dict['subs'] = entries['subs']
2018 if 'sectiontitle' in entries:
2019 pat = entries['sectiontitle']
2020 if not pat or not is_re(pat):
2021 raise EAsciiDoc,'malformed [titles] sectiontitle entry'
2023 Title.dump_dict['sectiontitle'] = pat
2024 if 'blocktitle' in entries:
2025 pat = entries['blocktitle']
2026 if not pat or not is_re(pat):
2027 raise EAsciiDoc,'malformed [titles] blocktitle entry'
2028 BlockTitle.pattern = pat
2029 Title.dump_dict['blocktitle'] = pat
2030 # Load single-line title patterns.
2031 for k in ('sect0','sect1','sect2','sect3','sect4'):
2034 if not pat or not is_re(pat):
2035 raise EAsciiDoc,'malformed [titles] %s entry' % k
2036 Title.dump_dict[k] = pat
2037 # TODO: Check we have either a Title.pattern or at least one
2038 # single-line title pattern -- can this be done here or do we need
2039 # check routine like the other block checkers?
2042 dump_section('titles',Title.dump_dict)
2046 Set Title section name:
2047 If the first positional or 'template' attribute is set use it,
2048 next search for section title in [specialsections],
2049 if not found use default 'sect<level>' name.
2051 sectname = AttributeList.attrs.get('1')
2052 if sectname and sectname != 'float':
2053 Title.sectname = sectname
2054 elif 'template' in AttributeList.attrs:
2055 Title.sectname = AttributeList.attrs['template']
2057 for pat,sect in config.specialsections.items():
2058 mo = re.match(pat,Title.attributes['title'])
2060 title = mo.groupdict().get('title')
2061 if title is not None:
2062 Title.attributes['title'] = title.strip()
2064 Title.attributes['title'] = mo.group().strip()
2065 Title.sectname = sect
2068 Title.sectname = 'sect%d' % Title.level
2070 def getnumber(level):
2071 """Return next section number at section 'level' formatted like
2074 for l in range(len(Title.section_numbers)):
2075 n = Title.section_numbers[l]
2079 number = '%s%d.' % (number, n)
2081 number = '%s%d.' % (number, n + 1)
2082 Title.section_numbers[l] = n + 1
2084 # Reset unprocessed section levels.
2085 Title.section_numbers[l] = 0
2089 class FloatingTitle(Title):
2090 '''Floated titles are translated differently.'''
2093 return Title.isnext() and AttributeList.style() == 'float'
2096 assert Lex.next() is FloatingTitle
2099 AttributeList.consume(Title.attributes)
2100 template = 'floatingtitle'
2101 if template in config.sections:
2102 stag,etag = config.section2tags(template,Title.attributes)
2103 writer.write(stag,trace='floating title')
2105 message.warning('missing template section: [%s]' % template)
2109 """Static methods and attributes only."""
2110 endtags = [] # Stack of currently open section (level,endtag) tuples.
2111 ids = [] # List of already used ids.
2113 raise AssertionError,'no class instances allowed'
2115 def savetag(level,etag):
2116 """Save section end."""
2117 Section.endtags.append((level,etag))
2119 def setlevel(level):
2120 """Set document level and write open section close tags up to level."""
2121 while Section.endtags and Section.endtags[-1][0] >= level:
2122 writer.write(Section.endtags.pop()[1],trace='section close')
2123 document.level = level
2127 The normalized value of the id attribute is an NCName according to
2128 the 'Namespaces in XML' Recommendation:
2129 NCName ::= NCNameStartChar NCNameChar*
2130 NCNameChar ::= NameChar - ':'
2131 NCNameStartChar ::= Letter | '_'
2132 NameChar ::= Letter | Digit | '.' | '-' | '_' | ':'
2134 # Replace non-alpha numeric characters in title with underscores and
2135 # convert to lower case.
2136 base_ident = char_encode(re.sub(r'(?u)\W+', '_',
2137 char_decode(title)).strip('_').lower())
2138 # Prefix the ID name with idprefix attribute or underscore if not
2139 # defined. Prefix ensures the ID does not clash with existing IDs.
2140 idprefix = document.attributes.get('idprefix','_')
2141 base_ident = idprefix + base_ident
2147 ident = '%s_%d' % (base_ident, i)
2148 if ident not in Section.ids:
2149 Section.ids.append(ident)
2156 if not document.attributes.get('sectids') is None \
2157 and 'id' not in AttributeList.attrs:
2158 # Generate ids for sections.
2159 AttributeList.attrs['id'] = Section.gen_id(Title.attributes['title'])
2162 assert Lex.next() is Title
2163 prev_sectname = Title.sectname
2165 if Title.level == 0 and document.doctype != 'book':
2166 message.error('only book doctypes can contain level 0 sections')
2167 if Title.level > document.level \
2168 and 'basebackend-docbook' in document.attributes \
2169 and prev_sectname in ('colophon','abstract', \
2170 'dedication','glossary','bibliography'):
2171 message.error('%s section cannot contain sub-sections' % prev_sectname)
2172 if Title.level > document.level+1:
2173 # Sub-sections of multi-part book level zero Preface and Appendices
2174 # are meant to be out of sequence.
2175 if document.doctype == 'book' \
2176 and document.level == 0 \
2177 and Title.level == 2 \
2178 and prev_sectname in ('preface','appendix'):
2181 message.warning('section title out of sequence: '
2182 'expected level %d, got level %d'
2183 % (document.level+1, Title.level))
2185 Section.setlevel(Title.level)
2186 if 'numbered' in document.attributes:
2187 Title.attributes['sectnum'] = Title.getnumber(document.level)
2189 Title.attributes['sectnum'] = ''
2190 AttributeList.consume(Title.attributes)
2191 stag,etag = config.section2tags(Title.sectname,Title.attributes)
2192 Section.savetag(Title.level,etag)
2193 writer.write(stag,trace='section open: level %d: %s' %
2194 (Title.level, Title.attributes['title']))
2195 Section.translate_body()
2197 def translate_body(terminator=Title):
2200 while next and next is not terminator:
2201 if isinstance(terminator,DelimitedBlock) and next is Title:
2202 message.error('section title not permitted in delimited block')
2206 # The section is not empty if contains a subsection.
2207 if next and isempty and Title.level > document.level:
2209 # Report empty sections if invalid markup will result.
2211 if document.backend == 'docbook' and Title.sectname != 'index':
2212 message.error('empty section is not valid')
2214 class AbstractBlock:
2216 # Configuration parameter names common to all blocks.
2217 self.CONF_ENTRIES = ('delimiter','options','subs','presubs','postsubs',
2218 'posattrs','style','.*-style','template','filter')
2219 self.start = None # File reader cursor at start delimiter.
2220 self.name=None # Configuration file section name.
2221 # Configuration parameters.
2222 self.delimiter=None # Regular expression matching block delimiter.
2223 self.delimiter_reo=None # Compiled delimiter.
2224 self.template=None # template section entry.
2225 self.options=() # options entry list.
2226 self.presubs=None # presubs/subs entry list.
2227 self.postsubs=() # postsubs entry list.
2228 self.filter=None # filter entry.
2229 self.posattrs=() # posattrs entry list.
2230 self.style=None # Default style.
2231 self.styles=OrderedDict() # Each entry is a styles dictionary.
2232 # Before a block is processed it's attributes (from it's
2233 # attributes list) are merged with the block configuration parameters
2234 # (by self.merge_attributes()) resulting in the template substitution
2235 # dictionary (self.attributes) and the block's processing parameters
2236 # (self.parameters).
2238 # The names of block parameters.
2239 self.PARAM_NAMES=('template','options','presubs','postsubs','filter')
2240 self.parameters=None
2241 # Leading delimiter match object.
2243 def short_name(self):
2244 """ Return the text following the last dash in the section name."""
2245 i = self.name.rfind('-')
2249 return self.name[i+1:]
2250 def error(self, msg, cursor=None, halt=False):
2251 message.error('[%s] %s' % (self.name,msg), cursor, halt)
2252 def is_conf_entry(self,param):
2253 """Return True if param matches an allowed configuration file entry
2255 for s in self.CONF_ENTRIES:
2256 if re.match('^'+s+'$',param):
2259 def load(self,name,entries):
2260 """Update block definition from section 'entries' dictionary."""
2262 self.update_parameters(entries, self, all=True)
2263 def update_parameters(self, src, dst=None, all=False):
2265 Parse processing parameters from src dictionary to dst object.
2266 dst defaults to self.parameters.
2267 If all is True then copy src entries that aren't parameter names.
2269 dst = dst or self.parameters
2270 msg = '[%s] malformed entry %%s: %%s' % self.name
2272 if isinstance(obj,dict):
2276 for k,v in src.items():
2277 if not re.match(r'\d+',k) and not is_name(k):
2278 raise EAsciiDoc, msg % (k,v)
2281 raise EAsciiDoc, msg % (k,v)
2285 elif k == 'options':
2286 if isinstance(v,str):
2287 v = parse_options(v, (), msg % (k,v))
2288 # Merge with existing options.
2289 v = tuple(set(dst.options).union(set(v)))
2291 elif k in ('subs','presubs','postsubs'):
2292 # Subs is an alias for presubs.
2293 if k == 'subs': k = 'presubs'
2294 if isinstance(v,str):
2295 v = parse_options(v, SUBS_OPTIONS, msg % (k,v))
2297 elif k == 'delimiter':
2301 raise EAsciiDoc, msg % (k,v)
2306 raise EAsciiDoc, msg % (k,v)
2307 elif k == 'posattrs':
2308 v = parse_options(v, (), msg % (k,v))
2311 mo = re.match(r'^(?P<style>.*)-style$',k)
2314 raise EAsciiDoc, msg % (k,v)
2315 style = mo.group('style')
2316 if not is_name(style):
2317 raise EAsciiDoc, msg % (k,v)
2319 if not parse_named_attributes(v,d):
2320 raise EAsciiDoc, msg % (k,v)
2322 # Subs is an alias for presubs.
2323 d['presubs'] = d['subs']
2325 self.styles[style] = d
2326 elif all or k in self.PARAM_NAMES:
2327 copy(dst,k,v) # Derived class specific entries.
2328 def get_param(self,name,params=None):
2330 Return named processing parameter from params dictionary.
2331 If the parameter is not in params look in self.parameters.
2333 if params and name in params:
2335 elif name in self.parameters:
2336 return self.parameters[name]
2339 def get_subs(self,params=None):
2341 Return (presubs,postsubs) tuple.
2343 presubs = self.get_param('presubs',params)
2344 postsubs = self.get_param('postsubs',params)
2345 return (presubs,postsubs)
2347 """Write block definition to stdout."""
2348 write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
2349 write('['+self.name+']')
2350 if self.is_conf_entry('delimiter'):
2351 write('delimiter='+self.delimiter)
2353 write('template='+self.template)
2355 write('options='+','.join(self.options))
2358 write('presubs='+','.join(self.presubs))
2360 write('subs='+','.join(self.presubs))
2362 write('postsubs='+','.join(self.postsubs))
2364 write('filter='+self.filter)
2366 write('posattrs='+','.join(self.posattrs))
2368 write('style='+self.style)
2370 for style,d in self.styles.items():
2372 for k,v in d.items(): s += '%s=%r,' % (k,v)
2373 write('%s-style=%s' % (style,s[:-1]))
2375 """Validate block after the complete configuration has been loaded."""
2376 if self.is_conf_entry('delimiter') and not self.delimiter:
2377 raise EAsciiDoc,'[%s] missing delimiter' % self.name
2379 if not is_name(self.style):
2380 raise EAsciiDoc, 'illegal style name: %s' % self.style
2381 if not self.style in self.styles:
2382 if not isinstance(self,List): # Lists don't have templates.
2383 message.warning('[%s] \'%s\' style not in %s' % (
2384 self.name,self.style,self.styles.keys()))
2385 # Check all styles for missing templates.
2386 all_styles_have_template = True
2387 for k,v in self.styles.items():
2388 t = v.get('template')
2389 if t and not t in config.sections:
2390 # Defer check if template name contains attributes.
2391 if not re.search(r'{.+}',t):
2392 message.warning('missing template section: [%s]' % t)
2394 all_styles_have_template = False
2395 # Check we have a valid template entry or alternatively that all the
2396 # styles have templates.
2397 if self.is_conf_entry('template') and not 'skip' in self.options:
2399 if not self.template in config.sections:
2400 # Defer check if template name contains attributes.
2401 if not re.search(r'{.+}',self.template):
2402 message.warning('missing template section: [%s]'
2404 elif not all_styles_have_template:
2405 if not isinstance(self,List): # Lists don't have templates.
2406 message.warning('missing styles templates: [%s]' % self.name)
2408 """Check if this block is next in document reader."""
2410 reader.skip_blank_lines()
2411 if reader.read_next():
2412 if not self.delimiter_reo:
2413 # Cache compiled delimiter optimization.
2414 self.delimiter_reo = re.compile(self.delimiter)
2415 mo = self.delimiter_reo.match(reader.read_next())
2420 def translate(self):
2421 """Translate block from document reader."""
2422 if not self.presubs:
2423 self.presubs = config.subsnormal
2425 self.start = reader.cursor[:]
2426 def merge_attributes(self,attrs,params=[]):
2428 Use the current blocks attribute list (attrs dictionary) to build a
2429 dictionary of block processing parameters (self.parameters) and tag
2430 substitution attributes (self.attributes).
2432 1. Copy the default parameters (self.*) to self.parameters.
2433 self.parameters are used internally to render the current block.
2434 Optional params array of additional parameters.
2436 2. Copy attrs to self.attributes. self.attributes are used for template
2437 and tag substitution in the current block.
2439 3. If a style attribute was specified update self.parameters with the
2440 corresponding style parameters; if there are any style parameters
2441 remaining add them to self.attributes (existing attribute list entries
2444 4. Set named positional attributes in self.attributes if self.posattrs
2447 5. Finally self.parameters is updated with any corresponding parameters
2452 def check_array_parameter(param):
2453 # Check the parameter is a sequence type.
2454 if not is_array(self.parameters[param]):
2455 message.error('malformed presubs attribute: %s' %
2456 self.parameters[param])
2457 # Revert to default value.
2458 self.parameters[param] = getattr(self,param)
2460 params = list(self.PARAM_NAMES) + params
2461 self.attributes = {}
2463 # If a default style is defined make it available in the template.
2464 self.attributes['style'] = self.style
2465 self.attributes.update(attrs)
2466 # Calculate dynamic block parameters.
2467 # Start with configuration file defaults.
2468 self.parameters = AttrDict()
2470 self.parameters[name] = getattr(self,name)
2471 # Load the selected style attributes.
2472 posattrs = self.posattrs
2473 if posattrs and posattrs[0] == 'style':
2474 # Positional attribute style has highest precedence.
2475 style = self.attributes.get('1')
2479 # Use explicit style attribute, fall back to default style.
2480 style = self.attributes.get('style',self.style)
2482 if not is_name(style):
2483 message.error('illegal style name: %s' % style)
2485 # Lists have implicit styles and do their own style checks.
2486 elif style not in self.styles and not isinstance(self,List):
2487 message.warning('missing style: [%s]: %s' % (self.name,style))
2489 if style in self.styles:
2490 self.attributes['style'] = style
2491 for k,v in self.styles[style].items():
2495 self.parameters[k] = v
2496 elif not k in self.attributes:
2497 # Style attributes don't take precedence over explicit.
2498 self.attributes[k] = v
2499 # Set named positional attributes.
2500 for i,v in enumerate(posattrs):
2501 if str(i+1) in self.attributes:
2502 self.attributes[v] = self.attributes[str(i+1)]
2503 # Override config and style attributes with attribute list attributes.
2504 self.update_parameters(attrs)
2505 check_array_parameter('options')
2506 check_array_parameter('presubs')
2507 check_array_parameter('postsubs')
2509 class AbstractBlocks:
2510 """List of block definitions."""
2511 PREFIX = '' # Conf file section name prefix set in derived classes.
2512 BLOCK_TYPE = None # Block type set in derived classes.
2515 self.blocks = [] # List of Block objects.
2516 self.default = None # Default Block.
2517 self.delimiters = None # Combined delimiters regular expression.
2518 def load(self,sections):
2519 """Load block definition from 'sections' dictionary."""
2520 for k in sections.keys():
2521 if re.match(r'^'+ self.PREFIX + r'.+$',k):
2523 parse_entries(sections.get(k,()),d)
2524 for b in self.blocks:
2528 b = self.BLOCK_TYPE()
2529 self.blocks.append(b)
2533 raise EAsciiDoc,'[%s] %s' % (k,str(e))
2535 for b in self.blocks:
2538 for b in self.blocks:
2544 """Validate the block definitions."""
2545 # Validate delimiters and build combined lists delimiter pattern.
2547 for b in self.blocks:
2548 assert b.__class__ is self.BLOCK_TYPE
2551 delimiters.append(b.delimiter)
2552 self.delimiters = re_join(delimiters)
2554 class Paragraph(AbstractBlock):
2556 AbstractBlock.__init__(self)
2557 self.text=None # Text in first line of paragraph.
2558 def load(self,name,entries):
2559 AbstractBlock.load(self,name,entries)
2561 AbstractBlock.dump(self)
2562 write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
2565 result = AbstractBlock.isnext(self)
2567 self.text = self.mo.groupdict().get('text')
2569 def translate(self):
2570 AbstractBlock.translate(self)
2571 attrs = self.mo.groupdict().copy()
2572 if 'text' in attrs: del attrs['text']
2573 BlockTitle.consume(attrs)
2574 AttributeList.consume(attrs)
2575 self.merge_attributes(attrs)
2576 reader.read() # Discard (already parsed item first line).
2577 body = reader.read_until(paragraphs.terminators)
2578 body = [self.text] + list(body)
2579 presubs = self.parameters.presubs
2580 postsubs = self.parameters.postsubs
2581 if document.attributes.get('plaintext') is None:
2582 body = Lex.set_margin(body) # Move body to left margin.
2583 body = Lex.subs(body,presubs)
2584 template = self.parameters.template
2585 template = subs_attrs(template,attrs)
2586 stag = config.section2tags(template, self.attributes,skipend=True)[0]
2587 if self.parameters.filter:
2588 body = filter_lines(self.parameters.filter,body,self.attributes)
2589 body = Lex.subs(body,postsubs)
2590 etag = config.section2tags(template, self.attributes,skipstart=True)[1]
2591 # Write start tag, content, end tag.
2592 writer.write(dovetail_tags(stag,body,etag),trace='paragraph')
2594 class Paragraphs(AbstractBlocks):
2595 """List of paragraph definitions."""
2596 BLOCK_TYPE = Paragraph
2599 AbstractBlocks.__init__(self)
2600 self.terminators=None # List of compiled re's.
2601 def initialize(self):
2602 self.terminators = [
2603 re.compile(r'^\+$|^$'),
2604 re.compile(AttributeList.pattern),
2605 re.compile(blocks.delimiters),
2606 re.compile(tables.delimiters),
2607 re.compile(tables_OLD.delimiters),
2609 def load(self,sections):
2610 AbstractBlocks.load(self,sections)
2612 AbstractBlocks.validate(self)
2613 # Check we have a default paragraph definition, put it last in list.
2614 for b in self.blocks:
2615 if b.name == 'paradef-default':
2616 self.blocks.append(b)
2618 self.blocks.remove(b)
2621 raise EAsciiDoc,'missing section: [paradef-default]'
2623 class List(AbstractBlock):
2624 NUMBER_STYLES= ('arabic','loweralpha','upperalpha','lowerroman',
2627 AbstractBlock.__init__(self)
2628 self.CONF_ENTRIES += ('type','tags')
2629 self.PARAM_NAMES += ('tags',)
2630 # tabledef conf file parameters.
2632 self.tags=None # Name of listtags-<tags> conf section.
2633 # Calculated parameters.
2634 self.tag=None # Current tags AttrDict.
2635 self.label=None # List item label (labeled lists).
2636 self.text=None # Text in first line of list item.
2637 self.index=None # Matched delimiter 'index' group (numbered lists).
2638 self.type=None # List type ('numbered','bulleted','labeled').
2639 self.ordinal=None # Current list item ordinal number (1..)
2640 self.number_style=None # Current numbered list style ('arabic'..)
2641 def load(self,name,entries):
2642 AbstractBlock.load(self,name,entries)
2644 AbstractBlock.dump(self)
2645 write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
2646 write('type='+self.type)
2647 write('tags='+self.tags)
2650 AbstractBlock.validate(self)
2652 tags += [s['tags'] for s in self.styles.values() if 'tags' in s]
2654 if t not in lists.tags:
2655 self.error('missing section: [listtags-%s]' % t,halt=True)
2657 result = AbstractBlock.isnext(self)
2659 self.label = self.mo.groupdict().get('label')
2660 self.text = self.mo.groupdict().get('text')
2661 self.index = self.mo.groupdict().get('index')
2663 def translate_entry(self):
2664 assert self.type == 'labeled'
2665 entrytag = subs_tag(self.tag.entry, self.attributes)
2666 labeltag = subs_tag(self.tag.label, self.attributes)
2667 writer.write(entrytag[0],trace='list entry open')
2668 writer.write(labeltag[0],trace='list label open')
2670 while Lex.next() is self:
2671 reader.read() # Discard (already parsed item first line).
2672 writer.write_tag(self.tag.term, [self.label],
2673 self.presubs, self.attributes,trace='list term')
2675 writer.write(labeltag[1],trace='list label close')
2677 self.translate_item()
2678 writer.write(entrytag[1],trace='list entry close')
2679 def translate_item(self):
2680 if self.type == 'callout':
2681 self.attributes['coids'] = calloutmap.calloutids(self.ordinal)
2682 itemtag = subs_tag(self.tag.item, self.attributes)
2683 writer.write(itemtag[0],trace='list item open')
2685 text = reader.read_until(lists.terminators)
2687 text = [self.text] + list(text)
2689 writer.write_tag(self.tag.text, text, self.presubs, self.attributes,trace='list text')
2690 # Process explicit and implicit list item continuations.
2692 continuation = reader.read_next() == '+'
2693 if continuation: reader.read() # Discard continuation line.
2694 while Lex.next() in (BlockTitle,AttributeList):
2695 # Consume continued element title and attributes.
2696 Lex.next().translate()
2697 if not continuation and BlockTitle.title:
2698 # Titled elements terminate the list.
2701 if next in lists.open:
2703 elif isinstance(next,List):
2705 elif isinstance(next,Paragraph) and 'listelement' in next.options:
2708 # This is where continued elements are processed.
2710 message.error('section title not allowed in list item',halt=True)
2714 writer.write(itemtag[1],trace='list item close')
2717 def calc_style(index):
2718 """Return the numbered list style ('arabic'...) of the list item index.
2719 Return None if unrecognized style."""
2720 if re.match(r'^\d+[\.>]$', index):
2722 elif re.match(r'^[ivx]+\)$', index):
2723 style = 'lowerroman'
2724 elif re.match(r'^[IVX]+\)$', index):
2725 style = 'upperroman'
2726 elif re.match(r'^[a-z]\.$', index):
2727 style = 'loweralpha'
2728 elif re.match(r'^[A-Z]\.$', index):
2729 style = 'upperalpha'
2735 def calc_index(index,style):
2736 """Return the ordinal number of (1...) of the list item index
2737 for the given list style."""
2738 def roman_to_int(roman):
2739 roman = roman.lower()
2740 digits = {'i':1,'v':5,'x':10}
2742 for i in range(len(roman)):
2743 digit = digits[roman[i]]
2744 # If next digit is larger this digit is negative.
2745 if i+1 < len(roman) and digits[roman[i+1]] > digit:
2751 if style == 'arabic':
2752 ordinal = int(index)
2753 elif style == 'lowerroman':
2754 ordinal = roman_to_int(index)
2755 elif style == 'upperroman':
2756 ordinal = roman_to_int(index)
2757 elif style == 'loweralpha':
2758 ordinal = ord(index) - ord('a') + 1
2759 elif style == 'upperalpha':
2760 ordinal = ord(index) - ord('A') + 1
2765 def check_index(self):
2766 """Check calculated self.ordinal (1,2,...) against the item number
2767 in the document (self.index) and check the number style is the same as
2768 the first item (self.number_style)."""
2769 assert self.type in ('numbered','callout')
2771 style = self.calc_style(self.index)
2772 if style != self.number_style:
2773 message.warning('list item style: expected %s got %s' %
2774 (self.number_style,style), offset=1)
2775 ordinal = self.calc_index(self.index,style)
2776 if ordinal != self.ordinal:
2777 message.warning('list item index: expected %s got %s' %
2778 (self.ordinal,ordinal), offset=1)
2780 def check_tags(self):
2781 """ Check that all necessary tags are present. """
2782 tags = set(Lists.TAGS)
2783 if self.type != 'labeled':
2784 tags = tags.difference(['entry','label','term'])
2785 missing = tags.difference(self.tag.keys())
2787 self.error('missing tag(s): %s' % ','.join(missing), halt=True)
2788 def translate(self):
2789 AbstractBlock.translate(self)
2790 if self.short_name() in ('bibliography','glossary','qanda'):
2791 message.deprecated('old %s list syntax' % self.short_name())
2792 lists.open.append(self)
2793 attrs = self.mo.groupdict().copy()
2794 for k in ('label','text','index'):
2795 if k in attrs: del attrs[k]
2797 # Set the numbering style from first list item.
2798 attrs['style'] = self.calc_style(self.index)
2799 BlockTitle.consume(attrs)
2800 AttributeList.consume(attrs)
2801 self.merge_attributes(attrs,['tags'])
2802 if self.type in ('numbered','callout'):
2803 self.number_style = self.attributes.get('style')
2804 if self.number_style not in self.NUMBER_STYLES:
2805 message.error('illegal numbered list style: %s' % self.number_style)
2806 # Fall back to default style.
2807 self.attributes['style'] = self.number_style = self.style
2808 self.tag = lists.tags[self.parameters.tags]
2810 if 'width' in self.attributes:
2811 # Set horizontal list 'labelwidth' and 'itemwidth' attributes.
2812 v = str(self.attributes['width'])
2813 mo = re.match(r'^(\d{1,2})%?$',v)
2815 labelwidth = int(mo.group(1))
2816 self.attributes['labelwidth'] = str(labelwidth)
2817 self.attributes['itemwidth'] = str(100-labelwidth)
2819 self.error('illegal attribute value: width="%s"' % v)
2820 stag,etag = subs_tag(self.tag.list, self.attributes)
2822 writer.write(stag,trace='list open')
2824 # Process list till list syntax changes or there is a new title.
2825 while Lex.next() is self and not BlockTitle.title:
2827 document.attributes['listindex'] = str(self.ordinal)
2828 if self.type in ('numbered','callout'):
2830 if self.type in ('bulleted','numbered','callout'):
2831 reader.read() # Discard (already parsed item first line).
2832 self.translate_item()
2833 elif self.type == 'labeled':
2834 self.translate_entry()
2836 raise AssertionError,'illegal [%s] list type' % self.name
2838 writer.write(etag,trace='list close')
2839 if self.type == 'callout':
2840 calloutmap.validate(self.ordinal)
2841 calloutmap.listclose()
2844 document.attributes['listindex'] = str(lists.open[-1].ordinal)
2846 class Lists(AbstractBlocks):
2847 """List of List objects."""
2850 TYPES = ('bulleted','numbered','labeled','callout')
2851 TAGS = ('list', 'entry','item','text', 'label','term')
2853 AbstractBlocks.__init__(self)
2854 self.open = [] # A stack of the current and parent lists.
2855 self.tags={} # List tags dictionary. Each entry is a tags AttrDict.
2856 self.terminators=None # List of compiled re's.
2857 def initialize(self):
2858 self.terminators = [
2859 re.compile(r'^\+$|^$'),
2860 re.compile(AttributeList.pattern),
2861 re.compile(lists.delimiters),
2862 re.compile(blocks.delimiters),
2863 re.compile(tables.delimiters),
2864 re.compile(tables_OLD.delimiters),
2866 def load(self,sections):
2867 AbstractBlocks.load(self,sections)
2868 self.load_tags(sections)
2869 def load_tags(self,sections):
2871 Load listtags-* conf file sections to self.tags.
2873 for section in sections.keys():
2874 mo = re.match(r'^listtags-(?P<name>\w+)$',section)
2876 name = mo.group('name')
2877 if name in self.tags:
2881 parse_entries(sections.get(section,()),d)
2883 if k not in self.TAGS:
2884 message.warning('[%s] contains illegal list tag: %s' %
2888 AbstractBlocks.validate(self)
2889 for b in self.blocks:
2890 # Check list has valid type.
2891 if not b.type in Lists.TYPES:
2892 raise EAsciiDoc,'[%s] illegal type' % b.name
2895 AbstractBlocks.dump(self)
2896 for k,v in self.tags.items():
2897 dump_section('listtags-'+k, v)
2900 class DelimitedBlock(AbstractBlock):
2902 AbstractBlock.__init__(self)
2903 def load(self,name,entries):
2904 AbstractBlock.load(self,name,entries)
2906 AbstractBlock.dump(self)
2907 write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
2910 return AbstractBlock.isnext(self)
2911 def translate(self):
2912 AbstractBlock.translate(self)
2913 reader.read() # Discard delimiter.
2915 if self.short_name() != 'comment':
2916 BlockTitle.consume(attrs)
2917 AttributeList.consume(attrs)
2918 self.merge_attributes(attrs)
2919 options = self.parameters.options
2920 if 'skip' in options:
2921 reader.read_until(self.delimiter,same_file=True)
2922 elif safe() and self.name == 'blockdef-backend':
2923 message.unsafe('Backend Block')
2924 reader.read_until(self.delimiter,same_file=True)
2926 template = self.parameters.template
2927 template = subs_attrs(template,attrs)
2928 name = self.short_name()+' block'
2929 if 'sectionbody' in options:
2930 # The body is treated like a section body.
2931 stag,etag = config.section2tags(template,self.attributes)
2932 writer.write(stag,trace=name+' open')
2933 Section.translate_body(self)
2934 writer.write(etag,trace=name+' close')
2936 stag = config.section2tags(template,self.attributes,skipend=True)[0]
2937 body = reader.read_until(self.delimiter,same_file=True)
2938 presubs = self.parameters.presubs
2939 postsubs = self.parameters.postsubs
2940 body = Lex.subs(body,presubs)
2941 if self.parameters.filter:
2942 body = filter_lines(self.parameters.filter,body,self.attributes)
2943 body = Lex.subs(body,postsubs)
2944 # Write start tag, content, end tag.
2945 etag = config.section2tags(template,self.attributes,skipstart=True)[1]
2946 writer.write(dovetail_tags(stag,body,etag),trace=name)
2947 trace(self.short_name()+' block close',etag)
2949 self.error('missing closing delimiter',self.start)
2951 delimiter = reader.read() # Discard delimiter line.
2952 assert re.match(self.delimiter,delimiter)
2954 class DelimitedBlocks(AbstractBlocks):
2955 """List of delimited blocks."""
2956 BLOCK_TYPE = DelimitedBlock
2957 PREFIX = 'blockdef-'
2959 AbstractBlocks.__init__(self)
2960 def load(self,sections):
2961 """Update blocks defined in 'sections' dictionary."""
2962 AbstractBlocks.load(self,sections)
2964 AbstractBlocks.validate(self)
2968 def __init__(self, width=None, align_spec=None, style=None):
2969 self.width = width or '1'
2970 self.halign, self.valign = Table.parse_align_spec(align_spec)
2971 self.style = style # Style name or None.
2972 # Calculated attribute values.
2973 self.abswidth = None # 1.. (page units).
2974 self.pcwidth = None # 1..99 (percentage).
2977 def __init__(self, data, span_spec=None, align_spec=None, style=None):
2979 self.span, self.vspan = Table.parse_span_spec(span_spec)
2980 self.halign, self.valign = Table.parse_align_spec(align_spec)
2983 return '<Cell: %d.%d %s.%s %s "%s">' % (
2984 self.span, self.vspan,
2985 self.halign, self.valign,
2989 class Table(AbstractBlock):
2990 ALIGN = {'<':'left', '>':'right', '^':'center'}
2991 VALIGN = {'<':'top', '>':'bottom', '^':'middle'}
2992 FORMATS = ('psv','csv','dsv')
2996 # The count and align group matches are not exact.
2997 psv=r'((?<!\S)((?P<span>[\d.]+)(?P<op>[*+]))?(?P<align>[<\^>.]{,3})?(?P<style>[a-z])?)?\|'
3000 AbstractBlock.__init__(self)
3001 self.CONF_ENTRIES += ('format','tags','separator')
3002 # tabledef conf file parameters.
3005 self.tags=None # Name of tabletags-<tags> conf section.
3006 # Calculated parameters.
3007 self.abswidth=None # 1.. (page units).
3008 self.pcwidth = None # 1..99 (percentage).
3009 self.rows=[] # Parsed rows, each row is a list of Cells.
3010 self.columns=[] # List of Columns.
3012 def parse_align_spec(align_spec):
3014 Parse AsciiDoc cell alignment specifier and return 2-tuple with
3015 horizonatal and vertical alignment names. Unspecified alignments
3018 result = (None, None)
3020 mo = re.match(r'^([<\^>])?(\.([<\^>]))?$', align_spec)
3022 result = (Table.ALIGN.get(mo.group(1)),
3023 Table.VALIGN.get(mo.group(3)))
3026 def parse_span_spec(span_spec):
3028 Parse AsciiDoc cell span specifier and return 2-tuple with horizonatal
3029 and vertical span counts. Set default values (1,1) if not
3032 result = (None, None)
3034 mo = re.match(r'^(\d+)?(\.(\d+))?$', span_spec)
3036 result = (mo.group(1) and int(mo.group(1)),
3037 mo.group(3) and int(mo.group(3)))
3038 return (result[0] or 1, result[1] or 1)
3039 def load(self,name,entries):
3040 AbstractBlock.load(self,name,entries)
3042 AbstractBlock.dump(self)
3043 write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
3044 write('format='+self.format)
3047 AbstractBlock.validate(self)
3048 if self.format not in Table.FORMATS:
3049 self.error('illegal format=%s' % self.format,halt=True)
3050 self.tags = self.tags or 'default'
3052 tags += [s['tags'] for s in self.styles.values() if 'tags' in s]
3054 if t not in tables.tags:
3055 self.error('missing section: [tabletags-%s]' % t,halt=True)
3057 # Evaluate escape characters.
3058 self.separator = eval('"'+self.separator+'"')
3059 #TODO: Move to class Tables
3060 # Check global table parameters.
3061 elif config.pagewidth is None:
3062 self.error('missing [miscellaneous] entry: pagewidth')
3063 elif config.pageunits is None:
3064 self.error('missing [miscellaneous] entry: pageunits')
3065 def validate_attributes(self):
3066 """Validate and parse table attributes."""
3068 format = self.format
3070 separator = self.separator
3071 abswidth = float(config.pagewidth)
3073 for k,v in self.attributes.items():
3075 if v not in self.FORMATS:
3076 self.error('illegal %s=%s' % (k,v))
3080 if v not in tables.tags:
3081 self.error('illegal %s=%s' % (k,v))
3084 elif k == 'separator':
3087 if not re.match(r'^\d{1,3}%$',v) or int(v[:-1]) > 100:
3088 self.error('illegal %s=%s' % (k,v))
3090 abswidth = float(v[:-1])/100 * config.pagewidth
3091 pcwidth = float(v[:-1])
3092 # Calculate separator if it has not been specified.
3094 separator = Table.SEPARATORS[format]
3096 if len(separator) > 1:
3097 self.error('illegal csv separator=%s' % separator)
3100 if not is_re(separator):
3101 self.error('illegal regular expression: separator=%s' %
3103 self.parameters.format = format
3104 self.parameters.tags = tags
3105 self.parameters.separator = separator
3106 self.abswidth = abswidth
3107 self.pcwidth = pcwidth
3108 def get_tags(self,params):
3109 tags = self.get_param('tags',params)
3110 assert(tags and tags in tables.tags)
3111 return tables.tags[tags]
3112 def get_style(self,prefix):
3114 Return the style dictionary whose name starts with 'prefix'.
3118 names = self.styles.keys()
3121 if name.startswith(prefix):
3122 return self.styles[name]
3124 self.error('missing style: %s*' % prefix)
3126 def parse_cols(self, cols, halign, valign):
3128 Build list of column objects from table 'cols', 'halign' and 'valign'
3131 # [<multiplier>*][<align>][<width>][<style>]
3132 COLS_RE1 = r'^((?P<count>\d+)\*)?(?P<align>[<\^>.]{,3})?(?P<width>\d+%?)?(?P<style>[a-z]\w*)?$'
3133 # [<multiplier>*][<width>][<align>][<style>]
3134 COLS_RE2 = r'^((?P<count>\d+)\*)?(?P<width>\d+%?)?(?P<align>[<\^>.]{,3})?(?P<style>[a-z]\w*)?$'
3135 reo1 = re.compile(COLS_RE1)
3136 reo2 = re.compile(COLS_RE2)
3138 if re.match(r'^\d+$',cols):
3139 for i in range(int(cols)):
3140 self.columns.append(Column())
3142 for col in re.split(r'\s*,\s*',cols):
3143 mo = reo1.match(col)
3145 mo = reo2.match(col)
3147 count = int(mo.groupdict().get('count') or 1)
3148 for i in range(count):
3149 self.columns.append(
3150 Column(mo.group('width'), mo.group('align'),
3151 self.get_style(mo.group('style')))
3154 self.error('illegal column spec: %s' % col,self.start)
3155 # Set column (and indirectly cell) default alignments.
3156 for col in self.columns:
3157 col.halign = col.halign or halign or document.attributes.get('halign') or 'left'
3158 col.valign = col.valign or valign or document.attributes.get('valign') or 'top'
3159 # Validate widths and calculate missing widths.
3160 n = 0; percents = 0; props = 0
3161 for col in self.columns:
3163 if col.width[-1] == '%': percents += int(col.width[:-1])
3164 else: props += int(col.width)
3166 if percents > 0 and props > 0:
3167 self.error('mixed percent and proportional widths: %s'
3169 pcunits = percents > 0
3170 # Fill in missing widths.
3171 if n < len(self.columns) and percents < 100:
3173 width = float(100 - percents)/float(len(self.columns) - n)
3176 for col in self.columns:
3179 col.width = str(int(width))+'%'
3182 col.width = str(width)
3184 # Calculate column alignment and absolute and percent width values.
3186 for col in self.columns:
3188 col.pcwidth = float(col.width[:-1])
3190 col.pcwidth = (float(col.width)/props)*100
3191 col.abswidth = self.abswidth * (col.pcwidth/100)
3192 if config.pageunits in ('cm','mm','in','em'):
3193 col.abswidth = '%.2f' % round(col.abswidth,2)
3195 col.abswidth = '%d' % round(col.abswidth)
3196 percents += col.pcwidth
3197 col.pcwidth = int(col.pcwidth)
3198 if round(percents) > 100:
3199 self.error('total width exceeds 100%%: %s' % cols,self.start)
3200 elif round(percents) < 100:
3201 self.error('total width less than 100%%: %s' % cols,self.start)
3202 def build_colspecs(self):
3204 Generate column related substitution attributes.
3208 for col in self.columns:
3209 colspec = self.get_tags(col.style).colspec
3211 self.attributes['halign'] = col.halign
3212 self.attributes['valign'] = col.valign
3213 self.attributes['colabswidth'] = col.abswidth
3214 self.attributes['colpcwidth'] = col.pcwidth
3215 self.attributes['colnumber'] = str(i)
3216 s = subs_attrs(colspec, self.attributes)
3218 message.warning('colspec dropped: contains undefined attribute')
3223 self.attributes['colspecs'] = writer.newline.join(cols)
3224 def parse_rows(self, text):
3226 Parse the table source text into self.rows (a list of rows, each row
3229 reserved = {} # Cols reserved by rowspans (indexed by row number).
3230 if self.parameters.format in ('psv','dsv'):
3231 ri = 0 # Current row index 0..
3232 cells = self.parse_psv_dsv(text)
3234 ci = 0 # Column counter 0..colcount
3236 colcount = len(self.columns) - reserved.get(ri,0)
3238 # Reserve spanned columns from ensuing rows.
3239 for i in range(1, cell.vspan):
3240 reserved[ri+i] = reserved.get(ri+i, 0) + cell.span
3245 self.rows.append(row)
3250 message.warning('table row %d: span exceeds number of columns'
3252 elif self.parameters.format == 'csv':
3253 self.rows = self.parse_csv(text)
3255 assert True,'illegal table format'
3256 # Check that all row spans match.
3257 for ri,row in enumerate(self.rows):
3260 row_span += cell.span
3261 row_span += reserved.get(ri,0)
3263 header_span = row_span
3264 if row_span < header_span:
3265 message.warning('table row %d: does not span all columns' % (ri+1))
3266 if row_span > header_span:
3267 message.warning('table row %d: exceeds columns span' % (ri+1))
3268 # Check that now row spans exceed the number of rows.
3269 if len([x for x in reserved.keys() if x >= len(self.rows)]) > 0:
3270 message.warning('one or more cell spans exceed the available rows')
3271 def subs_rows(self, rows, rowtype='body'):
3273 Return a string of output markup from a list of rows, each row
3274 is a list of raw data text.
3276 tags = tables.tags[self.parameters.tags]
3277 if rowtype == 'header':
3279 elif rowtype == 'footer':
3284 stag,etag = subs_tag(rtag,self.attributes)
3287 result += self.subs_row(row,rowtype)
3289 return writer.newline.join(result)
3290 def subs_row(self, row, rowtype):
3292 Substitute the list of Cells using the data tag.
3293 Returns a list of marked up table cell elements.
3298 if i >= len(self.columns):
3299 break # Skip cells outside the header width.
3300 col = self.columns[i]
3301 self.attributes['halign'] = cell.halign or col.halign
3302 self.attributes['valign'] = cell.valign or col.valign
3303 self.attributes['colabswidth'] = col.abswidth
3304 self.attributes['colpcwidth'] = col.pcwidth
3305 self.attributes['colnumber'] = str(i+1)
3306 self.attributes['colspan'] = str(cell.span)
3307 self.attributes['colstart'] = self.attributes['colnumber']
3308 self.attributes['colend'] = str(i+cell.span)
3309 self.attributes['rowspan'] = str(cell.vspan)
3310 self.attributes['morerows'] = str(cell.vspan-1)
3311 # Fill missing column data with blanks.
3312 if i > len(self.columns) - 1:
3316 if rowtype == 'header':
3317 # Use table style unless overriden by cell style.
3318 colstyle = cell.style
3320 # If the cell style is not defined use the column style.
3321 colstyle = cell.style or col.style
3322 tags = self.get_tags(colstyle)
3323 presubs,postsubs = self.get_subs(colstyle)
3325 data = Lex.subs(data, presubs)
3326 data = filter_lines(self.get_param('filter',colstyle),
3327 data, self.attributes)
3328 data = Lex.subs(data, postsubs)
3329 if rowtype != 'header':
3330 ptag = tags.paragraph
3332 stag,etag = subs_tag(ptag,self.attributes)
3333 text = '\n'.join(data).strip()
3335 for para in re.split(r'\n{2,}',text):
3336 data += dovetail_tags([stag],para.split('\n'),[etag])
3337 if rowtype == 'header':
3338 dtag = tags.headdata
3339 elif rowtype == 'footer':
3340 dtag = tags.footdata
3342 dtag = tags.bodydata
3343 stag,etag = subs_tag(dtag,self.attributes)
3344 result = result + dovetail_tags([stag],data,[etag])
3347 def parse_csv(self,text):
3349 Parse the table source text and return a list of rows, each row
3355 rdr = csv.reader(StringIO.StringIO('\r\n'.join(text)),
3356 delimiter=self.parameters.separator, skipinitialspace=True)
3359 rows.append([Cell(data) for data in row])
3361 self.error('csv parse error: %s' % row)
3363 def parse_psv_dsv(self,text):
3365 Parse list of PSV or DSV table source text lines and return a list of
3368 def append_cell(data, span_spec, op, align_spec, style):
3370 if op == '*': # Cell multiplier.
3371 span = Table.parse_span_spec(span_spec)[0]
3372 for i in range(span):
3373 cells.append(Cell(data, '1', align_spec, style))
3374 elif op == '+': # Column spanner.
3375 cells.append(Cell(data, span_spec, align_spec, style))
3377 self.error('illegal table cell operator')
3378 text = '\n'.join(text)
3379 separator = '(?msu)'+self.parameters.separator
3380 format = self.parameters.format
3388 for mo in re.finditer(separator,text):
3389 data += text[start:mo.start()]
3390 if data.endswith('\\'):
3391 data = data[:-1]+mo.group() # Reinstate escaped separators.
3393 append_cell(data, span, op, align, style)
3394 span = mo.groupdict().get('span')
3395 op = mo.groupdict().get('op')
3396 align = mo.groupdict().get('align')
3397 style = mo.groupdict().get('style')
3399 style = self.get_style(style)
3402 # Last cell follows final separator.
3403 data += text[start:]
3404 append_cell(data, span, op, align, style)
3405 # We expect a dummy blank item preceeding first PSV cell.
3407 if cells[0].data.strip() != '':
3408 self.error('missing leading separator: %s' % separator,
3413 def translate(self):
3414 AbstractBlock.translate(self)
3415 reader.read() # Discard delimiter.
3416 # Reset instance specific properties.
3420 BlockTitle.consume(attrs)
3421 # Mix in document attribute list.
3422 AttributeList.consume(attrs)
3423 self.merge_attributes(attrs)
3424 self.validate_attributes()
3425 # Add global and calculated configuration parameters.
3426 self.attributes['pagewidth'] = config.pagewidth
3427 self.attributes['pageunits'] = config.pageunits
3428 self.attributes['tableabswidth'] = int(self.abswidth)
3429 self.attributes['tablepcwidth'] = int(self.pcwidth)
3430 # Read the entire table.
3431 text = reader.read_until(self.delimiter)
3433 self.error('missing closing delimiter',self.start)
3435 delimiter = reader.read() # Discard closing delimiter.
3436 assert re.match(self.delimiter,delimiter)
3438 message.warning('[%s] table is empty' % self.name)
3440 cols = attrs.get('cols')
3442 # Calculate column count from number of items in first line.
3443 if self.parameters.format == 'csv':
3444 cols = text[0].count(self.parameters.separator) + 1
3447 for cell in self.parse_psv_dsv(text[:1]):
3449 self.parse_cols(cols, attrs.get('halign'), attrs.get('valign'))
3450 # Set calculated attributes.
3451 self.attributes['colcount'] = len(self.columns)
3452 self.build_colspecs()
3453 self.parse_rows(text)
3454 # The 'rowcount' attribute is used by the experimental LaTeX backend.
3455 self.attributes['rowcount'] = str(len(self.rows))
3456 # Generate headrows, footrows, bodyrows.
3457 # Headrow, footrow and bodyrow data replaces same named attributes in
3458 # the table markup template. In order to ensure this data does not get
3459 # a second attribute substitution (which would interfere with any
3460 # already substituted inline passthroughs) unique placeholders are used
3461 # (the tab character does not appear elsewhere since it is expanded on
3462 # input) which are replaced after template attribute substitution.
3463 headrows = footrows = bodyrows = None
3464 if self.rows and 'header' in self.parameters.options:
3465 headrows = self.subs_rows(self.rows[0:1],'header')
3466 self.attributes['headrows'] = '\x07headrows\x07'
3467 self.rows = self.rows[1:]
3468 if self.rows and 'footer' in self.parameters.options:
3469 footrows = self.subs_rows( self.rows[-1:], 'footer')
3470 self.attributes['footrows'] = '\x07footrows\x07'
3471 self.rows = self.rows[:-1]
3473 bodyrows = self.subs_rows(self.rows)
3474 self.attributes['bodyrows'] = '\x07bodyrows\x07'
3475 table = subs_attrs(config.sections[self.parameters.template],
3477 table = writer.newline.join(table)
3478 # Before we finish replace the table head, foot and body place holders
3479 # with the real data.
3481 table = table.replace('\x07headrows\x07', headrows, 1)
3483 table = table.replace('\x07footrows\x07', footrows, 1)
3485 table = table.replace('\x07bodyrows\x07', bodyrows, 1)
3486 writer.write(table,trace='table')
3488 class Tables(AbstractBlocks):
3489 """List of tables."""
3491 PREFIX = 'tabledef-'
3492 TAGS = ('colspec', 'headrow','footrow','bodyrow',
3493 'headdata','footdata', 'bodydata','paragraph')
3495 AbstractBlocks.__init__(self)
3496 # Table tags dictionary. Each entry is a tags dictionary.
3498 def load(self,sections):
3499 AbstractBlocks.load(self,sections)
3500 self.load_tags(sections)
3501 def load_tags(self,sections):
3503 Load tabletags-* conf file sections to self.tags.
3505 for section in sections.keys():
3506 mo = re.match(r'^tabletags-(?P<name>\w+)$',section)
3508 name = mo.group('name')
3509 if name in self.tags:
3513 parse_entries(sections.get(section,()),d)
3515 if k not in self.TAGS:
3516 message.warning('[%s] contains illegal table tag: %s' %
3520 AbstractBlocks.validate(self)
3521 # Check we have a default table definition,
3522 for i in range(len(self.blocks)):
3523 if self.blocks[i].name == 'tabledef-default':
3524 default = self.blocks[i]
3527 raise EAsciiDoc,'missing section: [tabledef-default]'
3528 # Propagate defaults to unspecified table parameters.
3529 for b in self.blocks:
3530 if b is not default:
3531 if b.format is None: b.format = default.format
3532 if b.template is None: b.template = default.template
3533 # Check tags and propagate default tags.
3534 if not 'default' in self.tags:
3535 raise EAsciiDoc,'missing section: [tabletags-default]'
3536 default = self.tags['default']
3537 for tag in ('bodyrow','bodydata','paragraph'): # Mandatory default tags.
3538 if tag not in default:
3539 raise EAsciiDoc,'missing [tabletags-default] entry: %s' % tag
3540 for t in self.tags.values():
3541 if t is not default:
3542 if t.colspec is None: t.colspec = default.colspec
3543 if t.headrow is None: t.headrow = default.headrow
3544 if t.footrow is None: t.footrow = default.footrow
3545 if t.bodyrow is None: t.bodyrow = default.bodyrow
3546 if t.headdata is None: t.headdata = default.headdata
3547 if t.footdata is None: t.footdata = default.footdata
3548 if t.bodydata is None: t.bodydata = default.bodydata
3549 if t.paragraph is None: t.paragraph = default.paragraph
3550 # Use body tags if header and footer tags are not specified.
3551 for t in self.tags.values():
3552 if not t.headrow: t.headrow = t.bodyrow
3553 if not t.footrow: t.footrow = t.bodyrow
3554 if not t.headdata: t.headdata = t.bodydata
3555 if not t.footdata: t.footdata = t.bodydata
3556 # Check table definitions are valid.
3557 for b in self.blocks:
3560 AbstractBlocks.dump(self)
3561 for k,v in self.tags.items():
3562 dump_section('tabletags-'+k, v)
3565 # Default system macro syntax.
3566 SYS_RE = r'(?u)^(?P<name>[\\]?\w(\w|-)*?)::(?P<target>\S*?)' + \
3567 r'(\[(?P<attrlist>.*?)\])$'
3569 self.macros = [] # List of Macros.
3570 self.current = None # The last matched block macro.
3571 self.passthroughs = []
3572 # Initialize default system macro.
3574 m.pattern = self.SYS_RE
3576 m.reo = re.compile(m.pattern)
3577 self.macros.append(m)
3578 def load(self,entries):
3579 for entry in entries:
3583 # Delete undefined macro.
3584 for i,m2 in enumerate(self.macros):
3585 if m2.pattern == m.pattern:
3589 message.warning('unable to delete missing macro: %s' % m.pattern)
3591 # Check for duplicates.
3592 for m2 in self.macros:
3593 if m2.pattern == m.pattern:
3594 message.verbose('macro redefinition: %s%s' % (m.prefix,m.name))
3597 self.macros.append(m)
3599 write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
3601 # Dump all macros except the first (built-in system) macro.
3602 for m in self.macros[1:]:
3603 # Escape = in pattern.
3604 macro = '%s=%s%s' % (m.pattern.replace('=',r'\='), m.prefix, m.name)
3605 if m.subslist is not None:
3606 macro += '[' + ','.join(m.subslist) + ']'
3610 # Check all named sections exist.
3612 for m in self.macros:
3613 if m.name and m.prefix != '+':
3615 def subs(self,text,prefix='',callouts=False):
3616 # If callouts is True then only callout macros are processed, if False
3617 # then all non-callout macros are processed.
3619 for m in self.macros:
3620 if m.prefix == prefix:
3621 if callouts ^ (m.name != 'callout'):
3622 result = m.subs(result)
3625 """Return matching macro if block macro is next on reader."""
3626 reader.skip_blank_lines()
3627 line = reader.read_next()
3629 for m in self.macros:
3631 if m.reo.match(line):
3635 def match(self,prefix,name,text):
3636 """Return re match object matching 'text' with macro type 'prefix',
3637 macro name 'name'."""
3638 for m in self.macros:
3639 if m.prefix == prefix:
3640 mo = m.reo.match(text)
3644 if re.match(name,mo.group('name')):
3647 def extract_passthroughs(self,text,prefix=''):
3648 """ Extract the passthrough text and replace with temporary
3650 self.passthroughs = []
3651 for m in self.macros:
3652 if m.has_passthrough() and m.prefix == prefix:
3653 text = m.subs_passthroughs(text, self.passthroughs)
3655 def restore_passthroughs(self,text):
3656 """ Replace passthough placeholders with the original passthrough
3658 for i,v in enumerate(self.passthroughs):
3659 text = text.replace('\x07'+str(i)+'\x07', self.passthroughs[i])
3664 self.pattern = None # Matching regular expression.
3665 self.name = '' # Conf file macro name (None if implicit).
3666 self.prefix = '' # '' if inline, '+' if system, '#' if block.
3667 self.reo = None # Compiled pattern re object.
3668 self.subslist = [] # Default subs for macros passtext group.
3669 def has_passthrough(self):
3670 return self.pattern.find(r'(?P<passtext>') >= 0
3671 def section_name(self,name=None):
3672 """Return macro markup template section name based on macro name and
3673 prefix. Return None section not found."""
3674 assert self.prefix != '+'
3678 if self.prefix == '#':
3679 suffix = '-blockmacro'
3681 suffix = '-inlinemacro'
3682 if name+suffix in config.sections:
3685 message.warning('missing macro section: [%s]' % (name+suffix))
3687 def load(self,entry):
3688 e = parse_entry(entry)
3690 # Only the macro pattern was specified, mark for deletion.
3692 self.pattern = entry
3695 raise EAsciiDoc,'illegal macro regular expression: %s' % e[0]
3697 if name and name[0] in ('+','#'):
3698 prefix, name = name[0], name[1:]
3701 # Parse passthrough subslist.
3702 mo = re.match(r'^(?P<name>[^[]*)(\[(?P<subslist>.*)\])?$', name)
3703 name = mo.group('name')
3704 if name and not is_name(name):
3705 raise EAsciiDoc,'illegal section name in macro entry: %s' % entry
3706 subslist = mo.group('subslist')
3707 if subslist is not None:
3708 # Parse and validate passthrough subs.
3709 subslist = parse_options(subslist, SUBS_OPTIONS,
3710 'illegal subs in macro entry: %s' % entry)
3711 self.pattern = pattern
3712 self.reo = re.compile(pattern)
3713 self.prefix = prefix
3715 self.subslist = subslist or []
3717 def subs(self,text):
3719 """Function called to perform macro substitution.
3720 Uses matched macro regular expression object and returns string
3721 containing the substituted macro body."""
3722 # Check if macro reference is escaped.
3723 if mo.group()[0] == '\\':
3724 return mo.group()[1:] # Strip leading backslash.
3726 # Delete groups that didn't participate in match.
3727 for k,v in d.items():
3728 if v is None: del d[k]
3733 message.warning('missing macro name group: %s' % mo.re.pattern)
3736 section_name = self.section_name(name)
3737 if not section_name:
3739 # If we're dealing with a block macro get optional block ID and
3741 if self.prefix == '#' and self.name != 'comment':
3742 AttributeList.consume(d)
3743 BlockTitle.consume(d)
3744 # Parse macro attributes.
3746 if d['attrlist'] in (None,''):
3749 if self.prefix == '':
3750 # Unescape ] characters in inline macros.
3751 d['attrlist'] = d['attrlist'].replace('\\]',']')
3752 parse_attributes(d['attrlist'],d)
3753 # Generate option attributes.
3755 options = parse_options(d['options'], (),
3756 '%s: illegal option name' % name)
3757 for option in options:
3758 d[option+'-option'] = ''
3759 # Substitute single quoted attribute values in block macros.
3760 if self.prefix == '#':
3761 AttributeList.subs(d)
3762 if name == 'callout':
3763 listindex =int(d['index'])
3764 d['coid'] = calloutmap.add(listindex)
3765 # The alt attribute is the first image macro positional attribute.
3766 if name == 'image' and '1' in d:
3768 # Unescape special characters in LaTeX target file names.
3769 if document.backend == 'latex' and 'target' in d and d['target']:
3771 d['0'] = d['target']
3772 d['target']= config.subs_specialchars_reverse(d['target'])
3773 # BUG: We've already done attribute substitution on the macro which
3774 # means that any escaped attribute references are now unescaped and
3775 # will be substituted by config.subs_section() below. As a partial
3776 # fix have withheld {0} from substitution but this kludge doesn't
3777 # fix it for other attributes containing unescaped references.
3778 # Passthrough macros don't have this problem.
3781 d['0'] = chr(0) # Replace temporarily with unused character.
3782 body = config.subs_section(section_name,d)
3785 elif len(body) == 1:
3788 if self.prefix == '#':
3789 result = writer.newline.join(body)
3791 # Internally processed inline macros use UNIX line
3793 result = '\n'.join(body)
3795 result = result.replace(chr(0), a0)
3798 return self.reo.sub(subs_func, text)
3800 def translate(self):
3801 """ Block macro translation."""
3802 assert self.prefix == '#'
3805 if self.has_passthrough():
3806 s = macros.extract_passthroughs(s,'#')
3810 if self.has_passthrough():
3811 s = macros.restore_passthroughs(s)
3813 trace('macro block',before,s)
3816 def subs_passthroughs(self, text, passthroughs):
3817 """ Replace macro attribute lists in text with placeholders.
3818 Substitute and append the passthrough attribute lists to the
3819 passthroughs list."""
3821 """Function called to perform inline macro substitution.
3822 Uses matched macro regular expression object and returns string
3823 containing the substituted macro body."""
3824 # Don't process escaped macro references.
3825 if mo.group()[0] == '\\':
3828 if not 'passtext' in d:
3829 message.warning('passthrough macro %s: missing passtext group' %
3832 passtext = d['passtext']
3833 if re.search('\x07\\d+\x07', passtext):
3834 message.warning('nested inline passthrough')
3836 if d.get('subslist'):
3837 if d['subslist'].startswith(':'):
3838 message.error('block macro cannot occur here: %s' % mo.group(),
3840 subslist = parse_options(d['subslist'], SUBS_OPTIONS,
3841 'illegal passthrough macro subs option')
3843 subslist = self.subslist
3844 passtext = Lex.subs_1(passtext,subslist)
3845 if passtext is None: passtext = ''
3846 if self.prefix == '':
3847 # Unescape ] characters in inline macros.
3848 passtext = passtext.replace('\\]',']')
3849 passthroughs.append(passtext)
3850 # Tabs guarantee the placeholders are unambiguous.
3852 text[mo.start():mo.start('passtext')] +
3853 '\x07' + str(len(passthroughs)-1) + '\x07' +
3854 text[mo.end('passtext'):mo.end()]
3858 return self.reo.sub(subs_func, text)
3863 self.comap = {} # key = list index, value = callouts list.
3864 self.calloutindex = 0 # Current callout index number.
3865 self.listnumber = 1 # Current callout list number.
3866 def listclose(self):
3867 # Called when callout list is closed.
3868 self.listnumber += 1
3869 self.calloutindex = 0
3871 def add(self,listindex):
3872 # Add next callout index to listindex map entry. Return the callout id.
3873 self.calloutindex += 1
3874 # Append the coindex to a list in the comap dictionary.
3875 if not listindex in self.comap:
3876 self.comap[listindex] = [self.calloutindex]
3878 self.comap[listindex].append(self.calloutindex)
3879 return self.calloutid(self.listnumber, self.calloutindex)
3881 def calloutid(listnumber,calloutindex):
3882 return 'CO%d-%d' % (listnumber,calloutindex)
3883 def calloutids(self,listindex):
3884 # Retieve list of callout indexes that refer to listindex.
3885 if listindex in self.comap:
3887 for coindex in self.comap[listindex]:
3888 result += ' ' + self.calloutid(self.listnumber,coindex)
3889 return result.strip()
3891 message.warning('no callouts refer to list item '+str(listindex))
3893 def validate(self,maxlistindex):
3894 # Check that all list indexes referenced by callouts exist.
3895 for listindex in self.comap.keys():
3896 if listindex > maxlistindex:
3897 message.warning('callout refers to non-existent list item '
3900 #---------------------------------------------------------------------------
3901 # Input stream Reader and output stream writer classes.
3902 #---------------------------------------------------------------------------
3904 UTF8_BOM = '\xef\xbb\xbf'
3907 """Line oriented AsciiDoc input file reader. Processes include and
3908 conditional inclusion system macros. Tabs are expanded and lines are right
3910 # This class is not used directly, use Reader class instead.
3911 READ_BUFFER_MIN = 10 # Read buffer low level.
3913 self.f = None # Input file object.
3914 self.fname = None # Input file name.
3915 self.next = [] # Read ahead buffer containing
3916 # [filename,linenumber,linetext] lists.
3917 self.cursor = None # Last read() [filename,linenumber,linetext].
3918 self.tabsize = 8 # Tab expansion number of spaces.
3919 self.parent = None # Included reader's parent reader.
3920 self._lineno = 0 # The last line read from file object f.
3921 self.current_depth = 0 # Current include depth.
3922 self.max_depth = 5 # Initial maxiumum allowed include depth.
3923 self.bom = None # Byte order mark (BOM).
3924 self.infile = None # Saved document 'infile' attribute.
3925 self.indir = None # Saved document 'indir' attribute.
3926 def open(self,fname):
3928 message.verbose('reading: '+fname)
3929 if fname == '<stdin>':
3934 self.f = open(fname,'rb')
3936 self.indir = os.path.dirname(fname)
3937 document.attributes['infile'] = self.infile
3938 document.attributes['indir'] = self.indir
3939 self._lineno = 0 # The last line read from file object f.
3941 # Prefill buffer by reading the first line and then pushing it back.
3942 if Reader1.read(self):
3943 if self.cursor[2].startswith(UTF8_BOM):
3944 self.cursor[2] = self.cursor[2][len(UTF8_BOM):]
3946 self.unread(self.cursor)
3948 def closefile(self):
3949 """Used by class methods to close nested include files."""
3955 def read(self, skip=False):
3956 """Read next line. Return None if EOF. Expand tabs. Strip trailing
3957 white space. Maintain self.next read ahead buffer. If skip=True then
3958 conditional exclusion is active (ifdef and ifndef macros)."""
3960 if len(self.next) <= self.READ_BUFFER_MIN:
3961 s = self.f.readline()
3963 self._lineno = self._lineno + 1
3965 if self.tabsize != 0:
3966 s = s.expandtabs(self.tabsize)
3968 self.next.append([self.fname,self._lineno,s])
3969 if len(self.next) > self.READ_BUFFER_MIN:
3971 s = self.f.readline()
3973 self._lineno = self._lineno + 1
3974 # Return first (oldest) buffer entry.
3975 if len(self.next) > 0:
3976 self.cursor = self.next[0]
3978 result = self.cursor[2]
3979 # Check for include macro.
3980 mo = macros.match('+',r'include[1]?',result)
3982 # Don't process include macro once the maximum depth is reached.
3983 if self.current_depth >= self.max_depth:
3985 # Perform attribute substitution on include macro file name.
3986 fname = subs_attrs(mo.group('target'))
3988 return Reader1.read(self) # Return next input line.
3989 if self.fname != '<stdin>':
3990 fname = os.path.expandvars(os.path.expanduser(fname))
3991 fname = safe_filename(fname, os.path.dirname(self.fname))
3993 return Reader1.read(self) # Return next input line.
3994 if mo.group('name') == 'include1':
3995 if not config.dumping:
3996 # Store the include file in memory for later
3997 # retrieval by the {include1:} system attribute.
3998 config.include1[fname] = [
3999 s.rstrip() for s in open(fname)]
4000 return '{include1:%s}' % fname
4002 # This is a configuration dump, just pass the macro
4005 # Parse include macro attributes.
4007 parse_attributes(mo.group('attrlist'),attrs)
4008 # Clone self and set as parent (self assumes the role of child).
4011 self.parent = parent
4012 # Set attributes in child.
4013 if 'tabsize' in attrs:
4014 self.tabsize = int(validate(attrs['tabsize'],
4016 'illegal include macro tabsize argument'))
4018 self.tabsize = config.tabsize
4019 if 'depth' in attrs:
4020 attrs['depth'] = int(validate(attrs['depth'],
4022 'illegal include macro depth argument'))
4023 self.max_depth = self.current_depth + attrs['depth']
4024 # Process included file.
4026 self.current_depth = self.current_depth + 1
4027 result = Reader1.read(self)
4029 if not Reader1.eof(self):
4030 result = Reader1.read(self)
4035 """Returns True if all lines have been read."""
4036 if len(self.next) == 0:
4037 # End of current file.
4040 assign(self,self.parent) # Restore parent reader.
4041 document.attributes['infile'] = self.infile
4042 document.attributes['indir'] = self.indir
4043 return Reader1.eof(self)
4048 def read_next(self):
4049 """Like read() but does not advance file pointer."""
4050 if Reader1.eof(self):
4053 return self.next[0][2]
4054 def unread(self,cursor):
4055 """Push the line (filename,linenumber,linetext) tuple back into the read
4056 buffer. Note that it's up to the caller to restore the previous
4059 self.next.insert(0,cursor)
4061 class Reader(Reader1):
4062 """ Wraps (well, sought of) Reader1 class and implements conditional text
4065 Reader1.__init__(self)
4066 self.depth = 0 # if nesting depth.
4067 self.skip = False # true if we're skipping ifdef...endif.
4068 self.skipname = '' # Name of current endif macro target.
4069 self.skipto = -1 # The depth at which skipping is reenabled.
4070 def read_super(self):
4071 result = Reader1.read(self,self.skip)
4072 if result is None and self.skip:
4073 raise EAsciiDoc,'missing endif::%s[]' % self.skipname
4076 result = self.read_super()
4080 mo = macros.match('+',r'ifdef|ifndef|ifeval|endif',result)
4082 name = mo.group('name')
4083 target = mo.group('target')
4084 attrlist = mo.group('attrlist')
4088 raise EAsciiDoc,'mismatched macro: %s' % result
4089 if self.depth == self.skipto:
4091 if target and self.skipname != target:
4092 raise EAsciiDoc,'mismatched macro: %s' % result
4094 if name in ('ifdef','ifndef'):
4096 raise EAsciiDoc,'missing macro target: %s' % result
4099 elif name == 'ifeval':
4101 raise EAsciiDoc,'missing ifeval condition: %s' % result
4103 result = self.read_super()
4106 mo = macros.match('+',r'ifdef|ifndef|ifeval|endif',result)
4108 name = mo.group('name')
4109 target = mo.group('target')
4110 attrlist = mo.group('attrlist')
4112 self.depth = self.depth-1
4114 if not target and name in ('ifdef','ifndef'):
4115 raise EAsciiDoc,'missing macro target: %s' % result
4116 defined = is_attr_defined(target, document.attributes)
4119 if defined: return attrlist
4121 self.skip = not defined
4122 elif name == 'ifndef':
4124 if not defined: return attrlist
4127 elif name == 'ifeval':
4129 raise EAsciiDoc,'missing ifeval condition: %s' % result
4131 attrlist = subs_attrs(attrlist)
4134 cond = eval(attrlist)
4136 raise EAsciiDoc,'error evaluating ifeval condition: %s: %s' % (result, str(e))
4137 self.skip = not cond
4138 if not attrlist or name == 'ifeval':
4140 self.skipto = self.depth
4141 self.skipname = target
4142 self.depth = self.depth+1
4143 result = self.read()
4145 # Expand executable block macros.
4146 mo = macros.match('+',r'eval|sys|sys2',result)
4148 action = mo.group('name')
4149 cmd = mo.group('attrlist')
4150 s = system(action, cmd, is_macro=True)
4152 self.cursor[2] = s # So we don't re-evaluate.
4155 # Unescape escaped system macros.
4156 if macros.match('+',r'\\eval|\\sys|\\sys2|\\ifdef|\\ifndef|\\endif|\\include|\\include1',result):
4160 return self.read_next() is None
4161 def read_next(self):
4162 save_cursor = self.cursor
4163 result = self.read()
4164 if result is not None:
4165 self.unread(self.cursor)
4166 self.cursor = save_cursor
4168 def read_lines(self,count=1):
4169 """Return tuple containing count lines."""
4172 while i < count and not self.eof():
4173 result.append(self.read())
4174 return tuple(result)
4175 def read_ahead(self,count=1):
4176 """Same as read_lines() but does not advance the file pointer."""
4179 save_cursor = self.cursor
4182 while i < count and not self.eof():
4183 result.append(self.read())
4184 putback.append(self.cursor)
4187 self.unread(putback.pop())
4189 self.cursor = save_cursor
4190 return tuple(result)
4191 def skip_blank_lines(self):
4192 reader.read_until(r'\s*\S+')
4193 def read_until(self,terminators,same_file=False):
4194 """Like read() but reads lines up to (but not including) the first line
4195 that matches the terminator regular expression, regular expression
4196 object or list of regular expression objects. If same_file is True then
4197 the terminating pattern must occur in the file the was being read when
4198 the routine was called."""
4200 fname = self.cursor[0]
4202 if not isinstance(terminators,list):
4203 if isinstance(terminators,basestring):
4204 terminators = [re.compile(terminators)]
4206 terminators = [terminators]
4207 while not self.eof():
4208 save_cursor = self.cursor
4210 if not same_file or fname == self.cursor[0]:
4211 for reo in terminators:
4213 self.unread(self.cursor)
4214 self.cursor = save_cursor
4215 return tuple(result)
4217 return tuple(result)
4220 """Writes lines to output file."""
4222 self.newline = '\r\n' # End of line terminator.
4223 self.f = None # Output file object.
4224 self.fname = None # Output file name.
4225 self.lines_out = 0 # Number of lines written.
4226 self.skip_blank_lines = False # If True don't output blank lines.
4227 def open(self,fname,bom=None):
4229 bom is optional byte order mark.
4230 http://en.wikipedia.org/wiki/Byte-order_mark
4233 if fname == '<stdout>':
4236 self.f = open(fname,'wb+')
4237 message.verbose('writing: '+writer.fname,False)
4242 if self.fname != '<stdout>':
4244 def write_line(self, line=None):
4245 if not (self.skip_blank_lines and (not line or not line.strip())):
4246 self.f.write((line or '') + self.newline)
4247 self.lines_out = self.lines_out + 1
4248 def write(self,*args,**kwargs):
4249 """Iterates arguments, writes tuple and list arguments one line per
4250 element, else writes argument as single line. If no arguments writes
4251 blank line. If argument is None nothing is written. self.newline is
4252 appended to each line."""
4253 if 'trace' in kwargs and len(args) > 0:
4254 trace(kwargs['trace'],args[0])
4257 self.lines_out = self.lines_out + 1
4263 elif arg is not None:
4264 self.write_line(arg)
4265 def write_tag(self,tag,content,subs=None,d=None,**kwargs):
4266 """Write content enveloped by tag.
4267 Substitutions specified in the 'subs' list are perform on the
4270 subs = config.subsnormal
4271 stag,etag = subs_tag(tag,d)
4272 content = Lex.subs(content,subs)
4273 if 'trace' in kwargs:
4274 trace(kwargs['trace'],[stag]+content+[etag])
4282 #---------------------------------------------------------------------------
4283 # Configuration file processing.
4284 #---------------------------------------------------------------------------
4285 def _subs_specialwords(mo):
4286 """Special word substitution function called by
4287 Config.subs_specialwords()."""
4288 word = mo.re.pattern # The special word.
4289 template = config.specialwords[word] # The corresponding markup template.
4290 if not template in config.sections:
4291 raise EAsciiDoc,'missing special word template [%s]' % template
4292 if mo.group()[0] == '\\':
4293 return mo.group()[1:] # Return escaped word.
4295 args['words'] = mo.group() # The full match string is argument 'words'.
4296 args.update(mo.groupdict()) # Add other named match groups to the arguments.
4297 # Delete groups that didn't participate in match.
4298 for k,v in args.items():
4299 if v is None: del args[k]
4300 lines = subs_attrs(config.sections[template],args)
4303 elif len(lines) == 1:
4306 result = writer.newline.join(lines)
4310 """Methods to process configuration files."""
4311 # Non-template section name regexp's.
4312 ENTRIES_SECTIONS= ('tags','miscellaneous','attributes','specialcharacters',
4313 'specialwords','macros','replacements','quotes','titles',
4314 r'paradef-.+',r'listdef-.+',r'blockdef-.+',r'tabledef-.+',
4315 r'tabletags-.+',r'listtags-.+','replacements2',
4318 self.sections = OrderedDict() # Keyed by section name containing
4319 # lists of section lines.
4320 # Command-line options.
4321 self.verbose = False
4322 self.header_footer = True # -s, --no-header-footer option.
4323 # [miscellaneous] section.
4325 self.textwidth = 70 # DEPRECATED: Old tables only.
4326 self.newline = '\r\n'
4327 self.pagewidth = None
4328 self.pageunits = None
4329 self.outfilesuffix = ''
4330 self.subsnormal = SUBS_NORMAL
4331 self.subsverbatim = SUBS_VERBATIM
4333 self.tags = {} # Values contain (stag,etag) tuples.
4334 self.specialchars = {} # Values of special character substitutions.
4335 self.specialwords = {} # Name is special word pattern, value is macro.
4336 self.replacements = OrderedDict() # Key is find pattern, value is
4338 self.replacements2 = OrderedDict()
4339 self.specialsections = {} # Name is special section name pattern, value
4340 # is corresponding section name.
4341 self.quotes = OrderedDict() # Values contain corresponding tag name.
4342 self.fname = '' # Most recently loaded configuration file name.
4343 self.conf_attrs = {} # Attributes entries from conf files.
4344 self.cmd_attrs = {} # Attributes from command-line -a options.
4345 self.loaded = [] # Loaded conf files.
4346 self.include1 = {} # Holds include1::[] files for {include1:}.
4347 self.dumping = False # True if asciidoc -c option specified.
4349 def init(self, cmd):
4351 Check Python version and locate the executable and configuration files
4353 cmd is the asciidoc command or asciidoc.py path.
4355 if float(sys.version[:3]) < MIN_PYTHON_VERSION:
4356 message.stderr('FAILED: Python 2.3 or better required')
4358 if not os.path.exists(cmd):
4359 message.stderr('FAILED: Missing asciidoc command: %s' % cmd)
4362 APP_FILE = os.path.realpath(cmd)
4364 APP_DIR = os.path.dirname(APP_FILE)
4366 USER_DIR = userdir()
4367 if USER_DIR is not None:
4368 USER_DIR = os.path.join(USER_DIR,'.asciidoc')
4369 if not os.path.isdir(USER_DIR):
4372 def load_file(self, fname, dir=None, include=[], exclude=[]):
4374 Loads sections dictionary with sections from file fname.
4375 Existing sections are overlaid.
4376 The 'include' list contains the section names to be loaded.
4377 The 'exclude' list contains section names not to be loaded.
4378 Return False if no file was found in any of the locations.
4381 fname = os.path.join(dir, fname)
4382 # Sliently skip missing configuration file.
4383 if not os.path.isfile(fname):
4385 # Don't load conf files twice (local and application conf files are the
4386 # same if the source file is in the application directory).
4387 if os.path.realpath(fname) in self.loaded:
4389 rdr = Reader() # Reader processes system macros.
4390 message.linenos = False # Disable document line numbers.
4392 message.linenos = None
4394 reo = re.compile(r'(?u)^\[(?P<section>[^\W\d][\w-]*)\]\s*$')
4395 sections = OrderedDict()
4396 section,contents = '',[]
4397 while not rdr.eof():
4399 if s and s[0] == '#': # Skip comment lines.
4401 if s[:2] == '\\#': # Unescape lines starting with '#'.
4404 found = reo.findall(s)
4406 if section: # Store previous section.
4407 if section in sections \
4408 and self.entries_section(section):
4409 if ''.join(contents):
4411 sections[section] = sections[section] + contents
4413 del sections[section]
4415 sections[section] = contents
4416 section = found[0].lower()
4420 if section and contents: # Store last section.
4421 if section in sections \
4422 and self.entries_section(section):
4423 if ''.join(contents):
4425 sections[section] = sections[section] + contents
4427 del sections[section]
4429 sections[section] = contents
4432 for s in set(sections) - set(include):
4435 for s in set(sections) & set(exclude):
4438 self.load_sections(sections,attrs)
4440 # If all sections are loaded mark this file as loaded.
4441 self.loaded.append(os.path.realpath(fname))
4442 document.update_attributes(attrs) # So they are available immediately.
4445 def load_sections(self,sections,attrs=None):
4447 Loads sections dictionary. Each dictionary entry contains a
4449 Updates 'attrs' with parsed [attributes] section entries.
4451 # Delete trailing blank lines from sections.
4452 for k in sections.keys():
4453 for i in range(len(sections[k])-1,-1,-1):
4454 if not sections[k][i]:
4456 elif not self.entries_section(k):
4458 # Add/overwrite new sections.
4459 self.sections.update(sections)
4461 # Internally [miscellaneous] section entries are just attributes.
4463 parse_entries(sections.get('miscellaneous',()), d, unquote=True,
4464 allow_name_only=True)
4465 parse_entries(sections.get('attributes',()), d, unquote=True,
4466 allow_name_only=True)
4467 update_attrs(self.conf_attrs,d)
4468 if attrs is not None:
4471 parse_entries(sections.get('titles',()),d)
4473 parse_entries(sections.get('specialcharacters',()),self.specialchars,escape_delimiter=False)
4474 parse_entries(sections.get('quotes',()),self.quotes)
4475 self.parse_specialwords()
4476 self.parse_replacements()
4477 self.parse_replacements('replacements2')
4478 self.parse_specialsections()
4479 paragraphs.load(sections)
4480 lists.load(sections)
4481 blocks.load(sections)
4482 tables_OLD.load(sections)
4483 tables.load(sections)
4484 macros.load(sections.get('macros',()))
4486 def get_load_dirs(self):
4488 Return list of well known paths with conf files.
4492 # Load from folders in asciidoc executable directory.
4493 result.append(APP_DIR)
4495 # Load from global configuration directory.
4496 result.append(CONF_DIR)
4497 # Load configuration files from ~/.asciidoc if it exists.
4498 if USER_DIR is not None:
4499 result.append(USER_DIR)
4502 def find_in_dirs(self, filename, dirs=None):
4504 Find conf files from dirs list.
4505 Return list of found file paths.
4506 Return empty list if not found in any of the locations.
4510 dirs = self.get_load_dirs()
4512 f = os.path.join(d,filename)
4513 if os.path.isfile(f):
4517 def load_from_dirs(self, filename, dirs=None, include=[]):
4519 Load conf file from dirs list.
4520 If dirs not specified try all the well known locations.
4521 Return False if no file was sucessfully loaded.
4524 for f in self.find_in_dirs(filename,dirs):
4525 if self.load_file(f, include=include):
4529 def load_backend(self, dirs=None):
4531 Load the backend configuration files from dirs list.
4532 If dirs not specified try all the well known locations.
4535 dirs = self.get_load_dirs()
4537 conf = document.backend + '.conf'
4538 self.load_file(conf,d)
4539 conf = document.backend + '-' + document.doctype + '.conf'
4540 self.load_file(conf,d)
4542 def load_filters(self, dirs=None):
4544 Load filter configuration files from 'filters' directory in dirs list.
4545 If dirs not specified try all the well known locations.
4548 dirs = self.get_load_dirs()
4550 # Load filter .conf files.
4551 filtersdir = os.path.join(d,'filters')
4552 for dirpath,dirnames,filenames in os.walk(filtersdir):
4554 if re.match(r'^.+\.conf$',f):
4555 self.load_file(f,dirpath)
4557 def load_miscellaneous(self,d):
4558 """Set miscellaneous configuration entries from dictionary 'd'."""
4559 def set_misc(name,rule='True',intval=False):
4561 errmsg = 'illegal [miscellaneous] %s entry' % name
4563 setattr(self, name, int(validate(d[name],rule,errmsg)))
4565 setattr(self, name, validate(d[name],rule,errmsg))
4566 set_misc('tabsize','int($)>0',intval=True)
4567 set_misc('textwidth','int($)>0',intval=True) # DEPRECATED: Old tables only.
4568 set_misc('pagewidth','"%f" % $')
4569 if 'pagewidth' in d:
4570 self.pagewidth = float(self.pagewidth)
4571 set_misc('pageunits')
4572 set_misc('outfilesuffix')
4574 # Convert escape sequences to their character values.
4575 self.newline = eval('"'+d['newline']+'"')
4576 if 'subsnormal' in d:
4577 self.subsnormal = parse_options(d['subsnormal'],SUBS_OPTIONS,
4578 'illegal [%s] %s: %s' %
4579 ('miscellaneous','subsnormal',d['subsnormal']))
4580 if 'subsverbatim' in d:
4581 self.subsverbatim = parse_options(d['subsverbatim'],SUBS_OPTIONS,
4582 'illegal [%s] %s: %s' %
4583 ('miscellaneous','subsverbatim',d['subsverbatim']))
4586 """Check the configuration for internal consistancy. Called after all
4587 configuration files have been loaded."""
4588 message.linenos = False # Disable document line numbers.
4589 # Heuristic to validate that at least one configuration file was loaded.
4590 if not self.specialchars or not self.tags or not lists:
4591 raise EAsciiDoc,'incomplete configuration files'
4592 # Check special characters are only one character long.
4593 for k in self.specialchars.keys():
4595 raise EAsciiDoc,'[specialcharacters] ' \
4596 'must be a single character: %s' % k
4597 # Check all special words have a corresponding inline macro body.
4598 for macro in self.specialwords.values():
4599 if not is_name(macro):
4600 raise EAsciiDoc,'illegal special word name: %s' % macro
4601 if not macro in self.sections:
4602 message.warning('missing special word macro: [%s]' % macro)
4603 # Check all text quotes have a corresponding tag.
4604 for q in self.quotes.keys()[:]:
4605 tag = self.quotes[q]
4607 del self.quotes[q] # Undefine quote.
4611 if not tag in self.tags:
4612 message.warning('[quotes] %s missing tag definition: %s' % (q,tag))
4613 # Check all specialsections section names exist.
4614 for k,v in self.specialsections.items():
4616 del self.specialsections[k]
4617 elif not v in self.sections:
4618 message.warning('missing specialsections section: [%s]' % v)
4619 paragraphs.validate()
4622 tables_OLD.validate()
4625 message.linenos = None
4627 def entries_section(self,section_name):
4629 Return True if conf file section contains entries, not a markup
4632 for name in self.ENTRIES_SECTIONS:
4633 if re.match(name,section_name):
4638 """Dump configuration to stdout."""
4641 hdr = hdr + '#' + writer.newline
4642 hdr = hdr + '# Generated by AsciiDoc %s for %s %s.%s' % \
4643 (VERSION,document.backend,document.doctype,writer.newline)
4644 t = time.asctime(time.localtime(time.time()))
4645 hdr = hdr + '# %s%s' % (t,writer.newline)
4646 hdr = hdr + '#' + writer.newline
4647 sys.stdout.write(hdr)
4648 # Dump special sections.
4649 # Dump only the configuration file and command-line attributes.
4650 # [miscellanous] entries are dumped as part of the [attributes].
4652 d.update(self.conf_attrs)
4653 d.update(self.cmd_attrs)
4654 dump_section('attributes',d)
4656 dump_section('quotes',self.quotes)
4657 dump_section('specialcharacters',self.specialchars)
4659 for k,v in self.specialwords.items():
4661 d[v] = '%s "%s"' % (d[v],k) # Append word list.
4664 dump_section('specialwords',d)
4665 dump_section('replacements',self.replacements)
4666 dump_section('replacements2',self.replacements2)
4667 dump_section('specialsections',self.specialsections)
4669 for k,v in self.tags.items():
4671 dump_section('tags',d)
4678 # Dump remaining sections.
4679 for k in self.sections.keys():
4680 if not self.entries_section(k):
4681 sys.stdout.write('[%s]%s' % (k,writer.newline))
4682 for line in self.sections[k]:
4683 sys.stdout.write('%s%s' % (line,writer.newline))
4684 sys.stdout.write(writer.newline)
4686 def subs_section(self,section,d):
4687 """Section attribute substitution using attributes from
4688 document.attributes and 'd'. Lines containing undefinded
4689 attributes are deleted."""
4690 if section in self.sections:
4691 return subs_attrs(self.sections[section],d)
4693 message.warning('missing section: [%s]' % section)
4696 def parse_tags(self):
4697 """Parse [tags] section entries into self.tags dictionary."""
4699 parse_entries(self.sections.get('tags',()),d)
4700 for k,v in d.items():
4705 self.tags[k] = (None,None)
4707 mo = re.match(r'(?P<stag>.*)\|(?P<etag>.*)',v)
4709 self.tags[k] = (mo.group('stag'), mo.group('etag'))
4711 raise EAsciiDoc,'[tag] %s value malformed' % k
4713 def tag(self, name, d=None):
4714 """Returns (starttag,endtag) tuple named name from configuration file
4715 [tags] section. Raise error if not found. If a dictionary 'd' is
4716 passed then merge with document attributes and perform attribute
4717 substitution on tags."""
4718 if not name in self.tags:
4719 raise EAsciiDoc, 'missing tag: %s' % name
4720 stag,etag = self.tags[name]
4722 # TODO: Should we warn if substitution drops a tag?
4724 stag = subs_attrs(stag,d)
4726 etag = subs_attrs(etag,d)
4727 if stag is None: stag = ''
4728 if etag is None: etag = ''
4731 def parse_specialsections(self):
4732 """Parse specialsections section to self.specialsections dictionary."""
4733 # TODO: This is virtually the same as parse_replacements() and should
4734 # be factored to single routine.
4736 parse_entries(self.sections.get('specialsections',()),d,unquote=True)
4737 for pat,sectname in d.items():
4738 pat = strip_quotes(pat)
4740 raise EAsciiDoc,'[specialsections] entry ' \
4741 'is not a valid regular expression: %s' % pat
4742 if sectname is None:
4743 if pat in self.specialsections:
4744 del self.specialsections[pat]
4746 self.specialsections[pat] = sectname
4748 def parse_replacements(self,sect='replacements'):
4749 """Parse replacements section into self.replacements dictionary."""
4751 parse_entries(self.sections.get(sect,()), d, unquote=True)
4752 for pat,rep in d.items():
4753 if not self.set_replacement(pat, rep, getattr(self,sect)):
4754 raise EAsciiDoc,'[%s] entry in %s is not a valid' \
4755 ' regular expression: %s' % (sect,self.fname,pat)
4758 def set_replacement(pat, rep, replacements):
4759 """Add pattern and replacement to replacements dictionary."""
4760 pat = strip_quotes(pat)
4764 if pat in replacements:
4765 del replacements[pat]
4767 replacements[pat] = strip_quotes(rep)
4770 def subs_replacements(self,s,sect='replacements'):
4771 """Substitute patterns from self.replacements in 's'."""
4773 for pat,rep in getattr(self,sect).items():
4774 result = re.sub(pat, rep, result)
4777 def parse_specialwords(self):
4778 """Parse special words section into self.specialwords dictionary."""
4779 reo = re.compile(r'(?:\s|^)(".+?"|[^"\s]+)(?=\s|$)')
4780 for line in self.sections.get('specialwords',()):
4781 e = parse_entry(line)
4783 raise EAsciiDoc,'[specialwords] entry in %s is malformed: %s' \
4786 if not is_name(name):
4787 raise EAsciiDoc,'[specialwords] name in %s is illegal: %s' \
4789 if wordlist is None:
4790 # Undefine all words associated with 'name'.
4791 for k,v in self.specialwords.items():
4793 del self.specialwords[k]
4795 words = reo.findall(wordlist)
4797 word = strip_quotes(word)
4799 raise EAsciiDoc,'[specialwords] entry in %s ' \
4800 'is not a valid regular expression: %s' \
4802 self.specialwords[word] = name
4804 def subs_specialchars(self,s):
4805 """Perform special character substitution on string 's'."""
4806 """It may seem like a good idea to escape special characters with a '\'
4807 character, the reason we don't is because the escape character itself
4808 then has to be escaped and this makes including code listings
4809 problematic. Use the predefined {amp},{lt},{gt} attributes instead."""
4812 result = result + self.specialchars.get(ch,ch)
4815 def subs_specialchars_reverse(self,s):
4816 """Perform reverse special character substitution on string 's'."""
4818 for k,v in self.specialchars.items():
4819 result = result.replace(v, k)
4822 def subs_specialwords(self,s):
4823 """Search for word patterns from self.specialwords in 's' and
4824 substitute using corresponding macro."""
4826 for word in self.specialwords.keys():
4827 result = re.sub(word, _subs_specialwords, result)
4830 def expand_templates(self,entries):
4831 """Expand any template::[] macros in a list of section entries."""
4833 for line in entries:
4834 mo = macros.match('+',r'template',line)
4836 s = mo.group('attrlist')
4837 if s in self.sections:
4838 result += self.expand_templates(self.sections[s])
4840 message.warning('missing section: [%s]' % s)
4846 def expand_all_templates(self):
4847 for k,v in self.sections.items():
4848 self.sections[k] = self.expand_templates(v)
4850 def section2tags(self, section, d={}, skipstart=False, skipend=False):
4851 """Perform attribute substitution on 'section' using document
4852 attributes plus 'd' attributes. Return tuple (stag,etag) containing
4853 pre and post | placeholder tags. 'skipstart' and 'skipend' are
4854 used to suppress substitution."""
4855 assert section is not None
4856 if section in self.sections:
4857 body = self.sections[section]
4859 message.warning('missing section: [%s]' % section)
4861 # Split macro body into start and end tag lists.
4867 mo = re.match(r'(?P<stag>.*)\|(?P<etag>.*)',s)
4869 if mo.group('stag'):
4870 stag.append(mo.group('stag'))
4871 if mo.group('etag'):
4872 etag.append(mo.group('etag'))
4878 # Do attribute substitution last so {brkbar} can be used to escape |.
4879 # But don't do attribute substitution on title -- we've already done it.
4880 title = d.get('title')
4882 d['title'] = chr(0) # Replace with unused character.
4884 stag = subs_attrs(stag, d)
4886 etag = subs_attrs(etag, d)
4887 # Put the {title} back.
4889 stag = map(lambda x: x.replace(chr(0), title), stag)
4890 etag = map(lambda x: x.replace(chr(0), title), etag)
4895 #---------------------------------------------------------------------------
4896 # Deprecated old table classes follow.
4897 # Naming convention is an _OLD name suffix.
4898 # These will be removed from future versions of AsciiDoc
4900 def join_lines_OLD(lines):
4901 """Return a list in which lines terminated with the backslash line
4902 continuation character are joined."""
4905 continuation = False
4907 if line and line[-1] == '\\':
4912 result.append(s+line)
4914 continuation = False
4924 self.colalign = None # 'left','right','center'
4925 self.rulerwidth = None
4926 self.colwidth = None # Output width in page units.
4928 class Table_OLD(AbstractBlock):
4929 COL_STOP = r"(`|'|\.)" # RE.
4930 ALIGNMENTS = {'`':'left', "'":'right', '.':'center'}
4931 FORMATS = ('fixed','csv','dsv')
4933 AbstractBlock.__init__(self)
4934 self.CONF_ENTRIES += ('template','fillchar','format','colspec',
4935 'headrow','footrow','bodyrow','headdata',
4936 'footdata', 'bodydata')
4937 # Configuration parameters.
4939 self.format=None # 'fixed','csv','dsv'
4947 # Calculated parameters.
4948 self.underline=None # RE matching current table underline.
4949 self.isnumeric=False # True if numeric ruler.
4950 self.tablewidth=None # Optional table width scale factor.
4951 self.columns=[] # List of Columns.
4953 self.check_msg='' # Message set by previous self.validate() call.
4954 def load(self,name,entries):
4955 AbstractBlock.load(self,name,entries)
4956 """Update table definition from section entries in 'entries'."""
4957 for k,v in entries.items():
4959 if v and len(v) == 1:
4962 raise EAsciiDoc,'malformed table fillchar: %s' % v
4964 if v in Table_OLD.FORMATS:
4967 raise EAsciiDoc,'illegal table format: %s' % v
4968 elif k == 'colspec':
4970 elif k == 'headrow':
4972 elif k == 'footrow':
4974 elif k == 'bodyrow':
4976 elif k == 'headdata':
4978 elif k == 'footdata':
4980 elif k == 'bodydata':
4983 AbstractBlock.dump(self)
4984 write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
4985 write('fillchar='+self.fillchar)
4986 write('format='+self.format)
4988 write('colspec='+self.colspec)
4990 write('headrow='+self.headrow)
4992 write('footrow='+self.footrow)
4993 write('bodyrow='+self.bodyrow)
4995 write('headdata='+self.headdata)
4997 write('footdata='+self.footdata)
4998 write('bodydata='+self.bodydata)
5001 AbstractBlock.validate(self)
5002 """Check table definition and set self.check_msg if invalid else set
5003 self.check_msg to blank string."""
5004 # Check global table parameters.
5005 if config.textwidth is None:
5006 self.check_msg = 'missing [miscellaneous] textwidth entry'
5007 elif config.pagewidth is None:
5008 self.check_msg = 'missing [miscellaneous] pagewidth entry'
5009 elif config.pageunits is None:
5010 self.check_msg = 'missing [miscellaneous] pageunits entry'
5011 elif self.headrow is None:
5012 self.check_msg = 'missing headrow entry'
5013 elif self.footrow is None:
5014 self.check_msg = 'missing footrow entry'
5015 elif self.bodyrow is None:
5016 self.check_msg = 'missing bodyrow entry'
5017 elif self.headdata is None:
5018 self.check_msg = 'missing headdata entry'
5019 elif self.footdata is None:
5020 self.check_msg = 'missing footdata entry'
5021 elif self.bodydata is None:
5022 self.check_msg = 'missing bodydata entry'
5027 return AbstractBlock.isnext(self)
5028 def parse_ruler(self,ruler):
5029 """Parse ruler calculating underline and ruler column widths."""
5030 fc = re.escape(self.fillchar)
5031 # Strip and save optional tablewidth from end of ruler.
5032 mo = re.match(r'^(.*'+fc+r'+)([\d\.]+)$',ruler)
5035 self.tablewidth = float(mo.group(2))
5036 self.attributes['tablewidth'] = str(float(self.tablewidth))
5038 self.tablewidth = None
5039 self.attributes['tablewidth'] = '100.0'
5040 # Guess whether column widths are specified numerically or not.
5041 if ruler[1] != self.fillchar:
5042 # If the first column does not start with a fillchar then numeric.
5043 self.isnumeric = True
5044 elif ruler[1:] == self.fillchar*len(ruler[1:]):
5045 # The case of one column followed by fillchars is numeric.
5046 self.isnumeric = True
5048 self.isnumeric = False
5049 # Underlines must be 3 or more fillchars.
5050 self.underline = r'^' + fc + r'{3,}$'
5051 splits = re.split(self.COL_STOP,ruler)[1:]
5052 # Build self.columns.
5053 for i in range(0,len(splits),2):
5055 c.colalign = self.ALIGNMENTS[splits[i]]
5058 # Strip trailing fillchars.
5059 s = re.sub(fc+r'+$','',s)
5063 c.rulerwidth = int(validate(s,'int($)>0',
5064 'malformed ruler: bad width'))
5065 else: # Calculate column width from inter-fillchar intervals.
5066 if not re.match(r'^'+fc+r'+$',s):
5067 raise EAsciiDoc,'malformed ruler: illegal fillchars'
5068 c.rulerwidth = len(s)+1
5069 self.columns.append(c)
5070 # Fill in unspecified ruler widths.
5072 if self.columns[0].rulerwidth is None:
5074 for c in self.columns:
5075 if c.rulerwidth is None:
5076 c.rulerwidth = prevwidth
5077 prevwidth = c.rulerwidth
5078 def build_colspecs(self):
5079 """Generate colwidths and colspecs. This can only be done after the
5080 table arguments have been parsed since we use the table format."""
5081 self.attributes['cols'] = len(self.columns)
5082 # Calculate total ruler width.
5084 for c in self.columns:
5085 totalwidth = totalwidth + c.rulerwidth
5087 raise EAsciiDoc,'zero width table'
5088 # Calculate marked up colwidths from rulerwidths.
5089 for c in self.columns:
5090 # Convert ruler width to output page width.
5091 width = float(c.rulerwidth)
5092 if self.format == 'fixed':
5093 if self.tablewidth is None:
5094 # Size proportional to ruler width.
5095 colfraction = width/config.textwidth
5097 # Size proportional to page width.
5098 colfraction = width/totalwidth
5100 # Size proportional to page width.
5101 colfraction = width/totalwidth
5102 c.colwidth = colfraction * config.pagewidth # To page units.
5103 if self.tablewidth is not None:
5104 c.colwidth = c.colwidth * self.tablewidth # Scale factor.
5105 if self.tablewidth > 1:
5106 c.colwidth = c.colwidth/100 # tablewidth is in percent.
5111 for c in self.columns:
5113 self.attributes['colalign'] = c.colalign
5114 self.attributes['colwidth'] = str(int(c.colwidth))
5115 self.attributes['colnumber'] = str(i + 1)
5116 s = subs_attrs(self.colspec,self.attributes)
5118 message.warning('colspec dropped: contains undefined attribute')
5121 self.attributes['colspecs'] = writer.newline.join(cols)
5122 def split_rows(self,rows):
5123 """Return a two item tuple containing a list of lines up to but not
5124 including the next underline (continued lines are joined ) and the
5125 tuple of all lines after the underline."""
5126 reo = re.compile(self.underline)
5128 while not reo.match(rows[i]):
5131 raise EAsciiDoc,'missing table rows'
5133 raise EAsciiDoc,'closing [%s] underline expected' % self.name
5134 return (join_lines_OLD(rows[:i]), rows[i+1:])
5135 def parse_rows(self, rows, rtag, dtag):
5136 """Parse rows list using the row and data tags. Returns a substituted
5137 list of output lines."""
5139 # Source rows are parsed as single block, rather than line by line, to
5140 # allow the CSV reader to handle multi-line rows.
5141 if self.format == 'fixed':
5142 rows = self.parse_fixed(rows)
5143 elif self.format == 'csv':
5144 rows = self.parse_csv(rows)
5145 elif self.format == 'dsv':
5146 rows = self.parse_dsv(rows)
5148 assert True,'illegal table format'
5149 # Substitute and indent all data in all rows.
5150 stag,etag = subs_tag(rtag,self.attributes)
5152 result.append(' '+stag)
5153 for data in self.subs_row(row,dtag):
5154 result.append(' '+data)
5155 result.append(' '+etag)
5157 def subs_row(self, data, dtag):
5158 """Substitute the list of source row data elements using the data tag.
5159 Returns a substituted list of output table data items."""
5161 if len(data) < len(self.columns):
5162 message.warning('fewer row data items then table columns')
5163 if len(data) > len(self.columns):
5164 message.warning('more row data items than table columns')
5165 for i in range(len(self.columns)):
5166 if i > len(data) - 1:
5167 d = '' # Fill missing column data with blanks.
5171 self.attributes['colalign'] = c.colalign
5172 self.attributes['colwidth'] = str(int(c.colwidth))
5173 self.attributes['colnumber'] = str(i + 1)
5174 stag,etag = subs_tag(dtag,self.attributes)
5175 # Insert AsciiDoc line break (' +') where row data has newlines
5176 # ('\n'). This is really only useful when the table format is csv
5177 # and the output markup is HTML. It's also a bit dubious in that it
5178 # assumes the user has not modified the shipped line break pattern.
5179 subs = self.get_subs()[0]
5180 if 'replacements' in subs:
5181 # Insert line breaks in cell data.
5182 d = re.sub(r'(?m)\n',r' +\n',d)
5183 d = d.split('\n') # So writer.newline is written.
5186 result = result + [stag] + Lex.subs(d,subs) + [etag]
5188 def parse_fixed(self,rows):
5189 """Parse the list of source table rows. Each row item in the returned
5190 list contains a list of cell data elements."""
5195 # build an encoded representation
5196 row = char_decode(row)
5197 for c in self.columns:
5198 end = start + c.rulerwidth
5199 if c is self.columns[-1]:
5200 # Text in last column can continue forever.
5201 # Use the encoded string to slice, but convert back
5202 # to plain string before further processing
5203 data.append(char_encode(row[start:]).strip())
5205 data.append(char_encode(row[start:end]).strip())
5209 def parse_csv(self,rows):
5210 """Parse the list of source table rows. Each row item in the returned
5211 list contains a list of cell data elements."""
5215 rdr = csv.reader(StringIO.StringIO('\r\n'.join(rows)),
5216 skipinitialspace=True)
5221 raise EAsciiDoc,'csv parse error: %s' % row
5223 def parse_dsv(self,rows):
5224 """Parse the list of source table rows. Each row item in the returned
5225 list contains a list of cell data elements."""
5226 separator = self.attributes.get('separator',':')
5227 separator = eval('"'+separator+'"')
5228 if len(separator) != 1:
5229 raise EAsciiDoc,'malformed dsv separator: %s' % separator
5230 # TODO If separator is preceeded by an odd number of backslashes then
5231 # it is escaped and should not delimit.
5235 if row == '': continue
5236 # Unescape escaped characters.
5237 row = eval('"'+row.replace('"','\\"')+'"')
5238 data = row.split(separator)
5239 data = [s.strip() for s in data]
5242 def translate(self):
5243 message.deprecated('old tables syntax')
5244 AbstractBlock.translate(self)
5245 # Reset instance specific properties.
5246 self.underline = None
5249 BlockTitle.consume(attrs)
5250 # Add relevant globals to table substitutions.
5251 attrs['pagewidth'] = str(config.pagewidth)
5252 attrs['pageunits'] = config.pageunits
5253 # Mix in document attribute list.
5254 AttributeList.consume(attrs)
5255 # Validate overridable attributes.
5256 for k,v in attrs.items():
5258 if v not in self.FORMATS:
5259 raise EAsciiDoc, 'illegal [%s] %s: %s' % (self.name,k,v)
5261 elif k == 'tablewidth':
5263 self.tablewidth = float(attrs['tablewidth'])
5265 raise EAsciiDoc, 'illegal [%s] %s: %s' % (self.name,k,v)
5266 self.merge_attributes(attrs)
5267 # Parse table ruler.
5268 ruler = reader.read()
5269 assert re.match(self.delimiter,ruler)
5270 self.parse_ruler(ruler)
5271 # Read the entire table.
5274 line = reader.read_next()
5275 # Table terminated by underline followed by a blank line or EOF.
5276 if len(table) > 0 and re.match(self.underline,table[-1]):
5277 if line in ('',None):
5280 raise EAsciiDoc,'closing [%s] underline expected' % self.name
5281 table.append(reader.read())
5282 # EXPERIMENTAL: The number of lines in the table, requested by Benjamin Klum.
5283 self.attributes['rows'] = str(len(table))
5284 if self.check_msg: # Skip if table definition was marked invalid.
5285 message.warning('skipping %s table: %s' % (self.name,self.check_msg))
5287 # Generate colwidths and colspecs.
5288 self.build_colspecs()
5289 # Generate headrows, footrows, bodyrows.
5290 # Headrow, footrow and bodyrow data replaces same named attributes in
5291 # the table markup template. In order to ensure this data does not get
5292 # a second attribute substitution (which would interfere with any
5293 # already substituted inline passthroughs) unique placeholders are used
5294 # (the tab character does not appear elsewhere since it is expanded on
5295 # input) which are replaced after template attribute substitution.
5296 headrows = footrows = []
5297 bodyrows,table = self.split_rows(table)
5300 bodyrows,table = self.split_rows(table)
5302 footrows,table = self.split_rows(table)
5304 headrows = self.parse_rows(headrows, self.headrow, self.headdata)
5305 headrows = writer.newline.join(headrows)
5306 self.attributes['headrows'] = '\x07headrows\x07'
5308 footrows = self.parse_rows(footrows, self.footrow, self.footdata)
5309 footrows = writer.newline.join(footrows)
5310 self.attributes['footrows'] = '\x07footrows\x07'
5311 bodyrows = self.parse_rows(bodyrows, self.bodyrow, self.bodydata)
5312 bodyrows = writer.newline.join(bodyrows)
5313 self.attributes['bodyrows'] = '\x07bodyrows\x07'
5314 table = subs_attrs(config.sections[self.template],self.attributes)
5315 table = writer.newline.join(table)
5316 # Before we finish replace the table head, foot and body place holders
5317 # with the real data.
5319 table = table.replace('\x07headrows\x07', headrows, 1)
5321 table = table.replace('\x07footrows\x07', footrows, 1)
5322 table = table.replace('\x07bodyrows\x07', bodyrows, 1)
5323 writer.write(table,trace='table')
5325 class Tables_OLD(AbstractBlocks):
5326 """List of tables."""
5327 BLOCK_TYPE = Table_OLD
5328 PREFIX = 'old_tabledef-'
5330 AbstractBlocks.__init__(self)
5331 def load(self,sections):
5332 AbstractBlocks.load(self,sections)
5334 # Does not call AbstractBlocks.validate().
5335 # Check we have a default table definition,
5336 for i in range(len(self.blocks)):
5337 if self.blocks[i].name == 'old_tabledef-default':
5338 default = self.blocks[i]
5341 raise EAsciiDoc,'missing section: [OLD_tabledef-default]'
5342 # Set default table defaults.
5343 if default.format is None: default.subs = 'fixed'
5344 # Propagate defaults to unspecified table parameters.
5345 for b in self.blocks:
5346 if b is not default:
5347 if b.fillchar is None: b.fillchar = default.fillchar
5348 if b.format is None: b.format = default.format
5349 if b.template is None: b.template = default.template
5350 if b.colspec is None: b.colspec = default.colspec
5351 if b.headrow is None: b.headrow = default.headrow
5352 if b.footrow is None: b.footrow = default.footrow
5353 if b.bodyrow is None: b.bodyrow = default.bodyrow
5354 if b.headdata is None: b.headdata = default.headdata
5355 if b.footdata is None: b.footdata = default.footdata
5356 if b.bodydata is None: b.bodydata = default.bodydata
5357 # Check all tables have valid fill character.
5358 for b in self.blocks:
5359 if not b.fillchar or len(b.fillchar) != 1:
5360 raise EAsciiDoc,'[%s] missing or illegal fillchar' % b.name
5361 # Build combined tables delimiter patterns and assign defaults.
5363 for b in self.blocks:
5365 # (ColStop,(ColWidth,FillChar+)?)+, FillChar+, TableWidth?
5366 b.delimiter = r'^(' + Table_OLD.COL_STOP \
5367 + r'(\d*|' + re.escape(b.fillchar) + r'*)' \
5369 + re.escape(b.fillchar) + r'+' \
5371 delimiters.append(b.delimiter)
5373 b.headrow = b.bodyrow
5375 b.footrow = b.bodyrow
5377 b.headdata = b.bodydata
5379 b.footdata = b.bodydata
5380 self.delimiters = re_join(delimiters)
5381 # Check table definitions are valid.
5382 for b in self.blocks:
5386 message.warning('[%s] table definition: %s' % (b.name,b.check_msg))
5388 # End of deprecated old table classes.
5389 #---------------------------------------------------------------------------
5391 #---------------------------------------------------------------------------
5393 #---------------------------------------------------------------------------
5394 import shutil, zipfile
5400 def unzip(zip_file, destdir):
5402 Unzip Zip file to destination directory.
5403 Throws exception if error occurs.
5405 zipo = zipfile.ZipFile(zip_file, 'r')
5407 for zi in zipo.infolist():
5408 outfile = zi.filename
5409 if not outfile.endswith('/'):
5410 d, outfile = os.path.split(outfile)
5411 directory = os.path.normpath(os.path.join(destdir, d))
5412 if not os.path.isdir(directory):
5413 os.makedirs(directory)
5414 outfile = os.path.join(directory, outfile)
5415 perms = (zi.external_attr >> 16) & 0777
5416 message.verbose('extracting: %s' % outfile)
5417 fh = os.open(outfile, os.O_CREAT | os.O_WRONLY, perms)
5419 os.write(fh, zipo.read(zi.filename))
5427 --filter option commands.
5431 def get_filters_dir():
5433 Return path of .asciidoc/filters in user's home direcory or None if
5434 user home not defined.
5438 result = os.path.join(result,'.asciidoc','filters')
5444 Install filter Zip file.
5445 args[0] is filter zip file path.
5446 args[1] is optional destination filters directory.
5448 if len(args) not in (1,2):
5449 die('invalid number of arguments: --filter install %s'
5452 if not os.path.isfile(zip_file):
5453 die('file not found: %s' % zip_file)
5454 reo = re.match(r'^\w+',os.path.split(zip_file)[1])
5456 die('filter file name does not start with legal filter name: %s'
5458 filter_name = reo.group()
5460 filters_dir = args[1]
5461 if not os.path.isdir(filters_dir):
5462 die('directory not found: %s' % filters_dir)
5464 filters_dir = Filter.get_filters_dir()
5466 die('user home directory is not defined')
5467 filter_dir = os.path.join(filters_dir, filter_name)
5468 if os.path.exists(filter_dir):
5469 die('filter is already installed: %s' % filter_dir)
5471 os.makedirs(filter_dir)
5473 die('failed to create filter directory: %s' % str(e))
5475 unzip(zip_file, filter_dir)
5477 die('failed to extract filter: %s' % str(e))
5482 Delete filter from .asciidoc/filters/ in user's home directory.
5483 args[0] is filter name.
5484 args[1] is optional filters directory.
5486 if len(args) not in (1,2):
5487 die('invalid number of arguments: --filter remove %s'
5489 filter_name = args[0]
5490 if not re.match(r'^\w+$',filter_name):
5491 die('illegal filter name: %s' % filter_name)
5494 if not os.path.isdir(d):
5495 die('directory not found: %s' % d)
5497 d = Filter.get_filters_dir()
5499 die('user directory is not defined')
5500 filter_dir = os.path.join(d, filter_name)
5501 if not os.path.isdir(filter_dir):
5502 die('cannot find filter: %s' % filter_dir)
5504 message.verbose('removing: %s' % filter_dir)
5505 shutil.rmtree(filter_dir)
5507 die('failed to delete filter: %s' % str(e))
5512 List all filter directories (global and local).
5514 for d in [os.path.join(d,'filters') for d in config.get_load_dirs()]:
5515 if os.path.isdir(d):
5516 for f in os.walk(d).next()[1]:
5517 message.stdout(os.path.join(d,f))
5520 #---------------------------------------------------------------------------
5522 #---------------------------------------------------------------------------
5525 APP_FILE = None # This file's full path.
5526 APP_DIR = None # This file's directory.
5527 USER_DIR = None # ~/.asciidoc
5528 # Global configuration files directory (set by Makefile build target).
5529 CONF_DIR = '/etc/asciidoc'
5530 HELP_FILE = 'help.conf' # Default (English) help file.
5534 document = Document() # The document being processed.
5535 config = Config() # Configuration file reader.
5536 reader = Reader() # Input stream line reader.
5537 writer = Writer() # Output stream line writer.
5538 message = Message() # Message functions.
5539 paragraphs = Paragraphs() # Paragraph definitions.
5540 lists = Lists() # List definitions.
5541 blocks = DelimitedBlocks() # DelimitedBlock definitions.
5542 tables_OLD = Tables_OLD() # Table_OLD definitions.
5543 tables = Tables() # Table definitions.
5544 macros = Macros() # Macro definitions.
5545 calloutmap = CalloutMap() # Coordinates callouts and callout list.
5546 trace = Trace() # Implements trace attribute processing.
5548 ### Used by asciidocapi.py ###
5549 # List of message strings written to stderr.
5550 messages = message.messages
5553 def asciidoc(backend, doctype, confiles, infile, outfile, options):
5554 """Convert AsciiDoc document to DocBook document of type doctype
5555 The AsciiDoc document is read from file object src the translated
5556 DocBook file written to file object dst."""
5557 def load_conffiles(include=[], exclude=[]):
5558 # Load conf files specified on the command-line and by the conf-files attribute.
5559 files = document.attributes.get('conf-files','')
5560 files = [f.strip() for f in files.split('|') if f.strip()]
5564 if os.path.isfile(f):
5565 config.load_file(f, include=include, exclude=exclude)
5567 raise EAsciiDoc,'configuration file %s missing' % f
5570 if doctype not in (None,'article','manpage','book'):
5571 raise EAsciiDoc,'illegal document type'
5572 # Set processing options.
5574 if o == '-c': config.dumping = True
5575 if o == '-s': config.header_footer = False
5576 if o == '-v': config.verbose = True
5577 document.update_attributes()
5578 if '-e' not in options:
5579 # Load asciidoc.conf files in two passes: the first for attributes
5580 # the second for everything. This is so that locally set attributes
5581 # available are in the global asciidoc.conf
5582 if not config.load_from_dirs('asciidoc.conf',include=['attributes']):
5583 raise EAsciiDoc,'configuration file asciidoc.conf missing'
5584 load_conffiles(include=['attributes'])
5585 config.load_from_dirs('asciidoc.conf')
5586 if infile != '<stdin>':
5587 indir = os.path.dirname(infile)
5588 config.load_file('asciidoc.conf', indir,
5589 include=['attributes','titles','specialchars'])
5591 load_conffiles(include=['attributes','titles','specialchars'])
5592 document.update_attributes()
5593 # Check the infile exists.
5594 if infile != '<stdin>':
5595 if not os.path.isfile(infile):
5596 raise EAsciiDoc,'input file %s missing' % infile
5597 document.infile = infile
5598 AttributeList.initialize()
5599 # Open input file and parse document header.
5600 reader.tabsize = config.tabsize
5602 has_header = document.parse_header(doctype,backend)
5603 # doctype is now finalized.
5604 document.attributes['doctype-'+document.doctype] = ''
5605 # Load backend configuration files.
5606 if '-e' not in options:
5607 f = document.backend + '.conf'
5608 if not config.find_in_dirs(f):
5609 message.warning('missing backend conf file: %s' % f, linenos=False)
5610 config.load_backend()
5611 # backend is now known.
5612 document.attributes['backend-'+document.backend] = ''
5613 document.attributes[document.backend+'-'+document.doctype] = ''
5615 if '-e' not in options:
5616 # Load filters and language file.
5617 config.load_filters()
5618 document.load_lang()
5619 if infile != '<stdin>':
5620 # Load local conf files (files in the source file directory).
5621 config.load_file('asciidoc.conf', indir)
5622 config.load_backend([indir])
5623 config.load_filters([indir])
5624 # Load document specific configuration files.
5625 f = os.path.splitext(infile)[0]
5627 f for f in (f+'.conf', f+'-'+document.backend+'.conf')
5628 if os.path.isfile(f) ]
5629 for f in doc_conffiles:
5632 # Build asciidoc-args attribute.
5634 # Add custom conf file arguments.
5635 for f in doc_conffiles + confiles:
5636 args += ' --conf-file "%s"' % f
5637 # Add command-line and header attributes.
5639 attrs.update(AttributeEntry.attributes)
5640 attrs.update(config.cmd_attrs)
5641 if 'title' in attrs: # Don't pass the header title.
5643 for k,v in attrs.items():
5645 args += ' --attribute "%s=%s"' % (k,v)
5647 args += ' --attribute "%s"' % k
5648 document.attributes['asciidoc-args'] = args
5649 # Build outfile name.
5651 outfile = os.path.splitext(infile)[0] + '.' + document.backend
5652 if config.outfilesuffix:
5653 # Change file extension.
5654 outfile = os.path.splitext(outfile)[0] + config.outfilesuffix
5655 document.outfile = outfile
5656 # Document header attributes override conf file attributes.
5657 document.attributes.update(AttributeEntry.attributes)
5658 document.update_attributes()
5659 # Configuration is fully loaded so can expand templates.
5660 config.expand_all_templates()
5661 # Check configuration for consistency.
5663 paragraphs.initialize()
5668 writer.newline = config.newline
5670 writer.open(outfile, reader.bom)
5672 document.translate(has_header) # Generate the output.
5677 except KeyboardInterrupt:
5681 if outfile and outfile != '<stdout>' and os.path.isfile(outfile):
5683 # Build and print error description.
5686 msg = message.format('', msg)
5687 if isinstance(e, EAsciiDoc):
5688 message.stderr('%s%s' % (msg,str(e)))
5690 if __name__ == '__main__':
5691 message.stderr(msg+'unexpected error:')
5692 message.stderr('-'*60)
5693 traceback.print_exc(file=sys.stderr)
5694 message.stderr('-'*60)
5696 message.stderr('%sunexpected error: %s' % (msg,str(e)))
5702 show_help('default', sys.stderr)
5704 def show_help(topic, f=None):
5705 """Print help topic to file object f."""
5709 lang = config.cmd_attrs.get('lang')
5710 if lang and lang != 'en':
5711 help_file = 'help-' + lang + '.conf'
5713 help_file = HELP_FILE
5714 # Print [topic] section from help file.
5715 config.load_from_dirs(help_file)
5716 if len(config.sections) == 0:
5717 # Default to English if specified language help files not found.
5718 help_file = HELP_FILE
5719 config.load_from_dirs(help_file)
5720 if len(config.sections) == 0:
5721 message.stderr('no help topics found')
5724 for k in config.sections:
5725 if re.match(re.escape(topic), k):
5727 lines = config.sections[k]
5729 if topic != 'topics':
5730 message.stderr('help topic not found: [%s] in %s' % (topic, help_file))
5731 message.stderr('available help topics: %s' % ', '.join(config.sections.keys()))
5734 message.stderr('ambiguous help topic: %s' % topic)
5739 ### Used by asciidocapi.py ###
5740 def execute(cmd,opts,args):
5742 Execute asciidoc with command-line options and arguments.
5743 cmd is asciidoc command or asciidoc.py path.
5744 opts and args conform to values returned by getopt.getopt().
5745 Raises SystemExit if an error occurs.
5752 >>> infile = StringIO.StringIO('Hello *{author}*')
5753 >>> outfile = StringIO.StringIO()
5755 >>> opts.append(('--backend','html4'))
5756 >>> opts.append(('--no-header-footer',None))
5757 >>> opts.append(('--attribute','author=Joe Bloggs'))
5758 >>> opts.append(('--out-file',outfile))
5759 >>> execute(__file__, opts, [infile])
5760 >>> print outfile.getvalue()
5761 <p>Hello <strong>Joe Bloggs</strong></p>
5768 usage('To many arguments')
5777 if o in ('--help','-h'):
5779 #DEPRECATED: --unsafe option.
5781 document.safe = False
5783 document.safe = True
5784 if o == '--version':
5785 print('asciidoc %s' % VERSION)
5787 if o in ('-b','--backend'):
5789 # config.cmd_attrs['backend'] = v
5790 if o in ('-c','--dump-conf'):
5791 options.append('-c')
5792 if o in ('-d','--doctype'):
5794 # config.cmd_attrs['doctype'] = v
5795 if o in ('-e','--no-conf'):
5796 options.append('-e')
5797 if o in ('-f','--conf-file'):
5799 if o in ('-n','--section-numbers'):
5802 if o in ('-a','--attribute'):
5803 e = parse_entry(v, allow_name_only=True)
5805 usage('Illegal -a option: %s' % v)
5808 # A @ suffix denotes don't override existing document attributes.
5809 if v and v[-1] == '@':
5810 document.attributes[k] = v[:-1]
5812 config.cmd_attrs[k] = v
5813 if o in ('-o','--out-file'):
5815 if o in ('-s','--no-header-footer'):
5816 options.append('-s')
5817 if o in ('-v','--verbose'):
5818 options.append('-v')
5821 show_help('default')
5825 if len(args) == 0 and len(opts) == 0:
5829 usage('No source file specified')
5832 # usage('No --backend option specified')
5834 stdin,stdout = sys.stdin,sys.stdout
5839 elif isinstance(infile, str):
5840 infile = os.path.abspath(infile)
5841 else: # Input file is file object from API call.
5845 outfile = '<stdout>'
5846 elif isinstance(outfile, str):
5847 outfile = os.path.abspath(outfile)
5848 elif outfile is None:
5849 if infile == '<stdin>':
5850 outfile = '<stdout>'
5851 else: # Output file is file object from API call.
5852 sys.stdout = outfile
5853 outfile = '<stdout>'
5855 asciidoc(backend, doctype, confiles, infile, outfile, options)
5856 if document.has_errors:
5859 sys.stdin,sys.stdout = stdin,stdout
5861 if __name__ == '__main__':
5862 # Process command line options.
5865 #DEPRECATED: --unsafe option.
5866 opts,args = getopt.getopt(sys.argv[1:],
5867 'a:b:cd:ef:hno:svw:',
5868 ['attribute=','backend=','conf-file=','doctype=','dump-conf',
5869 'help','no-conf','no-header-footer','out-file=',
5870 'section-numbers','verbose','version','safe','unsafe',
5871 'doctest','filter'])
5872 except getopt.GetoptError:
5873 message.stderr('illegal command options')
5875 if '--doctest' in [opt[0] for opt in opts]:
5876 # Run module doctests.
5878 options = doctest.NORMALIZE_WHITESPACE + doctest.ELLIPSIS
5879 failures,tries = doctest.testmod(optionflags=options)
5881 message.stderr('All doctests passed')
5885 if '--filter' in [opt[0] for opt in opts]:
5886 config.init(sys.argv[0])
5887 config.verbose = bool(set(['-v','--verbose']) & set([opt[0] for opt in opts]))
5889 die('missing --filter command')
5890 elif args[0] == 'install':
5891 Filter.install(args[1:])
5892 elif args[0] == 'remove':
5893 Filter.remove(args[1:])
5894 elif args[0] == 'list':
5897 die('illegal --filter command: %s' % args[0])
5900 execute(sys.argv[0],opts,args)
5901 except KeyboardInterrupt: