kea/src/bin/auth/gen-statisticsitems.py.pre.in

#!@PYTHON@

# Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

"""\
This script generates spec file, docbook XML and some part of statistics code
from statistics_msg_items.def.
"""

import os
import re
import sys
import json
from xml.etree import ElementTree

item_list = []
localstatedir = '@@LOCALSTATEDIR@@'
builddir = '@builddir@'
srcdir = '@srcdir@'
pre_suffix = '.pre'

xmldocument_command_name = 'b10-auth'

def need_generate(filepath, prepath, mtime):
    '''Check if we need to generate the specified file.

    To avoid unnecessary compilation, we skip (re)generating the file when
    the file already exists and newer than the base file, and definition file
    specified with mtime.
    '''
    if os.path.exists(filepath) and\
        (os.path.getmtime(filepath) > mtime and
         os.path.getmtime(filepath) > os.path.getmtime(prepath)):
        return False
    return True

def import_definitions():
    '''Load statsitics items definitions from statistics_msg_items.def.

    statistics_msg_items.def defines a tree of message statistics items.
    Syntax:
        Each line describes a node; branch node for subset of counters,
        leaf node for a counter item.
        Each fields are separated with one or more field separator (Tab).
        Field separator in the head of a line is ignored.

        branch node:
        (item name)\t+(internal branch name)\t+(description of the item)\t+'='
        leaf node:
        (item name)\t+(internal item counter name)\t+(description of the item)

        Branch nodes contain leaf nodes and/or branch nodes. The end of
        a branch node is indicated with ';' as item name (first column).

        Internal branch name and internal item counter name must be unique.

    Returns mtime of statistics_msg_items.def. It will be used to check
    auto-generated files need to be regenerated.
    '''
    global item_list

    items_definition_file = srcdir + os.sep + 'statistics_msg_items.def'
    with open(items_definition_file, 'r') as item_definition:
        re_splitter = re.compile('\t+')
        l = item_list
        lp = None
        for line in item_definition.readlines():
            element = re_splitter.split(line.rstrip())
            # pop first element if it is empty to skip indent
            if element[0] == '':
                element.pop(0)

            # last element is '=': a branch node definition.
            if element[-1] == '=':
                l.append({'name': element[0], 'child': [], 'index': element[1],
                          'description': element[2], 'parent': lp})
                lp = l
                l = l[-1]['child']
            # last element is ';': end of a branch node.
            elif element[-1] == ';':
                l = lp
                lp = l[-1]['parent']
            # otherwise, a leaf node definition.
            else:
                l.append({'name': element[0], 'child': None,
                          'index': element[1], 'description': element[2],
                          'parent': lp})
    return os.path.getmtime(items_definition_file)

def generate_specfile(specfile, def_mtime):
    '''Generate spec in specfile from skeleton (specfille+'.pre').
    If the specfile is newer than both skeleton and def_mtime, file generation
    will be skipped.

    This method reads the content of skeleton and appends statistics items
    definition into { "module_spec": { "statistics": } }.
    LOCALSTATEDIR is also expanded.

    Returns nothing.
    '''

    def convert_list(items, prefix=''):
        spec_list = []
        default_map = {}
        for item in items:
            full_item_name = prefix + item['name']
            if item['child'] is None:
                default_map[item['name']] = 0
                spec_list.append({
                        'item_name': item['name'],
                        'item_optional': False,
                        'item_type': 'integer',
                        'item_default': 0,
                        'item_title': full_item_name,
                        'item_description': item['description'],
                    })
            else:
                child_spec_list, child_default_map = \
                    convert_list(item['child'], full_item_name + '.')
                spec_list.append({
                        'item_name': item['name'],
                        'item_type': 'map',
                        'item_optional': False,
                        'item_title': full_item_name,
                        'item_description': item['description'],
                        'item_default': child_default_map,
                        'map_item_spec': child_spec_list,
                    })
                default_map[item['name']] = child_default_map
        return spec_list, default_map

    item_spec_list, item_default_map = convert_list(item_list)

    statistics_spec_list = [{
        'item_name': 'zones',
        'item_type': 'named_set',
        'item_optional': False,
        'item_title': 'Zone statistics',
        'item_description':
                'Zone statistics items. ' +
                "Items for all zones are stored in '_SERVER_'.",
        'item_default': { '_SERVER_': item_default_map },
        'named_set_item_spec': {
            'item_name': 'zone',
            'item_type': 'map',
            'item_optional': False,
            'item_default': {},
            'map_item_spec': item_spec_list,
            },
        }]

    if need_generate(builddir+os.sep+specfile,
                     builddir+os.sep+specfile+pre_suffix, def_mtime):
        with open(builddir+os.sep+specfile+pre_suffix, 'r') as stats_pre:
            # split LOCALSTATEDIR to avoid substitution
            stats_pre_json = \
                json.loads(stats_pre.read().replace('@@LOCAL'+'STATEDIR@@',
                                                    localstatedir))
        stats_pre_json['module_spec']['statistics'] = statistics_spec_list
        statistics_spec_json = json.dumps(stats_pre_json, sort_keys=True,
                                          indent=2)
        with open(builddir+os.sep+specfile, 'w') as stats_spec:
            stats_spec.write(statistics_spec_json)
    else:
        print('skip generating ' + specfile)
    return

def generate_docfile(docfile, def_mtime):
    '''Generate docbook XML in docfile from skeleton (docfile+'.pre').
    If the docfile is newer than both skeleton and def_mtime, file generation
    will be skipped.

    This method reads the content of skeleton and replaces
    <!-- ### STATISTICS DATA PLACEHOLDER ### --> with statistics items
    definition. LOCALSTATEDIR is also expanded.

    Returns nothing.
    '''
    def convert_list(items, tree, prefix=''):
        '''
        Build XML tree from items.
            <varlistentry>
              <term>##item_full_name##</term>
              <listitem><simpara>##item_description##</simpara></listitem>
            </varlistentry>
        xmldocument_command_name in item_description is put inside <command>
        element.
        '''
        for item in items:
            full_item_name = prefix + item['name']
            if item['child'] is None:
                # the item is a leaf node: build varlistentry
                child_element = ElementTree.SubElement(tree, 'varlistentry')
                term = ElementTree.SubElement(child_element, 'term')
                term.text = full_item_name
                list_item = ElementTree.SubElement(child_element, 'listitem')
                sim_para = ElementTree.SubElement(list_item, 'simpara')
                sim_para.text = ''
                prev = None
                # put xmldocument_command_name in <command> node
                for word in item['description'].split():
                    if word == xmldocument_command_name:
                        command = ElementTree.SubElement(sim_para, 'command')
                        command.text = word
                        # at this point command.tail is None
                        # append a space as trailing text for the next word
                        # so it can be concatenated with trailing words
                        command.tail = ' '
                        prev = command
                    else:
                        if prev is None:
                            sim_para.text += word + ' '
                        else:
                            prev.tail += word + ' '
                # remove trailing whitespaces at the end of the description
                if prev is None:
                    sim_para.text = sim_para.text.rstrip()
                else:
                    prev.tail = prev.tail.rstrip()
            else:
                # the item is a branch node: call myself for child nodes
                convert_list(item['child'], tree, full_item_name + '.')
        return

    if need_generate(builddir+os.sep+docfile,
                     srcdir+os.sep+docfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+docfile+pre_suffix, 'r') as doc_pre:
            # split LOCALSTATEDIR to avoid substitution
            doc_pre_xml = doc_pre.read().replace('@@LOCAL'+'STATEDIR@@',
                                                 localstatedir)

        variable_tree = ElementTree.Element('variablelist')
        convert_list(item_list, variable_tree)
        pretty_xml = ElementTree.tostring(variable_tree)
        if not isinstance(pretty_xml, str):
            pretty_xml = pretty_xml.decode('utf-8')
        # put newline around <variablelist> and <varlistentry> element
        pretty_xml = \
            re.sub(r'(</?var(?:iablelist|listentry)>)', r'\1\n', pretty_xml)
        # indent <term> and <listitem>
        pretty_xml = \
            re.sub(r'(<(?:term|listitem)>)', r'  \1', pretty_xml)
        # put newline after </term> and </listitem>
        pretty_xml = \
            re.sub(r'(</(?:term|listitem)>)', r'\1\n', pretty_xml)

        with open(builddir+os.sep+docfile, 'w') as doc:
            doc.write(doc_pre_xml.replace(
                '<!-- ### STATISTICS DATA PLACEHOLDER ### -->',
                pretty_xml))
    else:
        print('skip generating ' + docfile)
    return

def generate_cxx(itemsfile, ccfile, utfile, def_mtime):
    '''Generate some part of statistics code in itemsfile, ccfile, utfile from
    skeleton (itemsfile+'.pre', ccfile+'.pre', utfile+'.pre').
    If the file is newer than both skeleton and def_mtime, file generation
    will be skipped.

    This method reads the content of skeleton and replaces
    // ### STATISTICS ITEMS DEFINITION ### with statistics items definition in
    ccfile and utfile,
    // ### STATISTICS ITEMS DECLARATION ### with statistics items declaration
    in itemsfile.

    Returns nothing.
    '''
    msg_counter_types = ['enum MSGCounterType {']
    item_names =  ['// using -1 as counter_id to state it is not a '
                   + 'counter item']
    item_names += ['const int NOT_ITEM = -1;', '']

    def convert_list(items, item_head, msg_counter_types, item_names):
        '''Convert item tree to a set of C++ code fragment in given lists.

        This method recursively builds two lists:
        - msg_counter_types consists of strings for all leaf items, each
          defines one enum element with a comment, e.g.
          COUNTER_ITEM, ///< item's description
        - item_names consists of tuples of three elements, depending on
          whether it's a leaf element (no child from it) or not:
          (leaf)   ( "item_name", NULL, COUNTER_ITEM )
          (branch) ( "item_name", CHILD_NAME, NOT_ITEM )

        Each single call to this function builds a C++ structure beginning
        with the given item_head, which is a string that reads like
        'const struct CounterSpec some_counters[] = {'
        followed by the item_names tuples corresponding to that level.
        If some of the items of this level have a child, recursive calls
        to this function extends msg_counter_types and item_names.
        item_names for this level will be concatenated at the end end of
        the given item_names.

        '''
        item_names_current = [item_head]
        for item in items:
            item_spec = '    { "' + item['name'] + '", '
            if item['child'] is None:
                item_spec += 'NULL, ' + item['index']
                msg_counter_types.append('    ' + item['index'] + ',    ' +
                                         '///< ' + item['description'])
            else:
                item_spec += item['index'] + ', NOT_ITEM'
                child_head = 'const struct CounterSpec ' + \
                    item['index'] + '[] = {'
                convert_list(item['child'], child_head,
                             msg_counter_types, item_names)
            item_names_current.append(item_spec + ' },')

        item_names_current.append('    { NULL, NULL, NOT_ITEM }\n};')
        item_names.extend(item_names_current)

    convert_list(item_list, 'const struct CounterSpec msg_counter_tree[] = {',
                 msg_counter_types, item_names)
    msg_counter_types.extend([
            '    // End of counter types',
            '    MSG_COUNTER_TYPES  ///< The number of defined counters',
            '};'])

    item_decls = '\n'.join(msg_counter_types)
    item_defs = '\n'.join(item_names)

    if need_generate(builddir+os.sep+itemsfile,
                     srcdir+os.sep+itemsfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+itemsfile+pre_suffix, 'r') \
            as statistics_items_h_pre:
            items_pre = statistics_items_h_pre.read()

        with open(builddir+os.sep+itemsfile, 'w') as statistics_items_h:
            statistics_items_h.write(items_pre.replace(
                '// ### STATISTICS ITEMS DECLARATION ###', item_decls))
    else:
        print('skip generating ' + itemsfile)

    if need_generate(builddir+os.sep+ccfile,
                     srcdir+os.sep+ccfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+ccfile+pre_suffix, 'r') as statistics_cc_pre:
            items_pre = statistics_cc_pre.read()

        with open(builddir+os.sep+ccfile, 'w') as statistics_cc:
            statistics_cc.write(items_pre.replace(
                '// ### STATISTICS ITEMS DEFINITION ###', item_defs))
    else:
        print('skip generating ' + ccfile)

    if need_generate(builddir+os.sep+utfile,
                     srcdir+os.sep+utfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+utfile+pre_suffix, 'r') \
            as statistics_ut_cc_pre:
            items_pre = statistics_ut_cc_pre.read()

        with open(builddir+os.sep+utfile, 'w') as statistics_ut_cc:
            statistics_ut_cc.write(items_pre.replace(
                '// ### STATISTICS ITEMS DEFINITION ###', item_defs))
    else:
        print('skip generating ' + utfile)

    return

if __name__ == "__main__":
    try:
        def_mtime = import_definitions()
        generate_specfile('auth.spec', def_mtime)
        generate_docfile('b10-auth.xml', def_mtime)
        generate_cxx('statistics_items.h',
                     'statistics.cc',
                     'tests'+os.sep+'statistics_unittest.cc',
                     def_mtime)
    except:
        sys.stderr.write('File generation failed due to exception: %s\n' %
                         sys.exc_info()[1])
        exit(1)