kea/src/bin/auth/gen-statisticsitems.py.pre.in

#!@PYTHON@

# Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

"""\
This script generates spec file, docbook XML and some part of statistics code
from statistics_msg_items.def.
"""

import os
import re
import sys
import json
from xml.etree import ElementTree
from xml.dom.minidom import parseString

item_list = []
localstatedir = '@@LOCALSTATEDIR@@'
builddir = '@builddir@'
srcdir = '@srcdir@'
pre_suffix = '.pre'

xmldocument_command_name = 'b10-auth'

def need_generate(filepath, prepath, mtime):
    '''Check if we need to generate the specified file.

    To avoid unnecessary compilation, we skip (re)generating the file when
    the file already exists and newer than the base file, and definition file
    specified with mtime.
    '''
    if os.path.exists(filepath) and\
        (os.path.getmtime(filepath) > mtime and
         os.path.getmtime(filepath) > os.path.getmtime(prepath)):
        return False
    return True

def import_definitions():
    '''Load statsitics items definitions from statistics_msg_items.def.

    statistics_msg_items.def defines a tree of message statistics items.
    Syntax:
        Each line describes a node; branch node for subset of counters,
        leaf node for a counter item.
        Each fields are separated with one or more field separator (Tab).
        Field separator in the head of a line is ignored.

        branch node:
        (item name)\t+(internal branch name)\t+(description of the item)\t+'='
        leaf node:
        (item name)\t+(internal item counter name)\t+(description of the item)

        Branch nodes contains leaf nodes and/or branch nodes. The end of
        a branch node is indicated with ';' as item name (first column).

        Internal branch name and internal item counter name must be unique.

    Returns mtime of statistics_msg_items.def. It will be used to check
    auto-generated files need to be regenerated.
    '''
    global item_list

    items_definition_file = srcdir + os.sep + 'statistics_msg_items.def'
    with open(items_definition_file, 'r') as item_definition:
        re_splitter = re.compile('\t+')
        l = item_list
        lp = None
        for line in item_definition.readlines():
            element = re_splitter.split(line.rstrip())
            # pop first element if it is empty to skip indent
            if element[0] == '':
                element.pop(0)

            # last element is '=': a branch node definition.
            if element[-1] == '=':
                l.append({'name': element[0], 'child': [], 'index': element[1],
                          'description': element[2], 'parent': lp})
                lp = l
                l = l[-1]['child']
            # last element is ';': end of a branch node.
            elif element[-1] == ';':
                l = lp
                lp = l[-1]['parent']
            # otherwise, a leaf node definition.
            else:
                l.append({'name': element[0], 'child': None,
                          'index': element[1], 'description': element[2],
                          'parent': lp})
    return os.path.getmtime(items_definition_file)

def generate_specfile(specfile, def_mtime):
    '''Generate spec in specfile from skeleton (specfille+'.pre').
    If the specfile is newer than both skeleton and def_mtime, file generation
    will be skipped.

    This method reads the content of skeleton and appends statistics items
    definition into { "module_spec": { "statistics": } }.
    LOCALSTATEDIR is also expanded.

    Returns nothing.
    '''

    def convert_list(items, prefix=''):
        spec_list = []
        default_map = {}
        for item in items:
            full_item_name = prefix + item['name']
            if item['child'] is None:
                default_map[item['name']] = 0
                spec_list.append({
                        'item_name': item['name'],
                        'item_optional': False,
                        'item_type': 'integer',
                        'item_default': 0,
                        'item_title': full_item_name,
                        'item_description': item['description'],
                    })
            else:
                child_spec_list, child_default_map = \
                    convert_list(item['child'], full_item_name + '.')
                spec_list.append({
                        'item_name': item['name'],
                        'item_type': 'map',
                        'item_optional': False,
                        'item_title': full_item_name,
                        'item_description': item['description'],
                        'item_default': child_default_map,
                        'map_item_spec': child_spec_list,
                    })
                default_map[item['name']] = child_default_map
        return spec_list, default_map

    item_spec_list, item_default_map = convert_list(item_list)

    statistics_spec_list = [{
        'item_name': 'zones',
        'item_type': 'named_set',
        'item_optional': False,
        'item_title': 'Zone statistics',
        'item_description':
                'Zone statistics items. ' +
                "Items for all zones are stored in '_SERVER_'.",
        'item_default': { '_SERVER_': item_default_map },
        'named_set_item_spec': {
            'item_name': 'zone',
            'item_type': 'map',
            'item_optional': False,
            'item_default': {},
            'map_item_spec': item_spec_list,
            },
        }]

    if need_generate(builddir+os.sep+specfile,
                     builddir+os.sep+specfile+pre_suffix, def_mtime):
        with open(builddir+os.sep+specfile+pre_suffix, 'r') as stats_pre:
            # split LOCALSTATEDIR to avoid substitution
            stats_pre_json = \
                json.loads(stats_pre.read().replace('@@LOCAL'+'STATEDIR@@',
                                                    localstatedir))
        stats_pre_json['module_spec']['statistics'] = statistics_spec_list
        statistics_spec_json = json.dumps(stats_pre_json, sort_keys=True,
                                          indent=2)
        with open(builddir+os.sep+specfile, 'w') as stats_spec:
            stats_spec.write(statistics_spec_json)
    else:
        print('skip generating ' + specfile)
    return

def generate_docfile(docfile, def_mtime):
    '''Generate docbook XML in docfile from skeleton (docfile+'.pre').
    If the docfile is newer than both skeleton and def_mtime, file generation
    will be skipped.

    This method reads the content of skeleton and replaces
    <!-- ### STATISTICS DATA PLACEHOLDER ### --> with statistics items
    definition. LOCALSTATEDIR is also expanded.

    Returns nothing.
    '''
    def convert_list(items, tree, prefix=''):
        '''
        Build XML tree from items.
            <varlistentry>
              <term>##item_full_name##</term>
              <listitem>
              <simpara>
              ##item_description##
              </simpara>
              </listitem>
            <varlistentry>
        xmldocument_command_name in item_description is surrounded with
        <command>.
        '''
        for item in items:
            full_item_name = prefix + item['name']
            if item['child'] is None:
                # the item is a leaf node: build varlistentry
                child_element = ElementTree.SubElement(tree, 'varlistentry')
                term = ElementTree.SubElement(child_element, 'term')
                term.text = full_item_name
                list_item = ElementTree.SubElement(child_element, 'listitem')
                sim_para = ElementTree.SubElement(list_item, 'simpara')
                sim_para.text = ''
                prev = None
                # put xmldocument_command_name in <command> node
                for word in item['description'].split():
                    if word == xmldocument_command_name:
                        command = ElementTree.SubElement(sim_para, 'command')
                        command.text = word
                        # append empty string for trailing text
                        command.tail = ''
                        prev = command
                    else:
                        if prev is None:
                            sim_para.text += word + ' '
                        else:
                            prev.tail += word + ' '
                # remove extra trailing whitespaces
                sim_para.text = sim_para.text.rstrip()
                if prev is not None:
                    prev.tail = prev.tail.rstrip()
            else:
                # the item is a branch node: call myself for child nodes
                convert_list(item['child'], tree, full_item_name + '.')
        return

    if need_generate(builddir+os.sep+docfile,
                     srcdir+os.sep+docfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+docfile+pre_suffix, 'r') as doc_pre:
            # split LOCALSTATEDIR to avoid substitution
            doc_pre_xml = doc_pre.read().replace('@@LOCAL'+'STATEDIR@@',
                                                 localstatedir)

        variable_tree = ElementTree.Element('variablelist')
        convert_list(item_list, variable_tree)
        pretty_xml = \
            parseString(ElementTree.tostring(variable_tree)).toprettyxml()
        # remove XML declaration
        pretty_xml = re.sub('<\?xml[^?]+\?>', '', pretty_xml)
        # remove extra whitespaces inside <command> and <term>
        pretty_xml = \
            re.sub(r'<(command|term)>\s+([^<\s]+)\s+</\1>', r'<\1>\2</\1>',
                   pretty_xml)
        with open(builddir+os.sep+docfile, 'w') as doc:
            doc.write(doc_pre_xml.replace(
                '<!-- ### STATISTICS DATA PLACEHOLDER ### -->',
                pretty_xml))
    else:
        print('skip generating ' + docfile)
    return

def generate_cxx(itemsfile, ccfile, utfile, def_mtime):
    '''Generate some part of statistics code in itemsfile, ccfile, utfile from
    skeleton (itemsfile+'.pre', ccfile+'.pre', utfile+'.pre').
    If the file is newer than both skeleton and def_mtime, file generation
    will be skipped.

    This method reads the content of skeleton and replaces
    // ### STATISTICS ITEMS DEFINITION ### with statistics items definition in
    ccfile and utfile,
    // ### STATISTICS ITEMS DECLARATION ### with statistics items declaration
    in itemsfile.

    Returns nothing.
    '''
    msg_counter_types = 'enum MSGCounterType {\n'
    item_names =  ['// using -1 as counter_id to state it is not a '
                   + 'counter item\n']
    item_names += ['const int NOT_ITEM = -1;\n', '\n']

    def convert_list(items, msg_counter_types, item_names_current, item_names):
        for item in items:
            if item['child'] is None:
                msg_counter_types += '    ' + item['index'] + ',    ' +\
                                    '///< ' + item['description'] + '\n'
                item_names_current.append('    { "' + item['name'] +
                                          '", NULL, ' + item['index'] + ' },\n'
                                          )
            else:
                item_names_current_ = ['const struct CounterSpec ' +
                                       item['index'] + '[] = {\n']
                msg_counter_types, item_names_current_, item_names = \
                        convert_list(item['child'], msg_counter_types,
                                     item_names_current_, item_names)
                item_names_current_.append('    { NULL, NULL, NOT_ITEM }\n' +
                              '};\n')
                item_names.extend(item_names_current_)
                item_names_current.append('    { "' + item['name'] + '", ' +
                                          item['index'] + ', NOT_ITEM },\n')
        return msg_counter_types, item_names_current, item_names

    msg_counter_types, item_names_current, item_names = \
            convert_list(item_list, msg_counter_types, [], item_names)
    item_names.append('const struct CounterSpec msg_counter_tree[] = {\n')
    item_names.extend(item_names_current)
    item_names.append('    { NULL, NULL, NOT_ITEM }\n' +
                  '};\n')

    msg_counter_types += \
        '    // End of counter types\n' +\
        '    MSG_COUNTER_TYPES  ///< The number of defined counters\n' +\
        '};\n'

    item_decls = msg_counter_types
    item_defs = ''.join(item_names)

    if need_generate(builddir+os.sep+itemsfile,
                     srcdir+os.sep+itemsfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+itemsfile+pre_suffix, 'r') \
            as statistics_items_h_pre:
            items_pre = statistics_items_h_pre.read()

        with open(builddir+os.sep+itemsfile, 'w') as statistics_items_h:
            statistics_items_h.write(items_pre.replace(
                '// ### STATISTICS ITEMS DECLARATION ###', item_decls))
    else:
        print('skip generating ' + itemsfile)

    if need_generate(builddir+os.sep+ccfile,
                     srcdir+os.sep+ccfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+ccfile+pre_suffix, 'r') as statistics_cc_pre:
            items_pre = statistics_cc_pre.read()

        with open(builddir+os.sep+ccfile, 'w') as statistics_cc:
            statistics_cc.write(items_pre.replace(
                '// ### STATISTICS ITEMS DEFINITION ###', item_defs))
    else:
        print('skip generating ' + ccfile)

    if need_generate(builddir+os.sep+utfile,
                     srcdir+os.sep+utfile+pre_suffix, def_mtime):
        with open(srcdir+os.sep+utfile+pre_suffix, 'r') \
            as statistics_ut_cc_pre:
            items_pre = statistics_ut_cc_pre.read()

        with open(builddir+os.sep+utfile, 'w') as statistics_ut_cc:
            statistics_ut_cc.write(items_pre.replace(
                '// ### STATISTICS ITEMS DEFINITION ###', item_defs))
    else:
        print('skip generating ' + utfile)

    return

if __name__ == "__main__":
    try:
        def_mtime = import_definitions()
        generate_specfile('auth.spec', def_mtime)
        generate_docfile('b10-auth.xml', def_mtime)
        generate_cxx('statistics_items.h',
                     'statistics.cc',
                     'tests'+os.sep+'statistics_unittest.cc',
                     def_mtime)
    except:
        sys.stderr.write('File generation failed due to exception: %s\n' %
                         sys.exc_info()[1])
        exit(1)