eti2wireshark.py

#!/usr/bin/env python3

# Generate Wireshark Dissectors for eletronic trading/market data
# protocols such as ETI/EOBI.
#
# Targets Wireshark 3.5 or later.
#
# SPDX-FileCopyrightText: © 2021 Georg Sauthoff <mail@gms.tf>
# SPDX-License-Identifier: GPL-2.0-or-later


import argparse
import itertools
import re
import sys
import xml.etree.ElementTree as ET


from etimodel import get_min_sizes, get_max_sizes



def get_used_types(st):
    xs = set(y.get('type') for _, x in st.items() for y in x)
    return xs

def get_data_types(d):
    r = d.getroot()
    x = r.find('DataTypes')
    h = {}
    for e in x:
        h[e.get('name')] = e
    return h

def get_structs(d):
    r = d.getroot()
    x = r.find('Structures')
    h = {}
    for e in x:
        h[e.get('name')] = e
    return h

def get_templates(st):
    ts = []
    for k, v in st.items():
        if v.get('type') == 'Message':
            ts.append((int(v.get('numericID')), k))
    ts.sort()
    return ts


def gen_header(proto, desc, o=sys.stdout):
    if proto.startswith('eti') or proto.startswith('xti'):
        ph = '#include "packet-tcp.h"    // tcp_dissect_pdus()'
    else:
        ph = '#include "packet-udp.h"    // udp_dissect_pdus()'
    print(f'''// auto-generated by Georg Sauthoff's eti2wireshark.py

/* packet-eti.c
 * Routines for {proto.upper()} dissection
 * Copyright 2021, Georg Sauthoff <mail@gms.tf>
 *
 * Wireshark - Network traffic analyzer
 * By Gerald Combs <gerald@wireshark.org>
 * Copyright 1998 Gerald Combs
 *
 * SPDX-License-Identifier: GPL-2.0-or-later
 */

/*
 * The {desc} ({proto.upper()}) is an electronic trading protocol
 * that is used by a few exchanges (Eurex, Xetra, ...).
 *
 * It's a Length-Tag based protocol consisting of mostly fix sized
 * request/response messages.
 *
 * Links:
 * https://en.wikipedia.org/wiki/List_of_electronic_trading_protocols#Europe
 * https://github.com/gsauthof/python-eti#protocol-descriptions
 * https://github.com/gsauthof/python-eti#protocol-introduction
 *
 */

#include <config.h>


#include <epan/packet.h>   // Should be first Wireshark include (other than config.h)
{ph}
#include <epan/expert.h>   // expert info

#include <inttypes.h>
#include <stdio.h>         // snprintf()


/* Prototypes */
/* (Required to prevent [-Wmissing-prototypes] warnings */
void proto_reg_handoff_{proto}(void);
void proto_register_{proto}(void);
''', file=o)


def name2ident(name):
    ll = True
    xs = []
    for i, c in enumerate(name):
        if c.isupper():
            if i > 0 and ll:
                xs.append('_')
            xs.append(c.lower())
            ll = False
        else:
            xs.append(c)
            ll = True
    return ''.join(xs)

def gen_enums(dt, ts, o=sys.stdout):
    print('static const value_string template_id_vals[] = { // TemplateID', file=o)
    min_tid, max_tid = ts[0][0], ts[-1][0]
    xs = [None] * (max_tid - min_tid + 1)
    for tid, name in ts:
        xs[tid-min_tid] = name
    for i, name in enumerate(xs):
        if name is None:
            print(f'    {{ {min_tid + i}, "Unknown" }},', file=o)
        else:
            print(f'    {{ {min_tid + i}, "{name}" }},', file=o)
    print('''    { 0, NULL }
};
static value_string_ext template_id_vals_ext = VALUE_STRING_EXT_INIT(template_id_vals);''', file=o)
    name2access = { 'TemplateID': '&template_id_vals_ext' }

    dedup = {}
    for name, e in dt.items():
        vs = [ (x.get('value'), x.get('name')) for x in e.findall('ValidValue') ]
        if not vs:
            continue
        if e.get('rootType') == 'String' and e.get('size') != '1':
            continue

        ident = name2ident(name)

        nv = e.get('noValue')
        ws = [ v[0] for v in vs ]
        if nv not in ws:
            if nv.startswith('0x0') and e.get('rootType') == 'String':
                nv = '\0'
            vs.append( (nv, 'NO_VALUE') )

        if e.get('type') == 'int':
            vs.sort(key = lambda x : int(x[0], 0))
        else:
            vs.sort(key = lambda x : ord(x[0]))
        s = '-'.join(f'{v[0]}:{v[1]}' for v in vs)
        x = dedup.get(s)
        if x is None:
            dedup[s] = name
        else:
            name2access[name] = name2access[x]
            print(f'// {name} aliased by {x}', file=o)
            continue

        print(f'static const value_string {ident}_vals[] = {{ // {name}', file=o)
        for i, v in enumerate(vs):
            if e.get('rootType') == 'String':
                k = f"'{v[0]}'" if ord(v[0]) != 0 else '0'
                print(f'''    {{ {k}, "{v[1]}" }},''', file=o)
            else:
                print(f'    {{ {v[0]}, "{v[1]}" }},', file=o)
        print('''    { 0, NULL }
};''', file=o)

        if len(vs) > 7:
            print(f'static value_string_ext {ident}_vals_ext = VALUE_STRING_EXT_INIT({ident}_vals);', file=o)
            name2access[name] = f'&{ident}_vals_ext'
        else:
            name2access[name] = f'VALS({ident}_vals)'

    return name2access


def get_fields(st, dt):
    seen = {}
    for name, e in st.items():
        for m in e:
            t = dt.get(m.get('type'))
            if is_padding(t):
                continue
            if not (is_int(t) or is_fixed_string(t) or is_var_string(t)):
                continue
            name = m.get('name')
            if name in seen:
                if seen[name] != t:
                    raise RuntimeError(f'Mismatching type for: {name}')
            else:
                seen[name] = t
    vs = list(seen.items())
    vs.sort()
    return vs

def gen_field_handles(st, dt, proto, o=sys.stdout):
    print(f'''static expert_field ei_{proto}_counter_overflow = EI_INIT;
static expert_field ei_{proto}_invalid_template = EI_INIT;
static expert_field ei_{proto}_invalid_length = EI_INIT;''', file=o)
    if not proto.startswith('eobi'):
        print(f'static expert_field ei_{proto}_unaligned = EI_INIT;', file=o)
    print(f'''static expert_field ei_{proto}_missing = EI_INIT;
static expert_field ei_{proto}_overused = EI_INIT;
''', file=o)

    vs = get_fields(st, dt)
    s = ', '.join('-1' for i in range(len(vs)))
    print(f'static int hf_{proto}[] = {{ {s} }};', file=o)
    print(f'''static int hf_{proto}_dscp_exec_summary = -1;
static int hf_{proto}_dscp_improved = -1;
static int hf_{proto}_dscp_widened = -1;''', file=o)
    print('enum Field_Handle_Index {', file=o)
    for i, (name, _) in enumerate(vs):
        c = ' ' if i == 0 else ','
        print(f'    {c} {name.upper()}_FH_IDX', file=o)
    print('};', file=o)

def type2ft(t):
    if is_timestamp_ns(t):
        return 'FT_ABSOLUTE_TIME'
    if is_dscp(t):
        return 'FT_UINT8'
    if is_int(t):
        if t.get('rootType') == 'String':
            return 'FT_CHAR'
        u = 'U' if is_unsigned(t) else ''
        if t.get('size') is None:
            raise RuntimeError(f'None size: {t.get("name")}')
        size = int(t.get('size')) * 8
        return f'FT_{u}INT{size}'
    if is_fixed_string(t) or is_var_string(t):
        # NB: technically, ETI fixed-strings are blank-padded,
        # unless they are marked NO_VALUE, in that case
        # the first byte is zero, followed by unspecified content.
        # Also, some fixed-strings are zero-terminated, where again
        # the bytes following the terminator are unspecified.
        return 'FT_STRINGZTRUNC'
    raise RuntimeError('unexpected type')

def type2enc(t):
    if is_timestamp_ns(t):
        return 'ABSOLUTE_TIME_UTC'
    if is_dscp(t):
        return 'BASE_HEX'
    if is_int(t):
        if t.get('rootType') == 'String':
            # NB: basically only used when enum and value is unknown
            return 'BASE_HEX'
        else:
            return 'BASE_DEC'
    if is_fixed_string(t) or is_var_string(t):
        # previously 'STR_ASCII', which was removed upstream
        # cf. 19dcb725b61e384f665ad4b955f3b78f63e626d9
        return 'BASE_NONE'
    raise RuntimeError('unexpected type')

def gen_field_info(st, dt, n2enum, proto='eti', o=sys.stdout):
    print('    static hf_register_info hf[] ={', file=o)
    vs = get_fields(st, dt)
    for i, (name, t) in enumerate(vs):
        c = ' ' if i == 0 else ','
        ft = type2ft(t)
        enc = type2enc(t)
        if is_enum(t) and not is_dscp(t):
            vals = n2enum[t.get('name')]
            if vals.startswith('&'):
                extra_enc =  '| BASE_EXT_STRING'
            else:
                extra_enc = ''
        else:
            vals = 'NULL'
            extra_enc = ''
        print(f'''        {c} {{ &hf_{proto}[{name.upper()}_FH_IDX],
              {{ "{name}", "{proto}.{name.lower()}",
                {ft}, {enc}{extra_enc}, {vals}, 0x0,
                NULL, HFILL }}
          }}''', file=o)
    print(f'''        , {{ &hf_{proto}_dscp_exec_summary,
              {{ "DSCP_ExecSummary", "{proto}.dscp_execsummary",
                FT_BOOLEAN, 8, NULL, 0x10,
                NULL, HFILL }}
          }}
        , {{ &hf_{proto}_dscp_improved,
              {{ "DSCP_Improved", "{proto}.dscp_improved",
                FT_BOOLEAN, 8, NULL, 0x20,
                NULL, HFILL }}
          }}
        , {{ &hf_{proto}_dscp_widened,
              {{ "DSCP_Widened", "{proto}.dscp_widened",
                FT_BOOLEAN, 8, NULL, 0x40,
                NULL, HFILL }}
          }}''', file=o)
    print('    };', file=o)


def gen_subtree_handles(st, proto='eti', o=sys.stdout):
    ns = [ name for name, e in st.items() if e.get('type') != 'Message' ]
    ns.sort()
    s = ', '.join('-1' for i in range(len(ns) + 1))
    h = dict( (n, i) for i, n in enumerate(ns, 1) )
    print(f'static gint ett_{proto}[] = {{ {s} }};', file=o)
    print(f'static gint ett_{proto}_dscp = -1;', file=o)
    return h


def gen_subtree_array(st, proto='eti', o=sys.stdout):
    n = sum(1 for name, e in st.items() if e.get('type') != 'Message')
    n += 1
    s = ', '.join(f'&ett_{proto}[{i}]' for i in range(n))
    print(f'    static gint * const ett[] = {{ {s}, &ett_{proto}_dscp }};', file=o)


def gen_fields_table(st, dt, sh, o=sys.stdout):
    name2off = {}
    off = 0
    names = []
    for name, e in st.items():
        if e.get('type') == 'Message':
            continue
        if name.endswith('Comp'):
            s = name[:-4]
        name2off[name] = off
        off += len(s) + 1
        names.append(s)
    s = '\\0'.join(names)
    print(f'    static const char struct_names[] = "{s}";', file=o)

    xs  = [ x for x in st.items() if x[1].get('type') != 'Message' ]
    xs += [ x for x in st.items() if x[1].get('type') == 'Message' ]
    print('    static const struct ETI_Field fields[] = {', file=o)
    i = 0
    fields2idx = {}
    for name, e in xs:
        fields2idx[name] = i
        print(f'        // {name}@{i}', file=o)
        counters = {}
        cnt = 0
        for m in e:
            t = dt.get(m.get('type'))
            c = ' ' if i == 0 else ','
            typ = ''
            size = int(t.get('size')) if t is not None else 0
            rep = ''
            fh = f'{m.get("name").upper()}_FH_IDX'
            sub = ''
            if is_padding(t):
                print(f'        {c} {{ ETI_PADDING, 0, {size}, 0, 0 }}', file=o)
            elif is_fixed_point(t):
                if size != 8:
                    raise RuntimeError('only supporting 8 byte fixed point')
                fraction = int(t.get('precision'))
                if fraction > 16:
                    raise RuntimeError('unusual high precisio in fixed point')
                print(f'        {c} {{ ETI_FIXED_POINT, {fraction}, {size}, {fh}, 0 }}', file=o)
            elif is_timestamp_ns(t):
                if size != 8:
                    raise RuntimeError('only supporting timestamps')
                print(f'        {c} {{ ETI_TIMESTAMP_NS, 0, {size}, {fh}, 0 }}', file=o)
            elif is_dscp(t):
                print(f'        {c} {{ ETI_DSCP, 0, {size}, {fh}, 0 }}', file=o)
            elif is_int(t):
                u = 'U' if is_unsigned(t) else ''
                if t.get('rootType') == 'String':
                    typ = 'ETI_CHAR'
                else:
                    typ = f'ETI_{u}INT'
                    if is_enum(t):
                        typ += '_ENUM'
                if t.get('type') == 'Counter':
                    counters[m.get('name')] = cnt
                    suf = f' // <- counter@{cnt}'
                    if cnt > 7:
                        raise RuntimeError(f'too many counters in message: {name}')
                    rep = cnt
                    cnt += 1
                    if typ != 'ETI_UINT':
                        raise RuntimeError('only unsigned counters supported')
                    if size > 2:
                        raise RuntimeError('only smaller counters supported')
                    typ = 'ETI_COUNTER'
                    ett_idx = t.get('maxValue')
                else:
                    rep = 0
                    suf = ''
                    ett_idx = 0
                print(f'        {c} {{ {typ}, {rep}, {size}, {fh}, {ett_idx} }}{suf}', file=o)
            elif is_fixed_string(t):
                print(f'        {c} {{ ETI_STRING, 0, {size}, {fh}, 0 }}', file=o)
            elif is_var_string(t):
                k = m.get('counter')
                x = counters[k]
                print(f'        {c} {{ ETI_VAR_STRING, {x}, {size}, {fh}, 0 }}', file=o)
            else:
                a = m.get('type')
                fields_idx = fields2idx[a]
                k = m.get('counter')
                if k:
                    counter_off = counters[k]
                    typ = 'ETI_VAR_STRUCT'
                else:
                    counter_off = 0
                    typ = 'ETI_STRUCT'
                names_off = name2off[m.get('type')]
                ett_idx = sh[a]
                print(f'        {c} {{ {typ}, {counter_off}, {names_off}, {fields_idx}, {ett_idx} }} // {m.get("name")}', file=o)
            i += 1
        print('        , { ETI_EOF, 0, 0, 0, 0 }', file=o)
        i += 1
    print('    };', file=o)
    return fields2idx

def gen_template_table(min_templateid, n, ts, fields2idx, o=sys.stdout):
    xs = [ '-1' ] * n
    for tid, name in ts:
        xs[tid - min_templateid] = f'{fields2idx[name]} /* {name} */'
    s = '\n        , '.join(xs)
    print(f'    static const int16_t tid2fidx[] = {{\n          {s}\n    }};', file=o)

def gen_sizes_table(min_templateid, n, st, dt, ts, proto, o=sys.stdout):
    is_eobi = proto.startswith('eobi')
    xs = [ '0' if is_eobi else '{ 0, 0}' ] * n
    min_s = get_min_sizes(st, dt)
    max_s = get_max_sizes(st, dt)
    if is_eobi:
        for tid, name in ts:
            xs[tid - min_templateid] = f'{max_s[name]} /* {name} */'
    else:
        for tid, name in ts:
            xs[tid - min_templateid] = f'{{ {min_s[name]}, {max_s[name]} }} /* {name} */'
    s = '\n        , '.join(xs)
    if is_eobi:
        print(f'    static const uint32_t tid2size[] = {{\n          {s}\n    }};', file=o)
    else:
        print(f'    static const uint32_t tid2size[{n}][2] = {{\n          {s}\n    }};', file=o)


# yes, usage attribute of single fields depends on the context
# otherwise, we could just put the information into the fields table
# Example: EOBI.PacketHeader.MessageHeader.MsgSeqNum is unused whereas
# it's required in the EOBI ExecutionSummary and other messages
def gen_usage_table(min_templateid, n, ts, ams, o=sys.stdout):
    def map_usage(m):
        x = m.get('usage')
        if x == 'mandatory':
            return 0
        elif x == 'optional':
            return 1
        elif x == 'unused':
            return 2
        else:
            raise RuntimeError(f'unknown usage value: {x}')

    h = {}
    i = 0
    print('    static const unsigned char usages[] = {', file=o)
    for am in ams:
        name = am.get("name")
        tid = int(am.get('numericID'))
        print(f'        // {name}', file=o)
        h[tid] = i
        for e in am:
            if e.tag == 'Group':
                print(f'        //// {e.get("type")}', file=o)
                for m in e:
                    if m.get('hidden') == 'true' or pad_re.match(m.get('name')):
                        continue
                    k = ' ' if i == 0 else ','
                    print(f'        {k} {map_usage(m)} // {m.get("name")}#{i}', file=o)
                    i += 1
                print('        ///', file=o)
            else:
                if e.get('hidden') == 'true' or pad_re.match(e.get('name')):
                    continue
                k = ' ' if i == 0 else ','
                print(f'        {k} {map_usage(e)} // {e.get("name")}#{i}', file=o)
                i += 1

    # NB: the last element is a filler to simplify the out-of-bounds check
    #     (cf. the uidx DISSECTOR_ASSER_CMPUINIT() before the switch statement)
    #     when the ETI_EOF of the message whose usage information comes last
    #     is reached
    print(f'        , 0 // filler', file=o)
    print('    };', file=o)
    xs = [ '-1' ] * n
    t2n = dict(ts)
    for tid, uidx in h.items():
        name = t2n[tid]
        xs[tid - min_templateid] = f'{uidx} /* {name} */'
    s = '\n        , '.join(xs)
    print(f'    static const int16_t tid2uidx[] = {{\n        {s}\n    }};', file=o)


def gen_dscp_table(proto, o=sys.stdout):
    print(f'''    static int * const dscp_bits[] = {{
        &hf_{proto}_dscp_exec_summary,
        &hf_{proto}_dscp_improved,
        &hf_{proto}_dscp_widened,
        NULL
    }};''', file=o)


def mk_int_case(size, signed, proto):
    signed_str = 'i' if signed else ''
    unsigned_str = '' if signed else 'u'
    fmt_str = 'i' if signed else 'u'
    if size == 2:
        size_str = 's'
    elif size == 4:
        size_str = 'l'
    elif size == 8:
        size_str = '64'
    type_str = f'g{unsigned_str}int{size * 8}'
    no_value_str = f'INT{size * 8}_MIN' if signed else f'UINT{size * 8}_MAX'
    pt_size = '64' if size == 8 else ''
    if signed:
        hex_str = '0x80' + '00' * (size - 1)
    else:
        hex_str = '0x' + 'ff' * size
    if size == 1:
        fn = f'tvb_get_g{unsigned_str}int8'
    else:
        fn = f'tvb_get_letoh{signed_str}{size_str}'
    s = f'''case {size}:
                        {{
                            {type_str} x = {fn}(tvb, off);
                            if (x == {no_value_str}) {{
                                proto_item *e = proto_tree_add_{unsigned_str}int{pt_size}_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "NO_VALUE ({hex_str})");
                                if (!usages[uidx])
                                    expert_add_info_format(pinfo, e, &ei_{proto}_missing, "required value is missing");
                            }} else {{
                                proto_item *e = proto_tree_add_{unsigned_str}int{pt_size}_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "%" PRI{fmt_str}{size * 8}, x);
                                if (usages[uidx] == 2)
                                    expert_add_info_format(pinfo, e, &ei_{proto}_overused, "unused value is set");
                            }}
                        }}
                        break;'''
    return s


def gen_dissect_structs(o=sys.stdout):
    print('''
enum ETI_Type {
    ETI_EOF,
    ETI_PADDING,
    ETI_UINT,
    ETI_INT,
    ETI_UINT_ENUM,
    ETI_INT_ENUM,
    ETI_COUNTER,
    ETI_FIXED_POINT,
    ETI_TIMESTAMP_NS,
    ETI_CHAR,
    ETI_STRING,
    ETI_VAR_STRING,
    ETI_STRUCT,
    ETI_VAR_STRUCT,
    ETI_DSCP
};

struct ETI_Field {
    uint8_t  type;
    uint8_t  counter_off;      // offset into counter array
                               // if ETI_COUNTER => storage
                               // if ETI_VAR_STRING or ETI_VAR_STRUCT => load
                               // to get length or repeat count
                               // if ETI_FIXED_POINT: #fractional digits
    uint16_t size;             // or offset into struct_names if ETI_STRUCT/ETI_VAR_STRUCT
    uint16_t field_handle_idx; // or index into fields array if ETI_STRUCT/ETI_VAR_STRUT
    uint16_t ett_idx;          // index into ett array if ETI_STRUCT/ETI_VAR_STRUCT
                               // or max value if ETI_COUNTER
};
''', file=o)

def gen_dissect_fn(st, dt, ts, sh, ams, proto, o=sys.stdout):
    if proto.startswith('eti') or proto.startswith('xti'):
        bl_fn = 'tvb_get_letohl'
        template_off = 4
    else:
        bl_fn = 'tvb_get_letohs'
        template_off = 2
    print(f'''/* This method dissects fully reassembled messages */
static int
dissect_{proto}_message(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, void *data _U_)
{{
    col_set_str(pinfo->cinfo, COL_PROTOCOL, "{proto.upper()}");
    col_clear(pinfo->cinfo, COL_INFO);
    guint16 templateid = tvb_get_letohs(tvb, {template_off});
    const char *template_str = val_to_str_ext(templateid, &template_id_vals_ext, "Unknown {proto.upper()} template: 0x%04x");
    col_add_fstr(pinfo->cinfo, COL_INFO, "%s", template_str);

    /* create display subtree for the protocol */
    proto_item *ti = proto_tree_add_item(tree, proto_{proto}, tvb, 0, -1, ENC_NA);
    guint32 bodylen= {bl_fn}(tvb, 0);
    proto_item_append_text(ti, ", %s (%" PRIu16 "), BodyLen: %u", template_str, templateid, bodylen);
    proto_tree *root = proto_item_add_subtree(ti, ett_{proto}[0]);
''', file=o)

    min_templateid = ts[0][0]
    max_templateid = ts[-1][0]
    n = max_templateid - min_templateid + 1

    fields2idx = gen_fields_table(st, dt, sh, o)
    gen_template_table(min_templateid, n, ts, fields2idx, o)
    gen_sizes_table(min_templateid, n, st, dt, ts, proto, o)
    gen_usage_table(min_templateid, n, ts, ams, o)
    gen_dscp_table(proto, o)

    print(f'''    if (templateid < {min_templateid} || templateid > {max_templateid}) {{
        proto_tree_add_expert_format(root, pinfo, &ei_{proto}_invalid_template, tvb, {template_off}, 4,
            "Template ID out of range: %" PRIu16, templateid);
        return tvb_captured_length(tvb);
    }}
    int fidx = tid2fidx[templateid - {min_templateid}];
    if (fidx == -1) {{
        proto_tree_add_expert_format(root, pinfo, &ei_{proto}_invalid_template, tvb, {template_off}, 4,
            "Unallocated Template ID: %" PRIu16, templateid);
        return tvb_captured_length(tvb);
    }}''', file=o)

    if proto.startswith('eobi'):
        print(f'''    if (bodylen != tid2size[templateid - {min_templateid}]) {{
        proto_tree_add_expert_format(root, pinfo, &ei_{proto}_invalid_length, tvb, 0, {template_off},
                "Unexpected BodyLen value of %" PRIu32 ", expected:  %" PRIu32, bodylen, tid2size[templateid - {min_templateid}]);
    }}''', file=o)
    else:
        print(f'''    if (bodylen < tid2size[templateid - {min_templateid}][0] || bodylen > tid2size[templateid - {min_templateid}][1]) {{
        if (tid2size[templateid - {min_templateid}][0] != tid2size[templateid - {min_templateid}][1])
            proto_tree_add_expert_format(root, pinfo, &ei_{proto}_invalid_length, tvb, 0, {template_off},
                    "Unexpected BodyLen value of %" PRIu32 ", expected:  %" PRIu32 "..%" PRIu32, bodylen, tid2size[templateid - {min_templateid}][0], tid2size[templateid - {min_templateid}][1]);
        else
            proto_tree_add_expert_format(root, pinfo, &ei_{proto}_invalid_length, tvb, 0, {template_off},
                    "Unexpected BodyLen value of %" PRIu32 ", expected:  %" PRIu32, bodylen, tid2size[templateid - {min_templateid}][0]);
    }}
    if (bodylen % 8)
        proto_tree_add_expert_format(root, pinfo, &ei_{proto}_unaligned, tvb, 0, {template_off},
                "BodyLen value of %" PRIu32 " is not divisible by 8", bodylen);
''', file=o)

    print(f'''    int uidx = tid2uidx[templateid - {min_templateid}];
    DISSECTOR_ASSERT_CMPINT(uidx, >=, 0);
    DISSECTOR_ASSERT_CMPUINT(((size_t)uidx), <, (sizeof usages / sizeof usages[0]));
''', file=o)

    print(f'''    int old_fidx = 0;
    int old_uidx = 0;
    unsigned top = 1;
    unsigned counter[8] = {{0}};
    unsigned off = 0;
    unsigned struct_off = 0;
    unsigned repeats = 0;
    proto_tree *t = root;
    while (top) {{
        DISSECTOR_ASSERT_CMPINT(fidx, >=, 0);
        DISSECTOR_ASSERT_CMPUINT(((size_t)fidx), <, (sizeof fields / sizeof fields[0]));
        DISSECTOR_ASSERT_CMPINT(uidx, >=, 0);
        DISSECTOR_ASSERT_CMPUINT(((size_t)uidx), <, (sizeof usages / sizeof usages[0]));

        switch (fields[fidx].type) {{
            case ETI_EOF:
                DISSECTOR_ASSERT_CMPUINT(top, >=, 1);
                DISSECTOR_ASSERT_CMPUINT(top, <=, 2);
                if (t != root)
                    proto_item_set_len(t, off - struct_off);
                if (repeats) {{
                    --repeats;
                    fidx = fields[old_fidx].field_handle_idx;
                    uidx = old_uidx;
                    t = proto_tree_add_subtree(root, tvb, off, -1, ett_{proto}[fields[old_fidx].ett_idx], NULL, &struct_names[fields[old_fidx].size]);
                    struct_off = off;
                }} else {{
                    fidx = old_fidx + 1;
                    t = root;
                    --top;
                }}
                break;
            case ETI_VAR_STRUCT:
            case ETI_STRUCT:
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].counter_off, <, sizeof counter / sizeof counter[0]);
                repeats = fields[fidx].type == ETI_VAR_STRUCT ? counter[fields[fidx].counter_off] : 1;
                if (repeats) {{
                    --repeats;
                    t = proto_tree_add_subtree(root, tvb, off, -1, ett_{proto}[fields[fidx].ett_idx], NULL, &struct_names[fields[fidx].size]);
                    struct_off = off;
                    old_fidx = fidx;
                    old_uidx = uidx;
                    fidx = fields[fidx].field_handle_idx;
                    DISSECTOR_ASSERT_CMPUINT(top, ==, 1);
                    ++top;
                }} else {{
                    ++fidx;
                }}
                break;
            case ETI_PADDING:
                off += fields[fidx].size;
                ++fidx;
                break;
            case ETI_CHAR:
                proto_tree_add_item(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, ENC_ASCII);
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_STRING:
                {{
                    guint8 c = tvb_get_guint8(tvb, off);
                    if (c)
                        proto_tree_add_item(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, ENC_ASCII);
                    else {{
                        proto_item *e = proto_tree_add_string(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, "NO_VALUE ('0x00...')");
                        if (!usages[uidx])
                            expert_add_info_format(pinfo, e, &ei_{proto}_missing, "required value is missing");
                    }}
                }}
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_VAR_STRING:
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].counter_off, <, sizeof counter / sizeof counter[0]);
                proto_tree_add_item(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, counter[fields[fidx].counter_off], ENC_ASCII);
                off += counter[fields[fidx].counter_off];
                ++fidx;
                ++uidx;
                break;
            case ETI_COUNTER:
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].counter_off, <, sizeof counter / sizeof counter[0]);
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].size, <=, 2);
                {{
                    switch (fields[fidx].size) {{
                        case 1:
                            {{
                                guint8 x = tvb_get_guint8(tvb, off);
                                if (x == UINT8_MAX) {{
                                    proto_tree_add_uint_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "NO_VALUE (0xff)");
                                    counter[fields[fidx].counter_off] = 0;
                                }} else {{
                                    proto_item *e = proto_tree_add_uint_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "%" PRIu8, x);
                                    if (x > fields[fidx].ett_idx) {{
                                        counter[fields[fidx].counter_off] = fields[fidx].ett_idx;
                                        expert_add_info_format(pinfo, e, &ei_{proto}_counter_overflow, "Counter overflow: %" PRIu8 " > %" PRIu16, x, fields[fidx].ett_idx);
                                    }} else {{
                                        counter[fields[fidx].counter_off] = x;
                                    }}
                                }}
                            }}
                            break;
                        case 2:
                            {{
                                guint16 x = tvb_get_letohs(tvb, off);
                                if (x == UINT16_MAX) {{
                                    proto_tree_add_uint_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "NO_VALUE (0xffff)");
                                    counter[fields[fidx].counter_off] = 0;
                                }} else {{
                                    proto_item *e = proto_tree_add_uint_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "%" PRIu16, x);
                                    if (x > fields[fidx].ett_idx) {{
                                        counter[fields[fidx].counter_off] = fields[fidx].ett_idx;
                                        expert_add_info_format(pinfo, e, &ei_{proto}_counter_overflow, "Counter overflow: %" PRIu16 " > %" PRIu16, x, fields[fidx].ett_idx);
                                    }} else {{
                                        counter[fields[fidx].counter_off] = x;
                                    }}
                                }}
                            }}
                            break;
                    }}
                }}
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_UINT:
                switch (fields[fidx].size) {{
                    {mk_int_case(1, False, proto)}
                    {mk_int_case(2, False, proto)}
                    {mk_int_case(4, False, proto)}
                    {mk_int_case(8, False, proto)}
                }}
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_INT:
                switch (fields[fidx].size) {{
                    {mk_int_case(1, True, proto)}
                    {mk_int_case(2, True, proto)}
                    {mk_int_case(4, True, proto)}
                    {mk_int_case(8, True, proto)}
                }}
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_UINT_ENUM:
            case ETI_INT_ENUM:
                proto_tree_add_item(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, ENC_LITTLE_ENDIAN);
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_FIXED_POINT:
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].size, ==, 8);
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].counter_off, >, 0);
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].counter_off, <=, 16);
                {{
                    gint64 x = tvb_get_letohi64(tvb, off);
                    if (x == INT64_MIN) {{
                        proto_item *e = proto_tree_add_int64_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "NO_VALUE (0x8000000000000000)");
                        if (!usages[uidx])
                            expert_add_info_format(pinfo, e, &ei_{proto}_missing, "required value is missing");
                    }} else {{
                        unsigned slack = fields[fidx].counter_off + 1;
                        if (x < 0)
                            slack += 1;
                        char s[21];
                        int n = snprintf(s, sizeof s, "%0*" PRIi64, slack, x);
                        DISSECTOR_ASSERT_CMPUINT(n, >, 0);
                        unsigned k = n - fields[fidx].counter_off;
                        proto_tree_add_int64_format_value(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, x, "%.*s.%s", k, s, s + k);
                    }}
                }}
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_TIMESTAMP_NS:
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].size, ==, 8);
                proto_tree_add_item(t, hf_{proto}[fields[fidx].field_handle_idx], tvb, off, fields[fidx].size, ENC_LITTLE_ENDIAN | ENC_TIME_NSECS);
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
            case ETI_DSCP:
                DISSECTOR_ASSERT_CMPUINT(fields[fidx].size, ==, 1);
                proto_tree_add_bitmask(t, tvb, off, hf_{proto}[fields[fidx].field_handle_idx], ett_{proto}_dscp, dscp_bits, ENC_LITTLE_ENDIAN);
                off += fields[fidx].size;
                ++fidx;
                ++uidx;
                break;
        }}
    }}
''', file=o)

    print('''    return tvb_captured_length(tvb);
}
''', file=o)

    print(f'''/* determine PDU length of protocol {proto.upper()} */
static guint
get_{proto}_message_len(packet_info *pinfo _U_, tvbuff_t *tvb, int offset, void *data _U_)
{{
    return (guint){bl_fn}(tvb, offset);
}}
''', file=o)

    if proto.startswith('eobi'):
        print(f'''static int
dissect_{proto}(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree,
        void *data)
{{
    return udp_dissect_pdus(tvb, pinfo, tree, 4, NULL,
            get_{proto}_message_len, dissect_{proto}_message, data);
}}
''', file=o)
    else:
        print(f'''static int
dissect_{proto}(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree,
        void *data)
{{
    tcp_dissect_pdus(tvb, pinfo, tree, TRUE, 4 /* bytes to read for bodylen */,
            get_{proto}_message_len, dissect_{proto}_message, data);
    return tvb_captured_length(tvb);
}}
''', file=o)

def gen_register_fn(st, dt, n2enum, proto, desc, o=sys.stdout):
    print(f'''void
proto_register_{proto}(void)
{{''', file=o)
    gen_field_info(st, dt, n2enum, proto, o)

    print(f'''    static ei_register_info ei[] = {{
        {{
            &ei_{proto}_counter_overflow,
            {{ "{proto}.counter_overflow", PI_PROTOCOL, PI_WARN, "Counter Overflow", EXPFILL }}
        }},
        {{
            &ei_{proto}_invalid_template,
            {{ "{proto}.invalid_template", PI_PROTOCOL, PI_ERROR, "Invalid Template ID", EXPFILL }}
        }},
        {{
            &ei_{proto}_invalid_length,
            {{ "{proto}.invalid_length", PI_PROTOCOL, PI_ERROR, "Invalid Body Length", EXPFILL }}
        }},''', file=o)
    if not proto.startswith('eobi'):
        print(f'''        {{
            &ei_{proto}_unaligned,
            {{ "{proto}.unaligned", PI_PROTOCOL, PI_ERROR, "A Body Length not divisible by 8 leads to unaligned followup messages", EXPFILL }}
        }},''', file=o)
    print(f'''        {{
            &ei_{proto}_missing,
            {{ "{proto}.missing", PI_PROTOCOL, PI_WARN, "A required value is missing", EXPFILL }}
        }},
        {{
            &ei_{proto}_overused,
            {{ "{proto}.overused", PI_PROTOCOL, PI_WARN, "An unused value is set", EXPFILL }}
        }}
    }};''', file=o)

    print(f'''    proto_{proto} = proto_register_protocol("{desc}",
            "{proto.upper()}", "{proto}");''', file=o)

    print(f'''    expert_module_t *expert_{proto} = expert_register_protocol(proto_{proto});
    expert_register_field_array(expert_{proto}, ei, array_length(ei));''', file=o)

    print(f'    proto_register_field_array(proto_{proto}, hf, array_length(hf));',
        file=o)
    gen_subtree_array(st, proto, o)
    print('    proto_register_subtree_array(ett, array_length(ett));', file=o)
    print('}\n', file=o)


def gen_handoff_fn(proto, o=sys.stdout):
    print(f'''void
proto_reg_handoff_{proto}(void)
{{
    dissector_handle_t {proto}_handle = create_dissector_handle(dissect_{proto},
            proto_{proto});

    // cf. N7 Network Access Guide, e.g.
    // https://www.xetra.com/xetra-en/technology/t7/system-documentation/release10-0/Release-10.0-2692700?frag=2692724
    // https://www.xetra.com/resource/blob/2762078/388b727972b5122945eedf0e63c36920/data/N7-Network-Access-Guide-v2.0.59.pdf

''', file=o)
    if proto.startswith('eti'):
        print(f'''    // NB: can only be called once for a port/handle pair ...
    // dissector_add_uint_with_preference("tcp.port", 19006 /* LF PROD */, eti_handle);

    dissector_add_uint("tcp.port", 19006 /* LF PROD */, {proto}_handle);
    dissector_add_uint("tcp.port", 19043 /* PS PROD */, {proto}_handle);
    dissector_add_uint("tcp.port", 19506 /* LF SIMU */, {proto}_handle);
    dissector_add_uint("tcp.port", 19543 /* PS SIMU */, {proto}_handle);''', file=o)
    elif proto.startswith('xti'):
        print(f'''    // NB: unfortunately, Cash-ETI shares the same ports as Derivatives-ETI ...
    //     We thus can't really add a well-know port for XTI.
    //     Use Wireshark's `Decode As...` or tshark's `-d tcp.port=19043,xti` feature
    //     to switch from ETI to XTI dissection.
    dissector_add_uint_with_preference("tcp.port", 19042 /* dummy */, {proto}_handle);''', file=o)
    else:
        print(f'''    static const int ports[] = {{
        59000, // Snapshot    EUREX US-allowed    PROD
        59001, // Incremental EUREX US-allowed    PROD
        59032, // Snapshot    EUREX US-restricted PROD
        59033, // Incremental EUREX US-restricted PROD
        59500, // Snapshot    EUREX US-allowed    SIMU
        59501, // Incremental EUREX US-allowed    SIMU
        59532, // Snapshot    EUREX US-restricted SIMU
        59533, // Incremental EUREX US-restricted SIMU

        57000, // Snapshot    FX US-allowed    PROD
        57001, // Incremental FX US-allowed    PROD
        57032, // Snapshot    FX US-restricted PROD
        57033, // Incremental FX US-restricted PROD
        57500, // Snapshot    FX US-allowed    SIMU
        57501, // Incremental FX US-allowed    SIMU
        57532, // Snapshot    FX US-restricted SIMU
        57533, // Incremental FX US-restricted SIMU

        59000, // Snapshot    Xetra PROD
        59001, // Incremental Xetra PROD
        59500, // Snapshot    Xetra SIMU
        59501, // Incremental Xetra SIMU

        56000, // Snapshot    Boerse Frankfurt PROD
        56001, // Incremental Boerse Frankfurt PROD
        56500, // Snapshot    Boerse Frankfurt SIMU
        56501  // Incremental Boerse Frankfurt SIMU
    }};
    for (unsigned i = 0; i < sizeof ports / sizeof ports[0]; ++i)
        dissector_add_uint("udp.port", ports[i], {proto}_handle);''', file=o)
    print('}', file=o)

def is_int(t):
    if t is not None:
        r = t.get('rootType')
        return r in ('int', 'floatDecimal') or (r == 'String' and t.get('size') == '1')
    return False

def is_enum(t):
    if t is not None:
        r = t.get('rootType')
        if r == 'int' or (r == 'String' and t.get('size') == '1'):
            return t.find('ValidValue') is not None
    return False

def is_fixed_point(t):
    return t is not None and t.get('rootType') == 'floatDecimal'

def is_timestamp_ns(t):
    return t is not None and t.get('type') == 'UTCTimestamp'

def is_dscp(t):
    return t is not None and t.get('name') == 'DSCP'

pad_re = re.compile('Pad[1-9]')

def is_padding(t):
    if t is not None:
        return t.get('rootType') == 'String' and pad_re.match(t.get('name'))
    return False

def is_fixed_string(t):
    if t is not None:
        return t.get('rootType') in ('String', 'data') and not t.get('variableSize')
    return False

def is_var_string(t):
    if t is not None:
        return t.get('rootType') in ('String', 'data') and t.get('variableSize') is not None
    return False

def is_unsigned(t):
    v = t.get('minValue')
    return v is not None and not v.startswith('-')

def is_counter(t):
    return t.get('type') == 'Counter'

def type_to_fmt(t):
    if is_padding(t):
        return f'{t.get("size")}x'
    elif is_int(t):
        n = int(t.get('size'))
        if n == 1:
            return 'B'
        else:
            if n == 2:
                c = 'h'
            elif n == 4:
                c = 'i'
            elif n == 8:
                c = 'q'
            else:
                raise ValueError(f'unknown int size {n}')
            if is_unsigned(t):
                c = c.upper()
            return c
    elif is_fixed_string(t):
        return f'{t.get("size")}s'
    else:
        return '?'

def pp_int_type(t):
    if not is_int(t):
        return None
    s = 'i'
    if is_unsigned(t):
        s = 'u'
    n = int(t.get('size'))
    s += str(n)
    return s

def is_elementary(t):
    return t is not None and t.get('counter') is None

def group_members(e, dt):
    xs = []
    ms = []
    for m in e:
        t = dt.get(m.get('type'))
        if is_elementary(t):
            ms.append(m)
        else:
            if ms:
                xs.append(ms)
                ms = []
            xs.append([m])
    if ms:
        xs.append(ms)
    return xs



def parse_args():
    p = argparse.ArgumentParser(description='Generate Wireshark Dissector for ETI/EOBI style protocol specifictions')
    p.add_argument('filename', help='protocol description XML file')
    p.add_argument('--proto', default='eti',
            help='short protocol name (default: %(default)s)')
    p.add_argument('--desc', '-d',
            default='Enhanced Trading Interface',
            help='protocol description (default: %(default)s)')
    p.add_argument('--output', '-o', default='-',
            help='output filename (default: stdout)')
    args = p.parse_args()
    return args

def main():
    args = parse_args()
    filename = args.filename
    d = ET.parse(filename)
    o = sys.stdout if args.output == '-' else open(args.output, 'w')
    proto = args.proto

    version = (d.getroot().get('version'), d.getroot().get('subVersion'))
    desc = f'{args.desc} {version[0]}'

    dt = get_data_types(d)
    st = get_structs(d)
    used = get_used_types(st)
    for k in list(dt.keys()):
        if k not in used:
            del dt[k]
    ts = get_templates(st)
    ams = d.getroot().find('ApplicationMessages')

    gen_header(proto, desc, o)
    print(f'static int proto_{proto} = -1;', file=o)
    gen_field_handles(st, dt, proto, o)
    n2enum = gen_enums(dt, ts, o)
    gen_dissect_structs(o)
    sh = gen_subtree_handles(st, proto, o)
    gen_dissect_fn(st, dt, ts, sh, ams, proto, o)
    gen_register_fn(st, dt, n2enum, proto, desc, o)
    gen_handoff_fn(proto, o)


if __name__ == '__main__':
    sys.exit(main())