#!/usr/bin/env python3
"""probe-compare - compare a contract-probe.json with an expectation file.

The document is the status file the service writes with status_write: the
envelope of schema aep.status/v1 ("schema", "generated_at") with the report
of schema aep.contract-probe/v1 under "report". The expectation file has the
same form; the report in it has the format of ADL-DEMO-SERVICE.md section 5.
Its values are either literal (must be equal) or placeholder rules. The rules
that refer to the document (@fqdn, @one_of_nameservers) read the report. A
document without envelope (the bare report) is compared the same way with an
expectation without envelope.

  whole value
    @any                 any value, the key must exist
    @list                a list (any content, may be empty)
    @includes[a,b,...]   a list that contains every named entry
    @set[a,b,...]        a list with exactly the named entries, in any order
    @match[regex]        a string the regular expression matches as a whole
    @hex12               twelve lower case hex digits
    @uuid                a UUID (RFC 4122, lower case), e.g. the object
                         UIDs svc-ctrl writes into the service.aep.asag.io
                         labels
    @unixtime            integer > 0 (seconds, generated_at of the envelope)
    @annotation          the contract hash: non empty string, every field
                         marked @annotation carries the same value; with
                         --live annotation=<v> equal to <v>
    @generation          integer >= 1, every field marked @generation
                         carries the same value; with --live generation=<n>
                         equal to <n>
    @fqdn                the pod FQDN of the document (identity.fqdn)
    @instance            the instance name of the document
                         (identity.instance; svc-ctrl generates it from
                         spec.actual of the provision, the expectation
                         must not fix it)
    @one_of_nameservers  one entry of network.resolv_conf.nameservers
    @absent              the key must not exist (stage files)

  inside a string (value or key)
    @fqdn                the pod FQDN of the document
    @instance            the instance name of the document
                         ("zabbix-psk-@instance", "@instance-token")
    @pod                 one DNS label ("@pod.tick1.scale42.com")
    @any                 any non empty text ("portal.aep.asag.io/@any")

  key
    @anykey              any key; the object must have at least one key that
                         no other rule of the object claims, all of them
                         must satisfy the value rule

Top level keys of the expectation file that start with "@note" are
documentation and are not compared.

Keys of the document that the expectation does not know are listed as notes;
with --strict they are deviations.

Exit status: 0 no deviation, 1 deviations, 2 usage or input error.
"""

import argparse
import json
import re
import sys

SCHEMA = "aep.contract-probe/v1"

HEX12 = re.compile(r"^[0-9a-f]{12}$")
UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
LABEL = r"[a-z0-9]([-a-z0-9]*[a-z0-9])?"
FQDN = re.compile(r"^(?=.{1,253}$)" + LABEL + r"(\." + LABEL + r")+$")
INCLUDES = re.compile(r"^@includes\[(.*)\]$")
SET = re.compile(r"^@set\[(.*)\]$")
MATCH = re.compile(r"^@match\[(.*)\]$", re.S)
TOKEN = re.compile(r"@(fqdn|instance|pod|any)(?![a-z_0-9])")

WHOLE = ("@any", "@list", "@hex12", "@uuid", "@unixtime", "@annotation", "@generation", "@fqdn",
         "@instance", "@one_of_nameservers", "@absent")


class Context:
    """Values of the document that rules refer to, and what rules saw."""

    def __init__(self, doc, live):
        self.live = live
        self.fqdn = None
        self.instance = None
        self.nameservers = None
        self.seen = {"annotation": [], "generation": []}
        if isinstance(doc, dict) and isinstance(doc.get("report"), dict):
            doc = doc["report"]
        if isinstance(doc, dict):
            ident = doc.get("identity")
            if isinstance(ident, dict) and isinstance(ident.get("fqdn"), str):
                self.fqdn = ident["fqdn"]
            if isinstance(ident, dict) and isinstance(ident.get("instance"), str) \
                    and ident["instance"] != "":
                self.instance = ident["instance"]
            net = doc.get("network")
            if isinstance(net, dict):
                rc = net.get("resolv_conf")
                if isinstance(rc, dict) and isinstance(rc.get("nameservers"), list):
                    self.nameservers = rc["nameservers"]


class Report:
    def __init__(self):
        self.deviations = []
        self.notes = []

    def dev(self, path, text):
        self.deviations.append((path, text))

    def note(self, path, text):
        self.notes.append((path, text))


def show(value):
    text = json.dumps(value, ensure_ascii=False, sort_keys=True)
    if len(text) > 120:
        text = text[:117] + "..."
    return text


def is_rule(value):
    return isinstance(value, str) and (
        value in WHOLE or INCLUDES.match(value) is not None
        or SET.match(value) is not None or MATCH.match(value) is not None)


def has_token(text):
    return isinstance(text, str) and TOKEN.search(text) is not None


class Needs(Exception):
    """A rule refers to a field of the report that is missing."""

    def __init__(self, field):
        Exception.__init__(self, field)
        self.field = field

    def text(self):
        return "needs " + self.field + " of the report, which is missing"


def token_regex(text, ctx):
    """Regex for a string with embedded placeholders; Needs if a referenced field is unknown."""
    out = []
    pos = 0
    for m in TOKEN.finditer(text):
        out.append(re.escape(text[pos:m.start()]))
        name = m.group(1)
        if name == "fqdn":
            if ctx.fqdn is None:
                raise Needs("identity.fqdn")
            out.append(re.escape(ctx.fqdn))
        elif name == "instance":
            if ctx.instance is None:
                raise Needs("identity.instance")
            out.append(re.escape(ctx.instance))
        elif name == "pod":
            out.append(LABEL)
        else:
            out.append(r".+")
        pos = m.end()
    out.append(re.escape(text[pos:]))
    return re.compile("^" + "".join(out) + "$", re.S)


def check_rule(rule, actual, path, ctx, rep):
    if rule == "@any":
        return
    if rule == "@list":
        if not isinstance(actual, list):
            rep.dev(path, "expected a list, found " + show(actual))
        return
    m = INCLUDES.match(rule)
    if m:
        wanted = [w.strip() for w in m.group(1).split(",") if w.strip()]
        if not isinstance(actual, list):
            rep.dev(path, "expected a list with " + ", ".join(wanted) + ", found " + show(actual))
            return
        missing = [w for w in wanted if w not in actual]
        if missing:
            rep.dev(path, "list lacks " + ", ".join(missing) + "; found " + show(actual))
        return
    m = SET.match(rule)
    if m:
        wanted = [w.strip() for w in m.group(1).split(",") if w.strip()]
        if not isinstance(actual, list):
            rep.dev(path, "expected a list of " + ", ".join(wanted) + " in any order, found " + show(actual))
            return
        if not all(isinstance(a, str) for a in actual) or sorted(actual) != sorted(wanted):
            rep.dev(path, "expected exactly " + ", ".join(wanted) + " in any order, found " + show(actual))
        return
    m = MATCH.match(rule)
    if m:
        try:
            rx = re.compile(m.group(1), re.S)
        except re.error as ex:
            rep.dev(path, "rule " + show(rule) + " is not a regular expression: " + str(ex))
            return
        if not (isinstance(actual, str) and rx.fullmatch(actual)):
            rep.dev(path, "expected a string matching " + show(m.group(1)) + ", found " + show(actual))
        return
    if rule == "@hex12":
        if not (isinstance(actual, str) and HEX12.match(actual)):
            rep.dev(path, "expected 12 hex digits, found " + show(actual))
        return
    if rule == "@uuid":
        if not (isinstance(actual, str) and UUID.match(actual)):
            rep.dev(path, "expected a UUID, found " + show(actual))
        return
    if rule == "@instance":
        if ctx.instance is None:
            rep.dev(path, "@instance " + Needs("identity.instance").text())
        elif actual != ctx.instance:
            rep.dev(path, "expected the instance name " + show(ctx.instance) + ", found " + show(actual))
        return
    if rule == "@unixtime":
        if isinstance(actual, bool) or not isinstance(actual, int) or actual <= 0:
            rep.dev(path, "expected unix seconds (integer > 0), found " + show(actual))
        return
    if rule == "@annotation":
        if not (isinstance(actual, str) and actual != ""):
            rep.dev(path, "expected the contract hash (non empty string), found " + show(actual))
            return
        ctx.seen["annotation"].append((path, actual))
        return
    if rule == "@generation":
        if isinstance(actual, bool) or not isinstance(actual, int) or actual < 1:
            rep.dev(path, "expected a generation (integer >= 1), found " + show(actual))
            return
        ctx.seen["generation"].append((path, actual))
        return
    if rule == "@fqdn":
        if ctx.fqdn is None:
            rep.dev(path, "@fqdn needs identity.fqdn of the report, which is missing")
        elif actual != ctx.fqdn:
            rep.dev(path, "expected the pod FQDN " + show(ctx.fqdn) + ", found " + show(actual))
        elif not FQDN.match(actual):
            rep.dev(path, "pod FQDN is not a valid FQDN: " + show(actual))
        return
    if rule == "@one_of_nameservers":
        if ctx.nameservers is None:
            rep.dev(path, "@one_of_nameservers needs network.resolv_conf.nameservers, which is missing")
        elif actual not in ctx.nameservers:
            rep.dev(path, "expected one of " + show(ctx.nameservers) + ", found " + show(actual))
        return
    rep.dev(path, "unknown rule " + rule)


def join(path, key):
    return key if path == "" else path + "." + key


def compare_object(expected, actual, path, ctx, rep, strict):
    if not isinstance(actual, dict):
        rep.dev(path or "$", "expected an object, found " + show(actual))
        return
    claimed = set()
    anykey_rule = None
    for key, exp in expected.items():
        if path == "" and key.startswith("@note"):
            continue
        if key == "@anykey":
            anykey_rule = exp
            continue
        if exp == "@absent":
            if key in actual:
                rep.dev(join(path, key), "must be absent, found " + show(actual[key]))
                claimed.add(key)
            continue
        if has_token(key):
            try:
                rx = token_regex(key, ctx)
            except Needs as need:
                rep.dev(join(path, key), "key rule " + need.text())
                continue
            hits = [k for k in actual if rx.match(k)]
            if not hits:
                rep.dev(join(path, key), "no key matches; found keys " + show(sorted(actual)))
                continue
            for k in hits:
                claimed.add(k)
                compare(exp, actual[k], join(path, k), ctx, rep, strict)
            continue
        if key not in actual:
            rep.dev(join(path, key), "missing, expected " + show(exp))
            continue
        claimed.add(key)
        compare(exp, actual[key], join(path, key), ctx, rep, strict)
    rest = [k for k in actual if k not in claimed]
    if anykey_rule is not None:
        if not rest:
            rep.dev(join(path, "@anykey"), "expected at least one key, found none")
        for k in rest:
            compare(anykey_rule, actual[k], join(path, k), ctx, rep, strict)
        return
    for k in rest:
        text = "not in the expectation: " + show(actual[k])
        if strict:
            rep.dev(join(path, k), text)
        else:
            rep.note(join(path, k), text)


def compare(expected, actual, path, ctx, rep, strict):
    if is_rule(expected):
        check_rule(expected, actual, path, ctx, rep)
        return
    if isinstance(expected, str) and has_token(expected):
        try:
            rx = token_regex(expected, ctx)
        except Needs as need:
            rep.dev(path, "rule " + show(expected) + " " + need.text())
            return
        if not (isinstance(actual, str) and rx.match(actual)):
            rep.dev(path, "expected " + show(expected) + ", found " + show(actual))
        return
    if isinstance(expected, dict):
        compare_object(expected, actual, path, ctx, rep, strict)
        return
    if isinstance(expected, list):
        if not isinstance(actual, list):
            rep.dev(path, "expected a list, found " + show(actual))
            return
        if len(expected) != len(actual):
            rep.dev(path, "expected %d entries, found %d: %s" % (len(expected), len(actual), show(actual)))
        for i in range(min(len(expected), len(actual))):
            compare(expected[i], actual[i], "%s[%d]" % (path, i), ctx, rep, strict)
        return
    # literals: bool and number are distinct (true != 1)
    if type(expected) is not type(actual) and not (
            isinstance(expected, (int, float)) and isinstance(actual, (int, float))
            and not isinstance(expected, bool) and not isinstance(actual, bool)):
        rep.dev(path, "expected " + show(expected) + ", found " + show(actual))
        return
    if expected != actual:
        rep.dev(path, "expected " + show(expected) + ", found " + show(actual))


def check_groups(ctx, rep):
    for name in ("annotation", "generation"):
        seen = ctx.seen[name]
        values = sorted({json.dumps(v) for _, v in seen})
        if len(values) > 1:
            rep.dev("@" + name, "fields differ: " + ", ".join("%s=%s" % (p, show(v)) for p, v in seen))
        if name in ctx.live and seen:
            live = ctx.live[name]
            for p, v in seen:
                if str(v) != live:
                    rep.dev(p, "expected the live value %s, found %s" % (live, show(v)))


def strings_of(value, path=""):
    """Every string of a JSON value (keys and values) with its path."""
    if isinstance(value, dict):
        for k, v in value.items():
            yield join(path, k) + " (key)", k
            yield from strings_of(v, join(path, k))
    elif isinstance(value, list):
        for i, v in enumerate(value):
            yield from strings_of(v, "%s[%d]" % (path, i))
    elif isinstance(value, str):
        yield path, value


def secret_values(secrets):
    """Values of a test secrets document {secret: {key: value}}."""
    out = []
    if not isinstance(secrets, dict):
        raise ValueError("secrets document is not an object")
    for name, data in secrets.items():
        if not isinstance(data, dict):
            raise ValueError("secret %s is not an object" % name)
        for key, value in data.items():
            if isinstance(value, str) and value != "":
                out.append(("%s/%s" % (name, key), value))
    return out


def check_secrets(doc, doc_text, secrets, scans, rep):
    """T10: no known secret value in the file and in the scanned logs.

    A value that is an identity (psk-<fqdn>) is public by contract
    (ZABBIX-ONBOARD) and part of the block zabbix_agent; only the keys named
    *.identity are exempt.
    """
    for ref, value in secret_values(secrets):
        if ref.endswith(".identity"):
            continue
        for path, text in strings_of(doc):
            if value in text:
                rep.dev(path, "contains the value of secret " + ref)
        if value in doc_text and not any(value in t for _, t in strings_of(doc)):
            rep.dev("$", "file contains the value of secret " + ref)
        for name, text in scans:
            if value in text:
                rep.dev("scan:" + name, "contains the value of secret " + ref)


def load(path, what):
    try:
        with open(path, encoding="utf-8") as fh:
            text = fh.read()
    except OSError as err:
        raise SystemExit2("%s %s: %s" % (what, path, err.strerror))
    try:
        return json.loads(text), text
    except ValueError as err:
        raise SystemExit2("%s %s is not JSON: %s" % (what, path, err))


class SystemExit2(Exception):
    pass


def run(argv, out=sys.stdout, err=sys.stderr):
    ap = argparse.ArgumentParser(
        prog="probe-compare",
        description="Compare a contract-probe.json with an expectation file (ADL-DEMO-SERVICE section 5).")
    ap.add_argument("actual", help="the produced contract-probe.json")
    ap.add_argument("expected", help="expectation file, e.g. expected/tick1.json")
    ap.add_argument("--live", action="append", default=[], metavar="NAME=VALUE",
                    help="live value for @annotation / @generation (annotation=sha256:..., generation=3)")
    ap.add_argument("--secrets", metavar="FILE",
                    help="JSON {secret: {key: value}}: no value may occur in the file or the scanned files (T10)")
    ap.add_argument("--scan", action="append", default=[], metavar="FILE",
                    help="further file searched for the secret values (log, plan); needs --secrets")
    ap.add_argument("--strict", action="store_true",
                    help="keys of the document that the expectation does not know are deviations")
    ap.add_argument("--quiet", action="store_true", help="only the deviations")
    try:
        args = ap.parse_args(argv)
    except SystemExit as ex:
        return 2 if ex.code not in (0, None) else 0

    live = {}
    for item in args.live:
        name, sep, value = item.partition("=")
        if not sep or name not in ("annotation", "generation") or value == "":
            print("probe-compare: --live expects annotation=<value> or generation=<value>", file=err)
            return 2
        live[name] = value
    if args.scan and not args.secrets:
        print("probe-compare: --scan needs --secrets", file=err)
        return 2

    try:
        actual, actual_text = load(args.actual, "document")
        expected, _ = load(args.expected, "expectation")
        if not isinstance(expected, dict):
            raise SystemExit2("expectation %s is not an object" % args.expected)
        secrets = None
        scans = []
        if args.secrets:
            secrets, _ = load(args.secrets, "secrets")
            try:
                secret_values(secrets)
            except ValueError as ex:
                raise SystemExit2("secrets %s: %s" % (args.secrets, ex))
            for name in args.scan:
                try:
                    with open(name, encoding="utf-8", errors="replace") as fh:
                        scans.append((name, fh.read()))
                except OSError as ex:
                    raise SystemExit2("scan %s: %s" % (name, ex.strerror))
    except SystemExit2 as ex:
        print("probe-compare: %s" % ex, file=err)
        return 2

    rep = Report()
    ctx = Context(actual, live)
    compare(expected, actual, "", ctx, rep, args.strict)
    check_groups(ctx, rep)
    if secrets is not None:
        check_secrets(actual, actual_text, secrets, scans, rep)

    if not args.quiet:
        for path, text in rep.notes:
            print("NOTE  %s: %s" % (path, text), file=out)
    for path, text in rep.deviations:
        print("DIFF  %s: %s" % (path, text), file=out)
    if rep.deviations:
        print("probe-compare: %d deviation(s) between %s and %s"
              % (len(rep.deviations), args.actual, args.expected), file=out)
        return 1
    if not args.quiet:
        print("probe-compare: %s matches %s (%d note(s))"
              % (args.actual, args.expected, len(rep.notes)), file=out)
    return 0


if __name__ == "__main__":
    sys.exit(run(sys.argv[1:]))
