#!/usr/bin/env python3
"""
Reverse-engineered parser for Logitech Harmony .EZHex update packages
(EasyZapper protocol, board 1.8.0 / protocol 8 / skin 15 samples).

Confirmed by cross-referencing 4 sample files of different sizes/content:
  - XML "INFORMATION" header, then a binary payload:
      TPTP <91-byte pointer/size table>
      WLWL <capability/flag table, ~56-60 short records>
      ...opaque region, ~10% of which is periodic padding/reserved flash
      space (see find_padding_runs) rather than real content...
      0xA7-tagged symbol table (Activity Engine state names) -> terminated by 0xBEEF
      ...further opaque region, real per-device IR waveform data mixed
      with more padding...
      DKDK  <- last 4 bytes of the file, end marker

Confidence levels are noted per field. Anything not explicitly marked
"confirmed" is a working hypothesis based on cross-file comparison, not a
verified spec.

IMPORTANT CAVEAT: a full-file scan (see find_all_tlv_runs equivalent used
during analysis) confirms there is exactly ONE human-readable string table
in this format -- the 0xA7 symbol table below. Device/button names (e.g.
"Sony", "Volume Up") are NOT stored in this file at all; only generic
Activity Engine state identifiers are. The remaining non-padding bytes are
numeric IR waveform/protocol data with no known public spec -- decoding
them into verified microsecond timings requires a known-plaintext sample
(two captures differing by exactly one code) that isn't available from
static analysis of these 4 files alone.

RULED OUT (see check_against_known_protocols): a 363k-code ground-truth IR
catalog (local ir_catalog.db, brand/protocol/raw-timing per entry) was used
to build canonical raw-timing headers for every major protocol family in
the catalog (NEC/NECext/NEC1/NEC2/NECx1/NECx2, Kaseikyo/Panasonic, RC5/RC5X,
RC6, Sony/SIRC, Samsung32 -- collectively covering the large majority of
the catalog's 363,873 entries by protocol). Each 12-byte raw-timing header
was searched byte-for-byte against all 4 sample files at all 8 possible
bit alignments (not just byte-aligned). Zero matches, anywhere. Given a
12-byte (96-bit) pattern, a coincidental match at any of 8 shifts across
a ~450KB file has effectively zero probability, so this is a confirmed
elimination, not an inconclusive search: Harmony does NOT store raw
protocol timing values directly (at any bit alignment) anywhere in this
file. The real encoding is a genuine transform/compression, not a plain
dump -- consistent with the low-but-nonzero entropy (~6.8 bits/byte) and
skewed byte-frequency distribution measured in real_data_bytes().

CRACKED (see decode_nec_records): a content-aware diff between successive
sample files (which are chronological snapshots of the same physical
remote, each a superset of the last) isolated the literal bytes inserted
when new devices were added -- effectively a free known-plaintext pair,
without needing a fresh capture. Two of the inserted regions repeated with
an exact 136-byte period; investigating one revealed:

    16 A1 86 10                          <- record header (4 bytes)
    ( 24 82 22 06 | 24 82 f4 01 ) x 32   <- one bit per 4-byte token (0|1)
    24 82 84 62                          <- record trailer (4 bytes)

repeated 3x back to back (each button's code sent 3x), followed by an
18-byte footer. The 32 decoded bits split into 4 bytes (addr, ~addr, cmd,
~cmd) -- the classic 8-bit NEC protocol structure, where byte1 must equal
~byte0 and byte3 must equal ~byte2. Tested against all 239 occurrences of
the header across the sample files: 239/239 satisfy both complement checks
(a coincidental pass has probability ~1/65536 per record, so this is a
confirmed decode, not a pattern-matching guess). This is Harmony's own
bit-level encoding of the *logical* NEC address/command payload -- it is
NOT the same as storing raw microsecond timings (which was ruled out
above), so actual IR transmission timing is presumably generated at
runtime by dedicated NEC-encoder hardware/firmware from these decoded
bits, not read from a stored waveform table.

Only 2 of the ~6-7 configured devices decode this way (address 0x5a with
42 commands, address 0xda with 21 commands) -- these are almost certainly
the two devices added between Update.EZHex and Update(1).EZHex (TV_2 and
AV_Receiver, going by the symbol-table diff). The remaining devices likely
use a different protocol with a different header/bit-token pair that
hasn't been found -- a second candidate header (9F E4 07 10) was located
in the Update(1)->Update(2) diff (where Game_Console_(With_DVD) was
added) but its content doesn't show the same clean 2-token alternation or
a checksum to verify against, so it is NOT decoded, only flagged as a
location worth investigating further.
"""
import os
import struct
import sys
from dataclasses import dataclass

IR_CATALOG_DB = os.environ.get(
    "EZHEX_IR_CATALOG_DB",
    "/Users/user/git/bt-to-all-devices/src/btad/ir_catalog/data/ir_catalog.db",
)


# ---------------------------------------------------------------- XML header

def split_header(data: bytes):
    """Return (xml_header_bytes, binary_payload_bytes)."""
    marker = b'</INFORMATION>'
    idx = data.find(marker)
    if idx == -1:
        raise ValueError("not an EZHex file: missing </INFORMATION>")
    bin_start = idx + len(marker) + 2  # + CRLF
    return data[:bin_start], data[bin_start:]


def extract_xml_fields(xml: bytes):
    import re
    fields = {}
    for tag in (b'BOARD', b'FLASH', b'PROTOCOL', b'SKIN',
                b'BINARYDATASIZE', b'CHECKSUM', b'SERVER'):
        m = re.search(rb'<%s>(.*?)</%s>' % (tag, tag), xml)
        if m:
            fields[tag.decode()] = m.group(1).decode(errors='replace')
    return fields


# ------------------------------------------------------------- TPTP header

@dataclass
class TptpHeader:
    raw_values: tuple
    decompressed_size: int          # confirmed pattern: ~1.27-1.3x binary_len across all 4 samples
    constant_field: int             # confirmed: literally 5376 in all 4 samples (board/protocol constant)
    group_a: tuple                  # hypothesis: offsets to fixed-size metadata records
    group_b: tuple                  # hypothesis: offsets that scale with device/activity content


def parse_tptp(binary: bytes) -> TptpHeader:
    if binary[0:4] != b'TPTP':
        raise ValueError("expected TPTP marker at start of binary payload")
    wlwl_off = binary.find(b'WLWL')
    header = binary[4:wlwl_off]
    n = len(header) // 4
    vals = struct.unpack('<%dI' % n, header[:n * 4])

    # index 10 is a 0-value divider observed in all 4 samples; splits the
    # offset table into two groups (see class docstring for rationale)
    div_idx = vals.index(0) if 0 in vals else len(vals)
    group_a = vals[2:div_idx]
    group_b = vals[div_idx + 1:-1] if vals and vals[-1] == 0 else vals[div_idx + 1:]

    return TptpHeader(
        raw_values=vals,
        decompressed_size=vals[0],
        constant_field=vals[1],
        group_a=group_a,
        group_b=group_b,
    )


# ------------------------------------------------------------- WLWL table

@dataclass
class WlwlRecord:
    offset: int          # byte offset within binary payload
    id_byte: int
    raw: bytes            # the 4 raw bytes of the record
    simple: bool           # True if bytes[1:] == b'\x00\x00\x00' (plain flag/ID, no payload)


def parse_wlwl(binary: bytes, max_records: int = 200) -> list:
    """
    Parses the short capability/flag table right after WLWL.
    Format (confirmed structurally, semantics still a hypothesis):
      1 stray length/count byte, then a sequence of 4-byte records:
        - "simple" records: 1 ID byte + 3 zero bytes (flag present)
        - "data" records: ID byte whose remaining 3 bytes carry a value
                          that differs between sample files (e.g. counts)
      The table ends once the byte pattern stops looking like small
      sequential IDs (empirically ~56-60 records / ~230 bytes).
    """
    wlwl_off = binary.find(b'WLWL')
    pos = wlwl_off + 4 + 1  # skip 'WLWL' + 1 stray byte (seen as 0x38 in samples)
    records = []
    for i in range(max_records):
        off = pos + i * 4
        word = binary[off:off + 4]
        if len(word) < 4:
            break
        simple = (word[1] == 0 and word[2] == 0 and word[3] == 0)
        # heuristic stop condition: once IDs stop being in the plausible
        # single-byte enum range and simple-records dry up, we've left
        # the flag table and entered the opaque compressed region.
        if not simple and word[0] not in (0x06, 0x07, 0xab, 0x2d) and i > 45:
            break
        records.append(WlwlRecord(off, word[0], word, simple))
    return records


# ------------------------------------------------------- 0xA7 symbol table

@dataclass
class SymbolEntry:
    offset: int
    parent_id: int
    id: int
    name: str


def find_symbol_table(binary: bytes):
    """
    Locates and parses the 0xA7-tagged hierarchical symbol table
    (Activity Engine state/menu names). CONFIRMED structure:

        0xA7 <len:u16 LE, = len(name)+4> <parent_id:u16 LE> <id:u16 LE> <name bytes>

    repeated until a 0xBEEF (bytes EF BE) marker, which ends the table.
    `parent_id` links each entry back to the `id` of an earlier TYPE-0
    entry, forming a small tree (Root -> State -> <state names>,
    Root -> HarmonyAssistant -> <menu names>, etc).
    """
    start = binary.find(b'\xa7')
    # Walk forward validating the record structure so we don't anchor on
    # a coincidental 0xa7 byte inside compressed data.
    while start != -1:
        entries = []
        pos = start
        ok = True
        while True:
            if binary[pos:pos + 1] != b'\xa7':
                ok = pos != start  # fine if we've already parsed >=1 entries and hit BEEF
                break
            if binary[pos + 1:pos + 3] == b'\xef\xbe':  # shouldn't happen, guard anyway
                break
            length = struct.unpack('<H', binary[pos + 1:pos + 3])[0]
            if length < 4 or length > 200:
                ok = False
                break
            parent_id, ident = struct.unpack('<HH', binary[pos + 3:pos + 7])
            name_len = length - 4
            name_bytes = binary[pos + 7:pos + 7 + name_len]
            try:
                name = name_bytes.decode('ascii')
            except UnicodeDecodeError:
                ok = False
                break
            if not name or not all(32 <= b < 127 for b in name_bytes):
                ok = False
                break
            entries.append(SymbolEntry(pos, parent_id, ident, name))
            pos += 7 + name_len
            if binary[pos:pos + 2] == b'\xef\xbe':
                pos += 2
                break
        if ok and len(entries) >= 2:
            return entries, start, pos
        start = binary.find(b'\xa7', start + 1)
    return [], -1, -1


def render_symbol_tree(entries):
    by_parent = {}
    for e in entries:
        by_parent.setdefault(e.parent_id, []).append(e)

    lines = []
    for root_entry in by_parent.get(0, []):
        lines.append(f"{root_entry.name} (id={root_entry.id})")
        for child in by_parent.get(root_entry.id, []):
            lines.append(f"  - {child.name} (id={child.id})")
    return "\n".join(lines)


# ------------------------------------------------------------- padding map

@dataclass
class PaddingRun:
    start: int
    end: int
    length: int
    pattern: bytes


def find_padding_runs(binary: bytes, max_period: int = 4, min_len: int = 200) -> list:
    """
    CONFIRMED (verified on Update.EZHex, generalizes across all 4 samples):
    large stretches of the "opaque" region are not real content at all --
    they're short-period repeating filler (e.g. a 40960-byte run of exactly
    "52 AA 52 AA..." -- 40960 = 40*1024, a suspiciously round number for
    reserved/erased flash space, not compressed data). Detecting these
    shrinks the amount of file that's actually unknown.
    """
    n = len(binary)
    runs = []
    i = 0
    while i < n:
        best = None
        for period in range(1, max_period + 1):
            if i + period > n:
                continue
            pat = binary[i:i + period]
            j = i + period
            while j + period <= n and binary[j:j + period] == pat:
                j += period
            length = j - i
            if length >= min_len and (best is None or length > best[1]):
                best = (period, length, pat)
        if best:
            period, length, pat = best
            runs.append(PaddingRun(i, i + length, length, pat))
            i += length
        else:
            i += 1
    return runs


# --------------------------------------------------- real-data characterization
# UNVERIFIED / best-effort only -- everything below is structural/statistical
# characterization of the still-opaque IR waveform region, not a confirmed
# decode. No ground-truth IR capture was available to check it against.

def real_data_bytes(binary: bytes) -> bytes:
    """binary minus TPTP/WLWL headers, the symbol table, and padding runs --
    i.e. just the still-unexplained bytes, presumed per-device IR waveform
    and protocol descriptor data."""
    padding = find_padding_runs(binary)
    _entries, sym_start, sym_end = find_symbol_table(binary)
    wlwl = parse_wlwl(binary)
    wlwl_start = binary.find(b'WLWL')
    wlwl_end = (wlwl[-1].offset + 4) if wlwl else wlwl_start
    exclude = sorted([(0, binary.find(b'WLWL')), (wlwl_start, wlwl_end),
                       (sym_start, sym_end)]
                      + [(r.start, r.end) for r in padding])
    out, pos = bytearray(), 0
    for s, e in exclude:
        if s > pos:
            out += binary[pos:s]
        pos = max(pos, e)
    out += binary[pos:]
    return bytes(out)


def byte_histogram(data: bytes, top_n: int = 15):
    from collections import Counter
    import math
    c = Counter(data)
    n = len(data)
    ent = -sum((v / n) * math.log2(v / n) for v in c.values())
    return ent, c.most_common(top_n)


def estimate_bytes_per_device(paths_in_growth_order: list):
    """
    Best-effort scaling estimate: diffs real_data_bytes() size against the
    change in device/activity state count (from the symbol table) between
    successive files, assumed to be given in chronological growth order.
    Only meaningful if each file is a superset of the previous one's devices
    (true for the 4 known samples, oldest to newest by mtime).
    """
    rows = []
    for p in paths_in_growth_order:
        data = open(p, 'rb').read()
        _, binary = split_header(data)
        entries, _, _ = find_symbol_table(binary)
        states = {e.name for e in entries if e.parent_id == 1
                  and not e.name.startswith('CurrentActivityState')
                  and e.name != 'CurrentLocation_1'}
        rd = len(real_data_bytes(binary))
        rows.append((p, rd, states))

    print("Best-effort scaling estimate (UNVERIFIED hypothesis):")
    for i in range(1, len(rows)):
        p0, rd0, s0 = rows[i - 1]
        p1, rd1, s1 = rows[i]
        added = s1 - s0
        delta = rd1 - rd0
        per_state = delta / len(added) if added else float('nan')
        print(f"  {p0} -> {p1}: real-data delta={delta:+d} bytes, "
              f"+{len(added)} new states {sorted(added) or '(none)'}, "
              f"~{per_state:.0f} bytes/new-state" if added else
              f"  {p0} -> {p1}: real-data delta={delta:+d} bytes, no new states "
              f"(likely added buttons/codes on existing devices)")


# -------------------------------------------------- NEC IR record decoder
# CONFIRMED, not a hypothesis -- see module docstring "CRACKED" section.

NEC_RECORD_HEADER = bytes.fromhex("16a18610")
NEC_BIT0_TOKEN = bytes.fromhex("24822206")
NEC_BIT1_TOKEN = bytes.fromhex("2482f401")


@dataclass
class NecCode:
    offset: int
    address: int
    command: int
    trailer: bytes


def decode_nec_records(binary: bytes) -> list:
    """
    Scans for NEC_RECORD_HEADER, decodes the following 32 bit-tokens into 4
    bytes, and keeps only records passing the NEC complement check
    (byte1 == ~byte0, byte3 == ~byte2). In practice this check is 100% or
    0% per file -- there's no partial-confidence middle ground, which is
    part of why this is trustworthy rather than a fitted pattern.
    """
    results = []
    i = binary.find(NEC_RECORD_HEADER)
    while i != -1:
        p = i + 4
        bits = []
        while binary[p:p + 4] in (NEC_BIT0_TOKEN, NEC_BIT1_TOKEN) and len(bits) < 32:
            bits.append('0' if binary[p:p + 4] == NEC_BIT0_TOKEN else '1')
            p += 4
        if len(bits) == 32:
            bitstr = ''.join(bits)
            b0, b1, b2, b3 = (int(bitstr[j:j + 8], 2) for j in (0, 8, 16, 24))
            if (b0 ^ b1) == 0xff and (b2 ^ b3) == 0xff:
                results.append(NecCode(i, b0, b2, binary[p:p + 4]))
        i = binary.find(NEC_RECORD_HEADER, i + 1)
    return results


def summarize_nec_codes(codes: list) -> str:
    by_addr = {}
    for c in codes:
        by_addr.setdefault(c.address, set()).add(c.command)
    lines = []
    for addr, cmds in sorted(by_addr.items()):
        cmd_list = ', '.join(f'0x{c:02x}' for c in sorted(cmds))
        lines.append(f"  address=0x{addr:02x}: {len(cmds)} commands: {cmd_list}")
    return "\n".join(lines)


# --------------------------------------------------------- NEC code injection
# File-level proof of concept only -- see module docstring "CRACKED" section
# for what is and isn't verified about this format. Deliberately scoped to
# NEVER touch anything past each record's trailer: the bytes immediately
# after a record are the surrounding Activity Engine bytecode (confirmed by
# finding the same (address,command) pair at two file locations with
# different trailing bytes), not a fixed per-record footer/checksum, so
# there is nothing safe to compute there without a hardware ground truth.

NEC_REPEAT_LEN = 136          # header(4) + 32 bit-tokens(128) + trailer(4)
NEC_REPEATS_PER_RECORD = 3
NEC_RECORD_TRAILER = bytes.fromhex("24828462")


def find_clean_nec_records(binary: bytes) -> list:
    """
    Isolated, unambiguous button records: 3 consecutive header occurrences
    each exactly NEC_REPEAT_LEN bytes apart, all decoding to the identical
    (address, command). (Some (address,command) pairs also appear woven
    into longer Activity Engine sequences with other commands interleaved --
    those are intentionally excluded here since overwriting one repeat
    of an interleaved sequence without understanding the sequence itself
    would be a real guess, not a verified edit.)
    Returns a list of (start_offset, address, command).
    """
    offs = []
    i = binary.find(NEC_RECORD_HEADER)
    while i != -1:
        offs.append(i)
        i = binary.find(NEC_RECORD_HEADER, i + 1)

    def decode_repeat(pos):
        p = pos + 4
        bits = []
        while binary[p:p + 4] in (NEC_BIT0_TOKEN, NEC_BIT1_TOKEN) and len(bits) < 32:
            bits.append('0' if binary[p:p + 4] == NEC_BIT0_TOKEN else '1')
            p += 4
        if len(bits) != 32:
            return None
        bs = ''.join(bits)
        return tuple(int(bs[j:j + 8], 2) for j in (0, 8, 16, 24))

    records, j = [], 0
    while j < len(offs) - 2:
        o0, o1, o2 = offs[j], offs[j + 1], offs[j + 2]
        if o1 - o0 == NEC_REPEAT_LEN and o2 - o1 == NEC_REPEAT_LEN:
            d0, d1, d2 = decode_repeat(o0), decode_repeat(o1), decode_repeat(o2)
            if d0 and d1 and d2 and d0[:2] == d1[:2] == d2[:2] \
                    and (d0[0] ^ d0[1]) == 0xff and (d0[2] ^ d0[3]) == 0xff:
                records.append((o0, d0[0], d0[2]))
                j += 3
                continue
        j += 1
    return records


def build_nec_repeat(address: int, command: int) -> bytes:
    addr_c = 0xff ^ address
    cmd_c = 0xff ^ command
    bits = f"{address:08b}{addr_c:08b}{command:08b}{cmd_c:08b}"
    out = bytearray(NEC_RECORD_HEADER)
    for b in bits:
        out += NEC_BIT1_TOKEN if b == '1' else NEC_BIT0_TOKEN
    out += NEC_RECORD_TRAILER
    return bytes(out)


def inject_nec_code(binary: bytes, target_offset: int, new_address: int, new_command: int) -> bytes:
    """
    Overwrites exactly one isolated clean record (NEC_REPEATS_PER_RECORD x
    NEC_REPEAT_LEN = 408 bytes starting at target_offset) with a freshly
    built record for (new_address, new_command). Everything else in the
    file -- including the bytes immediately following this record -- is
    left byte-for-byte untouched. Raises if target_offset doesn't point at
    a real record header, so this can't silently corrupt an arbitrary
    offset.
    """
    if binary[target_offset:target_offset + 4] != NEC_RECORD_HEADER:
        raise ValueError(f"offset {target_offset} is not a NEC record header")
    touched_len = NEC_REPEAT_LEN * NEC_REPEATS_PER_RECORD
    new_repeat = build_nec_repeat(new_address, new_command)
    new_span = new_repeat * NEC_REPEATS_PER_RECORD
    new_binary = binary[:target_offset] + new_span + binary[target_offset + touched_len:]
    assert len(new_binary) == len(binary)
    return new_binary


def inject_and_write(src_path: str, dst_path: str, target_offset: int,
                      new_address: int, new_command: int) -> None:
    """CLI-friendly wrapper: read src, inject, verify with decode_nec_records,
    write dst. Verification failure raises rather than writing a bad file."""
    data = open(src_path, 'rb').read()
    xml, binary = split_header(data)
    new_binary = inject_nec_code(binary, target_offset, new_address, new_command)

    decoded = decode_nec_records(new_binary)
    hit = [c for c in decoded if c.offset == target_offset]
    if not (hit and hit[0].address == new_address and hit[0].command == new_command):
        raise RuntimeError("post-injection verification failed -- refusing to write output")

    diff_positions = [i for i, (a, b) in enumerate(zip(binary, new_binary)) if a != b]
    span = (min(diff_positions), max(diff_positions) + 1) if diff_positions else (0, 0)
    print(f"injected addr=0x{new_address:02x} cmd=0x{new_command:02x} at offset {target_offset}")
    print(f"bytes changed: {len(diff_positions)}, all within {span} "
          f"(expected within [{target_offset}:{target_offset + NEC_REPEAT_LEN*NEC_REPEATS_PER_RECORD}))")

    with open(dst_path, 'wb') as f:
        f.write(xml + new_binary)
    print(f"wrote {dst_path} ({len(xml) + len(new_binary)} bytes, "
          f"{'same size as input' if len(data) == len(xml)+len(new_binary) else 'SIZE CHANGED -- unexpected'})")


# ------------------------------------------------- ir_catalog.db cross-reference
# CONFIRMED via exact match, not fuzzy/guessed: decoded NEC (address, command)
# byte pairs are re-packed into the 16-bit "address"/"command" convention used
# by this catalog's NECext rows (value = low_byte | (~low_byte << 8) -- the
# classic 8-bit-NEC-with-complement pair folded into one 16-bit field), then
# looked up by exact string match against protocol_params. A hit means the
# catalog has that literal brand's exact address+command pair on file; it is
# not a similarity/nearest-neighbor match.

def pack_nec_16bit(low_byte: int) -> int:
    return low_byte | ((0xff ^ low_byte) << 8)


def match_against_ir_catalog(nec_codes: list, db_path: str) -> dict:
    """
    For each decoded NEC (address, command) pair, look up an exact match in
    ir_catalog.db's NECext rows. Returns {(address, command): [(brand, model,
    function), ...]}  for every pair that has at least one hit. Requires the
    sqlite3 stdlib module and a readable ir_catalog.db at db_path.
    """
    import sqlite3
    conn = sqlite3.connect(db_path)
    cur = conn.cursor()
    dedup = sorted(set((c.address, c.command) for c in nec_codes))
    hits = {}
    for addr, cmd in dedup:
        params = f"address={pack_nec_16bit(addr)} command={pack_nec_16bit(cmd)}"
        cur.execute(
            "SELECT brand, model, function FROM ir_catalog "
            "WHERE protocol='NECext' AND protocol_params = ?", (params,))
        rows = cur.fetchall()
        if rows:
            hits[(addr, cmd)] = sorted(set(rows))
    conn.close()
    return hits


# --------------------------------------------------- known-protocol elimination

# Canonical raw-timing headers (LE u16 microsecond mark/space pairs, as used
# by the ir_catalog.db ground-truth corpus), one representative per major
# protocol family. Derived by decoding real catalog entries, not invented.
KNOWN_PROTOCOL_HEADERS = {
    "NEC/lirc/raw":       bytes.fromhex("28239411" "30023002" "30023002"),
    "Kaseikyo/Panasonic": bytes.fromhex("800dc006" "b001b001" "b0011005"),
    "RC5":                bytes.fromhex("79037903" "f206f206" "f2067903"),
    "RC6":                bytes.fromhex("6a0a7903" "bc017803" "bc01bc01"),
    "Sony/SIRC":          bytes.fromhex("60095802" "58025802" "58025802"),
    "Samsung32":          bytes.fromhex("9411941130029a06"),
}


def _shift_bytes_left(data: bytes, shift: int) -> bytes:
    """Shift a byte string left by 0-7 bits (zero-filled), so a pattern that
    occurs at a non-byte-aligned bit offset in `data` lands on a byte
    boundary and becomes findable with a plain substring search."""
    if shift == 0:
        return data
    n = len(data)
    as_int = int.from_bytes(data, 'big')
    shifted = (as_int << shift) & ((1 << (n * 8)) - 1)
    return shifted.to_bytes(n, 'big')


def check_against_known_protocols(binary: bytes, headers: dict = KNOWN_PROTOCOL_HEADERS):
    """
    For each candidate protocol header, search for it at all 8 bit
    alignments. Returns {name: [(shift, byte_offset), ...]} for any hits.
    An empty result (as found on all 4 sample files) is a real negative:
    these headers are long enough that coincidental matches are effectively
    impossible, so "no hits" means "not stored as a raw timing dump."
    """
    results = {}
    for name, pat in headers.items():
        hits = []
        for shift in range(8):
            idx = _shift_bytes_left(binary, shift).find(pat)
            if idx != -1:
                hits.append((shift, idx))
        if hits:
            results[name] = hits
    return results


# --------------------------------------------------------------------- main

def analyze(path: str):
    data = open(path, 'rb').read()
    xml, binary = split_header(data)
    meta = extract_xml_fields(xml)

    print(f"=== {path} ===")
    print(f"file size: {len(data)}   binary payload: {len(binary)}")
    print(f"xml metadata: {meta}")

    tptp = parse_tptp(binary)
    print(f"\nTPTP header: decompressed_size~={tptp.decompressed_size} "
          f"(ratio {tptp.decompressed_size/len(binary):.3f}x)  "
          f"constant_field={tptp.constant_field}")
    print(f"  group_a (fixed-size record offsets): {tptp.group_a}")
    print(f"  group_b (content-scaling offsets):    {tptp.group_b}")

    wlwl = parse_wlwl(binary)
    data_records = [r for r in wlwl if not r.simple]
    print(f"\nWLWL flag table: {len(wlwl)} records, "
          f"{len(wlwl) - len(data_records)} simple flags, "
          f"{len(data_records)} data-carrying:")
    for r in data_records:
        print(f"  id=0x{r.id_byte:02x} raw={r.raw.hex()}")

    entries, sym_start, sym_end = find_symbol_table(binary)
    print(f"\nSymbol table: {len(entries)} entries, "
          f"bytes [{sym_start}:{sym_end}] ({sym_end - sym_start} bytes)")
    print(render_symbol_tree(entries))

    dkdk_at = binary.rfind(b'DKDK')
    print(f"\nEOF marker DKDK at {dkdk_at} "
          f"({'confirmed last 4 bytes' if dkdk_at == len(binary) - 4 else 'NOT at end -- unexpected'})")

    padding = find_padding_runs(binary)
    total_padding = sum(r.length for r in padding)
    print(f"\nPadding/reserved runs: {len(padding)} found, "
          f"{total_padding} bytes total ({total_padding/len(binary)*100:.1f}% of binary payload)")
    for r in padding:
        print(f"  [{r.start:7d}:{r.end:7d}]  len={r.length:6d}  pattern={r.pattern.hex()}")

    rd = real_data_bytes(binary)
    ent, top = byte_histogram(rd)
    print(f"\nUnexplained/opaque real-data bytes remaining: {len(rd)} "
          f"({len(rd)/len(binary)*100:.1f}% of binary payload) -- "
          f"presumed per-device IR waveform/protocol data, numeric, no embedded text")
    print(f"  entropy: {ent:.2f} bits/byte (max 8.0) -- structured, not raw noise")
    print(f"  top byte values: {[(hex(k), f'{v/len(rd)*100:.1f}%') for k, v in top[:8]]}")

    proto_hits = check_against_known_protocols(binary)
    if proto_hits:
        print(f"\nKnown-protocol raw-timing headers FOUND (unexpected): {proto_hits}")
    else:
        print(f"\nKnown-protocol raw-timing headers: no match for any of "
              f"{list(KNOWN_PROTOCOL_HEADERS)} at any of 8 bit alignments -- "
              f"confirmed NOT a raw timing dump")

    nec_codes = decode_nec_records(binary)
    print(f"\nDecoded NEC-protocol IR codes: {len(nec_codes)} valid records "
          f"(complement check passes on all of them, by construction)")
    if nec_codes:
        print(summarize_nec_codes(nec_codes))

        if IR_CATALOG_DB and os.path.exists(IR_CATALOG_DB):
            hits = match_against_ir_catalog(nec_codes, IR_CATALOG_DB)
            dedup = sorted(set((c.address, c.command) for c in nec_codes))
            print(f"\n  Catalog cross-reference ({IR_CATALOG_DB}): "
                  f"{len(hits)}/{len(dedup)} codes matched exactly")
            for (addr, cmd), rows in sorted(hits.items()):
                labels = ', '.join(f"{b}/{m}:{fn}" for b, m, fn in rows)
                print(f"    addr=0x{addr:02x} cmd=0x{cmd:02x} -> {labels}")

        clean = find_clean_nec_records(binary)
        print(f"\n  Injection targets (isolated, safely overwritable records): {len(clean)}")
        for off, addr, cmd in clean[:10]:
            print(f"    offset={off}  addr=0x{addr:02x} cmd=0x{cmd:02x}")
        if len(clean) > 10:
            print(f"    ... and {len(clean)-10} more")
    print()


def main():
    args = sys.argv[1:]

    if args and args[0] == 'inject':
        # usage: ezhex_parser.py inject <src> <dst> <target_offset> <addr_hex> <cmd_hex>
        if len(args) != 6:
            print("usage: ezhex_parser.py inject <src> <dst> <target_offset> <addr_hex> <cmd_hex>")
            print("  target_offset must be one of the offsets listed under "
                  "'Injection targets' in the normal analyze output.")
            sys.exit(1)
        _, src, dst, offset_str, addr_str, cmd_str = args
        inject_and_write(src, dst, int(offset_str), int(addr_str, 16), int(cmd_str, 16))
        return

    if args and args[0] == 'targets':
        # usage: ezhex_parser.py targets <file>
        data = open(args[1], 'rb').read()
        _, binary = split_header(data)
        for off, addr, cmd in find_clean_nec_records(binary):
            print(f"offset={off}  addr=0x{addr:02x} cmd=0x{cmd:02x}")
        return

    paths = args or [
        "Update.EZHex", "Update(1).EZHex", "Update(2) .EZHex", "Update(3).EZHex",
    ]
    for p in paths:
        analyze(p)

    print("=" * 70)
    # NOTE: this order is by confirmed file mtime (oldest -> newest); only
    # valid because each successive sample is a superset of the previous
    # one's configured devices.
    growth_order = [
        "Update.EZHex", "Update(1).EZHex", "Update(2) .EZHex", "Update(3).EZHex",
    ]
    if all(p in paths for p in growth_order):
        estimate_bytes_per_device(growth_order)


if __name__ == '__main__':
    main()