Keep OP_0 OP_IF data envelopes byte for byte

README.md

@@ -104,7 +104,10 @@ The script is split into **regions** the symbolic engine can model, separated
 by **barriers** it cannot: `OP_IF`/`OP_ELSE`/`OP_ENDIF`, `OP_CHECKMULTISIG`,
 `OP_DEPTH`, `OP_CODESEPARATOR`, a `PICK` whose index is computed at runtime,
 and so on. Barriers are never modified and nothing crosses them. Everything
-after a top-level `OP_RETURN` is kept byte for byte.
+after a top-level `OP_RETURN` is kept byte for byte, and so is every
+`OP_0 OP_IF … OP_ENDIF` block without an `OP_ELSE` of its own: it never runs,
+and it is where data envelopes such as inscriptions live, so rewriting its
+contents would change the data without changing what the script does.
 
 Each region goes through these passes, repeated while they keep finding savings:
 

src/analysis.js

@@ -11,7 +11,7 @@
 // very start of a script must stay: there it is the only thing that fails an
 // empty stack.
 
-const { OP, TAIL } = require('./script')
+const { OP, TAIL, DEAD } = require('./script')
 const { Interner, SymState } = require('./symbolic')
 
 function barrierEffect (op, g, ga, frames) {
@@ -58,6 +58,7 @@ function barrierEffect (op, g, ga, frames) {
       return [g, ga]
     default:
       if (c === TAIL) return [0, 0]
+      if (c === DEAD) return [g, ga] // pushes OP_0, pops it, runs nothing else
       return [Math.max(g - 3, 0), ga]
   }
 }

src/optimize.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { OP, TAIL, isPush, pushValue, pushOp, opSize, opsSize, parse, encode, toBuffer } = require('./script')
+const { OP, TAIL, DEAD, isPush, pushValue, pushOp, opSize, opsSize, parse, encode, toBuffer } = require('./script')
 const { equivalent } = require('./symbolic')
 const { analyze, heights } = require('./analysis')
 const { peephole } = require('./peephole')
@@ -77,7 +77,7 @@ function optimizeRegion (ops, g, ga, cache, cfg) {
 function normalizePushes (ops) {
   const rewrites = []
   const out = ops.map((op, index) => {
-    if (op.code === TAIL || !isPush(op)) return op
+    if (op.code === TAIL || op.code === DEAD || !isPush(op)) return op
     const min = pushOp(pushValue(op))
     if (opSize(min) >= opSize(op)) return op
     rewrites.push({ pass: 'push-encoding', index, before: [op], after: [min], saved: opSize(op) - opSize(min) })

src/profile.js

@@ -2,7 +2,7 @@
 
 // Byte profiler: where a script's serialized bytes go.
 
-const { OP, TAIL, isPush, pushValue, opSize, opName, toAsm } = require('./script')
+const { OP, TAIL, DEAD, isPush, pushValue, opSize, opName, toAsm } = require('./script')
 const { decodeNum } = require('./num')
 
 const CATEGORY = {}
@@ -35,6 +35,7 @@ function profile (ops, { ngrams = 10 } = {}) {
     const op = ops[i]
     const size = opSize(op)
     if (op.code === TAIL) { add('data (after OP_RETURN)', size); continue }
+    if (op.code === DEAD) { add('data (OP_0 OP_IF block)', size); continue }
     const name = isPush(op) ? (op.code === OP.OP_0 || op.code >= OP.OP_1NEGATE ? opName(op.code) : 'push') : opName(op.code)
     const e = opcodes[name] || (opcodes[name] = { count: 0, bytes: 0 })
     e.count++
@@ -60,7 +61,7 @@ function profile (ops, { ngrams = 10 } = {}) {
 
   if (!ngrams) return summary()
   const grams = new Map()
-  const tokens = ops.map(o => (o.code === TAIL ? '<tail>' : isPush(o) ? (pushValue(o).length <= 4 ? toAsm([o]) : `<${pushValue(o).length}b>`) : opName(o.code).replace(/^OP_/, '')))
+  const tokens = ops.map(o => (o.code === TAIL ? '<tail>' : o.code === DEAD ? '<dead>' : isPush(o) ? (pushValue(o).length <= 4 ? toAsm([o]) : `<${pushValue(o).length}b>`) : opName(o.code).replace(/^OP_/, '')))
   const sizes = ops.map(opSize)
   for (let n = 2; n <= 4; n++) {
     for (let i = 0; i + n <= ops.length; i++) {

src/script.js

@@ -14,8 +14,12 @@ NAMES[OP.OP_CHECKSEQUENCEVERIFY] = 'OP_CHECKSEQUENCEVERIFY'
 
 // An op is { code, data } for pushes, { code } otherwise. A tail op
 // ({ code: TAIL, raw }) holds bytes we never touch: everything after a
-// top-level OP_RETURN, or an unparseable remainder.
+// top-level OP_RETURN, or an unparseable remainder. A dead op
+// ({ code: DEAD, raw }) is an `OP_0 OP_IF ... OP_ENDIF` block with no OP_ELSE
+// of its own: it never executes and has no stack effect, and it is how data
+// envelopes (inscriptions and similar) are carried, so it is kept verbatim too.
 const TAIL = -1
+const DEAD = -2
 
 function isPush (op) {
   return op.code >= 0 && (op.code <= OP.OP_PUSHDATA4 || op.code === OP.OP_1NEGATE ||
@@ -31,7 +35,7 @@ function pushValue (op) {
 }
 
 function opSize (op) {
-  if (op.code === TAIL) return op.raw.length
+  if (op.code === TAIL || op.code === DEAD) return op.raw.length
   if (op.code === OP.OP_0 || op.code > OP.OP_PUSHDATA4) return 1
   if (op.code < OP.OP_PUSHDATA1) return 1 + op.data.length
   if (op.code === OP.OP_PUSHDATA1) return 2 + op.data.length
@@ -65,7 +69,37 @@ function numOp (n) {
   return pushOp(encodeNum(n))
 }
 
-function parse (buf) {
+// Where the `OP_0 OP_IF` block whose OP_IF sits at `i` ends (the index after
+// its OP_ENDIF), or -1 if it is not a dead block: it has an OP_ELSE of its own
+// (that branch runs), no matching OP_ENDIF, or a truncated push. A block holding
+// OP_VERIF or OP_VERNOTIF is not treated as dead either: in an unexecuted branch
+// they open a conditional after Chronicle but do nothing between Genesis and
+// Chronicle, so where the block ends depends on the era.
+function deadBlockEnd (buf, i) {
+  let depth = 0
+  while (i < buf.length) {
+    const code = buf[i++]
+    if (code > 0 && code <= OP.OP_PUSHDATA4) {
+      let len = code
+      if (code >= OP.OP_PUSHDATA1) {
+        const w = code === OP.OP_PUSHDATA1 ? 1 : code === OP.OP_PUSHDATA2 ? 2 : 4
+        if (i + w > buf.length) return -1
+        len = w === 1 ? buf[i] : w === 2 ? buf.readUInt16LE(i) : buf.readUInt32LE(i)
+        i += w
+      }
+      if (i + len > buf.length) return -1
+      i += len
+      continue
+    }
+    if (code === OP.OP_VERIF || code === OP.OP_VERNOTIF) return -1
+    if (code === OP.OP_IF || code === OP.OP_NOTIF) depth++
+    else if (code === OP.OP_ELSE && depth === 1) return -1
+    else if (code === OP.OP_ENDIF && --depth === 0) return i
+  }
+  return -1
+}
+
+function parse (buf, { deadBlocks = true } = {}) {
   const ops = []
   let i = 0
   let depth = 0
@@ -87,6 +121,16 @@ function parse (buf) {
       i += len
       continue
     }
+    if (deadBlocks && code === OP.OP_IF && start > 0 && buf[start - 1] === OP.OP_0 &&
+        ops.length && ops[ops.length - 1].code === OP.OP_0) {
+      const end = deadBlockEnd(buf, start)
+      if (end > 0) {
+        ops.pop()
+        ops.push({ code: DEAD, raw: Buffer.from(buf.slice(start - 1, end)) })
+        i = end
+        continue
+      }
+    }
     ops.push({ code })
     if (code === OP.OP_IF || code === OP.OP_NOTIF || code === OP.OP_VERIF || code === OP.OP_VERNOTIF) depth++
     else if (code === OP.OP_ENDIF) depth--
@@ -103,7 +147,7 @@ function parse (buf) {
 function encode (ops) {
   const parts = []
   for (const op of ops) {
-    if (op.code === TAIL) { parts.push(op.raw); continue }
+    if (op.code === TAIL || op.code === DEAD) { parts.push(op.raw); continue }
     if (op.code === OP.OP_0 || op.code > OP.OP_PUSHDATA4) { parts.push(Buffer.from([op.code])); continue }
     const len = op.data.length
     let head
@@ -122,6 +166,7 @@ function opName (code) {
 function toAsm (ops, { maxData = 0 } = {}) {
   return ops.map(op => {
     if (op.code === TAIL) return `<tail ${op.raw.length} bytes>`
+    if (op.code === DEAD) return toAsm(parse(op.raw, { deadBlocks: false }), { maxData })
     if (op.code === OP.OP_0) return 'OP_0'
     if (op.code > 0 && op.code <= OP.OP_PUSHDATA4) {
       const hex = op.data.toString('hex')
@@ -145,12 +190,12 @@ function toBuffer (input) {
 
 function sameOp (a, b) {
   if (a.code !== b.code) return false
-  if (a.code === TAIL) return a.raw.equals(b.raw)
+  if (a.code === TAIL || a.code === DEAD) return a.raw.equals(b.raw)
   if (a.data || b.data) return !!a.data && !!b.data && a.data.equals(b.data)
   return true
 }
 
 module.exports = {
-  OP, TAIL, isPush, pushValue, opSize, opsSize, pushOp, pushCost, numOp,
+  OP, TAIL, DEAD, isPush, pushValue, opSize, opsSize, pushOp, pushCost, numOp,
   parse, encode, toAsm, toBuffer, opName, sameOp
 }

test/fuzz.test.js

@@ -9,6 +9,7 @@ const assert = require('node:assert')
 const crypto = require('crypto')
 const { optimize } = require('../src')
 const { OP, numOp, encode, pushOp } = require('../src/script')
+const { encodeNum } = require('../src/num')
 const { differential } = require('../src/verify')
 
 function rng (seed) {
@@ -78,3 +79,69 @@ test('stack-only scripts: heavy fuzz', () => {
     assert.ok(diff.ok, `mismatch for ${buf.toString('hex')} -> ${res.script.toString('hex')}: ${JSON.stringify(diff)}`)
   }
 })
+
+// Every modelled opcode (including signatures, Chronicle opcodes and bit
+// shifts), unusual push encodings, boundary numbers, OP_NOTIF, OP_RETURN and
+// OP_0 OP_IF blocks inside branches, and PICK/ROLL with a computed index. Starting
+// stacks are small numbers, booleans and repeats, so checks pass more often.
+const WIDE = [...STACK, ...ARITH, ...OTHER, 'OP_NUMNOTEQUAL', 'OP_GREATERTHAN', 'OP_LESSTHANOREQUAL',
+  'OP_GREATERTHANOREQUAL', 'OP_SHA1', 'OP_RIPEMD160', 'OP_HASH256', 'OP_OR', 'OP_XOR', 'OP_BIN2NUM',
+  'OP_2DIV', 'OP_SUBSTR', 'OP_LEFT', 'OP_RIGHT', 'OP_CHECKSIG', 'OP_CODESEPARATOR']
+// Size and shift operands stay small so the interpreter does not build huge elements.
+const SIZED = ['OP_NUM2BIN', 'OP_LSHIFT', 'OP_RSHIFT', 'OP_LSHIFTNUM', 'OP_RSHIFTNUM', 'OP_2MUL']
+const ODD = [
+  () => ({ code: OP.OP_PUSHDATA1, data: Buffer.from([7]) }),
+  () => ({ code: 1, data: Buffer.from([3]) }),
+  () => ({ code: 2, data: Buffer.from([2, 0x00]) }),
+  () => ({ code: 1, data: Buffer.from([0x80]) }),
+  () => pushOp(Buffer.from('ffffff7f', 'hex')),
+  () => pushOp(Buffer.from('0000008000', 'hex')),
+  () => pushOp(Buffer.from('ffffffff', 'hex')),
+  () => ({ code: OP.OP_1NEGATE })
+]
+
+function wideScript (r, len) {
+  const ops = []
+  let depth = 0
+  for (let i = 0; i < len; i++) {
+    const k = r() % 100
+    if (k < 42) ops.push({ code: OP[WIDE[r() % WIDE.length]] })
+    else if (k < 46) { ops.push(numOp(r() % 9)); ops.push({ code: OP[SIZED[r() % SIZED.length]] }) } else if (k < 58) { ops.push(numOp(r() % 6)); ops.push({ code: r() % 2 ? OP.OP_PICK : OP.OP_ROLL }) } else if (k < 70) ops.push(numOp((r() % 12) - 3))
+    else if (k < 80) ops.push(ODD[r() % ODD.length]())
+    else if (k < 83) ops.push(pushOp(crypto.randomBytes(r() % 5)))
+    else if (k < 86) ops.push({ code: r() % 2 ? OP.OP_PICK : OP.OP_ROLL })
+    else if (k < 88) ops.push({ code: OP.OP_0 }, { code: OP.OP_IF }, numOp(r() % 4), { code: OP.OP_DUP }, { code: OP.OP_DROP }, { code: OP.OP_ENDIF })
+    else if (k < 93) { ops.push({ code: r() % 2 ? OP.OP_IF : OP.OP_NOTIF }); depth++ } else if (k < 95 && depth) ops.push({ code: OP.OP_ELSE })
+    else if (k < 97 && depth) { ops.push({ code: OP.OP_ENDIF }); depth-- } else if (k < 98 && depth) ops.push({ code: OP.OP_RETURN })
+  }
+  while (depth--) ops.push({ code: OP.OP_ENDIF })
+  return ops
+}
+
+test('wide opcode fuzz with stacks that pass checks', () => {
+  const r = rng(0xB5B)
+  const small = () => {
+    const k = r() % 12
+    if (k < 6) return encodeNum(BigInt(r() % 5))
+    if (k < 8) return Buffer.from([1])
+    if (k < 9) return Buffer.alloc(0)
+    if (k < 10) return Buffer.from([0x80])
+    return crypto.randomBytes(1 + (r() % 4))
+  }
+  const stacks = []
+  for (let i = 0; i < 40; i++) {
+    const st = []
+    const d = r() % 12
+    for (let j = 0; j < d; j++) st.push(j && r() % 4 === 0 ? st[r() % j] : small())
+    stacks.push(st)
+  }
+  let passing = 0
+  for (let i = 0; i < 150; i++) {
+    const buf = encode(wideScript(r, 3 + (r() % 40)))
+    const res = optimize(buf, { differential: false })
+    const diff = differential(buf, res.script, { runs: 20, maxDepth: 10, stacks })
+    assert.ok(diff.ok, `mismatch for ${buf.toString('hex')} -> ${res.script.toString('hex')}: ${JSON.stringify(diff)}`)
+    passing += diff.succeeded
+  }
+  assert.ok(passing > 0)
+})

test/unit.test.js

@@ -79,6 +79,37 @@ test('never touches barriers', () => {
   assert.match(asm(r), /OP_DEPTH OP_2 OP_CHECKMULTISIG OP_CODESEPARATOR/)
 })
 
+test('keeps OP_0 OP_IF data envelopes byte for byte and optimizes around them', () => {
+  const field = 'ab'.repeat(60)
+  const envelope = `OP_0 OP_IF 6f7264 OP_1 746578742f706c61696e OP_0 ${field} OP_0 ${field} OP_1 OP_1 OP_ADD OP_DROP OP_ENDIF`
+  const envelopeHex = toBuffer(envelope).toString('hex')
+  const r = opt('OP_DUP OP_DROP ' + envelope + ' OP_DUP OP_DROP OP_DUP OP_HASH160 ' + '11'.repeat(20) + ' OP_EQUALVERIFY OP_CHECKSIG')
+  assert.ok(r.script.toString('hex').includes(envelopeHex), 'envelope bytes changed')
+  // The leading DUP DROP is what fails an empty stack, so it stays. The block has
+  // no stack effect, so the one after it inherits that guarantee and goes.
+  const [before, after] = asm(r).split(/OP_0 OP_IF.*OP_ENDIF/)
+  assert.strictEqual(before.trim(), 'OP_DUP OP_DROP')
+  assert.ok(!/OP_DROP/.test(after))
+  assert.ok(r.report.verification.symbolic.ok)
+
+  const dead = parse(toBuffer(envelope))
+  assert.strictEqual(dead.length, 1)
+  assert.strictEqual(dead[0].code, -2)
+  assert.strictEqual(encode(dead).toString('hex'), envelopeHex)
+  assert.strictEqual(toAsm(dead), toAsm(parse(toBuffer(envelope), { deadBlocks: false })))
+  assert.ok(profile(envelope).categories.some(c => c.name === 'data (OP_0 OP_IF block)' && c.bytes === envelopeHex.length / 2))
+
+  // An OP_ELSE of its own runs, and a nested one does not count; an unterminated block is not dead.
+  assert.ok(parse(toBuffer('OP_0 OP_IF OP_1 OP_ELSE OP_2 OP_ENDIF')).every(o => o.code >= 0))
+  assert.strictEqual(parse(toBuffer('OP_0 OP_IF OP_1 OP_IF OP_2 OP_ELSE OP_3 OP_ENDIF OP_ENDIF'))[0].code, -2)
+  assert.ok(parse(toBuffer('OP_0 OP_IF OP_1')).every(o => o.code >= 0))
+  // Between Genesis and Chronicle an unexecuted OP_VERIF opens nothing, so this
+  // OP_ELSE belongs to the OP_0 OP_IF and the OP_DROP runs.
+  assert.ok(parse(toBuffer('OP_0 OP_IF OP_VERIF OP_ELSE OP_DROP OP_ENDIF OP_ENDIF')).every(o => o.code >= 0))
+  const verif = 'OP_DUP OP_DROP OP_1 OP_IF OP_0 OP_IF OP_VERIF OP_ELSE OP_DROP OP_ENDIF OP_ENDIF OP_DUP OP_DROP OP_1'
+  assert.match(asm(opt(verif, { chronicle: false })), /OP_ENDIF OP_DUP OP_DROP OP_1$/)
+})
+
 test('equivalence checker', () => {
   assert.ok(equivalent('OP_OVER OP_OVER', 'OP_2DUP'))
   assert.ok(!equivalent('OP_SWAP OP_SUB', 'OP_SUB'))