Check --no-chronicle results in both eras; write ASM only when it reads back exactly

README.md

@@ -58,7 +58,7 @@ cd scriptmin && npm install && npm test
 ```bash
 scriptmin script.hex                     # optimize, print report and hex
 scriptmin -o min.hex script.hex          # write the result
-scriptmin --asm -o min.asm script.asm    # ASM in, ASM out
+scriptmin --asm -o min.asm script.asm    # ASM in, ASM out (refused if ASM cannot hold it exactly)
 scriptmin --profile script.hex           # where the bytes go
 scriptmin --explain script.hex           # every rewrite, largest first
 scriptmin --json script.hex              # machine-readable report
@@ -159,14 +159,21 @@ The barrier sequences of the two scripts must be identical. If the proof fails,
 `@smartledger/bsv` interpreter with random starting stacks, plus any you supply
 with `--stacks`. Success, final stack and final alt stack must match. Random
 stacks rarely get past a script's first real check, so for large verifiers pass
-realistic unlocking stacks with `--stacks`.
+realistic unlocking stacks with `--stacks`. With `--no-chronicle` the tests run
+under the rules before Chronicle as well as after it.
 
 ### Assumptions and caveats
 
 - **Target rules are current BSV consensus (post-Genesis, post-Chronicle).**
   Chronicle opcodes (`OP_SUBSTR`, `OP_LEFT`, `OP_RIGHT`, `OP_LSHIFTNUM`,
   `OP_RSHIFTNUM`) are modelled with their Chronicle stack effects. Use
   `--no-chronicle` to treat them as barriers.
+- **ASM cannot hold every script.** It does not record how data was pushed, so
+  `--asm -o` refuses a result with a non-minimal push (optimized code is
+  minimal, but `OP_0 OP_IF` blocks and data after `OP_RETURN` are kept
+  verbatim), a truncated push, an unnamed opcode, or only data pushes, rather
+  than write a different script.
+  Hex and `--binary` output are always exact.
 - **Signatures commit to the script.** `OP_CHECKSIG` signs the script code, so
   signatures and OP_PUSH_TX preimages must be created against the optimized
   script. A covenant that embeds its own script hash, length or bytes must be
@@ -307,6 +314,7 @@ report.verification     // { symbolic, differential }
 profile(script)                    // categories, opcodes, PICK/ROLL depths, costly patterns
 proveEquivalent(a, b)              // { ok, regions, barriers } or { ok: false, reason }
 differential(a, b, { runs: 500 })  // { ok, runs, succeeded } or a counterexample
+exactAsm(script)                   // ASM that reads back byte for byte, or throws
 ```
 
 ## Benchmarks

bin/scriptmin.js

@@ -6,8 +6,7 @@ const fs = require('fs')
 // Output piped into something that stops reading (| head) is not an error.
 process.stdout.on('error', (e) => { if (e.code === 'EPIPE') process.exit(0); throw e })
 const path = require('path')
-const { optimize, profile, Cache, toAsm, toBuffer } = require('../src')
-const { parse } = require('../src/script')
+const { optimize, profile, Cache, toAsm, exactAsm, toBuffer } = require('../src')
 const { StackTable } = require('../src/superopt')
 const { EFFORT } = require('../src/optimize')
 
@@ -84,6 +83,15 @@ function die (msg) {
   process.exit(2)
 }
 
+// Written ASM must read back as the same script: refuse rather than write a different one.
+function asmOut (script) {
+  try {
+    return exactAsm(script) + '\n'
+  } catch (e) {
+    die(e.message)
+  }
+}
+
 const fmt = n => n.toLocaleString('en-US')
 const clip = (text, ops, width = 150) => (text.length <= width ? text : `${text.slice(0, width)}… (${fmt(ops)} ops)`)
 const pad = (s, n) => String(s).padEnd(n)
@@ -173,7 +181,7 @@ function compileMain (argv) {
   } catch (e) {
     die(e.message)
   }
-  if (a.out) fs.writeFileSync(a.out, a.asm ? toAsm(parse(res.script)) + '\n' : res.script.toString('hex') + '\n')
+  if (a.out) fs.writeFileSync(a.out, a.asm ? asmOut(res.script) : res.script.toString('hex') + '\n')
   const r = res.report
   if (a.json) {
     process.stdout.write(JSON.stringify({ script: res.script.toString('hex'), gates: r.gates, maxBits: r.maxBits, bytes: r.optimized, referenceBytes: r.reference, checked: r.checked, ms: r.ms }, null, 2) + '\n')
@@ -275,7 +283,7 @@ function main () {
   if (a.db && cache) fs.writeFileSync(a.db, JSON.stringify(cache.toJSON()))
 
   if (a.out) {
-    const data = a.binary ? res.script : a.asm ? toAsm(parse(res.script)) + '\n' : res.script.toString('hex') + '\n'
+    const data = a.binary ? res.script : a.asm ? asmOut(res.script) : res.script.toString('hex') + '\n'
     fs.mkdirSync(path.dirname(path.resolve(a.out)), { recursive: true })
     fs.writeFileSync(a.out, data)
   }

src/index.js

@@ -24,5 +24,6 @@ module.exports = {
   parse: script.parse,
   encode: script.encode,
   toAsm: script.toAsm,
+  exactAsm: script.exactAsm,
   toBuffer: script.toBuffer
 }

src/optimize.js

@@ -7,7 +7,7 @@ const { peephole } = require('./peephole')
 const { windowsRegion } = require('./windows')
 const { rescheduleFragment } = require('./schedule')
 const { Cache, StackTable } = require('./superopt')
-const { proveEquivalent, differential } = require('./verify')
+const { proveEquivalent, differential, eraFlags } = require('./verify')
 const { profile } = require('./profile')
 
 const EFFORT = {
@@ -173,7 +173,7 @@ function optimize (input, options = {}) {
     }
   }
   if (cfg.differential !== 0 && cfg.differential !== false) {
-    const diff = differential(buf, script, { runs: typeof cfg.differential === 'number' ? cfg.differential : 100, stacks: cfg.stacks || [] })
+    const diff = differential(buf, script, { runs: typeof cfg.differential === 'number' ? cfg.differential : 100, stacks: cfg.stacks || [], flags: eraFlags(cfg) })
     verification.differential = diff
     if (!diff.ok) {
       const err = new Error('internal error: optimized script disagrees with the original on a test input')

src/script.js

@@ -163,15 +163,23 @@ function opName (code) {
   return NAMES[code] || ('OP_UNKNOWN' + code)
 }
 
-function toAsm (ops, { maxData = 0 } = {}) {
+// ASM for reading. With `bare`, it is the ASM @smartledger/bsv reads: data as
+// bare hex whatever its push opcode, and the bytes after a top-level OP_RETURN
+// as ASM when they parse. See exactAsm for writing a script out.
+function toAsm (ops, { maxData = 0, bare = false } = {}) {
   return ops.map(op => {
-    if (op.code === TAIL) return `<tail ${op.raw.length} bytes>`
-    if (op.code === DEAD) return toAsm(parse(op.raw, { deadBlocks: false }), { maxData })
+    if (op.code === TAIL) {
+      if (bare) {
+        try { return bsv.Script.fromBuffer(op.raw).toASM() } catch (e) {}
+      }
+      return `<tail ${op.raw.length} bytes>`
+    }
+    if (op.code === DEAD) return toAsm(parse(op.raw, { deadBlocks: false }), { maxData, bare })
     if (op.code === OP.OP_0) return 'OP_0'
     if (op.code > 0 && op.code <= OP.OP_PUSHDATA4) {
       const hex = op.data.toString('hex')
       const shown = maxData && hex.length > maxData * 2 ? hex.slice(0, maxData * 2) + '…' : hex
-      return op.code >= OP.OP_PUSHDATA1 ? `${opName(op.code)}:${shown}` : shown
+      return op.code >= OP.OP_PUSHDATA1 && !bare ? `${opName(op.code)}:${shown}` : shown
     }
     return opName(op.code)
   }).join(' ')
@@ -188,6 +196,49 @@ function toBuffer (input) {
   return bsv.Script.fromASM(text.replace(/\s+/g, ' ')).toBuffer()
 }
 
+// Whether a push anywhere in buf, including after OP_RETURN, runs past its end.
+function truncated (buf) {
+  let i = 0
+  while (i < buf.length) {
+    const code = buf[i++]
+    if (code === 0 || code > OP.OP_PUSHDATA4) continue
+    let len = code
+    if (code >= OP.OP_PUSHDATA1) {
+      const w = code === OP.OP_PUSHDATA1 ? 1 : code === OP.OP_PUSHDATA2 ? 2 : 4
+      if (i + w > buf.length) return true
+      len = w === 1 ? buf[i] : w === 2 ? buf.readUInt16LE(i) : buf.readUInt32LE(i)
+      i += w
+    }
+    if (i + len > buf.length) return true
+    i += len
+  }
+  return false
+}
+
+// ASM that reads back as exactly `buf`, or an error. ASM cannot say how data
+// was pushed (OP_PUSHDATA1 with a short payload reads back as a direct push),
+// a truncated push, or an opcode with no name; and text of bare hex alone reads
+// as hex. Scripts like those have to be written as hex.
+function exactAsm (buf) {
+  const asm = toAsm(parse(buf), { bare: true })
+  let back = null
+  try { back = toBuffer(asm) } catch (e) {}
+  if (!back || !back.equals(buf)) {
+    const ops = parse(buf, { deadBlocks: false })
+    const why = truncated(buf)
+      ? 'it contains a truncated push'
+      : ops.some(o => isPush(o) && o.code !== TAIL && opSize(pushOp(pushValue(o))) !== opSize(o))
+        ? 'a push is not minimally encoded, and ASM cannot say how data was pushed'
+        : /OP_UNKNOWN|\bOP_INVALIDOPCODE\b/.test(asm) || ops.some(o => o.code >= 0 && !isPush(o) && !NAMES[o.code])
+          ? 'it uses an opcode that has no name in ASM'
+          : !/\bOP_/.test(asm)
+              ? 'it is only data pushes, and ASM with no opcodes reads back as hex'
+              : 'its ASM reads back as a different script'
+    throw new Error(`cannot write this script as ASM: ${why}; write it as hex`)
+  }
+  return asm
+}
+
 function sameOp (a, b) {
   if (a.code !== b.code) return false
   if (a.code === TAIL || a.code === DEAD) return a.raw.equals(b.raw)
@@ -197,5 +248,5 @@ function sameOp (a, b) {
 
 module.exports = {
   OP, TAIL, DEAD, isPush, pushValue, opSize, opsSize, pushOp, pushCost, numOp,
-  parse, encode, toAsm, toBuffer, opName, sameOp
+  parse, encode, toAsm, exactAsm, toBuffer, opName, sameOp
 }

src/verify.js

@@ -93,10 +93,33 @@ function randomElement (rnd) {
   return crypto.randomBytes(rnd() % 40)
 }
 
+// Consensus flags for the eras a script is checked in. Chronicle changes what
+// some opcodes do (OP_VERIF and OP_VERNOTIF in an unexecuted branch, OP_2MUL,
+// OP_SUBSTR, ...), so a script optimized without assuming Chronicle is checked
+// under the rules before it as well.
+function eraFlags ({ chronicle = true } = {}) {
+  const current = Interpreter.currentConsensusFlags()
+  if (chronicle) return [current]
+  return [current, current & ~(Interpreter.SCRIPT_UTXO_AFTER_CHRONICLE | Interpreter.SCRIPT_ENABLE_CHRONICLE)]
+}
+
 // Runs both scripts on random starting stacks (and any supplied ones) and
 // compares success, final stack and final alt stack. Failures are compared
-// only as failures: an optimized script may fail at a different op.
+// only as failures: an optimized script may fail at a different op. `flags`
+// may be a list of flag sets, one per era; the first era that disagrees is
+// returned with its counterexample.
 function differential (originalBuf, optimizedBuf, { runs = 200, maxDepth = 12, stacks = [], flags } = {}) {
+  if (Array.isArray(flags)) {
+    let total = 0
+    let succeeded = 0
+    for (const f of flags) {
+      const d = differential(originalBuf, optimizedBuf, { runs, maxDepth, stacks, flags: f })
+      if (!d.ok) return Object.assign(d, { flags: f })
+      total += d.runs
+      succeeded += d.succeeded
+    }
+    return { ok: true, runs: total, succeeded, eras: flags.length }
+  }
   flags = flags === undefined ? Interpreter.currentConsensusFlags() : flags
   const rnd = () => crypto.randomBytes(4).readUInt32LE(0)
   const inputs = stacks.slice()
@@ -121,4 +144,4 @@ function differential (originalBuf, optimizedBuf, { runs = 200, maxDepth = 12, s
   return { ok: true, runs: inputs.length, succeeded }
 }
 
-module.exports = { proveEquivalent, differential, evaluate, encode }
+module.exports = { proveEquivalent, differential, evaluate, encode, eraFlags }

test/fuzz.test.js

@@ -145,3 +145,22 @@ test('wide opcode fuzz with stacks that pass checks', () => {
   }
   assert.ok(passing > 0)
 })
+
+// OP_0 OP_IF blocks holding OP_VERIF/OP_VERNOTIF inside other conditionals: in
+// an unexecuted branch those open a conditional after Chronicle and do nothing
+// before it. Optimized without assuming Chronicle, and checked in both eras.
+test('conditionals whose meaning changes with Chronicle, checked in both eras', () => {
+  const { eraFlags } = require('../src/verify')
+  const r = rng(0xC4C)
+  const pick = a => a[r() % a.length]
+  const seq = () => Array.from({ length: r() % 3 }, () => pick(['OP_DUP', 'OP_DROP', 'OP_SWAP', 'OP_OVER', 'OP_1', 'OP_ADD', 'OP_NIP', 'OP_DUP OP_DROP', 'OP_RETURN'])).join(' ')
+  const { toBuffer } = require('../src/script')
+  for (let i = 0; i < 400; i++) {
+    const src = [seq(), pick(['OP_1', 'OP_0', '']), 'OP_IF', 'OP_0 OP_IF', pick(['OP_VERIF', 'OP_VERNOTIF']), seq(),
+      pick(['OP_ELSE', '']), seq(), 'OP_ENDIF', seq(), pick(['OP_ENDIF', '']), seq(), 'OP_DUP OP_DROP', seq()].join(' ').replace(/\s+/g, ' ').trim()
+    const buf = toBuffer(src)
+    const res = optimize(buf, { chronicle: false, differential: false })
+    const diff = differential(buf, res.script, { runs: 15, maxDepth: 5, flags: eraFlags({ chronicle: false }) })
+    assert.ok(diff.ok, `mismatch for ${src} -> ${res.script.toString('hex')}: ${JSON.stringify(diff)}`)
+  }
+})

test/unit.test.js

@@ -187,6 +187,47 @@ test('CLI optimizes, explains and profiles', () => {
   assert.strictEqual(OP.OP_DROP, 0x75)
 })
 
+test('ASM output reads back byte for byte, or is refused', () => {
+  const { exactAsm } = require('../src')
+  const { differential, eraFlags } = require('../src/verify')
+  for (const src of [
+    'OP_DUP OP_HASH160 ' + '11'.repeat(20) + ' OP_EQUALVERIFY OP_CHECKSIG OP_RETURN 74657374 ' + 'ab'.repeat(80),
+    'OP_0 OP_IF 6f7264 OP_1 746578742f706c61696e OP_0 ' + 'cd'.repeat(300) + ' OP_ENDIF OP_1',
+    'OP_1 OP_RETURN OP_INVALIDOPCODE 0'
+  ]) {
+    const buf = toBuffer(src)
+    assert.ok(toBuffer(exactAsm(buf)).equals(buf), src)
+  }
+  assert.throws(() => exactAsm(Buffer.from('4c03aabbcc51', 'hex')), /not minimally encoded/)
+  assert.throws(() => exactAsm(Buffer.from('516a05aabb', 'hex')), /truncated push/)
+  assert.throws(() => exactAsm(Buffer.from('4c50' + 'ab'.repeat(80), 'hex')), /only data pushes/)
+
+  const bin = path.join(__dirname, '..', 'bin', 'scriptmin.js')
+  const dir = require('fs').mkdtempSync(path.join(require('os').tmpdir(), 'scriptmin-asm-'))
+  try {
+    const src = 'OP_DUP OP_DROP OP_DUP OP_HASH160 ' + '11'.repeat(20) + ' OP_EQUALVERIFY OP_CHECKSIG OP_RETURN ' + 'ab'.repeat(40)
+    require('fs').writeFileSync(path.join(dir, 'in.asm'), src)
+    execFileSync('node', [bin, '--asm', '--tests', '0', '-o', path.join(dir, 'out.asm'), path.join(dir, 'in.asm')])
+    const written = require('fs').readFileSync(path.join(dir, 'out.asm'), 'utf8')
+    assert.ok(written.trim().endsWith('OP_RETURN ' + 'ab'.repeat(40)), written)
+    require('fs').writeFileSync(path.join(dir, 'odd.hex'), '76756a4c03aabbcc')
+    assert.throws(() => execFileSync('node', [bin, '--asm', '--tests', '0', '-o', path.join(dir, 'odd.asm'), path.join(dir, 'odd.hex')], { stdio: 'pipe' }), e => e.status === 2 && /write it as hex/.test(e.stderr))
+  } finally {
+    require('fs').rmSync(dir, { recursive: true, force: true })
+  }
+
+  // Scripts optimized without assuming Chronicle are checked under the rules before it too.
+  assert.strictEqual(eraFlags().length, 1)
+  assert.strictEqual(eraFlags({ chronicle: false }).length, 2)
+  // Between Genesis and Chronicle this OP_VERIF opens nothing, so the OP_DROP runs.
+  const orig = toBuffer('OP_1 OP_IF OP_0 OP_IF OP_VERIF OP_ELSE OP_DROP OP_ENDIF OP_ENDIF OP_DUP OP_DROP OP_1')
+  const wrong = toBuffer('OP_1 OP_IF OP_0 OP_IF OP_VERIF OP_ELSE OP_DROP OP_ENDIF OP_ENDIF OP_1')
+  const stacks = [[Buffer.from([7])]]
+  assert.ok(differential(orig, wrong, { runs: 0, stacks, flags: eraFlags() }).ok)
+  assert.ok(!differential(orig, wrong, { runs: 0, stacks, flags: eraFlags({ chronicle: false }) }).ok)
+  assert.strictEqual(opt('OP_DUP OP_DROP OP_1 OP_ADD', { chronicle: false }).report.verification.differential.eras, 2)
+})
+
 // A constant ROLL index in the tens of thousands makes the fragment need that
 // many inputs. Arranging them one by one was quadratic (75 s here); a schedule
 // is abandoned once it is far larger than the fragment it would replace.