diff --git a/README.md b/README.md index 809f0f3..6c54d49 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ cd scriptmin && npm install && npm test ```bash scriptmin script.hex # optimize, print report and hex scriptmin -o min.hex script.hex # write the result -scriptmin --asm -o min.asm script.asm # ASM in, ASM out +scriptmin --asm -o min.asm script.asm # ASM in, ASM out (refused if ASM cannot hold it exactly) scriptmin --profile script.hex # where the bytes go scriptmin --explain script.hex # every rewrite, largest first scriptmin --json script.hex # machine-readable report @@ -159,7 +159,8 @@ The barrier sequences of the two scripts must be identical. If the proof fails, `@smartledger/bsv` interpreter with random starting stacks, plus any you supply with `--stacks`. Success, final stack and final alt stack must match. Random stacks rarely get past a script's first real check, so for large verifiers pass -realistic unlocking stacks with `--stacks`. +realistic unlocking stacks with `--stacks`. With `--no-chronicle` the tests run +under the rules before Chronicle as well as after it. ### Assumptions and caveats @@ -167,6 +168,12 @@ realistic unlocking stacks with `--stacks`. Chronicle opcodes (`OP_SUBSTR`, `OP_LEFT`, `OP_RIGHT`, `OP_LSHIFTNUM`, `OP_RSHIFTNUM`) are modelled with their Chronicle stack effects. Use `--no-chronicle` to treat them as barriers. +- **ASM cannot hold every script.** It does not record how data was pushed, so + `--asm -o` refuses a result with a non-minimal push (optimized code is + minimal, but `OP_0 OP_IF` blocks and data after `OP_RETURN` are kept + verbatim), a truncated push, an unnamed opcode, or only data pushes, rather + than write a different script. + Hex and `--binary` output are always exact. - **Signatures commit to the script.** `OP_CHECKSIG` signs the script code, so signatures and OP_PUSH_TX preimages must be created against the optimized script. A covenant that embeds its own script hash, length or bytes must be @@ -307,6 +314,7 @@ report.verification // { symbolic, differential } profile(script) // categories, opcodes, PICK/ROLL depths, costly patterns proveEquivalent(a, b) // { ok, regions, barriers } or { ok: false, reason } differential(a, b, { runs: 500 }) // { ok, runs, succeeded } or a counterexample +exactAsm(script) // ASM that reads back byte for byte, or throws ``` ## Benchmarks diff --git a/bin/scriptmin.js b/bin/scriptmin.js index 89f147c..473b9c3 100755 --- a/bin/scriptmin.js +++ b/bin/scriptmin.js @@ -6,8 +6,7 @@ const fs = require('fs') // Output piped into something that stops reading (| head) is not an error. process.stdout.on('error', (e) => { if (e.code === 'EPIPE') process.exit(0); throw e }) const path = require('path') -const { optimize, profile, Cache, toAsm, toBuffer } = require('../src') -const { parse } = require('../src/script') +const { optimize, profile, Cache, toAsm, exactAsm, toBuffer } = require('../src') const { StackTable } = require('../src/superopt') const { EFFORT } = require('../src/optimize') @@ -84,6 +83,15 @@ function die (msg) { process.exit(2) } +// Written ASM must read back as the same script: refuse rather than write a different one. +function asmOut (script) { + try { + return exactAsm(script) + '\n' + } catch (e) { + die(e.message) + } +} + const fmt = n => n.toLocaleString('en-US') const clip = (text, ops, width = 150) => (text.length <= width ? text : `${text.slice(0, width)}… (${fmt(ops)} ops)`) const pad = (s, n) => String(s).padEnd(n) @@ -173,7 +181,7 @@ function compileMain (argv) { } catch (e) { die(e.message) } - if (a.out) fs.writeFileSync(a.out, a.asm ? toAsm(parse(res.script)) + '\n' : res.script.toString('hex') + '\n') + if (a.out) fs.writeFileSync(a.out, a.asm ? asmOut(res.script) : res.script.toString('hex') + '\n') const r = res.report if (a.json) { process.stdout.write(JSON.stringify({ script: res.script.toString('hex'), gates: r.gates, maxBits: r.maxBits, bytes: r.optimized, referenceBytes: r.reference, checked: r.checked, ms: r.ms }, null, 2) + '\n') @@ -275,7 +283,7 @@ function main () { if (a.db && cache) fs.writeFileSync(a.db, JSON.stringify(cache.toJSON())) if (a.out) { - const data = a.binary ? res.script : a.asm ? toAsm(parse(res.script)) + '\n' : res.script.toString('hex') + '\n' + const data = a.binary ? res.script : a.asm ? asmOut(res.script) : res.script.toString('hex') + '\n' fs.mkdirSync(path.dirname(path.resolve(a.out)), { recursive: true }) fs.writeFileSync(a.out, data) } diff --git a/src/index.js b/src/index.js index f244722..b1c6d5b 100644 --- a/src/index.js +++ b/src/index.js @@ -24,5 +24,6 @@ module.exports = { parse: script.parse, encode: script.encode, toAsm: script.toAsm, + exactAsm: script.exactAsm, toBuffer: script.toBuffer } diff --git a/src/optimize.js b/src/optimize.js index fb35a77..41858dc 100644 --- a/src/optimize.js +++ b/src/optimize.js @@ -7,7 +7,7 @@ const { peephole } = require('./peephole') const { windowsRegion } = require('./windows') const { rescheduleFragment } = require('./schedule') const { Cache, StackTable } = require('./superopt') -const { proveEquivalent, differential } = require('./verify') +const { proveEquivalent, differential, eraFlags } = require('./verify') const { profile } = require('./profile') const EFFORT = { @@ -173,7 +173,7 @@ function optimize (input, options = {}) { } } if (cfg.differential !== 0 && cfg.differential !== false) { - const diff = differential(buf, script, { runs: typeof cfg.differential === 'number' ? cfg.differential : 100, stacks: cfg.stacks || [] }) + const diff = differential(buf, script, { runs: typeof cfg.differential === 'number' ? cfg.differential : 100, stacks: cfg.stacks || [], flags: eraFlags(cfg) }) verification.differential = diff if (!diff.ok) { const err = new Error('internal error: optimized script disagrees with the original on a test input') diff --git a/src/script.js b/src/script.js index 0d92d63..8eb880b 100644 --- a/src/script.js +++ b/src/script.js @@ -163,15 +163,23 @@ function opName (code) { return NAMES[code] || ('OP_UNKNOWN' + code) } -function toAsm (ops, { maxData = 0 } = {}) { +// ASM for reading. With `bare`, it is the ASM @smartledger/bsv reads: data as +// bare hex whatever its push opcode, and the bytes after a top-level OP_RETURN +// as ASM when they parse. See exactAsm for writing a script out. +function toAsm (ops, { maxData = 0, bare = false } = {}) { return ops.map(op => { - if (op.code === TAIL) return `` - if (op.code === DEAD) return toAsm(parse(op.raw, { deadBlocks: false }), { maxData }) + if (op.code === TAIL) { + if (bare) { + try { return bsv.Script.fromBuffer(op.raw).toASM() } catch (e) {} + } + return `` + } + if (op.code === DEAD) return toAsm(parse(op.raw, { deadBlocks: false }), { maxData, bare }) if (op.code === OP.OP_0) return 'OP_0' if (op.code > 0 && op.code <= OP.OP_PUSHDATA4) { const hex = op.data.toString('hex') const shown = maxData && hex.length > maxData * 2 ? hex.slice(0, maxData * 2) + '…' : hex - return op.code >= OP.OP_PUSHDATA1 ? `${opName(op.code)}:${shown}` : shown + return op.code >= OP.OP_PUSHDATA1 && !bare ? `${opName(op.code)}:${shown}` : shown } return opName(op.code) }).join(' ') @@ -188,6 +196,49 @@ function toBuffer (input) { return bsv.Script.fromASM(text.replace(/\s+/g, ' ')).toBuffer() } +// Whether a push anywhere in buf, including after OP_RETURN, runs past its end. +function truncated (buf) { + let i = 0 + while (i < buf.length) { + const code = buf[i++] + if (code === 0 || code > OP.OP_PUSHDATA4) continue + let len = code + if (code >= OP.OP_PUSHDATA1) { + const w = code === OP.OP_PUSHDATA1 ? 1 : code === OP.OP_PUSHDATA2 ? 2 : 4 + if (i + w > buf.length) return true + len = w === 1 ? buf[i] : w === 2 ? buf.readUInt16LE(i) : buf.readUInt32LE(i) + i += w + } + if (i + len > buf.length) return true + i += len + } + return false +} + +// ASM that reads back as exactly `buf`, or an error. ASM cannot say how data +// was pushed (OP_PUSHDATA1 with a short payload reads back as a direct push), +// a truncated push, or an opcode with no name; and text of bare hex alone reads +// as hex. Scripts like those have to be written as hex. +function exactAsm (buf) { + const asm = toAsm(parse(buf), { bare: true }) + let back = null + try { back = toBuffer(asm) } catch (e) {} + if (!back || !back.equals(buf)) { + const ops = parse(buf, { deadBlocks: false }) + const why = truncated(buf) + ? 'it contains a truncated push' + : ops.some(o => isPush(o) && o.code !== TAIL && opSize(pushOp(pushValue(o))) !== opSize(o)) + ? 'a push is not minimally encoded, and ASM cannot say how data was pushed' + : /OP_UNKNOWN|\bOP_INVALIDOPCODE\b/.test(asm) || ops.some(o => o.code >= 0 && !isPush(o) && !NAMES[o.code]) + ? 'it uses an opcode that has no name in ASM' + : !/\bOP_/.test(asm) + ? 'it is only data pushes, and ASM with no opcodes reads back as hex' + : 'its ASM reads back as a different script' + throw new Error(`cannot write this script as ASM: ${why}; write it as hex`) + } + return asm +} + function sameOp (a, b) { if (a.code !== b.code) return false if (a.code === TAIL || a.code === DEAD) return a.raw.equals(b.raw) @@ -197,5 +248,5 @@ function sameOp (a, b) { module.exports = { OP, TAIL, DEAD, isPush, pushValue, opSize, opsSize, pushOp, pushCost, numOp, - parse, encode, toAsm, toBuffer, opName, sameOp + parse, encode, toAsm, exactAsm, toBuffer, opName, sameOp } diff --git a/src/verify.js b/src/verify.js index 6e85dbc..63e74f4 100644 --- a/src/verify.js +++ b/src/verify.js @@ -93,10 +93,33 @@ function randomElement (rnd) { return crypto.randomBytes(rnd() % 40) } +// Consensus flags for the eras a script is checked in. Chronicle changes what +// some opcodes do (OP_VERIF and OP_VERNOTIF in an unexecuted branch, OP_2MUL, +// OP_SUBSTR, ...), so a script optimized without assuming Chronicle is checked +// under the rules before it as well. +function eraFlags ({ chronicle = true } = {}) { + const current = Interpreter.currentConsensusFlags() + if (chronicle) return [current] + return [current, current & ~(Interpreter.SCRIPT_UTXO_AFTER_CHRONICLE | Interpreter.SCRIPT_ENABLE_CHRONICLE)] +} + // Runs both scripts on random starting stacks (and any supplied ones) and // compares success, final stack and final alt stack. Failures are compared -// only as failures: an optimized script may fail at a different op. +// only as failures: an optimized script may fail at a different op. `flags` +// may be a list of flag sets, one per era; the first era that disagrees is +// returned with its counterexample. function differential (originalBuf, optimizedBuf, { runs = 200, maxDepth = 12, stacks = [], flags } = {}) { + if (Array.isArray(flags)) { + let total = 0 + let succeeded = 0 + for (const f of flags) { + const d = differential(originalBuf, optimizedBuf, { runs, maxDepth, stacks, flags: f }) + if (!d.ok) return Object.assign(d, { flags: f }) + total += d.runs + succeeded += d.succeeded + } + return { ok: true, runs: total, succeeded, eras: flags.length } + } flags = flags === undefined ? Interpreter.currentConsensusFlags() : flags const rnd = () => crypto.randomBytes(4).readUInt32LE(0) const inputs = stacks.slice() @@ -121,4 +144,4 @@ function differential (originalBuf, optimizedBuf, { runs = 200, maxDepth = 12, s return { ok: true, runs: inputs.length, succeeded } } -module.exports = { proveEquivalent, differential, evaluate, encode } +module.exports = { proveEquivalent, differential, evaluate, encode, eraFlags } diff --git a/test/fuzz.test.js b/test/fuzz.test.js index 76fb208..06fd2b5 100644 --- a/test/fuzz.test.js +++ b/test/fuzz.test.js @@ -145,3 +145,22 @@ test('wide opcode fuzz with stacks that pass checks', () => { } assert.ok(passing > 0) }) + +// OP_0 OP_IF blocks holding OP_VERIF/OP_VERNOTIF inside other conditionals: in +// an unexecuted branch those open a conditional after Chronicle and do nothing +// before it. Optimized without assuming Chronicle, and checked in both eras. +test('conditionals whose meaning changes with Chronicle, checked in both eras', () => { + const { eraFlags } = require('../src/verify') + const r = rng(0xC4C) + const pick = a => a[r() % a.length] + const seq = () => Array.from({ length: r() % 3 }, () => pick(['OP_DUP', 'OP_DROP', 'OP_SWAP', 'OP_OVER', 'OP_1', 'OP_ADD', 'OP_NIP', 'OP_DUP OP_DROP', 'OP_RETURN'])).join(' ') + const { toBuffer } = require('../src/script') + for (let i = 0; i < 400; i++) { + const src = [seq(), pick(['OP_1', 'OP_0', '']), 'OP_IF', 'OP_0 OP_IF', pick(['OP_VERIF', 'OP_VERNOTIF']), seq(), + pick(['OP_ELSE', '']), seq(), 'OP_ENDIF', seq(), pick(['OP_ENDIF', '']), seq(), 'OP_DUP OP_DROP', seq()].join(' ').replace(/\s+/g, ' ').trim() + const buf = toBuffer(src) + const res = optimize(buf, { chronicle: false, differential: false }) + const diff = differential(buf, res.script, { runs: 15, maxDepth: 5, flags: eraFlags({ chronicle: false }) }) + assert.ok(diff.ok, `mismatch for ${src} -> ${res.script.toString('hex')}: ${JSON.stringify(diff)}`) + } +}) diff --git a/test/unit.test.js b/test/unit.test.js index b19d846..0d99899 100644 --- a/test/unit.test.js +++ b/test/unit.test.js @@ -187,6 +187,47 @@ test('CLI optimizes, explains and profiles', () => { assert.strictEqual(OP.OP_DROP, 0x75) }) +test('ASM output reads back byte for byte, or is refused', () => { + const { exactAsm } = require('../src') + const { differential, eraFlags } = require('../src/verify') + for (const src of [ + 'OP_DUP OP_HASH160 ' + '11'.repeat(20) + ' OP_EQUALVERIFY OP_CHECKSIG OP_RETURN 74657374 ' + 'ab'.repeat(80), + 'OP_0 OP_IF 6f7264 OP_1 746578742f706c61696e OP_0 ' + 'cd'.repeat(300) + ' OP_ENDIF OP_1', + 'OP_1 OP_RETURN OP_INVALIDOPCODE 0' + ]) { + const buf = toBuffer(src) + assert.ok(toBuffer(exactAsm(buf)).equals(buf), src) + } + assert.throws(() => exactAsm(Buffer.from('4c03aabbcc51', 'hex')), /not minimally encoded/) + assert.throws(() => exactAsm(Buffer.from('516a05aabb', 'hex')), /truncated push/) + assert.throws(() => exactAsm(Buffer.from('4c50' + 'ab'.repeat(80), 'hex')), /only data pushes/) + + const bin = path.join(__dirname, '..', 'bin', 'scriptmin.js') + const dir = require('fs').mkdtempSync(path.join(require('os').tmpdir(), 'scriptmin-asm-')) + try { + const src = 'OP_DUP OP_DROP OP_DUP OP_HASH160 ' + '11'.repeat(20) + ' OP_EQUALVERIFY OP_CHECKSIG OP_RETURN ' + 'ab'.repeat(40) + require('fs').writeFileSync(path.join(dir, 'in.asm'), src) + execFileSync('node', [bin, '--asm', '--tests', '0', '-o', path.join(dir, 'out.asm'), path.join(dir, 'in.asm')]) + const written = require('fs').readFileSync(path.join(dir, 'out.asm'), 'utf8') + assert.ok(written.trim().endsWith('OP_RETURN ' + 'ab'.repeat(40)), written) + require('fs').writeFileSync(path.join(dir, 'odd.hex'), '76756a4c03aabbcc') + assert.throws(() => execFileSync('node', [bin, '--asm', '--tests', '0', '-o', path.join(dir, 'odd.asm'), path.join(dir, 'odd.hex')], { stdio: 'pipe' }), e => e.status === 2 && /write it as hex/.test(e.stderr)) + } finally { + require('fs').rmSync(dir, { recursive: true, force: true }) + } + + // Scripts optimized without assuming Chronicle are checked under the rules before it too. + assert.strictEqual(eraFlags().length, 1) + assert.strictEqual(eraFlags({ chronicle: false }).length, 2) + // Between Genesis and Chronicle this OP_VERIF opens nothing, so the OP_DROP runs. + const orig = toBuffer('OP_1 OP_IF OP_0 OP_IF OP_VERIF OP_ELSE OP_DROP OP_ENDIF OP_ENDIF OP_DUP OP_DROP OP_1') + const wrong = toBuffer('OP_1 OP_IF OP_0 OP_IF OP_VERIF OP_ELSE OP_DROP OP_ENDIF OP_ENDIF OP_1') + const stacks = [[Buffer.from([7])]] + assert.ok(differential(orig, wrong, { runs: 0, stacks, flags: eraFlags() }).ok) + assert.ok(!differential(orig, wrong, { runs: 0, stacks, flags: eraFlags({ chronicle: false }) }).ok) + assert.strictEqual(opt('OP_DUP OP_DROP OP_1 OP_ADD', { chronicle: false }).report.verification.differential.eras, 2) +}) + // A constant ROLL index in the tens of thousands makes the fragment need that // many inputs. Arranging them one by one was quadratic (75 s here); a schedule // is abandoned once it is far larger than the fragment it would replace.