diff --git a/.gitignore b/.gitignore index bd77bab..e9cd4e3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ node_modules/ patterns.json overview.md .env +.survey/ diff --git a/README.md b/README.md index 40d6f33..48876fe 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ On a real, hand-optimized BLS12-381 Miller loop (the 333 KB script that ran on mainnet) it removes **40%**, 333,031 → 199,147 bytes, and every test case of the library it came from still passes. See [docs/real-world.md](docs/real-world.md). +Across a sample of mainnet, the saving is concentrated in contracts that check +their own transaction: about 20% of their bytes, and under 1% of all +locking-script bytes, since most of the chain is P2PKH (already minimal) and +data. See [docs/mainnet-survey.md](docs/mainnet-survey.md), which you can +reproduce with `examples/mainnet-survey.js`. + Built on [`@smartledger/bsv`](https://www.npmjs.com/package/@smartledger/bsv) for script parsing, hashing and the reference interpreter used in testing. @@ -109,6 +115,25 @@ after a top-level `OP_RETURN` is kept byte for byte, and so is every and it is where data envelopes such as inscriptions live, so rewriting its contents would change the data without changing what the script does. +Two more things are kept as they are, because a script can be shorter without +being the same output: + +- **Data that nothing uses.** A push no operation takes as an operand and that + is not left on the stack is a data carrier: ` OP_DROP`, a document + before a `OP_CHECKSIG`, protocol fields signed and dropped in pairs. Dropping + it would not change what the script does, but the output would no longer + carry what it was made to carry. Such pushes are barriers, kept byte for + byte with their encoding; where a region carries data, its one-byte fields + and empty pushes are kept too. `keepData: false` (`--drop-unused-data`) + removes them, and `dataMinBytes` (default 2) sets how long an unused push + must be for its region to count as carrying data. +- **Standard output templates.** P2PKH, P2PK, P2SH, bare multisig and + `OP_RETURN` data outputs are returned unchanged. Wallets, indexers and relay + policy recognise them by their exact bytes: a bare multisig that lists the + same key twice could reuse it with `OP_OVER` and save 65 bytes, and stop + being a bare multisig. `templates: false` (`--rewrite-templates`) optimizes + them anyway. + Each region goes through these passes, repeated while they keep finding savings: | Pass | What it does | @@ -303,6 +328,9 @@ const { script, ops, report } = optimize(hexOrAsmOrBuffer, { stacks: [], // extra starting stacks (arrays of Buffers) verify: true, // symbolic proof chronicle: true, + keepData: true, // keep pushes nothing uses (they carry data) + dataMinBytes: 2, // unused push length that marks a region as carrying data + templates: true, // leave standard outputs (P2PKH, bare multisig, ...) unchanged cache: new Cache() // share across calls to reuse solutions }) @@ -311,6 +339,7 @@ report.passes // [{ name, saved }] report.byCategory // [{ name, before, after, saved }] stack, arithmetic, data pushes... report.rewrites // [{ pass, rule?, before, after, saved, regionOffset }] report.verification // { symbolic, differential } +report.kept // { template, dataPushes } profile(script) // categories, opcodes, PICK/ROLL depths, costly patterns proveEquivalent(a, b) // { ok, regions, barriers } or { ok: false, reason } diff --git a/bin/scriptmin.js b/bin/scriptmin.js index 473b9c3..6dfe465 100755 --- a/bin/scriptmin.js +++ b/bin/scriptmin.js @@ -44,6 +44,8 @@ Options: --stacks JSON array of starting stacks (arrays of hex) to test against --no-verify skip the symbolic equivalence proof (not recommended) --no-chronicle treat Chronicle opcodes (OP_SUBSTR, OP_LEFT, ...) as barriers + --drop-unused-data let pushes that nothing uses be removed (kept by default: they carry data) + --rewrite-templates optimize standard outputs (P2PKH, P2PK, P2SH, bare multisig) too -h, --help show this help ` @@ -66,6 +68,8 @@ function parseArgs (argv) { case '--stacks': a.stacks = next(); break case '--no-verify': a.verify = false; break case '--no-chronicle': a.chronicle = false; break + case '--drop-unused-data': a.keepData = false; break + case '--rewrite-templates': a.templates = false; break case '-h': case '--help': process.stdout.write(USAGE); process.exit(0); break default: if (x.startsWith('-') && x !== '-') die(`unknown option ${x}`) @@ -270,6 +274,8 @@ function main () { cache, verify: a.verify, chronicle: a.chronicle, + keepData: a.keepData, + templates: a.templates, differential: a.tests, stacks }) diff --git a/docs/mainnet-survey.md b/docs/mainnet-survey.md new file mode 100644 index 0000000..6a93579 --- /dev/null +++ b/docs/mainnet-survey.md @@ -0,0 +1,103 @@ +# What is on the chain, and what scriptmin saves + +Everything before this was measured on scripts written for this project or for +one library. This is a survey of what mainnet actually holds, and what the +optimizer would do to it. + +Reproduce it with: + +```bash +node examples/mainnet-survey.js collect # ~20 min, resumable, reads a public API +node examples/mainnet-survey.js report +``` + +## The sample + +240 blocks spread evenly from the Genesis upgrade (height 620,538, February +2020) to height 967,180 (17 September 2026), read through WhatsOnChain. Blocks +with more transactions than the cap are sampled down to 400 of them, so a busy +block counts for as much as a quiet one. That gives 15,564 transactions and +59,323 outputs holding 27,485,018 bytes of locking script. + +## Where the bytes are + +| script | outputs | | bytes | | +| --- | ---: | ---: | ---: | ---: | +| non-standard | 1,144 | 1.9% | 14,382,348 | 52.3% | +| `OP_RETURN` data output | 26,707 | 45.0% | 11,311,633 | 41.2% | +| P2PKH + inscription | 230 | 0.4% | 1,003,538 | 3.7% | +| P2PKH | 31,175 | 52.6% | 779,375 | 2.8% | +| P2PKH + data | 9 | 0.0% | 6,094 | 0.0% | +| P2PK | 58 | 0.1% | 2,030 | 0.0% | + +Half the outputs are P2PKH, which is already minimal, and they hold under 3% of +the bytes. Most bytes are data: `OP_RETURN` payloads, inscription envelopes, and +two single scripts over 5 MB. Non-standard scripts are 1.9% of outputs but half +the bytes, and they are growing: in an earlier, wider sample of 385 blocks they +went from 94 outputs in the 2021 blocks to 922 in the 2026 ones. + +## What scriptmin saves + +The non-standard scripts fall into 41 templates (the same code, with different +data). One script per template was optimized at `--effort medium`; each passed +the equivalence proof and 20 interpreter runs. + +**0.40% of all locking-script bytes; 2.9% of the non-standard ones.** The saving +is concentrated in scripts that check their own transaction with OP_PUSH_TX: + +| template | outputs | bytes | saved each | +| --- | ---: | ---: | ---: | +| compiled contract, 2021–2023 (`OP_1 40 76 88 a9 ac …`) | 5 | 7,366 | 2,600 (35%) | +| the same family, other builds | 6 | 3,652 | 1,257 (34%) | +| | 6 | 5,626 | 1,225 (22%) | +| large contract, 2026 (`00 6a 76 88 a9 ac …`) | 2 | 200,262 | 32,081 (16%) | +| time-locked contract, 2024 | 1 | 934 | 240 (26%) | +| token contract, 2022–2023 | 29 | 784 | 129 (16%) | +| covenant parsing its own preimage, 2024–2026 | 120 | 1,341 | 5 (0.4%) | + +A wider sample of 385 blocks, with more contract outputs in it, put the same +figure at 0.96% of all locking bytes and 21% of OP_PUSH_TX contract bytes. How +much a sample catches depends on which contract outputs it happens to include; +the shape of the answer does not change. + +Nothing was saved on P2PKH, hashlocks, inscription envelopes or `OP_RETURN` +data. Those are already minimal, or they are data, which is kept. + +## What this means + +- **The audience is contract authors, not the network.** Rewriting the chain's + scripts is not worth it; generating smaller ones is. A contract that checks + its own transaction commits to its own code, so the saving is real only for a + contract deployed from the optimized script — the natural place is a compiler, + next to whatever emits the script. +- **A contract that was already hand-tuned has little left.** The most common + covenant in the sample gives up 5 bytes of 1,341. Compiler output gives up a + fifth to a third. + +## What the survey changed in scriptmin + +Three rewrites in the first run were correct and still wrong to make. They are +why `keepData` and `templates` exist, both on by default: + +| script | what the optimizer did | why it is wrong | +| --- | --- | --- | +| ` OP_DROP OP_CHECKSIG` (103 outputs here, 289 in the wider sample) | 121 B → 35 B | the output exists to carry the document | +| ` OP_CHECKSIG OP_2DROP OP_2DROP OP_DROP` (39 outputs) | 451 B → 35 B | the same, for signed protocol fields | +| ` 21e8 OP_SIZE … OP_DROP OP_CHECKSIG` (13 outputs, a Boost puzzle) | 46 B → 10 B | the hash names the content being worked on | +| `OP_2 OP_3 OP_CHECKMULTISIG` | 201 B → 136 B | reusing the repeated key with `OP_OVER` stops it being a bare multisig | + +A push that no operation uses, and that is not left on the stack, is data. +It is kept byte for byte, and so are standard output templates. Contract savings +are unaffected: across all templates they are the same as with the protections +off. + +## Caveats + +- One sample, one script per template. Two samples of the same chain gave 0.40% + and 0.96% of locking bytes. +- Outputs are counted per sampled transaction, not weighted by how much of the + chain's traffic each block carries. +- Locking scripts only. Unlocking scripts (signatures, contract arguments) are + not measured here. +- Savings assume the contract is deployed from the optimized script. Scripts + already on the chain cannot be rewritten. diff --git a/examples/mainnet-survey.js b/examples/mainnet-survey.js new file mode 100644 index 0000000..a6df96d --- /dev/null +++ b/examples/mainnet-survey.js @@ -0,0 +1,256 @@ +'use strict' + +// What mainnet locking scripts are made of, and what scriptmin would save. +// +// node examples/mainnet-survey.js collect [--blocks 240] [--tx 400] [--seconds 540] +// node examples/mainnet-survey.js report [--effort medium] +// +// `collect` samples blocks evenly from the Genesis upgrade to the chain tip +// through WhatsOnChain's public API, and records every output's locking script: +// standard ones as a class and a count, the rest in full. One file per block +// under .survey/, so it resumes where it stopped; the public API is paced, so a +// full sample takes about twenty minutes. +// +// `report` groups the non-standard scripts into templates (opcodes kept, data +// pushes replaced by their size), optimizes one script per template, and prints +// what that would save, weighted by how often the template appears. +// +// Nothing here spends, broadcasts or needs a key: it only reads public data. + +const fs = require('fs') +const path = require('path') +const https = require('https') +const crypto = require('crypto') +const bsv = require('@smartledger/bsv') +const { optimize } = require('../src') +const { parse, toAsm, isPush, pushValue, TAIL, DEAD, standardTemplate } = require('../src/script') + +const DIR = path.join(__dirname, '..', '.survey') +const GENESIS = 620538 // the Genesis upgrade, February 2020 +const HOST = 'api.whatsonchain.com' +const BASE = '/v1/bsv/main' +const MIN_GAP_MS = 350 + +const arg = (name, dflt) => { + const i = process.argv.indexOf('--' + name) + return i < 0 ? dflt : process.argv[i + 1] +} + +let lastCall = 0 +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) + +function request (method, p, body) { + return new Promise((resolve, reject) => { + const payload = body === undefined ? null : JSON.stringify(body) + const req = https.request({ + host: HOST, + path: BASE + p, + method, + headers: payload ? { 'content-type': 'application/json', 'content-length': Buffer.byteLength(payload) } : {} + }, res => { + const chunks = [] + res.on('data', c => chunks.push(c)) + res.on('end', () => { + const text = Buffer.concat(chunks).toString() + if (res.statusCode >= 400) return reject(new Error(`${res.statusCode} on ${p}: ${text.slice(0, 120)}`)) + try { resolve(JSON.parse(text)) } catch (e) { resolve(text) } + }) + }) + req.setTimeout(60000, () => req.destroy(new Error('timeout on ' + p))) + req.on('error', reject) + if (payload) req.write(payload) + req.end() + }) +} + +// Paced and retried: this is someone else's public API. +async function api (method, p, body) { + for (let attempt = 0; ; attempt++) { + const wait = Math.max(0, lastCall + MIN_GAP_MS - Date.now()) + if (wait) await sleep(wait) + lastCall = Date.now() + try { + return await request(method, p, body) + } catch (e) { + if (attempt >= 5 || !/^(429|5\d\d)|timeout|ECONNRESET|socket/.test(e.message)) throw e + await sleep(1000 * 2 ** attempt) + } + } +} + +// Standard templates plus the shapes built from them, so that "other" means a +// script someone wrote rather than a wallet's output. +function classify (buf) { + const std = standardTemplate(buf) + if (std) return std + if (buf.length === 0) return 'empty' + const hex = buf.toString('hex') + if (hex.includes('0063036f7264') && /76a914[0-9a-f]{40}88ac/.test(hex)) { + const stripped = hex.replace(/0063036f7264[0-9a-f]*68$/, '').replace(/^0063036f7264[0-9a-f]*?68(?=76a914)/, '') + if (/^76a914[0-9a-f]{40}88ac$/.test(stripped)) return 'P2PKH + inscription' + } + if (/^76a914[0-9a-f]{40}88ac6a/.test(hex)) return 'P2PKH + data' + return 'other' +} + +async function sampleBlock (height, maxTx) { + const file = path.join(DIR, height + '.json') + if (fs.existsSync(file)) return false + const b = await api('GET', '/block/height/' + height) + let txids = b.tx || [] + if (b.pages && b.pages.size) { + // A large block lists its txids in pages; take up to four at random. + txids = [] + const pages = new Set() + while (pages.size < Math.min(4, b.pages.size)) pages.add(1 + crypto.randomInt(b.pages.size)) + for (const page of pages) { + const more = await api('GET', `/block/hash/${b.hash}/page/${page}`) + if (Array.isArray(more)) txids.push(...more) + } + } + const picked = txids.length > maxTx + ? txids.map(t => [crypto.randomInt(1e9), t]).sort((x, y) => x[0] - y[0]).slice(0, maxTx).map(x => x[1]) + : txids + const counts = {} + const bytes = {} + const others = {} + let outputs = 0 + for (let i = 0; i < picked.length; i += 20) { + const res = await api('POST', '/txs/hex', { txids: picked.slice(i, i + 20) }) + for (const r of res) { + if (!r.hex) continue + let tx + try { tx = new bsv.Transaction(r.hex) } catch (e) { continue } + for (const o of tx.outputs) { + const s = o.script.toBuffer() + const c = classify(s) + outputs++ + counts[c] = (counts[c] || 0) + 1 + bytes[c] = (bytes[c] || 0) + s.length + if (c === 'other' || c === 'bare multisig' || c === 'P2PKH + data') { + const k = crypto.createHash('sha256').update(s).digest('hex') + if (!others[k]) others[k] = { hex: s.length <= 2e6 ? s.toString('hex') : null, size: s.length, n: 0, cls: c, txid: r.txid } + others[k].n++ + } + } + } + } + fs.writeFileSync(file, JSON.stringify({ height, hash: b.hash, time: b.time, size: b.size, txcount: b.txcount || txids.length, sampled: picked.length, outputs, counts, bytes, others })) + return true +} + +async function collect () { + fs.mkdirSync(DIR, { recursive: true }) + const nblocks = Number(arg('blocks', 240)) + const maxTx = Number(arg('tx', 400)) + const stopAt = Date.now() + Number(arg('seconds', 540)) * 1000 + // The tip is pinned in .survey/tip so that resuming picks the same blocks. + const tipFile = path.join(DIR, 'tip') + let tip + if (fs.existsSync(tipFile)) tip = Number(fs.readFileSync(tipFile, 'utf8')) + else { + tip = (await api('GET', '/chain/info')).blocks - 6 + fs.writeFileSync(tipFile, String(tip)) + } + const heights = [] + for (let i = 0; i < nblocks; i++) heights.push(Math.round(GENESIS + (tip - GENESIS) * (i + 0.5) / nblocks)) + let done = 0 + let had = 0 + for (const h of heights) { + if (Date.now() > stopAt) break + try { + if (await sampleBlock(h, maxTx)) done++ + else had++ + } catch (e) { + console.error('block', h, e.message) + } + } + console.log(`fetched ${done}, already had ${had}, of ${heights.length} blocks up to height ${tip}`) + if (done + had < heights.length) console.log('run again to continue') +} + +// Opcodes as they are, pushes of 8 bytes or more as , verbatim blocks as +// what they are: two scripts share a template when they are the same code. +function templateOf (buf) { + return parse(buf).map(o => { + if (o.code === TAIL) return '' + if (o.code === DEAD) return '' + if (isPush(o) && pushValue(o).length >= 8) return '' + return toAsm([o]) + }).join(' ') +} + +function report () { + if (!fs.existsSync(DIR)) { console.error('no .survey directory: run "collect" first'); process.exit(2) } + const files = fs.readdirSync(DIR).filter(f => f.endsWith('.json')) + const blocks = files.map(f => JSON.parse(fs.readFileSync(path.join(DIR, f)))) + const counts = {} + const bytes = {} + const others = new Map() + let outputs = 0 + let txs = 0 + for (const b of blocks) { + outputs += b.outputs + txs += b.sampled + for (const [k, v] of Object.entries(b.counts)) counts[k] = (counts[k] || 0) + v + for (const [k, v] of Object.entries(b.bytes)) bytes[k] = (bytes[k] || 0) + v + for (const [h, o] of Object.entries(b.others)) { + const e = others.get(h) || Object.assign({}, o, { n: 0 }) + e.n += o.n + others.set(h, e) + } + } + const total = Object.values(bytes).reduce((a, x) => a + x, 0) + console.log(`${blocks.length} blocks, ${txs} transactions, ${outputs} outputs, ${total} bytes of locking script\n`) + console.log('class'.padEnd(22) + 'outputs'.padStart(9) + '%'.padStart(7) + 'bytes'.padStart(12) + '%'.padStart(7)) + for (const c of Object.keys(counts).sort((a, b) => bytes[b] - bytes[a])) { + console.log(c.padEnd(22) + String(counts[c]).padStart(9) + (100 * counts[c] / outputs).toFixed(1).padStart(7) + + String(bytes[c]).padStart(12) + (100 * bytes[c] / total).toFixed(1).padStart(7)) + } + + const templates = new Map() + let hugeCount = 0 + let hugeBytes = 0 + for (const o of others.values()) { + // Scripts over 2 MB are counted but not stored: they are inscriptions, all data. + if (!o.hex) { hugeCount += o.n; hugeBytes += o.size * o.n; continue } + const buf = Buffer.from(o.hex, 'hex') + let t + try { t = templateOf(buf) } catch (e) { t = 'unparseable' } + const e = templates.get(t) || { t, n: 0, bytes: 0, example: buf, txid: o.txid } + e.n += o.n + e.bytes += o.size * o.n + templates.set(t, e) + } + const effort = arg('effort', 'medium') + console.log(`\n${templates.size} templates among the non-standard scripts; optimizing one script of each at --effort ${effort}\n`) + console.log('outputs'.padStart(8) + 'bytes'.padStart(10) + 'saved'.padStart(9) + ' kept template') + let before = 0 + let after = 0 + const rows = [...templates.values()].sort((a, b) => b.bytes - a.bytes) + for (const e of rows) { + let r + try { + r = optimize(e.example, { effort, differential: 20 }) + } catch (err) { + console.log(String(e.n).padStart(8) + String(e.example.length).padStart(10) + ' -' + ' ' + err.message.slice(0, 40)) + continue + } + const saved = (e.example.length - r.script.length) * e.n + before += e.bytes + after += e.bytes - saved + const kept = r.report.kept.template || (r.report.kept.dataPushes ? r.report.kept.dataPushes + ' data' : '') + console.log(String(e.n).padStart(8) + String(e.example.length).padStart(10) + String(saved).padStart(9) + ' ' + + kept.padEnd(6) + ' ' + (e.t.length > 70 ? e.t.slice(0, 70) + '…' : e.t)) + } + if (!before) { console.log('\nno non-standard scripts in this sample'); return } + if (hugeCount) console.log(`\n${hugeCount} script${hugeCount === 1 ? '' : 's'} over 2 MB (${hugeBytes} bytes) were counted but not kept, so not optimized here`) + console.log(`\nnon-standard scripts: ${before} -> ${after} bytes (${(100 * (before - after) / before).toFixed(1)}%)`) + console.log(`all locking scripts: ${(100 * (before - after) / total).toFixed(2)}% of ${total} bytes`) + console.log('Savings are code only: data and standard templates are kept (see README).') +} + +const cmd = process.argv[2] +if (cmd === 'collect') collect().catch(e => { console.error(e.message); process.exit(1) }) +else if (cmd === 'report') report() +else { console.error('usage: mainnet-survey.js collect|report [options]'); process.exit(2) } diff --git a/src/analysis.js b/src/analysis.js index f6ff6c8..307301d 100644 --- a/src/analysis.js +++ b/src/analysis.js @@ -11,11 +11,12 @@ // very start of a script must stay: there it is the only thing that fails an // empty stack. -const { OP, TAIL, DEAD } = require('./script') +const { OP, TAIL, DEAD, isPush, pushValue } = require('./script') const { Interner, SymState } = require('./symbolic') function barrierEffect (op, g, ga, frames) { const c = op.code + if (op.keep) return [g + 1, ga] // a data push kept verbatim switch (c) { case OP.OP_IF: case OP.OP_NOTIF: { @@ -91,6 +92,35 @@ function analyze (ops, opts = {}) { return { regions, barriers } } +// Marks, with `keep: true`, the pushes whose value nothing uses: no operation +// takes it as an operand, and it is not left on the stack for code after its +// region. Such a push only carries data (a protocol tag, a content hash, a +// document), so removing it would not change what the script does but would +// lose what the output was for. A region is treated as carrying data when one +// of its unused pushes is at least `minBytes` long; then every unused push in +// it is kept. A kept push is a barrier, left byte for byte by every pass and +// by the proof. Returns the number of pushes marked. +function markUnusedData (ops, { minBytes = 2, chronicle = true } = {}) { + const { regions } = analyze(ops, { chronicle }) + let marked = 0 + for (const r of regions) { + const I = new Interner() + const st = new SymState(I, { chronicle, trackUses: true }) + for (let i = r.start; i < r.end; i++) st.step(ops[i]) + const live = new Set([...st.uses, ...st.main, ...st.alt]) + const unused = [] + for (let i = r.start; i < r.end; i++) { + const op = ops[i] + if (isPush(op) && !op.keep && !live.has(I.konst(pushValue(op)))) unused.push(op) + } + // A region carrying data keeps all of it, one-byte fields and empty pushes + // included: those are as much a part of a protocol's payload as the rest. + if (!unused.some(op => pushValue(op).length >= minBytes)) continue + for (const op of unused) { op.keep = true; marked++ } + } + return marked +} + // Guaranteed main/alt heights before each op of a region, given its entry guarantee. function heights (ops, g, ga, opts = {}) { const I = new Interner() @@ -105,4 +135,4 @@ function heights (ops, g, ga, opts = {}) { return { hm, ha } } -module.exports = { analyze, heights } +module.exports = { analyze, heights, markUnusedData } diff --git a/src/optimize.js b/src/optimize.js index 41858dc..dd9dfbf 100644 --- a/src/optimize.js +++ b/src/optimize.js @@ -1,8 +1,8 @@ 'use strict' -const { OP, TAIL, DEAD, isPush, pushValue, pushOp, opSize, opsSize, parse, encode, toBuffer } = require('./script') +const { OP, TAIL, DEAD, isPush, pushValue, pushOp, opSize, opsSize, parse, encode, toBuffer, standardTemplate } = require('./script') const { equivalent } = require('./symbolic') -const { analyze, heights } = require('./analysis') +const { analyze, heights, markUnusedData } = require('./analysis') const { peephole } = require('./peephole') const { windowsRegion } = require('./windows') const { rescheduleFragment } = require('./schedule') @@ -77,7 +77,7 @@ function optimizeRegion (ops, g, ga, cache, cfg) { function normalizePushes (ops) { const rewrites = [] const out = ops.map((op, index) => { - if (op.code === TAIL || op.code === DEAD || !isPush(op)) return op + if (op.code === TAIL || op.code === DEAD || op.keep || !isPush(op)) return op const min = pushOp(pushValue(op)) if (opSize(min) >= opSize(op)) return op rewrites.push({ pass: 'push-encoding', index, before: [op], after: [min], saved: opSize(op) - opSize(min) }) @@ -86,8 +86,14 @@ function normalizePushes (ops) { return { ops: out, rewrites } } -function warningsFor (ops) { +function warningsFor (ops, kept) { const w = [] + if (kept.template) { + w.push(`The script is a standard ${kept.template} output, recognised by its exact bytes; it was left unchanged (templates: false to optimize it anyway).`) + } + if (kept.dataPushes) { + w.push(`${kept.dataPushes} data push${kept.dataPushes === 1 ? '' : 'es'} that nothing in the script uses ${kept.dataPushes === 1 ? 'was' : 'were'} kept byte for byte: removing ${kept.dataPushes === 1 ? 'it' : 'them'} would not change what the script does, but would lose the data the output carries (keepData: false to allow it).`) + } const has = codes => ops.some(o => codes.includes(o.code)) if (has([OP.OP_CHECKSIG, OP.OP_CHECKSIGVERIFY, OP.OP_CHECKMULTISIG, OP.OP_CHECKMULTISIGVERIFY])) { w.push('The script checks signatures. Signatures commit to the script code, so they (and any OP_PUSH_TX preimage) must be produced against the optimized script. Covenants that embed their own script hash, length or bytes must be regenerated from it.') @@ -111,10 +117,18 @@ function optimize (input, options = {}) { cfg.cache = cache const buf = toBuffer(input) const original = parse(buf) + const kept = { template: null, dataPushes: 0 } + if (cfg.templates !== false) kept.template = standardTemplate(buf) + // Before push normalization, so kept pushes keep their encoding too. + if (!kept.template && cfg.keepData !== false) { + kept.dataPushes = markUnusedData(original, { minBytes: cfg.dataMinBytes ?? 2, chronicle: cfg.chronicle }) + } let rewrites = [] let ops = original - if (cfg.pushes !== false) { + if (kept.template) { + // Nothing to do: fall through with no regions rewritten. + } else if (cfg.pushes !== false) { const res = normalizePushes(ops) ops = res.ops rewrites = rewrites.concat(res.rewrites) @@ -126,6 +140,7 @@ function optimize (input, options = {}) { for (const op of original) { offsets.push(off); off += opSize(op) } const { regions, barriers } = analyze(ops, cfg) + if (kept.template) regions.length = 0 const out = [] let cursor = 0 let reverted = 0 @@ -214,7 +229,8 @@ function optimize (input, options = {}) { reverted, verification, cache: { hits: cache.hits, entries: cache.added, searches: cache.searches }, - warnings: warningsFor(original), + kept, + warnings: warningsFor(original, kept), ms: Date.now() - t0 } } diff --git a/src/script.js b/src/script.js index cc20182..5d1ad6a 100644 --- a/src/script.js +++ b/src/script.js @@ -251,6 +251,27 @@ function exactAsm (buf) { return asm } +// The name of the standard output template `buf` matches exactly, or null. +// Wallets, indexers and relay policy recognise these by their bytes, so they are +// returned unchanged even where a shorter equivalent exists (a bare multisig +// listing the same key twice could reuse it with OP_OVER, and stop being one). +function standardTemplate (buf) { + const n = buf.length + if (n === 25 && buf[0] === OP.OP_DUP && buf[1] === OP.OP_HASH160 && buf[2] === 20 && buf[23] === OP.OP_EQUALVERIFY && buf[24] === OP.OP_CHECKSIG) return 'P2PKH' + if (n === 23 && buf[0] === OP.OP_HASH160 && buf[1] === 20 && buf[22] === OP.OP_EQUAL) return 'P2SH' + if ((n === 35 && buf[0] === 33 && buf[34] === OP.OP_CHECKSIG) || (n === 67 && buf[0] === 65 && buf[66] === OP.OP_CHECKSIG)) return 'P2PK' + if (n >= 1 && (buf[0] === OP.OP_RETURN || (n >= 2 && buf[0] === OP.OP_0 && buf[1] === OP.OP_RETURN))) return 'data output' + if (n >= 3 && buf[n - 1] === OP.OP_CHECKMULTISIG) { + const ops = parse(buf, { deadBlocks: false }) + const m = ops[0].code - OP.OP_1 + 1 + const k = ops[ops.length - 2].code - OP.OP_1 + 1 + const keys = ops.slice(1, -2) + if (m >= 1 && m <= 16 && k >= 1 && k <= 16 && m <= k && keys.length === k && + keys.every(o => o.data && (o.data.length === 33 || o.data.length === 65) && o.code === o.data.length)) return 'bare multisig' + } + return null +} + function sameOp (a, b) { if (a.code !== b.code) return false if (a.code === TAIL || a.code === DEAD) return a.raw.equals(b.raw) @@ -260,5 +281,5 @@ function sameOp (a, b) { module.exports = { OP, TAIL, DEAD, isPush, pushValue, opSize, opsSize, pushOp, pushCost, numOp, - parse, encode, toAsm, exactAsm, toBuffer, opName, sameOp + parse, encode, toAsm, exactAsm, toBuffer, opName, sameOp, standardTemplate } diff --git a/src/symbolic.js b/src/symbolic.js index 8c2ce98..f526487 100644 --- a/src/symbolic.js +++ b/src/symbolic.js @@ -172,7 +172,7 @@ const STACK_OPS = new Set([ ]) class SymState { - constructor (interner, { record = false, chronicle = true } = {}) { + constructor (interner, { record = false, chronicle = true, trackUses = false } = {}) { this.I = interner this.main = [] this.alt = [] @@ -182,6 +182,9 @@ class SymState { this.apps = record ? [] : null this.usedAlt = false this.chronicle = chronicle + // Values some operation consumed (as an operand or a PICK/ROLL index), + // whether or not it was folded. Stack moves and drops do not count. + this.uses = trackUses ? new Set() : null } ensure (k) { @@ -209,7 +212,7 @@ class SymState { // True if `op` can be modelled; false leaves the state untouched. supports (op) { const code = op.code - if (code < 0) return false + if (code < 0 || op.keep) return false if (isPush(op)) return true if (STACK_OPS.has(code)) { if (code === OP.OP_PICK || code === OP.OP_ROLL) { @@ -269,7 +272,9 @@ class SymState { } case OP.OP_PICK: case OP.OP_ROLL: { - const n = Number(decodeNum(this.I.constBuf(m.pop()), 4)) + const index = m.pop() + if (this.uses) this.uses.add(index) + const n = Number(decodeNum(this.I.constBuf(index), 4)) this.ensure(n + 1) const s = this.main const idx = s.length - 1 - n @@ -293,6 +298,7 @@ class SymState { const s = this.main const args = sp.pops ? s.splice(s.length - sp.pops) : [] const I = this.I + if (this.uses) for (const a of args) this.uses.add(a) if (code === OP.OP_VERIFY) { const x = args[0] diff --git a/test/fuzz.test.js b/test/fuzz.test.js index 06fd2b5..0fc8227 100644 --- a/test/fuzz.test.js +++ b/test/fuzz.test.js @@ -164,3 +164,41 @@ test('conditionals whose meaning changes with Chronicle, checked in both eras', assert.ok(diff.ok, `mismatch for ${src} -> ${res.script.toString('hex')}: ${JSON.stringify(diff)}`) } }) + +// Data pushes interleaved with code and dropped in every shape: whatever the +// optimizer does to the code, each unused data push of two or more bytes comes +// out byte for byte and in order, and the script still behaves the same. +test('data that nothing uses survives optimization, in order', () => { + const r = rng(0xDA7A) + const { parse } = require('../src/script') + const { markUnusedData } = require('../src/analysis') + const CODE = ['OP_DUP', 'OP_DROP', 'OP_SWAP', 'OP_OVER', 'OP_ADD', 'OP_NIP', 'OP_ROT', 'OP_1ADD', 'OP_SHA256', 'OP_SIZE', 'OP_EQUAL', 'OP_TOALTSTACK', 'OP_FROMALTSTACK'] + let checked = 0 + for (let i = 0; i < 300; i++) { + const ops = [] + const n = 3 + (r() % 16) + for (let k = 0; k < n; k++) { + const x = r() % 10 + if (x < 3) ops.push(pushOp(crypto.randomBytes(r() % 6))) + else if (x < 5) ops.push(numOp(r() % 4)) + else if (x < 6) ops.push({ code: [OP.OP_DROP, OP.OP_2DROP, OP.OP_NIP][r() % 3] }) + else ops.push({ code: OP[CODE[r() % CODE.length]] }) + } + const buf = encode(ops) + const marked = parse(buf) + markUnusedData(marked) + const kept = marked.filter(o => o.keep).map(o => encode([o]).toString('hex')) + const res = optimize(buf, { differential: false }) + let at = 0 + const hex = res.script.toString('hex') + for (const k of kept) { + const j = hex.indexOf(k, at) + assert.ok(j >= 0, `lost ${k} from ${buf.toString('hex')} -> ${hex}`) + at = j + k.length + } + checked += kept.length + const diff = differential(buf, res.script, { runs: 20, maxDepth: 6 }) + assert.ok(diff.ok, `mismatch for ${buf.toString('hex')} -> ${hex}: ${JSON.stringify(diff)}`) + } + assert.ok(checked > 50) +}) diff --git a/test/unit.test.js b/test/unit.test.js index ea7e8ed..3a9f02a 100644 --- a/test/unit.test.js +++ b/test/unit.test.js @@ -110,6 +110,48 @@ test('keeps OP_0 OP_IF data envelopes byte for byte and optimizes around them', assert.match(asm(opt(verif, { chronicle: false })), /OP_ENDIF OP_DUP OP_DROP OP_1$/) }) +test('keeps data that nothing uses, and standard templates, byte for byte', () => { + const key = '02' + '11'.repeat(32) + // Data carriers seen on mainnet: a tag dropped on its own, a document before + // a P2PK check, and signed protocol fields dropped in pairs. + for (const src of [ + '64796c616e31 OP_DROP', + 'OP_PUSHDATA1 ' + '20'.repeat(80) + ' OP_DROP ' + key + ' OP_CHECKSIG', + key + ' OP_CHECKSIG 6167696431 ' + '33'.repeat(20) + ' 32 OP_0 7c OP_2DROP OP_2DROP OP_DROP' + ]) { + const buf = toBuffer(src.replace('OP_PUSHDATA1 ' + '20'.repeat(80), '4c50' + '20'.repeat(80))) + const r = opt(buf) + assert.ok(r.script.equals(buf), `changed: ${src.slice(0, 40)} -> ${toAsm(r.ops, { maxData: 4 })}`) + assert.ok(r.report.kept.dataPushes > 0) + assert.ok(r.report.warnings.some(w => /kept byte for byte/.test(w))) + // Allowed when asked. + assert.ok(opt(buf, { keepData: false }).script.length < buf.length) + } + + // A Boost-style puzzle: the content hash is kept, the code around it is still optimized. + const hash = 'ab'.repeat(32) + const boost = toBuffer(`${hash} 21e8 OP_SIZE OP_4 OP_PICK OP_SHA256 OP_SWAP OP_SPLIT OP_DROP OP_EQUALVERIFY OP_DROP OP_CHECKSIG`) + const rb = opt(boost) + assert.ok(rb.script.toString('hex').includes('20' + hash)) + assert.ok(rb.script.length < boost.length) + assert.ok(rb.report.verification.symbolic.ok) + + // Used constants are not data: they still fold and deduplicate. + assert.strictEqual(opt('aabbccdd aabbccdd OP_EQUAL OP_VERIFY OP_1').script.toString('hex'), '51') + + // A bare multisig listing a key twice is left as the template, not rewritten with OP_OVER. + const k1 = '04' + 'ac'.repeat(64) + const k2 = '04' + '3e'.repeat(64) + const ms = toBuffer(`OP_2 ${k1} ${k2} ${k1} OP_3 OP_CHECKMULTISIG`) + const rm = opt(ms) + assert.ok(rm.script.equals(ms)) + assert.strictEqual(rm.report.kept.template, 'bare multisig') + assert.ok(opt(ms, { templates: false }).script.length < ms.length) + for (const [t, name] of [['76a914' + '11'.repeat(20) + '88ac', 'P2PKH'], ['a914' + '11'.repeat(20) + '87', 'P2SH'], ['21' + key + 'ac', 'P2PK'], ['006a0474657374', 'data output']]) { + assert.strictEqual(opt(t).report.kept.template, name) + } +}) + test('equivalence checker', () => { assert.ok(equivalent('OP_OVER OP_OVER', 'OP_2DUP')) assert.ok(!equivalent('OP_SWAP OP_SUB', 'OP_SUB'))