diff --git a/ProtoDef b/ProtoDef index 8e07785..a2b1cd2 160000 --- a/ProtoDef +++ b/ProtoDef @@ -1 +1 @@ -Subproject commit 8e07785e94626882fa3333184bb366e3f6625356 +Subproject commit a2b1cd2d84488012b7949fbe578d3781cc3329a4 diff --git a/doc/compiler.md b/doc/compiler.md index a97a078..c946c05 100644 --- a/doc/compiler.md +++ b/doc/compiler.md @@ -228,4 +228,24 @@ compiledProto.setVariable('noArraySizeCheck', true); // Use it as if it were a normal ProtoDef const buffer = compiledProto.createPacketBuffer('mainType', result) const result = compiledProto.parsePacketBuffer('mainType', buffer) -``` \ No newline at end of file +``` +### Sizing inside a writer + +A parametrizable type is compiled by one compiler at a time, so a writer normally has no way to know how large a nested value will be. When it must serialize part of the value before writing (a checksum of it, for example), `WriteCompiler.callTypeSize(value, type)` returns code calling the sizer for `type`, which the sizeOf context already holds since it is generated first. It is only available when the types are compiled through `ProtoDefCompiler`, and `type` has to be a named type, since an anonymous one has no function in that context to call. + +A datatype that needs a helper function in its generated code registers it as a context type, which copies the function's source into the compiled output, rather than reaching for something outside it. The `hash` datatype is built on both: + +```javascript +Write: { + _crc32c: ['context', crc32c], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = ctx._${alg}(bodyBuffer)\n` + code += 'return ' + compiler.callType('hash', type) + return compiler.wrapCode(code) + }] +} +``` + +The context is shared with the protocol's own type names, and a context entry wins over a type of the same name, so a helper's name is underscored to keep it out of the way. diff --git a/src/compiler.js b/src/compiler.js index 9c8f057..24c5aa1 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -12,6 +12,7 @@ class ProtoDefCompiler { this.readCompiler = new ReadCompiler() this.writeCompiler = new WriteCompiler() this.sizeOfCompiler = new SizeOfCompiler() + this.writeCompiler.sizeOfCompiler = this.sizeOfCompiler } addTypes (types) { @@ -62,6 +63,8 @@ class CompiledProtodef { this.sizeOfCtx = sizeOfCtx this.writeCtx = writeCtx this.readCtx = readCtx + // Code from callTypeSize runs against the sizeOf context + writeCtx.sizeOfCtx = sizeOfCtx } read (buffer, cursor, type) { @@ -361,11 +364,26 @@ class WriteCompiler extends Compiler { if (args.length > 0) return '(' + code + `)(${value}, buffer, ${offsetExpr}, ` + args.map(name => this.getField(name)).join(', ') + ')' return '(' + code + `)(${value}, buffer, ${offsetExpr})` } + + /** + * Code computing the size of `value` as `type`, for writers that need to + * serialize part of a value before they can write it. The sizer is a + * function in the sizeOf context, which is generated first, so `type` has + * to be a named one for there to be a function to call. + */ + callTypeSize (value, type) { + if (!this.sizeOfCompiler) throw new Error('sizeOfCtx is only available when compiling with ProtoDefCompiler') + if (typeof type !== 'string' || !this.sizeOfCompiler.types[type]) { + throw new Error('cannot size ' + JSON.stringify(type) + ' from a writer, it is not a named type') + } + return `ctx.sizeOfCtx.${type}(${value})` + } } class SizeOfCompiler extends Compiler { constructor () { super() + this.constants = {} this.addTypes(conditionalDatatypes.SizeOf) this.addTypes(structuresDatatypes.SizeOf) @@ -390,12 +408,22 @@ class SizeOfCompiler extends Compiler { this.primitiveTypes[type] = `native.${type}` if (!isNaN(fn)) { this.native[type] = (value) => { return fn } + this.constants[type] = fn } else { this.native[type] = fn } this.types[type] = 'native' } + /** + * The size of `type` when it doesn't depend on the value, following + * aliases down to a fixed-size native; undefined otherwise + */ + constantSize (type) { + while (typeof type === 'string' && typeof this.types[type] === 'string' && this.types[type] !== 'native') type = this.types[type] + return this.constants[type] + } + compileType (type) { if (type instanceof Array) { if (this.parameterizableTypes[type[0]]) { return this.parameterizableTypes[type[0]](this, type[1]) } diff --git a/src/datatypes/compiler-utils.js b/src/datatypes/compiler-utils.js index d60eda5..0b28e2d 100644 --- a/src/datatypes/compiler-utils.js +++ b/src/datatypes/compiler-utils.js @@ -1,3 +1,5 @@ +const { algorithms: hashAlgorithms } = require('./hash') + module.exports = { Read: { pstring: ['parametrizable', (compiler, string) => { @@ -83,10 +85,17 @@ return { value, size } let code = 'const { value, size } = ' + compiler.callType(mapper.type) + '\n' code += 'return { value: ' + JSON.stringify(sanitizeMappings(mapper.mappings)) + '[value] || value, size }' return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { type }) => { + return compiler.wrapCode('return ' + compiler.callType(type)) }] }, Write: { + // A hash digest is taken in generated code, so the digest function is + // copied into the compiled context rather than reached for outside it. + // Underscored: the context is shared with the protocol's own type names. + _crc32c: ['context', hashAlgorithms.crc32c.digest], pstring: ['parametrizable', (compiler, string) => { let code = `const length = Buffer.byteLength(value, "${string.encoding || 'utf8'}")\n` if (string.countType) { @@ -163,6 +172,20 @@ return (ctx.${type})(val, buffer, offset) code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + hash: ['parametrizable', (compiler, { alg, type, body }) => { + if (!hashAlgorithms[alg]) throw new Error('Unknown hash algorithm: ' + alg) + let code = `const bodyBuffer = Buffer.alloc(${compiler.callTypeSize('value', body)})\n` + code += `;((buffer) => ${compiler.callType('value', body, '0')})(bodyBuffer)\n` + code += `const hash = ctx._${alg}(bodyBuffer)\n` + // A CRC is unsigned; a signed `type` takes its two's complement + code += 'try {\n' + code += ' return ' + compiler.callType('hash', type) + '\n' + code += '} catch (e) {\n' + code += ' if (!(e instanceof RangeError)) throw e\n' + code += ' return ' + compiler.callType('hash | 0', type) + '\n' + code += '}' + return compiler.wrapCode(code) }] }, @@ -217,6 +240,14 @@ return (ctx.${type})(val) code += 'if (mapped === undefined) throw new Error(value + \' is not in the mappings value\')\n' code += 'return ' + compiler.callType('mapped', mapper.type) return compiler.wrapCode(code) + }], + // The digest has a fixed width, so a hash is sized without hashing: its + // size is the size of `type`, which the spec requires to be constant + hash: ['parametrizable', (compiler, { alg, type }) => { + const size = compiler.constantSize(type) + if (size === undefined) throw new Error('hash type must be of constant size, ' + JSON.stringify(type) + ' is not') + if (size < hashAlgorithms[alg].bytes) throw new Error('hash type is too small for a ' + alg + ' digest') + return String(size) }] } } diff --git a/src/datatypes/hash.js b/src/datatypes/hash.js new file mode 100644 index 0000000..89622e3 --- /dev/null +++ b/src/datatypes/hash.js @@ -0,0 +1,66 @@ +const { getFieldInfo } = require('../utils') + +// CRC-32C (Castagnoli): reflected table-driven, all-ones init and final xor. +// The compiler copies this function into the code it generates, so it has to +// stand on its own: no imports, no module scope, its table cached on itself. +function crc32c (buffer) { + let table = crc32c.table + if (!table) { + table = crc32c.table = new Int32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0x82F63B78 ^ (c >>> 1) : c >>> 1 + table[n] = c + } + } + let c = -1 + for (let i = 0; i < buffer.length; i++) c = table[(c ^ buffer[i]) & 0xff] ^ (c >>> 8) + return (c ^ -1) >>> 0 +} + +// `alg` is an explicit list in the spec, so a protocol means the same thing in +// every implementation; adding an algorithm here is a spec change. The width of +// the digest is what lets a hash be sized without hashing. +const algorithms = { + crc32c: { bytes: 4, digest: crc32c } +} + +function digest (alg, buffer) { + const algorithm = algorithms[alg] + if (!algorithm) throw new Error('Unknown hash algorithm: ' + alg) + return algorithm.digest(buffer) +} + +function readHash (buffer, offset, { type }, rootNode) { + return this.read(buffer, offset, type, rootNode) +} + +// A CRC is unsigned; a signed `type` takes its two's complement. +function writeHash (value, buffer, offset, { alg, type, body }, rootNode) { + if (typeof body !== 'string') throw new Error('hash body must be a named type, ' + JSON.stringify(body) + ' is not') + const bodyBuffer = Buffer.alloc(this.sizeOf(value, body, rootNode)) + this.write(value, bodyBuffer, 0, body, rootNode) + const hash = digest(alg, bodyBuffer) + try { + return this.write(hash, buffer, offset, type, rootNode) + } catch (e) { + if (!(e instanceof RangeError)) throw e + return this.write(hash | 0, buffer, offset, type, rootNode) + } +} + +// The digest has a fixed width, so the size of a hash never depends on the +// value: `type` is required to be of constant size and is looked up as one. +function sizeOfHash (value, { alg, type }, rootNode) { + const functions = this.types[getFieldInfo(type).type] + const size = functions ? functions[2] : undefined + if (typeof size !== 'number') throw new Error('hash type must be of constant size, ' + JSON.stringify(type) + ' is not') + if (size < algorithms[alg].bytes) throw new Error('hash type is too small for a ' + alg + ' digest') + return size +} + +module.exports = { + digest, + algorithms, + hash: [readHash, writeHash, sizeOfHash, require('../../ProtoDef/schemas/utils.json').hash] +} diff --git a/src/datatypes/utils.js b/src/datatypes/utils.js index 2f1722b..5dead75 100644 --- a/src/datatypes/utils.js +++ b/src/datatypes/utils.js @@ -9,6 +9,7 @@ module.exports = { bitflags: [readBitflags, writeBitflags, sizeOfBitflags, require('../../ProtoDef/schemas/utils.json').bitflags], cstring: [readCString, writeCString, sizeOfCString, require('../../ProtoDef/schemas/utils.json').cstring], mapper: [readMapper, writeMapper, sizeOfMapper, require('../../ProtoDef/schemas/utils.json').mapper], + hash: require('./hash').hash, ...require('./varint') } diff --git a/test/misc.js b/test/misc.js index dab6359..c823bd0 100644 --- a/test/misc.js +++ b/test/misc.js @@ -25,3 +25,81 @@ describe('mapper', () => { }) } }) + +describe('hash', () => { + const { digest } = require('../src/datatypes/hash') + const varintHash = ['hash', { alg: 'crc32c', type: 'varint', body: 'Body' }] + const inlineBody = ['hash', { alg: 'crc32c', type: 'u32', body: ['buffer', { count: 9 }] }] + const types = { + Body: ['buffer', { count: 9 }], + crc32c: ['hash', { alg: 'crc32c', type: 'u32', body: 'Body' }], + signed: ['hash', { alg: 'crc32c', type: 'HashCode', body: 'Body' }], + HashCode: 'i32', + // A hash over a list of hashes + entry: ['container', [{ name: 'key', type: ['pstring', { countType: 'u8' }] }, { name: 'value', type: 'li32' }]], + list: ['array', { countType: 'u8', type: ['hash', { alg: 'crc32c', type: 'lu32', body: 'entry' }] }], + nested: ['hash', { alg: 'crc32c', type: 'lu32', body: 'list' }] + } + const proto = new ProtoDef() + proto.addTypes(types) + const compiler = new ProtoDefCompiler() + compiler.addTypesToCompile(types) + const compiled = compiler.compileProtoDefSync() + const check = Buffer.from('123456789') + const u32 = n => { const b = Buffer.alloc(4); b.writeUInt32BE(n); return b } + const lu32 = n => { const b = Buffer.alloc(4); b.writeUInt32LE(n); return b } + + it('crc32c matches its check value', () => { + assert.strictEqual(digest('crc32c', check), 0xE3069283) + }) + + it('rejects an algorithm the spec does not define', () => { + assert.throws(() => digest('sha256', check), /Unknown hash algorithm/) + const log = console.log // the validator dumps the type it rejected + console.log = () => {} + try { + assert.throws(() => new ProtoDef().addTypes({ bad: ['hash', { alg: 'sha256', type: 'u32', body: 'u8' }] })) + } finally { + console.log = log + } + }) + + it('rejects a body that is not a named type', () => { + assert.throws(() => proto.write(check, Buffer.alloc(4), 0, inlineBody), /named type/) + const c = new ProtoDefCompiler() + c.addTypesToCompile({ withInlineBody: inlineBody }) + assert.throws(() => c.compileProtoDefSync(), /named type/) + }) + + it('rejects a hash written as a variable-size type', () => { + assert.throws(() => proto.sizeOf(check, varintHash), /constant size/) + const c = new ProtoDefCompiler() + c.addTypesToCompile({ asVarint: varintHash }) + assert.throws(() => c.compileProtoDefSync(), /constant size/) + }) + + for (const [label, p] of [['interpreted', proto], ['compiled', compiled]]) { + describe(label, () => { + it('writes the hash of the serialized body', () => { + assert.deepStrictEqual(p.createPacketBuffer('crc32c', check), u32(0xE3069283)) + }) + it('reads the hash, not the value', () => { + assert.strictEqual(p.parsePacketBuffer('crc32c', u32(0xE3069283)).data, 0xE3069283) + }) + it('writes a signed type in two\'s complement', () => { + const buffer = p.createPacketBuffer('signed', check) + assert.deepStrictEqual(buffer, u32(0xE3069283)) + assert.strictEqual(p.parsePacketBuffer('signed', buffer).data, 0xE3069283 | 0) + }) + it('sizes without hashing', () => { + assert.strictEqual(p.sizeOf(check, 'signed'), 4) + }) + it('nests hashes of hashes', () => { + const value = [{ key: 'a', value: 1 }, { key: 'b', value: 2 }] + const list = Buffer.concat([Buffer.from([2]), ...value.map(entry => lu32(digest('crc32c', p.createPacketBuffer('entry', entry))))]) + assert.deepStrictEqual(p.createPacketBuffer('list', value), list) + assert.deepStrictEqual(p.createPacketBuffer('nested', value), lu32(digest('crc32c', list))) + }) + }) + } +})