refactor: use buffer-layout to clean up buffer encoding

web3.js/flow-typed/buffer-layout.js

@@ -0,0 +1,4 @@
+declare module 'buffer-layout' {
+  // TODO: Fill in types
+  declare module.exports: any;
+}

web3.js/package-lock.json

@@ -2439,6 +2439,11 @@
       "integrity": "sha1-qfuAbOgUXVQoUQznLyeLs2OmOL8=",
       "dev": true
     },
+    "buffer-layout": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/buffer-layout/-/buffer-layout-1.2.0.tgz",
+      "integrity": "sha512-iiyRoho/ERzBUv6kFvfsrLNgTlVwOkqQcSQN7WrO3Y+c5SeuEhCn6+y1KwhM0V3ndptF5mI/RI44zkw0qcR5Jg=="
+    },
     "buffer-shims": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz",

web3.js/package.json

@@ -53,6 +53,7 @@
     "babel-runtime": "^6.26.0",
     "bn.js": "^4.11.8",
     "bs58": "^4.0.1",
+    "buffer-layout": "^1.2.0",
     "jayson": "^2.0.6",
     "node-fetch": "^2.2.0",
     "superstruct": "^0.6.0",

web3.js/src/budget-program.js

@@ -1,7 +1,10 @@
 // @flow
 
+import * as BufferLayout from 'buffer-layout';
+
 import {Transaction} from './transaction';
 import {PublicKey} from './publickey';
+import * as Layout from './layout';
 
 /**
  * Represents a condition that is met by executing a `applySignature()`
@@ -108,11 +111,20 @@ function serializeCondition(condition: BudgetCondition) {
   }
   case 'signature':
   {
-    const from = condition.from.toBuffer();
+    const userdataLayout = BufferLayout.struct([
+      BufferLayout.u32('condition'),
+      Layout.publicKey('from'),
+    ]);
 
+    const from = condition.from.toBuffer();
     const userdata = Buffer.alloc(4 + from.length);
-    userdata.writeUInt32LE(1, 0); // Condition enum = Signature
+    userdataLayout.encode(
-    from.copy(userdata, 4);
+      {
+        instruction: 1, // Signature
+        from
+      },
+      userdata,
+    );
     return userdata;
   }
   default:
@@ -310,8 +322,17 @@ export class BudgetProgram {
    * pending payment to proceed.
    */
   static applySignature(from: PublicKey, program: PublicKey, to: PublicKey): Transaction {
-    const userdata = Buffer.alloc(4);
+    const userdataLayout = BufferLayout.struct([
-    userdata.writeUInt32LE(2, 0); // ApplySignature instruction
+      BufferLayout.u32('instruction'),
+    ]);
+
+    const userdata = Buffer.alloc(userdataLayout.span);
+    userdataLayout.encode(
+      {
+        instruction: 2, // ApplySignature instruction
+      },
+      userdata,
+    );
 
     return new Transaction({
       fee: 0,

web3.js/src/layout.js

@@ -0,0 +1,18 @@
+// @flow
+
+import * as BufferLayout from 'buffer-layout';
+
+
+/**
+ * Layout for a public key
+ */
+export const publicKey = (property: string = 'publicKey'): Object => {
+  return BufferLayout.blob(32, property);
+};
+
+/**
+ * Layout for a 256bit unsigned value
+ */
+export const uint256 = (property: string = 'uint256'): Object => {
+  return BufferLayout.blob(32, property);
+};

web3.js/src/system-program.js

@@ -1,9 +1,10 @@
 // @flow
 
-import assert from 'assert';
+import * as BufferLayout from 'buffer-layout';
 
 import {Transaction} from './transaction';
 import {PublicKey} from './publickey';
+import * as Layout from './layout';
 
 /**
  * Factory class for transactions to interact with the System program
@@ -26,23 +27,24 @@ export class SystemProgram {
     space: number,
     programId: PublicKey
   ): Transaction {
-    const userdata = Buffer.alloc(4 + 8 + 8 + 1 + 32);
-    let pos = 0;
 
-    userdata.writeUInt32LE(0, pos); // Create Account instruction
+    const userdataLayout = BufferLayout.struct([
-    pos += 4;
+      BufferLayout.u32('instruction'),
+      BufferLayout.ns64('tokens'),
+      BufferLayout.ns64('space'),
+      Layout.publicKey('programId'),
+    ]);
 
-    userdata.writeUInt32LE(tokens, pos); // tokens as i64
+    const userdata = Buffer.alloc(userdataLayout.span);
-    pos += 8;
+    userdataLayout.encode(
-
+      {
-    userdata.writeUInt32LE(space, pos); // space as u64
+        instruction: 0, // Create Account instruction
-    pos += 8;
+        tokens,
-
+        space,
-    const programIdBytes = programId.toBuffer();
+        programId: programId.toBuffer(),
-    programIdBytes.copy(userdata, pos);
+      },
-    pos += programIdBytes.length;
+      userdata,
-
+    );
-    assert(pos <= userdata.length);
 
     return new Transaction({
       fee: 0,
@@ -56,15 +58,19 @@ export class SystemProgram {
    * Generate a Transaction that moves tokens from one account to another
    */
   static move(from: PublicKey, to: PublicKey, amount: number): Transaction {
-    const userdata = Buffer.alloc(4 + 8);
+    const userdataLayout = BufferLayout.struct([
-    let pos = 0;
+      BufferLayout.u32('instruction'),
-    userdata.writeUInt32LE(2, pos); // Move instruction
+      BufferLayout.ns64('amount'),
-    pos += 4;
+    ]);
 
-    userdata.writeUInt32LE(amount, pos); // amount as u64
+    const userdata = Buffer.alloc(userdataLayout.span);
-    pos += 8;
+    userdataLayout.encode(
-
+      {
-    assert(pos === userdata.length);
+        instruction: 2, // Move instruction
+        amount,
+      },
+      userdata,
+    );
 
     return new Transaction({
       fee: 0,
@@ -78,17 +84,19 @@ export class SystemProgram {
    * Generate a Transaction that assigns an account to a program
    */
   static assign(from: PublicKey, programId: PublicKey): Transaction {
-    const userdata = Buffer.alloc(4 + 32);
+    const userdataLayout = BufferLayout.struct([
-    let pos = 0;
+      BufferLayout.u32('instruction'),
+      Layout.publicKey('programId'),
+    ]);
 
-    userdata.writeUInt32LE(1, pos); // Assign instruction
+    const userdata = Buffer.alloc(userdataLayout.span);
-    pos += 4;
+    userdataLayout.encode(
-
+      {
-    const programIdBytes = programId.toBuffer();
+        instruction: 1, // Assign instruction
-    programIdBytes.copy(userdata, pos);
+        programId: programId.toBuffer(),
-    pos += programIdBytes.length;
+      },
-
+      userdata,
-    assert(pos === userdata.length);
+    );
 
     return new Transaction({
       fee: 0,

web3.js/src/transaction.js

@@ -1,9 +1,11 @@
 // @flow
 
 import assert from 'assert';
+import * as BufferLayout from 'buffer-layout';
 import nacl from 'tweetnacl';
 import bs58 from 'bs58';
 
+import * as Layout from './layout';
 import type {Account} from './account';
 import type {PublicKey} from './publickey';
 
@@ -54,7 +56,7 @@ export class Transaction {
   /**
    * Program Id to execute
    */
-  programId: ?PublicKey;
+  programId: PublicKey;
 
   /**
    * A recent transaction id.  Must be populated by the caller
@@ -79,53 +81,46 @@ export class Transaction {
    * @private
    */
   _getSignData(): Buffer {
-    const {lastId} = this;
+    const {lastId, keys, programId, userdata} = this;
     if (!lastId) {
       throw new Error('Transaction lastId required');
     }
 
-    // Start with a Buffer that should be large enough to fit any Transaction
+    const signDataLayout = BufferLayout.struct([
-    const transactionData = Buffer.alloc(2048);
+      BufferLayout.ns64('keysLength'),
+      BufferLayout.seq(
+        Layout.publicKey('key'),
+        keys.length,
+        'keys'
+      ),
+      Layout.publicKey('programId'),
+      Layout.publicKey('lastId'),
+      BufferLayout.ns64('fee'),
+      BufferLayout.ns64('userdataLength'),
+      BufferLayout.blob(userdata.length, 'userdata'),
+    ]);
 
-    let pos = 0;
+    let signData = Buffer.alloc(2048);
-
+    let length = signDataLayout.encode(
-    // serialize `this.keys`
-    transactionData.writeUInt32LE(this.keys.length, pos);  // u64
-    pos += 8;
-    for (let key of this.keys) {
-      const keyBytes = key.toBuffer();
-      keyBytes.copy(transactionData, pos);
-      pos += 32;
-    }
-
-    // serialize `this.programId`
-    if (this.programId) {
-      const keyBytes = this.programId.toBuffer();
-      keyBytes.copy(transactionData, pos);
-    }
-    pos += 32;
-
-    // serialize `this.lastId`
       {
-      const lastIdBytes = Buffer.from(bs58.decode(lastId));
+        keysLength: keys.length,
-      assert(lastIdBytes.length === 32);
+        keys: keys.map((key) => key.toBuffer()),
-      lastIdBytes.copy(transactionData, pos);
+        programId: programId.toBuffer(),
-      pos += 32;
+        lastId: Buffer.from(bs58.decode(lastId)),
+        fee: 0,
+        userdataLength: userdata.length,
+        userdata,
+      },
+      signData
+    );
+
+    if (userdata.length === 0) {
+      // If userdata is empty, strip the 64bit 'userdataLength' field from
+      // the end of signData
+      length -= 8;
     }
-
+    signData = signData.slice(0, length);
-    // serialize `this.fee`
+    return signData;
-    transactionData.writeUInt32LE(this.fee, pos);        // u64
-    pos += 8;
-
-    // serialize `this.userdata`
-    if (this.userdata.length > 0) {
-      transactionData.writeUInt32LE(this.userdata.length, pos);  // u64
-      pos += 8;
-      this.userdata.copy(transactionData, pos);
-      pos += this.userdata.length;
-    }
-
-    return transactionData.slice(0, pos);
   }
 
   /**
@@ -134,8 +129,8 @@ export class Transaction {
    * The Transaction must be assigned a valid `lastId` before invoking this method
    */
   sign(from: Account) {
-    const transactionData = this._getSignData();
+    const signData = this._getSignData();
-    this.signature = nacl.sign.detached(transactionData, from.secretKey);
+    this.signature = nacl.sign.detached(signData, from.secretKey);
     assert(this.signature.length === 64);
   }
 
@@ -150,13 +145,13 @@ export class Transaction {
       throw new Error('Transaction has not been signed');
     }
 
-    const transactionData = this._getSignData();
+    const signData = this._getSignData();
     const wireTransaction = Buffer.alloc(
-      signature.length + transactionData.length
+      signature.length + signData.length
     );
 
     Buffer.from(signature).copy(wireTransaction, 0);
-    transactionData.copy(wireTransaction, signature.length);
+    signData.copy(wireTransaction, signature.length);
     return wireTransaction;
   }
 }