From 7d0a91d7011d5fd08f4d72193bd799fec2e30f2f Mon Sep 17 00:00:00 2001 From: Wyatt Barnes Date: Tue, 18 Jul 2023 15:31:32 -1000 Subject: [PATCH] Add missing `blockHeaderSchema` properties (#6243) * Add missing blockHeaderSchema properties * Update CHANGELOGs * Update BlockHeaderOutput * Refactor subscript new heads integration test * Add try/catch to new heads sub test for faster failing * Init e2e new heads subscription test * Debug failing tests * Debugging failing tests for cypress --- CHANGELOG.md | 5 ++ packages/web3-eth/CHANGELOG.md | 4 + packages/web3-eth/package.json | 2 + packages/web3-eth/src/schemas.ts | 62 ++++++++++++- packages/web3-eth/test/e2e/accounts.json | 1 + packages/web3-eth/test/e2e/e2e_utils.ts | 85 ++++++++++++++++++ packages/web3-eth/test/e2e/jest.config.js | 36 ++++++++ packages/web3-eth/test/e2e/setup.js | 24 +++++ .../test/e2e/subscription_new_heads.test.ts | 88 +++++++++++++++++++ .../integration/subscription_heads.test.ts | 22 ++++- packages/web3-types/src/eth_types.ts | 36 ++++++-- 11 files changed, 357 insertions(+), 8 deletions(-) create mode 120000 packages/web3-eth/test/e2e/accounts.json create mode 100644 packages/web3-eth/test/e2e/e2e_utils.ts create mode 100644 packages/web3-eth/test/e2e/jest.config.js create mode 100644 packages/web3-eth/test/e2e/setup.js create mode 100644 packages/web3-eth/test/e2e/subscription_new_heads.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d793b37b407..956f5c62f64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1815,4 +1815,9 @@ If there are any bugs, improvements, optimizations or any new feature proposal f #### web3-validator +- Rpc method `getPastLogs` accept blockHash as a parameter https://ethereum.org/en/developers/docs/apis/json-rpc/#eth_getlogs (#6181) + +#### web3-eth + +- Missing `blockHeaderSchema` properties causing some properties to not appear in response of `newHeads` subscription (#6243) - Type `RawValidationError` was removed (#6264) diff --git a/packages/web3-eth/CHANGELOG.md b/packages/web3-eth/CHANGELOG.md index 56ef46cf30e..37151b39dd6 100644 --- a/packages/web3-eth/CHANGELOG.md +++ b/packages/web3-eth/CHANGELOG.md @@ -162,3 +162,7 @@ Documentation: - Dependencies updated ## [Unreleased] + +### Fixed + +- Missing `blockHeaderSchema` properties causing some properties to not appear in response of `newHeads` subscription (#6243) diff --git a/packages/web3-eth/package.json b/packages/web3-eth/package.json index 97fa69ac0ac..4475b2acaa3 100644 --- a/packages/web3-eth/package.json +++ b/packages/web3-eth/package.json @@ -36,6 +36,8 @@ "test": "jest --config=./test/unit/jest.config.js", "test:coverage:unit": "jest --config=./test/unit/jest.config.js --coverage=true --coverage-reporters=text", "test:ci": "jest --coverage=true --coverage-reporters=json --verbose", + "test:e2e:mainnet": "jest --config=./test/e2e/jest.config.js --forceExit", + "test:e2e:sepolia": "jest --config=./test/e2e/jest.config.js --forceExit", "test:watch": "npm test -- --watch", "test:unit": "jest --config=./test/unit/jest.config.js", "test:integration": "jest --config=./test/integration/jest.config.js --runInBand --forceExit", diff --git a/packages/web3-eth/src/schemas.ts b/packages/web3-eth/src/schemas.ts index f5b200204db..3a25068f049 100644 --- a/packages/web3-eth/src/schemas.ts +++ b/packages/web3-eth/src/schemas.ts @@ -321,13 +321,37 @@ export const blockSchema = { }, }; +export const withdrawalsSchema = { + type: 'object', + properties: { + index: { + format: 'uint', + }, + validatorIndex: { + format: 'uint', + }, + address: { + format: 'bytes32', + }, + amount: { + format: 'uint', + }, + }, +}; + export const blockHeaderSchema = { type: 'object', properties: { + author: { + format: 'bytes32', + }, + hash: { + format: 'bytes32', + }, parentHash: { format: 'bytes32', }, - receiptRoot: { + receiptsRoot: { format: 'bytes32', }, miner: { @@ -339,12 +363,18 @@ export const blockHeaderSchema = { transactionsRoot: { format: 'bytes32', }, + withdrawalsRoot: { + format: 'bytes32', + }, logsBloom: { format: 'bytes256', }, difficulty: { format: 'uint', }, + totalDifficulty: { + format: 'uint', + }, number: { format: 'uint', }, @@ -366,6 +396,36 @@ export const blockHeaderSchema = { sha3Uncles: { format: 'bytes32', }, + size: { + format: 'uint', + }, + baseFeePerGas: { + format: 'uint', + }, + excessDataGas: { + format: 'uint', + }, + mixHash: { + format: 'bytes32', + }, + transactions: { + type: 'array', + items: { + format: 'bytes32', + }, + }, + uncles: { + type: 'array', + items: { + format: 'bytes32', + }, + }, + withdrawals: { + type: 'array', + items: { + ...withdrawalsSchema, + }, + }, }, }; diff --git a/packages/web3-eth/test/e2e/accounts.json b/packages/web3-eth/test/e2e/accounts.json new file mode 120000 index 00000000000..7dbcddb60a2 --- /dev/null +++ b/packages/web3-eth/test/e2e/accounts.json @@ -0,0 +1 @@ +../../../../scripts/accounts.json \ No newline at end of file diff --git a/packages/web3-eth/test/e2e/e2e_utils.ts b/packages/web3-eth/test/e2e/e2e_utils.ts new file mode 100644 index 00000000000..6cc6d49e60e --- /dev/null +++ b/packages/web3-eth/test/e2e/e2e_utils.ts @@ -0,0 +1,85 @@ +/* +This file is part of web3.js. + +web3.js is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +web3.js is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with web3.js. If not, see . +*/ + +/** + * @NOTE This Util method is kept seperate from shared system_test_utils.ts file because + * of it's import of .secrets.json. For this method to exist in shared system_test_utils.ts + * file, the import path would be ../.secrets.json which doesn't resolve when the file is + * copied over to each package's test directory. Because web3 package is the only package + * running these E2E tests that use Sepolia and Mainnet, this util exists here for now. + */ + +import { getSystemTestBackend } from '../fixtures/system_test_utils'; +// eslint-disable-next-line import/no-relative-packages +import secrets from '../../../../.secrets.json'; + +export const getSystemE2ETestProvider = (): string => { + if (process.env.WEB3_SYTEM_TEST_MODE === 'http') { + return getSystemTestBackend() === 'sepolia' + ? process.env.INFURA_SEPOLIA_HTTP ?? secrets.SEPOLIA.HTTP + : process.env.INFURA_MAINNET_HTTP ?? secrets.MAINNET.HTTP; + } + return getSystemTestBackend() === 'sepolia' + ? process.env.INFURA_SEPOLIA_WS ?? secrets.SEPOLIA.WS + : process.env.INFURA_MAINNET_WS ?? secrets.MAINNET.WS; +}; + +export const getE2ETestAccountAddress = (): string => { + if (process.env.TEST_ACCOUNT_ADDRESS !== undefined) { + return process.env.TEST_ACCOUNT_ADDRESS; + // eslint-disable-next-line no-else-return + } else if (getSystemTestBackend() === 'sepolia' || getSystemTestBackend() === 'mainnet') { + return secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET'].ACCOUNT + .address; + } + + throw new Error('Unable to get test account address'); +}; + +export const getE2ETestContractAddress = () => + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET'] + .DEPLOYED_TEST_CONTRACT_ADDRESS as string; + +export const getAllowedSendTransaction = (): boolean => { + if (process.env.ALLOWED_SEND_TRANSACTION !== undefined) { + // https://github.com/actions/runner/issues/1483 + if (process.env.ALLOWED_SEND_TRANSACTION === 'false') { + return false; + } + + return Boolean(process.env.ALLOWED_SEND_TRANSACTION); + // eslint-disable-next-line no-else-return + } else if (getSystemTestBackend() === 'sepolia' || getSystemTestBackend() === 'mainnet') { + return secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET'] + .ALLOWED_SEND_TRANSACTION; + } + + return false; +}; + +export const getE2ETestAccountPrivateKey = (): string => { + if (process.env.TEST_ACCOUNT_PRIVATE_KEY !== undefined) { + return process.env.TEST_ACCOUNT_PRIVATE_KEY; + // eslint-disable-next-line no-else-return + } else if (getSystemTestBackend() === 'sepolia' || getSystemTestBackend() === 'mainnet') { + return secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET'].ACCOUNT + .privateKey; + } + + throw new Error('Unable to get test account private key'); +}; diff --git a/packages/web3-eth/test/e2e/jest.config.js b/packages/web3-eth/test/e2e/jest.config.js new file mode 100644 index 00000000000..cc3c69f72ff --- /dev/null +++ b/packages/web3-eth/test/e2e/jest.config.js @@ -0,0 +1,36 @@ +'use strict'; + +const base = require('../config/jest.config'); + +module.exports = { + ...base, + setupFilesAfterEnv: ['/test/e2e/setup.js'], + testMatch: [ + `/test/e2e/*.(spec|test).(js|ts)`, + `/test/e2e/${process.env.WEB3_SYSTEM_TEST_BACKEND}/**/*.(spec|test).(js|ts)`, + ], + /** + * restoreMocks [boolean] + * + * Default: false + * + * Automatically restore mock state between every test. + * Equivalent to calling jest.restoreAllMocks() between each test. + * This will lead to any mocks having their fake implementations removed + * and restores their initial implementation. + */ + restoreMocks: true, + + /** + * resetModules [boolean] + * + * Default: false + * + * By default, each test file gets its own independent module registry. + * Enabling resetModules goes a step further and resets the module registry before running each individual test. + * This is useful to isolate modules for every test so that local module state doesn't conflict between tests. + * This can be done programmatically using jest.resetModules(). + */ + resetModules: true, + coverageDirectory: `.coverage/e2e/${process.env.WEB3_SYSTEM_TEST_BACKEND}`, +}; diff --git a/packages/web3-eth/test/e2e/setup.js b/packages/web3-eth/test/e2e/setup.js new file mode 100644 index 00000000000..fddbec59a1e --- /dev/null +++ b/packages/web3-eth/test/e2e/setup.js @@ -0,0 +1,24 @@ +/* +This file is part of web3.js. + +web3.js is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +web3.js is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with web3.js. If not, see . +*/ + +// Have to use `require` because of Jest issue https://jestjs.io/docs/ecmascript-modules +// eslint-disable-next-line @typescript-eslint/no-require-imports +require('../config/setup'); + +const jestTimeout = 30000; // Sometimes `in3` takes long time because of its decentralized nature. + +jest.setTimeout(jestTimeout); diff --git a/packages/web3-eth/test/e2e/subscription_new_heads.test.ts b/packages/web3-eth/test/e2e/subscription_new_heads.test.ts new file mode 100644 index 00000000000..01bb6b5fc4e --- /dev/null +++ b/packages/web3-eth/test/e2e/subscription_new_heads.test.ts @@ -0,0 +1,88 @@ +/* +This file is part of web3.js. + +web3.js is free software: you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +web3.js is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with web3.js. If not, see . +*/ +// eslint-disable-next-line import/no-extraneous-dependencies +import Web3, { BlockHeaderOutput } from 'web3'; + +import { + closeOpenConnection, + getSystemTestBackend, + itIf, + waitForOpenConnection, +} from '../fixtures/system_test_utils'; +import { getSystemE2ETestProvider } from './e2e_utils'; + +describe(`${getSystemTestBackend()} tests - subscription newHeads`, () => { + const provider = getSystemE2ETestProvider(); + const expectedNumberOfNewHeads = 1; + + let web3: Web3; + + beforeAll(() => { + web3 = new Web3(provider); + }); + + afterAll(async () => { + await closeOpenConnection(web3); + }); + + itIf(provider.startsWith('ws'))( + `should subscribe to newHeads and receive ${expectedNumberOfNewHeads}`, + async () => { + const newHeadsSubscription = await web3.eth.subscribe('newHeads'); + + let numberOfNewHeadsReceived = 0; + + await waitForOpenConnection(web3.eth); + const assertionPromise = new Promise((resolve, reject) => { + newHeadsSubscription.on('data', (data: BlockHeaderOutput) => { + try { + expect(data).toMatchObject({ + hash: expect.any(String), + parentHash: expect.any(String), + receiptsRoot: expect.any(String), + miner: expect.any(String), + stateRoot: expect.any(String), + transactionsRoot: expect.any(String), + logsBloom: expect.any(String), + difficulty: expect.any(BigInt), + number: expect.any(BigInt), + gasLimit: expect.any(BigInt), + gasUsed: expect.any(BigInt), + timestamp: expect.any(BigInt), + extraData: expect.any(String), + nonce: expect.any(BigInt), + sha3Uncles: expect.any(String), + baseFeePerGas: expect.any(BigInt), + mixHash: expect.any(String), + withdrawalsRoot: expect.any(String), + }); + } catch (error) { + reject(error); + } + + numberOfNewHeadsReceived += 1; + if (numberOfNewHeadsReceived === expectedNumberOfNewHeads) resolve(undefined); + }); + + newHeadsSubscription.on('error', error => reject(error)); + }); + + await assertionPromise; + await web3.eth.subscriptionManager?.removeSubscription(newHeadsSubscription); + }, + ); +}); diff --git a/packages/web3-eth/test/integration/subscription_heads.test.ts b/packages/web3-eth/test/integration/subscription_heads.test.ts index 44ef600efc1..c801e5e83ef 100644 --- a/packages/web3-eth/test/integration/subscription_heads.test.ts +++ b/packages/web3-eth/test/integration/subscription_heads.test.ts @@ -43,7 +43,27 @@ describeIf(isSocket)('subscription', () => { let times = 0; const pr = new Promise((resolve: Resolve, reject) => { sub.on('data', (data: BlockHeaderOutput) => { - expect(typeof data.parentHash).toBe('string'); + try { + expect(typeof data.hash).toBe('string'); + expect(typeof data.parentHash).toBe('string'); + expect(typeof data.receiptsRoot).toBe('string'); + expect(typeof data.miner).toBe('string'); + expect(typeof data.stateRoot).toBe('string'); + expect(typeof data.transactionsRoot).toBe('string'); + expect(typeof data.logsBloom).toBe('string'); + expect(typeof data.difficulty).toBe('bigint'); + expect(typeof data.number).toBe('bigint'); + expect(typeof data.gasLimit).toBe('bigint'); + expect(typeof data.gasUsed).toBe('bigint'); + expect(typeof data.timestamp).toBe('bigint'); + expect(typeof data.extraData).toBe('string'); + expect(typeof data.nonce).toBe('bigint'); + expect(typeof data.sha3Uncles).toBe('string'); + expect(typeof data.baseFeePerGas).toBe('bigint'); + expect(typeof data.mixHash).toBe('string'); + } catch (error) { + reject(error); + } times += 1; expect(times).toBeGreaterThanOrEqual(times); diff --git a/packages/web3-types/src/eth_types.ts b/packages/web3-types/src/eth_types.ts index f8452e5471f..6a9a4aa8e88 100644 --- a/packages/web3-types/src/eth_types.ts +++ b/packages/web3-types/src/eth_types.ts @@ -144,18 +144,42 @@ export interface BlockOutput { readonly parentHash?: HexString32Bytes; } +export interface Withdrawals { + readonly index: Numbers; + readonly validatorIndex: Numbers; + readonly address: Address; + readonly amount: Numbers; +} + export interface BlockHeaderOutput { + readonly hash?: HexString32Bytes; + readonly parentHash?: HexString32Bytes; + readonly receiptsRoot?: HexString32Bytes; + readonly miner?: HexString; + readonly stateRoot?: HexString32Bytes; + readonly transactionsRoot?: HexString32Bytes; + readonly withdrawalsRoot?: HexString32Bytes; + readonly logsBloom?: Bytes; + readonly difficulty?: Numbers; + readonly number?: Numbers; readonly gasLimit: Numbers; readonly gasUsed: Numbers; readonly timestamp: Numbers; - readonly number?: Numbers; - readonly difficulty?: Numbers; + readonly extraData?: Bytes; + readonly nonce?: Numbers; + readonly sha3Uncles: HexString32Bytes[]; + readonly baseFeePerGas?: Numbers; + + // These fields are returned when the RPC client is Nethermind, + // but aren't available in other clients such as Geth + readonly author?: Address; readonly totalDifficulty?: Numbers; + readonly size?: Numbers; + readonly excessDataGas?: Numbers; + readonly mixHash?: HexString32Bytes; readonly transactions?: TransactionOutput[]; - readonly miner?: HexString; - readonly baseFeePerGas?: Numbers; - readonly parentHash?: HexString32Bytes; - readonly sha3Uncles: HexString32Bytes[]; + readonly uncles?: Uncles; + readonly withdrawals?: Withdrawals[]; } export interface ReceiptInput {