Add missing blockHeaderSchema properties (#6243)

* Add missing blockHeaderSchema properties

* Update CHANGELOGs

* Update BlockHeaderOutput

* Refactor subscript new heads integration test

* Add try/catch to new heads sub test for faster failing

* Init e2e new heads subscription test

* Debug failing tests

* Debugging failing tests for cypress

CHANGELOG.md

@@ -1815,4 +1815,9 @@ If there are any bugs, improvements, optimizations or any new feature proposal f
 
 #### web3-validator
 
+-   Rpc method `getPastLogs` accept blockHash as a parameter https://ethereum.org/en/developers/docs/apis/json-rpc/#eth_getlogs (#6181)
+
+#### web3-eth
+
+-   Missing `blockHeaderSchema` properties causing some properties to not appear in response of `newHeads` subscription (#6243)
 -   Type `RawValidationError` was removed (#6264)

packages/web3-eth/CHANGELOG.md

@@ -162,3 +162,7 @@ Documentation:
 -   Dependencies updated
 
 ## [Unreleased]
+
+### Fixed
+
+-   Missing `blockHeaderSchema` properties causing some properties to not appear in response of `newHeads` subscription (#6243)

packages/web3-eth/package.json

@@ -36,6 +36,8 @@
 		"test": "jest --config=./test/unit/jest.config.js",
 		"test:coverage:unit": "jest --config=./test/unit/jest.config.js --coverage=true --coverage-reporters=text",
 		"test:ci": "jest --coverage=true --coverage-reporters=json --verbose",
+		"test:e2e:mainnet": "jest --config=./test/e2e/jest.config.js --forceExit",
+		"test:e2e:sepolia": "jest --config=./test/e2e/jest.config.js --forceExit",
 		"test:watch": "npm test -- --watch",
 		"test:unit": "jest --config=./test/unit/jest.config.js",
 		"test:integration": "jest --config=./test/integration/jest.config.js --runInBand --forceExit",

packages/web3-eth/src/schemas.ts

@@ -321,13 +321,37 @@ export const blockSchema = {
 	},
 };
 
+export const withdrawalsSchema = {
+	type: 'object',
+	properties: {
+		index: {
+			format: 'uint',
+		},
+		validatorIndex: {
+			format: 'uint',
+		},
+		address: {
+			format: 'bytes32',
+		},
+		amount: {
+			format: 'uint',
+		},
+	},
+};
+
 export const blockHeaderSchema = {
 	type: 'object',
 	properties: {
+		author: {
+			format: 'bytes32',
+		},
+		hash: {
+			format: 'bytes32',
+		},
 		parentHash: {
 			format: 'bytes32',
 		},
-		receiptRoot: {
+		receiptsRoot: {
 			format: 'bytes32',
 		},
 		miner: {
@@ -339,12 +363,18 @@ export const blockHeaderSchema = {
 		transactionsRoot: {
 			format: 'bytes32',
 		},
+		withdrawalsRoot: {
+			format: 'bytes32',
+		},
 		logsBloom: {
 			format: 'bytes256',
 		},
 		difficulty: {
 			format: 'uint',
 		},
+		totalDifficulty: {
+			format: 'uint',
+		},
 		number: {
 			format: 'uint',
 		},
@@ -366,6 +396,36 @@ export const blockHeaderSchema = {
 		sha3Uncles: {
 			format: 'bytes32',
 		},
+		size: {
+			format: 'uint',
+		},
+		baseFeePerGas: {
+			format: 'uint',
+		},
+		excessDataGas: {
+			format: 'uint',
+		},
+		mixHash: {
+			format: 'bytes32',
+		},
+		transactions: {
+			type: 'array',
+			items: {
+				format: 'bytes32',
+			},
+		},
+		uncles: {
+			type: 'array',
+			items: {
+				format: 'bytes32',
+			},
+		},
+		withdrawals: {
+			type: 'array',
+			items: {
+				...withdrawalsSchema,
+			},
+		},
 	},
 };
 

packages/web3-eth/test/e2e/accounts.json

@@ -0,0 +1 @@
+../../../../scripts/accounts.json

packages/web3-eth/test/e2e/e2e_utils.ts

@@ -0,0 +1,85 @@
+/*
+This file is part of web3.js.
+
+web3.js is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+web3.js is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with web3.js.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/**
+ * @NOTE This Util method is kept seperate from shared system_test_utils.ts file because
+ * of it's import of .secrets.json. For this method to exist in shared system_test_utils.ts
+ * file, the import path would be ../.secrets.json which doesn't resolve when the file is
+ * copied over to each package's test directory. Because web3 package is the only package
+ * running these E2E tests that use Sepolia and Mainnet, this util exists here for now.
+ */
+
+import { getSystemTestBackend } from '../fixtures/system_test_utils';
+// eslint-disable-next-line import/no-relative-packages
+import secrets from '../../../../.secrets.json';
+
+export const getSystemE2ETestProvider = (): string => {
+	if (process.env.WEB3_SYTEM_TEST_MODE === 'http') {
+		return getSystemTestBackend() === 'sepolia'
+			? process.env.INFURA_SEPOLIA_HTTP ?? secrets.SEPOLIA.HTTP
+			: process.env.INFURA_MAINNET_HTTP ?? secrets.MAINNET.HTTP;
+	}
+	return getSystemTestBackend() === 'sepolia'
+		? process.env.INFURA_SEPOLIA_WS ?? secrets.SEPOLIA.WS
+		: process.env.INFURA_MAINNET_WS ?? secrets.MAINNET.WS;
+};
+
+export const getE2ETestAccountAddress = (): string => {
+	if (process.env.TEST_ACCOUNT_ADDRESS !== undefined) {
+		return process.env.TEST_ACCOUNT_ADDRESS;
+		// eslint-disable-next-line no-else-return
+	} else if (getSystemTestBackend() === 'sepolia' || getSystemTestBackend() === 'mainnet') {
+		return secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET'].ACCOUNT
+			.address;
+	}
+
+	throw new Error('Unable to get test account address');
+};
+
+export const getE2ETestContractAddress = () =>
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
+	secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET']
+		.DEPLOYED_TEST_CONTRACT_ADDRESS as string;
+
+export const getAllowedSendTransaction = (): boolean => {
+	if (process.env.ALLOWED_SEND_TRANSACTION !== undefined) {
+		// https://github.com/actions/runner/issues/1483
+		if (process.env.ALLOWED_SEND_TRANSACTION === 'false') {
+			return false;
+		}
+
+		return Boolean(process.env.ALLOWED_SEND_TRANSACTION);
+		// eslint-disable-next-line no-else-return
+	} else if (getSystemTestBackend() === 'sepolia' || getSystemTestBackend() === 'mainnet') {
+		return secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET']
+			.ALLOWED_SEND_TRANSACTION;
+	}
+
+	return false;
+};
+
+export const getE2ETestAccountPrivateKey = (): string => {
+	if (process.env.TEST_ACCOUNT_PRIVATE_KEY !== undefined) {
+		return process.env.TEST_ACCOUNT_PRIVATE_KEY;
+		// eslint-disable-next-line no-else-return
+	} else if (getSystemTestBackend() === 'sepolia' || getSystemTestBackend() === 'mainnet') {
+		return secrets[getSystemTestBackend().toUpperCase() as 'SEPOLIA' | 'MAINNET'].ACCOUNT
+			.privateKey;
+	}
+
+	throw new Error('Unable to get test account private key');
+};

packages/web3-eth/test/e2e/jest.config.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const base = require('../config/jest.config');
+
+module.exports = {
+	...base,
+	setupFilesAfterEnv: ['<rootDir>/test/e2e/setup.js'],
+	testMatch: [
+		`<rootDir>/test/e2e/*.(spec|test).(js|ts)`,
+		`<rootDir>/test/e2e/${process.env.WEB3_SYSTEM_TEST_BACKEND}/**/*.(spec|test).(js|ts)`,
+	],
+	/**
+	 * restoreMocks [boolean]
+	 *
+	 * Default: false
+	 *
+	 * Automatically restore mock state between every test.
+	 * Equivalent to calling jest.restoreAllMocks() between each test.
+	 * This will lead to any mocks having their fake implementations removed
+	 * and restores their initial implementation.
+	 */
+	restoreMocks: true,
+
+	/**
+	 * resetModules [boolean]
+	 *
+	 * Default: false
+	 *
+	 * By default, each test file gets its own independent module registry.
+	 * Enabling resetModules goes a step further and resets the module registry before running each individual test.
+	 * This is useful to isolate modules for every test so that local module state doesn't conflict between tests.
+	 * This can be done programmatically using jest.resetModules().
+	 */
+	resetModules: true,
+	coverageDirectory: `.coverage/e2e/${process.env.WEB3_SYSTEM_TEST_BACKEND}`,
+};

packages/web3-eth/test/e2e/setup.js

@@ -0,0 +1,24 @@
+/*
+This file is part of web3.js.
+
+web3.js is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+web3.js is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with web3.js.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// Have to use `require` because of Jest issue https://jestjs.io/docs/ecmascript-modules
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+require('../config/setup');
+
+const jestTimeout = 30000; // Sometimes `in3` takes long time because of its decentralized nature.
+
+jest.setTimeout(jestTimeout);

packages/web3-eth/test/e2e/subscription_new_heads.test.ts

@@ -0,0 +1,88 @@
+/*
+This file is part of web3.js.
+
+web3.js is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+web3.js is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with web3.js.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// eslint-disable-next-line import/no-extraneous-dependencies
+import Web3, { BlockHeaderOutput } from 'web3';
+
+import {
+	closeOpenConnection,
+	getSystemTestBackend,
+	itIf,
+	waitForOpenConnection,
+} from '../fixtures/system_test_utils';
+import { getSystemE2ETestProvider } from './e2e_utils';
+
+describe(`${getSystemTestBackend()} tests - subscription newHeads`, () => {
+	const provider = getSystemE2ETestProvider();
+	const expectedNumberOfNewHeads = 1;
+
+	let web3: Web3;
+
+	beforeAll(() => {
+		web3 = new Web3(provider);
+	});
+
+	afterAll(async () => {
+		await closeOpenConnection(web3);
+	});
+
+	itIf(provider.startsWith('ws'))(
+		`should subscribe to newHeads and receive ${expectedNumberOfNewHeads}`,
+		async () => {
+			const newHeadsSubscription = await web3.eth.subscribe('newHeads');
+
+			let numberOfNewHeadsReceived = 0;
+
+			await waitForOpenConnection(web3.eth);
+			const assertionPromise = new Promise((resolve, reject) => {
+				newHeadsSubscription.on('data', (data: BlockHeaderOutput) => {
+					try {
+						expect(data).toMatchObject<BlockHeaderOutput>({
+							hash: expect.any(String),
+							parentHash: expect.any(String),
+							receiptsRoot: expect.any(String),
+							miner: expect.any(String),
+							stateRoot: expect.any(String),
+							transactionsRoot: expect.any(String),
+							logsBloom: expect.any(String),
+							difficulty: expect.any(BigInt),
+							number: expect.any(BigInt),
+							gasLimit: expect.any(BigInt),
+							gasUsed: expect.any(BigInt),
+							timestamp: expect.any(BigInt),
+							extraData: expect.any(String),
+							nonce: expect.any(BigInt),
+							sha3Uncles: expect.any(String),
+							baseFeePerGas: expect.any(BigInt),
+							mixHash: expect.any(String),
+							withdrawalsRoot: expect.any(String),
+						});
+					} catch (error) {
+						reject(error);
+					}
+
+					numberOfNewHeadsReceived += 1;
+					if (numberOfNewHeadsReceived === expectedNumberOfNewHeads) resolve(undefined);
+				});
+
+				newHeadsSubscription.on('error', error => reject(error));
+			});
+
+			await assertionPromise;
+			await web3.eth.subscriptionManager?.removeSubscription(newHeadsSubscription);
+		},
+	);
+});

packages/web3-eth/test/integration/subscription_heads.test.ts

@@ -43,7 +43,27 @@ describeIf(isSocket)('subscription', () => {
 			let times = 0;
 			const pr = new Promise((resolve: Resolve, reject) => {
 				sub.on('data', (data: BlockHeaderOutput) => {
-					expect(typeof data.parentHash).toBe('string');
+					try {
+						expect(typeof data.hash).toBe('string');
+						expect(typeof data.parentHash).toBe('string');
+						expect(typeof data.receiptsRoot).toBe('string');
+						expect(typeof data.miner).toBe('string');
+						expect(typeof data.stateRoot).toBe('string');
+						expect(typeof data.transactionsRoot).toBe('string');
+						expect(typeof data.logsBloom).toBe('string');
+						expect(typeof data.difficulty).toBe('bigint');
+						expect(typeof data.number).toBe('bigint');
+						expect(typeof data.gasLimit).toBe('bigint');
+						expect(typeof data.gasUsed).toBe('bigint');
+						expect(typeof data.timestamp).toBe('bigint');
+						expect(typeof data.extraData).toBe('string');
+						expect(typeof data.nonce).toBe('bigint');
+						expect(typeof data.sha3Uncles).toBe('string');
+						expect(typeof data.baseFeePerGas).toBe('bigint');
+						expect(typeof data.mixHash).toBe('string');
+					} catch (error) {
+						reject(error);
+					}
 
 					times += 1;
 					expect(times).toBeGreaterThanOrEqual(times);