-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: form committees and find majority results (#304)
Group accepted measurements to committees on a per-retrieval-task basis. Evaluate each committee to find an absolute majority result. Reject results that are in minority. When the committee is too small to give us confidence in a majority being honest, or if we cannot find an absolute majority, then reject all measurements in such committee. Signed-off-by: Miroslav Bajtoš <oss@bajtos.net> Co-authored-by: Julian Gruber <julian@juliangruber.com>
- Loading branch information
1 parent
fc7e4c4
commit 4925298
Showing
11 changed files
with
706 additions
and
86 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,214 @@ | ||
| import assert from 'node:assert' | ||
| import createDebug from 'debug' | ||
| import { getTaskId } from './retrieval-stats.js' | ||
|
|
||
| /** @import {Measurement} from './preprocess.js' */ | ||
| /** @import {RetrievalResult, CommitteeCheckError} from './typings.js' */ | ||
|
|
||
| const debug = createDebug('spark:committee') | ||
|
|
||
| /** @typedef {Map<string, Committee>} TaskIdToCommitteeMap */ | ||
|
|
||
| /** @typedef {{ | ||
| indexerResult: string | CommitteeCheckError; | ||
| retrievalResult: RetrievalResult | ||
| }} CommitteeEvaluation | ||
| */ | ||
| export class Committee { | ||
| /** @type {Measurement[]} */ | ||
| #measurements | ||
|
|
||
| /** | ||
| * @param {Pick<Measurement, 'cid' | 'minerId'>} retrievalTask | ||
| */ | ||
| constructor ({ cid, minerId }) { | ||
| this.retrievalTask = { minerId, cid } | ||
|
|
||
| this.#measurements = [] | ||
|
|
||
| /** @type {CommitteeEvaluation | undefined} */ | ||
| this.evaluation = undefined | ||
| } | ||
|
|
||
| get size () { | ||
| return this.#measurements.length | ||
| } | ||
|
|
||
| get measurements () { | ||
| const ret = [...this.#measurements] | ||
| Object.freeze(ret) | ||
| return ret | ||
| } | ||
|
|
||
| /** | ||
| * @param {Measurement} m | ||
| */ | ||
| addMeasurement (m) { | ||
| assert.strictEqual(m.cid, this.retrievalTask.cid, 'cid must match') | ||
| assert.strictEqual(m.minerId, this.retrievalTask.minerId, 'minerId must match') | ||
| assert.strictEqual(m.fraudAssessment, 'OK', 'only accepted measurements can be added') | ||
| this.#measurements.push(m) | ||
| } | ||
|
|
||
| /** | ||
| * @param {object} args | ||
| * @param {number} args.requiredCommitteeSize | ||
| * @returns | ||
| */ | ||
| evaluate ({ requiredCommitteeSize }) { | ||
| debug( | ||
| 'Evaluating task %o with a committee of %s measurements', | ||
| this.retrievalTask, | ||
| this.#measurements.length | ||
| ) | ||
|
|
||
| if (this.#measurements.length < requiredCommitteeSize) { | ||
| debug('→ committee is too small (size=%s required=%s); retrievalResult=COMMITTEE_TOO_SMALL', | ||
| this.#measurements.length, | ||
| requiredCommitteeSize | ||
| ) | ||
| this.evaluation = { | ||
| indexerResult: 'COMMITTEE_TOO_SMALL', | ||
| retrievalResult: 'COMMITTEE_TOO_SMALL' | ||
| } | ||
| for (const m of this.#measurements) m.fraudAssessment = 'COMMITTEE_TOO_SMALL' | ||
| return | ||
| } | ||
|
|
||
| debug('- searching for majority in indexer results') | ||
| /** @type {(keyof Measurement)[]} */ | ||
| const indexerResultProps = [ | ||
| 'providerId', | ||
| 'indexerResult', | ||
| 'provider_address', | ||
| 'protocol' | ||
| ] | ||
| const indexerResultMajority = this.#findMajority(indexerResultProps) | ||
| const indexerResult = indexerResultMajority | ||
| ? indexerResultMajority.majorityValue.indexerResult | ||
| : 'MAJORITY_NOT_FOUND' | ||
|
|
||
| debug('- searching for majority in retrieval results') | ||
| /** @type {(keyof Measurement)[]} */ | ||
| const retrievalResultProps = [ | ||
| ...indexerResultProps, | ||
| // NOTE: We are not checking the fields that were used to calculate | ||
| // the retrievalResult value: | ||
| // - status_code | ||
| // - timeout | ||
| // - car_too_large | ||
| // If there is an agreement on the retrieval result, then those fields | ||
| // must have the same value too. | ||
| 'retrievalResult', | ||
| 'byte_length', | ||
| 'carChecksum' | ||
| ] | ||
|
|
||
| const retrievalResultMajority = this.#findMajority(retrievalResultProps) | ||
| /** @type {CommitteeEvaluation['retrievalResult']} */ | ||
| let retrievalResult | ||
| if (retrievalResultMajority) { | ||
| retrievalResult = retrievalResultMajority.majorityValue.retrievalResult | ||
| for (const m of retrievalResultMajority.minorityMeasurements) { | ||
| m.fraudAssessment = 'MINORITY_RESULT' | ||
| } | ||
| } else { | ||
| retrievalResult = 'MAJORITY_NOT_FOUND' | ||
| for (const m of this.#measurements) m.fraudAssessment = 'MAJORITY_NOT_FOUND' | ||
| } | ||
|
|
||
| this.evaluation = { | ||
| indexerResult, | ||
| retrievalResult | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @param {(keyof Measurement)[]} measurementFields | ||
| */ | ||
| #findMajority (measurementFields) { | ||
| /** @param {Measurement} m */ | ||
| const getResult = m => pick(m, ...measurementFields) | ||
|
|
||
| /** @type {Map<string, Measurement[]>} */ | ||
| const resultGroups = new Map() | ||
|
|
||
| // 1. Group measurements using the result as the grouping key | ||
| for (const m of this.#measurements) { | ||
| const key = JSON.stringify(getResult(m)) | ||
| let list = resultGroups.get(key) | ||
| if (!list) { | ||
| list = [] | ||
| resultGroups.set(key, list) | ||
| } | ||
| list.push(m) | ||
| } | ||
|
|
||
| if (debug.enabled) { | ||
| debug('- results found:') | ||
| for (const k of resultGroups.keys()) { | ||
| debug(' %o', JSON.parse(k)) | ||
| } | ||
| } | ||
|
|
||
| // 2. Sort the measurement groups by their size | ||
| const keys = Array.from(resultGroups.keys()) | ||
| keys.sort( | ||
| (a, b) => (resultGroups.get(a)?.length ?? 0) - (resultGroups.get(b)?.length ?? 0) | ||
| ) | ||
| const measurementGroups = keys.map(k => resultGroups.get(k)) | ||
|
|
||
| // 3. Find the majority | ||
| const majorityMeasurements = measurementGroups.pop() | ||
| const majoritySize = majorityMeasurements.length | ||
| const majorityValue = getResult(majorityMeasurements[0]) | ||
|
|
||
| debug('- majority=%s committee-size=%s value=%o', majoritySize, this.size, majorityValue) | ||
| if (majoritySize <= this.size / 2) { | ||
| debug('→ majority is not absolute; result=MAJORITY_NOT_FOUND') | ||
| return undefined | ||
| } else { | ||
| debug('→ majority agrees on result=%o', majorityValue) | ||
| return { | ||
| majorityValue, | ||
| majorityMeasurements, | ||
| minorityMeasurements: measurementGroups.flat() | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @template T | ||
| * @template {keyof T} K | ||
| * @param {T} obj | ||
| * @param {K[]} keys | ||
| * @returns {Pick<T, K>} | ||
| */ | ||
| function pick (obj, ...keys) { | ||
| /** @type {any} */ | ||
| const ret = {} | ||
| keys.forEach(key => { | ||
| ret[key] = obj[key] | ||
| }) | ||
| return ret | ||
| } | ||
|
|
||
| /** | ||
| * @param {Iterable<Measurement>} measurements | ||
| * @returns {TaskIdToCommitteeMap} | ||
| */ | ||
| export function groupMeasurementsToCommittees (measurements) { | ||
| /** @type {TaskIdToCommitteeMap} */ | ||
| const committeesMap = new Map() | ||
| for (const m of measurements) { | ||
| const key = getTaskId(m) | ||
| let c = committeesMap.get(key) | ||
| if (!c) { | ||
| c = new Committee(m) | ||
| committeesMap.set(key, c) | ||
| } | ||
| c.addMeasurement(m) | ||
| } | ||
| return committeesMap | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.