Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add top earnings participant endpoint #170

Merged
merged 5 commits into from
Jul 4, 2024
Merged

feat: add top earnings participant endpoint #170

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7d1e1b9
feat: add top earnings participant endpoint
PatrickNercessian Jul 2, 2024
633f194
constrant range query to today, endpoint path change
PatrickNercessian Jul 3, 2024
3e5cd6b
Merge branch 'main' into feat-add-top-earnings-endpoints
PatrickNercessian Jul 3, 2024
384ddc1
Merge branch 'main' into feat-add-top-earnings-endpoints
bajtos Jul 4, 2024
9332282
fixup! document why filter.to must be today
bajtos Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions observer/test/observer.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ describe('observer', () => {
beforeEach(async () => {
await pgPools.evaluate.query('DELETE FROM daily_participants')
await pgPools.evaluate.query('DELETE FROM participants')
await pgPools.stats.query('DELETE FROM daily_scheduled_rewards')
juliangruber marked this conversation as resolved.
Show resolved Hide resolved
await givenDailyParticipants(pgPools.evaluate, today(), ['0xCURRENT'])
await givenDailyParticipants(pgPools.evaluate, '2000-01-01', ['0xOLD'])
})
Expand Down
5 changes: 4 additions & 1 deletion stats/lib/platform-routes.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import {
fetchDailyStationCount,
fetchMonthlyStationCount,
fetchDailyRewardTransfers,
fetchDailyStationAcceptedMeasurementCount
fetchDailyStationAcceptedMeasurementCount,
fetchTopEarningParticipants
} from './platform-stats-fetchers.js'

const createRespondWithFetchFn = (pathname, searchParams, res) => (pgPool, fetchFn) => {
Expand Down Expand Up @@ -32,6 +33,8 @@ export const handlePlatformRoutes = async (req, res, pgPools) => {
await respond(pgPools.evaluate, fetchDailyStationAcceptedMeasurementCount)
} else if (req.method === 'GET' && url === '/transfers/daily') {
await respond(pgPools.stats, fetchDailyRewardTransfers)
} else if (req.method === 'GET' && url === '/participants/top-earnings') {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
} else if (req.method === 'GET' && url === '/participants/top-earnings') {
} else if (req.method === 'GET' && url === '/participants/top-earning') {

This route is about the top earning participants, and not the top earnings among participants (subtle but I think important)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please add this route to the README as well

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, addressed in 633f194

await respond(pgPools.stats, fetchTopEarningParticipants)
} else {
return false
}
Expand Down
24 changes: 24 additions & 0 deletions stats/lib/platform-stats-fetchers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,27 @@ export const fetchDailyRewardTransfers = async (pgPool, filter) => {
`, [filter.from, filter.to])
return rows
}

/**
* @param {Queryable} pgPool
* @param {import('./typings.js').DateRangeFilter} filter
*/
export const fetchTopEarningParticipants = async (pgPool, filter) => {
const { rows } = await pgPool.query(`
WITH latest_scheduled_rewards AS (
SELECT DISTINCT ON (participant_address) participant_address, scheduled_rewards
FROM daily_scheduled_rewards
ORDER BY participant_address, day DESC
)
SELECT
COALESCE(drt.to_address, lsr.participant_address) as participant_address,
COALESCE(SUM(drt.amount), 0) + COALESCE(lsr.scheduled_rewards, 0) as total_rewards
bajtos marked this conversation as resolved.
Show resolved Hide resolved
FROM daily_reward_transfers drt
FULL OUTER JOIN latest_scheduled_rewards lsr
ON drt.to_address = lsr.participant_address
WHERE drt.day >= $1 AND drt.day <= $2 OR drt.day IS NULL
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I never remember the precedence of AND & OR operators. Could you please add parenthesis to make it explicit?

For example:

WHERE (drt.day >= $1 AND drt.day <= $2) OR drt.day IS NULL

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure! 633f194

GROUP BY COALESCE(drt.to_address, lsr.participant_address), lsr.scheduled_rewards
ORDER BY total_rewards DESC
bajtos marked this conversation as resolved.
Show resolved Hide resolved
`, [filter.from, filter.to])
return rows
}
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another set of eyes here would be nice for this complex query.

And also if we can run it on production data to see latency expectations.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

spark_stats=# EXPLAIN WITH latest_scheduled_rewards AS (
      SELECT DISTINCT ON (participant_address) participant_address, scheduled_rewards
      FROM daily_scheduled_rewards
      ORDER BY participant_address, day DESC
    )
    SELECT
      COALESCE(drt.to_address, lsr.participant_address) as participant_address,
      COALESCE(SUM(drt.amount), 0) + COALESCE(lsr.scheduled_rewards, 0) as total_rewards
    FROM daily_reward_transfers drt
    FULL OUTER JOIN latest_scheduled_rewards lsr
      ON drt.to_address = lsr.participant_address
    WHERE drt.day >= '2024-06-26' AND drt.day <= '2024-07-03' OR drt.day IS NULL
    GROUP BY COALESCE(drt.to_address, lsr.participant_address), lsr.scheduled_rewards
    ORDER BY total_rewards DESC;
                                                             QUERY PLAN
-------------------------------------------------------------------------------------------------------------------------------------
 Sort  (cost=14296.01..14304.06 rows=3220 width=75)
   Sort Key: ((COALESCE(sum(drt.amount), '0'::numeric) + COALESCE(daily_scheduled_rewards.scheduled_rewards, '0'::numeric))) DESC
   ->  HashAggregate  (cost=14060.10..14108.40 rows=3220 width=75)
         Group Key: COALESCE(drt.to_address, daily_scheduled_rewards.participant_address), daily_scheduled_rewards.scheduled_rewards
         ->  Hash Full Join  (cost=13345.69..14035.95 rows=3220 width=56)
               Hash Cond: (daily_scheduled_rewards.participant_address = drt.to_address)
               Filter: (((drt.day >= '2024-06-26'::date) AND (drt.day <= '2024-07-03'::date)) OR (drt.day IS NULL))
               ->  Unique  (cost=13295.05..13816.97 rows=8980 width=58)
                     ->  Sort  (cost=13295.05..13556.01 rows=104383 width=58)
                           Sort Key: daily_scheduled_rewards.participant_address, daily_scheduled_rewards.day DESC
                           ->  Seq Scan on daily_scheduled_rewards  (cost=0.00..2293.83 rows=104383 width=58)
               ->  Hash  (cost=31.95..31.95 rows=1495 width=60)
                     ->  Seq Scan on daily_reward_transfers drt  (cost=0.00..31.95 rows=1495 width=60)
(13 rows)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the last 7 days, and returning 11k rows, this query took

  Time: 309.534 ms

which is ok for me

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can shave off ~40ms from the query by adding an index ON daily_scheduled_rewards (participant_address, day), but I guess the improvement is small.

I agree that the query duration in the 300-400ms range is acceptable.

EXPLAIN ANALYZE WITH latest_scheduled_rewards AS (
      SELECT DISTINCT ON (participant_address) participant_address, scheduled_rewards
      FROM daily_scheduled_rewards
      ORDER BY participant_address, day DESC
    )
    SELECT
      COALESCE(drt.to_address, lsr.participant_address) as participant_address,
      COALESCE(SUM(drt.amount), 0) + COALESCE(lsr.scheduled_rewards, 0) as total_rewards
    FROM daily_reward_transfers drt
    FULL OUTER JOIN latest_scheduled_rewards lsr
      ON drt.to_address = lsr.participant_address
    WHERE drt.day >= '2024-06-26' AND drt.day <= '2024-07-03' OR drt.day IS NULL
    GROUP BY COALESCE(drt.to_address, lsr.participant_address), lsr.scheduled_rewards
    ORDER BY total_rewards DESC;
                                                                          QUERY PLAN                                                                          
--------------------------------------------------------------------------------------------------------------------------------------------------------------
 Sort  (cost=14693.79..14701.93 rows=3256 width=75) (actual time=380.119..381.786 rows=11118 loops=1)
   Sort Key: ((COALESCE(sum(drt.amount), '0'::numeric) + COALESCE(daily_scheduled_rewards.scheduled_rewards, '0'::numeric))) DESC
   Sort Method: quicksort  Memory: 1948kB
   ->  HashAggregate  (cost=14454.98..14503.82 rows=3256 width=75) (actual time=370.194..376.246 rows=11118 loops=1)
         Group Key: COALESCE(drt.to_address, daily_scheduled_rewards.participant_address), daily_scheduled_rewards.scheduled_rewards
         Batches: 5  Memory Usage: 4273kB  Disk Usage: 248kB
         ->  Hash Full Join  (cost=13724.09..14430.56 rows=3256 width=56) (actual time=309.549..364.272 rows=11118 loops=1)
               Hash Cond: (daily_scheduled_rewards.participant_address = drt.to_address)
               Filter: (((drt.day >= '2024-06-26'::date) AND (drt.day <= '2024-07-03'::date)) OR (drt.day IS NULL))
               Rows Removed by Filter: 959
               ->  Unique  (cost=13673.45..14209.91 rows=9082 width=58) (actual time=308.787..359.491 rows=11644 loops=1)
                     ->  Sort  (cost=13673.45..13941.68 rows=107292 width=58) (actual time=308.783..348.916 rows=107429 loops=1)
                           Sort Key: daily_scheduled_rewards.participant_address, daily_scheduled_rewards.day DESC
                           Sort Method: external merge  Disk: 7368kB
                           ->  Seq Scan on daily_scheduled_rewards  (cost=0.00..2344.92 rows=107292 width=58) (actual time=0.012..23.680 rows=107429 loops=1)
               ->  Hash  (cost=31.95..31.95 rows=1495 width=60) (actual time=0.747..0.749 rows=1495 loops=1)
                     Buckets: 2048  Batches: 1  Memory Usage: 151kB
                     ->  Seq Scan on daily_reward_transfers drt  (cost=0.00..31.95 rows=1495 width=60) (actual time=0.040..0.383 rows=1495 loops=1)
 Planning Time: 0.377 ms
 Execution Time: 384.109 ms
(20 rows)

78 changes: 78 additions & 0 deletions stats/test/platform-routes.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ describe('Platform Routes HTTP request handler', () => {
beforeEach(async () => {
await pgPools.evaluate.query('DELETE FROM daily_stations')
await pgPools.stats.query('DELETE FROM daily_reward_transfers')
await pgPools.stats.query('DELETE FROM daily_scheduled_rewards')
})

describe('GET /stations/daily', () => {
Expand Down Expand Up @@ -185,6 +186,83 @@ describe('Platform Routes HTTP request handler', () => {
])
})
})

describe('GET /participants/top-earnings', () => {
const setupScheduledRewardsData = async () => {
await pgPools.stats.query(`
INSERT INTO daily_scheduled_rewards (day, participant_address, scheduled_rewards)
VALUES
('2024-01-11', 'address1', 10),
('2024-01-11', 'address2', 20),
('2024-01-11', 'address3', 30),
('2024-01-12', 'address1', 15),
('2024-01-12', 'address2', 25),
('2024-01-12', 'address3', 35)
`)
}
it('returns top earning participants for the given date range', async () => {
// Set up test data
await givenDailyRewardTransferMetrics(pgPools.stats, '2024-01-09', [
{ toAddress: 'address1', amount: 100, lastCheckedBlock: 1 },
{ toAddress: 'address2', amount: 100, lastCheckedBlock: 1 },
{ toAddress: 'address3', amount: 100, lastCheckedBlock: 1 }
])

await givenDailyRewardTransferMetrics(pgPools.stats, '2024-01-10', [
{ toAddress: 'address1', amount: 100, lastCheckedBlock: 1 }
])
await givenDailyRewardTransferMetrics(pgPools.stats, '2024-01-11', [
{ toAddress: 'address2', amount: 150, lastCheckedBlock: 1 },
{ toAddress: 'address1', amount: 50, lastCheckedBlock: 1 }
])
await givenDailyRewardTransferMetrics(pgPools.stats, '2024-01-12', [
{ toAddress: 'address3', amount: 200, lastCheckedBlock: 1 },
{ toAddress: 'address2', amount: 100, lastCheckedBlock: 1 }
])

// Set up scheduled rewards data
await setupScheduledRewardsData()

const res = await fetch(
new URL(
'/participants/top-earnings?from=2024-01-10&to=2024-01-12',
baseUrl
), {
redirect: 'manual'
}
)
await assertResponseStatus(res, 200)
const topEarners = await res.json()
assert.deepStrictEqual(topEarners, [
{ participant_address: 'address2', total_rewards: '275' },
{ participant_address: 'address3', total_rewards: '235' },
{ participant_address: 'address1', total_rewards: '165' }
])
})
it('returns top earning participants for the given date range with no existing reward transfers', async () => {
await setupScheduledRewardsData()

await givenDailyRewardTransferMetrics(pgPools.stats, '2024-01-10', [
{ toAddress: 'address1', amount: 100, lastCheckedBlock: 1 }
])

const res = await fetch(
new URL(
'/participants/top-earnings?from=2024-01-10&to=2024-01-12',
baseUrl
), {
redirect: 'manual'
}
)
await assertResponseStatus(res, 200)
const topEarners = await res.json()
assert.deepStrictEqual(topEarners, [
{ participant_address: 'address1', total_rewards: '115' },
{ participant_address: 'address3', total_rewards: '35' },
{ participant_address: 'address2', total_rewards: '25' }
])
})
})
})

const givenDailyStationMetrics = async (pgPoolEvaluate, day, stationStats) => {
Expand Down
Loading